Compare commits

..

12 commits

Author SHA1 Message Date
9120e97ac5 [#203] Add go1.21 to CI
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2023-08-31 15:26:07 +03:00
2fc328a6d2 [#195] Add log constants linter
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2023-08-28 12:58:44 +03:00
b5fce5c8d2 [#168] Skip only invalid policies and copies instead of ignoring all of them
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-08-25 12:05:30 +03:00
41a128b1aa [#185] Update CHANGELOG.md
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-08-24 18:03:11 +03:00
6617adc22b [#185] Use correct object size when object is combined or encrypted
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-08-24 18:03:08 +03:00
631d9d83b6 [#185] Fix payload reader
When we use io.CopyBuffer it check for exact io.EOF matching,
so we need keep original EOF error otherwise io.CopyBuffer returns error

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-08-24 18:01:08 +03:00
adec93af54 [#185] tree: Fix getSubTreeMultipartUploads
Every tree node contains only FileName
but key in multipart info must contain FilePath

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-08-24 18:01:08 +03:00
8898c2ec08 [#185] Add tests for list multipart uploads
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-08-24 18:01:08 +03:00
8efcc957ea [#96] Move log messages to constants
Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2023-08-23 18:32:31 +03:00
6b728fef87 [#192] Add tests to make sure client_cut flag is passed to sdk
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-08-23 06:26:55 +00:00
6b1f365e65 [#192] Support client cut
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-08-23 06:26:55 +00:00
fcf1c45ad2 [#188] Fix url escaping
Url escaping has already been done in `net/http/request.go`

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-08-22 11:27:39 +03:00
71 changed files with 909 additions and 359 deletions

View file

@ -1,4 +1,4 @@
FROM golang:1.19 as builder
FROM golang:1.21 as builder
ARG BUILD=now
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw

View file

@ -6,7 +6,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
go_versions: [ '1.19', '1.20' ]
go_versions: [ '1.20', '1.21' ]
fail-fast: false
steps:
- uses: actions/checkout@v3

View file

@ -12,7 +12,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v3
with:
go-version: '1.20'
go-version: '1.21'
- name: Run commit format checker
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v1

View file

@ -7,17 +7,24 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: golangci-lint
uses: https://github.com/golangci/golangci-lint-action@v2
- name: Set up Go
uses: actions/setup-go@v3
with:
version: latest
go-version: '1.21'
cache: true
- name: Install linters
run: make lint-install
- name: Run linters
run: make lint
tests:
name: Tests
runs-on: ubuntu-latest
strategy:
matrix:
go_versions: [ '1.19', '1.20' ]
go_versions: [ '1.20', '1.21' ]
fail-fast: false
steps:
- uses: actions/checkout@v3

View file

@ -12,7 +12,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v3
with:
go-version: '1.20'
go-version: '1.21'
- name: Install govulncheck
run: go install golang.org/x/vuln/cmd/govulncheck@latest

View file

@ -24,6 +24,16 @@ linters-settings:
govet:
# report about shadowed variables
check-shadowing: false
custom:
truecloudlab-linters:
path: bin/external_linters.so
original-url: git.frostfs.info/TrueCloudLab/linters.git
settings:
noliteral:
enable: true
target-methods: ["Fatal"]
disable-packages: ["codes", "tc"]
constants-package: "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
linters:
enable:
@ -45,6 +55,7 @@ linters:
- gofmt
- whitespace
- goimports
- truecloudlab-linters
disable-all: true
fast: false

View file

@ -30,13 +30,20 @@ repos:
hooks:
- id: shellcheck
- repo: https://github.com/golangci/golangci-lint
rev: v1.51.2
hooks:
- id: golangci-lint
- repo: local
hooks:
- id: make-lint-install
name: install linters
entry: make lint-install
language: system
pass_filenames: false
- id: make-lint
name: run linters
entry: make lint
language: system
pass_filenames: false
- id: go-unit-tests
name: go unit tests
entry: make test

View file

@ -13,6 +13,8 @@ This document outlines major changes between releases.
- Replace part on re-upload when use multipart upload (#176)
- Fix goroutine leak on put object error (#178)
- Fix parsing signed headers in presigned urls (#182)
- Fix url escaping (#188)
- Use correct keys in `list-multipart-uploads` response (#185)
### Added
- Add a metric with addresses of nodes of the same and highest priority that are currently healthy (#51)
@ -25,6 +27,7 @@ This document outlines major changes between releases.
- Support multiple version credentials using GSet (#135)
- Implement chunk uploading (#106)
- Add new `kludge.bypass_content_encoding_check_in_chunks` config param (#146)
- Add new `frostfs.client_cut` config param (#192)
### Changed
- Update prometheus to v1.15.0 (#94)
@ -38,6 +41,7 @@ This document outlines major changes between releases.
- Use chi router instead of archived gorlilla/mux (#149)
- Complete multipart upload doesn't unnecessary copy now. Thus, the total time of multipart upload was reduced by 2 times (#63)
- Use gate key to form object owner (#175)
- Apply placement policies and copies if there is at least one valid value (#168)
### Removed
- Drop `tree.service` param (now endpoints from `peers` section are used) (#133)

View file

@ -3,8 +3,9 @@
# Common variables
REPO ?= $(shell go list -m)
VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")
GO_VERSION ?= 1.19
LINT_VERSION ?= 1.49.0
GO_VERSION ?= 1.20
LINT_VERSION ?= 1.54.0
TRUECLOUDLAB_LINT_VERSION ?= 0.0.2
BINDIR = bin
METRICS_DUMP_OUT ?= ./metrics-dump.json
@ -18,6 +19,10 @@ REPO_BASENAME = $(shell basename `go list -m`)
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
OUTPUT_LINT_DIR ?= $(shell pwd)/bin
LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
TMP_DIR := .cache
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
# .deb package versioning
@ -101,9 +106,23 @@ dirty-image:
-f .docker/Dockerfile.dirty \
-t $(HUB_IMAGE)-dirty:$(HUB_TAG) .
# Install linters
lint-install:
@mkdir -p $(TMP_DIR)
@rm -rf $(TMP_DIR)/linters
@git -c advice.detachedHead=false clone --branch v$(TRUECLOUDLAB_LINT_VERSION) https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters
@@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR)
@rm -rf $(TMP_DIR)/linters
@rmdir $(TMP_DIR) 2>/dev/null || true
@CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION)
# Run linters
lint:
@golangci-lint --timeout=5m run
@if [ ! -d "$(LINT_DIR)" ]; then \
echo "Run make lint-install"; \
exit 1; \
fi
$(LINT_DIR)/golangci-lint --timeout=5m run
# Run linters in Docker
docker/lint:

View file

@ -4,6 +4,7 @@ import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/bluele/gcache"
"go.uber.org/zap"
@ -46,7 +47,7 @@ func (o *AccessControlCache) Get(owner user.ID, key string) bool {
result, ok := entry.(bool)
if !ok {
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return false
}

View file

@ -5,6 +5,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/bluele/gcache"
"go.uber.org/zap"
@ -57,7 +58,7 @@ func (o *AccessBoxCache) Get(address oid.Address) *accessbox.Box {
result, ok := entry.(*accessbox.Box)
if !ok {
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return nil
}

View file

@ -5,6 +5,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"github.com/bluele/gcache"
"go.uber.org/zap"
)
@ -46,7 +47,7 @@ func (o *BucketCache) Get(key string) *data.BucketInfo {
result, ok := entry.(*data.BucketInfo)
if !ok {
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return nil
}

3
api/cache/names.go vendored
View file

@ -4,6 +4,7 @@ import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/bluele/gcache"
"go.uber.org/zap"
@ -48,7 +49,7 @@ func (o *ObjectsNameCache) Get(key string) *oid.Address {
result, ok := entry.(oid.Address)
if !ok {
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return nil
}

View file

@ -5,6 +5,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/bluele/gcache"
"go.uber.org/zap"
@ -47,7 +48,7 @@ func (o *ObjectsCache) GetObject(address oid.Address) *data.ExtendedObjectInfo {
result, ok := entry.(*data.ExtendedObjectInfo)
if !ok {
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return nil
}

View file

@ -7,6 +7,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"github.com/bluele/gcache"
"go.uber.org/zap"
@ -75,7 +76,7 @@ func (l *ObjectsListCache) GetVersions(key ObjectsListKey) []*data.NodeVersion {
result, ok := entry.([]*data.NodeVersion)
if !ok {
l.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return nil
}
@ -94,7 +95,7 @@ func (l *ObjectsListCache) CleanCacheEntriesContainingObject(objectName string,
for _, key := range keys {
k, ok := key.(ObjectsListKey)
if !ok {
l.logger.Warn("invalid cache key type", zap.String("actual", fmt.Sprintf("%T", key)),
l.logger.Warn(logs.InvalidCacheKeyType, zap.String("actual", fmt.Sprintf("%T", key)),
zap.String("expected", fmt.Sprintf("%T", k)))
continue
}

9
api/cache/system.go vendored
View file

@ -5,6 +5,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"github.com/bluele/gcache"
"go.uber.org/zap"
)
@ -48,7 +49,7 @@ func (o *SystemCache) GetObject(key string) *data.ObjectInfo {
result, ok := entry.(*data.ObjectInfo)
if !ok {
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return nil
}
@ -79,7 +80,7 @@ func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration {
result, ok := entry.(*data.CORSConfiguration)
if !ok {
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return nil
}
@ -95,7 +96,7 @@ func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
result, ok := entry.(*data.BucketSettings)
if !ok {
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return nil
}
@ -111,7 +112,7 @@ func (o *SystemCache) GetNotificationConfiguration(key string) *data.Notificatio
result, ok := entry.(*data.NotificationConfiguration)
if !ok {
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return nil
}

View file

@ -322,7 +322,7 @@ var errorCodes = errorCodeMap{
ErrInvalidMaxUploads: {
ErrCode: ErrInvalidMaxUploads,
Code: "InvalidArgument",
Description: "Argument max-uploads must be an integer between 0 and 2147483647",
Description: "Argument max-uploads must be an integer from 1 to 1000",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMaxKeys: {

View file

@ -20,6 +20,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -470,7 +471,7 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
ReqInfo: reqInfo,
}
if err = h.sendNotifications(ctx, s); err != nil {
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
}
}
w.WriteHeader(http.StatusOK)
@ -1460,7 +1461,7 @@ func (h *handler) encodeObjectACL(ctx context.Context, bucketACL *layer.BucketAC
if read {
permission = aclFullControl
} else {
h.reqLogger(ctx).Warn("some acl not fully mapped")
h.reqLogger(ctx).Warn(logs.SomeACLNotFullyMapped)
}
var grantee *Grantee

View file

@ -11,6 +11,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"go.uber.org/zap"
)
@ -56,13 +57,6 @@ type (
}
)
const (
// DefaultPolicy is a default policy of placing containers in FrostFS if it's not set at the request.
DefaultPolicy = "REP 3"
// DefaultCopiesNumber is a default number of object copies that is enough to consider put successful if it's not set in config.
DefaultCopiesNumber uint32 = 0
)
var _ api.Handler = (*handler)(nil)
// New creates new api.Handler using given logger and client.
@ -75,7 +69,7 @@ func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg *Config
}
if !cfg.NotificatorEnabled {
log.Warn("notificator is disabled, s3 won't produce notification events")
log.Warn(logs.NotificatorIsDisabledS3WontProduceNotificationEvents)
} else if notificator == nil {
return nil, errors.New("empty notificator")
}

View file

@ -17,7 +17,7 @@ func TestGetObjectPartsAttributes(t *testing.T) {
createTestBucket(hc, bktName)
putObject(t, hc, bktName, objName)
putObject(hc, bktName, objName)
result := getObjectAttributes(hc, bktName, objName, objectParts)
require.Nil(t, result.ObjectParts)

View file

@ -12,6 +12,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"go.uber.org/zap"
)
@ -117,7 +118,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
if srcSize, err := getObjectSize(extendedSrcObjInfo, encryptionParams); err != nil {
if srcSize, err := layer.GetObjectSize(srcObjInfo); err != nil {
h.logAndSendError(w, "failed to get source object size", reqInfo, err)
return
} else if srcSize > layer.UploadMaxSize { //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html
@ -249,7 +250,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
}
}
h.reqLogger(ctx).Info("object is copied", zap.Stringer("object_id", dstObjInfo.ID))
h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID))
s := &SendNotificationParams{
Event: EventObjectCreatedCopy,
@ -258,7 +259,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
ReqInfo: reqInfo,
}
if err = h.sendNotifications(ctx, s); err != nil {
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
}
if encryptionParams.Enabled() {

View file

@ -9,6 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"go.uber.org/zap"
)
@ -99,13 +100,13 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
}
bktInfo, err := h.obj.GetBucketInfo(ctx, reqInfo.BucketName)
if err != nil {
h.reqLogger(ctx).Warn("get bucket info", zap.Error(err))
h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err))
return
}
cors, err := h.obj.GetBucketCORS(ctx, bktInfo)
if err != nil {
h.reqLogger(ctx).Warn("get bucket cors", zap.Error(err))
h.reqLogger(ctx).Warn(logs.GetBucketCors, zap.Error(err))
return
}

View file

@ -11,6 +11,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
@ -114,7 +115,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
var objID oid.ID
if len(versionID) != 0 {
if err = objID.DecodeString(versionID); err != nil {
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
}
}
@ -130,7 +131,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
}
if err = h.sendNotifications(ctx, m); err != nil {
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
}
if deletedObject.VersionID != "" {
@ -261,7 +262,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
zap.Array("objects", marshaler),
zap.Errors("errors", errs),
}
h.reqLogger(ctx).Error("couldn't delete objects", fields...)
h.reqLogger(ctx).Error(logs.CouldntDeleteObjects, fields...)
}
if err = middleware.EncodeToResponse(w, response); err != nil {

View file

@ -25,7 +25,7 @@ func TestDeleteBucketOnAlreadyRemovedError(t *testing.T) {
bktName, objName := "bucket-for-removal", "object-to-delete"
bktInfo := createTestBucket(hc, bktName)
putObject(t, hc, bktName, objName)
putObject(hc, bktName, objName)
addr := getAddressOfLastVersion(hc, bktInfo, objName)
hc.tp.SetObjectError(addr, &apistatus.ObjectAlreadyRemoved{})
@ -66,7 +66,7 @@ func TestDeleteBucketOnNotFoundError(t *testing.T) {
bktName, objName := "bucket-for-removal", "object-to-delete"
bktInfo := createTestBucket(hc, bktName)
putObject(t, hc, bktName, objName)
putObject(hc, bktName, objName)
nodeVersion, err := hc.tree.GetUnversioned(hc.context, bktInfo, objName)
require.NoError(t, err)
@ -98,7 +98,7 @@ func TestDeleteObjectFromSuspended(t *testing.T) {
bktName, objName := "bucket-versioned-for-removal", "object-to-delete"
createSuspendedBucket(t, tc, bktName)
putObject(t, tc, bktName, objName)
putObject(tc, bktName, objName)
versionID, isDeleteMarker := deleteObject(t, tc, bktName, objName, emptyVersion)
require.True(t, isDeleteMarker)
@ -255,7 +255,7 @@ func TestDeleteMarkerSuspended(t *testing.T) {
t.Run("remove last unversioned non delete marker", func(t *testing.T) {
objName := "obj3"
putObject(t, tc, bktName, objName)
putObject(tc, bktName, objName)
nodeVersion, err := tc.tree.GetUnversioned(tc.Context(), bktInfo, objName)
require.NoError(t, err)
@ -333,7 +333,7 @@ func TestDeleteObjectFromListCache(t *testing.T) {
bktName, objName := "bucket-for-removal", "object-to-delete"
bktInfo, objInfo := createVersionedBucketAndObject(t, tc, bktName, objName)
versions := listObjectsV1(t, tc, bktName, "", "", "", -1)
versions := listObjectsV1(tc, bktName, "", "", "", -1)
require.Len(t, versions.Contents, 1)
checkFound(t, tc, bktName, objName, objInfo.VersionID())
@ -341,7 +341,7 @@ func TestDeleteObjectFromListCache(t *testing.T) {
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
// check cache is clean after object removal
versions = listObjectsV1(t, tc, bktName, "", "", "", -1)
versions = listObjectsV1(tc, bktName, "", "", "", -1)
require.Len(t, versions.Contents, 0)
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo))
@ -475,11 +475,11 @@ func getVersion(resp *ListObjectsVersionsResponse, objName string) []*ObjectVers
return res
}
func putObject(t *testing.T, tc *handlerContext, bktName, objName string) {
func putObject(hc *handlerContext, bktName, objName string) {
body := bytes.NewReader([]byte("content"))
w, r := prepareTestPayloadRequest(tc, bktName, objName, body)
tc.Handler().PutObjectHandler(w, r)
assertStatus(t, w, http.StatusOK)
w, r := prepareTestPayloadRequest(hc, bktName, objName, body)
hc.Handler().PutObjectHandler(w, r)
assertStatus(hc.t, w, http.StatusOK)
}
func createSuspendedBucket(t *testing.T, tc *handlerContext, bktName string) *data.BucketInfo {

View file

@ -12,7 +12,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"go.uber.org/zap"
)
@ -168,7 +167,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
fullSize, err := getObjectSize(extendedInfo, encryptionParams)
fullSize, err := layer.GetObjectSize(info)
if err != nil {
h.logAndSendError(w, "invalid size header", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
return
@ -233,23 +232,6 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
}
}
func getObjectSize(extendedInfo *data.ExtendedObjectInfo, encryptionParams encryption.Params) (uint64, error) {
var err error
fullSize := extendedInfo.ObjectInfo.Size
if encryptionParams.Enabled() {
if fullSize, err = strconv.ParseUint(extendedInfo.ObjectInfo.Headers[layer.AttributeDecryptedSize], 10, 64); err != nil {
return 0, fmt.Errorf("invalid decrypted size header: %w", err)
}
} else if extendedInfo.NodeVersion.IsCombined {
if fullSize, err = strconv.ParseUint(extendedInfo.ObjectInfo.Headers[layer.MultipartObjectSize], 10, 64); err != nil {
return 0, fmt.Errorf("invalid multipart size header: %w", err)
}
}
return fullSize, nil
}
func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs) error {
if len(args.IfMatch) > 0 && args.IfMatch != info.HashSum {
return fmt.Errorf("%w: etag mismatched: '%s', '%s'", errors.GetAPIError(errors.ErrPreconditionFailed), args.IfMatch, info.HashSum)

View file

@ -186,7 +186,7 @@ func TestGetObject(t *testing.T) {
bktName, objName := "bucket", "obj"
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
putObject(hc.t, hc, bktName, objName)
putObject(hc, bktName, objName)
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
checkFound(hc.t, hc, bktName, objName, emptyVersion)

View file

@ -3,9 +3,9 @@ package handler
import (
"bytes"
"context"
"crypto/rand"
"encoding/xml"
"io"
"math/rand"
"net/http"
"net/http/httptest"
"net/url"
@ -38,6 +38,8 @@ type handlerContext struct {
tree *tree.Tree
context context.Context
kludge *kludgeSettingsMock
layerFeatures *layer.FeatureSettingsMock
}
func (hc *handlerContext) Handler() *handler {
@ -123,11 +125,14 @@ func prepareHandlerContextBase(t *testing.T, minCache bool) *handlerContext {
cacheCfg = getMinCacheConfig(l)
}
features := &layer.FeatureSettingsMock{}
layerCfg := &layer.Config{
Caches: cacheCfg,
AnonKey: layer.AnonymousKey{Key: key},
Resolver: testResolver,
TreeService: treeMock,
Features: features,
}
var pp netmap.PlacementPolicy
@ -154,6 +159,8 @@ func prepareHandlerContextBase(t *testing.T, minCache bool) *handlerContext {
tree: treeMock,
context: middleware.SetBoxData(context.Background(), newTestAccessBox(t, key)),
kludge: kludge,
layerFeatures: features,
}
}

View file

@ -114,7 +114,7 @@ func TestHeadObject(t *testing.T) {
bktName, objName := "bucket", "obj"
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
putObject(hc.t, hc, bktName, objName)
putObject(hc, bktName, objName)
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
checkFound(hc.t, hc, bktName, objName, emptyVersion)

View file

@ -536,7 +536,7 @@ func TestPutObjectWithLock(t *testing.T) {
createTestBucketWithLock(hc, bktName, lockConfig)
objDefault := "obj-default-retention"
putObject(t, hc, bktName, objDefault)
putObject(hc, bktName, objDefault)
getObjectRetentionApproximate(hc, bktName, objDefault, governanceMode, time.Now().Add(24*time.Hour))
getObjectLegalHold(hc, bktName, objDefault, legalHoldOff)
@ -587,7 +587,7 @@ func TestPutLockErrors(t *testing.T) {
headers[api.AmzObjectLockRetainUntilDate] = "dummy"
putObjectWithLockFailed(t, hc, bktName, objName, headers, apiErrors.ErrInvalidRetentionDate)
putObject(t, hc, bktName, objName)
putObject(hc, bktName, objName)
retention := &data.Retention{Mode: governanceMode}
putObjectRetentionFailed(t, hc, bktName, objName, retention, apiErrors.ErrMalformedXML)

View file

@ -14,6 +14,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"github.com/google/uuid"
"go.uber.org/zap"
)
@ -92,6 +93,13 @@ type (
const (
uploadIDHeaderName = "uploadId"
partNumberHeaderName = "partNumber"
prefixQueryName = "prefix"
delimiterQueryName = "delimiter"
maxUploadsQueryName = "max-uploads"
encodingTypeQueryName = "encoding-type"
keyMarkerQueryName = "key-marker"
uploadIDMarkerQueryName = "upload-id-marker"
)
func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
@ -511,7 +519,7 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
ReqInfo: reqInfo,
}
if err = h.sendNotifications(ctx, s); err != nil {
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
}
return objInfo, nil
@ -528,30 +536,27 @@ func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Req
var (
queryValues = reqInfo.URL.Query()
delimiter = queryValues.Get("delimiter")
prefix = queryValues.Get("prefix")
maxUploadsStr = queryValues.Get(maxUploadsQueryName)
maxUploads = layer.MaxSizeUploadsList
)
if queryValues.Get("max-uploads") != "" {
val, err := strconv.Atoi(queryValues.Get("max-uploads"))
if err != nil || val < 0 {
if maxUploadsStr != "" {
val, err := strconv.Atoi(maxUploadsStr)
if err != nil || val < 1 || val > 1000 {
h.logAndSendError(w, "invalid maxUploads", reqInfo, errors.GetAPIError(errors.ErrInvalidMaxUploads))
return
}
if val < maxUploads {
maxUploads = val
}
}
p := &layer.ListMultipartUploadsParams{
Bkt: bktInfo,
Delimiter: delimiter,
EncodingType: queryValues.Get("encoding-type"),
KeyMarker: queryValues.Get("key-marker"),
Delimiter: queryValues.Get(delimiterQueryName),
EncodingType: queryValues.Get(encodingTypeQueryName),
KeyMarker: queryValues.Get(keyMarkerQueryName),
MaxUploads: maxUploads,
Prefix: prefix,
UploadIDMarker: queryValues.Get("upload-id-marker"),
Prefix: queryValues.Get(prefixQueryName),
UploadIDMarker: queryValues.Get(uploadIDMarkerQueryName),
}
list, err := h.obj.ListMultipartUploads(r.Context(), p)

View file

@ -3,12 +3,16 @@ package handler
import (
"bytes"
"encoding/xml"
"fmt"
"net/http"
"net/url"
"strconv"
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
s3Errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/stretchr/testify/require"
)
@ -105,6 +109,164 @@ func TestMultipartReUploadPart(t *testing.T) {
equalDataSlices(t, append(data1, data2...), data)
}
func TestListMultipartUploads(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-to-list-uploads"
createTestBucket(hc, bktName)
objName1 := "/my/object/name"
uploadInfo1 := createMultipartUpload(hc, bktName, objName1, map[string]string{})
objName2 := "/my/object2"
uploadInfo2 := createMultipartUpload(hc, bktName, objName2, map[string]string{})
objName3 := "/zzz/object/name3"
uploadInfo3 := createMultipartUpload(hc, bktName, objName3, map[string]string{})
t.Run("check upload key", func(t *testing.T) {
listUploads := listAllMultipartUploads(hc, bktName)
require.Len(t, listUploads.Uploads, 3)
for i, upload := range []*InitiateMultipartUploadResponse{uploadInfo1, uploadInfo2, uploadInfo3} {
require.Equal(t, upload.UploadID, listUploads.Uploads[i].UploadID)
require.Equal(t, upload.Key, listUploads.Uploads[i].Key)
}
})
t.Run("check max uploads", func(t *testing.T) {
listUploads := listMultipartUploadsBase(hc, bktName, "", "", "", "", 2)
require.Len(t, listUploads.Uploads, 2)
require.Equal(t, uploadInfo1.UploadID, listUploads.Uploads[0].UploadID)
require.Equal(t, uploadInfo2.UploadID, listUploads.Uploads[1].UploadID)
})
t.Run("check prefix", func(t *testing.T) {
listUploads := listMultipartUploadsBase(hc, bktName, "/my", "", "", "", -1)
require.Len(t, listUploads.Uploads, 2)
require.Equal(t, uploadInfo1.UploadID, listUploads.Uploads[0].UploadID)
require.Equal(t, uploadInfo2.UploadID, listUploads.Uploads[1].UploadID)
})
t.Run("check markers", func(t *testing.T) {
t.Run("check only key-marker", func(t *testing.T) {
listUploads := listMultipartUploadsBase(hc, bktName, "", "", "", objName2, -1)
require.Len(t, listUploads.Uploads, 1)
// If upload-id-marker is not specified, only the keys lexicographically greater than the specified key-marker will be included in the list.
require.Equal(t, uploadInfo3.UploadID, listUploads.Uploads[0].UploadID)
})
t.Run("check only upload-id-marker", func(t *testing.T) {
uploadIDMarker := uploadInfo1.UploadID
if uploadIDMarker > uploadInfo2.UploadID {
uploadIDMarker = uploadInfo2.UploadID
}
listUploads := listMultipartUploadsBase(hc, bktName, "", "", uploadIDMarker, "", -1)
// If key-marker is not specified, the upload-id-marker parameter is ignored.
require.Len(t, listUploads.Uploads, 3)
})
t.Run("check key-marker along with upload-id-marker", func(t *testing.T) {
uploadIDMarker := "00000000-0000-0000-0000-000000000000"
listUploads := listMultipartUploadsBase(hc, bktName, "", "", uploadIDMarker, objName3, -1)
require.Len(t, listUploads.Uploads, 1)
// If upload-id-marker is specified, any multipart uploads for a key equal to the key-marker might also be included,
// provided those multipart uploads have upload IDs lexicographically greater than the specified upload-id-marker.
require.Equal(t, uploadInfo3.UploadID, listUploads.Uploads[0].UploadID)
})
})
}
func TestMultipartUploadSize(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName := "bucket-for-test-multipart-size", "object-multipart"
createTestBucket(hc, bktName)
partSize := layer.UploadMinSize
objLen := 2 * partSize
headers := map[string]string{}
data := multipartUpload(hc, bktName, objName, headers, objLen, partSize)
require.Equal(t, objLen, len(data))
t.Run("check correct size in list v1", func(t *testing.T) {
listV1 := listObjectsV1(hc, bktName, "", "", "", -1)
require.Len(t, listV1.Contents, 1)
require.Equal(t, objLen, int(listV1.Contents[0].Size))
require.Equal(t, objName, listV1.Contents[0].Key)
})
t.Run("check correct size in list v2", func(t *testing.T) {
listV2 := listObjectsV2(hc, bktName, "", "", "", "", -1)
require.Len(t, listV2.Contents, 1)
require.Equal(t, objLen, int(listV2.Contents[0].Size))
require.Equal(t, objName, listV2.Contents[0].Key)
})
t.Run("check correct get", func(t *testing.T) {
_, hdr := getObject(hc, bktName, objName)
require.Equal(t, strconv.Itoa(objLen), hdr.Get(api.ContentLength))
part := getObjectRange(t, hc, bktName, objName, partSize, objLen-1)
equalDataSlices(t, data[partSize:], part)
})
t.Run("check correct size when part copy", func(t *testing.T) {
objName2 := "obj2"
uploadInfo := createMultipartUpload(hc, bktName, objName2, headers)
sourceCopy := bktName + "/" + objName
uploadPartCopy(hc, bktName, objName2, uploadInfo.UploadID, 1, sourceCopy, 0, 0)
uploadPartCopy(hc, bktName, objName2, uploadInfo.UploadID, 2, sourceCopy, 0, partSize)
})
}
func uploadPartCopy(hc *handlerContext, bktName, objName, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
return uploadPartCopyBase(hc, bktName, objName, false, uploadID, num, srcObj, start, end)
}
func uploadPartCopyBase(hc *handlerContext, bktName, objName string, encrypted bool, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
query := make(url.Values)
query.Set(uploadIDQuery, uploadID)
query.Set(partNumberQuery, strconv.Itoa(num))
w, r := prepareTestRequestWithQuery(hc, bktName, objName, query, nil)
if encrypted {
setEncryptHeaders(r)
}
r.Header.Set(api.AmzCopySource, srcObj)
if start+end > 0 {
r.Header.Set(api.AmzCopySourceRange, fmt.Sprintf("bytes=%d-%d", start, end))
}
hc.Handler().UploadPartCopy(w, r)
uploadPartCopyResponse := &UploadPartCopyResponse{}
readResponse(hc.t, w, http.StatusOK, uploadPartCopyResponse)
return uploadPartCopyResponse
}
func listAllMultipartUploads(hc *handlerContext, bktName string) *ListMultipartUploadsResponse {
return listMultipartUploadsBase(hc, bktName, "", "", "", "", -1)
}
func listMultipartUploadsBase(hc *handlerContext, bktName, prefix, delimiter, uploadIDMarker, keyMarker string, maxUploads int) *ListMultipartUploadsResponse {
query := make(url.Values)
query.Set(prefixQueryName, prefix)
query.Set(delimiterQueryName, delimiter)
query.Set(uploadIDMarkerQueryName, uploadIDMarker)
query.Set(keyMarkerQueryName, keyMarker)
if maxUploads != -1 {
query.Set(maxUploadsQueryName, strconv.Itoa(maxUploads))
}
w, r := prepareTestRequestWithQuery(hc, bktName, "", query, nil)
hc.Handler().ListMultipartUploadsHandler(w, r)
listPartsResponse := &ListMultipartUploadsResponse{}
readResponse(hc.t, w, http.StatusOK, listPartsResponse)
return listPartsResponse
}
func listParts(hc *handlerContext, bktName, objName string, uploadID string) *ListPartsResponse {
return listPartsBase(hc, bktName, objName, false, uploadID)
}

View file

@ -12,6 +12,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"github.com/google/uuid"
)
@ -202,7 +203,7 @@ func (h *handler) checkBucketConfiguration(ctx context.Context, conf *data.Notif
return
}
} else {
h.reqLogger(ctx).Warn("failed to send test event because notifications is disabled")
h.reqLogger(ctx).Warn(logs.FailedToSendTestEventBecauseNotificationsIsDisabled)
}
if q.ID == "" {

View file

@ -198,6 +198,10 @@ func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) []Obje
ETag: obj.HashSum,
}
if size, err := layer.GetObjectSize(obj); err == nil {
res.Size = size
}
if fetchOwner {
res.Owner = &Owner{
ID: obj.Owner.String(),

View file

@ -67,11 +67,11 @@ func TestS3CompatibilityBucketListV2BothContinuationTokenStartAfter(t *testing.T
createTestObject(tc, bktInfo, objName)
}
listV2Response1 := listObjectsV2(t, tc, bktName, "", "", "bar", "", 1)
listV2Response1 := listObjectsV2(tc, bktName, "", "", "bar", "", 1)
nextContinuationToken := listV2Response1.NextContinuationToken
require.Equal(t, "baz", listV2Response1.Contents[0].Key)
listV2Response2 := listObjectsV2(t, tc, bktName, "", "", "bar", nextContinuationToken, -1)
listV2Response2 := listObjectsV2(tc, bktName, "", "", "bar", nextContinuationToken, -1)
require.Equal(t, nextContinuationToken, listV2Response2.ContinuationToken)
require.Equal(t, "bar", listV2Response2.StartAfter)
@ -92,7 +92,7 @@ func TestS3BucketListDelimiterBasic(t *testing.T) {
createTestObject(tc, bktInfo, objName)
}
listV1Response := listObjectsV1(t, tc, bktName, "", "/", "", -1)
listV1Response := listObjectsV1(tc, bktName, "", "/", "", -1)
require.Equal(t, "/", listV1Response.Delimiter)
require.Equal(t, "asdf", listV1Response.Contents[0].Key)
require.Len(t, listV1Response.CommonPrefixes, 2)
@ -100,6 +100,26 @@ func TestS3BucketListDelimiterBasic(t *testing.T) {
require.Equal(t, "quux/", listV1Response.CommonPrefixes[1].Prefix)
}
func TestS3BucketListV2DelimiterPercentage(t *testing.T) {
tc := prepareHandlerContext(t)
bktName := "bucket-for-listing"
objects := []string{"b%ar", "b%az", "c%ab", "foo"}
bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])
for _, objName := range objects[1:] {
createTestObject(tc, bktInfo, objName)
}
listV2Response := listObjectsV2(tc, bktName, "", "%", "", "", -1)
require.Equal(t, "%", listV2Response.Delimiter)
require.Len(t, listV2Response.Contents, 1)
require.Equal(t, "foo", listV2Response.Contents[0].Key)
require.Len(t, listV2Response.CommonPrefixes, 2)
require.Equal(t, "b%", listV2Response.CommonPrefixes[0].Prefix)
require.Equal(t, "c%", listV2Response.CommonPrefixes[1].Prefix)
}
func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
tc := prepareHandlerContext(t)
@ -129,7 +149,7 @@ func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
validateListV2(t, tc, bktName, prefix, delim, "", 2, false, true, []string{"boo/bar"}, []string{"boo/baz/"})
}
func listObjectsV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
func listObjectsV2(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
if len(startAfter) != 0 {
query.Add("start-after", startAfter)
@ -138,17 +158,17 @@ func listObjectsV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter,
query.Add("continuation-token", continuationToken)
}
w, r := prepareTestFullRequest(tc, bktName, "", query, nil)
tc.Handler().ListObjectsV2Handler(w, r)
assertStatus(t, w, http.StatusOK)
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
hc.Handler().ListObjectsV2Handler(w, r)
assertStatus(hc.t, w, http.StatusOK)
res := &ListObjectsV2Response{}
parseTestResponse(t, w, res)
parseTestResponse(hc.t, w, res)
return res
}
func validateListV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, continuationToken string, maxKeys int,
isTruncated, last bool, checkObjects, checkPrefixes []string) string {
response := listObjectsV2(t, tc, bktName, prefix, delimiter, "", continuationToken, maxKeys)
response := listObjectsV2(tc, bktName, prefix, delimiter, "", continuationToken, maxKeys)
require.Equal(t, isTruncated, response.IsTruncated)
require.Equal(t, last, len(response.NextContinuationToken) == 0)
@ -182,16 +202,16 @@ func prepareCommonListObjectsQuery(prefix, delimiter string, maxKeys int) url.Va
return query
}
func listObjectsV1(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int) *ListObjectsV1Response {
func listObjectsV1(hc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int) *ListObjectsV1Response {
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
if len(marker) != 0 {
query.Add("marker", marker)
}
w, r := prepareTestFullRequest(tc, bktName, "", query, nil)
tc.Handler().ListObjectsV1Handler(w, r)
assertStatus(t, w, http.StatusOK)
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
hc.Handler().ListObjectsV1Handler(w, r)
assertStatus(hc.t, w, http.StatusOK)
res := &ListObjectsV1Response{}
parseTestResponse(t, w, res)
parseTestResponse(hc.t, w, res)
return res
}

View file

@ -24,6 +24,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"go.uber.org/zap"
@ -277,7 +278,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
ReqInfo: reqInfo,
}
if err = h.sendNotifications(ctx, s); err != nil {
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
}
if containsACL {
@ -494,7 +495,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
ReqInfo: reqInfo,
}
if err = h.sendNotifications(ctx, s); err != nil {
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
}
if acl := auth.MultipartFormValue(r, "acl"); acl != "" {
@ -539,7 +540,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
}
if settings, err := h.obj.GetBucketSettings(ctx, bktInfo); err != nil {
h.reqLogger(ctx).Warn("couldn't get bucket versioning", zap.String("bucket name", reqInfo.BucketName), zap.Error(err))
h.reqLogger(ctx).Warn(logs.CouldntGetBucketVersioning, zap.String("bucket name", reqInfo.BucketName), zap.Error(err))
} else if settings.VersioningEnabled() {
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
}
@ -778,7 +779,7 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
return
}
h.reqLogger(ctx).Info("bucket is created", zap.Stringer("container_id", bktInfo.CID))
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
if p.ObjectLockEnabled {
sp := &layer.PutSettingsParams{

View file

@ -23,6 +23,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/stretchr/testify/require"
)
@ -309,3 +310,37 @@ func TestCreateBucket(t *testing.T) {
box2, _ := createAccessBox(t)
createBucketAssertS3Error(hc, bktName, box2, s3errors.ErrBucketAlreadyExists)
}
func TestPutObjectClientCut(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName1, objName2 := "bkt-name", "obj-name1", "obj-name2"
createTestBucket(hc, bktName)
putObject(hc, bktName, objName1)
obj1 := getObjectFromLayer(hc, objName1)[0]
require.Empty(t, getObjectAttribute(obj1, "s3-client-cut"))
hc.layerFeatures.SetClientCut(true)
putObject(hc, bktName, objName2)
obj2 := getObjectFromLayer(hc, objName2)[0]
require.Equal(t, "true", getObjectAttribute(obj2, "s3-client-cut"))
}
func getObjectFromLayer(hc *handlerContext, objName string) []*object.Object {
var res []*object.Object
for _, o := range hc.tp.Objects() {
if objName == getObjectAttribute(o, object.AttributeFilePath) {
res = append(res, o)
}
}
return res
}
func getObjectAttribute(obj *object.Object, attrName string) string {
for _, attr := range obj.Attributes() {
if attr.Key() == attrName {
return attr.Value()
}
}
return ""
}

View file

@ -13,6 +13,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"go.uber.org/zap"
)
@ -66,7 +67,7 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
ReqInfo: reqInfo,
}
if err = h.sendNotifications(ctx, s); err != nil {
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
}
w.WriteHeader(http.StatusOK)
@ -143,7 +144,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
ReqInfo: reqInfo,
}
if err = h.sendNotifications(ctx, s); err != nil {
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
}
w.WriteHeader(http.StatusNoContent)

View file

@ -13,6 +13,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
frosterrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"go.uber.org/zap"
)
@ -36,7 +37,7 @@ func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo
zap.String("description", logText),
zap.Error(err)}
fields = append(fields, additional...)
h.log.Error("request failed", fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
h.log.Error(logs.RequestFailed, fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
}
func (h *handler) logAndSendErrorNoHeader(w http.ResponseWriter, logText string, reqInfo *middleware.ReqInfo, err error, additional ...zap.Field) {
@ -49,7 +50,7 @@ func (h *handler) logAndSendErrorNoHeader(w http.ResponseWriter, logText string,
zap.String("description", logText),
zap.Error(err)}
fields = append(fields, additional...)
h.log.Error("request failed", fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
h.log.Error(logs.RequestFailed, fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
}
func transformToS3Error(err error) error {

View file

@ -3,6 +3,7 @@ package layer
import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
@ -61,7 +62,7 @@ func (c *Cache) GetBucket(name string) *data.BucketInfo {
func (c *Cache) PutBucket(bktInfo *data.BucketInfo) {
if err := c.bucketCache.Put(bktInfo); err != nil {
c.logger.Warn("couldn't put bucket info into cache",
c.logger.Warn(logs.CouldntPutBucketInfoIntoCache,
zap.String("bucket name", bktInfo.Name),
zap.Stringer("bucket cid", bktInfo.CID),
zap.Error(err))
@ -104,13 +105,13 @@ func (c *Cache) GetLastObject(owner user.ID, bktName, objName string) *data.Exte
func (c *Cache) PutObject(owner user.ID, extObjInfo *data.ExtendedObjectInfo) {
if err := c.objCache.PutObject(extObjInfo); err != nil {
c.logger.Warn("couldn't add object to cache", zap.Error(err),
c.logger.Warn(logs.CouldntAddObjectToCache, zap.Error(err),
zap.String("object_name", extObjInfo.ObjectInfo.Name), zap.String("bucket_name", extObjInfo.ObjectInfo.Bucket),
zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString()))
}
if err := c.accessCache.Put(owner, extObjInfo.ObjectInfo.Address().EncodeToString()); err != nil {
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
}
}
@ -118,7 +119,7 @@ func (c *Cache) PutObjectWithName(owner user.ID, extObjInfo *data.ExtendedObject
c.PutObject(owner, extObjInfo)
if err := c.namesCache.Put(extObjInfo.ObjectInfo.NiceName(), extObjInfo.ObjectInfo.Address()); err != nil {
c.logger.Warn("couldn't put obj address to name cache",
c.logger.Warn(logs.CouldntPutObjAddressToNameCache,
zap.String("obj nice name", extObjInfo.ObjectInfo.NiceName()),
zap.Error(err))
}
@ -134,11 +135,11 @@ func (c *Cache) GetList(owner user.ID, key cache.ObjectsListKey) []*data.NodeVer
func (c *Cache) PutList(owner user.ID, key cache.ObjectsListKey, list []*data.NodeVersion) {
if err := c.listsCache.PutVersions(key, list); err != nil {
c.logger.Warn("couldn't cache list of objects", zap.Error(err))
c.logger.Warn(logs.CouldntCacheListOfObjects, zap.Error(err))
}
if err := c.accessCache.Put(owner, key.String()); err != nil {
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
}
}
@ -152,11 +153,11 @@ func (c *Cache) GetTagging(owner user.ID, key string) map[string]string {
func (c *Cache) PutTagging(owner user.ID, key string, tags map[string]string) {
if err := c.systemCache.PutTagging(key, tags); err != nil {
c.logger.Error("couldn't cache tags", zap.Error(err))
c.logger.Error(logs.CouldntCacheTags, zap.Error(err))
}
if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
}
}
@ -174,11 +175,11 @@ func (c *Cache) GetLockInfo(owner user.ID, key string) *data.LockInfo {
func (c *Cache) PutLockInfo(owner user.ID, key string, lockInfo *data.LockInfo) {
if err := c.systemCache.PutLockInfo(key, lockInfo); err != nil {
c.logger.Error("couldn't cache lock info", zap.Error(err))
c.logger.Error(logs.CouldntCacheLockInfo, zap.Error(err))
}
if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
}
}
@ -195,11 +196,11 @@ func (c *Cache) GetSettings(owner user.ID, bktInfo *data.BucketInfo) *data.Bucke
func (c *Cache) PutSettings(owner user.ID, bktInfo *data.BucketInfo, settings *data.BucketSettings) {
key := bktInfo.Name + bktInfo.SettingsObjectName()
if err := c.systemCache.PutSettings(key, settings); err != nil {
c.logger.Warn("couldn't cache bucket settings", zap.String("bucket", bktInfo.Name), zap.Error(err))
c.logger.Warn(logs.CouldntCacheBucketSettings, zap.String("bucket", bktInfo.Name), zap.Error(err))
}
if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
}
}
@ -217,11 +218,11 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf
key := bkt.Name + bkt.CORSObjectName()
if err := c.systemCache.PutCORS(key, cors); err != nil {
c.logger.Warn("couldn't cache cors", zap.String("bucket", bkt.Name), zap.Error(err))
c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err))
}
if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
}
}
@ -242,10 +243,10 @@ func (c *Cache) GetNotificationConfiguration(owner user.ID, bktInfo *data.Bucket
func (c *Cache) PutNotificationConfiguration(owner user.ID, bktInfo *data.BucketInfo, configuration *data.NotificationConfiguration) {
key := bktInfo.Name + bktInfo.NotificationConfigurationObjectName()
if err := c.systemCache.PutNotificationConfiguration(key, configuration); err != nil {
c.logger.Warn("couldn't cache notification configuration", zap.String("bucket", bktInfo.Name), zap.Error(err))
c.logger.Warn(logs.CouldntCacheNotificationConfiguration, zap.String("bucket", bktInfo.Name), zap.Error(err))
}
if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
}
}

View file

@ -9,6 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@ -63,7 +64,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
if len(attrLockEnabled) > 0 {
info.ObjectLockEnabled, err = strconv.ParseBool(attrLockEnabled)
if err != nil {
log.Error("could not parse container object lock enabled attribute",
log.Error(logs.CouldNotParseContainerObjectLockEnabledAttribute,
zap.String("lock_enabled", attrLockEnabled),
zap.Error(err),
)
@ -78,7 +79,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
res, err := n.frostFS.UserContainers(ctx, n.BearerOwner(ctx))
if err != nil {
n.reqLogger(ctx).Error("could not list user containers", zap.Error(err))
n.reqLogger(ctx).Error(logs.CouldNotListUserContainers, zap.Error(err))
return nil, err
}
@ -86,7 +87,7 @@ func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
for i := range res {
info, err := n.containerInfo(ctx, res[i])
if err != nil {
n.reqLogger(ctx).Error("could not fetch container info", zap.Error(err))
n.reqLogger(ctx).Error(logs.CouldNotFetchContainerInfo, zap.Error(err))
continue
}

View file

@ -10,6 +10,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"go.uber.org/zap"
)
@ -57,7 +58,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
if !objIDToDeleteNotFound {
if err = n.objectDelete(ctx, p.BktInfo, objIDToDelete); err != nil {
n.reqLogger(ctx).Error("couldn't delete cors object", zap.Error(err),
n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err),
zap.String("cnrID", p.BktInfo.CID.EncodeToString()),
zap.String("objID", objIDToDelete.EncodeToString()))
}

View file

@ -111,6 +111,9 @@ type PrmObjectCreate struct {
// Number of object copies that is enough to consider put successful.
CopiesNumber []uint32
// Enables client side object preparing.
ClientCut bool
}
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.

View file

@ -25,6 +25,18 @@ import (
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
)
type FeatureSettingsMock struct {
clientCut bool
}
func (k *FeatureSettingsMock) ClientCut() bool {
return k.clientCut
}
func (k *FeatureSettingsMock) SetClientCut(clientCut bool) {
k.clientCut = clientCut
}
type TestFrostFS struct {
FrostFS
@ -222,6 +234,13 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.
attrs = append(attrs, *a)
}
if prm.ClientCut {
a := object.NewAttribute()
a.SetKey("s3-client-cut")
a.SetValue("true")
attrs = append(attrs, *a)
}
for i := range prm.Attributes {
a := object.NewAttribute()
a.SetKey(prm.Attributes[i][0])

View file

@ -16,6 +16,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@ -45,6 +46,10 @@ type (
Resolve(ctx context.Context, name string) (cid.ID, error)
}
FeatureSettings interface {
ClientCut() bool
}
layer struct {
frostFS FrostFS
gateOwner user.ID
@ -54,6 +59,7 @@ type (
ncontroller EventListener
cache *Cache
treeService TreeService
features FeatureSettings
}
Config struct {
@ -63,6 +69,7 @@ type (
AnonKey AnonymousKey
Resolver BucketResolver
TreeService TreeService
Features FeatureSettings
}
// AnonymousKey contains data for anonymous requests.
@ -301,6 +308,7 @@ func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) Client {
resolver: config.Resolver,
cache: NewCache(config.Caches),
treeService: config.TreeService,
features: config.Features,
}
}
@ -555,7 +563,7 @@ func (n *layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams)
return nil, err
}
n.reqLogger(ctx).Debug("get object",
n.reqLogger(ctx).Debug(logs.GetObject,
zap.Stringer("cid", p.BktInfo.CID),
zap.Stringer("oid", objInfo.ObjectInfo.ID))
@ -686,7 +694,7 @@ func (n *layer) handleNotFoundError(bkt *data.BucketInfo, obj *VersionedObject)
func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject, nodeID uint64) *VersionedObject {
if client.IsErrObjectAlreadyRemoved(obj.Error) {
n.reqLogger(ctx).Debug("object already removed",
n.reqLogger(ctx).Debug(logs.ObjectAlreadyRemoved,
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID))
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeID)
@ -698,7 +706,7 @@ func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketIn
}
if client.IsErrObjectNotFound(obj.Error) {
n.reqLogger(ctx).Debug("object not found",
n.reqLogger(ctx).Debug(logs.ObjectNotFound,
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID))
obj.Error = nil
@ -776,7 +784,7 @@ func (n *layer) ResolveBucket(ctx context.Context, name string) (cid.ID, error)
return cid.ID{}, err
}
n.reqLogger(ctx).Info("resolve bucket", zap.Stringer("cid", cnrID))
n.reqLogger(ctx).Info(logs.ResolveBucket, zap.Stringer("cid", cnrID))
}
return cnrID, nil

View file

@ -16,6 +16,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/minio/sio"
@ -202,7 +203,7 @@ func (n *layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, er
func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) {
encInfo := FormEncryptionInfo(multipartInfo.Meta)
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
n.reqLogger(ctx).Warn("mismatched obj encryptionInfo", zap.Error(err))
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
return nil, s3errors.GetAPIError(s3errors.ErrInvalidEncryptionParameters)
}
@ -237,7 +238,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
size = decSize
}
n.reqLogger(ctx).Debug("upload part",
n.reqLogger(ctx).Debug(logs.UploadPart,
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
@ -258,7 +259,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
}
if !oldPartIDNotFound {
if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil {
n.reqLogger(ctx).Error("couldn't delete old part object", zap.Error(err),
n.reqLogger(ctx).Error(logs.CouldntDeleteOldPartObject, zap.Error(err),
zap.String("cid", bktInfo.CID.EncodeToString()),
zap.String("oid", oldPartID.EncodeToString()))
}
@ -288,10 +289,16 @@ func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
}
size := p.SrcObjInfo.Size
srcObjectSize := p.SrcObjInfo.Size
if objSize, err := GetObjectSize(p.SrcObjInfo); err == nil {
srcObjectSize = objSize
}
if p.Range != nil {
size = p.Range.End - p.Range.Start + 1
if p.Range.End > p.SrcObjInfo.Size {
return nil, fmt.Errorf("%w: %d-%d/%d", s3errors.GetAPIError(s3errors.ErrInvalidCopyPartRangeSource), p.Range.Start, p.Range.End, p.SrcObjInfo.Size)
if p.Range.End > srcObjectSize {
return nil, fmt.Errorf("%w: %d-%d/%d", s3errors.GetAPIError(s3errors.ErrInvalidCopyPartRangeSource), p.Range.Start, p.Range.End, srcObjectSize)
}
}
if size > UploadMaxSize {
@ -412,7 +419,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
CopiesNumbers: multipartInfo.CopiesNumbers,
})
if err != nil {
n.reqLogger(ctx).Error("could not put a completed object (multipart upload)",
n.reqLogger(ctx).Error(logs.CouldNotPutCompletedObject,
zap.String("uploadID", p.Info.UploadID),
zap.String("uploadKey", p.Info.Key),
zap.Error(err))
@ -424,7 +431,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
addr.SetContainer(p.Info.Bkt.CID)
for _, partInfo := range partsInfo {
if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil {
n.reqLogger(ctx).Warn("could not delete upload part",
n.reqLogger(ctx).Warn(logs.CouldNotDeleteUploadPart,
zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID),
zap.Error(err))
}
@ -503,7 +510,7 @@ func (n *layer) AbortMultipartUpload(ctx context.Context, p *UploadInfoParams) e
for _, info := range parts {
if err = n.objectDelete(ctx, p.Bkt, info.OID); err != nil {
n.reqLogger(ctx).Warn("couldn't delete part", zap.String("cid", p.Bkt.CID.EncodeToString()),
n.reqLogger(ctx).Warn(logs.CouldntDeletePart, zap.String("cid", p.Bkt.CID.EncodeToString()),
zap.String("oid", info.OID.EncodeToString()), zap.Int("part number", info.Number), zap.Error(err))
}
}
@ -520,7 +527,7 @@ func (n *layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
encInfo := FormEncryptionInfo(multipartInfo.Meta)
if err = p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
n.reqLogger(ctx).Warn("mismatched obj encryptionInfo", zap.Error(err))
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
return nil, s3errors.GetAPIError(s3errors.ErrInvalidEncryptionParameters)
}
@ -584,7 +591,7 @@ func (n *layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
oids[i] = part.OID.EncodeToString()
}
n.reqLogger(ctx).Debug("part details",
n.reqLogger(ctx).Debug(logs.PartDetails,
zap.Stringer("cid", p.Bkt.CID),
zap.String("upload id", p.UploadID),
zap.Ints("part numbers", partsNumbers),

View file

@ -9,6 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"go.uber.org/zap"
)
@ -46,7 +47,7 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
if !objIDToDeleteNotFound {
if err = n.objectDelete(ctx, p.BktInfo, objIDToDelete); err != nil {
n.reqLogger(ctx).Error("couldn't delete notification configuration object", zap.Error(err),
n.reqLogger(ctx).Error(logs.CouldntDeleteNotificationConfigurationObject, zap.Error(err),
zap.String("cid", p.BktInfo.CID.EncodeToString()),
zap.String("oid", objIDToDelete.EncodeToString()))
}

View file

@ -19,6 +19,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -291,7 +292,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
return nil, err
}
n.reqLogger(ctx).Debug("put object", zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
newVersion := &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{
@ -458,6 +459,7 @@ func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idOb
// Returns object ID and payload sha256 hash.
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (uint64, oid.ID, []byte, error) {
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
prm.ClientCut = n.features.ClientCut()
var size uint64
hash := sha256.New()
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
@ -467,7 +469,7 @@ func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktIn
id, err := n.frostFS.CreateObject(ctx, prm)
if err != nil {
if _, errDiscard := io.Copy(io.Discard, prm.Payload); errDiscard != nil {
n.reqLogger(ctx).Warn("failed to discard put payload, probably goroutine leaks", zap.Error(errDiscard))
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard))
}
return 0, oid.ID{}, nil, err
@ -656,7 +658,7 @@ func (n *layer) initWorkerPool(ctx context.Context, size int, p allObjectParams,
})
if err != nil {
wg.Done()
reqLog.Warn("failed to submit task to pool", zap.Error(err))
reqLog.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err))
}
}(node)
}
@ -798,7 +800,7 @@ func (n *layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo
meta, err := n.objectHead(ctx, bktInfo, node.OID)
if err != nil {
n.reqLogger(ctx).Warn("could not fetch object meta", zap.Error(err))
n.reqLogger(ctx).Warn(logs.CouldNotFetchObjectMeta, zap.Error(err))
return nil
}

View file

@ -7,6 +7,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
@ -180,7 +181,7 @@ func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (
}
if err == nil && version != nil && !version.IsDeleteMarker() {
n.reqLogger(ctx).Debug("get tree node",
n.reqLogger(ctx).Debug(logs.GetTreeNode,
zap.Stringer("cid", objVersion.BktInfo.CID), zap.Stringer("oid", version.OID))
}

View file

@ -97,6 +97,23 @@ func objectInfoFromMeta(bkt *data.BucketInfo, meta *object.Object) *data.ObjectI
}
}
func GetObjectSize(objInfo *data.ObjectInfo) (uint64, error) {
var err error
fullSize := objInfo.Size
if objInfo.Headers[AttributeDecryptedSize] != "" {
if fullSize, err = strconv.ParseUint(objInfo.Headers[AttributeDecryptedSize], 10, 64); err != nil {
return 0, fmt.Errorf("invalid decrypted size header: %w", err)
}
} else if objInfo.Headers[MultipartObjectSize] != "" {
if fullSize, err = strconv.ParseUint(objInfo.Headers[MultipartObjectSize], 10, 64); err != nil {
return 0, fmt.Errorf("invalid multipart size header: %w", err)
}
}
return fullSize, nil
}
func FormEncryptionInfo(headers map[string]string) encryption.ObjectEncryption {
algorithm := headers[AttributeEncryptionAlgorithm]
return encryption.ObjectEncryption{

View file

@ -170,6 +170,7 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
Caches: config,
AnonKey: AnonymousKey{Key: key},
TreeService: NewTreeService(),
Features: &FeatureSettingsMock{},
}
return &testContext{

View file

@ -5,6 +5,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"go.uber.org/zap"
)
@ -15,9 +16,9 @@ func Auth(center auth.Center, log *zap.Logger) Func {
box, err := center.Authenticate(r)
if err != nil {
if err == auth.ErrNoAuthorizationHeader {
reqLogOrDefault(ctx, log).Debug("couldn't receive access box for gate key, random key will be used")
reqLogOrDefault(ctx, log).Debug(logs.CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed)
} else {
reqLogOrDefault(ctx, log).Error("failed to pass authentication", zap.Error(err))
reqLogOrDefault(ctx, log).Error(logs.FailedToPassAuthentication, zap.Error(err))
if _, ok := err.(errors.Error); !ok {
err = errors.GetAPIError(errors.ErrAccessDenied)
}

View file

@ -10,6 +10,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"go.uber.org/zap"
@ -141,7 +142,7 @@ func resolveCID(log *zap.Logger, resolveBucket BucketResolveFunc) cidResolveFunc
bktInfo, err := resolveBucket(ctx, reqInfo.BucketName)
if err != nil {
reqLogOrDefault(ctx, log).Debug("failed to resolve CID", zap.Error(err))
reqLogOrDefault(ctx, log).Debug(logs.FailedToResolveCID, zap.Error(err))
return ""
}

View file

@ -9,6 +9,7 @@ import (
"strings"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"go.uber.org/zap"
@ -207,7 +208,7 @@ func Request(log *zap.Logger) Func {
reqLogger := log.With(zap.String("request_id", reqInfo.RequestID))
r = r.WithContext(SetReqLogger(r.Context(), reqLogger))
reqLogger.Info("request start", zap.String("host", r.Host),
reqLogger.Info(logs.RequestStart, zap.String("host", r.Host),
zap.String("remote_host", reqInfo.RemoteHost))
// continue execution
@ -242,14 +243,7 @@ func AddObjectName(l *zap.Logger) Func {
rctx := chi.RouteContext(ctx)
// trim leading slash (always present)
obj := rctx.RoutePath[1:]
object, err := url.PathUnescape(obj)
if err != nil {
object = obj
}
reqInfo.ObjectName = object
reqInfo.ObjectName = rctx.RoutePath[1:]
reqLogger := reqLogOrDefault(ctx, l)
r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("object", reqInfo.ObjectName))))

View file

@ -9,6 +9,7 @@ import (
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
"go.uber.org/zap"
)
@ -319,7 +320,7 @@ func LogSuccessResponse(l *zap.Logger) Func {
reqLogger := reqLogOrDefault(ctx, l)
reqInfo := GetReqInfo(ctx)
reqLogger.Info("request end",
reqLogger.Info(logs.RequestEnd,
zap.String("method", reqInfo.API),
zap.String("bucket", reqInfo.BucketName),
zap.String("object", reqInfo.ObjectName),

View file

@ -9,6 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
@ -168,9 +169,9 @@ func (c *Controller) Listen(ctx context.Context) {
select {
case msg := <-stream.ch:
if err := stream.h.HandleMessage(ctx, msg); err != nil {
c.logger.Error("could not handle message", zap.Error(err))
c.logger.Error(logs.CouldNotHandleMessage, zap.Error(err))
} else if err = msg.Ack(); err != nil {
c.logger.Error("could not ACK message", zap.Error(err))
c.logger.Error(logs.CouldNotACKMessage, zap.Error(err))
}
case <-ctx.Done():
return
@ -187,10 +188,10 @@ func (c *Controller) SendNotifications(topics map[string]string, p *handler.Send
event.Records[0].S3.ConfigurationID = id
msg, err := json.Marshal(event)
if err != nil {
c.logger.Error("couldn't marshal an event", zap.String("subject", topic), zap.Error(err))
c.logger.Error(logs.CouldntMarshalAnEvent, zap.String("subject", topic), zap.Error(err))
}
if err = c.publish(topic, msg); err != nil {
c.logger.Error("couldn't send an event to topic", zap.String("subject", topic), zap.Error(err))
c.logger.Error(logs.CouldntSendAnEventToTopic, zap.String("subject", topic), zap.Error(err))
}
}

View file

@ -9,6 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
s3middleware "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
@ -135,7 +136,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
})
if log := s3middleware.GetReqLog(ctx); log != nil {
log.Error("request unmatched", zap.String("method", reqInfo.API))
log.Error(logs.RequestUnmatched, zap.String("method", reqInfo.API))
}
}

View file

@ -61,6 +61,51 @@ func TestRouterObjectWithSlashes(t *testing.T) {
require.Equal(t, objName, resp.ReqInfo.ObjectName)
}
func TestRouterObjectEscaping(t *testing.T) {
chiRouter := prepareRouter(t)
bktName := "dkirillov"
for _, tc := range []struct {
name string
expectedObjName string
objName string
}{
{
name: "simple",
expectedObjName: "object",
objName: "object",
},
{
name: "with slashes",
expectedObjName: "fix/object",
objName: "fix/object",
},
{
name: "with percentage",
expectedObjName: "fix/object%ac",
objName: "fix/object%ac",
},
{
name: "with percentage escaped",
expectedObjName: "fix/object%ac",
objName: "fix/object%25ac",
},
} {
t.Run(tc.name, func(t *testing.T) {
target := fmt.Sprintf("/%s/%s", bktName, tc.objName)
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodPut, target, nil)
chiRouter.ServeHTTP(w, r)
resp := readResponse(t, w)
require.Equal(t, "PutObject", resp.Method)
require.Equal(t, tc.expectedObjName, resp.ReqInfo.ObjectName)
})
}
}
func prepareRouter(t *testing.T) *chi.Mux {
throttleOps := middleware.ThrottleOpts{
Limit: 10,

View file

@ -16,6 +16,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
@ -168,11 +169,11 @@ type (
func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwner user.ID) (cid.ID, error) {
if !opts.ID.Equals(cid.ID{}) {
a.log.Info("check container", zap.Stringer("cid", opts.ID))
a.log.Info(logs.CheckContainer, zap.Stringer("cid", opts.ID))
return opts.ID, a.frostFS.ContainerExists(ctx, opts.ID)
}
a.log.Info("create container",
a.log.Info(logs.CreateContainer,
zap.String("friendly_name", opts.FriendlyName),
zap.String("placement_policy", opts.PlacementPolicy))
@ -267,7 +268,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
return fmt.Errorf("check container: %w", err)
}
a.log.Info("store bearer token into FrostFS",
a.log.Info(logs.StoreBearerTokenIntoFrostFS,
zap.Stringer("owner_tkn", idOwner))
creds := tokens.New(a.frostFS, secrets.EphemeralKey, cache.DefaultAccessBoxConfig(a.log))
@ -345,7 +346,7 @@ func (a *Agent) UpdateSecret(ctx context.Context, w io.Writer, options *UpdateSe
var idOwner user.ID
user.IDFromKey(&idOwner, options.FrostFSKey.PrivateKey.PublicKey)
a.log.Info("update access cred object into FrostFS",
a.log.Info(logs.UpdateAccessCredObjectIntoFrostFS,
zap.Stringer("owner_tkn", idOwner))
oldAddr := options.Address

View file

@ -10,6 +10,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/spf13/viper"
@ -27,7 +28,7 @@ type PoolConfig struct {
}
func createFrostFS(ctx context.Context, log *zap.Logger, cfg PoolConfig) (authmate.FrostFS, error) {
log.Debug("prepare connection pool")
log.Debug(logs.PrepareConnectionPool)
var prm pool.InitParameters
prm.SetKey(&cfg.Key.PrivateKey)

View file

@ -3,7 +3,6 @@ package main
import (
"context"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"os"
@ -24,6 +23,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/services"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/xml"
@ -71,6 +71,7 @@ type (
xmlDecoder *xml.DecoderProvider
maxClient maxClientsConfig
bypassContentEncodingInChunks atomic.Bool
clientCut atomic.Bool
}
maxClientsConfig struct {
@ -130,7 +131,7 @@ func (a *App) initLayer(ctx context.Context) {
// prepare random key for anonymous requests
randomKey, err := keys.NewPrivateKey()
if err != nil {
a.log.Fatal("couldn't generate random key", zap.Error(err))
a.log.Fatal(logs.CouldntGenerateRandomKey, zap.Error(err))
}
var gateOwner user.ID
@ -144,6 +145,7 @@ func (a *App) initLayer(ctx context.Context) {
GateOwner: gateOwner,
Resolver: a.bucketResolver,
TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log),
Features: a.settings,
}
// prepare object layer
@ -153,29 +155,25 @@ func (a *App) initLayer(ctx context.Context) {
nopts := getNotificationsOptions(a.cfg, a.log)
a.nc, err = notifications.NewController(nopts, a.log)
if err != nil {
a.log.Fatal("failed to enable notifications", zap.Error(err))
a.log.Fatal(logs.FailedToEnableNotifications, zap.Error(err))
}
if err = a.obj.Initialize(ctx, a.nc); err != nil {
a.log.Fatal("couldn't initialize layer", zap.Error(err))
a.log.Fatal(logs.CouldntInitializeLayer, zap.Error(err))
}
}
}
func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
policies, err := newPlacementPolicy(log.logger, v)
if err != nil {
log.logger.Fatal("failed to create new policy mapping", zap.Error(err))
}
settings := &appSettings{
logLevel: log.lvl,
policies: policies,
policies: newPlacementPolicy(log.logger, v),
xmlDecoder: xml.NewDecoderProvider(v.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload)),
maxClient: newMaxClients(v),
}
settings.setBypassContentEncodingInChunks(v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
settings.setClientCut(v.GetBool(cfgClientCut))
return settings
}
@ -188,6 +186,14 @@ func (s *appSettings) setBypassContentEncodingInChunks(bypass bool) {
s.bypassContentEncodingInChunks.Store(bypass)
}
func (s *appSettings) ClientCut() bool {
return s.clientCut.Load()
}
func (s *appSettings) setClientCut(clientCut bool) {
s.clientCut.Store(clientCut)
}
func (a *App) initAPI(ctx context.Context) {
a.initLayer(ctx)
a.initHandler()
@ -202,7 +208,7 @@ func (a *App) initResolver() {
var err error
a.bucketResolver, err = resolver.NewBucketResolver(a.getResolverConfig())
if err != nil {
a.log.Fatal("failed to create resolver", zap.Error(err))
a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err))
}
}
@ -215,11 +221,11 @@ func (a *App) getResolverConfig() ([]string, *resolver.Config) {
order := a.cfg.GetStringSlice(cfgResolveOrder)
if resolveCfg.RPCAddress == "" {
order = remove(order, resolver.NNSResolver)
a.log.Warn(fmt.Sprintf("resolver '%s' won't be used since '%s' isn't provided", resolver.NNSResolver, cfgRPCEndpoint))
a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided)
}
if len(order) == 0 {
a.log.Info("container resolver will be disabled because of resolvers 'resolver_order' is empty")
a.log.Info(logs.ContainerResolverWillBeDisabled)
}
return order, resolveCfg
@ -240,10 +246,10 @@ func (a *App) initTracing(ctx context.Context) {
}
updated, err := tracing.Setup(ctx, cfg)
if err != nil {
a.log.Warn("failed to initialize tracing", zap.Error(err))
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err))
}
if updated {
a.log.Info("tracing config updated")
a.log.Info(logs.TracingConfigUpdated)
}
}
@ -253,7 +259,7 @@ func (a *App) shutdownTracing() {
defer cancel()
if err := tracing.Shutdown(shdnCtx); err != nil {
a.log.Warn("failed to shutdown tracing", zap.Error(err))
a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err))
}
}
@ -274,12 +280,12 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.
password := wallet.GetPassword(cfg, cfgWalletPassphrase)
key, err := wallet.GetKeyFromPath(cfg.GetString(cfgWalletPath), cfg.GetString(cfgWalletAddress), password)
if err != nil {
logger.Fatal("could not load FrostFS private key", zap.Error(err))
logger.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err))
}
prm.SetKey(&key.PrivateKey)
prmTree.SetKey(key)
logger.Info("using credentials", zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())))
logger.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())))
for _, peer := range fetchPeers(logger, cfg) {
prm.AddNode(peer)
@ -322,34 +328,28 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.
p, err := pool.NewPool(prm)
if err != nil {
logger.Fatal("failed to create connection pool", zap.Error(err))
logger.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err))
}
if err = p.Dial(ctx); err != nil {
logger.Fatal("failed to dial connection pool", zap.Error(err))
logger.Fatal(logs.FailedToDialConnectionPool, zap.Error(err))
}
treePool, err := treepool.NewPool(prmTree)
if err != nil {
logger.Fatal("failed to create tree pool", zap.Error(err))
logger.Fatal(logs.FailedToCreateTreePool, zap.Error(err))
}
if err = treePool.Dial(ctx); err != nil {
logger.Fatal("failed to dial tree pool", zap.Error(err))
logger.Fatal(logs.FailedToDialTreePool, zap.Error(err))
}
return p, treePool, key
}
func newPlacementPolicy(l *zap.Logger, v *viper.Viper) (*placementPolicy, error) {
policies := &placementPolicy{
regionMap: make(map[string]netmap.PlacementPolicy),
defaultCopiesNumbers: []uint32{handler.DefaultCopiesNumber},
}
policies.updateCopiesNumbers(l, v)
policies.updateDefaultCopiesNumbers(l, v)
return policies, policies.updatePolicy(v)
func newPlacementPolicy(l *zap.Logger, v *viper.Viper) *placementPolicy {
var policies placementPolicy
policies.update(l, v)
return &policies
}
func (p *placementPolicy) DefaultPlacementPolicy() netmap.PlacementPolicy {
@ -381,56 +381,18 @@ func (p *placementPolicy) DefaultCopiesNumbers() []uint32 {
}
func (p *placementPolicy) update(l *zap.Logger, v *viper.Viper) {
if err := p.updatePolicy(v); err != nil {
l.Warn("policies won't be updated", zap.Error(err))
}
p.updateCopiesNumbers(l, v)
p.updateDefaultCopiesNumbers(l, v)
}
func (p *placementPolicy) updatePolicy(v *viper.Viper) error {
defaultPlacementPolicy, err := fetchDefaultPolicy(v)
if err != nil {
return err
}
regionMap, err := fetchRegionMappingPolicies(v)
if err != nil {
return err
}
defaultPolicy := fetchDefaultPolicy(l, v)
regionMap := fetchRegionMappingPolicies(l, v)
defaultCopies := fetchDefaultCopiesNumbers(l, v)
copiesNumbers := fetchCopiesNumbers(l, v)
p.mu.Lock()
p.defaultPolicy = defaultPlacementPolicy
defer p.mu.Unlock()
p.defaultPolicy = defaultPolicy
p.regionMap = regionMap
p.mu.Unlock()
return nil
}
func (p *placementPolicy) updateCopiesNumbers(l *zap.Logger, v *viper.Viper) {
if newCopiesNumbers, err := fetchCopiesNumbers(l, v); err != nil {
l.Warn("copies numbers won't be updated", zap.Error(err))
} else {
p.mu.Lock()
p.copiesNumbers = newCopiesNumbers
p.mu.Unlock()
}
}
func (p *placementPolicy) updateDefaultCopiesNumbers(l *zap.Logger, v *viper.Viper) {
configuredValues, err := fetchDefaultCopiesNumbers(v)
if err == nil {
p.mu.Lock()
p.defaultCopiesNumbers = configuredValues
p.mu.Unlock()
l.Info("default copies numbers", zap.Uint32s("vector", p.defaultCopiesNumbers))
return
}
l.Error("cannot parse default copies numbers", zap.Error(err))
l.Warn("default copies numbers won't be updated", zap.Uint32s("current value", p.DefaultCopiesNumbers()))
p.defaultCopiesNumbers = defaultCopies
p.copiesNumbers = copiesNumbers
}
func remove(list []string, element string) []string {
@ -448,7 +410,7 @@ func remove(list []string, element string) []string {
// version (version.Version) and its name (frostfs-s3-gw). At the end, it writes
// about the stop to the log.
func (a *App) Wait() {
a.log.Info("application started",
a.log.Info(logs.ApplicationStarted,
zap.String("name", "frostfs-s3-gw"),
zap.String("version", version.Version),
)
@ -458,7 +420,7 @@ func (a *App) Wait() {
<-a.webDone // wait for web-server to be stopped
a.log.Info("application finished")
a.log.Info(logs.ApplicationFinished)
}
func (a *App) setHealthStatus() {
@ -469,7 +431,7 @@ func (a *App) setHealthStatus() {
func (a *App) Serve(ctx context.Context) {
// Attach S3 API:
domains := a.cfg.GetStringSlice(cfgListenDomains)
a.log.Info("fetch domains, prepare to use API", zap.Strings("domains", domains))
a.log.Info(logs.FetchDomainsPrepareToUseAPI, zap.Strings("domains", domains))
throttleOps := middleware.ThrottleOpts{
Limit: a.settings.maxClient.count,
@ -488,10 +450,10 @@ func (a *App) Serve(ctx context.Context) {
for i := range a.servers {
go func(i int) {
a.log.Info("starting server", zap.String("address", a.servers[i].Address()))
a.log.Info(logs.StartingServer, zap.String("address", a.servers[i].Address()))
if err := srv.Serve(a.servers[i].Listener()); err != nil && err != http.ErrServerClosed {
a.log.Fatal("listen and serve", zap.Error(err))
a.log.Fatal(logs.ListenAndServe, zap.Error(err))
}
}(i)
}
@ -512,7 +474,7 @@ LOOP:
ctx, cancel := shutdownContext()
defer cancel()
a.log.Info("stopping server", zap.Error(srv.Shutdown(ctx)))
a.log.Info(logs.StoppingServer, zap.Error(srv.Shutdown(ctx)))
a.metrics.Shutdown()
a.stopServices()
@ -526,23 +488,23 @@ func shutdownContext() (context.Context, context.CancelFunc) {
}
func (a *App) configReload(ctx context.Context) {
a.log.Info("SIGHUP config reload started")
a.log.Info(logs.SIGHUPConfigReloadStarted)
if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) {
a.log.Warn("failed to reload config because it's missed")
a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed)
return
}
if err := readInConfig(a.cfg); err != nil {
a.log.Warn("failed to reload config", zap.Error(err))
a.log.Warn(logs.FailedToReloadConfig, zap.Error(err))
return
}
if err := a.bucketResolver.UpdateResolvers(a.getResolverConfig()); err != nil {
a.log.Warn("failed to reload resolvers", zap.Error(err))
a.log.Warn(logs.FailedToReloadResolvers, zap.Error(err))
}
if err := a.updateServers(); err != nil {
a.log.Warn("failed to reload server parameters", zap.Error(err))
a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err))
}
a.stopServices()
@ -554,12 +516,12 @@ func (a *App) configReload(ctx context.Context) {
a.initTracing(ctx)
a.setHealthStatus()
a.log.Info("SIGHUP config reload completed")
a.log.Info(logs.SIGHUPConfigReloadCompleted)
}
func (a *App) updateSettings() {
if lvl, err := getLogLevel(a.cfg); err != nil {
a.log.Warn("log level won't be updated", zap.Error(err))
a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err))
} else {
a.settings.logLevel.SetLevel(lvl)
}
@ -568,6 +530,7 @@ func (a *App) updateSettings() {
a.settings.xmlDecoder.UseDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload))
a.settings.setBypassContentEncodingInChunks(a.cfg.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
a.settings.setClientCut(a.cfg.GetBool(cfgClientCut))
}
func (a *App) startServices() {
@ -593,16 +556,16 @@ func (a *App) initServers(ctx context.Context) {
}
srv, err := newServer(ctx, serverInfo)
if err != nil {
a.log.Warn("failed to add server", append(fields, zap.Error(err))...)
a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...)
continue
}
a.servers = append(a.servers, srv)
a.log.Info("add server", fields...)
a.log.Info(logs.AddServer, fields...)
}
if len(a.servers) == 0 {
a.log.Fatal("no healthy servers")
a.log.Fatal(logs.NoHealthyServers)
}
}
@ -713,29 +676,6 @@ func (a *App) initHandler() {
var err error
a.api, err = handler.New(a.log, a.obj, a.nc, cfg)
if err != nil {
a.log.Fatal("could not initialize API handler", zap.Error(err))
a.log.Fatal(logs.CouldNotInitializeAPIHandler, zap.Error(err))
}
}
func readRegionMap(filePath string) (map[string]string, error) {
regionMap := make(map[string]string)
if filePath == "" {
return regionMap, nil
}
data, err := os.ReadFile(filePath)
if err != nil {
return nil, fmt.Errorf("coudln't read file '%s'", filePath)
}
if err = json.Unmarshal(data, &regionMap); err != nil {
return nil, fmt.Errorf("unmarshal policies: %w", err)
}
if _, ok := regionMap[api.DefaultLocationConstraint]; ok {
return nil, fmt.Errorf("config overrides %s location constraint", api.DefaultLocationConstraint)
}
return regionMap, nil
}

View file

@ -1,6 +1,7 @@
package main
import (
"encoding/json"
"fmt"
"os"
"path"
@ -10,9 +11,11 @@ import (
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/notifications"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
@ -30,11 +33,14 @@ const (
defaultShutdownTimeout = 15 * time.Second
defaultPoolErrorThreshold uint32 = 100
defaultPlacementPolicy = "REP 3"
defaultMaxClientsCount = 100
defaultMaxClientsDeadline = time.Second * 30
)
var defaultCopiesNumbers = []uint32{0}
const ( // Settings.
// Logger.
cfgLoggerLevel = "logger.level"
@ -138,6 +144,8 @@ const ( // Settings.
// Configuration of parameters of requests to FrostFS.
// Number of the object copies to consider PUT to FrostFS successful.
cfgSetCopiesNumber = "frostfs.set_copies_number"
// Enabling client side object preparing for PUT operations.
cfgClientCut = "frostfs.client_cut"
// List of allowed AccessKeyID prefixes.
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
@ -222,24 +230,30 @@ func fetchMaxClientsDeadline(cfg *viper.Viper) time.Duration {
return maxClientsDeadline
}
func fetchDefaultPolicy(cfg *viper.Viper) (netmap.PlacementPolicy, error) {
defaultPolicyStr := handler.DefaultPolicy
func fetchDefaultPolicy(l *zap.Logger, cfg *viper.Viper) netmap.PlacementPolicy {
var policy netmap.PlacementPolicy
if cfg.IsSet(cfgPolicyDefault) {
defaultPolicyStr = cfg.GetString(cfgPolicyDefault)
policyStr := cfg.GetString(cfgPolicyDefault)
if err := policy.DecodeString(policyStr); err != nil {
l.Warn(logs.FailedToParseDefaultLocationConstraint,
zap.String("policy", policyStr), zap.String("default", defaultPlacementPolicy), zap.Error(err))
} else {
return policy
}
}
var defaultPlacementPolicy netmap.PlacementPolicy
if err := defaultPlacementPolicy.DecodeString(defaultPolicyStr); err != nil {
return netmap.PlacementPolicy{}, fmt.Errorf("parse default policy '%s': %w", defaultPolicyStr, err)
if err := policy.DecodeString(defaultPlacementPolicy); err != nil {
l.Fatal(logs.FailedToParseDefaultDefaultLocationConstraint, zap.String("policy", defaultPlacementPolicy))
}
return defaultPlacementPolicy, nil
return policy
}
func fetchNATSTimeout(cfg *viper.Viper, l *zap.Logger) time.Duration {
timeout := cfg.GetDuration(cfgNATSTimeout)
if timeout <= 0 {
l.Error("invalid lifetime, using default value (in seconds)",
l.Error(logs.InvalidLifetimeUsingDefaultValue,
zap.String("parameter", cfgNATSTimeout),
zap.Duration("value in config", timeout),
zap.Duration("default", notifications.DefaultTimeout))
@ -253,7 +267,7 @@ func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultV
if v.IsSet(cfgEntry) {
lifetime := v.GetDuration(cfgEntry)
if lifetime <= 0 {
l.Error("invalid lifetime, using default value (in seconds)",
l.Error(logs.InvalidLifetimeUsingDefaultValue,
zap.String("parameter", cfgEntry),
zap.Duration("value in config", lifetime),
zap.Duration("default", defaultValue))
@ -269,7 +283,7 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue
if v.IsSet(cfgEntry) {
size := v.GetInt(cfgEntry)
if size <= 0 {
l.Error("invalid cache size, using default value",
l.Error(logs.InvalidCacheSizeUsingDefaultValue,
zap.String("parameter", cfgEntry),
zap.Int("value in config", size),
zap.Int("default", defaultValue))
@ -288,7 +302,7 @@ func fetchDefaultMaxAge(cfg *viper.Viper, l *zap.Logger) int {
defaultMaxAge = cfg.GetInt(cfgDefaultMaxAge)
if defaultMaxAge <= 0 && defaultMaxAge != -1 {
l.Fatal("invalid defaultMaxAge",
l.Fatal(logs.InvalidDefaultMaxAge,
zap.String("parameter", cfgDefaultMaxAge),
zap.String("value in config", strconv.Itoa(defaultMaxAge)))
}
@ -297,14 +311,21 @@ func fetchDefaultMaxAge(cfg *viper.Viper, l *zap.Logger) int {
return defaultMaxAge
}
func fetchRegionMappingPolicies(cfg *viper.Viper) (map[string]netmap.PlacementPolicy, error) {
regionPolicyMap, err := readRegionMap(cfg.GetString(cfgPolicyRegionMapFile))
func fetchRegionMappingPolicies(l *zap.Logger, cfg *viper.Viper) map[string]netmap.PlacementPolicy {
filepath := cfg.GetString(cfgPolicyRegionMapFile)
regionPolicyMap, err := readRegionMap(filepath)
if err != nil {
return nil, fmt.Errorf("read region map file: %w", err)
l.Warn(logs.FailedToReadRegionMapFilePolicies, zap.String("file", filepath), zap.Error(err))
return make(map[string]netmap.PlacementPolicy)
}
regionMap := make(map[string]netmap.PlacementPolicy, len(regionPolicyMap))
for region, policy := range regionPolicyMap {
if region == api.DefaultLocationConstraint {
l.Warn(logs.DefaultLocationConstraintCantBeOverriden, zap.String("policy", policy))
continue
}
var pp netmap.PlacementPolicy
if err = pp.DecodeString(policy); err == nil {
regionMap[region] = pp
@ -316,29 +337,49 @@ func fetchRegionMappingPolicies(cfg *viper.Viper) (map[string]netmap.PlacementPo
continue
}
return nil, fmt.Errorf("parse region '%s' to policy mapping: %w", region, err)
l.Warn(logs.FailedToParseLocationConstraint, zap.String("region", region), zap.String("policy", policy))
}
return regionMap
}
func readRegionMap(filePath string) (map[string]string, error) {
regionMap := make(map[string]string)
if filePath == "" {
return regionMap, nil
}
data, err := os.ReadFile(filePath)
if err != nil {
return nil, fmt.Errorf("coudln't read file '%s'", filePath)
}
if err = json.Unmarshal(data, &regionMap); err != nil {
return nil, fmt.Errorf("unmarshal policies: %w", err)
}
return regionMap, nil
}
func fetchDefaultCopiesNumbers(v *viper.Viper) ([]uint32, error) {
func fetchDefaultCopiesNumbers(l *zap.Logger, v *viper.Viper) []uint32 {
unparsed := v.GetStringSlice(cfgSetCopiesNumber)
var result []uint32
result := make([]uint32, len(unparsed))
for i := range unparsed {
parsedValue, err := strconv.ParseUint(unparsed[i], 10, 32)
if err != nil {
return nil, err
l.Warn(logs.FailedToParseDefaultCopiesNumbers,
zap.Strings("copies numbers", unparsed), zap.Uint32s("default", defaultCopiesNumbers), zap.Error(err))
return defaultCopiesNumbers
}
result = append(result, uint32(parsedValue))
result[i] = uint32(parsedValue)
}
return result, nil
return result
}
func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) (map[string][]uint32, error) {
var copiesNums = make(map[string][]uint32)
func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) map[string][]uint32 {
copiesNums := make(map[string][]uint32)
for i := 0; ; i++ {
key := cfgCopiesNumbers + "." + strconv.Itoa(i) + "."
constraint := v.GetString(key + "location_constraint")
@ -352,15 +393,17 @@ func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) (map[string][]uint32, err
for j := range vector {
parsedValue, err := strconv.ParseUint(vector[j], 10, 32)
if err != nil {
return nil, err
l.Warn(logs.FailedToParseCopiesNumbers, zap.String("location", constraint),
zap.Strings("copies numbers", vector), zap.Error(err))
continue
}
vector32[j] = uint32(parsedValue)
}
copiesNums[constraint] = vector32
l.Info("constraint added", zap.String("location", constraint), zap.Strings("copies numbers", vector))
l.Info(logs.ConstraintAdded, zap.String("location", constraint), zap.Strings("copies numbers", vector))
}
return copiesNums, nil
return copiesNums
}
func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
@ -372,7 +415,7 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
priority := v.GetInt(key + "priority")
if address == "" {
l.Warn("skip, empty address")
l.Warn(logs.SkipEmptyAddress)
break
}
if weight <= 0 { // unspecified or wrong
@ -384,7 +427,7 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
nodes = append(nodes, pool.NewNodeParam(priority, address, weight))
l.Info("added storage peer",
l.Info(logs.AddedStoragePeer,
zap.Int("priority", priority),
zap.String("address", address),
zap.Float64("weight", weight))

View file

@ -4,6 +4,7 @@ import (
"context"
"net/http"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"go.uber.org/zap"
)
@ -18,21 +19,21 @@ type Service struct {
// Start runs http service with the exposed endpoint on the configured port.
func (ms *Service) Start() {
if ms.enabled {
ms.log.Info("service is running", zap.String("endpoint", ms.Addr))
ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr))
err := ms.ListenAndServe()
if err != nil && err != http.ErrServerClosed {
ms.log.Warn("service couldn't start on configured port")
ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort)
}
} else {
ms.log.Info("service hasn't started since it's disabled")
ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled)
}
}
// ShutDown stops the service.
func (ms *Service) ShutDown(ctx context.Context) {
ms.log.Info("shutting down service", zap.String("endpoint", ms.Addr))
ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr))
err := ms.Shutdown(ctx)
if err != nil {
ms.log.Panic("can't shut down service")
ms.log.Panic(logs.CantShutDownService)
}
}

View file

@ -125,6 +125,8 @@ S3_GW_CORS_DEFAULT_MAX_AGE=600
# to consider PUT to FrostFS successful.
# `0` or empty list means that object will be processed according to the container's placement policy
S3_GW_FROSTFS_SET_COPIES_NUMBER=0
# This flag enables client side object preparing.
S3_GW_FROSTFS_CLIENT_CUT=false
# List of allowed AccessKeyID prefixes
# If not set, S3 GW will accept all AccessKeyIDs

View file

@ -150,6 +150,8 @@ frostfs:
# Numbers of the object copies (for each replica) to consider PUT to FrostFS successful.
# `[0]` or empty list means that object will be processed according to the container's placement policy
set_copies_number: [0]
# This flag enables client side object preparing.
client_cut: false
# List of allowed AccessKeyID prefixes
# If the parameter is omitted, S3 GW will accept all AccessKeyIDs

View file

@ -500,17 +500,20 @@ tracing:
# `frostfs` section
Contains parameters of requests to FrostFS.
This value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
The `set_copies_number` value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
header for `PutObject`, `CopyObject`, `CreateMultipartUpload`.
```yaml
frostfs:
set_copies_number: [0]
client_cut: false
```
| Parameter | Type | Default value | Description |
|---------------------|------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `set_copies_number` | `[]uint32` | `[0]` | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful. <br/>Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
| Parameter | Type | SIGHUP reload | Default value | Description |
|---------------------|------------|---------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `set_copies_number` | `[]uint32` | yes | `[0]` | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful. <br/>Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
| `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. |
# `resolve_bucket` section

2
go.mod
View file

@ -1,6 +1,6 @@
module git.frostfs.info/TrueCloudLab/frostfs-s3-gw
go 1.19
go 1.20
require (
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44

2
go.sum
View file

@ -44,8 +44,6 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3 h1:GBRTOTRrtIvxi2TgxG7z/J7uRXiyb1SxR4247FaYCgU=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05 h1:OuViMF54N87FXmaBEpYw3jhzaLrJ/EWOlPL1wUkimE0=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=

View file

@ -243,6 +243,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (
prmPut.SetHeader(*obj)
prmPut.SetPayload(prm.Payload)
prmPut.SetCopiesNumberVector(prm.CopiesNumber)
prmPut.SetClientCut(prm.ClientCut)
if prm.BearerToken != nil {
prmPut.UseBearer(*prm.BearerToken)
@ -262,6 +263,9 @@ type payloadReader struct {
func (x payloadReader) Read(p []byte) (int, error) {
n, err := x.ReadCloser.Read(p)
if err != nil && errors.Is(err, io.EOF) {
return n, err
}
return n, handleObjectError("read payload", err)
}

115
internal/logs/logs.go Normal file
View file

@ -0,0 +1,115 @@
package logs
const (
RequestUnmatched = "request unmatched" // Error in ../../api/router.go
CheckContainer = "check container" // Info in ../../authmate/authmate.go
CreateContainer = "create container" // Info in ../../authmate/authmate.go
StoreBearerTokenIntoFrostFS = "store bearer token into FrostFS" // Info in ../../authmate/authmate.go
UpdateAccessCredObjectIntoFrostFS = "update access cred object into FrostFS" // Info in ../../authmate/authmate.go
MetricsAreDisabled = "metrics are disabled" // Warn in ../../metrics/app.go
FoundMoreThanOneUnversionedNode = "found more than one unversioned node" // Debug in ../../pkg/service/tree/tree.go
ServiceIsRunning = "service is running" // Info in ../../cmd/s3-gw/service.go
ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../cmd/s3-gw/service.go
ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../cmd/s3-gw/service.go
ShuttingDownService = "shutting down service" // Info in ../../cmd/s3-gw/service.go
ContainerResolverWillBeDisabled = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../cmd/s3-gw/app.go
FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../cmd/s3-gw/app.go
TracingConfigUpdated = "tracing config updated" // Info in ../../cmd/s3-gw/app.go
FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../cmd/s3-gw/app.go
UsingCredentials = "using credentials" // Info in ../../cmd/s3-gw/app.go
ApplicationStarted = "application started" // Info in ../../cmd/s3-gw/app.go
ApplicationFinished = "application finished" // Info in ../../cmd/s3-gw/app.go
FetchDomainsPrepareToUseAPI = "fetch domains, prepare to use API" // Info in ../../cmd/s3-gw/app.go
StartingServer = "starting server" // Info in ../../cmd/s3-gw/app.go
StoppingServer = "stopping server" // Info in ../../cmd/s3-gw/app.go
SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../cmd/s3-gw/app.go
FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../cmd/s3-gw/app.go
FailedToReloadConfig = "failed to reload config" // Warn in ../../cmd/s3-gw/app.go
FailedToReloadResolvers = "failed to reload resolvers" // Warn in ../../cmd/s3-gw/app.go
FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../cmd/s3-gw/app.go
SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../cmd/s3-gw/app.go
LogLevelWontBeUpdated = "log level won't be updated" // Warn in ../../cmd/s3-gw/app.go
FailedToAddServer = "failed to add server" // Warn in ../../cmd/s3-gw/app.go
AddServer = "add server" // Info in ../../cmd/s3-gw/app.go
ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver 'nns' won't be used since 'rpc_endpoint' isn't provided" // Warn in ../../cmd/s3-gw/app.go
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/s3-gw/app_settings.go
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/s3-gw/app_settings.go
FailedToParseDefaultLocationConstraint = "failed to parse 'default' location constraint, default one will be used" // Warn in cmd/s3-gw/app_settings.go
FailedToReadRegionMapFilePolicies = "failed to read region map file, policies will be empty" // Warn in cmd/s3-gw/app_settings.go
DefaultLocationConstraintCantBeOverriden = "'default' location constraint can't be overriden by custom policy, use 'placement_policy.default'" // Warn in cmd/s3-gw/app_settings.go
FailedToParseLocationConstraint = "failed to parse location constraint, it cannot be used" // Warn in cmd/s3-gw/app_settings.go
FailedToParseDefaultCopiesNumbers = "failed to parse 'default' copies numbers, default one will be used" // Warn in cmd/s3-gw/app_settings.go
FailedToParseCopiesNumbers = "failed to parse copies numbers, skip" // Warn in cmd/s3-gw/app_settings.go
FailedToParseDefaultDefaultLocationConstraint = "failed to parse default 'default' location constraint" // Fatal in cmd/s3-gw/app_settings.go
ConstraintAdded = "constraint added" // Info in ../../cmd/s3-gw/app_settings.go
SkipEmptyAddress = "skip, empty address" // Warn in ../../cmd/s3-gw/app_settings.go
AddedStoragePeer = "added storage peer" // Info in ../../cmd/s3-gw/app_settings.go
PrepareConnectionPool = "prepare connection pool" // Debug in ../../cmd/s3-authmate/modules/utils.go
InvalidCacheEntryType = "invalid cache entry type" // Warn in ../../api/cache/*
InvalidCacheKeyType = "invalid cache key type" // Warn in ../../api/cache/objectslist.go
ObjectIsCopied = "object is copied" // Info in ../../api/handler/copy.go
CouldntSendNotification = "couldn't send notification: %w" // Error in ../../api/handler/*
FailedToSendTestEventBecauseNotificationsIsDisabled = "failed to send test event because notifications is disabled" // Warn in ../../api/handler/notifications.go
RequestFailed = "request failed" // Error in ../../api/handler/util.go
GetBucketInfo = "get bucket info" // Warn in ../../api/handler/cors.go
GetBucketCors = "get bucket cors" // Warn in ../../api/handler/cors.go
SomeACLNotFullyMapped = "some acl not fully mapped" // Warn in ../../api/handler/acl.go
CouldntDeleteObjects = "couldn't delete objects" // Error in ../../api/handler/delete.go
NotificatorIsDisabledS3WontProduceNotificationEvents = "notificator is disabled, s3 won't produce notification events" // Warn in ../../api/handler/api.go
CouldntGetBucketVersioning = "couldn't get bucket versioning" // Warn in ../../api/handler/put.go
BucketIsCreated = "bucket is created" // Info in ../../api/handler/put.go
CouldntDeleteNotificationConfigurationObject = "couldn't delete notification configuration object" // Error in ../../api/layer/notifications.go
CouldNotParseContainerObjectLockEnabledAttribute = "could not parse container object lock enabled attribute" // Error in ../../api/layer/container.go
CouldNotListUserContainers = "could not list user containers" // Error in ../../api/layer/container.go
CouldNotFetchContainerInfo = "could not fetch container info" // Error in ../../api/layer/container.go
MismatchedObjEncryptionInfo = "mismatched obj encryptionInfo" // Warn in ../../api/layer/multipart_upload.go
UploadPart = "upload part" // Debug in ../../api/layer/multipart_upload.go
CouldntDeleteOldPartObject = "couldn't delete old part object" // Error in ../../api/layer/multipart_upload.go
CouldNotPutCompletedObject = "could not put a completed object (multipart upload)" // Error in ../../api/layer/multipart_upload.go
CouldNotDeleteUploadPart = "could not delete upload part" // Warn in ../../api/layer/multipart_upload.go
CouldntDeletePart = "couldn't delete part" // Warn in ../../api/layer/multipart_upload.go
PartDetails = "part details" // Debug in ../../api/layer/multipart_upload.go
GetObject = "get object" // Debug in ../../api/layer/layer.go
ObjectAlreadyRemoved = "object already removed" // Debug in ../../api/layer/layer.go
ObjectNotFound = "object not found" // Debug in ../../api/layer/layer.go
ResolveBucket = "resolve bucket" // Info in ../../api/layer/layer.go
CouldntDeleteCorsObject = "couldn't delete cors object" // Error in ../../api/layer/cors.go
PutObject = "put object" // Debug in ../../api/layer/object.go
FailedToDiscardPutPayloadProbablyGoroutineLeaks = "failed to discard put payload, probably goroutine leaks" // Warn in ../../api/layer/object.go
FailedToSubmitTaskToPool = "failed to submit task to pool" // Warn in ../../api/layer/object.go
CouldNotFetchObjectMeta = "could not fetch object meta" // Warn in ../../api/layer/object.go
GetTreeNode = "get tree node" // Debug in ../../api/layer/tagging.go
CouldntPutBucketInfoIntoCache = "couldn't put bucket info into cache" // Warn in ../../api/layer/cache.go
CouldntAddObjectToCache = "couldn't add object to cache" // Warn in ../../api/layer/cache.go
CouldntCacheAccessControlOperation = "couldn't cache access control operation" // Warn in ../../api/layer/cache.go
CouldntPutObjAddressToNameCache = "couldn't put obj address to name cache" // Warn in ../../api/layer/cache.go
CouldntCacheListOfObjects = "couldn't cache list of objects" // Warn in ../../api/layer/cache.go
CouldntCacheTags = "couldn't cache tags" // Error in ../../api/layer/cache.go
CouldntCacheLockInfo = "couldn't cache lock info" // Error in ../../api/layer/cache.go
CouldntCacheBucketSettings = "couldn't cache bucket settings" // Warn in ../../api/layer/cache.go
CouldntCacheCors = "couldn't cache cors" // Warn in ../../api/layer/cache.go
CouldntCacheNotificationConfiguration = "couldn't cache notification configuration" // Warn in ../../api/layer/cache.go
RequestEnd = "request end" // Info in ../../api/middleware/response.go
CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed = "couldn't receive access box for gate key, random key will be used" // Debug in ../../api/middleware/auth.go
FailedToPassAuthentication = "failed to pass authentication" // Error in ../../api/middleware/auth.go
FailedToResolveCID = "failed to resolve CID" // Debug in ../../api/middleware/metrics.go
RequestStart = "request start" // Info in ../../api/middleware/reqinfo.go
CouldNotHandleMessage = "could not handle message" // Error in ../../api/notifications/controller.go
CouldNotACKMessage = "could not ACK message" // Error in ../../api/notifications/controller.go
CouldntMarshalAnEvent = "couldn't marshal an event" // Error in ../../api/notifications/controller.go
CouldntSendAnEventToTopic = "couldn't send an event to topic" // Error in ../../api/notifications/controller.go
InvalidDefaultMaxAge = "invalid defaultMaxAge" // Fatal in ../../cmd/s3-gw/app_settings.go
CantShutDownService = "can't shut down service" // Panic in ../../cmd/s3-gw/service.go
CouldntGenerateRandomKey = "couldn't generate random key" // Fatal in ../../cmd/s3-gw/app.go
FailedToEnableNotifications = "failed to enable notifications" // Fatal in ../../cmd/s3-gw/app.go
CouldntInitializeLayer = "couldn't initialize layer" // Fatal in ../../cmd/s3-gw/app.go
FailedToCreateResolver = "failed to create resolver" // Fatal in ../../cmd/s3-gw/app.go
CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../cmd/s3-gw/app.go
FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../cmd/s3-gw/app.go
FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../cmd/s3-gw/app.go
FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../cmd/s3-gw/app.go
FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../cmd/s3-gw/app.go
ListenAndServe = "listen and serve" // Fatal in ../../cmd/s3-gw/app.go
NoHealthyServers = "no healthy servers" // Fatal in ../../cmd/s3-gw/app.go
CouldNotInitializeAPIHandler = "could not initialize API handler" // Fatal in ../../cmd/s3-gw/app.go
)

View file

@ -4,6 +4,7 @@ import (
"net/http"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
dto "github.com/prometheus/client_model/go"
"go.uber.org/zap"
)
@ -17,7 +18,7 @@ type AppMetrics struct {
func NewAppMetrics(logger *zap.Logger, poolStatistics StatisticScraper, enabled bool) *AppMetrics {
if !enabled {
logger.Warn("metrics are disabled")
logger.Warn(logs.MetricsAreDisabled)
}
return &AppMetrics{
logger: logger,
@ -28,7 +29,7 @@ func NewAppMetrics(logger *zap.Logger, poolStatistics StatisticScraper, enabled
func (m *AppMetrics) SetEnabled(enabled bool) {
if !enabled {
m.logger.Warn("metrics are disabled")
m.logger.Warn(logs.MetricsAreDisabled)
}
m.mu.Lock()

View file

@ -12,6 +12,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"go.uber.org/zap"
@ -220,6 +221,30 @@ func newNodeVersionFromTreeNode(filePath string, treeNode *treeNode) *data.NodeV
return version
}
func newMultipartInfoFromTreeNode(filePath string, treeNode *treeNode) (*data.MultipartInfo, error) {
uploadID, _ := treeNode.Get(uploadIDKV)
if uploadID == "" {
return nil, fmt.Errorf("it's not a multipart node")
}
multipartInfo := &data.MultipartInfo{
ID: treeNode.ID,
Key: filePath,
UploadID: uploadID,
Meta: treeNode.Meta,
}
ownerID, _ := treeNode.Get(ownerKV)
_ = multipartInfo.Owner.DecodeString(ownerID)
created, _ := treeNode.Get(createdKV)
if utcMilli, err := strconv.ParseInt(created, 10, 64); err == nil {
multipartInfo.Created = time.UnixMilli(utcMilli)
}
return multipartInfo, nil
}
func newMultipartInfo(node NodeResponse) (*data.MultipartInfo, error) {
multipartInfo := &data.MultipartInfo{
ID: node.GetNodeID(),
@ -829,7 +854,7 @@ func (c *Tree) getUnversioned(ctx context.Context, bktInfo *data.BucketInfo, tre
}
if len(nodes) > 1 {
c.reqLogger(ctx).Debug("found more than one unversioned node",
c.reqLogger(ctx).Debug(logs.FoundMoreThanOneUnversionedNode,
zap.String("treeID", treeID), zap.String("filepath", filepath))
}
@ -857,14 +882,14 @@ func (c *Tree) CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketIn
}
func (c *Tree) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.MultipartInfo, error) {
subTreeNodes, _, err := c.getSubTreeByPrefix(ctx, bktInfo, systemTree, prefix, false)
subTreeNodes, headPrefix, err := c.getSubTreeByPrefix(ctx, bktInfo, systemTree, prefix, false)
if err != nil {
return nil, err
}
var result []*data.MultipartInfo
for _, node := range subTreeNodes {
multipartUploads, err := c.getSubTreeMultipartUploads(ctx, bktInfo, node.GetNodeID())
multipartUploads, err := c.getSubTreeMultipartUploads(ctx, bktInfo, node.GetNodeID(), headPrefix)
if err != nil {
return nil, err
}
@ -874,19 +899,55 @@ func (c *Tree) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.Bu
return result, nil
}
func (c *Tree) getSubTreeMultipartUploads(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) ([]*data.MultipartInfo, error) {
func (c *Tree) getSubTreeMultipartUploads(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, parentFilePath string) ([]*data.MultipartInfo, error) {
subTree, err := c.service.GetSubTree(ctx, bktInfo, systemTree, nodeID, maxGetSubTreeDepth)
if err != nil {
return nil, err
}
result := make([]*data.MultipartInfo, 0, len(subTree))
for _, node := range subTree {
multipartInfo, err := newMultipartInfo(node)
if err != nil { // missed uploadID (it's a part node)
var parentPrefix string
if parentFilePath != "" { // The root of subTree can also have a parent
parentPrefix = strings.TrimSuffix(parentFilePath, separator) + separator // To avoid 'foo//bar'
}
var filepath string
namesMap := make(map[uint64]string, len(subTree))
multiparts := make(map[string][]*data.MultipartInfo, len(subTree))
for i, node := range subTree {
treeNode, fileName, err := parseTreeNode(node)
if err != nil {
continue
}
result = append(result, multipartInfo)
if i != 0 {
if filepath, err = formFilePath(node, fileName, namesMap); err != nil {
return nil, fmt.Errorf("invalid node order: %w", err)
}
} else {
filepath = parentPrefix + fileName
namesMap[treeNode.ID] = filepath
}
multipartInfo, err := newMultipartInfoFromTreeNode(filepath, treeNode)
if err != nil {
continue
}
key := formLatestNodeKey(node.GetParentID(), fileName)
multipartInfos, ok := multiparts[key]
if !ok {
multipartInfos = []*data.MultipartInfo{multipartInfo}
} else {
multipartInfos = append(multipartInfos, multipartInfo)
}
multiparts[key] = multipartInfos
}
result := make([]*data.MultipartInfo, 0, len(multiparts))
for _, multipartInfo := range multiparts {
result = append(result, multipartInfo...)
}
return result, nil