Compare commits
1 commit
master
...
fix/govuln
Author | SHA1 | Date | |
---|---|---|---|
2f46ae4beb |
93 changed files with 953 additions and 2469 deletions
|
@ -16,7 +16,8 @@ jobs:
|
||||||
- name: Setup Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@v3
|
uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: '1.23.6'
|
go-version: '1.23'
|
||||||
|
check-latest: true
|
||||||
|
|
||||||
- name: Install govulncheck
|
- name: Install govulncheck
|
||||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||||
|
|
47
CHANGELOG.md
47
CHANGELOG.md
|
@ -4,44 +4,6 @@ This document outlines major changes between releases.
|
||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
## [0.32.11] - 2025-02-28
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- ListObjects could return empty result from priority storage node with failed shard (#651)
|
|
||||||
|
|
||||||
## [0.32.10] - 2025-02-14
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Chunk streaming empty body (#642)
|
|
||||||
|
|
||||||
## [0.32.9] - 2025-02-12
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Make `Content-Md5` header check optional (#612)
|
|
||||||
|
|
||||||
## [0.32.8] - 2025-02-11
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Return 404 instead of 500 when object is missing in object storage and available in the tree (#626)
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- `tree_stream_timeout` configuration parameter (#627)
|
|
||||||
|
|
||||||
## [0.32.7] - 2025-02-06
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Correct passing copies number during multipart upload (#623)
|
|
||||||
|
|
||||||
## [0.32.6] - 2025-02-05
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Connection leak when `feature.tree_pool_netmap_support` is enabled (#622)
|
|
||||||
|
|
||||||
## [0.32.5] - 2025-02-04
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Support trailing headers signature during aws-chunk upload (#607)
|
|
||||||
|
|
||||||
## [0.32.4] - 2025-02-03
|
## [0.32.4] - 2025-02-03
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
@ -457,11 +419,4 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs
|
||||||
[0.32.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.1...v0.32.2
|
[0.32.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.1...v0.32.2
|
||||||
[0.32.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.2...v0.32.3
|
[0.32.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.2...v0.32.3
|
||||||
[0.32.4]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.3...v0.32.4
|
[0.32.4]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.3...v0.32.4
|
||||||
[0.32.5]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.4...v0.32.5
|
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.4...master
|
||||||
[0.32.6]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.5...v0.32.6
|
|
||||||
[0.32.7]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.6...v0.32.7
|
|
||||||
[0.32.8]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.7...v0.32.8
|
|
||||||
[0.32.9]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.8...v0.32.9
|
|
||||||
[0.32.10]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.9...v0.32.10
|
|
||||||
[0.32.11]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.10...v0.32.11
|
|
||||||
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.11...master
|
|
2
VERSION
2
VERSION
|
@ -1 +1 @@
|
||||||
v0.32.11
|
v0.32.4
|
||||||
|
|
|
@ -33,8 +33,8 @@ var (
|
||||||
// AuthorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
|
// AuthorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
|
||||||
AuthorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
|
AuthorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
|
||||||
|
|
||||||
// AuthorizationFieldV4aRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
|
// authorizationFieldV4aRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
|
||||||
AuthorizationFieldV4aRegexp = regexp.MustCompile(`AWS4-ECDSA-P256-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
|
authorizationFieldV4aRegexp = regexp.MustCompile(`AWS4-ECDSA-P256-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
|
||||||
|
|
||||||
// postPolicyCredentialRegexp -- is regexp for credentials when uploading file using POST with policy.
|
// postPolicyCredentialRegexp -- is regexp for credentials when uploading file using POST with policy.
|
||||||
postPolicyCredentialRegexp = regexp.MustCompile(`(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request`)
|
postPolicyCredentialRegexp = regexp.MustCompile(`(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request`)
|
||||||
|
@ -107,7 +107,7 @@ func New(creds tokens.Credentials, prefixes []string, settings CenterSettings) *
|
||||||
return &Center{
|
return &Center{
|
||||||
cli: creds,
|
cli: creds,
|
||||||
reg: NewRegexpMatcher(AuthorizationFieldRegexp),
|
reg: NewRegexpMatcher(AuthorizationFieldRegexp),
|
||||||
regV4a: NewRegexpMatcher(AuthorizationFieldV4aRegexp),
|
regV4a: NewRegexpMatcher(authorizationFieldV4aRegexp),
|
||||||
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
|
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
|
||||||
allowedAccessKeyIDPrefixes: prefixes,
|
allowedAccessKeyIDPrefixes: prefixes,
|
||||||
settings: settings,
|
settings: settings,
|
||||||
|
@ -115,8 +115,8 @@ func New(creds tokens.Credentials, prefixes []string, settings CenterSettings) *
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
SignaturePreambleSigV4 = "AWS4-HMAC-SHA256"
|
signaturePreambleSigV4 = "AWS4-HMAC-SHA256"
|
||||||
SignaturePreambleSigV4A = "AWS4-ECDSA-P256-SHA256"
|
signaturePreambleSigV4A = "AWS4-ECDSA-P256-SHA256"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (c *Center) parseAuthHeader(authHeader string, headers http.Header) (*AuthHeader, error) {
|
func (c *Center) parseAuthHeader(authHeader string, headers http.Header) (*AuthHeader, error) {
|
||||||
|
@ -128,13 +128,13 @@ func (c *Center) parseAuthHeader(authHeader string, headers http.Header) (*AuthH
|
||||||
)
|
)
|
||||||
|
|
||||||
switch preamble {
|
switch preamble {
|
||||||
case SignaturePreambleSigV4:
|
case signaturePreambleSigV4:
|
||||||
submatches = c.reg.GetSubmatches(authHeader)
|
submatches = c.reg.GetSubmatches(authHeader)
|
||||||
if len(submatches) != authHeaderPartsNum {
|
if len(submatches) != authHeaderPartsNum {
|
||||||
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrAuthorizationHeaderMalformed), authHeader)
|
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrAuthorizationHeaderMalformed), authHeader)
|
||||||
}
|
}
|
||||||
region = submatches["region"]
|
region = submatches["region"]
|
||||||
case SignaturePreambleSigV4A:
|
case signaturePreambleSigV4A:
|
||||||
submatches = c.regV4a.GetSubmatches(authHeader)
|
submatches = c.regV4a.GetSubmatches(authHeader)
|
||||||
if len(submatches) != authHeaderV4aPartsNum {
|
if len(submatches) != authHeaderV4aPartsNum {
|
||||||
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrAuthorizationHeaderMalformed), authHeader)
|
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrAuthorizationHeaderMalformed), authHeader)
|
||||||
|
@ -161,7 +161,7 @@ func IsStandardContentSHA256(key string) bool {
|
||||||
return ok
|
return ok
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Center) Authenticate(ctx context.Context, r *http.Request) (*middleware.Box, error) {
|
func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
authHdr *AuthHeader
|
authHdr *AuthHeader
|
||||||
|
@ -170,7 +170,7 @@ func (c *Center) Authenticate(ctx context.Context, r *http.Request) (*middleware
|
||||||
)
|
)
|
||||||
|
|
||||||
queryValues := r.URL.Query()
|
queryValues := r.URL.Query()
|
||||||
if queryValues.Get(AmzAlgorithm) == SignaturePreambleSigV4 {
|
if queryValues.Get(AmzAlgorithm) == signaturePreambleSigV4 {
|
||||||
creds := strings.Split(queryValues.Get(AmzCredential), "/")
|
creds := strings.Split(queryValues.Get(AmzCredential), "/")
|
||||||
if len(creds) != 5 || creds[4] != "aws4_request" {
|
if len(creds) != 5 || creds[4] != "aws4_request" {
|
||||||
return nil, fmt.Errorf("bad X-Amz-Credential")
|
return nil, fmt.Errorf("bad X-Amz-Credential")
|
||||||
|
@ -183,7 +183,7 @@ func (c *Center) Authenticate(ctx context.Context, r *http.Request) (*middleware
|
||||||
SignedFields: strings.Split(queryValues.Get(AmzSignedHeaders), ";"),
|
SignedFields: strings.Split(queryValues.Get(AmzSignedHeaders), ";"),
|
||||||
Date: creds[1],
|
Date: creds[1],
|
||||||
IsPresigned: true,
|
IsPresigned: true,
|
||||||
Preamble: SignaturePreambleSigV4,
|
Preamble: signaturePreambleSigV4,
|
||||||
PayloadHash: r.Header.Get(AmzContentSHA256),
|
PayloadHash: r.Header.Get(AmzContentSHA256),
|
||||||
}
|
}
|
||||||
authHdr.Expiration, err = time.ParseDuration(queryValues.Get(AmzExpires) + "s")
|
authHdr.Expiration, err = time.ParseDuration(queryValues.Get(AmzExpires) + "s")
|
||||||
|
@ -191,7 +191,7 @@ func (c *Center) Authenticate(ctx context.Context, r *http.Request) (*middleware
|
||||||
return nil, fmt.Errorf("%w: couldn't parse X-Amz-Expires %v", apierr.GetAPIError(apierr.ErrMalformedExpires), err)
|
return nil, fmt.Errorf("%w: couldn't parse X-Amz-Expires %v", apierr.GetAPIError(apierr.ErrMalformedExpires), err)
|
||||||
}
|
}
|
||||||
signatureDateTimeStr = queryValues.Get(AmzDate)
|
signatureDateTimeStr = queryValues.Get(AmzDate)
|
||||||
} else if queryValues.Get(AmzAlgorithm) == SignaturePreambleSigV4A {
|
} else if queryValues.Get(AmzAlgorithm) == signaturePreambleSigV4A {
|
||||||
creds := strings.Split(queryValues.Get(AmzCredential), "/")
|
creds := strings.Split(queryValues.Get(AmzCredential), "/")
|
||||||
if len(creds) != 4 || creds[3] != "aws4_request" {
|
if len(creds) != 4 || creds[3] != "aws4_request" {
|
||||||
return nil, fmt.Errorf("bad X-Amz-Credential")
|
return nil, fmt.Errorf("bad X-Amz-Credential")
|
||||||
|
@ -204,7 +204,7 @@ func (c *Center) Authenticate(ctx context.Context, r *http.Request) (*middleware
|
||||||
SignedFields: strings.Split(queryValues.Get(AmzSignedHeaders), ";"),
|
SignedFields: strings.Split(queryValues.Get(AmzSignedHeaders), ";"),
|
||||||
Date: creds[1],
|
Date: creds[1],
|
||||||
IsPresigned: true,
|
IsPresigned: true,
|
||||||
Preamble: SignaturePreambleSigV4A,
|
Preamble: signaturePreambleSigV4A,
|
||||||
PayloadHash: r.Header.Get(AmzContentSHA256),
|
PayloadHash: r.Header.Get(AmzContentSHA256),
|
||||||
}
|
}
|
||||||
authHdr.Expiration, err = time.ParseDuration(queryValues.Get(AmzExpires) + "s")
|
authHdr.Expiration, err = time.ParseDuration(queryValues.Get(AmzExpires) + "s")
|
||||||
|
@ -216,7 +216,7 @@ func (c *Center) Authenticate(ctx context.Context, r *http.Request) (*middleware
|
||||||
authHeaderField := r.Header[AuthorizationHdr]
|
authHeaderField := r.Header[AuthorizationHdr]
|
||||||
if len(authHeaderField) != 1 {
|
if len(authHeaderField) != 1 {
|
||||||
if strings.HasPrefix(r.Header.Get(ContentTypeHdr), "multipart/form-data") {
|
if strings.HasPrefix(r.Header.Get(ContentTypeHdr), "multipart/form-data") {
|
||||||
return c.checkFormData(ctx, r)
|
return c.checkFormData(r)
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("%w: %v", middleware.ErrNoAuthorizationHeader, authHeaderField)
|
return nil, fmt.Errorf("%w: %v", middleware.ErrNoAuthorizationHeader, authHeaderField)
|
||||||
}
|
}
|
||||||
|
@ -242,7 +242,7 @@ func (c *Center) Authenticate(ctx context.Context, r *http.Request) (*middleware
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
box, attrs, err := c.cli.GetBox(ctx, cnrID, authHdr.AccessKeyID)
|
box, attrs, err := c.cli.GetBox(r.Context(), cnrID, authHdr.AccessKeyID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("get box by access key '%s': %w", authHdr.AccessKeyID, err)
|
return nil, fmt.Errorf("get box by access key '%s': %w", authHdr.AccessKeyID, err)
|
||||||
}
|
}
|
||||||
|
@ -315,7 +315,7 @@ func (c Center) checkAccessKeyID(accessKeyID string) error {
|
||||||
return fmt.Errorf("%w: accesskeyID prefix isn't allowed", apierr.GetAPIError(apierr.ErrAccessDenied))
|
return fmt.Errorf("%w: accesskeyID prefix isn't allowed", apierr.GetAPIError(apierr.ErrAccessDenied))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Center) checkFormData(ctx context.Context, r *http.Request) (*middleware.Box, error) {
|
func (c *Center) checkFormData(r *http.Request) (*middleware.Box, error) {
|
||||||
if err := r.ParseMultipartForm(maxFormSizeMemory); err != nil {
|
if err := r.ParseMultipartForm(maxFormSizeMemory); err != nil {
|
||||||
return nil, fmt.Errorf("%w: parse multipart form with max size %d", apierr.GetAPIError(apierr.ErrInvalidArgument), maxFormSizeMemory)
|
return nil, fmt.Errorf("%w: parse multipart form with max size %d", apierr.GetAPIError(apierr.ErrInvalidArgument), maxFormSizeMemory)
|
||||||
}
|
}
|
||||||
|
@ -347,7 +347,7 @@ func (c *Center) checkFormData(ctx context.Context, r *http.Request) (*middlewar
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
box, attrs, err := c.cli.GetBox(ctx, cnrID, accessKeyID)
|
box, attrs, err := c.cli.GetBox(r.Context(), cnrID, accessKeyID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("get box by accessKeyID '%s': %w", accessKeyID, err)
|
return nil, fmt.Errorf("get box by accessKeyID '%s': %w", accessKeyID, err)
|
||||||
}
|
}
|
||||||
|
@ -402,7 +402,7 @@ func (c *Center) checkSign(ctx context.Context, authHeader *AuthHeader, box *acc
|
||||||
}
|
}
|
||||||
|
|
||||||
switch authHeader.Preamble {
|
switch authHeader.Preamble {
|
||||||
case SignaturePreambleSigV4:
|
case signaturePreambleSigV4:
|
||||||
creds := aws.Credentials{
|
creds := aws.Credentials{
|
||||||
AccessKeyID: authHeader.AccessKeyID,
|
AccessKeyID: authHeader.AccessKeyID,
|
||||||
SecretAccessKey: box.Gate.SecretKey,
|
SecretAccessKey: box.Gate.SecretKey,
|
||||||
|
@ -437,7 +437,7 @@ func (c *Center) checkSign(ctx context.Context, authHeader *AuthHeader, box *acc
|
||||||
authHeader.Signature, signature, authHeader.SignedFields)
|
authHeader.Signature, signature, authHeader.SignedFields)
|
||||||
}
|
}
|
||||||
|
|
||||||
case SignaturePreambleSigV4A:
|
case signaturePreambleSigV4A:
|
||||||
signer := v4a.NewSigner(func(options *v4a.SignerOptions) {
|
signer := v4a.NewSigner(func(options *v4a.SignerOptions) {
|
||||||
options.DisableURIPathEscaping = true
|
options.DisableURIPathEscaping = true
|
||||||
})
|
})
|
||||||
|
|
|
@ -69,7 +69,7 @@ func TestAuthHeaderParse(t *testing.T) {
|
||||||
Signature: "2811ccb9e242f41426738fb1f",
|
Signature: "2811ccb9e242f41426738fb1f",
|
||||||
SignedFields: []string{"host", "x-amz-content-sha256", "x-amz-date"},
|
SignedFields: []string{"host", "x-amz-content-sha256", "x-amz-date"},
|
||||||
Date: "20210809",
|
Date: "20210809",
|
||||||
Preamble: SignaturePreambleSigV4,
|
Preamble: signaturePreambleSigV4,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
@ -567,7 +567,7 @@ func TestAuthenticate(t *testing.T) {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
creds := tokens.New(bigConfig)
|
creds := tokens.New(bigConfig)
|
||||||
cntr := New(creds, tc.prefixes, ¢erSettingsMock{})
|
cntr := New(creds, tc.prefixes, ¢erSettingsMock{})
|
||||||
box, err := cntr.Authenticate(ctx, tc.request)
|
box, err := cntr.Authenticate(tc.request)
|
||||||
|
|
||||||
if tc.err {
|
if tc.err {
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
|
@ -600,7 +600,6 @@ func TestHTTPPostAuthenticate(t *testing.T) {
|
||||||
region = "default"
|
region = "default"
|
||||||
)
|
)
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
key, err := keys.NewPrivateKey()
|
key, err := keys.NewPrivateKey()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -752,7 +751,7 @@ func TestHTTPPostAuthenticate(t *testing.T) {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
creds := tokens.New(bigConfig)
|
creds := tokens.New(bigConfig)
|
||||||
cntr := New(creds, tc.prefixes, ¢erSettingsMock{})
|
cntr := New(creds, tc.prefixes, ¢erSettingsMock{})
|
||||||
box, err := cntr.Authenticate(ctx, tc.request)
|
box, err := cntr.Authenticate(tc.request)
|
||||||
|
|
||||||
if tc.err {
|
if tc.err {
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
|
|
|
@ -99,14 +99,12 @@ func TestCheckSign(t *testing.T) {
|
||||||
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
|
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
|
||||||
settings: ¢erSettingsMock{},
|
settings: ¢erSettingsMock{},
|
||||||
}
|
}
|
||||||
box, err := c.Authenticate(ctx, req)
|
box, err := c.Authenticate(req)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.EqualValues(t, expBox, box.AccessBox)
|
require.EqualValues(t, expBox, box.AccessBox)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCheckSignV4a(t *testing.T) {
|
func TestCheckSignV4a(t *testing.T) {
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
var accessKeyAddr oid.Address
|
var accessKeyAddr oid.Address
|
||||||
err := accessKeyAddr.DecodeString("8N7CYBY74kxZXoyvA5UNdmovaXqFpwNfvEPsqaN81es2/3tDwq5tR8fByrJcyJwyiuYX7Dae8tyDT7pd8oaL1MBto")
|
err := accessKeyAddr.DecodeString("8N7CYBY74kxZXoyvA5UNdmovaXqFpwNfvEPsqaN81es2/3tDwq5tR8fByrJcyJwyiuYX7Dae8tyDT7pd8oaL1MBto")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -147,10 +145,10 @@ func TestCheckSignV4a(t *testing.T) {
|
||||||
|
|
||||||
c := &Center{
|
c := &Center{
|
||||||
cli: mock,
|
cli: mock,
|
||||||
regV4a: NewRegexpMatcher(AuthorizationFieldV4aRegexp),
|
regV4a: NewRegexpMatcher(authorizationFieldV4aRegexp),
|
||||||
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
|
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
|
||||||
}
|
}
|
||||||
box, err := c.Authenticate(ctx, req)
|
box, err := c.Authenticate(req)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.EqualValues(t, expBox, box.AccessBox)
|
require.EqualValues(t, expBox, box.AccessBox)
|
||||||
}
|
}
|
||||||
|
|
2
api/cache/access_control.go
vendored
2
api/cache/access_control.go
vendored
|
@ -48,7 +48,7 @@ func (o *AccessControlCache) Get(owner user.ID, key string) bool {
|
||||||
result, ok := entry.(bool)
|
result, ok := entry.(bool)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
2
api/cache/accessbox.go
vendored
2
api/cache/accessbox.go
vendored
|
@ -67,7 +67,7 @@ func (o *AccessBoxCache) Get(accessKeyID string) *AccessBoxCacheValue {
|
||||||
result, ok := entry.(*AccessBoxCacheValue)
|
result, ok := entry.(*AccessBoxCacheValue)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
4
api/cache/buckets.go
vendored
4
api/cache/buckets.go
vendored
|
@ -65,7 +65,7 @@ func (o *BucketCache) GetByCID(cnrID cid.ID) *data.BucketInfo {
|
||||||
key, ok := entry.(string)
|
key, ok := entry.(string)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", key)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", key)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -81,7 +81,7 @@ func (o *BucketCache) get(key string) *data.BucketInfo {
|
||||||
result, ok := entry.(*data.BucketInfo)
|
result, ok := entry.(*data.BucketInfo)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
2
api/cache/frostfsid.go
vendored
2
api/cache/frostfsid.go
vendored
|
@ -69,7 +69,7 @@ func get[T any](c *FrostfsIDCache, key any) *T {
|
||||||
result, ok := entry.(*T)
|
result, ok := entry.(*T)
|
||||||
if !ok {
|
if !ok {
|
||||||
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
4
api/cache/listsession.go
vendored
4
api/cache/listsession.go
vendored
|
@ -52,7 +52,7 @@ func NewListSessionCache(config *Config) *ListSessionCache {
|
||||||
session, ok := val.(*data.ListSession)
|
session, ok := val.(*data.ListSession)
|
||||||
if !ok {
|
if !ok {
|
||||||
config.Logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", val)),
|
config.Logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", val)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", session)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", session)))
|
||||||
}
|
}
|
||||||
|
|
||||||
if !session.Acquired.Load() {
|
if !session.Acquired.Load() {
|
||||||
|
@ -72,7 +72,7 @@ func (l *ListSessionCache) GetListSession(key ListSessionKey) *data.ListSession
|
||||||
result, ok := entry.(*data.ListSession)
|
result, ok := entry.(*data.ListSession)
|
||||||
if !ok {
|
if !ok {
|
||||||
l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
2
api/cache/names.go
vendored
2
api/cache/names.go
vendored
|
@ -50,7 +50,7 @@ func (o *ObjectsNameCache) Get(key string) *oid.Address {
|
||||||
result, ok := entry.(oid.Address)
|
result, ok := entry.(oid.Address)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
4
api/cache/network.go
vendored
4
api/cache/network.go
vendored
|
@ -54,7 +54,7 @@ func (c *NetworkCache) GetNetworkInfo() *netmap.NetworkInfo {
|
||||||
result, ok := entry.(netmap.NetworkInfo)
|
result, ok := entry.(netmap.NetworkInfo)
|
||||||
if !ok {
|
if !ok {
|
||||||
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -74,7 +74,7 @@ func (c *NetworkCache) GetNetmap() *netmap.NetMap {
|
||||||
result, ok := entry.(netmap.NetMap)
|
result, ok := entry.(netmap.NetMap)
|
||||||
if !ok {
|
if !ok {
|
||||||
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
2
api/cache/objects.go
vendored
2
api/cache/objects.go
vendored
|
@ -49,7 +49,7 @@ func (o *ObjectsCache) GetObject(address oid.Address) *data.ExtendedObjectInfo {
|
||||||
result, ok := entry.(*data.ExtendedObjectInfo)
|
result, ok := entry.(*data.ExtendedObjectInfo)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
10
api/cache/objects_test.go
vendored
10
api/cache/objects_test.go
vendored
|
@ -8,14 +8,14 @@ import (
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test"
|
objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap/zaptest"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
func getTestConfig(t *testing.T) *Config {
|
func getTestConfig() *Config {
|
||||||
return &Config{
|
return &Config{
|
||||||
Size: 10,
|
Size: 10,
|
||||||
Lifetime: 5 * time.Second,
|
Lifetime: 5 * time.Second,
|
||||||
Logger: zaptest.NewLogger(t),
|
Logger: zap.NewExample(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -44,7 +44,7 @@ func TestCache(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("check get", func(t *testing.T) {
|
t.Run("check get", func(t *testing.T) {
|
||||||
cache := New(getTestConfig(t))
|
cache := New(getTestConfig())
|
||||||
err := cache.PutObject(extObjInfo)
|
err := cache.PutObject(extObjInfo)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -53,7 +53,7 @@ func TestCache(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("check delete", func(t *testing.T) {
|
t.Run("check delete", func(t *testing.T) {
|
||||||
cache := New(getTestConfig(t))
|
cache := New(getTestConfig())
|
||||||
err := cache.PutObject(extObjInfo)
|
err := cache.PutObject(extObjInfo)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
|
4
api/cache/objectslist.go
vendored
4
api/cache/objectslist.go
vendored
|
@ -77,7 +77,7 @@ func (l *ObjectsListCache) GetVersions(key ObjectsListKey) []*data.NodeVersion {
|
||||||
result, ok := entry.([]*data.NodeVersion)
|
result, ok := entry.([]*data.NodeVersion)
|
||||||
if !ok {
|
if !ok {
|
||||||
l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -96,7 +96,7 @@ func (l *ObjectsListCache) CleanCacheEntriesContainingObject(objectName string,
|
||||||
k, ok := key.(ObjectsListKey)
|
k, ok := key.(ObjectsListKey)
|
||||||
if !ok {
|
if !ok {
|
||||||
l.logger.Warn(logs.InvalidCacheKeyType, zap.String("actual", fmt.Sprintf("%T", key)),
|
l.logger.Warn(logs.InvalidCacheKeyType, zap.String("actual", fmt.Sprintf("%T", key)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", k)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", k)))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if cnr.Equals(k.cid) && strings.HasPrefix(objectName, k.prefix) {
|
if cnr.Equals(k.cid) && strings.HasPrefix(objectName, k.prefix) {
|
||||||
|
|
22
api/cache/objectslist_test.go
vendored
22
api/cache/objectslist_test.go
vendored
|
@ -8,17 +8,17 @@ import (
|
||||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap/zaptest"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
const testingCacheLifetime = 5 * time.Second
|
const testingCacheLifetime = 5 * time.Second
|
||||||
const testingCacheSize = 10
|
const testingCacheSize = 10
|
||||||
|
|
||||||
func getTestObjectsListConfig(t *testing.T) *Config {
|
func getTestObjectsListConfig() *Config {
|
||||||
return &Config{
|
return &Config{
|
||||||
Size: testingCacheSize,
|
Size: testingCacheSize,
|
||||||
Lifetime: testingCacheLifetime,
|
Lifetime: testingCacheLifetime,
|
||||||
Logger: zaptest.NewLogger(t),
|
Logger: zap.NewExample(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -35,7 +35,7 @@ func TestObjectsListCache(t *testing.T) {
|
||||||
|
|
||||||
t.Run("lifetime", func(t *testing.T) {
|
t.Run("lifetime", func(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
config = getTestObjectsListConfig(t)
|
config = getTestObjectsListConfig()
|
||||||
cache = NewObjectsListCache(config)
|
cache = NewObjectsListCache(config)
|
||||||
listKey = ObjectsListKey{cid: cidKey}
|
listKey = ObjectsListKey{cid: cidKey}
|
||||||
)
|
)
|
||||||
|
@ -53,7 +53,7 @@ func TestObjectsListCache(t *testing.T) {
|
||||||
|
|
||||||
t.Run("get cache with empty prefix", func(t *testing.T) {
|
t.Run("get cache with empty prefix", func(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
cache = NewObjectsListCache(getTestObjectsListConfig(t))
|
cache = NewObjectsListCache(getTestObjectsListConfig())
|
||||||
listKey = ObjectsListKey{cid: cidKey}
|
listKey = ObjectsListKey{cid: cidKey}
|
||||||
)
|
)
|
||||||
err := cache.PutVersions(listKey, versions)
|
err := cache.PutVersions(listKey, versions)
|
||||||
|
@ -73,7 +73,7 @@ func TestObjectsListCache(t *testing.T) {
|
||||||
prefix: "dir",
|
prefix: "dir",
|
||||||
}
|
}
|
||||||
|
|
||||||
cache := NewObjectsListCache(getTestObjectsListConfig(t))
|
cache := NewObjectsListCache(getTestObjectsListConfig())
|
||||||
err := cache.PutVersions(listKey, versions)
|
err := cache.PutVersions(listKey, versions)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -98,7 +98,7 @@ func TestObjectsListCache(t *testing.T) {
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
cache := NewObjectsListCache(getTestObjectsListConfig(t))
|
cache := NewObjectsListCache(getTestObjectsListConfig())
|
||||||
err := cache.PutVersions(listKey, versions)
|
err := cache.PutVersions(listKey, versions)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -116,7 +116,7 @@ func TestObjectsListCache(t *testing.T) {
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
cache := NewObjectsListCache(getTestObjectsListConfig(t))
|
cache := NewObjectsListCache(getTestObjectsListConfig())
|
||||||
err := cache.PutVersions(listKey, versions)
|
err := cache.PutVersions(listKey, versions)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -137,7 +137,7 @@ func TestCleanCacheEntriesChangedWithPutObject(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("put object to the root of the bucket", func(t *testing.T) {
|
t.Run("put object to the root of the bucket", func(t *testing.T) {
|
||||||
config := getTestObjectsListConfig(t)
|
config := getTestObjectsListConfig()
|
||||||
config.Lifetime = time.Minute
|
config.Lifetime = time.Minute
|
||||||
cache := NewObjectsListCache(config)
|
cache := NewObjectsListCache(config)
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
|
@ -156,7 +156,7 @@ func TestCleanCacheEntriesChangedWithPutObject(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("put object to dir/", func(t *testing.T) {
|
t.Run("put object to dir/", func(t *testing.T) {
|
||||||
config := getTestObjectsListConfig(t)
|
config := getTestObjectsListConfig()
|
||||||
config.Lifetime = time.Minute
|
config.Lifetime = time.Minute
|
||||||
cache := NewObjectsListCache(config)
|
cache := NewObjectsListCache(config)
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
|
@ -175,7 +175,7 @@ func TestCleanCacheEntriesChangedWithPutObject(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("put object to dir/lol/", func(t *testing.T) {
|
t.Run("put object to dir/lol/", func(t *testing.T) {
|
||||||
config := getTestObjectsListConfig(t)
|
config := getTestObjectsListConfig()
|
||||||
config.Lifetime = time.Minute
|
config.Lifetime = time.Minute
|
||||||
cache := NewObjectsListCache(config)
|
cache := NewObjectsListCache(config)
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
|
|
2
api/cache/policy.go
vendored
2
api/cache/policy.go
vendored
|
@ -54,7 +54,7 @@ func (o *MorphPolicyCache) Get(key MorphPolicyCacheKey) []*chain.Chain {
|
||||||
result, ok := entry.([]*chain.Chain)
|
result, ok := entry.([]*chain.Chain)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
8
api/cache/system.go
vendored
8
api/cache/system.go
vendored
|
@ -50,7 +50,7 @@ func (o *SystemCache) GetObject(key string) *data.ObjectInfo {
|
||||||
result, ok := entry.(*data.ObjectInfo)
|
result, ok := entry.(*data.ObjectInfo)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -81,7 +81,7 @@ func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration {
|
||||||
result, ok := entry.(*data.CORSConfiguration)
|
result, ok := entry.(*data.CORSConfiguration)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -97,7 +97,7 @@ func (o *SystemCache) GetLifecycleConfiguration(key string) *data.LifecycleConfi
|
||||||
result, ok := entry.(*data.LifecycleConfiguration)
|
result, ok := entry.(*data.LifecycleConfiguration)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -113,7 +113,7 @@ func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
|
||||||
result, ok := entry.(*data.BucketSettings)
|
result, ok := entry.(*data.BucketSettings)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -60,6 +60,7 @@ const (
|
||||||
ErrMalformedACL
|
ErrMalformedACL
|
||||||
ErrMalformedXML
|
ErrMalformedXML
|
||||||
ErrMissingContentLength
|
ErrMissingContentLength
|
||||||
|
ErrMissingContentMD5
|
||||||
ErrMissingRequestBodyError
|
ErrMissingRequestBodyError
|
||||||
ErrMissingSecurityHeader
|
ErrMissingSecurityHeader
|
||||||
ErrNoSuchBucket
|
ErrNoSuchBucket
|
||||||
|
@ -477,6 +478,12 @@ var errorCodes = errorCodeMap{
|
||||||
Description: "You must provide the Content-Length HTTP header.",
|
Description: "You must provide the Content-Length HTTP header.",
|
||||||
HTTPStatusCode: http.StatusLengthRequired,
|
HTTPStatusCode: http.StatusLengthRequired,
|
||||||
},
|
},
|
||||||
|
ErrMissingContentMD5: {
|
||||||
|
ErrCode: ErrMissingContentMD5,
|
||||||
|
Code: "MissingContentMD5",
|
||||||
|
Description: "Missing required header for this request: Content-Md5.",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrMissingSecurityHeader: {
|
ErrMissingSecurityHeader: {
|
||||||
ErrCode: ErrMissingSecurityHeader,
|
ErrCode: ErrMissingSecurityHeader,
|
||||||
Code: "MissingSecurityHeader",
|
Code: "MissingSecurityHeader",
|
||||||
|
|
|
@ -11,7 +11,6 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
@ -48,9 +47,7 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetBucketACL")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -102,7 +99,7 @@ func (h *handler) encodePrivateCannedACL(ctx context.Context, bktInfo *data.Buck
|
||||||
ownerEncodedID := ownerDisplayName
|
ownerEncodedID := ownerDisplayName
|
||||||
|
|
||||||
if settings.OwnerKey == nil {
|
if settings.OwnerKey == nil {
|
||||||
h.reqLogger(ctx).Warn(logs.BucketOwnerKeyIsMissing, zap.String("owner", bktInfo.Owner.String()), logs.TagField(logs.TagDatapath))
|
h.reqLogger(ctx).Warn(logs.BucketOwnerKeyIsMissing, zap.String("owner", bktInfo.Owner.String()))
|
||||||
} else {
|
} else {
|
||||||
ownerDisplayName = settings.OwnerKey.Address()
|
ownerDisplayName = settings.OwnerKey.Address()
|
||||||
ownerEncodedID = hex.EncodeToString(settings.OwnerKey.Bytes())
|
ownerEncodedID = hex.EncodeToString(settings.OwnerKey.Bytes())
|
||||||
|
@ -130,9 +127,7 @@ func getTokenIssuerKey(box *accessbox.Box) (*keys.PublicKey, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutBucketACL")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -155,7 +150,7 @@ func (h *handler) putBucketACLAPEHandler(w http.ResponseWriter, r *http.Request,
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
if errBody := r.Body.Close(); errBody != nil {
|
if errBody := r.Body.Close(); errBody != nil {
|
||||||
h.reqLogger(ctx).Warn(logs.CouldNotCloseRequestBody, zap.Error(errBody), logs.TagField(logs.TagDatapath))
|
h.reqLogger(ctx).Warn(logs.CouldNotCloseRequestBody, zap.Error(errBody))
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
@ -199,9 +194,7 @@ func (h *handler) putBucketACLAPEHandler(w http.ResponseWriter, r *http.Request,
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetObjectACL")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -223,9 +216,7 @@ func (h *handler) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutObjectACL")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
if _, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName); err != nil {
|
if _, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName); err != nil {
|
||||||
|
@ -237,9 +228,7 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) GetBucketPolicyStatusHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketPolicyStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetBucketPolicyStatus")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -282,9 +271,7 @@ func (h *handler) GetBucketPolicyStatusHandler(w http.ResponseWriter, r *http.Re
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetBucketPolicy")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -311,9 +298,7 @@ func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.DeleteBucketPolicy")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -343,9 +328,7 @@ func checkOwner(info *data.BucketInfo, owner string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutBucketPolicy")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -399,7 +382,7 @@ func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
|
||||||
h.logAndSendError(ctx, w, "could not convert s3 policy to native chain policy", reqInfo, err)
|
h.logAndSendError(ctx, w, "could not convert s3 policy to native chain policy", reqInfo, err)
|
||||||
return
|
return
|
||||||
} else {
|
} else {
|
||||||
h.reqLogger(ctx).Warn(logs.PolicyCouldntBeConvertedToNativeRules, logs.TagField(logs.TagDatapath))
|
h.reqLogger(ctx).Warn(logs.PolicyCouldntBeConvertedToNativeRules)
|
||||||
}
|
}
|
||||||
|
|
||||||
chainsToSave := []*chain.Chain{s3Chain}
|
chainsToSave := []*chain.Chain{s3Chain}
|
||||||
|
|
|
@ -42,7 +42,6 @@ type (
|
||||||
RetryMaxBackoff() time.Duration
|
RetryMaxBackoff() time.Duration
|
||||||
RetryStrategy() RetryStrategy
|
RetryStrategy() RetryStrategy
|
||||||
TLSTerminationHeader() string
|
TLSTerminationHeader() string
|
||||||
ListingKeepaliveThrottle() time.Duration
|
|
||||||
}
|
}
|
||||||
|
|
||||||
FrostFSID interface {
|
FrostFSID interface {
|
||||||
|
|
|
@ -8,7 +8,6 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
@ -71,9 +70,7 @@ var validAttributes = map[string]struct{}{
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetObjectAttributes")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
params, err := parseGetObjectAttributeArgs(r, h.reqLogger(ctx))
|
params, err := parseGetObjectAttributeArgs(r, h.reqLogger(ctx))
|
||||||
|
|
|
@ -5,7 +5,6 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
@ -15,9 +14,7 @@ const maxBucketList = 10000
|
||||||
|
|
||||||
// ListBucketsHandler handles bucket listing requests.
|
// ListBucketsHandler handles bucket listing requests.
|
||||||
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.ListBuckets")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
params, err := parseListBucketParams(r)
|
params, err := parseListBucketParams(r)
|
||||||
|
|
|
@ -6,7 +6,6 @@ import (
|
||||||
"regexp"
|
"regexp"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
@ -41,18 +40,17 @@ func path2BucketObject(path string) (string, string, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.CopyObject")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
versionID string
|
versionID string
|
||||||
metadata map[string]string
|
metadata map[string]string
|
||||||
tagSet map[string]string
|
tagSet map[string]string
|
||||||
)
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
ctx = r.Context()
|
||||||
cannedACLStatus := aclHeadersStatus(r)
|
reqInfo = middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
|
cannedACLStatus = aclHeadersStatus(r)
|
||||||
|
)
|
||||||
|
|
||||||
src := r.Header.Get(api.AmzCopySource)
|
src := r.Header.Get(api.AmzCopySource)
|
||||||
// Check https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
|
// Check https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
|
||||||
|
@ -246,7 +244,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID), logs.TagField(logs.TagExternalStorage))
|
h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID))
|
||||||
|
|
||||||
if dstEncryptionParams.Enabled() {
|
if dstEncryptionParams.Enabled() {
|
||||||
addSSECHeaders(w.Header(), r.Header)
|
addSSECHeaders(w.Header(), r.Header)
|
||||||
|
|
|
@ -5,13 +5,10 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -23,10 +20,7 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetBucketCors")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -48,10 +42,7 @@ func (h *handler) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutBucketCors")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -85,10 +76,7 @@ func (h *handler) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.DeleteBucketCors")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -105,9 +93,6 @@ func (h *handler) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.AppendCORSHeaders")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
if r.Method == http.MethodOptions {
|
if r.Method == http.MethodOptions {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -116,20 +101,20 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
|
ctx := r.Context()
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
if reqInfo.BucketName == "" {
|
if reqInfo.BucketName == "" {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
bktInfo, err := h.getBucketInfo(ctx, reqInfo.BucketName)
|
bktInfo, err := h.getBucketInfo(ctx, reqInfo.BucketName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
|
h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
cors, err := h.obj.GetBucketCORS(ctx, bktInfo, h.cfg.NewXMLDecoder)
|
cors, err := h.obj.GetBucketCORS(ctx, bktInfo, h.cfg.NewXMLDecoder)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.reqLogger(ctx).Warn(logs.GetBucketCors, zap.Error(err), logs.TagField(logs.TagDatapath))
|
h.reqLogger(ctx).Warn(logs.GetBucketCors, zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -168,10 +153,7 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) Preflight(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) Preflight(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.Preflight")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
bktInfo, err := h.getBucketInfo(ctx, reqInfo.BucketName)
|
bktInfo, err := h.getBucketInfo(ctx, reqInfo.BucketName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -7,7 +7,6 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
|
@ -62,9 +61,7 @@ type DeleteObjectsResponse struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.DeleteObject")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
versionID := reqInfo.URL.Query().Get(api.QueryVersionID)
|
versionID := reqInfo.URL.Query().Get(api.QueryVersionID)
|
||||||
versionedObject := []*layer.VersionedObject{{
|
versionedObject := []*layer.VersionedObject{{
|
||||||
|
@ -131,11 +128,16 @@ func isErrObjectLocked(err error) bool {
|
||||||
|
|
||||||
// DeleteMultipleObjectsHandler handles multiple delete requests.
|
// DeleteMultipleObjectsHandler handles multiple delete requests.
|
||||||
func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.DeleteMultipleObjects")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
|
// Content-Md5 is required and should be set
|
||||||
|
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||||
|
if _, ok := r.Header[api.ContentMD5]; !ok {
|
||||||
|
h.logAndSendError(ctx, w, "missing Content-MD5", reqInfo, errors.GetAPIError(errors.ErrMissingContentMD5))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// Content-Length is required and should be non-zero
|
// Content-Length is required and should be non-zero
|
||||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||||
if r.ContentLength <= 0 {
|
if r.ContentLength <= 0 {
|
||||||
|
@ -235,9 +237,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.DeleteBucket")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -542,6 +542,7 @@ func deleteObjectsBase(hc *handlerContext, bktName string, objVersions [][2]stri
|
||||||
}
|
}
|
||||||
|
|
||||||
w, r := prepareTestRequest(hc, bktName, "", req)
|
w, r := prepareTestRequest(hc, bktName, "", req)
|
||||||
|
r.Header.Set(api.ContentMD5, "")
|
||||||
hc.Handler().DeleteMultipleObjectsHandler(w, r)
|
hc.Handler().DeleteMultipleObjectsHandler(w, r)
|
||||||
assertStatus(hc.t, w, http.StatusOK)
|
assertStatus(hc.t, w, http.StatusOK)
|
||||||
|
|
||||||
|
|
|
@ -8,7 +8,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
@ -149,11 +148,12 @@ func writeHeaders(h http.Header, requestHeader http.Header, extendedInfo *data.E
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetObject")
|
var (
|
||||||
defer span.End()
|
params *layer.RangeParams
|
||||||
|
|
||||||
var params *layer.RangeParams
|
ctx = r.Context()
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo = middleware.GetReqInfo(ctx)
|
||||||
|
)
|
||||||
|
|
||||||
conditional := parseConditionalHeaders(r.Header, h.reqLogger(ctx))
|
conditional := parseConditionalHeaders(r.Header, h.reqLogger(ctx))
|
||||||
|
|
||||||
|
@ -255,7 +255,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = objPayload.StreamTo(w); err != nil {
|
if err = objPayload.StreamTo(w); err != nil {
|
||||||
h.logError(ctx, "could not stream object payload", reqInfo, err)
|
h.logAndSendError(ctx, w, "could not stream object payload", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -296,12 +296,12 @@ func parseConditionalHeaders(headers http.Header, log *zap.Logger) *conditionalA
|
||||||
if httpTime, err := parseHTTPTime(headers.Get(api.IfModifiedSince)); err == nil {
|
if httpTime, err := parseHTTPTime(headers.Get(api.IfModifiedSince)); err == nil {
|
||||||
args.IfModifiedSince = httpTime
|
args.IfModifiedSince = httpTime
|
||||||
} else {
|
} else {
|
||||||
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfModifiedSince, headers.Get(api.IfModifiedSince)), zap.Error(err), logs.TagField(logs.TagDatapath))
|
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfModifiedSince, headers.Get(api.IfModifiedSince)), zap.Error(err))
|
||||||
}
|
}
|
||||||
if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil {
|
if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil {
|
||||||
args.IfUnmodifiedSince = httpTime
|
args.IfUnmodifiedSince = httpTime
|
||||||
} else {
|
} else {
|
||||||
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err), logs.TagField(logs.TagDatapath))
|
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
return args
|
return args
|
||||||
|
|
|
@ -2,7 +2,6 @@ package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
@ -198,46 +197,6 @@ func TestGetObject(t *testing.T) {
|
||||||
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, apierr.ErrNoSuchKey)
|
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, apierr.ErrNoSuchKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetObjectStreamError(t *testing.T) {
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
bktName, objName := "bucket", "obj"
|
|
||||||
info := createBucket(hc, bktName)
|
|
||||||
|
|
||||||
putObject(hc, bktName, objName)
|
|
||||||
addr := getAddressOfLastVersion(hc, info.BktInfo, objName)
|
|
||||||
hc.tp.SetObjectStreamError(addr, 4, context.Canceled)
|
|
||||||
|
|
||||||
d, _ := getObject(hc, bktName, objName)
|
|
||||||
require.Equal(t, "cont", string(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetDeletedObject(t *testing.T) {
|
|
||||||
hc := prepareHandlerContextWithMinCache(t)
|
|
||||||
bktName, objName := "bucket", "obj"
|
|
||||||
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
|
|
||||||
|
|
||||||
putObject(hc, bktName, objName)
|
|
||||||
|
|
||||||
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
|
|
||||||
checkFound(hc.t, hc, bktName, objName, emptyVersion)
|
|
||||||
|
|
||||||
addr := getAddressOfLastVersion(hc, bktInfo, objName)
|
|
||||||
t.Run("not found error", func(_ *testing.T) {
|
|
||||||
hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
|
|
||||||
hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectNotFound{})
|
|
||||||
|
|
||||||
getObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), apierr.ErrNoSuchVersion)
|
|
||||||
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, apierr.ErrNoSuchKey)
|
|
||||||
})
|
|
||||||
t.Run("already removed error", func(_ *testing.T) {
|
|
||||||
hc.tp.SetObjectError(addr, &apistatus.ObjectAlreadyRemoved{})
|
|
||||||
hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectAlreadyRemoved{})
|
|
||||||
|
|
||||||
getObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), apierr.ErrNoSuchVersion)
|
|
||||||
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, apierr.ErrNoSuchKey)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetObjectEnabledMD5(t *testing.T) {
|
func TestGetObjectEnabledMD5(t *testing.T) {
|
||||||
hc := prepareHandlerContext(t)
|
hc := prepareHandlerContext(t)
|
||||||
bktName, objName := "bucket", "obj"
|
bktName, objName := "bucket", "obj"
|
||||||
|
|
|
@ -22,7 +22,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
engineiam "git.frostfs.info/TrueCloudLab/policy-engine/iam"
|
engineiam "git.frostfs.info/TrueCloudLab/policy-engine/iam"
|
||||||
utils "github.com/trailofbits/go-fuzz-utils"
|
utils "github.com/trailofbits/go-fuzz-utils"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap/zaptest"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
@ -40,9 +40,9 @@ const (
|
||||||
func createTestBucketAndInitContext() {
|
func createTestBucketAndInitContext() {
|
||||||
fuzzt = new(tt.T)
|
fuzzt = new(tt.T)
|
||||||
|
|
||||||
log := zap.NewExample()
|
log := zaptest.NewLogger(fuzzt)
|
||||||
var err error
|
var err error
|
||||||
fuzzHc, err = prepareHandlerContextBase(layer.DefaultCachesConfigs(log), log)
|
fuzzHc, err = prepareHandlerContextBase(layer.DefaultCachesConfigs(log))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
@ -171,9 +171,9 @@ func generateHeaders(tp *utils.TypeProvider, r *http.Request, params []string) e
|
||||||
func InitFuzzCreateBucketHandler() {
|
func InitFuzzCreateBucketHandler() {
|
||||||
fuzzt = new(tt.T)
|
fuzzt = new(tt.T)
|
||||||
|
|
||||||
log := zap.NewExample()
|
log := zaptest.NewLogger(fuzzt)
|
||||||
var err error
|
var err error
|
||||||
fuzzHc, err = prepareHandlerContextBase(layer.DefaultCachesConfigs(log), log)
|
fuzzHc, err = prepareHandlerContextBase(layer.DefaultCachesConfigs(log))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,7 +35,6 @@ import (
|
||||||
"github.com/panjf2000/ants/v2"
|
"github.com/panjf2000/ants/v2"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
"go.uber.org/zap/zaptest"
|
|
||||||
"golang.org/x/exp/slices"
|
"golang.org/x/exp/slices"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -154,17 +153,12 @@ func (c *configMock) TLSTerminationHeader() string {
|
||||||
return c.tlsTerminationHeader
|
return c.tlsTerminationHeader
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *configMock) ListingKeepaliveThrottle() time.Duration {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *configMock) putLocationConstraint(constraint string) {
|
func (c *configMock) putLocationConstraint(constraint string) {
|
||||||
c.placementPolicies[constraint] = c.defaultPolicy
|
c.placementPolicies[constraint] = c.defaultPolicy
|
||||||
}
|
}
|
||||||
|
|
||||||
func prepareHandlerContext(t *testing.T) *handlerContext {
|
func prepareHandlerContext(t *testing.T) *handlerContext {
|
||||||
log := zaptest.NewLogger(t)
|
hc, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(zap.NewExample()))
|
||||||
hc, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(log), log)
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
return &handlerContext{
|
return &handlerContext{
|
||||||
handlerContextBase: hc,
|
handlerContextBase: hc,
|
||||||
|
@ -173,8 +167,7 @@ func prepareHandlerContext(t *testing.T) *handlerContext {
|
||||||
}
|
}
|
||||||
|
|
||||||
func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext {
|
func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext {
|
||||||
log := zaptest.NewLogger(t)
|
hc, err := prepareHandlerContextBase(getMinCacheConfig(zap.NewExample()))
|
||||||
hc, err := prepareHandlerContextBase(getMinCacheConfig(log), log)
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
return &handlerContext{
|
return &handlerContext{
|
||||||
handlerContextBase: hc,
|
handlerContextBase: hc,
|
||||||
|
@ -182,12 +175,13 @@ func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func prepareHandlerContextBase(cacheCfg *layer.CachesConfig, log *zap.Logger) (*handlerContextBase, error) {
|
func prepareHandlerContextBase(cacheCfg *layer.CachesConfig) (*handlerContextBase, error) {
|
||||||
key, err := keys.NewPrivateKey()
|
key, err := keys.NewPrivateKey()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log := zap.NewExample()
|
||||||
tp := layer.NewTestFrostFS(key)
|
tp := layer.NewTestFrostFS(key)
|
||||||
|
|
||||||
testResolver := &resolver.Resolver{Name: "test_resolver"}
|
testResolver := &resolver.Resolver{Name: "test_resolver"}
|
||||||
|
@ -203,7 +197,7 @@ func prepareHandlerContextBase(cacheCfg *layer.CachesConfig, log *zap.Logger) (*
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
treeMock := tree.NewTree(memCli, log)
|
treeMock := tree.NewTree(memCli, zap.NewExample())
|
||||||
|
|
||||||
features := &layer.FeatureSettingsMock{}
|
features := &layer.FeatureSettingsMock{}
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,6 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
@ -28,9 +27,7 @@ func getRangeToDetectContentType(maxSize uint64) *layer.RangeParams {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.HeadObject")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -126,9 +123,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.HeadBucket")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
|
|
@ -3,14 +3,11 @@ package handler
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetBucketLocation")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
|
|
@ -10,8 +10,6 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
@ -29,10 +27,7 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetBucketLifecycle")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -54,28 +49,31 @@ func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutBucketLifecycle")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
|
|
||||||
tee := io.TeeReader(r.Body, &buf)
|
tee := io.TeeReader(r.Body, &buf)
|
||||||
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
|
ctx := r.Context()
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
cfg := new(data.LifecycleConfiguration)
|
// Content-Md5 is required and should be set
|
||||||
if err := h.cfg.NewXMLDecoder(tee, r.UserAgent()).Decode(cfg); err != nil {
|
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html
|
||||||
h.logAndSendError(ctx, w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
|
if _, ok := r.Header[api.ContentMD5]; !ok {
|
||||||
|
h.logAndSendError(ctx, w, "missing Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrMissingContentMD5))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := r.Header[api.ContentMD5]; ok {
|
|
||||||
headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5))
|
headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(ctx, w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
h.logAndSendError(ctx, w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cfg := new(data.LifecycleConfiguration)
|
||||||
|
if err = h.cfg.NewXMLDecoder(tee, r.UserAgent()).Decode(cfg); err != nil {
|
||||||
|
h.logAndSendError(ctx, w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
bodyMD5, err := getContentMD5(&buf)
|
bodyMD5, err := getContentMD5(&buf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(ctx, w, "could not get content md5", reqInfo, err)
|
h.logAndSendError(ctx, w, "could not get content md5", reqInfo, err)
|
||||||
|
@ -86,7 +84,6 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
|
||||||
h.logAndSendError(ctx, w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
h.logAndSendError(ctx, w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -123,10 +120,7 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.DeleteBucketLifecycle")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
|
|
@ -29,8 +29,6 @@ func TestPutBucketLifecycleConfiguration(t *testing.T) {
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
body *data.LifecycleConfiguration
|
body *data.LifecycleConfiguration
|
||||||
headers map[string]string
|
|
||||||
addMD5 bool
|
|
||||||
errorCode apierr.ErrorCode
|
errorCode apierr.ErrorCode
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
|
@ -72,22 +70,6 @@ func TestPutBucketLifecycleConfiguration(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
|
|
||||||
name: "correct Content-Md5 header",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
ID: "rule",
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Days: ptr(21),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
addMD5: true,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "too many rules",
|
name: "too many rules",
|
||||||
body: func() *data.LifecycleConfiguration {
|
body: func() *data.LifecycleConfiguration {
|
||||||
|
@ -425,44 +407,14 @@ func TestPutBucketLifecycleConfiguration(t *testing.T) {
|
||||||
},
|
},
|
||||||
errorCode: apierr.ErrInvalidRequest,
|
errorCode: apierr.ErrInvalidRequest,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "invalid Content-Md5 header",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Days: ptr(21),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
headers: map[string]string{api.ContentMD5: "invalid"},
|
|
||||||
errorCode: apierr.ErrInvalidDigest,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Content-Md5 header does not match body md5 hash",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Days: ptr(21),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
headers: map[string]string{api.ContentMD5: base64.StdEncoding.EncodeToString([]byte("some-hash"))},
|
|
||||||
errorCode: apierr.ErrInvalidDigest,
|
|
||||||
},
|
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
if tc.errorCode > 0 {
|
if tc.errorCode > 0 {
|
||||||
putBucketLifecycleConfigurationErr(hc, bktName, tc.body, tc.headers, apierr.GetAPIError(tc.errorCode))
|
putBucketLifecycleConfigurationErr(hc, bktName, tc.body, apierr.GetAPIError(tc.errorCode))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
putBucketLifecycleConfiguration(hc, bktName, tc.body, tc.headers, tc.addMD5)
|
putBucketLifecycleConfiguration(hc, bktName, tc.body)
|
||||||
|
|
||||||
cfg := getBucketLifecycleConfiguration(hc, bktName)
|
cfg := getBucketLifecycleConfiguration(hc, bktName)
|
||||||
require.Equal(t, tc.body.Rules, cfg.Rules)
|
require.Equal(t, tc.body.Rules, cfg.Rules)
|
||||||
|
@ -496,13 +448,45 @@ func TestPutBucketLifecycleIDGeneration(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
putBucketLifecycleConfiguration(hc, bktName, lifecycle, nil, false)
|
putBucketLifecycleConfiguration(hc, bktName, lifecycle)
|
||||||
cfg := getBucketLifecycleConfiguration(hc, bktName)
|
cfg := getBucketLifecycleConfiguration(hc, bktName)
|
||||||
require.Len(t, cfg.Rules, 2)
|
require.Len(t, cfg.Rules, 2)
|
||||||
require.NotEmpty(t, cfg.Rules[0].ID)
|
require.NotEmpty(t, cfg.Rules[0].ID)
|
||||||
require.NotEmpty(t, cfg.Rules[1].ID)
|
require.NotEmpty(t, cfg.Rules[1].ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPutBucketLifecycleInvalidMD5(t *testing.T) {
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
bktName := "bucket-lifecycle-md5"
|
||||||
|
createBucket(hc, bktName)
|
||||||
|
|
||||||
|
lifecycle := &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
w, r := prepareTestRequest(hc, bktName, "", lifecycle)
|
||||||
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
|
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMissingContentMD5))
|
||||||
|
|
||||||
|
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
|
||||||
|
r.Header.Set(api.ContentMD5, "")
|
||||||
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
|
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
||||||
|
|
||||||
|
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
|
||||||
|
r.Header.Set(api.ContentMD5, "some-hash")
|
||||||
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
|
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
||||||
|
}
|
||||||
|
|
||||||
func TestPutBucketLifecycleInvalidXML(t *testing.T) {
|
func TestPutBucketLifecycleInvalidXML(t *testing.T) {
|
||||||
hc := prepareHandlerContext(t)
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
@ -521,32 +505,25 @@ func TestPutBucketLifecycleInvalidXML(t *testing.T) {
|
||||||
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMalformedXML))
|
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMalformedXML))
|
||||||
}
|
}
|
||||||
|
|
||||||
func putBucketLifecycleConfiguration(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, headers map[string]string, addMD5 bool) {
|
func putBucketLifecycleConfiguration(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) {
|
||||||
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg, headers, addMD5)
|
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
|
||||||
assertStatus(hc.t, w, http.StatusOK)
|
assertStatus(hc.t, w, http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
func putBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, headers map[string]string, err apierr.Error) {
|
func putBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, err apierr.Error) {
|
||||||
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg, headers, false)
|
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
|
||||||
assertS3Error(hc.t, w, err)
|
assertS3Error(hc.t, w, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func putBucketLifecycleConfigurationBase(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, headers map[string]string, addMD5 bool) *httptest.ResponseRecorder {
|
func putBucketLifecycleConfigurationBase(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) *httptest.ResponseRecorder {
|
||||||
w, r := prepareTestRequest(hc, bktName, "", cfg)
|
w, r := prepareTestRequest(hc, bktName, "", cfg)
|
||||||
|
|
||||||
for k, v := range headers {
|
|
||||||
r.Header.Set(k, v)
|
|
||||||
}
|
|
||||||
|
|
||||||
if addMD5 {
|
|
||||||
rawBody, err := xml.Marshal(cfg)
|
rawBody, err := xml.Marshal(cfg)
|
||||||
require.NoError(hc.t, err)
|
require.NoError(hc.t, err)
|
||||||
|
|
||||||
hash := md5.New()
|
hash := md5.New()
|
||||||
hash.Write(rawBody)
|
hash.Write(rawBody)
|
||||||
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(hash.Sum(nil)))
|
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(hash.Sum(nil)))
|
||||||
}
|
|
||||||
|
|
||||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
return w
|
return w
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,7 +7,6 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
@ -27,9 +26,7 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutBucketObjectLockConfig")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -77,9 +74,7 @@ func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetBucketObjectLockConfig")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -113,9 +108,7 @@ func (h *handler) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutObjectLegalHold")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -168,9 +161,7 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetObjectLegalHold")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -208,9 +199,7 @@ func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutObjectRetention")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -258,9 +247,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetObjectRetention")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
|
|
@ -10,7 +10,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
@ -105,9 +104,7 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.CreateMultipartUpload")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
uploadID := uuid.New()
|
uploadID := uuid.New()
|
||||||
cannedACLStatus := aclHeadersStatus(r)
|
cannedACLStatus := aclHeadersStatus(r)
|
||||||
|
@ -177,9 +174,7 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.UploadPart")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -252,16 +247,15 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.UploadPartCopy")
|
var (
|
||||||
defer span.End()
|
versionID string
|
||||||
|
ctx = r.Context()
|
||||||
var versionID string
|
reqInfo = middleware.GetReqInfo(ctx)
|
||||||
|
queryValues = reqInfo.URL.Query()
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
uploadID = queryValues.Get(uploadIDHeaderName)
|
||||||
queryValues := reqInfo.URL.Query()
|
partNumStr = queryValues.Get(partNumberHeaderName)
|
||||||
uploadID := queryValues.Get(uploadIDHeaderName)
|
additional = []zap.Field{zap.String("uploadID", uploadID), zap.String("partNumber", partNumStr)}
|
||||||
partNumStr := queryValues.Get(partNumberHeaderName)
|
)
|
||||||
additional := []zap.Field{zap.String("uploadID", uploadID), zap.String("partNumber", partNumStr)}
|
|
||||||
|
|
||||||
partNumber, err := strconv.Atoi(partNumStr)
|
partNumber, err := strconv.Atoi(partNumStr)
|
||||||
if err != nil || partNumber < layer.UploadMinPartNumber || partNumber > layer.UploadMaxPartNumber {
|
if err != nil || partNumber < layer.UploadMinPartNumber || partNumber > layer.UploadMaxPartNumber {
|
||||||
|
@ -387,9 +381,7 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.CompleteMultipartUpload")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -516,9 +508,7 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.ListMultipartUploads")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -569,9 +559,7 @@ func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Req
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.ListParts")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -635,9 +623,7 @@ func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.AbortMultipartUpload")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
|
|
@ -1,32 +1,25 @@
|
||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"encoding/xml"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"go.uber.org/zap"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||||
|
|
||||||
// ListObjectsV1Handler handles objects listing requests for API version 1.
|
// ListObjectsV1Handler handles objects listing requests for API version 1.
|
||||||
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.ListObjectsV1")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
params, err := parseListObjectsArgsV1(reqInfo)
|
params, err := parseListObjectsArgsV1(reqInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -39,28 +32,14 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
ch := make(chan struct{})
|
|
||||||
defer close(ch)
|
|
||||||
params.Chan = ch
|
|
||||||
|
|
||||||
stopPeriodicResponseWriter := periodicXMLWriter(w, h.cfg.ListingKeepaliveThrottle(), ch)
|
|
||||||
|
|
||||||
list, err := h.obj.ListObjectsV1(ctx, params)
|
list, err := h.obj.ListObjectsV1(ctx, params)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logAndSendError := h.periodicWriterErrorSender(stopPeriodicResponseWriter())
|
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
|
||||||
logAndSendError(ctx, w, "could not list objects v1", reqInfo, err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
headerIsWritten := stopPeriodicResponseWriter()
|
|
||||||
if headerIsWritten {
|
|
||||||
if err = middleware.EncodeToResponseNoHeader(w, h.encodeV1(params, list)); err != nil {
|
|
||||||
h.logAndSendErrorNoHeader(ctx, w, "could not encode listing v1 response", reqInfo, err)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if err = middleware.EncodeToResponse(w, h.encodeV1(params, list)); err != nil {
|
if err = middleware.EncodeToResponse(w, h.encodeV1(params, list)); err != nil {
|
||||||
h.logAndSendError(ctx, w, "could not encode listing v1 response", reqInfo, err)
|
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -85,9 +64,7 @@ func (h *handler) encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjects
|
||||||
|
|
||||||
// ListObjectsV2Handler handles objects listing requests for API version 2.
|
// ListObjectsV2Handler handles objects listing requests for API version 2.
|
||||||
func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.ListObjectsV2")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
params, err := parseListObjectsArgsV2(reqInfo)
|
params, err := parseListObjectsArgsV2(reqInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -100,28 +77,14 @@ func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
ch := make(chan struct{})
|
|
||||||
defer close(ch)
|
|
||||||
params.Chan = ch
|
|
||||||
|
|
||||||
stopPeriodicResponseWriter := periodicXMLWriter(w, h.cfg.ListingKeepaliveThrottle(), ch)
|
|
||||||
|
|
||||||
list, err := h.obj.ListObjectsV2(ctx, params)
|
list, err := h.obj.ListObjectsV2(ctx, params)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logAndSendError := h.periodicWriterErrorSender(stopPeriodicResponseWriter())
|
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
|
||||||
logAndSendError(ctx, w, "could not list objects v2", reqInfo, err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
headerIsWritten := stopPeriodicResponseWriter()
|
|
||||||
if headerIsWritten {
|
|
||||||
if err = middleware.EncodeToResponseNoHeader(w, h.encodeV2(params, list)); err != nil {
|
|
||||||
h.logAndSendErrorNoHeader(ctx, w, "could not encode listing v2 response", reqInfo, err)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if err = middleware.EncodeToResponse(w, h.encodeV2(params, list)); err != nil {
|
if err = middleware.EncodeToResponse(w, h.encodeV2(params, list)); err != nil {
|
||||||
h.logAndSendError(ctx, w, "could not encode listing v2 response", reqInfo, err)
|
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -260,9 +223,7 @@ func fillContents(src []*data.ExtendedNodeVersion, encode string, fetchOwner, md
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.ListBucketObjectVersions")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
p, err := parseListObjectVersionsRequest(reqInfo)
|
p, err := parseListObjectVersionsRequest(reqInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -275,29 +236,15 @@ func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
ch := make(chan struct{})
|
info, err := h.obj.ListObjectVersions(ctx, p)
|
||||||
defer close(ch)
|
|
||||||
p.Chan = ch
|
|
||||||
|
|
||||||
stopPeriodicResponseWriter := periodicXMLWriter(w, h.cfg.ListingKeepaliveThrottle(), ch)
|
|
||||||
|
|
||||||
list, err := h.obj.ListObjectVersions(ctx, p)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logAndSendError := h.periodicWriterErrorSender(stopPeriodicResponseWriter())
|
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
|
||||||
logAndSendError(ctx, w, "could not list objects versions", reqInfo, err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
response := encodeListObjectVersionsToResponse(p, list, p.BktInfo.Name, h.cfg.MD5Enabled())
|
response := encodeListObjectVersionsToResponse(p, info, p.BktInfo.Name, h.cfg.MD5Enabled())
|
||||||
headerIsWritten := stopPeriodicResponseWriter()
|
|
||||||
if headerIsWritten {
|
|
||||||
if err = middleware.EncodeToResponseNoHeader(w, response); err != nil {
|
|
||||||
h.logAndSendErrorNoHeader(ctx, w, "could not encode listing versions response", reqInfo, err)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if err = middleware.EncodeToResponse(w, response); err != nil {
|
if err = middleware.EncodeToResponse(w, response); err != nil {
|
||||||
h.logAndSendError(ctx, w, "could not encode listing versions response", reqInfo, err)
|
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -380,70 +327,3 @@ func encodeListObjectVersionsToResponse(p *layer.ListObjectVersionsParams, info
|
||||||
|
|
||||||
return &res
|
return &res
|
||||||
}
|
}
|
||||||
|
|
||||||
// periodicWriterErrorSender returns handler function to send error. If header is
|
|
||||||
// already written by periodic XML writer, do not send HTTP and XML headers.
|
|
||||||
func (h *handler) periodicWriterErrorSender(headerWritten bool) func(context.Context, http.ResponseWriter, string, *middleware.ReqInfo, error, ...zap.Field) {
|
|
||||||
if headerWritten {
|
|
||||||
return h.logAndSendErrorNoHeader
|
|
||||||
}
|
|
||||||
return h.logAndSendError
|
|
||||||
}
|
|
||||||
|
|
||||||
// periodicXMLWriter creates go routine to write xml header and whitespaces
|
|
||||||
// over time to avoid connection drop from the client. To work properly,
|
|
||||||
// pass `http.ResponseWriter` with implemented `http.Flusher` interface.
|
|
||||||
// Returns stop function which returns boolean if writer has been used
|
|
||||||
// during goroutine execution. Zero or negative duration disable writing.
|
|
||||||
func periodicXMLWriter(w io.Writer, dur time.Duration, ch <-chan struct{}) (stop func() bool) {
|
|
||||||
if dur <= 0 {
|
|
||||||
return func() bool { return false }
|
|
||||||
}
|
|
||||||
|
|
||||||
whitespaceChar := []byte(" ")
|
|
||||||
closer := make(chan struct{})
|
|
||||||
done := make(chan struct{})
|
|
||||||
headerWritten := false
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer close(done)
|
|
||||||
|
|
||||||
var lastEvent time.Time
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case _, ok := <-ch:
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if time.Since(lastEvent) < dur {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
lastEvent = time.Now()
|
|
||||||
|
|
||||||
if !headerWritten {
|
|
||||||
_, err := w.Write([]byte(xml.Header))
|
|
||||||
headerWritten = err == nil
|
|
||||||
}
|
|
||||||
_, err := w.Write(whitespaceChar)
|
|
||||||
if err != nil {
|
|
||||||
return // is there anything we can do better than ignore error?
|
|
||||||
}
|
|
||||||
if buffered, ok := w.(http.Flusher); ok {
|
|
||||||
buffered.Flush()
|
|
||||||
}
|
|
||||||
case <-closer:
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
stop = func() bool {
|
|
||||||
close(closer)
|
|
||||||
<-done // wait for goroutine to stop
|
|
||||||
return headerWritten
|
|
||||||
}
|
|
||||||
|
|
||||||
return stop
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,9 +1,7 @@
|
||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"encoding/xml"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
@ -105,7 +103,7 @@ func TestListObjectsVersionsSkipLogTaggingNodesError(t *testing.T) {
|
||||||
loggerCore, observedLog := observer.New(zap.DebugLevel)
|
loggerCore, observedLog := observer.New(zap.DebugLevel)
|
||||||
log := zap.New(loggerCore)
|
log := zap.New(loggerCore)
|
||||||
|
|
||||||
hcBase, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(log), log)
|
hcBase, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(log))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
hc := &handlerContext{
|
hc := &handlerContext{
|
||||||
handlerContextBase: hcBase,
|
handlerContextBase: hcBase,
|
||||||
|
@ -178,7 +176,7 @@ func TestListObjectsContextCanceled(t *testing.T) {
|
||||||
layerCfg.SessionList.Lifetime = time.Hour
|
layerCfg.SessionList.Lifetime = time.Hour
|
||||||
layerCfg.SessionList.Size = 1
|
layerCfg.SessionList.Size = 1
|
||||||
|
|
||||||
hcBase, err := prepareHandlerContextBase(layerCfg, log)
|
hcBase, err := prepareHandlerContextBase(layerCfg)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
hc := &handlerContext{
|
hc := &handlerContext{
|
||||||
handlerContextBase: hcBase,
|
handlerContextBase: hcBase,
|
||||||
|
@ -843,101 +841,6 @@ func TestListingsWithInvalidEncodingType(t *testing.T) {
|
||||||
listObjectsV1Err(hc, bktName, "invalid", apierr.GetAPIError(apierr.ErrInvalidEncodingMethod))
|
listObjectsV1Err(hc, bktName, "invalid", apierr.GetAPIError(apierr.ErrInvalidEncodingMethod))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPeriodicWriter(t *testing.T) {
|
|
||||||
const dur = 100 * time.Millisecond
|
|
||||||
const whitespaces = 8
|
|
||||||
expected := []byte(xml.Header)
|
|
||||||
for i := 0; i < whitespaces; i++ {
|
|
||||||
expected = append(expected, []byte(" ")...)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("writes data", func(t *testing.T) {
|
|
||||||
ch := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
defer close(ch)
|
|
||||||
|
|
||||||
for range whitespaces {
|
|
||||||
ch <- struct{}{}
|
|
||||||
time.Sleep(dur)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
buf := bytes.NewBuffer(nil)
|
|
||||||
stop := periodicXMLWriter(buf, time.Nanosecond, ch)
|
|
||||||
|
|
||||||
// N number of whitespaces + half durations to guarantee at least N writes in buffer
|
|
||||||
time.Sleep(whitespaces*dur + dur/2)
|
|
||||||
require.True(t, stop())
|
|
||||||
require.Equal(t, string(expected), buf.String())
|
|
||||||
|
|
||||||
t.Run("no additional data after stop", func(t *testing.T) {
|
|
||||||
time.Sleep(2 * dur)
|
|
||||||
require.Equal(t, string(expected), buf.String())
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("does not write data", func(t *testing.T) {
|
|
||||||
buf := bytes.NewBuffer(nil)
|
|
||||||
|
|
||||||
ch := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
defer close(ch)
|
|
||||||
|
|
||||||
for range whitespaces {
|
|
||||||
time.Sleep(2 * dur)
|
|
||||||
ch <- struct{}{}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
stop := periodicXMLWriter(buf, time.Nanosecond, ch)
|
|
||||||
require.False(t, stop())
|
|
||||||
require.Empty(t, buf.Bytes())
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("throttling works", func(t *testing.T) {
|
|
||||||
buf := bytes.NewBuffer(nil)
|
|
||||||
|
|
||||||
ch := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
defer close(ch)
|
|
||||||
|
|
||||||
for range whitespaces {
|
|
||||||
ch <- struct{}{}
|
|
||||||
time.Sleep(dur / 2)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
stop := periodicXMLWriter(buf, dur, ch)
|
|
||||||
// N number of whitespaces + half durations to guarantee at least N writes in buffer
|
|
||||||
time.Sleep(whitespaces*dur + dur/2)
|
|
||||||
require.True(t, stop())
|
|
||||||
require.Equal(t, string(expected[:len(expected)-4]), buf.String())
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("disabled", func(t *testing.T) {
|
|
||||||
buf := bytes.NewBuffer(nil)
|
|
||||||
|
|
||||||
ch := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
defer close(ch)
|
|
||||||
|
|
||||||
for range whitespaces {
|
|
||||||
select {
|
|
||||||
case ch <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
time.Sleep(dur / 2)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
stop := periodicXMLWriter(buf, 0, ch)
|
|
||||||
// N number of whitespaces + half durations to guarantee at least N writes in buffer
|
|
||||||
time.Sleep(whitespaces*dur + dur/2)
|
|
||||||
require.False(t, stop())
|
|
||||||
require.Empty(t, buf.String())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkVersionsNames(t *testing.T, versions *ListObjectsVersionsResponse, names []string) {
|
func checkVersionsNames(t *testing.T, versions *ListObjectsVersionsResponse, names []string) {
|
||||||
for i, v := range versions.Version {
|
for i, v := range versions.Version {
|
||||||
require.Equal(t, names[i], v.Key)
|
require.Equal(t, names[i], v.Key)
|
||||||
|
|
|
@ -7,7 +7,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
@ -20,9 +19,7 @@ import (
|
||||||
const maxPatchSize = 5 * 1024 * 1024 * 1024 // 5GB
|
const maxPatchSize = 5 * 1024 * 1024 * 1024 // 5GB
|
||||||
|
|
||||||
func (h *handler) PatchObjectHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PatchObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PatchObject")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
if _, ok := r.Header[api.ContentRange]; !ok {
|
if _, ok := r.Header[api.ContentRange]; !ok {
|
||||||
|
@ -145,7 +142,7 @@ func parsePatchConditionalHeaders(headers http.Header, log *zap.Logger) *conditi
|
||||||
if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil {
|
if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil {
|
||||||
args.IfUnmodifiedSince = httpTime
|
args.IfUnmodifiedSince = httpTime
|
||||||
} else {
|
} else {
|
||||||
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err), logs.TagField(logs.TagDatapath))
|
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
return args
|
return args
|
||||||
|
|
|
@ -18,7 +18,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
@ -55,29 +54,17 @@ func (p *postPolicy) condition(key string) *policyCondition {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *postPolicy) CheckContentLength(size uint64) error {
|
func (p *postPolicy) CheckContentLength(size uint64) bool {
|
||||||
if p.empty {
|
if p.empty {
|
||||||
return nil
|
return true
|
||||||
}
|
}
|
||||||
for _, condition := range p.Conditions {
|
for _, condition := range p.Conditions {
|
||||||
if condition.Matching == "content-length-range" {
|
if condition.Matching == "content-length-range" {
|
||||||
start, err := strconv.ParseUint(condition.Key, 10, 64)
|
length := strconv.FormatUint(size, 10)
|
||||||
if err != nil {
|
return condition.Key <= length && length <= condition.Value
|
||||||
return errInvalidCondition
|
|
||||||
}
|
|
||||||
|
|
||||||
end, err := strconv.ParseUint(condition.Value, 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return errInvalidCondition
|
|
||||||
}
|
|
||||||
|
|
||||||
if start <= size && size <= end {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return fmt.Errorf("length of the content did not fall within the range specified in the condition")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *policyCondition) match(value string) bool {
|
func (p *policyCondition) match(value string) bool {
|
||||||
|
@ -197,13 +184,12 @@ type createBucketParams struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutObject")
|
var (
|
||||||
defer span.End()
|
err error
|
||||||
|
cannedACLStatus = aclHeadersStatus(r)
|
||||||
var err error
|
ctx = r.Context()
|
||||||
|
reqInfo = middleware.GetReqInfo(ctx)
|
||||||
cannedACLStatus := aclHeadersStatus(r)
|
)
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -480,7 +466,7 @@ func (h *handler) isTLSCheckRequired(r *http.Request) bool {
|
||||||
|
|
||||||
tlsTermination, err := strconv.ParseBool(tlsTerminationStr)
|
tlsTermination, err := strconv.ParseBool(tlsTerminationStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.reqLogger(r.Context()).Warn(logs.WarnInvalidTypeTLSTerminationHeader, zap.String("header", tlsTerminationStr), zap.Error(err), logs.TagField(logs.TagDatapath))
|
h.reqLogger(r.Context()).Warn(logs.WarnInvalidTypeTLSTerminationHeader, zap.String("header", tlsTerminationStr), zap.Error(err))
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -488,18 +474,16 @@ func (h *handler) isTLSCheckRequired(r *http.Request) bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PostObject")
|
var (
|
||||||
defer span.End()
|
tagSet map[string]string
|
||||||
|
ctx = r.Context()
|
||||||
var tagSet map[string]string
|
reqInfo = middleware.GetReqInfo(ctx)
|
||||||
|
metadata = make(map[string]string)
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
)
|
||||||
metadata := make(map[string]string)
|
|
||||||
|
|
||||||
policy, err := checkPostPolicy(r, reqInfo, metadata)
|
policy, err := checkPostPolicy(r, reqInfo, metadata)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(ctx, w, "failed check policy", reqInfo,
|
h.logAndSendError(ctx, w, "failed check policy", reqInfo, err)
|
||||||
fmt.Errorf("%w: %v", apierr.GetAPIError(apierr.ErrInvalidArgument), err))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -549,8 +533,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
if reqInfo.ObjectName == "" || strings.Contains(reqInfo.ObjectName, "${filename}") {
|
if reqInfo.ObjectName == "" || strings.Contains(reqInfo.ObjectName, "${filename}") {
|
||||||
_, head, err := r.FormFile("file")
|
_, head, err := r.FormFile("file")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(ctx, w, "could not parse file field", reqInfo,
|
h.logAndSendError(ctx, w, "could not parse file field", reqInfo, err)
|
||||||
fmt.Errorf("%w: %v", apierr.GetAPIError(apierr.ErrInvalidArgument), err))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
filename = head.Filename
|
filename = head.Filename
|
||||||
|
@ -559,8 +542,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
var head *multipart.FileHeader
|
var head *multipart.FileHeader
|
||||||
contentReader, head, err = r.FormFile("file")
|
contentReader, head, err = r.FormFile("file")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(ctx, w, "could not parse file field", reqInfo,
|
h.logAndSendError(ctx, w, "could not parse file field", reqInfo, err)
|
||||||
fmt.Errorf("%w: %v", apierr.GetAPIError(apierr.ErrInvalidArgument), err))
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
size = uint64(head.Size)
|
size = uint64(head.Size)
|
||||||
|
@ -578,8 +560,8 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := policy.CheckContentLength(size); err != nil {
|
if !policy.CheckContentLength(size) {
|
||||||
h.logAndSendError(ctx, w, err.Error(), reqInfo, apierr.GetAPIError(apierr.ErrPostPolicyConditionInvalidFormat))
|
h.logAndSendError(ctx, w, "invalid content-length", reqInfo, apierr.GetAPIError(apierr.ErrInvalidArgument))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -605,7 +587,6 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
ObjectName: objInfo.Name,
|
ObjectName: objInfo.Name,
|
||||||
VersionID: objInfo.VersionID(),
|
VersionID: objInfo.VersionID(),
|
||||||
},
|
},
|
||||||
TagSet: tagSet,
|
|
||||||
NodeVersion: extendedObjInfo.NodeVersion,
|
NodeVersion: extendedObjInfo.NodeVersion,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -783,10 +764,7 @@ func parseCannedACL(header http.Header) (string, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.CreateBucket")
|
h.createBucketHandlerPolicy(w, r)
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
h.createBucketHandlerPolicy(w, r.WithContext(ctx))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) parseCommonCreateBucketParams(reqInfo *middleware.ReqInfo, boxData *accessbox.Box, r *http.Request) (*keys.PublicKey, *layer.CreateBucketParams, error) {
|
func (h *handler) parseCommonCreateBucketParams(reqInfo *middleware.ReqInfo, boxData *accessbox.Box, r *http.Request) (*keys.PublicKey, *layer.CreateBucketParams, error) {
|
||||||
|
@ -850,7 +828,7 @@ func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Reque
|
||||||
h.logAndSendError(ctx, w, "could not create bucket", reqInfo, err)
|
h.logAndSendError(ctx, w, "could not create bucket", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID), logs.TagField(logs.TagExternalStorage))
|
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
|
||||||
|
|
||||||
chains := bucketCannedACLToAPERules(cannedACL, reqInfo, bktInfo.CID)
|
chains := bucketCannedACLToAPERules(cannedACL, reqInfo, bktInfo.CID)
|
||||||
if err = h.ape.SaveACLChains(bktInfo.CID.EncodeToString(), chains); err != nil {
|
if err = h.ape.SaveACLChains(bktInfo.CID.EncodeToString(), chains); err != nil {
|
||||||
|
@ -1063,7 +1041,7 @@ func isLockEnabled(log *zap.Logger, header http.Header) bool {
|
||||||
|
|
||||||
lockEnabled, err := strconv.ParseBool(lockEnabledStr)
|
lockEnabled, err := strconv.ParseBool(lockEnabledStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn(logs.InvalidBucketObjectLockEnabledHeader, zap.String("header", lockEnabledStr), zap.Error(err), logs.TagField(logs.TagDatapath))
|
log.Warn(logs.InvalidBucketObjectLockEnabledHeader, zap.String("header", lockEnabledStr), zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
return lockEnabled
|
return lockEnabled
|
||||||
|
|
|
@ -9,10 +9,7 @@ import (
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"encoding/xml"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"hash/crc32"
|
|
||||||
"io"
|
"io"
|
||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
@ -149,31 +146,8 @@ func TestPostObject(t *testing.T) {
|
||||||
filename string
|
filename string
|
||||||
content string
|
content string
|
||||||
objName string
|
objName string
|
||||||
tagging string
|
|
||||||
err bool
|
err bool
|
||||||
}{
|
}{
|
||||||
{
|
|
||||||
key: "user/user1/${filename}",
|
|
||||||
filename: "object",
|
|
||||||
content: "content",
|
|
||||||
objName: "user/user1/object",
|
|
||||||
tagging: "<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><TagSet><Tag><Key>Environment</Key><Value>Production</Value></Tag></TagSet></Tagging>",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
key: "user/user1/${filename}",
|
|
||||||
filename: "object",
|
|
||||||
content: "content",
|
|
||||||
objName: "user/user1/object",
|
|
||||||
tagging: "wrong tagging",
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
key: "user/user1/${filename}",
|
|
||||||
filename: "object",
|
|
||||||
content: "content",
|
|
||||||
objName: "user/user1/object",
|
|
||||||
tagging: "<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><TagSet></TagSet></Tagging>",
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
key: "user/user1/${filename}",
|
key: "user/user1/${filename}",
|
||||||
filename: "object",
|
filename: "object",
|
||||||
|
@ -231,21 +205,14 @@ func TestPostObject(t *testing.T) {
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.key+";"+tc.filename, func(t *testing.T) {
|
t.Run(tc.key+";"+tc.filename, func(t *testing.T) {
|
||||||
w := postObjectBase(hc, ns, bktName, tc.key, tc.filename, tc.content, tc.tagging)
|
w := postObjectBase(hc, ns, bktName, tc.key, tc.filename, tc.content)
|
||||||
if tc.err {
|
if tc.err {
|
||||||
assertStatus(hc.t, w, http.StatusBadRequest)
|
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInternalError))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
assertStatus(hc.t, w, http.StatusNoContent)
|
assertStatus(hc.t, w, http.StatusNoContent)
|
||||||
content, _ := getObject(hc, bktName, tc.objName)
|
content, _ := getObject(hc, bktName, tc.objName)
|
||||||
require.Equal(t, tc.content, string(content))
|
require.Equal(t, tc.content, string(content))
|
||||||
|
|
||||||
if tc.tagging != "" {
|
|
||||||
tagging := getObjectTagging(t, hc, bktName, tc.objName, "")
|
|
||||||
strtags, err := xml.Marshal(tagging)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, tc.tagging, string(strtags))
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -459,7 +426,7 @@ func TestPutObjectWithStreamBodyAWSExampleTrailing(t *testing.T) {
|
||||||
createTestBucket(hc, bktName)
|
createTestBucket(hc, bktName)
|
||||||
|
|
||||||
t.Run("valid trailer signature", func(t *testing.T) {
|
t.Run("valid trailer signature", func(t *testing.T) {
|
||||||
w, req, chunk := getChunkedRequestAWSExampleTrailing(t, bktName, objName)
|
w, req, chunk := getChunkedRequestTrailing(hc.context, t, bktName, objName)
|
||||||
hc.Handler().PutObjectHandler(w, req)
|
hc.Handler().PutObjectHandler(w, req)
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
|
||||||
|
@ -473,7 +440,7 @@ func TestPutObjectWithStreamBodyAWSExampleTrailing(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("invalid trailer signature", func(t *testing.T) {
|
t.Run("invalid trailer signature", func(t *testing.T) {
|
||||||
w, req, _ := getChunkedRequestAWSExampleTrailing(t, bktName, objName)
|
w, req, _ := getChunkedRequestTrailing(hc.context, t, bktName, objName)
|
||||||
body := req.Body.(*customNopCloser)
|
body := req.Body.(*customNopCloser)
|
||||||
body.Bytes()[body.Len()-2] = 'a'
|
body.Bytes()[body.Len()-2] = 'a'
|
||||||
hc.Handler().PutObjectHandler(w, req)
|
hc.Handler().PutObjectHandler(w, req)
|
||||||
|
@ -487,7 +454,7 @@ func TestPutObjectWithStreamBodyAWSExample(t *testing.T) {
|
||||||
bktName, objName := "examplebucket", "chunkObject.txt"
|
bktName, objName := "examplebucket", "chunkObject.txt"
|
||||||
createTestBucket(hc, bktName)
|
createTestBucket(hc, bktName)
|
||||||
|
|
||||||
w, req, chunk := getChunkedRequestAWSExample(t, bktName, objName)
|
w, req, chunk := getChunkedRequest(hc.context, t, bktName, objName)
|
||||||
hc.Handler().PutObjectHandler(w, req)
|
hc.Handler().PutObjectHandler(w, req)
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
|
||||||
|
@ -502,17 +469,13 @@ func TestPutObjectWithStreamBodyAWSExample(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPutObjectWithStreamEmptyBodyAWSExampleWithContentType(t *testing.T) {
|
func TestPutObjectWithStreamEmptyBodyAWSExample(t *testing.T) {
|
||||||
hc := prepareHandlerContext(t)
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
bktName, objName := "dkirillov", "tmp"
|
bktName, objName := "dkirillov", "tmp"
|
||||||
createTestBucket(hc, bktName)
|
createTestBucket(hc, bktName)
|
||||||
|
|
||||||
signTime, err := time.Parse("20060102T150405Z", "20241003T100055Z")
|
w, req := getEmptyChunkedRequest(hc.context, t, bktName, objName)
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
extra := [2]string{api.ContentType, "text/plain; charset=UTF-8"}
|
|
||||||
w, req := getChunkedRequestBase(t, bktName, objName, nil, api.StreamingContentSHA256, signTime, extra)
|
|
||||||
hc.Handler().PutObjectHandler(w, req)
|
hc.Handler().PutObjectHandler(w, req)
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
|
||||||
|
@ -526,94 +489,13 @@ func TestPutObjectWithStreamEmptyBodyAWSExampleWithContentType(t *testing.T) {
|
||||||
require.Empty(t, res.Contents[0].Size)
|
require.Empty(t, res.Contents[0].Size)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPutObjectWithStreamEmptyBody(t *testing.T) {
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
|
|
||||||
bktName := "bucket"
|
|
||||||
createTestBucket(hc, bktName)
|
|
||||||
|
|
||||||
signTime, err := time.Parse("20060102T150405Z", "20241003T100055Z")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Run("unsigned", func(t *testing.T) {
|
|
||||||
t.Run("trailer", func(t *testing.T) {
|
|
||||||
objName := "unsigned trailer"
|
|
||||||
|
|
||||||
w, req := getEmptyChunkedRequestUnsigned(t, bktName, objName)
|
|
||||||
req.Header.Del(api.ContentType)
|
|
||||||
hc.Handler().PutObjectHandler(w, req)
|
|
||||||
assertStatus(t, w, http.StatusOK)
|
|
||||||
|
|
||||||
d, h := getObject(hc, bktName, objName)
|
|
||||||
require.Empty(t, d)
|
|
||||||
require.Equal(t, "0", h.Get(api.ContentLength))
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("sigv4", func(t *testing.T) {
|
|
||||||
t.Run("trailer", func(t *testing.T) {
|
|
||||||
objName := "sigv4 trailer"
|
|
||||||
|
|
||||||
w, req := getChunkedRequestBase(t, bktName, objName, nil, api.StreamingContentSHA256Trailer, signTime)
|
|
||||||
req.Header.Del(api.ContentType)
|
|
||||||
hc.Handler().PutObjectHandler(w, req)
|
|
||||||
assertStatus(t, w, http.StatusOK)
|
|
||||||
|
|
||||||
d, h := getObject(hc, bktName, objName)
|
|
||||||
require.Empty(t, d)
|
|
||||||
require.Equal(t, "0", h.Get(api.ContentLength))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("no trailer", func(t *testing.T) {
|
|
||||||
objName := "sigv4 no trailer"
|
|
||||||
|
|
||||||
w, req := getChunkedRequestBase(t, bktName, objName, nil, api.StreamingContentSHA256, signTime)
|
|
||||||
req.Header.Del(api.ContentType)
|
|
||||||
hc.Handler().PutObjectHandler(w, req)
|
|
||||||
assertStatus(t, w, http.StatusOK)
|
|
||||||
|
|
||||||
d, h := getObject(hc, bktName, objName)
|
|
||||||
require.Empty(t, d)
|
|
||||||
require.Equal(t, "0", h.Get(api.ContentLength))
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("sigv4a", func(t *testing.T) {
|
|
||||||
t.Run("trailer", func(t *testing.T) {
|
|
||||||
objName := "sigv4a trailer"
|
|
||||||
|
|
||||||
w, req := getEmptyChunkedRequestSigv4aWithTrailers(t, bktName, objName)
|
|
||||||
req.Header.Del(api.ContentType)
|
|
||||||
hc.Handler().PutObjectHandler(w, req)
|
|
||||||
assertStatus(t, w, http.StatusOK)
|
|
||||||
|
|
||||||
d, h := getObject(hc, bktName, objName)
|
|
||||||
require.Empty(t, d)
|
|
||||||
require.Equal(t, "0", h.Get(api.ContentLength))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("no trailer", func(t *testing.T) {
|
|
||||||
objName := "sigv4a no trailer"
|
|
||||||
|
|
||||||
w, req := getEmptyChunkedRequestSigv4a(t, bktName, objName)
|
|
||||||
req.Header.Del(api.ContentType)
|
|
||||||
hc.Handler().PutObjectHandler(w, req)
|
|
||||||
assertStatus(t, w, http.StatusOK)
|
|
||||||
|
|
||||||
d, h := getObject(hc, bktName, objName)
|
|
||||||
require.Empty(t, d)
|
|
||||||
require.Equal(t, "0", h.Get(api.ContentLength))
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPutChunkedTestContentEncoding(t *testing.T) {
|
func TestPutChunkedTestContentEncoding(t *testing.T) {
|
||||||
hc := prepareHandlerContext(t)
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
bktName, objName := "examplebucket", "chunkObject.txt"
|
bktName, objName := "examplebucket", "chunkObject.txt"
|
||||||
createTestBucket(hc, bktName)
|
createTestBucket(hc, bktName)
|
||||||
|
|
||||||
w, req, _ := getChunkedRequestAWSExample(t, bktName, objName)
|
w, req, _ := getChunkedRequest(hc.context, t, bktName, objName)
|
||||||
req.Header.Set(api.ContentEncoding, api.AwsChunked+",gzip")
|
req.Header.Set(api.ContentEncoding, api.AwsChunked+",gzip")
|
||||||
|
|
||||||
hc.Handler().PutObjectHandler(w, req)
|
hc.Handler().PutObjectHandler(w, req)
|
||||||
|
@ -622,13 +504,13 @@ func TestPutChunkedTestContentEncoding(t *testing.T) {
|
||||||
resp := headObjectBase(hc, bktName, objName, emptyVersion)
|
resp := headObjectBase(hc, bktName, objName, emptyVersion)
|
||||||
require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding))
|
require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding))
|
||||||
|
|
||||||
w, req, _ = getChunkedRequestAWSExample(t, bktName, objName)
|
w, req, _ = getChunkedRequest(hc.context, t, bktName, objName)
|
||||||
req.Header.Set(api.ContentEncoding, "gzip")
|
req.Header.Set(api.ContentEncoding, "gzip")
|
||||||
hc.Handler().PutObjectHandler(w, req)
|
hc.Handler().PutObjectHandler(w, req)
|
||||||
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidEncodingMethod))
|
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidEncodingMethod))
|
||||||
|
|
||||||
hc.config.bypassContentEncodingInChunks = true
|
hc.config.bypassContentEncodingInChunks = true
|
||||||
w, req, _ = getChunkedRequestAWSExample(t, bktName, objName)
|
w, req, _ = getChunkedRequest(hc.context, t, bktName, objName)
|
||||||
req.Header.Set(api.ContentEncoding, "gzip")
|
req.Header.Set(api.ContentEncoding, "gzip")
|
||||||
hc.Handler().PutObjectHandler(w, req)
|
hc.Handler().PutObjectHandler(w, req)
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
@ -637,9 +519,9 @@ func TestPutChunkedTestContentEncoding(t *testing.T) {
|
||||||
require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding))
|
require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding))
|
||||||
}
|
}
|
||||||
|
|
||||||
// getChunkedRequestAWSExample implements request example from
|
// getChunkedRequest implements request example from
|
||||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
|
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
|
||||||
func getChunkedRequestAWSExample(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
|
func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
|
||||||
chunk := make([]byte, 65*1024)
|
chunk := make([]byte, 65*1024)
|
||||||
for i := range chunk {
|
for i := range chunk {
|
||||||
chunk[i] = 'a'
|
chunk[i] = 'a'
|
||||||
|
@ -647,8 +529,12 @@ func getChunkedRequestAWSExample(t *testing.T, bktName, objName string) (*httpte
|
||||||
chunk1 := chunk[:64*1024]
|
chunk1 := chunk[:64*1024]
|
||||||
chunk2 := chunk[64*1024:]
|
chunk2 := chunk[64*1024:]
|
||||||
|
|
||||||
|
AWSAccessKeyID := "AKIAIOSFODNN7EXAMPLE"
|
||||||
AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||||
|
|
||||||
|
awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey}
|
||||||
|
signer := v4.NewSigner()
|
||||||
|
|
||||||
reqBody := bytes.NewBufferString("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n")
|
reqBody := bytes.NewBufferString("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n")
|
||||||
_, err := reqBody.Write(chunk1)
|
_, err := reqBody.Write(chunk1)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -666,14 +552,32 @@ func getChunkedRequestAWSExample(t *testing.T, bktName, objName string) (*httpte
|
||||||
req.Header.Set("x-amz-content-sha256", api.StreamingContentSHA256)
|
req.Header.Set("x-amz-content-sha256", api.StreamingContentSHA256)
|
||||||
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
|
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
|
||||||
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
||||||
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9")
|
|
||||||
|
|
||||||
signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z")
|
signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = signer.SignHTTP(ctx, awsCreds, req, auth.UnsignedPayload, "s3", "us-east-1", signTime)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
req.Body = io.NopCloser(reqBody)
|
req.Body = io.NopCloser(reqBody)
|
||||||
|
|
||||||
w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
w := httptest.NewRecorder()
|
||||||
|
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
||||||
|
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
||||||
|
req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{
|
||||||
|
ClientTime: signTime,
|
||||||
|
AuthHeaders: &middleware.AuthHeader{
|
||||||
|
AccessKeyID: AWSAccessKeyID,
|
||||||
|
SignatureV4: "4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9",
|
||||||
|
Region: "us-east-1",
|
||||||
|
},
|
||||||
|
AccessBox: &accessbox.Box{
|
||||||
|
Gate: &accessbox.GateData{
|
||||||
|
SecretKey: AWSSecretAccessKey,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
return w, req, chunk
|
return w, req, chunk
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -685,9 +589,9 @@ func (c *customNopCloser) Close() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getChunkedRequestAWSExampleTrailing implements request example from
|
// getChunkedRequestTrailing implements request example from
|
||||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html
|
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html
|
||||||
func getChunkedRequestAWSExampleTrailing(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
|
func getChunkedRequestTrailing(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
|
||||||
chunk := make([]byte, 65*1024)
|
chunk := make([]byte, 65*1024)
|
||||||
for i := range chunk {
|
for i := range chunk {
|
||||||
chunk[i] = 'a'
|
chunk[i] = 'a'
|
||||||
|
@ -695,8 +599,12 @@ func getChunkedRequestAWSExampleTrailing(t *testing.T, bktName, objName string)
|
||||||
chunk1 := chunk[:64*1024]
|
chunk1 := chunk[:64*1024]
|
||||||
chunk2 := chunk[64*1024:]
|
chunk2 := chunk[64*1024:]
|
||||||
|
|
||||||
|
AWSAccessKeyID := "AKIAIOSFODNN7EXAMPLE"
|
||||||
AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||||
|
|
||||||
|
awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey}
|
||||||
|
signer := v4.NewSigner()
|
||||||
|
|
||||||
reqBody := bytes.NewBufferString("10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n")
|
reqBody := bytes.NewBufferString("10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n")
|
||||||
_, err := reqBody.Write(chunk1)
|
_, err := reqBody.Write(chunk1)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -726,14 +634,32 @@ func getChunkedRequestAWSExampleTrailing(t *testing.T, bktName, objName string)
|
||||||
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
|
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
|
||||||
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
||||||
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32c")
|
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32c")
|
||||||
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e")
|
|
||||||
|
|
||||||
signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z")
|
signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = signer.SignHTTP(ctx, awsCreds, req, api.StreamingContentSHA256Trailer, "s3", "us-east-1", signTime)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
req.Body = &customNopCloser{Buffer: reqBody}
|
req.Body = &customNopCloser{Buffer: reqBody}
|
||||||
|
|
||||||
w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
w := httptest.NewRecorder()
|
||||||
|
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
||||||
|
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
||||||
|
req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{
|
||||||
|
ClientTime: signTime,
|
||||||
|
AuthHeaders: &middleware.AuthHeader{
|
||||||
|
AccessKeyID: AWSAccessKeyID,
|
||||||
|
SignatureV4: "106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e",
|
||||||
|
Region: "us-east-1",
|
||||||
|
},
|
||||||
|
AccessBox: &accessbox.Box{
|
||||||
|
Gate: &accessbox.GateData{
|
||||||
|
SecretKey: AWSSecretAccessKey,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
return w, req, chunk
|
return w, req, chunk
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -742,6 +668,8 @@ func getChunkedRequestUnsignedTrailing(ctx context.Context, t *testing.T, bktNam
|
||||||
for i := range chunk {
|
for i := range chunk {
|
||||||
chunk[i] = 'a'
|
chunk[i] = 'a'
|
||||||
}
|
}
|
||||||
|
//chunk1 := chunk[:64*1024]
|
||||||
|
//chunk2 := chunk[64*1024:]
|
||||||
|
|
||||||
AWSAccessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6"
|
AWSAccessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6"
|
||||||
AWSSecretAccessKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e"
|
AWSSecretAccessKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e"
|
||||||
|
@ -758,6 +686,7 @@ func getChunkedRequestUnsignedTrailing(ctx context.Context, t *testing.T, bktNam
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
req, err := http.NewRequest("PUT", "https://localhost:8184/"+bktName+"/"+objName, nil)
|
req, err := http.NewRequest("PUT", "https://localhost:8184/"+bktName+"/"+objName, nil)
|
||||||
|
//req, err := http.NewRequest("PUT", "https://localhost:8184/test2/body", nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
req.Header.Set("x-amz-sdk-checksum-algorithm", "CRC64NVME")
|
req.Header.Set("x-amz-sdk-checksum-algorithm", "CRC64NVME")
|
||||||
req.Header.Set("content-encoding", api.AwsChunked)
|
req.Header.Set("content-encoding", api.AwsChunked)
|
||||||
|
@ -774,7 +703,23 @@ func getChunkedRequestUnsignedTrailing(ctx context.Context, t *testing.T, bktNam
|
||||||
|
|
||||||
req.Body = io.NopCloser(reqBody)
|
req.Body = io.NopCloser(reqBody)
|
||||||
|
|
||||||
w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
w := httptest.NewRecorder()
|
||||||
|
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
||||||
|
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
||||||
|
req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{
|
||||||
|
ClientTime: signTime,
|
||||||
|
AuthHeaders: &middleware.AuthHeader{
|
||||||
|
AccessKeyID: AWSAccessKeyID,
|
||||||
|
SignatureV4: "a075c83779d1c3c02254fbe4c9eff0a21556d15556fc6a25db69147c4838226b",
|
||||||
|
Region: "ru",
|
||||||
|
},
|
||||||
|
AccessBox: &accessbox.Box{
|
||||||
|
Gate: &accessbox.GateData{
|
||||||
|
SecretKey: AWSSecretAccessKey,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
return w, req, chunk
|
return w, req, chunk
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -812,183 +757,65 @@ func getChunkedRequestUnsignedTrailingSmall(ctx context.Context, t *testing.T, b
|
||||||
|
|
||||||
req.Body = io.NopCloser(reqBody)
|
req.Body = io.NopCloser(reqBody)
|
||||||
|
|
||||||
w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
w := httptest.NewRecorder()
|
||||||
|
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
||||||
|
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
||||||
|
req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{
|
||||||
|
ClientTime: signTime,
|
||||||
|
AuthHeaders: &middleware.AuthHeader{
|
||||||
|
AccessKeyID: AWSAccessKeyID,
|
||||||
|
SignatureV4: "a075c83779d1c3c02254fbe4c9eff0a21556d15556fc6a25db69147c4838226b",
|
||||||
|
Region: "ru",
|
||||||
|
},
|
||||||
|
AccessBox: &accessbox.Box{
|
||||||
|
Gate: &accessbox.GateData{
|
||||||
|
SecretKey: AWSSecretAccessKey,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
return w, req, []byte(chunk)
|
return w, req, []byte(chunk)
|
||||||
}
|
}
|
||||||
|
|
||||||
func getChunkedRequestBase(t *testing.T, bktName, objName string, chunks [][]byte, shaType string, signTime time.Time, extraHeaders ...[2]string) (*httptest.ResponseRecorder, *http.Request) {
|
func getEmptyChunkedRequest(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) {
|
||||||
creds := aws.Credentials{
|
AWSAccessKeyID := "48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh"
|
||||||
AccessKeyID: "48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh",
|
AWSSecretAccessKey := "09260955b4eb0279dc017ba20a1ddac909cbd226c86cbb2d868e55534c8e64b0"
|
||||||
SecretAccessKey: "09260955b4eb0279dc017ba20a1ddac909cbd226c86cbb2d868e55534c8e64b0",
|
|
||||||
}
|
|
||||||
region := "us-east-1"
|
|
||||||
service := "s3"
|
|
||||||
|
|
||||||
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, nil)
|
reqBody := bytes.NewBufferString("0;chunk-signature=311a7142c8f3a07972c3aca65c36484b513a8fee48ab7178c7225388f2ae9894\r\n\r\n")
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
payloadLength := 0
|
|
||||||
for _, chunk := range chunks {
|
|
||||||
payloadLength += len(chunk)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, kv := range extraHeaders {
|
|
||||||
req.Header.Set(kv[0], kv[1])
|
|
||||||
}
|
|
||||||
|
|
||||||
req.Header.Set(api.ContentEncoding, api.AwsChunked)
|
|
||||||
req.Header.Set(api.AmzDecodedContentLength, strconv.Itoa(payloadLength))
|
|
||||||
req.Header.Set(api.AmzDate, signTime.Format("20060102T150405Z"))
|
|
||||||
req.Header.Set(api.AmzContentSha256, shaType)
|
|
||||||
if shaType == api.StreamingContentSHA256Trailer {
|
|
||||||
req.Header.Set(api.AmzTrailer, "x-amz-checksum-crc32")
|
|
||||||
}
|
|
||||||
|
|
||||||
signer := v4.NewSigner()
|
|
||||||
err = signer.SignHTTP(req.Context(), creds, req, shaType, service, region, signTime)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
seedSignature := strings.Split(req.Header.Get(api.Authorization), "Signature=")[1]
|
|
||||||
seed, err := hex.DecodeString(seedSignature)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var reqBody bytes.Buffer
|
|
||||||
|
|
||||||
hash := crc32.NewIEEE()
|
|
||||||
|
|
||||||
newStreamSigner := v4.NewStreamSigner(creds, service, region, seed)
|
|
||||||
for _, chunk := range chunks {
|
|
||||||
_, err = hash.Write(chunk)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
signature, err := newStreamSigner.GetSignature(req.Context(), nil, chunk, signTime)
|
|
||||||
require.NoError(t, err)
|
|
||||||
reqBody.WriteString(fmt.Sprintf("%x;chunk-signature=%x\r\n", len(chunk), signature))
|
|
||||||
reqBody.Write(chunk)
|
|
||||||
reqBody.WriteString("\r\n")
|
|
||||||
}
|
|
||||||
signature, err := newStreamSigner.GetSignature(req.Context(), nil, nil, signTime)
|
|
||||||
require.NoError(t, err)
|
|
||||||
reqBody.WriteString(fmt.Sprintf("0;chunk-signature=%x\r\n", signature))
|
|
||||||
|
|
||||||
if shaType == api.StreamingContentSHA256Trailer {
|
|
||||||
crc32Res := hash.Sum(nil)
|
|
||||||
checksumStr := "x-amz-checksum-crc32:" + base64.StdEncoding.EncodeToString(crc32Res)
|
|
||||||
reqBody.WriteString(fmt.Sprintf("%s\r\n", checksumStr))
|
|
||||||
trailerSignature, err := newStreamSigner.GetTrailerSignature([]byte(checksumStr+"\n"), signTime)
|
|
||||||
require.NoError(t, err)
|
|
||||||
reqBody.WriteString(fmt.Sprintf("x-amz-trailer-signature:%x\r\n", trailerSignature))
|
|
||||||
}
|
|
||||||
reqBody.WriteString("\r\n")
|
|
||||||
|
|
||||||
req.Body = io.NopCloser(&reqBody)
|
|
||||||
|
|
||||||
return prepareReqMiddlewares(req, signTime, creds.SecretAccessKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
func prepareReqMiddlewares(req *http.Request, signTime time.Time, secretAccessKey string) (*httptest.ResponseRecorder, *http.Request) {
|
|
||||||
authHeader := req.Header.Get(api.Authorization)
|
|
||||||
var parsed map[string]string
|
|
||||||
var region string
|
|
||||||
if strings.HasPrefix(authHeader, auth.SignaturePreambleSigV4) {
|
|
||||||
parsed = auth.NewRegexpMatcher(auth.AuthorizationFieldRegexp).GetSubmatches(authHeader)
|
|
||||||
region = parsed["region"]
|
|
||||||
} else {
|
|
||||||
parsed = auth.NewRegexpMatcher(auth.AuthorizationFieldV4aRegexp).GetSubmatches(authHeader)
|
|
||||||
region = req.Header.Get("X-Amz-Region-Set")
|
|
||||||
}
|
|
||||||
|
|
||||||
bktObj := strings.Split(req.URL.Path, "/")
|
|
||||||
|
|
||||||
box := &middleware.Box{
|
|
||||||
ClientTime: signTime,
|
|
||||||
AuthHeaders: &middleware.AuthHeader{
|
|
||||||
AccessKeyID: parsed["access_key_id"],
|
|
||||||
SignatureV4: parsed["v4_signature"],
|
|
||||||
Region: region,
|
|
||||||
},
|
|
||||||
AccessBox: &accessbox.Box{Gate: &accessbox.GateData{SecretKey: secretAccessKey}},
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktObj[1], Object: bktObj[2]}, "")
|
|
||||||
req = req.WithContext(middleware.SetReqInfo(req.Context(), reqInfo))
|
|
||||||
req = req.WithContext(middleware.SetBox(req.Context(), box))
|
|
||||||
|
|
||||||
return w, req
|
|
||||||
}
|
|
||||||
|
|
||||||
func getEmptyChunkedRequestUnsigned(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) {
|
|
||||||
AWSSecretAccessKey := "f1a0d650b650149f1a83140418e88a3c5572a0103e912e326492a91c19c4488a"
|
|
||||||
|
|
||||||
reqBody := bytes.NewBufferString("0\r\nx-amz-checksum-crc64nvme:AAAAAAAAAAA=\r\n\r\n")
|
|
||||||
|
|
||||||
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, reqBody)
|
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, reqBody)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
req.Header.Set(api.Authorization, "AWS4-HMAC-SHA256 Credential=3jNrmDtHtuj1uLcixaSMA4KNUhNYhv1EpUNdFnbTXgUP071pGdSZfHSLtoC8gzjF5HoD6sC3Scq33t1WvvEvjmPnt/20250213/ru/s3/aws4_request, SignedHeaders=content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-sdk-checksum-algorithm;x-amz-trailer, Signature=1231b012c0ac313770c5a95ccf77b95b6c9b1c3760d6aa24cb8309801d56eb4a")
|
req.Header.Set("Amz-Sdk-Invocation-Id", "8a8cd4be-aef8-8034-f08d-a6144ade41f9")
|
||||||
req.Header.Set(api.ContentEncoding, api.AwsChunked)
|
|
||||||
req.Header.Set(api.AmzDate, "20250213T124858Z")
|
|
||||||
req.Header.Set(api.AmzContentSha256, api.StreamingUnsignedPayloadTrailer)
|
|
||||||
req.Header.Set(api.AmzDecodedContentLength, "0")
|
|
||||||
req.Header.Set("X-Amz-Trailer", "x-amz-checksum-crc64nvme")
|
|
||||||
req.Header.Set("X-Amz-Sdk-Checksum-Algorithm", "CRC64NVME")
|
|
||||||
|
|
||||||
signTime, err := time.Parse("20060102T150405Z", req.Header.Get(api.AmzDate))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
func getEmptyChunkedRequestSigv4aWithTrailers(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) {
|
|
||||||
AWSSecretAccessKey := "f1a0d650b650149f1a83140418e88a3c5572a0103e912e326492a91c19c4488a"
|
|
||||||
|
|
||||||
body := "0;chunk-signature=3046022100ab9229a80d70f4d004768992881821a441a4ad4102e18de567e68216659bf497022100ec47a7a445351683557eedf893e6ed250c97af4b0415814671770b83766d69be\r\n" +
|
|
||||||
"x-amz-checksum-crc32:AAAAAA==\r\n" +
|
|
||||||
"x-amz-trailer-signature:3046022100a0a66c1adcee8d99460b4631b23c95fbad9eb4e6c56f1afb9e255715ba141169022100b2cfc8adc8036eb985f1ab0e770b575284c5fc8ca75c226558d3142cbaab83ce\r\n\r\n"
|
|
||||||
|
|
||||||
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, bytes.NewBufferString(body))
|
|
||||||
require.NoError(t, err)
|
|
||||||
req.Header.Set(api.Authorization, "AWS4-ECDSA-P256-SHA256 Credential=3jNrmDtHtuj1uLcixaSMA4KNUhNYhv1EpUNdFnbTXgUP071pGdSZfHSLtoC8gzjF5HoD6sC3Scq33t1WvvEvjmPnt/20250213/s3/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-region-set;x-amz-sdk-checksum-algorithm;x-amz-trailer, Signature=304402202e1f1efcc56c588d9a94a3d8f20368686df8bfd5e8aad01fc4eff569ff38f1800220215198e3f1ba785492fe6703c4722872909ce8a09e8c9a13da90a9230c7a24b7")
|
|
||||||
req.Header.Set("Amz-Sdk-Invocation-Id", "d42dc16d-7899-55fb-5b72-a654bd482f4f")
|
|
||||||
req.Header.Set("Amz-Sdk-Request", "attempt=1; max=2")
|
req.Header.Set("Amz-Sdk-Request", "attempt=1; max=2")
|
||||||
req.Header.Set(api.ContentEncoding, api.AwsChunked)
|
req.Header.Set(api.Authorization, "AWS4-HMAC-SHA256 Credential=48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh/20241003/us-east-1/s3/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-encoding;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length, Signature=4b530ab4af2381f214941af591266b209968264a2c94337fa1efc048c7dff352")
|
||||||
req.Header.Set(api.AmzDate, "20250213T132401Z")
|
req.Header.Set(api.ContentEncoding, "aws-chunked")
|
||||||
req.Header.Set(api.AmzContentSha256, api.StreamingContentV4aSHA256Trailer)
|
req.Header.Set(api.ContentLength, "86")
|
||||||
|
req.Header.Set(api.ContentType, "text/plain; charset=UTF-8")
|
||||||
|
req.Header.Set(api.AmzDate, "20241003T100055Z")
|
||||||
|
req.Header.Set(api.AmzContentSha256, "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
|
||||||
req.Header.Set(api.AmzDecodedContentLength, "0")
|
req.Header.Set(api.AmzDecodedContentLength, "0")
|
||||||
req.Header.Set(api.ContentLength, "367")
|
|
||||||
req.Header.Set(api.ContentType, "text/plain: charset=UTF-8")
|
|
||||||
req.Header.Set("X-Amz-Region-Set", "us-east-1")
|
|
||||||
req.Header.Set("X-Amz-Trailer", "x-amz-checksum-crc32")
|
|
||||||
req.Header.Set("X-Amz-Sdk-Checksum-Algorithm", "CRC32")
|
|
||||||
|
|
||||||
signTime, err := time.Parse("20060102T150405Z", req.Header.Get(api.AmzDate))
|
signTime, err := time.Parse("20060102T150405Z", "20241003T100055Z")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
return prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
w := httptest.NewRecorder()
|
||||||
}
|
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
||||||
|
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
||||||
|
req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{
|
||||||
|
ClientTime: signTime,
|
||||||
|
AuthHeaders: &middleware.AuthHeader{
|
||||||
|
AccessKeyID: AWSAccessKeyID,
|
||||||
|
SignatureV4: "4b530ab4af2381f214941af591266b209968264a2c94337fa1efc048c7dff352",
|
||||||
|
Region: "us-east-1",
|
||||||
|
},
|
||||||
|
AccessBox: &accessbox.Box{
|
||||||
|
Gate: &accessbox.GateData{
|
||||||
|
SecretKey: AWSSecretAccessKey,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
func getEmptyChunkedRequestSigv4a(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) {
|
return w, req
|
||||||
AWSSecretAccessKey := "f1a0d650b650149f1a83140418e88a3c5572a0103e912e326492a91c19c4488a"
|
|
||||||
|
|
||||||
body := "0;chunk-signature=304502203f7c598a2e9a6673bf1ca30f5f6bebd0d76a4e9d3c16531448e96c2cda22d16a0221009e7ed578da0a9781366f1461a1484e64f15707f26d4310e59514db6ff9f7e0f1**\r\n\r\n"
|
|
||||||
|
|
||||||
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, bytes.NewBufferString(body))
|
|
||||||
require.NoError(t, err)
|
|
||||||
req.Header.Set(api.Authorization, "AWS4-ECDSA-P256-SHA256 Credential=3jNrmDtHtuj1uLcixaSMA4KNUhNYhv1EpUNdFnbTXgUP071pGdSZfHSLtoC8gzjF5HoD6sC3Scq33t1WvvEvjmPnt/20250213/s3/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-region-set, Signature=3046022100dc589ea513448b996809db4b314a0b8a4a775c1165c6203c7104b2f1aae1243c0221009bf3a256e7c33415eaad20c1dbfb4e14cb00b362758bc4d2aaf94ca96a5f13f9")
|
|
||||||
req.Header.Set("Amz-Sdk-Invocation-Id", "f0814a40-0d74-066f-d01f-ed14f28ebfa4")
|
|
||||||
req.Header.Set("Amz-Sdk-Request", "attempt=1; max=2")
|
|
||||||
req.Header.Set(api.ContentEncoding, api.AwsChunked)
|
|
||||||
req.Header.Set(api.AmzDate, "20250213T135717Z")
|
|
||||||
req.Header.Set(api.AmzContentSha256, api.StreamingContentV4aSHA256)
|
|
||||||
req.Header.Set(api.AmzDecodedContentLength, "0")
|
|
||||||
req.Header.Set(api.ContentLength, "166")
|
|
||||||
req.Header.Set(api.ContentType, "text/plain: charset=UTF-8")
|
|
||||||
req.Header.Set("X-Amz-Region-Set", "use-east-1")
|
|
||||||
|
|
||||||
signTime, err := time.Parse("20060102T150405Z", req.Header.Get(api.AmzDate))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCreateBucket(t *testing.T) {
|
func TestCreateBucket(t *testing.T) {
|
||||||
|
@ -1237,104 +1064,6 @@ func TestFormEncryptionParamsBase(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCheckContentLength(t *testing.T) {
|
|
||||||
contentLength := "content-length-range"
|
|
||||||
notFallError := "length of the content did not fall within the range specified in the condition"
|
|
||||||
parseError := "invalid condition"
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
matching string
|
|
||||||
key string
|
|
||||||
value string
|
|
||||||
size uint64
|
|
||||||
errMsg string
|
|
||||||
emptyPolicy bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "valid",
|
|
||||||
matching: contentLength,
|
|
||||||
key: "0",
|
|
||||||
value: "1000",
|
|
||||||
size: 50,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid lower limit",
|
|
||||||
matching: contentLength,
|
|
||||||
key: "5",
|
|
||||||
value: "100",
|
|
||||||
size: 5,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid upper limit",
|
|
||||||
matching: contentLength,
|
|
||||||
key: "5",
|
|
||||||
value: "100",
|
|
||||||
size: 100,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid size value (too small)",
|
|
||||||
matching: contentLength,
|
|
||||||
key: "5",
|
|
||||||
value: "100",
|
|
||||||
size: 2,
|
|
||||||
errMsg: notFallError,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid size value (to high)",
|
|
||||||
matching: contentLength,
|
|
||||||
key: "5",
|
|
||||||
value: "100",
|
|
||||||
size: 200,
|
|
||||||
errMsg: notFallError,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no matching",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid key type",
|
|
||||||
matching: contentLength,
|
|
||||||
key: "invalid",
|
|
||||||
value: "100",
|
|
||||||
size: 10,
|
|
||||||
errMsg: parseError,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid value type",
|
|
||||||
matching: contentLength,
|
|
||||||
key: "5",
|
|
||||||
value: "invalid",
|
|
||||||
size: 10,
|
|
||||||
errMsg: parseError,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty policy",
|
|
||||||
emptyPolicy: true,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
policy := &postPolicy{
|
|
||||||
Conditions: []*policyCondition{
|
|
||||||
{
|
|
||||||
Matching: tc.matching,
|
|
||||||
Key: tc.key,
|
|
||||||
Value: tc.value,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
empty: tc.emptyPolicy,
|
|
||||||
}
|
|
||||||
|
|
||||||
err := policy.CheckContentLength(tc.size)
|
|
||||||
if tc.errMsg != "" {
|
|
||||||
require.Error(t, err)
|
|
||||||
require.Contains(t, err.Error(), tc.errMsg)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func prepareRequestForEncryption(hc *handlerContext, algo, key, md5, tlsTermination string, reqWithoutTLS, reqWithoutSSE, isCopySource bool) *http.Request {
|
func prepareRequestForEncryption(hc *handlerContext, algo, key, md5, tlsTermination string, reqWithoutTLS, reqWithoutSSE, isCopySource bool) *http.Request {
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||||
|
|
||||||
|
@ -1363,8 +1092,8 @@ func prepareRequestForEncryption(hc *handlerContext, algo, key, md5, tlsTerminat
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
func postObjectBase(hc *handlerContext, ns, bktName, key, filename, content, tagging string) *httptest.ResponseRecorder {
|
func postObjectBase(hc *handlerContext, ns, bktName, key, filename, content string) *httptest.ResponseRecorder {
|
||||||
policy := "eyJleHBpcmF0aW9uIjogIjIwMjUtMTItMDFUMTI6MDA6MDAuMDAwWiIsImNvbmRpdGlvbnMiOiBbCiBbInN0YXJ0cy13aXRoIiwgIiR4LWFtei1jcmVkZW50aWFsIiwgIiJdLAogWyJzdGFydHMtd2l0aCIsICIkeC1hbXotZGF0ZSIsICIiXSwKIFsic3RhcnRzLXdpdGgiLCAiJGtleSIsICIiXSwKIFsic3RhcnRzLXdpdGgiLCAiJHRhZ2dpbmciLCAiIl0KXX0K"
|
policy := "eyJleHBpcmF0aW9uIjogIjIwMjUtMTItMDFUMTI6MDA6MDAuMDAwWiIsImNvbmRpdGlvbnMiOiBbCiBbInN0YXJ0cy13aXRoIiwgIiR4LWFtei1jcmVkZW50aWFsIiwgIiJdLAogWyJzdGFydHMtd2l0aCIsICIkeC1hbXotZGF0ZSIsICIiXSwKIFsic3RhcnRzLXdpdGgiLCAiJGtleSIsICIiXQpdfQ=="
|
||||||
|
|
||||||
timeToSign := time.Now()
|
timeToSign := time.Now()
|
||||||
timeToSignStr := timeToSign.Format("20060102T150405Z")
|
timeToSignStr := timeToSign.Format("20060102T150405Z")
|
||||||
|
@ -1377,7 +1106,7 @@ func postObjectBase(hc *handlerContext, ns, bktName, key, filename, content, tag
|
||||||
creds := getCredsStr(accessKeyID, timeToSignStr, region, service)
|
creds := getCredsStr(accessKeyID, timeToSignStr, region, service)
|
||||||
sign := auth.SignStr(secretKey, service, region, timeToSign, policy)
|
sign := auth.SignStr(secretKey, service, region, timeToSign, policy)
|
||||||
|
|
||||||
body, contentType, err := getMultipartFormBody(policy, creds, timeToSignStr, sign, key, filename, content, tagging)
|
body, contentType, err := getMultipartFormBody(policy, creds, timeToSignStr, sign, key, filename, content)
|
||||||
require.NoError(hc.t, err)
|
require.NoError(hc.t, err)
|
||||||
|
|
||||||
w, r := prepareTestPostRequest(hc, bktName, body)
|
w, r := prepareTestPostRequest(hc, bktName, body)
|
||||||
|
@ -1395,7 +1124,7 @@ func getCredsStr(accessKeyID, timeToSign, region, service string) string {
|
||||||
return accessKeyID + "/" + timeToSign + "/" + region + "/" + service + "/aws4_request"
|
return accessKeyID + "/" + timeToSign + "/" + region + "/" + service + "/aws4_request"
|
||||||
}
|
}
|
||||||
|
|
||||||
func getMultipartFormBody(policy, creds, date, sign, key, filename, content, tagging string) (io.Reader, string, error) {
|
func getMultipartFormBody(policy, creds, date, sign, key, filename, content string) (io.Reader, string, error) {
|
||||||
body := &bytes.Buffer{}
|
body := &bytes.Buffer{}
|
||||||
writer := multipart.NewWriter(body)
|
writer := multipart.NewWriter(body)
|
||||||
defer writer.Close()
|
defer writer.Close()
|
||||||
|
@ -1416,9 +1145,6 @@ func getMultipartFormBody(policy, creds, date, sign, key, filename, content, tag
|
||||||
if err := writer.WriteField(strings.ToLower(auth.AmzSignature), sign); err != nil {
|
if err := writer.WriteField(strings.ToLower(auth.AmzSignature), sign); err != nil {
|
||||||
return nil, "", err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
if err := writer.WriteField("tagging", tagging); err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := writer.CreateFormFile("file", filename)
|
file, err := writer.CreateFormFile("file", filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -59,10 +59,6 @@ func (c *s3ChunkReader) Read(buf []byte) (num int, err error) {
|
||||||
buf = buf[num:]
|
buf = buf[num:]
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.err != nil {
|
|
||||||
return 0, c.err
|
|
||||||
}
|
|
||||||
|
|
||||||
var size int
|
var size int
|
||||||
for {
|
for {
|
||||||
b, err := c.reader.ReadByte()
|
b, err := c.reader.ReadByte()
|
||||||
|
|
|
@ -31,10 +31,6 @@ func (c *s3UnsignedChunkReader) Read(buf []byte) (num int, err error) {
|
||||||
buf = buf[num:]
|
buf = buf[num:]
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.err != nil {
|
|
||||||
return 0, c.err
|
|
||||||
}
|
|
||||||
|
|
||||||
var size int
|
var size int
|
||||||
var b byte
|
var b byte
|
||||||
for {
|
for {
|
||||||
|
|
|
@ -46,10 +46,6 @@ func (c *s3v4aChunkReader) Read(buf []byte) (num int, err error) {
|
||||||
buf = buf[num:]
|
buf = buf[num:]
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.err != nil {
|
|
||||||
return 0, c.err
|
|
||||||
}
|
|
||||||
|
|
||||||
var size int
|
var size int
|
||||||
for {
|
for {
|
||||||
b, err := c.reader.ReadByte()
|
b, err := c.reader.ReadByte()
|
||||||
|
|
|
@ -6,7 +6,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"unicode"
|
"unicode"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
@ -22,9 +21,7 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutObjectTagging")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
tagSet, err := h.readTagSet(reqInfo.Tagging)
|
tagSet, err := h.readTagSet(reqInfo.Tagging)
|
||||||
|
@ -57,9 +54,7 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetObjectTagging")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -97,9 +92,7 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.DeleteObjectTagging")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -123,9 +116,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutBucketTagging")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
tagSet, err := h.readTagSet(reqInfo.Tagging)
|
tagSet, err := h.readTagSet(reqInfo.Tagging)
|
||||||
|
@ -147,9 +138,7 @@ func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetBucketTagging")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
@ -171,9 +160,7 @@ func (h *handler) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.DeleteBucketTagging")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
|
|
@ -39,40 +39,9 @@ func (h *handler) logAndSendError(ctx context.Context, w http.ResponseWriter, lo
|
||||||
zap.String("object", reqInfo.ObjectName),
|
zap.String("object", reqInfo.ObjectName),
|
||||||
zap.String("description", logText),
|
zap.String("description", logText),
|
||||||
zap.String("user", reqInfo.User),
|
zap.String("user", reqInfo.User),
|
||||||
zap.Error(err),
|
zap.Error(err)}
|
||||||
}
|
|
||||||
fields = append(fields, additional...)
|
fields = append(fields, additional...)
|
||||||
h.reqLogger(ctx).Error(logs.RequestFailed, append(fields, logs.TagField(logs.TagDatapath))...)
|
h.reqLogger(ctx).Error(logs.RequestFailed, fields...)
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) logAndSendErrorNoHeader(ctx context.Context, w http.ResponseWriter, logText string, reqInfo *middleware.ReqInfo, err error, additional ...zap.Field) {
|
|
||||||
err = handleDeleteMarker(w, err)
|
|
||||||
if wrErr := middleware.WriteErrorResponseNoHeader(w, reqInfo, apierr.TransformToS3Error(err)); wrErr != nil {
|
|
||||||
additional = append(additional, zap.NamedError("write_response_error", wrErr))
|
|
||||||
}
|
|
||||||
fields := []zap.Field{
|
|
||||||
zap.String("method", reqInfo.API),
|
|
||||||
zap.String("bucket", reqInfo.BucketName),
|
|
||||||
zap.String("object", reqInfo.ObjectName),
|
|
||||||
zap.String("description", logText),
|
|
||||||
zap.String("user", reqInfo.User),
|
|
||||||
zap.Error(err),
|
|
||||||
}
|
|
||||||
fields = append(fields, additional...)
|
|
||||||
h.reqLogger(ctx).Error(logs.RequestFailed, append(fields, logs.TagField(logs.TagDatapath))...)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) logError(ctx context.Context, logText string, reqInfo *middleware.ReqInfo, err error, additional ...zap.Field) {
|
|
||||||
fields := []zap.Field{
|
|
||||||
zap.String("method", reqInfo.API),
|
|
||||||
zap.String("bucket", reqInfo.BucketName),
|
|
||||||
zap.String("object", reqInfo.ObjectName),
|
|
||||||
zap.String("description", logText),
|
|
||||||
zap.String("user", reqInfo.User),
|
|
||||||
zap.Error(err),
|
|
||||||
}
|
|
||||||
fields = append(fields, additional...)
|
|
||||||
h.reqLogger(ctx).Error(logs.RequestFailed, append(fields, logs.TagField(logs.TagDatapath))...)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func handleDeleteMarker(w http.ResponseWriter, err error) error {
|
func handleDeleteMarker(w http.ResponseWriter, err error) error {
|
||||||
|
|
|
@ -3,7 +3,6 @@ package handler
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
|
@ -11,9 +10,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutBucketVersioning")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
configuration := new(VersioningConfiguration)
|
configuration := new(VersioningConfiguration)
|
||||||
|
@ -60,9 +57,7 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ
|
||||||
|
|
||||||
// GetBucketVersioningHandler implements bucket versioning getter handler.
|
// GetBucketVersioningHandler implements bucket versioning getter handler.
|
||||||
func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetBucketVersioning")
|
ctx := r.Context()
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
|
|
@ -63,7 +63,6 @@ const (
|
||||||
AmzPartNumberMarker = "X-Amz-Part-Number-Marker"
|
AmzPartNumberMarker = "X-Amz-Part-Number-Marker"
|
||||||
AmzStorageClass = "X-Amz-Storage-Class"
|
AmzStorageClass = "X-Amz-Storage-Class"
|
||||||
AmzForceBucketDelete = "X-Amz-Force-Delete-Bucket"
|
AmzForceBucketDelete = "X-Amz-Force-Delete-Bucket"
|
||||||
AmzTrailer = "X-Amz-Trailer"
|
|
||||||
|
|
||||||
AmzServerSideEncryptionCustomerAlgorithm = "x-amz-server-side-encryption-customer-algorithm"
|
AmzServerSideEncryptionCustomerAlgorithm = "x-amz-server-side-encryption-customer-algorithm"
|
||||||
AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"
|
AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"
|
||||||
|
|
|
@ -76,8 +76,7 @@ func (c *Cache) PutBucket(bktInfo *data.BucketInfo) {
|
||||||
zap.String("zone", bktInfo.Zone),
|
zap.String("zone", bktInfo.Zone),
|
||||||
zap.String("bucket name", bktInfo.Name),
|
zap.String("bucket name", bktInfo.Name),
|
||||||
zap.Stringer("bucket cid", bktInfo.CID),
|
zap.Stringer("bucket cid", bktInfo.CID),
|
||||||
zap.Error(err),
|
zap.Error(err))
|
||||||
logs.TagField(logs.TagDatapath))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -119,12 +118,11 @@ func (c *Cache) PutObject(owner user.ID, extObjInfo *data.ExtendedObjectInfo) {
|
||||||
if err := c.objCache.PutObject(extObjInfo); err != nil {
|
if err := c.objCache.PutObject(extObjInfo); err != nil {
|
||||||
c.logger.Warn(logs.CouldntAddObjectToCache, zap.Error(err),
|
c.logger.Warn(logs.CouldntAddObjectToCache, zap.Error(err),
|
||||||
zap.String("object_name", extObjInfo.ObjectInfo.Name), zap.String("bucket_name", extObjInfo.ObjectInfo.Bucket),
|
zap.String("object_name", extObjInfo.ObjectInfo.Name), zap.String("bucket_name", extObjInfo.ObjectInfo.Bucket),
|
||||||
zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString()),
|
zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString()))
|
||||||
logs.TagField(logs.TagDatapath))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, extObjInfo.ObjectInfo.Address().EncodeToString()); err != nil {
|
if err := c.accessCache.Put(owner, extObjInfo.ObjectInfo.Address().EncodeToString()); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -134,8 +132,7 @@ func (c *Cache) PutObjectWithName(owner user.ID, extObjInfo *data.ExtendedObject
|
||||||
if err := c.namesCache.Put(extObjInfo.ObjectInfo.NiceName(), extObjInfo.ObjectInfo.Address()); err != nil {
|
if err := c.namesCache.Put(extObjInfo.ObjectInfo.NiceName(), extObjInfo.ObjectInfo.Address()); err != nil {
|
||||||
c.logger.Warn(logs.CouldntPutObjAddressToNameCache,
|
c.logger.Warn(logs.CouldntPutObjAddressToNameCache,
|
||||||
zap.String("obj nice name", extObjInfo.ObjectInfo.NiceName()),
|
zap.String("obj nice name", extObjInfo.ObjectInfo.NiceName()),
|
||||||
zap.Error(err),
|
zap.Error(err))
|
||||||
logs.TagField(logs.TagDatapath))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -149,11 +146,11 @@ func (c *Cache) GetList(owner user.ID, key cache.ObjectsListKey) []*data.NodeVer
|
||||||
|
|
||||||
func (c *Cache) PutList(owner user.ID, key cache.ObjectsListKey, list []*data.NodeVersion) {
|
func (c *Cache) PutList(owner user.ID, key cache.ObjectsListKey, list []*data.NodeVersion) {
|
||||||
if err := c.listsCache.PutVersions(key, list); err != nil {
|
if err := c.listsCache.PutVersions(key, list); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheListOfObjects, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheListOfObjects, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key.String()); err != nil {
|
if err := c.accessCache.Put(owner, key.String()); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -167,11 +164,11 @@ func (c *Cache) GetListSession(owner user.ID, key cache.ListSessionKey) *data.Li
|
||||||
|
|
||||||
func (c *Cache) PutListSession(owner user.ID, key cache.ListSessionKey, session *data.ListSession) {
|
func (c *Cache) PutListSession(owner user.ID, key cache.ListSessionKey, session *data.ListSession) {
|
||||||
if err := c.sessionListCache.PutListSession(key, session); err != nil {
|
if err := c.sessionListCache.PutListSession(key, session); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheListSession, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheListSession, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key.String()); err != nil {
|
if err := c.accessCache.Put(owner, key.String()); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -190,11 +187,11 @@ func (c *Cache) GetTagging(owner user.ID, key string) map[string]string {
|
||||||
|
|
||||||
func (c *Cache) PutTagging(owner user.ID, key string, tags map[string]string) {
|
func (c *Cache) PutTagging(owner user.ID, key string, tags map[string]string) {
|
||||||
if err := c.systemCache.PutTagging(key, tags); err != nil {
|
if err := c.systemCache.PutTagging(key, tags); err != nil {
|
||||||
c.logger.Error(logs.CouldntCacheTags, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Error(logs.CouldntCacheTags, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key); err != nil {
|
if err := c.accessCache.Put(owner, key); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -212,11 +209,11 @@ func (c *Cache) GetLockInfo(owner user.ID, key string) *data.LockInfo {
|
||||||
|
|
||||||
func (c *Cache) PutLockInfo(owner user.ID, key string, lockInfo *data.LockInfo) {
|
func (c *Cache) PutLockInfo(owner user.ID, key string, lockInfo *data.LockInfo) {
|
||||||
if err := c.systemCache.PutLockInfo(key, lockInfo); err != nil {
|
if err := c.systemCache.PutLockInfo(key, lockInfo); err != nil {
|
||||||
c.logger.Error(logs.CouldntCacheLockInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Error(logs.CouldntCacheLockInfo, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key); err != nil {
|
if err := c.accessCache.Put(owner, key); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -233,11 +230,11 @@ func (c *Cache) GetSettings(owner user.ID, bktInfo *data.BucketInfo) *data.Bucke
|
||||||
func (c *Cache) PutSettings(owner user.ID, bktInfo *data.BucketInfo, settings *data.BucketSettings) {
|
func (c *Cache) PutSettings(owner user.ID, bktInfo *data.BucketInfo, settings *data.BucketSettings) {
|
||||||
key := bktInfo.Name + bktInfo.SettingsObjectName()
|
key := bktInfo.Name + bktInfo.SettingsObjectName()
|
||||||
if err := c.systemCache.PutSettings(key, settings); err != nil {
|
if err := c.systemCache.PutSettings(key, settings); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheBucketSettings, zap.String("bucket", bktInfo.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheBucketSettings, zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key); err != nil {
|
if err := c.accessCache.Put(owner, key); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -255,11 +252,11 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf
|
||||||
key := bkt.CORSObjectName()
|
key := bkt.CORSObjectName()
|
||||||
|
|
||||||
if err := c.systemCache.PutCORS(key, cors); err != nil {
|
if err := c.systemCache.PutCORS(key, cors); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key); err != nil {
|
if err := c.accessCache.Put(owner, key); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -281,11 +278,11 @@ func (c *Cache) PutLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo, c
|
||||||
key := bkt.LifecycleConfigurationObjectName()
|
key := bkt.LifecycleConfigurationObjectName()
|
||||||
|
|
||||||
if err := c.systemCache.PutLifecycleConfiguration(key, cfg); err != nil {
|
if err := c.systemCache.PutLifecycleConfiguration(key, cfg); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key); err != nil {
|
if err := c.accessCache.Put(owner, key); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -299,7 +296,7 @@ func (c *Cache) GetNetworkInfo() *netmap.NetworkInfo {
|
||||||
|
|
||||||
func (c *Cache) PutNetworkInfo(info netmap.NetworkInfo) {
|
func (c *Cache) PutNetworkInfo(info netmap.NetworkInfo) {
|
||||||
if err := c.networkCache.PutNetworkInfo(info); err != nil {
|
if err := c.networkCache.PutNetworkInfo(info); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheNetworkInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheNetworkInfo, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -309,7 +306,7 @@ func (c *Cache) GetNetmap() *netmap.NetMap {
|
||||||
|
|
||||||
func (c *Cache) PutNetmap(nm netmap.NetMap) {
|
func (c *Cache) PutNetmap(nm netmap.NetMap) {
|
||||||
if err := c.networkCache.PutNetmap(nm); err != nil {
|
if err := c.networkCache.PutNetmap(nm); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheNetmap, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.logger.Warn(logs.CouldntCacheNetmap, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -5,16 +5,12 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *Layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) {
|
func (n *Layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetObjectTaggingAndLock")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
|
|
||||||
|
|
|
@ -64,7 +64,6 @@ func (n *Layer) containerInfo(ctx context.Context, prm frostfs.PrmContainer) (*d
|
||||||
log.Error(logs.CouldNotParseContainerObjectLockEnabledAttribute,
|
log.Error(logs.CouldNotParseContainerObjectLockEnabledAttribute,
|
||||||
zap.String("lock_enabled", attrLockEnabled),
|
zap.String("lock_enabled", attrLockEnabled),
|
||||||
zap.Error(err),
|
zap.Error(err),
|
||||||
logs.TagField(logs.TagDatapath),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -89,7 +88,7 @@ func (n *Layer) containerList(ctx context.Context, listParams ListBucketsParams)
|
||||||
|
|
||||||
res, err := n.frostFS.UserContainers(ctx, prm)
|
res, err := n.frostFS.UserContainers(ctx, prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Error(logs.CouldNotListUserContainers, zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
n.reqLogger(ctx).Error(logs.CouldNotListUserContainers, zap.Error(err))
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -101,7 +100,7 @@ func (n *Layer) containerList(ctx context.Context, listParams ListBucketsParams)
|
||||||
}
|
}
|
||||||
info, err := n.containerInfo(ctx, getPrm)
|
info, err := n.containerInfo(ctx, getPrm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Error(logs.CouldNotFetchContainerInfo, zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
n.reqLogger(ctx).Error(logs.CouldNotFetchContainerInfo, zap.Error(err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -8,7 +8,6 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
||||||
|
@ -24,9 +23,6 @@ const wildcard = "*"
|
||||||
var supportedMethods = map[string]struct{}{"GET": {}, "HEAD": {}, "POST": {}, "PUT": {}, "DELETE": {}}
|
var supportedMethods = map[string]struct{}{"GET": {}, "HEAD": {}, "POST": {}, "PUT": {}, "DELETE": {}}
|
||||||
|
|
||||||
func (n *Layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
func (n *Layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.PutBucketCORS")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
buf bytes.Buffer
|
buf bytes.Buffer
|
||||||
tee = io.TeeReader(p.Reader, &buf)
|
tee = io.TeeReader(p.Reader, &buf)
|
||||||
|
@ -96,15 +92,11 @@ func (n *Layer) deleteCORSObject(ctx context.Context, bktInfo *data.BucketInfo,
|
||||||
if err := n.objectDeleteWithAuth(ctx, corsBkt, addr.Object(), prmAuth); err != nil {
|
if err := n.objectDeleteWithAuth(ctx, corsBkt, addr.Object(), prmAuth); err != nil {
|
||||||
n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err),
|
n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err),
|
||||||
zap.String("cnrID", corsBkt.CID.EncodeToString()),
|
zap.String("cnrID", corsBkt.CID.EncodeToString()),
|
||||||
zap.String("objID", addr.Object().EncodeToString()),
|
zap.String("objID", addr.Object().EncodeToString()))
|
||||||
logs.TagField(logs.TagExternalStorage))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, decoder func(io.Reader, string) *xml.Decoder) (*data.CORSConfiguration, error) {
|
func (n *Layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, decoder func(io.Reader, string) *xml.Decoder) (*data.CORSConfiguration, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetBucketCORS")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
cors, err := n.getCORS(ctx, bktInfo, decoder)
|
cors, err := n.getCORS(ctx, bktInfo, decoder)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -114,9 +106,6 @@ func (n *Layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, dec
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) error {
|
func (n *Layer) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.DeleteBucketCORS")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
objs, err := n.treeService.DeleteBucketCORS(ctx, bktInfo)
|
objs, err := n.treeService.DeleteBucketCORS(ctx, bktInfo)
|
||||||
objNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
|
objNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
|
||||||
if err != nil && !objNotFound {
|
if err != nil && !objNotFound {
|
||||||
|
|
|
@ -74,16 +74,10 @@ func (k *FeatureSettingsMock) FormContainerZone(ns string) string {
|
||||||
|
|
||||||
var _ frostfs.FrostFS = (*TestFrostFS)(nil)
|
var _ frostfs.FrostFS = (*TestFrostFS)(nil)
|
||||||
|
|
||||||
type offsetError struct {
|
|
||||||
offset int
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
type TestFrostFS struct {
|
type TestFrostFS struct {
|
||||||
objects map[string]*object.Object
|
objects map[string]*object.Object
|
||||||
copiesNumbers map[string][]uint32
|
copiesNumbers map[string][]uint32
|
||||||
objectErrors map[string]error
|
objectErrors map[string]error
|
||||||
objectStreamErrors map[string]offsetError
|
|
||||||
objectPutErrors map[string]error
|
objectPutErrors map[string]error
|
||||||
containers map[string]*container.Container
|
containers map[string]*container.Container
|
||||||
chains map[string][]chain.Chain
|
chains map[string][]chain.Chain
|
||||||
|
@ -97,7 +91,6 @@ func NewTestFrostFS(key *keys.PrivateKey) *TestFrostFS {
|
||||||
objects: make(map[string]*object.Object),
|
objects: make(map[string]*object.Object),
|
||||||
copiesNumbers: make(map[string][]uint32),
|
copiesNumbers: make(map[string][]uint32),
|
||||||
objectErrors: make(map[string]error),
|
objectErrors: make(map[string]error),
|
||||||
objectStreamErrors: make(map[string]offsetError),
|
|
||||||
objectPutErrors: make(map[string]error),
|
objectPutErrors: make(map[string]error),
|
||||||
containers: make(map[string]*container.Container),
|
containers: make(map[string]*container.Container),
|
||||||
chains: make(map[string][]chain.Chain),
|
chains: make(map[string][]chain.Chain),
|
||||||
|
@ -117,14 +110,6 @@ func (t *TestFrostFS) SetObjectError(addr oid.Address, err error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) SetObjectStreamError(addr oid.Address, offset int, err error) {
|
|
||||||
if err == nil {
|
|
||||||
delete(t.objectStreamErrors, addr.EncodeToString())
|
|
||||||
} else {
|
|
||||||
t.objectStreamErrors[addr.EncodeToString()] = offsetError{offset: offset, err: err}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TestFrostFS) SetObjectPutError(fileName string, err error) {
|
func (t *TestFrostFS) SetObjectPutError(fileName string, err error) {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
delete(t.objectPutErrors, fileName)
|
delete(t.objectPutErrors, fileName)
|
||||||
|
@ -277,32 +262,10 @@ func (t *TestFrostFS) GetObject(ctx context.Context, prm frostfs.PrmObjectGet) (
|
||||||
|
|
||||||
return &frostfs.Object{
|
return &frostfs.Object{
|
||||||
Header: *obj,
|
Header: *obj,
|
||||||
Payload: &objPayload{
|
Payload: io.NopCloser(bytes.NewReader(obj.Payload())),
|
||||||
r: bytes.NewReader(obj.Payload()),
|
|
||||||
offsetErr: t.objectStreamErrors[prm.Container.EncodeToString()+"/"+prm.Object.EncodeToString()],
|
|
||||||
},
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type objPayload struct {
|
|
||||||
offset int
|
|
||||||
r io.Reader
|
|
||||||
offsetErr offsetError
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *objPayload) Read(p []byte) (n int, err error) {
|
|
||||||
n, err = o.r.Read(p)
|
|
||||||
if o.offsetErr.err != nil && o.offset+n > o.offsetErr.offset {
|
|
||||||
return o.offsetErr.offset - o.offset, o.offsetErr.err
|
|
||||||
}
|
|
||||||
o.offset += n
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *objPayload) Close() error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TestFrostFS) RangeObject(ctx context.Context, prm frostfs.PrmObjectRange) (io.ReadCloser, error) {
|
func (t *TestFrostFS) RangeObject(ctx context.Context, prm frostfs.PrmObjectRange) (io.ReadCloser, error) {
|
||||||
obj, err := t.retrieveObject(ctx, prm.Container, prm.Object)
|
obj, err := t.retrieveObject(ctx, prm.Container, prm.Object)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -15,7 +15,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
@ -194,7 +193,6 @@ type (
|
||||||
Prefix string
|
Prefix string
|
||||||
VersionIDMarker string
|
VersionIDMarker string
|
||||||
Encode string
|
Encode string
|
||||||
Chan chan<- struct{}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ListBucketsParams struct {
|
ListBucketsParams struct {
|
||||||
|
@ -338,9 +336,6 @@ func (n *Layer) prepareAuthParameters(ctx context.Context, prm *frostfs.PrmAuth,
|
||||||
|
|
||||||
// GetBucketInfo returns bucket info by name.
|
// GetBucketInfo returns bucket info by name.
|
||||||
func (n *Layer) GetBucketInfo(ctx context.Context, name string) (*data.BucketInfo, error) {
|
func (n *Layer) GetBucketInfo(ctx context.Context, name string) (*data.BucketInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetBucketInfo")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
name, err := url.QueryUnescape(name)
|
name, err := url.QueryUnescape(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("unescape bucket name: %w", err)
|
return nil, fmt.Errorf("unescape bucket name: %w", err)
|
||||||
|
@ -389,9 +384,6 @@ func (n *Layer) ResolveCID(ctx context.Context, name string) (cid.ID, error) {
|
||||||
// ListBuckets returns all user containers. The name of the bucket is a container
|
// ListBuckets returns all user containers. The name of the bucket is a container
|
||||||
// id. Timestamp is omitted since it is not saved in frostfs container.
|
// id. Timestamp is omitted since it is not saved in frostfs container.
|
||||||
func (n *Layer) ListBuckets(ctx context.Context, params ListBucketsParams) (ListBucketsResult, error) {
|
func (n *Layer) ListBuckets(ctx context.Context, params ListBucketsParams) (ListBucketsResult, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.ListBuckets")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var result ListBucketsResult
|
var result ListBucketsResult
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
|
@ -413,9 +405,6 @@ func (n *Layer) ListBuckets(ctx context.Context, params ListBucketsParams) (List
|
||||||
|
|
||||||
// GetObject from storage.
|
// GetObject from storage.
|
||||||
func (n *Layer) GetObject(ctx context.Context, p *GetObjectParams) (*ObjectPayload, error) {
|
func (n *Layer) GetObject(ctx context.Context, p *GetObjectParams) (*ObjectPayload, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetObject")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var params getParams
|
var params getParams
|
||||||
|
|
||||||
params.objInfo = p.ObjectInfo
|
params.objInfo = p.ObjectInfo
|
||||||
|
@ -530,9 +519,6 @@ func getDecrypter(p *GetObjectParams) (*encryption.Decrypter, error) {
|
||||||
|
|
||||||
// GetObjectInfo returns meta information about the object.
|
// GetObjectInfo returns meta information about the object.
|
||||||
func (n *Layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error) {
|
func (n *Layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetObjectInfo")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
extendedObjectInfo, err := n.GetExtendedObjectInfo(ctx, p)
|
extendedObjectInfo, err := n.GetExtendedObjectInfo(ctx, p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -543,13 +529,8 @@ func (n *Layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.O
|
||||||
|
|
||||||
// GetExtendedObjectInfo returns meta information and corresponding info from the tree service about the object.
|
// GetExtendedObjectInfo returns meta information and corresponding info from the tree service about the object.
|
||||||
func (n *Layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) {
|
func (n *Layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetExtendedObjectInfo")
|
var objInfo *data.ExtendedObjectInfo
|
||||||
defer span.End()
|
var err error
|
||||||
|
|
||||||
var (
|
|
||||||
objInfo *data.ExtendedObjectInfo
|
|
||||||
err error
|
|
||||||
)
|
|
||||||
|
|
||||||
if p.Versioned() {
|
if p.Versioned() {
|
||||||
objInfo, err = n.headVersion(ctx, p.BktInfo, p)
|
objInfo, err = n.headVersion(ctx, p.BktInfo, p)
|
||||||
|
@ -562,18 +543,13 @@ func (n *Layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams)
|
||||||
|
|
||||||
n.reqLogger(ctx).Debug(logs.GetObject,
|
n.reqLogger(ctx).Debug(logs.GetObject,
|
||||||
zap.Stringer("cid", p.BktInfo.CID),
|
zap.Stringer("cid", p.BktInfo.CID),
|
||||||
zap.Stringer("oid", objInfo.ObjectInfo.ID),
|
zap.Stringer("oid", objInfo.ObjectInfo.ID))
|
||||||
logs.TagField(logs.TagDatapath),
|
|
||||||
)
|
|
||||||
|
|
||||||
return objInfo, nil
|
return objInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CopyObject from one bucket into another bucket.
|
// CopyObject from one bucket into another bucket.
|
||||||
func (n *Layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ExtendedObjectInfo, error) {
|
func (n *Layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ExtendedObjectInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.CopyObject")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
objPayload, err := n.GetObject(ctx, &GetObjectParams{
|
objPayload, err := n.GetObject(ctx, &GetObjectParams{
|
||||||
ObjectInfo: p.SrcObject,
|
ObjectInfo: p.SrcObject,
|
||||||
Versioned: p.SrcVersioned,
|
Versioned: p.SrcVersioned,
|
||||||
|
@ -620,8 +596,8 @@ func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
||||||
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
|
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting, zap.Stringer("cid", bkt.CID),
|
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting,
|
||||||
zap.String("oid", obj.VersionID), zap.Error(obj.Error), logs.TagField(logs.TagExternalStorage))
|
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error))
|
||||||
}
|
}
|
||||||
|
|
||||||
if obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID); obj.Error != nil {
|
if obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID); obj.Error != nil {
|
||||||
|
@ -659,8 +635,8 @@ func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
||||||
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
|
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting, zap.Stringer("cid", bkt.CID),
|
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting,
|
||||||
zap.String("oid", obj.VersionID), zap.Error(obj.Error), logs.TagField(logs.TagExternalStorage))
|
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -775,7 +751,7 @@ func (n *Layer) getNodeVersionsToDelete(ctx context.Context, bkt *data.BucketInf
|
||||||
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", apierr.GetAPIError(apierr.ErrNoSuchVersion))
|
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", apierr.GetAPIError(apierr.ErrNoSuchVersion))
|
||||||
}
|
}
|
||||||
|
|
||||||
n.reqLogger(ctx).Debug(logs.GetTreeNodeToDelete, zap.Stringer("cid", bkt.CID), zap.Strings("oids", oids), logs.TagField(logs.TagDatapath))
|
n.reqLogger(ctx).Debug(logs.GetTreeNodeToDelete, zap.Stringer("cid", bkt.CID), zap.Strings("oids", oids))
|
||||||
|
|
||||||
return versionsToDelete, nil
|
return versionsToDelete, nil
|
||||||
}
|
}
|
||||||
|
@ -842,13 +818,10 @@ func (n *Layer) removeCombinedObject(ctx context.Context, bkt *data.BucketInfo,
|
||||||
|
|
||||||
// DeleteObjects from the storage.
|
// DeleteObjects from the storage.
|
||||||
func (n *Layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*VersionedObject {
|
func (n *Layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*VersionedObject {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.DeleteObjects")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
for i, obj := range p.Objects {
|
for i, obj := range p.Objects {
|
||||||
p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.Settings, obj, p.NetworkInfo)
|
p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.Settings, obj, p.NetworkInfo)
|
||||||
if p.IsMultiple && p.Objects[i].Error != nil {
|
if p.IsMultiple && p.Objects[i].Error != nil {
|
||||||
n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error), logs.TagField(logs.TagExternalStorage))
|
n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -856,9 +829,6 @@ func (n *Layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*Ver
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) {
|
func (n *Layer) CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.CreateBucket")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
bktInfo, err := n.GetBucketInfo(ctx, p.Name)
|
bktInfo, err := n.GetBucketInfo(ctx, p.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if apierr.IsS3Error(err, apierr.ErrNoSuchBucket) {
|
if apierr.IsS3Error(err, apierr.ErrNoSuchBucket) {
|
||||||
|
@ -881,16 +851,13 @@ func (n *Layer) ResolveBucket(ctx context.Context, zone, name string) (cid.ID, e
|
||||||
return cid.ID{}, err
|
return cid.ID{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
n.reqLogger(ctx).Info(logs.ResolveBucket, zap.Stringer("cid", cnrID), logs.TagField(logs.TagDatapath))
|
n.reqLogger(ctx).Info(logs.ResolveBucket, zap.Stringer("cid", cnrID))
|
||||||
}
|
}
|
||||||
|
|
||||||
return cnrID, nil
|
return cnrID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.DeleteBucket")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
if !p.SkipCheck {
|
if !p.SkipCheck {
|
||||||
res, _, err := n.getAllObjectsVersions(ctx, commonVersionsListingParams{
|
res, _, err := n.getAllObjectsVersions(ctx, commonVersionsListingParams{
|
||||||
BktInfo: p.BktInfo,
|
BktInfo: p.BktInfo,
|
||||||
|
@ -909,12 +876,12 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
|
|
||||||
corsObj, err := n.treeService.GetBucketCORS(ctx, p.BktInfo)
|
corsObj, err := n.treeService.GetBucketCORS(ctx, p.BktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Error(logs.GetBucketCorsFromTree, zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
|
n.reqLogger(ctx).Error(logs.GetBucketCors, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
lifecycleObj, treeErr := n.treeService.GetBucketLifecycleConfiguration(ctx, p.BktInfo)
|
lifecycleObj, treeErr := n.treeService.GetBucketLifecycleConfiguration(ctx, p.BktInfo)
|
||||||
if treeErr != nil {
|
if treeErr != nil {
|
||||||
n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr), logs.TagField(logs.TagExternalStorageTree))
|
n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr))
|
||||||
}
|
}
|
||||||
|
|
||||||
err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
||||||
|
@ -934,9 +901,6 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) DeleteContainer(ctx context.Context, p *DeleteBucketParams) error {
|
func (n *Layer) DeleteContainer(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.DeleteContainer")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
n.cache.DeleteBucket(p.BktInfo)
|
n.cache.DeleteBucket(p.BktInfo)
|
||||||
if err := n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken); err != nil {
|
if err := n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken); err != nil {
|
||||||
return fmt.Errorf("delete container: %w", err)
|
return fmt.Errorf("delete container: %w", err)
|
||||||
|
@ -945,9 +909,6 @@ func (n *Layer) DeleteContainer(ctx context.Context, p *DeleteBucketParams) erro
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) GetNetworkInfo(ctx context.Context) (netmap.NetworkInfo, error) {
|
func (n *Layer) GetNetworkInfo(ctx context.Context) (netmap.NetworkInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetNetworkInfo")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
cachedInfo := n.cache.GetNetworkInfo()
|
cachedInfo := n.cache.GetNetworkInfo()
|
||||||
if cachedInfo != nil {
|
if cachedInfo != nil {
|
||||||
return *cachedInfo, nil
|
return *cachedInfo, nil
|
||||||
|
|
|
@ -7,7 +7,6 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
||||||
|
@ -24,9 +23,6 @@ type PutBucketLifecycleParams struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucketLifecycleParams) error {
|
func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucketLifecycleParams) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.PutBucketLifecycleConfiguration")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
cfgBytes, err := xml.Marshal(p.LifecycleCfg)
|
cfgBytes, err := xml.Marshal(p.LifecycleCfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("marshal lifecycle configuration: %w", err)
|
return fmt.Errorf("marshal lifecycle configuration: %w", err)
|
||||||
|
@ -83,16 +79,11 @@ func (n *Layer) deleteLifecycleObject(ctx context.Context, bktInfo *data.BucketI
|
||||||
if err := n.objectDeleteWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth); err != nil {
|
if err := n.objectDeleteWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth); err != nil {
|
||||||
n.reqLogger(ctx).Error(logs.CouldntDeleteLifecycleObject, zap.Error(err),
|
n.reqLogger(ctx).Error(logs.CouldntDeleteLifecycleObject, zap.Error(err),
|
||||||
zap.String("cid", lifecycleBkt.CID.EncodeToString()),
|
zap.String("cid", lifecycleBkt.CID.EncodeToString()),
|
||||||
zap.String("oid", addr.Object().EncodeToString()),
|
zap.String("oid", addr.Object().EncodeToString()))
|
||||||
logs.TagField(logs.TagExternalStorage),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (*data.LifecycleConfiguration, error) {
|
func (n *Layer) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (*data.LifecycleConfiguration, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetBucketLifecycleConfiguration")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
if cfg := n.cache.GetLifecycleConfiguration(owner, bktInfo); cfg != nil {
|
if cfg := n.cache.GetLifecycleConfiguration(owner, bktInfo); cfg != nil {
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
|
@ -138,9 +129,6 @@ func (n *Layer) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *da
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) error {
|
func (n *Layer) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.DeleteBucketLifecycleConfiguration")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
objs, err := n.treeService.DeleteBucketLifecycleConfiguration(ctx, bktInfo)
|
objs, err := n.treeService.DeleteBucketLifecycleConfiguration(ctx, bktInfo)
|
||||||
objsNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
|
objsNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
|
||||||
if err != nil && !objsNotFound {
|
if err != nil && !objsNotFound {
|
||||||
|
|
|
@ -9,7 +9,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
@ -27,7 +26,6 @@ type (
|
||||||
Encode string
|
Encode string
|
||||||
MaxKeys int
|
MaxKeys int
|
||||||
Prefix string
|
Prefix string
|
||||||
Chan chan<- struct{}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListObjectsParamsV1 contains params for ListObjectsV1.
|
// ListObjectsParamsV1 contains params for ListObjectsV1.
|
||||||
|
@ -82,8 +80,6 @@ type (
|
||||||
MaxKeys int
|
MaxKeys int
|
||||||
Marker string
|
Marker string
|
||||||
Bookmark string
|
Bookmark string
|
||||||
// Chan is a channel to prevent client from context canceling during long listing.
|
|
||||||
Chan chan<- struct{}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
commonLatestVersionsListingParams struct {
|
commonLatestVersionsListingParams struct {
|
||||||
|
@ -101,9 +97,6 @@ const (
|
||||||
|
|
||||||
// ListObjectsV1 returns objects in a bucket for requests of Version 1.
|
// ListObjectsV1 returns objects in a bucket for requests of Version 1.
|
||||||
func (n *Layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error) {
|
func (n *Layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.ListObjectsV1")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var result ListObjectsInfoV1
|
var result ListObjectsInfoV1
|
||||||
|
|
||||||
prm := commonLatestVersionsListingParams{
|
prm := commonLatestVersionsListingParams{
|
||||||
|
@ -114,7 +107,6 @@ func (n *Layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*Lis
|
||||||
MaxKeys: p.MaxKeys,
|
MaxKeys: p.MaxKeys,
|
||||||
Marker: p.Marker,
|
Marker: p.Marker,
|
||||||
Bookmark: p.Marker,
|
Bookmark: p.Marker,
|
||||||
Chan: p.Chan,
|
|
||||||
},
|
},
|
||||||
ListType: ListObjectsV1Type,
|
ListType: ListObjectsV1Type,
|
||||||
}
|
}
|
||||||
|
@ -136,9 +128,6 @@ func (n *Layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*Lis
|
||||||
|
|
||||||
// ListObjectsV2 returns objects in a bucket for requests of Version 2.
|
// ListObjectsV2 returns objects in a bucket for requests of Version 2.
|
||||||
func (n *Layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error) {
|
func (n *Layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.ListObjectsV2")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var result ListObjectsInfoV2
|
var result ListObjectsInfoV2
|
||||||
|
|
||||||
prm := commonLatestVersionsListingParams{
|
prm := commonLatestVersionsListingParams{
|
||||||
|
@ -149,7 +138,6 @@ func (n *Layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
|
||||||
MaxKeys: p.MaxKeys,
|
MaxKeys: p.MaxKeys,
|
||||||
Marker: p.StartAfter,
|
Marker: p.StartAfter,
|
||||||
Bookmark: p.ContinuationToken,
|
Bookmark: p.ContinuationToken,
|
||||||
Chan: p.Chan,
|
|
||||||
},
|
},
|
||||||
ListType: ListObjectsV2Type,
|
ListType: ListObjectsV2Type,
|
||||||
}
|
}
|
||||||
|
@ -170,9 +158,6 @@ func (n *Layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
func (n *Layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.ListObjectVersions")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
prm := commonVersionsListingParams{
|
prm := commonVersionsListingParams{
|
||||||
BktInfo: p.BktInfo,
|
BktInfo: p.BktInfo,
|
||||||
Delimiter: p.Delimiter,
|
Delimiter: p.Delimiter,
|
||||||
|
@ -180,7 +165,6 @@ func (n *Layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar
|
||||||
MaxKeys: p.MaxKeys,
|
MaxKeys: p.MaxKeys,
|
||||||
Marker: p.KeyMarker,
|
Marker: p.KeyMarker,
|
||||||
Bookmark: p.VersionIDMarker,
|
Bookmark: p.VersionIDMarker,
|
||||||
Chan: p.Chan,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
objects, isTruncated, err := n.getAllObjectsVersions(ctx, prm)
|
objects, isTruncated, err := n.getAllObjectsVersions(ctx, prm)
|
||||||
|
@ -224,10 +208,6 @@ func (n *Layer) getLatestObjectsVersions(ctx context.Context, p commonLatestVers
|
||||||
objects = append(objects, session.Next...)
|
objects = append(objects, session.Next...)
|
||||||
for obj := range objOutCh {
|
for obj := range objOutCh {
|
||||||
objects = append(objects, obj)
|
objects = append(objects, obj)
|
||||||
select {
|
|
||||||
case p.Chan <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = <-errorCh; err != nil {
|
if err = <-errorCh; err != nil {
|
||||||
|
@ -297,11 +277,6 @@ func handleGeneratedVersions(objOutCh <-chan *data.ExtendedNodeVersion, p common
|
||||||
allObjects = append(allObjects, eoi)
|
allObjects = append(allObjects, eoi)
|
||||||
}
|
}
|
||||||
lastName = name
|
lastName = name
|
||||||
|
|
||||||
select {
|
|
||||||
case p.Chan <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
formVersionsListRow(allObjects, listRowStartIndex, session)
|
formVersionsListRow(allObjects, listRowStartIndex, session)
|
||||||
|
@ -566,7 +541,7 @@ func (n *Layer) initWorkerPool(ctx context.Context, size int, p commonVersionsLi
|
||||||
|
|
||||||
realSize, err := GetObjectSize(oi)
|
realSize, err := GetObjectSize(oi)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
reqLog.Debug(logs.FailedToGetRealObjectSize, zap.Error(err), logs.TagField(logs.TagDatapath))
|
reqLog.Debug(logs.FailedToGetRealObjectSize, zap.Error(err))
|
||||||
realSize = oi.Size
|
realSize = oi.Size
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -579,7 +554,7 @@ func (n *Layer) initWorkerPool(ctx context.Context, size int, p commonVersionsLi
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
wg.Done()
|
wg.Done()
|
||||||
reqLog.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath))
|
reqLog.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err))
|
||||||
}
|
}
|
||||||
}(node)
|
}(node)
|
||||||
}
|
}
|
||||||
|
@ -670,7 +645,7 @@ func (n *Layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo
|
||||||
|
|
||||||
meta, err := n.objectHead(ctx, bktInfo, node.OID)
|
meta, err := n.objectHead(ctx, bktInfo, node.OID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Warn(logs.CouldNotFetchObjectMeta, zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
n.reqLogger(ctx).Warn(logs.CouldNotFetchObjectMeta, zap.Error(err))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -15,7 +15,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
@ -152,9 +151,6 @@ type (
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartParams) error {
|
func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartParams) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.CreateMultipartUpload")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
metaSize := len(p.Header)
|
metaSize := len(p.Header)
|
||||||
if p.Data != nil {
|
if p.Data != nil {
|
||||||
metaSize += len(p.Data.TagSet)
|
metaSize += len(p.Data.TagSet)
|
||||||
|
@ -194,9 +190,6 @@ func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, error) {
|
func (n *Layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.UploadPart")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
|
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||||
|
@ -220,7 +213,7 @@ func (n *Layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, er
|
||||||
func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) {
|
func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) {
|
||||||
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
||||||
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
||||||
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
|
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
|
||||||
return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters)
|
return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -273,10 +266,7 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||||
err = n.frostFS.DeleteObject(ctx, prm)
|
err = n.frostFS.DeleteObject(ctx, prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||||
zap.Stringer("cid", bktInfo.CID),
|
|
||||||
zap.Stringer("oid", createdObj.ID),
|
|
||||||
logs.TagField(logs.TagExternalStorage))
|
|
||||||
}
|
}
|
||||||
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
|
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
|
||||||
}
|
}
|
||||||
|
@ -293,10 +283,7 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
|
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
|
||||||
err = n.objectDelete(ctx, bktInfo, createdObj.ID)
|
err = n.objectDelete(ctx, bktInfo, createdObj.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||||
zap.Stringer("cid", bktInfo.CID),
|
|
||||||
zap.Stringer("oid", createdObj.ID),
|
|
||||||
logs.TagField(logs.TagExternalStorage))
|
|
||||||
}
|
}
|
||||||
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
|
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
|
||||||
}
|
}
|
||||||
|
@ -304,7 +291,7 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
|
|
||||||
n.reqLogger(ctx).Debug(logs.UploadPart,
|
n.reqLogger(ctx).Debug(logs.UploadPart,
|
||||||
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
|
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
|
||||||
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID), logs.TagField(logs.TagDatapath))
|
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||||
|
|
||||||
partInfo := &data.PartInfo{
|
partInfo := &data.PartInfo{
|
||||||
Key: p.Info.Key,
|
Key: p.Info.Key,
|
||||||
|
@ -327,8 +314,7 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil {
|
if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil {
|
||||||
n.reqLogger(ctx).Error(logs.CouldntDeleteOldPartObject, zap.Error(err),
|
n.reqLogger(ctx).Error(logs.CouldntDeleteOldPartObject, zap.Error(err),
|
||||||
zap.String("cid", bktInfo.CID.EncodeToString()),
|
zap.String("cid", bktInfo.CID.EncodeToString()),
|
||||||
zap.String("oid", oldPartID.EncodeToString()),
|
zap.String("oid", oldPartID.EncodeToString()))
|
||||||
logs.TagField(logs.TagExternalStorage))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -349,9 +335,6 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.ObjectInfo, error) {
|
func (n *Layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.ObjectInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.UploadPartCopy")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
|
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||||
|
@ -401,9 +384,6 @@ func (n *Layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipartParams) (*UploadData, *data.ExtendedObjectInfo, error) {
|
func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipartParams) (*UploadData, *data.ExtendedObjectInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.CompleteMultipartUpload")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
for i := 1; i < len(p.Parts); i++ {
|
for i := 1; i < len(p.Parts); i++ {
|
||||||
if p.Parts[i].PartNumber <= p.Parts[i-1].PartNumber {
|
if p.Parts[i].PartNumber <= p.Parts[i-1].PartNumber {
|
||||||
return nil, nil, apierr.GetAPIError(apierr.ErrInvalidPartOrder)
|
return nil, nil, apierr.GetAPIError(apierr.ErrInvalidPartOrder)
|
||||||
|
@ -501,8 +481,7 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
||||||
n.reqLogger(ctx).Error(logs.CouldNotPutCompletedObject,
|
n.reqLogger(ctx).Error(logs.CouldNotPutCompletedObject,
|
||||||
zap.String("uploadID", p.Info.UploadID),
|
zap.String("uploadID", p.Info.UploadID),
|
||||||
zap.String("uploadKey", p.Info.Key),
|
zap.String("uploadKey", p.Info.Key),
|
||||||
zap.Error(err),
|
zap.Error(err))
|
||||||
logs.TagField(logs.TagExternalStorage))
|
|
||||||
|
|
||||||
return nil, nil, apierr.GetAPIError(apierr.ErrInternalError)
|
return nil, nil, apierr.GetAPIError(apierr.ErrInternalError)
|
||||||
}
|
}
|
||||||
|
@ -514,8 +493,7 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
||||||
if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil {
|
if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil {
|
||||||
n.reqLogger(ctx).Warn(logs.CouldNotDeleteUploadPart,
|
n.reqLogger(ctx).Warn(logs.CouldNotDeleteUploadPart,
|
||||||
zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID),
|
zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID),
|
||||||
zap.Error(err),
|
zap.Error(err))
|
||||||
logs.TagField(logs.TagExternalStorage))
|
|
||||||
}
|
}
|
||||||
addr.SetObject(partInfo.OID)
|
addr.SetObject(partInfo.OID)
|
||||||
n.cache.DeleteObject(addr)
|
n.cache.DeleteObject(addr)
|
||||||
|
@ -526,9 +504,6 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) ListMultipartUploads(ctx context.Context, p *ListMultipartUploadsParams) (*ListMultipartUploadsInfo, error) {
|
func (n *Layer) ListMultipartUploads(ctx context.Context, p *ListMultipartUploadsParams) (*ListMultipartUploadsInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.ListMultipartUploads")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var result ListMultipartUploadsInfo
|
var result ListMultipartUploadsInfo
|
||||||
if p.MaxUploads == 0 {
|
if p.MaxUploads == 0 {
|
||||||
return &result, nil
|
return &result, nil
|
||||||
|
@ -589,9 +564,6 @@ func (n *Layer) ListMultipartUploads(ctx context.Context, p *ListMultipartUpload
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) AbortMultipartUpload(ctx context.Context, p *UploadInfoParams) error {
|
func (n *Layer) AbortMultipartUpload(ctx context.Context, p *UploadInfoParams) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.AbortMultipartUpload")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
multipartInfo, parts, err := n.getUploadParts(ctx, p)
|
multipartInfo, parts, err := n.getUploadParts(ctx, p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -615,7 +587,7 @@ func (n *Layer) deleteUploadedParts(ctx context.Context, bkt *data.BucketInfo, p
|
||||||
oids, err := relations.ListAllRelations(ctx, n.frostFS.Relations(), bkt.CID, info.OID, tokens)
|
oids, err := relations.ListAllRelations(ctx, n.frostFS.Relations(), bkt.CID, info.OID, tokens)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Warn(logs.FailedToListAllObjectRelations, zap.String("cid", bkt.CID.EncodeToString()),
|
n.reqLogger(ctx).Warn(logs.FailedToListAllObjectRelations, zap.String("cid", bkt.CID.EncodeToString()),
|
||||||
zap.String("oid", info.OID.EncodeToString()), zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
zap.String("oid", info.OID.EncodeToString()), zap.Error(err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
members = append(members, append(oids, info.OID)...)
|
members = append(members, append(oids, info.OID)...)
|
||||||
|
@ -624,14 +596,11 @@ func (n *Layer) deleteUploadedParts(ctx context.Context, bkt *data.BucketInfo, p
|
||||||
|
|
||||||
err := n.putTombstones(ctx, bkt, networkInfo, members)
|
err := n.putTombstones(ctx, bkt, networkInfo, members)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Warn(logs.FailedToPutTombstones, zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
n.reqLogger(ctx).Warn(logs.FailedToPutTombstones, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsInfo, error) {
|
func (n *Layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.ListParts")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var res ListPartsInfo
|
var res ListPartsInfo
|
||||||
multipartInfo, partsInfo, err := n.getUploadParts(ctx, p.Info)
|
multipartInfo, partsInfo, err := n.getUploadParts(ctx, p.Info)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -640,7 +609,7 @@ func (n *Layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
|
||||||
|
|
||||||
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
||||||
if err = p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
if err = p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
||||||
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
|
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
|
||||||
return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters)
|
return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -732,8 +701,7 @@ func (n *Layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
|
||||||
zap.Stringer("cid", p.Bkt.CID),
|
zap.Stringer("cid", p.Bkt.CID),
|
||||||
zap.String("upload id", p.UploadID),
|
zap.String("upload id", p.UploadID),
|
||||||
zap.Ints("part numbers", partsNumbers),
|
zap.Ints("part numbers", partsNumbers),
|
||||||
zap.Strings("oids", oids),
|
zap.Strings("oids", oids))
|
||||||
logs.TagField(logs.TagDatapath))
|
|
||||||
|
|
||||||
return multipartInfo, res, nil
|
return multipartInfo, res, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,7 +17,6 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
@ -228,9 +227,6 @@ func ParseCompletedPartHeader(hdr string) (*Part, error) {
|
||||||
|
|
||||||
// PutObject stores object into FrostFS, took payload from io.Reader.
|
// PutObject stores object into FrostFS, took payload from io.Reader.
|
||||||
func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error) {
|
func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.PutObject")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
bktSettings, err := n.GetBucketSettings(ctx, p.BktInfo)
|
bktSettings, err := n.GetBucketSettings(ctx, p.BktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("couldn't get versioning settings object: %w", err)
|
return nil, fmt.Errorf("couldn't get versioning settings object: %w", err)
|
||||||
|
@ -299,11 +295,7 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
if !bytes.Equal(headerMd5Hash, createdObj.MD5Sum) {
|
if !bytes.Equal(headerMd5Hash, createdObj.MD5Sum) {
|
||||||
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
|
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||||
zap.Stringer("cid", p.BktInfo.CID),
|
|
||||||
zap.Stringer("oid", createdObj.ID),
|
|
||||||
logs.TagField(logs.TagExternalStorage),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
return nil, apierr.GetAPIError(apierr.ErrBadDigest)
|
return nil, apierr.GetAPIError(apierr.ErrBadDigest)
|
||||||
}
|
}
|
||||||
|
@ -317,17 +309,13 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
|
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
|
||||||
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
|
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||||
zap.Stringer("cid", p.BktInfo.CID),
|
|
||||||
zap.Stringer("oid", createdObj.ID),
|
|
||||||
logs.TagField(logs.TagExternalStorage))
|
|
||||||
}
|
}
|
||||||
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
|
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID),
|
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||||
zap.Stringer("oid", createdObj.ID), logs.TagField(logs.TagExternalStorage))
|
|
||||||
now := TimeNow(ctx)
|
now := TimeNow(ctx)
|
||||||
newVersion := &data.NodeVersion{
|
newVersion := &data.NodeVersion{
|
||||||
BaseNodeVersion: data.BaseNodeVersion{
|
BaseNodeVersion: data.BaseNodeVersion{
|
||||||
|
@ -422,7 +410,7 @@ func (n *Layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.Bucke
|
||||||
|
|
||||||
meta, err := n.objectHead(ctx, bkt, node.OID)
|
meta, err := n.objectHead(ctx, bkt, node.OID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if client.IsErrObjectNotFound(err) || client.IsErrObjectAlreadyRemoved(err) {
|
if client.IsErrObjectNotFound(err) {
|
||||||
return nil, fmt.Errorf("%w: %s; %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error(), node.OID.EncodeToString())
|
return nil, fmt.Errorf("%w: %s; %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error(), node.OID.EncodeToString())
|
||||||
}
|
}
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -479,7 +467,7 @@ func (n *Layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
|
||||||
|
|
||||||
meta, err := n.objectHead(ctx, bkt, foundVersion.OID)
|
meta, err := n.objectHead(ctx, bkt, foundVersion.OID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if client.IsErrObjectNotFound(err) || client.IsErrObjectAlreadyRemoved(err) {
|
if client.IsErrObjectNotFound(err) {
|
||||||
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchVersion), err.Error())
|
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchVersion), err.Error())
|
||||||
}
|
}
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -552,7 +540,7 @@ func (n *Layer) objectPutAndHash(ctx context.Context, prm frostfs.PrmObjectCreat
|
||||||
func (n *Layer) payloadDiscard(ctx context.Context, payload io.Reader) {
|
func (n *Layer) payloadDiscard(ctx context.Context, payload io.Reader) {
|
||||||
if payload != nil {
|
if payload != nil {
|
||||||
if _, errDiscard := io.Copy(io.Discard, payload); errDiscard != nil {
|
if _, errDiscard := io.Copy(io.Discard, payload); errDiscard != nil {
|
||||||
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard), logs.TagField(logs.TagDatapath))
|
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -562,7 +550,7 @@ type logWrapper struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *logWrapper) Printf(format string, args ...interface{}) {
|
func (l *logWrapper) Printf(format string, args ...interface{}) {
|
||||||
l.log.Info(fmt.Sprintf(format, args...), logs.TagField(logs.TagDatapath))
|
l.log.Info(fmt.Sprintf(format, args...))
|
||||||
}
|
}
|
||||||
|
|
||||||
func IsSystemHeader(key string) bool {
|
func IsSystemHeader(key string) bool {
|
||||||
|
|
|
@ -10,7 +10,6 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
||||||
|
@ -26,9 +25,6 @@ type PatchObjectParams struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) PatchObject(ctx context.Context, p *PatchObjectParams) (*data.ExtendedObjectInfo, error) {
|
func (n *Layer) PatchObject(ctx context.Context, p *PatchObjectParams) (*data.ExtendedObjectInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.PatchObject")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
if p.Object.ObjectInfo.Headers[AttributeDecryptedSize] != "" {
|
if p.Object.ObjectInfo.Headers[AttributeDecryptedSize] != "" {
|
||||||
return nil, fmt.Errorf("patch encrypted object")
|
return nil, fmt.Errorf("patch encrypted object")
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,7 +10,6 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
||||||
|
@ -33,9 +32,6 @@ type PutLockInfoParams struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err error) {
|
func (n *Layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.PutLockInfo")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
newLock := p.NewLock
|
newLock := p.NewLock
|
||||||
versionNode := p.NodeVersion
|
versionNode := p.NodeVersion
|
||||||
// sometimes node version can be provided from executing context
|
// sometimes node version can be provided from executing context
|
||||||
|
@ -143,9 +139,6 @@ func (n *Layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion) (*data.LockInfo, error) {
|
func (n *Layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion) (*data.LockInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetLockInfo")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
if lockInfo := n.cache.GetLockInfo(owner, lockObjectKey(objVersion)); lockInfo != nil {
|
if lockInfo := n.cache.GetLockInfo(owner, lockObjectKey(objVersion)); lockInfo != nil {
|
||||||
return lockInfo, nil
|
return lockInfo, nil
|
||||||
|
@ -213,9 +206,6 @@ func lockObjectKey(objVersion *data.ObjectVersion) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
|
func (n *Layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetBucketSettings")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
if settings := n.cache.GetSettings(owner, bktInfo); settings != nil {
|
if settings := n.cache.GetSettings(owner, bktInfo); settings != nil {
|
||||||
return settings, nil
|
return settings, nil
|
||||||
|
@ -227,7 +217,7 @@ func (n *Layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
settings = &data.BucketSettings{Versioning: data.VersioningUnversioned}
|
settings = &data.BucketSettings{Versioning: data.VersioningUnversioned}
|
||||||
n.reqLogger(ctx).Debug(logs.BucketSettingsNotFoundUseDefaults, logs.TagField(logs.TagDatapath))
|
n.reqLogger(ctx).Debug(logs.BucketSettingsNotFoundUseDefaults)
|
||||||
}
|
}
|
||||||
|
|
||||||
n.cache.PutSettings(owner, bktInfo, settings)
|
n.cache.PutSettings(owner, bktInfo, settings)
|
||||||
|
@ -236,9 +226,6 @@ func (n *Layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) PutBucketSettings(ctx context.Context, p *PutSettingsParams) error {
|
func (n *Layer) PutBucketSettings(ctx context.Context, p *PutSettingsParams) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.PutBucketSettings")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
if err := n.treeService.PutSettingsNode(ctx, p.BktInfo, p.Settings); err != nil {
|
if err := n.treeService.PutSettingsNode(ctx, p.BktInfo, p.Settings); err != nil {
|
||||||
return fmt.Errorf("failed to get settings node: %w", err)
|
return fmt.Errorf("failed to get settings node: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,7 +5,6 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
|
||||||
|
@ -17,9 +16,6 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *Layer) GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error) {
|
func (n *Layer) GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetObjectTagging")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
|
|
||||||
|
@ -56,9 +52,6 @@ func (n *Layer) GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingPa
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingParams) (err error) {
|
func (n *Layer) PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingParams) (err error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.PutObjectTagging")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
nodeVersion := p.NodeVersion
|
nodeVersion := p.NodeVersion
|
||||||
if nodeVersion == nil {
|
if nodeVersion == nil {
|
||||||
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
|
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
|
||||||
|
@ -82,9 +75,6 @@ func (n *Layer) PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingPa
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion) error {
|
func (n *Layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.DeleteObjectTagging")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
version, err := n.getNodeVersion(ctx, p)
|
version, err := n.getNodeVersion(ctx, p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -106,9 +96,6 @@ func (n *Layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) {
|
func (n *Layer) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetBucketTagging")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
|
|
||||||
if tags := n.cache.GetTagging(owner, bucketTaggingCacheKey(bktInfo.CID)); tags != nil {
|
if tags := n.cache.GetTagging(owner, bucketTaggingCacheKey(bktInfo.CID)); tags != nil {
|
||||||
|
@ -126,9 +113,6 @@ func (n *Layer) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error {
|
func (n *Layer) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.PutBucketTagging")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
if err := n.treeService.PutBucketTagging(ctx, bktInfo, tagSet); err != nil {
|
if err := n.treeService.PutBucketTagging(ctx, bktInfo, tagSet); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -139,9 +123,6 @@ func (n *Layer) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error {
|
func (n *Layer) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "layer.DeleteBucketTagging")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
n.cache.DeleteTagging(bucketTaggingCacheKey(bktInfo.CID))
|
n.cache.DeleteTagging(bucketTaggingCacheKey(bktInfo.CID))
|
||||||
|
|
||||||
return n.treeService.DeleteBucketTagging(ctx, bktInfo)
|
return n.treeService.DeleteBucketTagging(ctx, bktInfo)
|
||||||
|
@ -187,8 +168,7 @@ func (n *Layer) getNodeVersion(ctx context.Context, objVersion *data.ObjectVersi
|
||||||
|
|
||||||
if err == nil && version != nil && !version.IsDeleteMarker {
|
if err == nil && version != nil && !version.IsDeleteMarker {
|
||||||
n.reqLogger(ctx).Debug(logs.GetTreeNode,
|
n.reqLogger(ctx).Debug(logs.GetTreeNode,
|
||||||
zap.Stringer("cid", objVersion.BktInfo.CID),
|
zap.Stringer("cid", objVersion.BktInfo.CID), zap.Stringer("oid", version.OID))
|
||||||
zap.Stringer("oid", version.OID), logs.TagField(logs.TagExternalStorageTree))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return version, err
|
return version, err
|
||||||
|
|
|
@ -65,13 +65,13 @@ func (n *Layer) submitPutTombstone(ctx context.Context, bkt *data.BucketInfo, me
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
|
||||||
if err := n.putTombstoneObject(ctx, tomb, bkt); err != nil {
|
if err := n.putTombstoneObject(ctx, tomb, bkt); err != nil {
|
||||||
n.reqLogger(ctx).Warn(logs.FailedToPutTombstoneObject, zap.String("cid", bkt.CID.EncodeToString()), zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
n.reqLogger(ctx).Warn(logs.FailedToPutTombstoneObject, zap.String("cid", bkt.CID.EncodeToString()), zap.Error(err))
|
||||||
errCh <- fmt.Errorf("put tombstone object: %w", err)
|
errCh <- fmt.Errorf("put tombstone object: %w", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
wg.Done()
|
wg.Done()
|
||||||
n.reqLogger(ctx).Warn(logs.FailedToSubmitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath))
|
n.reqLogger(ctx).Warn(logs.FailedToSubmitTaskToPool, zap.Error(err))
|
||||||
errCh <- fmt.Errorf("submit task to pool: %w", err)
|
errCh <- fmt.Errorf("submit task to pool: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -106,7 +106,7 @@ func (n *Layer) getMembers(ctx context.Context, cnrID cid.ID, objID oid.ID, toke
|
||||||
}
|
}
|
||||||
|
|
||||||
n.reqLogger(ctx).Warn(logs.FailedToListAllObjectRelations, zap.String("cid", cnrID.EncodeToString()),
|
n.reqLogger(ctx).Warn(logs.FailedToListAllObjectRelations, zap.String("cid", cnrID.EncodeToString()),
|
||||||
zap.String("oid", objID.EncodeToString()), zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
zap.String("oid", objID.EncodeToString()), zap.Error(err))
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return append(oids, objID), nil
|
return append(oids, objID), nil
|
||||||
|
|
|
@ -18,7 +18,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap/zaptest"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (tc *testContext) putObject(content []byte) *data.ObjectInfo {
|
func (tc *testContext) putObject(content []byte) *data.ObjectInfo {
|
||||||
|
@ -139,7 +139,7 @@ type testContext struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
||||||
logger := zaptest.NewLogger(t)
|
logger := zap.NewExample()
|
||||||
|
|
||||||
key, err := keys.NewPrivateKey()
|
key, err := keys.NewPrivateKey()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
|
@ -110,7 +110,7 @@ func preparePathStyleAddress(reqInfo *ReqInfo, r *http.Request, reqLogger *zap.L
|
||||||
// https://github.com/go-chi/chi/issues/641
|
// https://github.com/go-chi/chi/issues/641
|
||||||
// https://github.com/go-chi/chi/issues/642
|
// https://github.com/go-chi/chi/issues/642
|
||||||
if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil {
|
if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil {
|
||||||
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err), logs.TagField(logs.TagDatapath))
|
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
reqInfo.ObjectName = obj
|
reqInfo.ObjectName = obj
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,14 +1,12 @@
|
||||||
package middleware
|
package middleware
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"crypto/elliptic"
|
"crypto/elliptic"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
|
@ -32,9 +30,7 @@ type (
|
||||||
Center interface {
|
Center interface {
|
||||||
// Authenticate validate and authenticate request.
|
// Authenticate validate and authenticate request.
|
||||||
// Must return ErrNoAuthorizationHeader if auth header is missed.
|
// Must return ErrNoAuthorizationHeader if auth header is missed.
|
||||||
// Authenticate uses a separate context so that the authorization
|
Authenticate(request *http.Request) (*Box, error)
|
||||||
// span middleware does not contain all subsequent spans.
|
|
||||||
Authenticate(ctx context.Context, request *http.Request) (*Box, error)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//nolint:revive
|
//nolint:revive
|
||||||
|
@ -51,38 +47,34 @@ var ErrNoAuthorizationHeader = errors.New("no authorization header")
|
||||||
func Auth(center Center, log *zap.Logger) Func {
|
func Auth(center Center, log *zap.Logger) Func {
|
||||||
return func(h http.Handler) http.Handler {
|
return func(h http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
reqCtx := r.Context()
|
ctx := r.Context()
|
||||||
ctx, span := tracing.StartSpanFromContext(reqCtx, "middleware.Auth")
|
reqInfo := GetReqInfo(ctx)
|
||||||
|
|
||||||
reqInfo := GetReqInfo(reqCtx)
|
|
||||||
reqInfo.User = "anon"
|
reqInfo.User = "anon"
|
||||||
box, err := center.Authenticate(ctx, r)
|
box, err := center.Authenticate(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, ErrNoAuthorizationHeader) {
|
if errors.Is(err, ErrNoAuthorizationHeader) {
|
||||||
reqLogOrDefault(reqCtx, log).Debug(logs.CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed, zap.Error(err), logs.TagField(logs.TagDatapath))
|
reqLogOrDefault(ctx, log).Debug(logs.CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed, zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
reqLogOrDefault(reqCtx, log).Error(logs.FailedToPassAuthentication, zap.Error(err), logs.TagField(logs.TagDatapath))
|
reqLogOrDefault(ctx, log).Error(logs.FailedToPassAuthentication, zap.Error(err))
|
||||||
err = apierr.TransformToS3Error(err)
|
err = apierr.TransformToS3Error(err)
|
||||||
if err.(apierr.Error).ErrCode == apierr.ErrInternalError {
|
if err.(apierr.Error).ErrCode == apierr.ErrInternalError {
|
||||||
err = apierr.GetAPIError(apierr.ErrAccessDenied)
|
err = apierr.GetAPIError(apierr.ErrAccessDenied)
|
||||||
}
|
}
|
||||||
if _, wrErr := WriteErrorResponse(w, GetReqInfo(r.Context()), err); wrErr != nil {
|
if _, wrErr := WriteErrorResponse(w, GetReqInfo(r.Context()), err); wrErr != nil {
|
||||||
reqLogOrDefault(reqCtx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath))
|
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
|
||||||
}
|
}
|
||||||
span.End()
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
reqCtx = SetBox(reqCtx, box)
|
ctx = SetBox(ctx, box)
|
||||||
|
|
||||||
if box.AccessBox.Gate.BearerToken != nil {
|
if box.AccessBox.Gate.BearerToken != nil {
|
||||||
reqInfo.User = bearer.ResolveIssuer(*box.AccessBox.Gate.BearerToken).String()
|
reqInfo.User = bearer.ResolveIssuer(*box.AccessBox.Gate.BearerToken).String()
|
||||||
}
|
}
|
||||||
reqLogOrDefault(reqCtx, log).Debug(logs.SuccessfulAuth, zap.String("accessKeyID", box.AuthHeaders.AccessKeyID), logs.TagField(logs.TagDatapath))
|
reqLogOrDefault(ctx, log).Debug(logs.SuccessfulAuth, zap.String("accessKeyID", box.AuthHeaders.AccessKeyID))
|
||||||
}
|
}
|
||||||
|
|
||||||
span.End()
|
h.ServeHTTP(w, r.WithContext(ctx))
|
||||||
h.ServeHTTP(w, r.WithContext(reqCtx))
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -94,26 +86,22 @@ type FrostFSIDValidator interface {
|
||||||
func FrostfsIDValidation(frostfsID FrostFSIDValidator, log *zap.Logger) Func {
|
func FrostfsIDValidation(frostfsID FrostFSIDValidator, log *zap.Logger) Func {
|
||||||
return func(h http.Handler) http.Handler {
|
return func(h http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "middleware.FrostfsIDValidation")
|
ctx := r.Context()
|
||||||
|
|
||||||
bd, err := GetBoxData(ctx)
|
bd, err := GetBoxData(ctx)
|
||||||
if err != nil || bd.Gate.BearerToken == nil {
|
if err != nil || bd.Gate.BearerToken == nil {
|
||||||
reqLogOrDefault(ctx, log).Debug(logs.AnonRequestSkipFrostfsIDValidation, logs.TagField(logs.TagDatapath))
|
reqLogOrDefault(ctx, log).Debug(logs.AnonRequestSkipFrostfsIDValidation)
|
||||||
span.End()
|
|
||||||
h.ServeHTTP(w, r)
|
h.ServeHTTP(w, r)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = validateBearerToken(frostfsID, bd.Gate.BearerToken); err != nil {
|
if err = validateBearerToken(frostfsID, bd.Gate.BearerToken); err != nil {
|
||||||
reqLogOrDefault(ctx, log).Error(logs.FrostfsIDValidationFailed, zap.Error(err), logs.TagField(logs.TagDatapath))
|
reqLogOrDefault(ctx, log).Error(logs.FrostfsIDValidationFailed, zap.Error(err))
|
||||||
if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil {
|
if _, wrErr := WriteErrorResponse(w, GetReqInfo(r.Context()), err); wrErr != nil {
|
||||||
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath))
|
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
|
||||||
}
|
}
|
||||||
span.End()
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
span.End()
|
|
||||||
h.ServeHTTP(w, r)
|
h.ServeHTTP(w, r)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,7 +23,7 @@ type (
|
||||||
)
|
)
|
||||||
|
|
||||||
func LogHTTP(l *zap.Logger, _ LogHTTPSettings) Func {
|
func LogHTTP(l *zap.Logger, _ LogHTTPSettings) Func {
|
||||||
l.Warn(logs.LogHTTPDisabledInThisBuild, logs.TagField(logs.TagApp))
|
l.Warn(logs.LogHTTPDisabledInThisBuild)
|
||||||
return func(h http.Handler) http.Handler {
|
return func(h http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
h.ServeHTTP(w, r)
|
h.ServeHTTP(w, r)
|
||||||
|
|
|
@ -139,7 +139,7 @@ func resolveCID(log *zap.Logger, resolveContainerID ContainerIDResolveFunc) cidR
|
||||||
|
|
||||||
containerID, err := resolveContainerID(ctx, reqInfo.BucketName)
|
containerID, err := resolveContainerID(ctx, reqInfo.BucketName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
reqLogOrDefault(ctx, log).Debug(logs.FailedToResolveCID, zap.Error(err), logs.TagField(logs.TagDatapath))
|
reqLogOrDefault(ctx, log).Debug(logs.FailedToResolveCID, zap.Error(err))
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -10,7 +10,6 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
|
@ -89,35 +88,32 @@ type PolicyConfig struct {
|
||||||
func PolicyCheck(cfg PolicyConfig) Func {
|
func PolicyCheck(cfg PolicyConfig) Func {
|
||||||
return func(h http.Handler) http.Handler {
|
return func(h http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx, span := tracing.StartSpanFromContext(r.Context(), "middleware.PolicyCheck")
|
ctx := r.Context()
|
||||||
|
if err := policyCheck(r, cfg); err != nil {
|
||||||
if err := policyCheck(ctx, r, cfg); err != nil {
|
reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err))
|
||||||
reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err), logs.TagField(logs.TagDatapath))
|
|
||||||
err = apierr.TransformToS3Error(err)
|
err = apierr.TransformToS3Error(err)
|
||||||
if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil {
|
if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil {
|
||||||
reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath))
|
reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
|
||||||
}
|
}
|
||||||
span.End()
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
span.End()
|
|
||||||
h.ServeHTTP(w, r)
|
h.ServeHTTP(w, r)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func policyCheck(ctx context.Context, r *http.Request, cfg PolicyConfig) error {
|
func policyCheck(r *http.Request, cfg PolicyConfig) error {
|
||||||
reqInfo := GetReqInfo(ctx)
|
reqInfo := GetReqInfo(r.Context())
|
||||||
|
|
||||||
req, userKey, userGroups, err := getPolicyRequest(ctx, r, cfg, reqInfo.RequestType, reqInfo.BucketName, reqInfo.ObjectName)
|
req, userKey, userGroups, err := getPolicyRequest(r, cfg, reqInfo.RequestType, reqInfo.BucketName, reqInfo.ObjectName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var bktInfo *data.BucketInfo
|
var bktInfo *data.BucketInfo
|
||||||
if reqInfo.RequestType != noneType && !strings.HasSuffix(req.Operation(), CreateBucketOperation) {
|
if reqInfo.RequestType != noneType && !strings.HasSuffix(req.Operation(), CreateBucketOperation) {
|
||||||
bktInfo, err = cfg.BucketResolver(ctx, reqInfo.BucketName)
|
bktInfo, err = cfg.BucketResolver(r.Context(), reqInfo.BucketName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -165,7 +161,7 @@ func policyCheck(ctx context.Context, r *http.Request, cfg PolicyConfig) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getPolicyRequest(ctx context.Context, r *http.Request, cfg PolicyConfig, reqType ReqType, bktName string, objName string) (*testutil.Request, *keys.PublicKey, []string, error) {
|
func getPolicyRequest(r *http.Request, cfg PolicyConfig, reqType ReqType, bktName string, objName string) (*testutil.Request, *keys.PublicKey, []string, error) {
|
||||||
var (
|
var (
|
||||||
owner string
|
owner string
|
||||||
groups []string
|
groups []string
|
||||||
|
@ -173,6 +169,7 @@ func getPolicyRequest(ctx context.Context, r *http.Request, cfg PolicyConfig, re
|
||||||
pk *keys.PublicKey
|
pk *keys.PublicKey
|
||||||
)
|
)
|
||||||
|
|
||||||
|
ctx := r.Context()
|
||||||
bd, err := GetBoxData(ctx)
|
bd, err := GetBoxData(ctx)
|
||||||
if err == nil && bd.Gate.BearerToken != nil {
|
if err == nil && bd.Gate.BearerToken != nil {
|
||||||
pk, err = keys.NewPublicKeyFromBytes(bd.Gate.BearerToken.SigningKeyBytes(), elliptic.P256())
|
pk, err = keys.NewPublicKeyFromBytes(bd.Gate.BearerToken.SigningKeyBytes(), elliptic.P256())
|
||||||
|
@ -196,16 +193,14 @@ func getPolicyRequest(ctx context.Context, r *http.Request, cfg PolicyConfig, re
|
||||||
res = fmt.Sprintf(s3.ResourceFormatS3Bucket, bktName)
|
res = fmt.Sprintf(s3.ResourceFormatS3Bucket, bktName)
|
||||||
}
|
}
|
||||||
|
|
||||||
requestProps, resourceProps, err := determineProperties(ctx, r, cfg.Decoder, cfg.BucketResolver, cfg.Tagging, reqType, op, bktName, objName, owner, groups, tags)
|
requestProps, resourceProps, err := determineProperties(r, cfg.Decoder, cfg.BucketResolver, cfg.Tagging, reqType, op, bktName, objName, owner, groups, tags)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, fmt.Errorf("determine properties: %w", err)
|
return nil, nil, nil, fmt.Errorf("determine properties: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
reqLogOrDefault(r.Context(), cfg.Log).Debug(logs.PolicyRequest, zap.String("action", op),
|
reqLogOrDefault(r.Context(), cfg.Log).Debug(logs.PolicyRequest, zap.String("action", op),
|
||||||
zap.String("resource", res), zap.Any("request properties", requestProps),
|
zap.String("resource", res), zap.Any("request properties", requestProps),
|
||||||
zap.Any("resource properties", resourceProps),
|
zap.Any("resource properties", resourceProps))
|
||||||
logs.TagField(logs.TagDatapath),
|
|
||||||
)
|
|
||||||
|
|
||||||
return testutil.NewRequest(op, testutil.NewResource(res, resourceProps), requestProps), pk, groups, nil
|
return testutil.NewRequest(op, testutil.NewResource(res, resourceProps), requestProps), pk, groups, nil
|
||||||
}
|
}
|
||||||
|
@ -423,7 +418,7 @@ func determineGeneralOperation(r *http.Request) string {
|
||||||
return "UnmatchedOperation"
|
return "UnmatchedOperation"
|
||||||
}
|
}
|
||||||
|
|
||||||
func determineProperties(ctx context.Context, r *http.Request, decoder XMLDecoder, resolver BucketResolveFunc, tagging ResourceTagging, reqType ReqType,
|
func determineProperties(r *http.Request, decoder XMLDecoder, resolver BucketResolveFunc, tagging ResourceTagging, reqType ReqType,
|
||||||
op, bktName, objName, owner string, groups []string, userClaims map[string]string) (requestProperties map[string]string, resourceProperties map[string]string, err error) {
|
op, bktName, objName, owner string, groups []string, userClaims map[string]string) (requestProperties map[string]string, resourceProperties map[string]string, err error) {
|
||||||
requestProperties = map[string]string{
|
requestProperties = map[string]string{
|
||||||
s3.PropertyKeyOwner: owner,
|
s3.PropertyKeyOwner: owner,
|
||||||
|
@ -471,7 +466,7 @@ func determineProperties(ctx context.Context, r *http.Request, decoder XMLDecode
|
||||||
requestProperties[k] = v
|
requestProperties[k] = v
|
||||||
}
|
}
|
||||||
|
|
||||||
resourceProperties, err = determineResourceTags(ctx, reqType, op, bktName, objName, queries.Get(QueryVersionID), resolver, tagging)
|
resourceProperties, err = determineResourceTags(r.Context(), reqType, op, bktName, objName, queries.Get(QueryVersionID), resolver, tagging)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("determine resource tags: %w", err)
|
return nil, nil, fmt.Errorf("determine resource tags: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -166,7 +166,7 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
|
||||||
// generate random UUIDv4
|
// generate random UUIDv4
|
||||||
id, err := uuid.NewRandom()
|
id, err := uuid.NewRandom()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(logs.FailedToGenerateRequestID, zap.Error(err), logs.TagField(logs.TagDatapath))
|
log.Error(logs.FailedToGenerateRequestID, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
// set request id into response header
|
// set request id into response header
|
||||||
|
@ -198,8 +198,7 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
|
||||||
r = r.WithContext(SetReqLogger(ctx, reqLogger))
|
r = r.WithContext(SetReqLogger(ctx, reqLogger))
|
||||||
|
|
||||||
reqLogger.Info(logs.RequestStart, zap.String("host", r.Host),
|
reqLogger.Info(logs.RequestStart, zap.String("host", r.Host),
|
||||||
zap.String("remote_host", reqInfo.RemoteHost), zap.String("namespace", reqInfo.Namespace),
|
zap.String("remote_host", reqInfo.RemoteHost), zap.String("namespace", reqInfo.Namespace))
|
||||||
logs.TagField(logs.TagDatapath))
|
|
||||||
|
|
||||||
// continue execution
|
// continue execution
|
||||||
h.ServeHTTP(lw, r)
|
h.ServeHTTP(lw, r)
|
||||||
|
|
|
@ -144,17 +144,6 @@ func WriteErrorResponse(w http.ResponseWriter, reqInfo *ReqInfo, err error) (int
|
||||||
return code, nil
|
return code, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteErrorResponseNoHeader writes XML encoded error to the response body.
|
|
||||||
func WriteErrorResponseNoHeader(w http.ResponseWriter, reqInfo *ReqInfo, err error) error {
|
|
||||||
errorResponse := getAPIErrorResponse(reqInfo, err)
|
|
||||||
encodedErrorResponse, err := EncodeResponseNoHeader(errorResponse)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return WriteResponseBody(w, encodedErrorResponse)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write http common headers.
|
// Write http common headers.
|
||||||
func setCommonHeaders(w http.ResponseWriter) {
|
func setCommonHeaders(w http.ResponseWriter) {
|
||||||
w.Header().Set(hdrServerInfo, version.Server)
|
w.Header().Set(hdrServerInfo, version.Server)
|
||||||
|
@ -211,18 +200,6 @@ func EncodeResponse(response interface{}) ([]byte, error) {
|
||||||
return bytesBuffer.Bytes(), nil
|
return bytesBuffer.Bytes(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// EncodeResponseNoHeader encodes response without setting xml.Header.
|
|
||||||
// Should be used with periodicXMLWriter which sends xml.Header to the client
|
|
||||||
// with whitespaces to keep connection alive.
|
|
||||||
func EncodeResponseNoHeader(response interface{}) ([]byte, error) {
|
|
||||||
var bytesBuffer bytes.Buffer
|
|
||||||
if err := xml.NewEncoder(&bytesBuffer).Encode(response); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return bytesBuffer.Bytes(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// EncodeToResponse encodes the response into ResponseWriter.
|
// EncodeToResponse encodes the response into ResponseWriter.
|
||||||
func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
|
func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
|
@ -354,7 +331,7 @@ func LogSuccessResponse(l *zap.Logger) Func {
|
||||||
fields = append(fields, zap.String("user", reqInfo.User))
|
fields = append(fields, zap.String("user", reqInfo.User))
|
||||||
}
|
}
|
||||||
|
|
||||||
reqLogger.Info(logs.RequestEnd, append(fields, logs.TagField(logs.TagDatapath))...)
|
reqLogger.Info(logs.RequestEnd, fields...)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -245,14 +245,13 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
zap.String("method", reqInfo.API),
|
zap.String("method", reqInfo.API),
|
||||||
zap.String("http method", r.Method),
|
zap.String("http method", r.Method),
|
||||||
zap.String("url", r.RequestURI),
|
zap.String("url", r.RequestURI),
|
||||||
logs.TagField(logs.TagDatapath),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if wrErr != nil {
|
if wrErr != nil {
|
||||||
fields = append(fields, zap.NamedError("write_response_error", wrErr))
|
fields = append(fields, zap.NamedError("write_response_error", wrErr))
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Error(logs.RequestUnmatched, append(fields, logs.TagField(logs.TagDatapath))...)
|
log.Error(logs.RequestUnmatched, fields...)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -267,14 +266,13 @@ func notSupportedHandler() http.HandlerFunc {
|
||||||
fields := []zap.Field{
|
fields := []zap.Field{
|
||||||
zap.String("http method", r.Method),
|
zap.String("http method", r.Method),
|
||||||
zap.String("url", r.RequestURI),
|
zap.String("url", r.RequestURI),
|
||||||
logs.TagField(logs.TagDatapath),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if wrErr != nil {
|
if wrErr != nil {
|
||||||
fields = append(fields, zap.NamedError("write_response_error", wrErr))
|
fields = append(fields, zap.NamedError("write_response_error", wrErr))
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Error(logs.NotSupported, append(fields, logs.TagField(logs.TagDatapath))...)
|
log.Error(logs.NotSupported, fields...)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -45,7 +45,7 @@ type centerMock struct {
|
||||||
key *keys.PrivateKey
|
key *keys.PrivateKey
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *centerMock) Authenticate(context.Context, *http.Request) (*middleware.Box, error) {
|
func (c *centerMock) Authenticate(*http.Request) (*middleware.Box, error) {
|
||||||
if c.noAuthHeader {
|
if c.noAuthHeader {
|
||||||
return nil, middleware.ErrNoAuthorizationHeader
|
return nil, middleware.ErrNoAuthorizationHeader
|
||||||
}
|
}
|
||||||
|
|
179
cmd/s3-gw/app.go
179
cmd/s3-gw/app.go
|
@ -20,7 +20,6 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-contract/commonclient"
|
"git.frostfs.info/TrueCloudLab/frostfs-contract/commonclient"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
||||||
grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc"
|
grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc"
|
||||||
qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
|
@ -54,7 +53,6 @@ import (
|
||||||
"github.com/panjf2000/ants/v2"
|
"github.com/panjf2000/ants/v2"
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
"go.uber.org/zap/zapcore"
|
|
||||||
"golang.org/x/exp/slices"
|
"golang.org/x/exp/slices"
|
||||||
"golang.org/x/text/encoding/ianaindex"
|
"golang.org/x/text/encoding/ianaindex"
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
|
@ -93,10 +91,6 @@ type (
|
||||||
wrkDone chan struct{}
|
wrkDone chan struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
tagsConfig struct {
|
|
||||||
tagLogs sync.Map
|
|
||||||
}
|
|
||||||
|
|
||||||
loggerSettings struct {
|
loggerSettings struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
appMetrics *metrics.AppMetrics
|
appMetrics *metrics.AppMetrics
|
||||||
|
@ -105,7 +99,6 @@ type (
|
||||||
appSettings struct {
|
appSettings struct {
|
||||||
logLevel zap.AtomicLevel
|
logLevel zap.AtomicLevel
|
||||||
httpLogging s3middleware.LogHTTPConfig
|
httpLogging s3middleware.LogHTTPConfig
|
||||||
tagsConfig *tagsConfig
|
|
||||||
maxClient maxClientsConfig
|
maxClient maxClientsConfig
|
||||||
defaultMaxAge int
|
defaultMaxAge int
|
||||||
reconnectInterval time.Duration
|
reconnectInterval time.Duration
|
||||||
|
@ -139,61 +132,19 @@ type (
|
||||||
tombstoneMembersSize int
|
tombstoneMembersSize int
|
||||||
tombstoneLifetime uint64
|
tombstoneLifetime uint64
|
||||||
tlsTerminationHeader string
|
tlsTerminationHeader string
|
||||||
listingKeepaliveThrottle time.Duration
|
|
||||||
}
|
}
|
||||||
|
|
||||||
maxClientsConfig struct {
|
maxClientsConfig struct {
|
||||||
deadline time.Duration
|
deadline time.Duration
|
||||||
count int
|
count int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Logger struct {
|
||||||
|
logger *zap.Logger
|
||||||
|
lvl zap.AtomicLevel
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
func (t *tagsConfig) LevelEnabled(tag string, tgtLevel zapcore.Level) bool {
|
|
||||||
lvl, ok := t.tagLogs.Load(tag)
|
|
||||||
if !ok {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return lvl.(zapcore.Level).Enabled(tgtLevel)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tagsConfig) update(cfg *viper.Viper) error {
|
|
||||||
tags, err := fetchLogTagsConfig(cfg)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
t.tagLogs.Range(func(key, value any) bool {
|
|
||||||
k := key.(string)
|
|
||||||
v := value.(zapcore.Level)
|
|
||||||
|
|
||||||
if lvl, ok := tags[k]; ok {
|
|
||||||
if lvl != v {
|
|
||||||
t.tagLogs.Store(key, lvl)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
t.tagLogs.Delete(key)
|
|
||||||
delete(tags, k)
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
})
|
|
||||||
|
|
||||||
for k, v := range tags {
|
|
||||||
t.tagLogs.Store(k, v)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTagsConfig(v *viper.Viper) *tagsConfig {
|
|
||||||
var t tagsConfig
|
|
||||||
if err := t.update(v); err != nil {
|
|
||||||
// panic here is analogue of the similar panic during common log level initialization.
|
|
||||||
panic(err.Error())
|
|
||||||
}
|
|
||||||
return &t
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *loggerSettings) DroppedLogsInc() {
|
func (s *loggerSettings) DroppedLogsInc() {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
defer s.mu.RUnlock()
|
defer s.mu.RUnlock()
|
||||||
|
@ -212,10 +163,8 @@ func (s *loggerSettings) setMetrics(appMetrics *metrics.AppMetrics) {
|
||||||
|
|
||||||
func newApp(ctx context.Context, cfg *appCfg) *App {
|
func newApp(ctx context.Context, cfg *appCfg) *App {
|
||||||
logSettings := &loggerSettings{}
|
logSettings := &loggerSettings{}
|
||||||
tagConfig := newTagsConfig(cfg.config())
|
log := pickLogger(cfg.config(), logSettings)
|
||||||
log := pickLogger(cfg.config(), logSettings, tagConfig)
|
|
||||||
settings := newAppSettings(log, cfg.config())
|
settings := newAppSettings(log, cfg.config())
|
||||||
settings.tagsConfig = tagConfig
|
|
||||||
appCache := layer.NewCache(getCacheOptions(cfg.config(), log.logger))
|
appCache := layer.NewCache(getCacheOptions(cfg.config(), log.logger))
|
||||||
|
|
||||||
app := &App{
|
app := &App{
|
||||||
|
@ -256,7 +205,7 @@ func (a *App) initAuthCenter(ctx context.Context) {
|
||||||
if a.config().IsSet(cfgContainersAccessBox) {
|
if a.config().IsSet(cfgContainersAccessBox) {
|
||||||
cnrID, err := a.resolveContainerID(ctx, cfgContainersAccessBox)
|
cnrID, err := a.resolveContainerID(ctx, cfgContainersAccessBox)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal(logs.CouldNotFetchAccessBoxContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.CouldNotFetchAccessBoxContainerInfo, zap.Error(err))
|
||||||
}
|
}
|
||||||
a.settings.accessbox = &cnrID
|
a.settings.accessbox = &cnrID
|
||||||
}
|
}
|
||||||
|
@ -275,7 +224,7 @@ func (a *App) initLayer(ctx context.Context) {
|
||||||
// prepare random key for anonymous requests
|
// prepare random key for anonymous requests
|
||||||
randomKey, err := keys.NewPrivateKey()
|
randomKey, err := keys.NewPrivateKey()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal(logs.CouldntGenerateRandomKey, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.CouldntGenerateRandomKey, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
var gateOwner user.ID
|
var gateOwner user.ID
|
||||||
|
@ -285,7 +234,7 @@ func (a *App) initLayer(ctx context.Context) {
|
||||||
if a.config().IsSet(cfgContainersCORS) {
|
if a.config().IsSet(cfgContainersCORS) {
|
||||||
corsCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersCORS)
|
corsCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersCORS)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal(logs.CouldNotFetchCORSContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.CouldNotFetchCORSContainerInfo, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -293,7 +242,7 @@ func (a *App) initLayer(ctx context.Context) {
|
||||||
if a.config().IsSet(cfgContainersLifecycle) {
|
if a.config().IsSet(cfgContainersLifecycle) {
|
||||||
lifecycleCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersLifecycle)
|
lifecycleCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersLifecycle)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal(logs.CouldNotFetchLifecycleContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.CouldNotFetchLifecycleContainerInfo, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -319,7 +268,7 @@ func (a *App) initLayer(ctx context.Context) {
|
||||||
func (a *App) initWorkerPool() *ants.Pool {
|
func (a *App) initWorkerPool() *ants.Pool {
|
||||||
workerPool, err := ants.NewPool(a.settings.workerPoolSize)
|
workerPool, err := ants.NewPool(a.settings.workerPoolSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err))
|
||||||
}
|
}
|
||||||
return workerPool
|
return workerPool
|
||||||
}
|
}
|
||||||
|
@ -328,7 +277,6 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
|
||||||
settings := &appSettings{
|
settings := &appSettings{
|
||||||
logLevel: log.lvl,
|
logLevel: log.lvl,
|
||||||
httpLogging: s3middleware.LogHTTPConfig{},
|
httpLogging: s3middleware.LogHTTPConfig{},
|
||||||
tagsConfig: newTagsConfig(v),
|
|
||||||
maxClient: newMaxClients(v),
|
maxClient: newMaxClients(v),
|
||||||
defaultMaxAge: fetchDefaultMaxAge(v, log.logger),
|
defaultMaxAge: fetchDefaultMaxAge(v, log.logger),
|
||||||
reconnectInterval: fetchReconnectInterval(v),
|
reconnectInterval: fetchReconnectInterval(v),
|
||||||
|
@ -375,7 +323,6 @@ func (s *appSettings) update(v *viper.Viper, log *zap.Logger) {
|
||||||
tombstoneMembersSize := fetchTombstoneMembersSize(v)
|
tombstoneMembersSize := fetchTombstoneMembersSize(v)
|
||||||
tombstoneLifetime := fetchTombstoneLifetime(v)
|
tombstoneLifetime := fetchTombstoneLifetime(v)
|
||||||
tlsTerminationHeader := v.GetString(cfgEncryptionTLSTerminationHeader)
|
tlsTerminationHeader := v.GetString(cfgEncryptionTLSTerminationHeader)
|
||||||
listingKeepaliveThrottle := v.GetDuration(cfgKludgeListingKeepAliveThrottle)
|
|
||||||
|
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
@ -409,7 +356,6 @@ func (s *appSettings) update(v *viper.Viper, log *zap.Logger) {
|
||||||
s.tombstoneMembersSize = tombstoneMembersSize
|
s.tombstoneMembersSize = tombstoneMembersSize
|
||||||
s.tombstoneLifetime = tombstoneLifetime
|
s.tombstoneLifetime = tombstoneLifetime
|
||||||
s.tlsTerminationHeader = tlsTerminationHeader
|
s.tlsTerminationHeader = tlsTerminationHeader
|
||||||
s.listingKeepaliveThrottle = listingKeepaliveThrottle
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *appSettings) prepareVHSNamespaces(v *viper.Viper, log *zap.Logger, defaultNamespaces []string) map[string]bool {
|
func (s *appSettings) prepareVHSNamespaces(v *viper.Viper, log *zap.Logger, defaultNamespaces []string) map[string]bool {
|
||||||
|
@ -647,12 +593,6 @@ func (s *appSettings) TombstoneLifetime() uint64 {
|
||||||
return s.tombstoneLifetime
|
return s.tombstoneLifetime
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *appSettings) ListingKeepaliveThrottle() time.Duration {
|
|
||||||
s.mu.RLock()
|
|
||||||
defer s.mu.RUnlock()
|
|
||||||
return s.listingKeepaliveThrottle
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *App) initAPI(ctx context.Context) {
|
func (a *App) initAPI(ctx context.Context) {
|
||||||
a.initLayer(ctx)
|
a.initLayer(ctx)
|
||||||
a.initHandler()
|
a.initHandler()
|
||||||
|
@ -683,7 +623,7 @@ func (a *App) initFrostfsID(ctx context.Context) {
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal(logs.InitFrostfsIDFailed, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.InitFrostfsIDContractFailed, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
a.frostfsid, err = frostfsid.NewFrostFSID(frostfsid.Config{
|
a.frostfsid, err = frostfsid.NewFrostFSID(frostfsid.Config{
|
||||||
|
@ -692,7 +632,7 @@ func (a *App) initFrostfsID(ctx context.Context) {
|
||||||
Logger: a.log,
|
Logger: a.log,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal(logs.InitFrostfsIDFailed, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.InitFrostfsIDContractFailed, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -708,7 +648,7 @@ func (a *App) initPolicyStorage(ctx context.Context) {
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal(logs.InitPolicyContractFailed, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.InitPolicyContractFailed, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
a.policyStorage = policy.NewStorage(policy.StorageConfig{
|
a.policyStorage = policy.NewStorage(policy.StorageConfig{
|
||||||
|
@ -722,7 +662,7 @@ func (a *App) initResolver() {
|
||||||
var err error
|
var err error
|
||||||
a.bucketResolver, err = resolver.NewBucketResolver(a.getResolverOrder(), a.getResolverConfig())
|
a.bucketResolver, err = resolver.NewBucketResolver(a.getResolverOrder(), a.getResolverConfig())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -737,11 +677,11 @@ func (a *App) getResolverOrder() []string {
|
||||||
order := a.config().GetStringSlice(cfgResolveOrder)
|
order := a.config().GetStringSlice(cfgResolveOrder)
|
||||||
if a.config().GetString(cfgRPCEndpoint) == "" {
|
if a.config().GetString(cfgRPCEndpoint) == "" {
|
||||||
order = remove(order, resolver.NNSResolver)
|
order = remove(order, resolver.NNSResolver)
|
||||||
a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided, logs.TagField(logs.TagApp))
|
a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(order) == 0 {
|
if len(order) == 0 {
|
||||||
a.log.Info(logs.ContainerResolverWillBeDisabled, logs.TagField(logs.TagApp))
|
a.log.Info(logs.ContainerResolverWillBeDisabled)
|
||||||
}
|
}
|
||||||
|
|
||||||
return order
|
return order
|
||||||
|
@ -764,13 +704,13 @@ func (a *App) initTracing(ctx context.Context) {
|
||||||
if trustedCa := a.config().GetString(cfgTracingTrustedCa); trustedCa != "" {
|
if trustedCa := a.config().GetString(cfgTracingTrustedCa); trustedCa != "" {
|
||||||
caBytes, err := os.ReadFile(trustedCa)
|
caBytes, err := os.ReadFile(trustedCa)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
certPool := x509.NewCertPool()
|
certPool := x509.NewCertPool()
|
||||||
ok := certPool.AppendCertsFromPEM(caBytes)
|
ok := certPool.AppendCertsFromPEM(caBytes)
|
||||||
if !ok {
|
if !ok {
|
||||||
a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert"), logs.TagField(logs.TagApp))
|
a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
cfg.ServerCaCertPool = certPool
|
cfg.ServerCaCertPool = certPool
|
||||||
|
@ -778,17 +718,17 @@ func (a *App) initTracing(ctx context.Context) {
|
||||||
|
|
||||||
attributes, err := fetchTracingAttributes(a.config())
|
attributes, err := fetchTracingAttributes(a.config())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
cfg.Attributes = attributes
|
cfg.Attributes = attributes
|
||||||
|
|
||||||
updated, err := tracing.Setup(ctx, cfg)
|
updated, err := tracing.Setup(ctx, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err))
|
||||||
}
|
}
|
||||||
if updated {
|
if updated {
|
||||||
a.log.Info(logs.TracingConfigUpdated, logs.TagField(logs.TagApp))
|
a.log.Info(logs.TracingConfigUpdated)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -798,7 +738,7 @@ func (a *App) shutdownTracing() {
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
if err := tracing.Shutdown(shdnCtx); err != nil {
|
if err := tracing.Shutdown(shdnCtx); err != nil {
|
||||||
a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -815,7 +755,7 @@ func newMaxClients(cfg *viper.Viper) maxClientsConfig {
|
||||||
func getDialerSource(logger *zap.Logger, cfg *viper.Viper) *internalnet.DialerSource {
|
func getDialerSource(logger *zap.Logger, cfg *viper.Viper) *internalnet.DialerSource {
|
||||||
source, err := internalnet.NewDialerSource(fetchMultinetConfig(cfg, logger))
|
source, err := internalnet.NewDialerSource(fetchMultinetConfig(cfg, logger))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err), logs.TagField(logs.TagApp))
|
logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err))
|
||||||
}
|
}
|
||||||
return source
|
return source
|
||||||
}
|
}
|
||||||
|
@ -827,12 +767,12 @@ func (a *App) initPools(ctx context.Context) {
|
||||||
password := wallet.GetPassword(a.config(), cfgWalletPassphrase)
|
password := wallet.GetPassword(a.config(), cfgWalletPassphrase)
|
||||||
key, err := wallet.GetKeyFromPath(a.config().GetString(cfgWalletPath), a.config().GetString(cfgWalletAddress), password)
|
key, err := wallet.GetKeyFromPath(a.config().GetString(cfgWalletPath), a.config().GetString(cfgWalletAddress), password)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
prm.SetKey(&key.PrivateKey)
|
prm.SetKey(&key.PrivateKey)
|
||||||
prmTree.SetKey(key)
|
prmTree.SetKey(key)
|
||||||
a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())), logs.TagField(logs.TagApp))
|
a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())))
|
||||||
|
|
||||||
for _, peer := range fetchPeers(a.log, a.config()) {
|
for _, peer := range fetchPeers(a.log, a.config()) {
|
||||||
prm.AddNode(peer)
|
prm.AddNode(peer)
|
||||||
|
@ -859,8 +799,8 @@ func (a *App) initPools(ctx context.Context) {
|
||||||
|
|
||||||
prm.SetGracefulCloseOnSwitchTimeout(fetchSetGracefulCloseOnSwitchTimeout(a.config()))
|
prm.SetGracefulCloseOnSwitchTimeout(fetchSetGracefulCloseOnSwitchTimeout(a.config()))
|
||||||
|
|
||||||
prm.SetLogger(a.log.With(logs.TagField(logs.TagDatapath)))
|
prm.SetLogger(a.log)
|
||||||
prmTree.SetLogger(a.log.With(logs.TagField(logs.TagDatapath)))
|
prmTree.SetLogger(a.log)
|
||||||
|
|
||||||
prmTree.SetMaxRequestAttempts(a.config().GetInt(cfgTreePoolMaxAttempts))
|
prmTree.SetMaxRequestAttempts(a.config().GetInt(cfgTreePoolMaxAttempts))
|
||||||
|
|
||||||
|
@ -868,19 +808,17 @@ func (a *App) initPools(ctx context.Context) {
|
||||||
grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()),
|
grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()),
|
||||||
grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()),
|
grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()),
|
||||||
grpc.WithContextDialer(a.settings.dialerSource.GrpcContextDialer()),
|
grpc.WithContextDialer(a.settings.dialerSource.GrpcContextDialer()),
|
||||||
grpc.WithChainUnaryInterceptor(qostagging.NewUnaryClientInteceptor()),
|
|
||||||
grpc.WithChainStreamInterceptor(qostagging.NewStreamClientInterceptor()),
|
|
||||||
}
|
}
|
||||||
prm.SetGRPCDialOptions(interceptors...)
|
prm.SetGRPCDialOptions(interceptors...)
|
||||||
prmTree.SetGRPCDialOptions(interceptors...)
|
prmTree.SetGRPCDialOptions(interceptors...)
|
||||||
|
|
||||||
p, err := pool.NewPool(prm)
|
p, err := pool.NewPool(prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = p.Dial(ctx); err != nil {
|
if err = p.Dial(ctx); err != nil {
|
||||||
a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if a.config().GetBool(cfgTreePoolNetmapSupport) {
|
if a.config().GetBool(cfgTreePoolNetmapSupport) {
|
||||||
|
@ -889,10 +827,10 @@ func (a *App) initPools(ctx context.Context) {
|
||||||
|
|
||||||
treePool, err := treepool.NewPool(prmTree)
|
treePool, err := treepool.NewPool(prmTree)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err))
|
||||||
}
|
}
|
||||||
if err = treePool.Dial(ctx); err != nil {
|
if err = treePool.Dial(ctx); err != nil {
|
||||||
a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
a.treePool = treePool
|
a.treePool = treePool
|
||||||
|
@ -918,7 +856,6 @@ func (a *App) Wait() {
|
||||||
a.log.Info(logs.ApplicationStarted,
|
a.log.Info(logs.ApplicationStarted,
|
||||||
zap.String("name", "frostfs-s3-gw"),
|
zap.String("name", "frostfs-s3-gw"),
|
||||||
zap.String("version", version.Version),
|
zap.String("version", version.Version),
|
||||||
logs.TagField(logs.TagApp),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
a.metrics.State().SetVersion(version.Version)
|
a.metrics.State().SetVersion(version.Version)
|
||||||
|
@ -926,7 +863,7 @@ func (a *App) Wait() {
|
||||||
|
|
||||||
<-a.webDone // wait for web-server to be stopped
|
<-a.webDone // wait for web-server to be stopped
|
||||||
|
|
||||||
a.log.Info(logs.ApplicationFinished, logs.TagField(logs.TagApp))
|
a.log.Info(logs.ApplicationFinished)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) setHealthStatus() {
|
func (a *App) setHealthStatus() {
|
||||||
|
@ -972,11 +909,11 @@ func (a *App) Serve(ctx context.Context) {
|
||||||
|
|
||||||
for i := range servs {
|
for i := range servs {
|
||||||
go func(i int) {
|
go func(i int) {
|
||||||
a.log.Info(logs.StartingServer, zap.String("address", servs[i].Address()), logs.TagField(logs.TagApp))
|
a.log.Info(logs.StartingServer, zap.String("address", servs[i].Address()))
|
||||||
|
|
||||||
if err := srv.Serve(servs[i].Listener()); err != nil && err != http.ErrServerClosed {
|
if err := srv.Serve(servs[i].Listener()); err != nil && err != http.ErrServerClosed {
|
||||||
a.metrics.MarkUnhealthy(servs[i].Address())
|
a.metrics.MarkUnhealthy(servs[i].Address())
|
||||||
a.log.Fatal(logs.ListenAndServe, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.ListenAndServe, zap.Error(err))
|
||||||
}
|
}
|
||||||
}(i)
|
}(i)
|
||||||
}
|
}
|
||||||
|
@ -1001,7 +938,7 @@ LOOP:
|
||||||
ctx, cancel := shutdownContext()
|
ctx, cancel := shutdownContext()
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
a.log.Info(logs.StoppingServer, zap.Error(srv.Shutdown(ctx)), logs.TagField(logs.TagApp))
|
a.log.Info(logs.StoppingServer, zap.Error(srv.Shutdown(ctx)))
|
||||||
|
|
||||||
a.metrics.Shutdown()
|
a.metrics.Shutdown()
|
||||||
a.stopServices()
|
a.stopServices()
|
||||||
|
@ -1015,23 +952,23 @@ func shutdownContext() (context.Context, context.CancelFunc) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) configReload(ctx context.Context) {
|
func (a *App) configReload(ctx context.Context) {
|
||||||
a.log.Info(logs.SIGHUPConfigReloadStarted, logs.TagField(logs.TagApp))
|
a.log.Info(logs.SIGHUPConfigReloadStarted)
|
||||||
|
|
||||||
if !a.config().IsSet(cmdConfig) && !a.config().IsSet(cmdConfigDir) {
|
if !a.config().IsSet(cmdConfig) && !a.config().IsSet(cmdConfigDir) {
|
||||||
a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed, logs.TagField(logs.TagApp))
|
a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := a.cfg.reload(); err != nil {
|
if err := a.cfg.reload(); err != nil {
|
||||||
a.log.Warn(logs.FailedToReloadConfig, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Warn(logs.FailedToReloadConfig, zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := a.bucketResolver.UpdateResolvers(a.getResolverOrder()); err != nil {
|
if err := a.bucketResolver.UpdateResolvers(a.getResolverOrder()); err != nil {
|
||||||
a.log.Warn(logs.FailedToReloadResolvers, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Warn(logs.FailedToReloadResolvers, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := a.updateServers(); err != nil {
|
if err := a.updateServers(); err != nil {
|
||||||
a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
a.setRuntimeParameters()
|
a.setRuntimeParameters()
|
||||||
|
@ -1045,22 +982,18 @@ func (a *App) configReload(ctx context.Context) {
|
||||||
a.initTracing(ctx)
|
a.initTracing(ctx)
|
||||||
a.setHealthStatus()
|
a.setHealthStatus()
|
||||||
|
|
||||||
a.log.Info(logs.SIGHUPConfigReloadCompleted, logs.TagField(logs.TagApp))
|
a.log.Info(logs.SIGHUPConfigReloadCompleted)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) updateSettings() {
|
func (a *App) updateSettings() {
|
||||||
if lvl, err := getLogLevel(a.config()); err != nil {
|
if lvl, err := getLogLevel(a.config()); err != nil {
|
||||||
a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
a.settings.logLevel.SetLevel(lvl)
|
a.settings.logLevel.SetLevel(lvl)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.config(), a.log)); err != nil {
|
if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.config(), a.log)); err != nil {
|
||||||
a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err))
|
||||||
}
|
|
||||||
|
|
||||||
if err := a.settings.tagsConfig.update(a.config()); err != nil {
|
|
||||||
a.log.Warn(logs.TagsLogConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
a.settings.update(a.config(), a.log)
|
a.settings.update(a.config(), a.log)
|
||||||
|
@ -1091,17 +1024,17 @@ func (a *App) initServers(ctx context.Context) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.unbindServers = append(a.unbindServers, serverInfo)
|
a.unbindServers = append(a.unbindServers, serverInfo)
|
||||||
a.metrics.MarkUnhealthy(serverInfo.Address)
|
a.metrics.MarkUnhealthy(serverInfo.Address)
|
||||||
a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err), logs.TagField(logs.TagApp))...)
|
a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
a.metrics.MarkHealthy(serverInfo.Address)
|
a.metrics.MarkHealthy(serverInfo.Address)
|
||||||
|
|
||||||
a.servers = append(a.servers, srv)
|
a.servers = append(a.servers, srv)
|
||||||
a.log.Info(logs.AddServer, append(fields, logs.TagField(logs.TagApp))...)
|
a.log.Info(logs.AddServer, fields...)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(a.servers) == 0 {
|
if len(a.servers) == 0 {
|
||||||
a.log.Fatal(logs.NoHealthyServers, logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.NoHealthyServers)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1205,7 +1138,7 @@ func (a *App) initHandler() {
|
||||||
|
|
||||||
a.api, err = handler.New(a.log, a.obj, a.settings, a.policyStorage, a.frostfsid)
|
a.api, err = handler.New(a.log, a.obj, a.settings, a.policyStorage, a.frostfsid)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal(logs.CouldNotInitializeAPIHandler, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Fatal(logs.CouldNotInitializeAPIHandler, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1237,7 +1170,7 @@ func (a *App) getServers() []Server {
|
||||||
func (a *App) setRuntimeParameters() {
|
func (a *App) setRuntimeParameters() {
|
||||||
if len(os.Getenv("GOMEMLIMIT")) != 0 {
|
if len(os.Getenv("GOMEMLIMIT")) != 0 {
|
||||||
// default limit < yaml limit < app env limit < GOMEMLIMIT
|
// default limit < yaml limit < app env limit < GOMEMLIMIT
|
||||||
a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT, logs.TagField(logs.TagApp))
|
a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1246,9 +1179,7 @@ func (a *App) setRuntimeParameters() {
|
||||||
if softMemoryLimit != previous {
|
if softMemoryLimit != previous {
|
||||||
a.log.Info(logs.RuntimeSoftMemoryLimitUpdated,
|
a.log.Info(logs.RuntimeSoftMemoryLimitUpdated,
|
||||||
zap.Int64("new_value", softMemoryLimit),
|
zap.Int64("new_value", softMemoryLimit),
|
||||||
zap.Int64("old_value", previous),
|
zap.Int64("old_value", previous))
|
||||||
logs.TagField(logs.TagApp),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1274,7 +1205,7 @@ func (a *App) tryReconnect(ctx context.Context, sr *http.Server) bool {
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
defer a.mu.Unlock()
|
defer a.mu.Unlock()
|
||||||
|
|
||||||
a.log.Info(logs.ServerReconnecting, logs.TagField(logs.TagApp))
|
a.log.Info(logs.ServerReconnecting)
|
||||||
var failedServers []ServerInfo
|
var failedServers []ServerInfo
|
||||||
|
|
||||||
for _, serverInfo := range a.unbindServers {
|
for _, serverInfo := range a.unbindServers {
|
||||||
|
@ -1285,23 +1216,23 @@ func (a *App) tryReconnect(ctx context.Context, sr *http.Server) bool {
|
||||||
|
|
||||||
srv, err := newServer(ctx, serverInfo)
|
srv, err := newServer(ctx, serverInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Warn(logs.ServerReconnectFailed, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Warn(logs.ServerReconnectFailed, zap.Error(err))
|
||||||
failedServers = append(failedServers, serverInfo)
|
failedServers = append(failedServers, serverInfo)
|
||||||
a.metrics.MarkUnhealthy(serverInfo.Address)
|
a.metrics.MarkUnhealthy(serverInfo.Address)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
a.log.Info(logs.StartingServer, zap.String("address", srv.Address()), logs.TagField(logs.TagApp))
|
a.log.Info(logs.StartingServer, zap.String("address", srv.Address()))
|
||||||
a.metrics.MarkHealthy(serverInfo.Address)
|
a.metrics.MarkHealthy(serverInfo.Address)
|
||||||
if err = sr.Serve(srv.Listener()); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
if err = sr.Serve(srv.Listener()); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||||
a.log.Warn(logs.ListenAndServe, zap.Error(err), logs.TagField(logs.TagApp))
|
a.log.Warn(logs.ListenAndServe, zap.Error(err))
|
||||||
a.metrics.MarkUnhealthy(serverInfo.Address)
|
a.metrics.MarkUnhealthy(serverInfo.Address)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
a.servers = append(a.servers, srv)
|
a.servers = append(a.servers, srv)
|
||||||
a.log.Info(logs.ServerReconnectedSuccessfully, append(fields, logs.TagField(logs.TagApp))...)
|
a.log.Info(logs.ServerReconnectedSuccessfully, fields...)
|
||||||
}
|
}
|
||||||
|
|
||||||
a.unbindServers = failedServers
|
a.unbindServers = failedServers
|
||||||
|
|
|
@ -21,13 +21,20 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
|
"git.frostfs.info/TrueCloudLab/zapjournald"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
"github.com/ssgreg/journald"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
"go.uber.org/zap/zapcore"
|
"go.uber.org/zap/zapcore"
|
||||||
)
|
)
|
||||||
|
|
||||||
const wildcardPlaceholder = "<wildcard>"
|
const (
|
||||||
|
destinationStdout = "stdout"
|
||||||
|
destinationJournald = "journald"
|
||||||
|
|
||||||
|
wildcardPlaceholder = "<wildcard>"
|
||||||
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
defaultRebalanceInterval = 60 * time.Second
|
defaultRebalanceInterval = 60 * time.Second
|
||||||
|
@ -82,8 +89,7 @@ var (
|
||||||
defaultDefaultNamespaces = []string{"", "root"}
|
defaultDefaultNamespaces = []string{"", "root"}
|
||||||
)
|
)
|
||||||
|
|
||||||
// Settings.
|
const ( // Settings.
|
||||||
const (
|
|
||||||
// Logger.
|
// Logger.
|
||||||
cfgLoggerLevel = "logger.level"
|
cfgLoggerLevel = "logger.level"
|
||||||
cfgLoggerDestination = "logger.destination"
|
cfgLoggerDestination = "logger.destination"
|
||||||
|
@ -93,11 +99,6 @@ const (
|
||||||
cfgLoggerSamplingThereafter = "logger.sampling.thereafter"
|
cfgLoggerSamplingThereafter = "logger.sampling.thereafter"
|
||||||
cfgLoggerSamplingInterval = "logger.sampling.interval"
|
cfgLoggerSamplingInterval = "logger.sampling.interval"
|
||||||
|
|
||||||
cfgLoggerTags = "logger.tags"
|
|
||||||
cfgLoggerTagsPrefixTmpl = cfgLoggerTags + ".%d."
|
|
||||||
cfgLoggerTagsNameTmpl = cfgLoggerTagsPrefixTmpl + "name"
|
|
||||||
cfgLoggerTagsLevelTmpl = cfgLoggerTagsPrefixTmpl + "level"
|
|
||||||
|
|
||||||
// HttpLogging.
|
// HttpLogging.
|
||||||
cfgHTTPLoggingEnabled = "http_logging.enabled"
|
cfgHTTPLoggingEnabled = "http_logging.enabled"
|
||||||
cfgHTTPLoggingMaxBody = "http_logging.max_body"
|
cfgHTTPLoggingMaxBody = "http_logging.max_body"
|
||||||
|
@ -202,8 +203,6 @@ const (
|
||||||
cfgKludgeBypassContentEncodingCheckInChunks = "kludge.bypass_content_encoding_check_in_chunks"
|
cfgKludgeBypassContentEncodingCheckInChunks = "kludge.bypass_content_encoding_check_in_chunks"
|
||||||
cfgKludgeDefaultNamespaces = "kludge.default_namespaces"
|
cfgKludgeDefaultNamespaces = "kludge.default_namespaces"
|
||||||
cfgKludgeProfile = "kludge.profile"
|
cfgKludgeProfile = "kludge.profile"
|
||||||
cfgKludgeListingKeepAliveThrottle = "kludge.listing_keepalive_throttle"
|
|
||||||
|
|
||||||
// Web.
|
// Web.
|
||||||
cfgWebReadTimeout = "web.read_timeout"
|
cfgWebReadTimeout = "web.read_timeout"
|
||||||
cfgWebReadHeaderTimeout = "web.read_header_timeout"
|
cfgWebReadHeaderTimeout = "web.read_header_timeout"
|
||||||
|
@ -471,18 +470,14 @@ func fetchDefaultPolicy(l *zap.Logger, cfg *viper.Viper) netmap.PlacementPolicy
|
||||||
policyStr := cfg.GetString(cfgPolicyDefault)
|
policyStr := cfg.GetString(cfgPolicyDefault)
|
||||||
if err := policy.DecodeString(policyStr); err != nil {
|
if err := policy.DecodeString(policyStr); err != nil {
|
||||||
l.Warn(logs.FailedToParseDefaultLocationConstraint,
|
l.Warn(logs.FailedToParseDefaultLocationConstraint,
|
||||||
zap.String("policy", policyStr), zap.String("default", defaultPlacementPolicy),
|
zap.String("policy", policyStr), zap.String("default", defaultPlacementPolicy), zap.Error(err))
|
||||||
zap.Error(err), logs.TagField(logs.TagApp))
|
|
||||||
} else {
|
} else {
|
||||||
return policy
|
return policy
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := policy.DecodeString(defaultPlacementPolicy); err != nil {
|
if err := policy.DecodeString(defaultPlacementPolicy); err != nil {
|
||||||
l.Fatal(logs.FailedToParseDefaultDefaultLocationConstraint,
|
l.Fatal(logs.FailedToParseDefaultDefaultLocationConstraint, zap.String("policy", defaultPlacementPolicy))
|
||||||
zap.String("policy", defaultPlacementPolicy),
|
|
||||||
logs.TagField(logs.TagApp),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return policy
|
return policy
|
||||||
|
@ -495,9 +490,7 @@ func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultV
|
||||||
l.Error(logs.InvalidLifetimeUsingDefaultValue,
|
l.Error(logs.InvalidLifetimeUsingDefaultValue,
|
||||||
zap.String("parameter", cfgEntry),
|
zap.String("parameter", cfgEntry),
|
||||||
zap.Duration("value in config", lifetime),
|
zap.Duration("value in config", lifetime),
|
||||||
zap.Duration("default", defaultValue),
|
zap.Duration("default", defaultValue))
|
||||||
logs.TagField(logs.TagApp),
|
|
||||||
)
|
|
||||||
} else {
|
} else {
|
||||||
return lifetime
|
return lifetime
|
||||||
}
|
}
|
||||||
|
@ -513,9 +506,7 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue
|
||||||
l.Error(logs.InvalidCacheSizeUsingDefaultValue,
|
l.Error(logs.InvalidCacheSizeUsingDefaultValue,
|
||||||
zap.String("parameter", cfgEntry),
|
zap.String("parameter", cfgEntry),
|
||||||
zap.Int("value in config", size),
|
zap.Int("value in config", size),
|
||||||
zap.Int("default", defaultValue),
|
zap.Int("default", defaultValue))
|
||||||
logs.TagField(logs.TagApp),
|
|
||||||
)
|
|
||||||
} else {
|
} else {
|
||||||
return size
|
return size
|
||||||
}
|
}
|
||||||
|
@ -537,8 +528,7 @@ func fetchRemovingCheckInterval(v *viper.Viper, l *zap.Logger) time.Duration {
|
||||||
l.Error(logs.InvalidAccessBoxCacheRemovingCheckInterval,
|
l.Error(logs.InvalidAccessBoxCacheRemovingCheckInterval,
|
||||||
zap.String("parameter", cfgAccessBoxCacheRemovingCheckInterval),
|
zap.String("parameter", cfgAccessBoxCacheRemovingCheckInterval),
|
||||||
zap.Duration("value in config", duration),
|
zap.Duration("value in config", duration),
|
||||||
zap.Duration("default", defaultAccessBoxCacheRemovingCheckInterval),
|
zap.Duration("default", defaultAccessBoxCacheRemovingCheckInterval))
|
||||||
logs.TagField(logs.TagApp))
|
|
||||||
|
|
||||||
return defaultAccessBoxCacheRemovingCheckInterval
|
return defaultAccessBoxCacheRemovingCheckInterval
|
||||||
}
|
}
|
||||||
|
@ -552,9 +542,7 @@ func fetchDefaultMaxAge(cfg *viper.Viper, l *zap.Logger) int {
|
||||||
if defaultMaxAge <= 0 && defaultMaxAge != -1 {
|
if defaultMaxAge <= 0 && defaultMaxAge != -1 {
|
||||||
l.Fatal(logs.InvalidDefaultMaxAge,
|
l.Fatal(logs.InvalidDefaultMaxAge,
|
||||||
zap.String("parameter", cfgDefaultMaxAge),
|
zap.String("parameter", cfgDefaultMaxAge),
|
||||||
zap.String("value in config", strconv.Itoa(defaultMaxAge)),
|
zap.String("value in config", strconv.Itoa(defaultMaxAge)))
|
||||||
logs.TagField(logs.TagApp),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -565,19 +553,14 @@ func fetchRegionMappingPolicies(l *zap.Logger, cfg *viper.Viper) map[string]netm
|
||||||
filepath := cfg.GetString(cfgPolicyRegionMapFile)
|
filepath := cfg.GetString(cfgPolicyRegionMapFile)
|
||||||
regionPolicyMap, err := readRegionMap(filepath)
|
regionPolicyMap, err := readRegionMap(filepath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
l.Warn(logs.FailedToReadRegionMapFilePolicies,
|
l.Warn(logs.FailedToReadRegionMapFilePolicies, zap.String("file", filepath), zap.Error(err))
|
||||||
zap.String("file", filepath),
|
|
||||||
zap.Error(err),
|
|
||||||
logs.TagField(logs.TagApp))
|
|
||||||
return make(map[string]netmap.PlacementPolicy)
|
return make(map[string]netmap.PlacementPolicy)
|
||||||
}
|
}
|
||||||
|
|
||||||
regionMap := make(map[string]netmap.PlacementPolicy, len(regionPolicyMap))
|
regionMap := make(map[string]netmap.PlacementPolicy, len(regionPolicyMap))
|
||||||
for region, policy := range regionPolicyMap {
|
for region, policy := range regionPolicyMap {
|
||||||
if region == api.DefaultLocationConstraint {
|
if region == api.DefaultLocationConstraint {
|
||||||
l.Warn(logs.DefaultLocationConstraintCantBeOverriden,
|
l.Warn(logs.DefaultLocationConstraintCantBeOverriden, zap.String("policy", policy))
|
||||||
zap.String("policy", policy),
|
|
||||||
logs.TagField(logs.TagApp))
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -592,10 +575,7 @@ func fetchRegionMappingPolicies(l *zap.Logger, cfg *viper.Viper) map[string]netm
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
l.Warn(logs.FailedToParseLocationConstraint,
|
l.Warn(logs.FailedToParseLocationConstraint, zap.String("region", region), zap.String("policy", policy))
|
||||||
zap.String("region", region),
|
|
||||||
zap.String("policy", policy),
|
|
||||||
logs.TagField(logs.TagApp))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return regionMap
|
return regionMap
|
||||||
|
@ -627,11 +607,7 @@ func fetchDefaultCopiesNumbers(l *zap.Logger, v *viper.Viper) []uint32 {
|
||||||
parsedValue, err := strconv.ParseUint(unparsed[i], 10, 32)
|
parsedValue, err := strconv.ParseUint(unparsed[i], 10, 32)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
l.Warn(logs.FailedToParseDefaultCopiesNumbers,
|
l.Warn(logs.FailedToParseDefaultCopiesNumbers,
|
||||||
zap.Strings("copies numbers", unparsed),
|
zap.Strings("copies numbers", unparsed), zap.Uint32s("default", defaultCopiesNumbers), zap.Error(err))
|
||||||
zap.Uint32s("default", defaultCopiesNumbers),
|
|
||||||
zap.Error(err),
|
|
||||||
logs.TagField(logs.TagApp),
|
|
||||||
)
|
|
||||||
return defaultCopiesNumbers
|
return defaultCopiesNumbers
|
||||||
}
|
}
|
||||||
result[i] = uint32(parsedValue)
|
result[i] = uint32(parsedValue)
|
||||||
|
@ -687,17 +663,15 @@ func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) map[string][]uint32 {
|
||||||
for j := range vector {
|
for j := range vector {
|
||||||
parsedValue, err := strconv.ParseUint(vector[j], 10, 32)
|
parsedValue, err := strconv.ParseUint(vector[j], 10, 32)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
l.Warn(logs.FailedToParseCopiesNumbers,
|
l.Warn(logs.FailedToParseCopiesNumbers, zap.String("location", constraint),
|
||||||
zap.String("location", constraint),
|
zap.Strings("copies numbers", vector), zap.Error(err))
|
||||||
zap.Strings("copies numbers", vector), zap.Error(err),
|
|
||||||
logs.TagField(logs.TagApp))
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
vector32[j] = uint32(parsedValue)
|
vector32[j] = uint32(parsedValue)
|
||||||
}
|
}
|
||||||
|
|
||||||
copiesNums[constraint] = vector32
|
copiesNums[constraint] = vector32
|
||||||
l.Info(logs.ConstraintAdded, zap.String("location", constraint), zap.Strings("copies numbers", vector), logs.TagField(logs.TagApp))
|
l.Info(logs.ConstraintAdded, zap.String("location", constraint), zap.Strings("copies numbers", vector))
|
||||||
}
|
}
|
||||||
return copiesNums
|
return copiesNums
|
||||||
}
|
}
|
||||||
|
@ -706,9 +680,7 @@ func fetchDefaultNamespaces(l *zap.Logger, v *viper.Viper) []string {
|
||||||
defaultNamespaces := v.GetStringSlice(cfgKludgeDefaultNamespaces)
|
defaultNamespaces := v.GetStringSlice(cfgKludgeDefaultNamespaces)
|
||||||
if len(defaultNamespaces) == 0 {
|
if len(defaultNamespaces) == 0 {
|
||||||
defaultNamespaces = defaultDefaultNamespaces
|
defaultNamespaces = defaultDefaultNamespaces
|
||||||
l.Warn(logs.DefaultNamespacesCannotBeEmpty,
|
l.Warn(logs.DefaultNamespacesCannotBeEmpty, zap.Strings("namespaces", defaultNamespaces))
|
||||||
zap.Strings("namespaces", defaultNamespaces),
|
|
||||||
logs.TagField(logs.TagApp))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := range defaultNamespaces { // to be set namespaces in env variable as `S3_GW_KLUDGE_DEFAULT_NAMESPACES="" 'root'`
|
for i := range defaultNamespaces { // to be set namespaces in env variable as `S3_GW_KLUDGE_DEFAULT_NAMESPACES="" 'root'`
|
||||||
|
@ -732,7 +704,7 @@ func fetchNamespacesConfig(l *zap.Logger, v *viper.Viper) (NamespacesConfig, []s
|
||||||
|
|
||||||
nsConfig, err := readNamespacesConfig(v.GetString(cfgNamespacesConfig))
|
nsConfig, err := readNamespacesConfig(v.GetString(cfgNamespacesConfig))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
l.Warn(logs.FailedToParseNamespacesConfig, zap.Error(err), logs.TagField(logs.TagApp))
|
l.Warn(logs.FailedToParseNamespacesConfig, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
defaultNamespacesNames := fetchDefaultNamespaces(l, v)
|
defaultNamespacesNames := fetchDefaultNamespaces(l, v)
|
||||||
|
@ -746,13 +718,11 @@ func fetchNamespacesConfig(l *zap.Logger, v *viper.Viper) (NamespacesConfig, []s
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(overrideDefaults) > 0 {
|
if len(overrideDefaults) > 0 {
|
||||||
l.Warn(logs.DefaultNamespaceConfigValuesBeOverwritten, logs.TagField(logs.TagApp))
|
l.Warn(logs.DefaultNamespaceConfigValuesBeOverwritten)
|
||||||
defaultNSValue.LocationConstraints = overrideDefaults[0].LocationConstraints
|
defaultNSValue.LocationConstraints = overrideDefaults[0].LocationConstraints
|
||||||
defaultNSValue.CopiesNumbers = overrideDefaults[0].CopiesNumbers
|
defaultNSValue.CopiesNumbers = overrideDefaults[0].CopiesNumbers
|
||||||
if len(overrideDefaults) > 1 {
|
if len(overrideDefaults) > 1 {
|
||||||
l.Warn(logs.MultipleDefaultOverridesFound,
|
l.Warn(logs.MultipleDefaultOverridesFound, zap.String("name", overrideDefaults[0].Name))
|
||||||
zap.String("name", overrideDefaults[0].Name),
|
|
||||||
logs.TagField(logs.TagApp))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -795,7 +765,7 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
|
||||||
priority := v.GetInt(key + "priority")
|
priority := v.GetInt(key + "priority")
|
||||||
|
|
||||||
if address == "" {
|
if address == "" {
|
||||||
l.Warn(logs.SkipEmptyAddress, logs.TagField(logs.TagApp))
|
l.Warn(logs.SkipEmptyAddress)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if weight <= 0 { // unspecified or wrong
|
if weight <= 0 { // unspecified or wrong
|
||||||
|
@ -810,9 +780,7 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
|
||||||
l.Info(logs.AddedStoragePeer,
|
l.Info(logs.AddedStoragePeer,
|
||||||
zap.Int("priority", priority),
|
zap.Int("priority", priority),
|
||||||
zap.String("address", address),
|
zap.String("address", address),
|
||||||
zap.Float64("weight", weight),
|
zap.Float64("weight", weight))
|
||||||
logs.TagField(logs.TagApp),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nodes
|
return nodes
|
||||||
|
@ -836,7 +804,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo {
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := seen[serverInfo.Address]; ok {
|
if _, ok := seen[serverInfo.Address]; ok {
|
||||||
log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address), logs.TagField(logs.TagApp))
|
log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
seen[serverInfo.Address] = struct{}{}
|
seen[serverInfo.Address] = struct{}{}
|
||||||
|
@ -865,13 +833,13 @@ func fetchVHSNamespaces(v *viper.Viper, log *zap.Logger) map[string]bool {
|
||||||
nsMap := v.GetStringMap(cfgVHSNamespaces)
|
nsMap := v.GetStringMap(cfgVHSNamespaces)
|
||||||
for ns, val := range nsMap {
|
for ns, val := range nsMap {
|
||||||
if _, ok := vhsNamespacesEnabled[ns]; ok {
|
if _, ok := vhsNamespacesEnabled[ns]; ok {
|
||||||
log.Warn(logs.WarnDuplicateNamespaceVHS, zap.String("namespace", ns), logs.TagField(logs.TagApp))
|
log.Warn(logs.WarnDuplicateNamespaceVHS, zap.String("namespace", ns))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
enabledFlag, ok := val.(bool)
|
enabledFlag, ok := val.(bool)
|
||||||
if !ok {
|
if !ok {
|
||||||
log.Warn(logs.WarnValueVHSEnabledFlagWrongType, zap.String("namespace", ns), logs.TagField(logs.TagApp))
|
log.Warn(logs.WarnValueVHSEnabledFlagWrongType, zap.String("namespace", ns))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -955,41 +923,6 @@ func fetchTombstoneWorkerPoolSize(v *viper.Viper) int {
|
||||||
return tombstoneWorkerPoolSize
|
return tombstoneWorkerPoolSize
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchLogTagsConfig(v *viper.Viper) (map[string]zapcore.Level, error) {
|
|
||||||
res := make(map[string]zapcore.Level)
|
|
||||||
|
|
||||||
defaultLevel := v.GetString(cfgLoggerLevel)
|
|
||||||
var defaultLvl zapcore.Level
|
|
||||||
if err := defaultLvl.Set(defaultLevel); err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to parse log level, unknown level: '%s'", defaultLevel)
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := 0; ; i++ {
|
|
||||||
name := v.GetString(fmt.Sprintf(cfgLoggerTagsNameTmpl, i))
|
|
||||||
if name == "" {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
lvl := defaultLvl
|
|
||||||
level := v.GetString(fmt.Sprintf(cfgLoggerTagsLevelTmpl, i))
|
|
||||||
if level != "" {
|
|
||||||
if err := lvl.Set(level); err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to parse log tags config, unknown level: '%s'", level)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
res[name] = lvl
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(res) == 0 && !v.IsSet(cfgLoggerTags) {
|
|
||||||
for _, tag := range defaultTags {
|
|
||||||
res[tag] = defaultLvl
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func newViper(flags *pflag.FlagSet) (*viper.Viper, error) {
|
func newViper(flags *pflag.FlagSet) (*viper.Viper, error) {
|
||||||
v := viper.New()
|
v := viper.New()
|
||||||
|
|
||||||
|
@ -1300,19 +1233,129 @@ type LoggerAppSettings interface {
|
||||||
DroppedLogsInc()
|
DroppedLogsInc()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func pickLogger(v *viper.Viper, settings LoggerAppSettings) *Logger {
|
||||||
|
lvl, err := getLogLevel(v)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
dest := v.GetString(cfgLoggerDestination)
|
||||||
|
|
||||||
|
switch dest {
|
||||||
|
case destinationStdout:
|
||||||
|
return newStdoutLogger(v, lvl, settings)
|
||||||
|
case destinationJournald:
|
||||||
|
return newJournaldLogger(v, lvl, settings)
|
||||||
|
default:
|
||||||
|
panic(fmt.Sprintf("wrong destination for logger: %s", dest))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// newStdoutLogger constructs a Logger instance for the current application.
|
||||||
|
// Panics on failure.
|
||||||
|
//
|
||||||
|
// Logger contains a logger is built from zap's production logging configuration with:
|
||||||
|
// - parameterized level (debug by default)
|
||||||
|
// - console encoding
|
||||||
|
// - ISO8601 time encoding
|
||||||
|
// - sampling intervals
|
||||||
|
//
|
||||||
|
// and atomic log level to dynamically change it.
|
||||||
|
//
|
||||||
|
// Logger records a stack trace for all messages at or above fatal level.
|
||||||
|
//
|
||||||
|
// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace.
|
||||||
|
func newStdoutLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger {
|
||||||
|
stdout := zapcore.AddSync(os.Stderr)
|
||||||
|
level := zap.NewAtomicLevelAt(lvl)
|
||||||
|
|
||||||
|
consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level)
|
||||||
|
|
||||||
|
consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, settings)
|
||||||
|
|
||||||
|
return &Logger{
|
||||||
|
logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))),
|
||||||
|
lvl: level,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newJournaldLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger {
|
||||||
|
level := zap.NewAtomicLevelAt(lvl)
|
||||||
|
|
||||||
|
encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields)
|
||||||
|
|
||||||
|
core := zapjournald.NewCore(level, encoder, &journald.Journal{}, zapjournald.SyslogFields)
|
||||||
|
coreWithContext := core.With([]zapcore.Field{
|
||||||
|
zapjournald.SyslogFacility(zapjournald.LogDaemon),
|
||||||
|
zapjournald.SyslogIdentifier(),
|
||||||
|
zapjournald.SyslogPid(),
|
||||||
|
})
|
||||||
|
|
||||||
|
coreWithContext = applyZapCoreMiddlewares(coreWithContext, v, settings)
|
||||||
|
|
||||||
|
l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
|
||||||
|
|
||||||
|
return &Logger{
|
||||||
|
logger: l,
|
||||||
|
lvl: level,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newLogEncoder() zapcore.Encoder {
|
||||||
|
c := zap.NewProductionEncoderConfig()
|
||||||
|
c.EncodeTime = zapcore.ISO8601TimeEncoder
|
||||||
|
|
||||||
|
return zapcore.NewConsoleEncoder(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, settings LoggerAppSettings) zapcore.Core {
|
||||||
|
if v.GetBool(cfgLoggerSamplingEnabled) {
|
||||||
|
core = zapcore.NewSamplerWithOptions(core,
|
||||||
|
v.GetDuration(cfgLoggerSamplingInterval),
|
||||||
|
v.GetInt(cfgLoggerSamplingInitial),
|
||||||
|
v.GetInt(cfgLoggerSamplingThereafter),
|
||||||
|
zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) {
|
||||||
|
if dec&zapcore.LogDropped > 0 {
|
||||||
|
settings.DroppedLogsInc()
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
return core
|
||||||
|
}
|
||||||
|
|
||||||
|
func getLogLevel(v *viper.Viper) (zapcore.Level, error) {
|
||||||
|
var lvl zapcore.Level
|
||||||
|
lvlStr := v.GetString(cfgLoggerLevel)
|
||||||
|
err := lvl.UnmarshalText([]byte(lvlStr))
|
||||||
|
if err != nil {
|
||||||
|
return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+
|
||||||
|
"value should be one of %v", lvlStr, err, [...]zapcore.Level{
|
||||||
|
zapcore.DebugLevel,
|
||||||
|
zapcore.InfoLevel,
|
||||||
|
zapcore.WarnLevel,
|
||||||
|
zapcore.ErrorLevel,
|
||||||
|
zapcore.DPanicLevel,
|
||||||
|
zapcore.PanicLevel,
|
||||||
|
zapcore.FatalLevel,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return lvl, nil
|
||||||
|
}
|
||||||
|
|
||||||
func validateDomains(domains []string, log *zap.Logger) []string {
|
func validateDomains(domains []string, log *zap.Logger) []string {
|
||||||
validDomains := make([]string, 0, len(domains))
|
validDomains := make([]string, 0, len(domains))
|
||||||
LOOP:
|
LOOP:
|
||||||
for _, domain := range domains {
|
for _, domain := range domains {
|
||||||
if strings.Contains(domain, ":") {
|
if strings.Contains(domain, ":") {
|
||||||
log.Warn(logs.WarnDomainContainsPort, zap.String("domain", domain), logs.TagField(logs.TagApp))
|
log.Warn(logs.WarnDomainContainsPort, zap.String("domain", domain))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
domainParts := strings.Split(domain, ".")
|
domainParts := strings.Split(domain, ".")
|
||||||
for _, part := range domainParts {
|
for _, part := range domainParts {
|
||||||
if strings.ContainsAny(part, "<>") && part != wildcardPlaceholder {
|
if strings.ContainsAny(part, "<>") && part != wildcardPlaceholder {
|
||||||
log.Warn(logs.WarnDomainContainsInvalidPlaceholder, zap.String("domain", domain), logs.TagField(logs.TagApp))
|
log.Warn(logs.WarnDomainContainsInvalidPlaceholder, zap.String("domain", domain))
|
||||||
continue LOOP
|
continue LOOP
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,210 +0,0 @@
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
|
||||||
"git.frostfs.info/TrueCloudLab/zapjournald"
|
|
||||||
"github.com/spf13/viper"
|
|
||||||
"github.com/ssgreg/journald"
|
|
||||||
"go.uber.org/zap"
|
|
||||||
"go.uber.org/zap/zapcore"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
destinationStdout string = "stdout"
|
|
||||||
destinationJournald string = "journald"
|
|
||||||
)
|
|
||||||
|
|
||||||
var defaultTags = []string{logs.TagApp, logs.TagDatapath, logs.TagExternalStorage, logs.TagExternalStorageTree, logs.TagExternalBlockchain}
|
|
||||||
|
|
||||||
type Logger struct {
|
|
||||||
logger *zap.Logger
|
|
||||||
lvl zap.AtomicLevel
|
|
||||||
}
|
|
||||||
|
|
||||||
func pickLogger(v *viper.Viper, loggerSettings LoggerAppSettings, tagSettings TagFilterSettings) *Logger {
|
|
||||||
dest := v.GetString(cfgLoggerDestination)
|
|
||||||
|
|
||||||
switch dest {
|
|
||||||
case destinationStdout:
|
|
||||||
return newStdoutLogger(v, loggerSettings, tagSettings)
|
|
||||||
case destinationJournald:
|
|
||||||
return newJournaldLogger(v, loggerSettings, tagSettings)
|
|
||||||
default:
|
|
||||||
panic(fmt.Sprintf("wrong destination for logger: %s", dest))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// newStdoutLogger constructs a Logger instance for the current application.
|
|
||||||
// Panics on failure.
|
|
||||||
//
|
|
||||||
// Logger contains a logger is built from zap's production logging configuration with:
|
|
||||||
// - parameterized level (debug by default)
|
|
||||||
// - console encoding
|
|
||||||
// - ISO8601 time encoding
|
|
||||||
// - sampling intervals
|
|
||||||
//
|
|
||||||
// and atomic log level to dynamically change it.
|
|
||||||
//
|
|
||||||
// Logger records a stack trace for all messages at or above fatal level.
|
|
||||||
//
|
|
||||||
// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace.
|
|
||||||
func newStdoutLogger(v *viper.Viper, loggerSettings LoggerAppSettings, tagSettings TagFilterSettings) *Logger {
|
|
||||||
c := newZapLogConfig(v)
|
|
||||||
|
|
||||||
out, errSink, err := openZapSinks(c)
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("open zap sinks: %v", err.Error()))
|
|
||||||
}
|
|
||||||
|
|
||||||
core := zapcore.NewCore(zapcore.NewConsoleEncoder(c.EncoderConfig), out, c.Level)
|
|
||||||
core = applyZapCoreMiddlewares(core, v, loggerSettings, tagSettings)
|
|
||||||
l := zap.New(core, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), zap.ErrorOutput(errSink))
|
|
||||||
|
|
||||||
return &Logger{logger: l, lvl: c.Level}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newJournaldLogger(v *viper.Viper, logSettings LoggerAppSettings, tagSettings TagFilterSettings) *Logger {
|
|
||||||
c := newZapLogConfig(v)
|
|
||||||
|
|
||||||
// We can use NewJSONEncoder instead if, say, frontend
|
|
||||||
// would like to access journald logs and parse them easily.
|
|
||||||
encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields)
|
|
||||||
|
|
||||||
journalCore := zapjournald.NewCore(c.Level, encoder, &journald.Journal{}, zapjournald.SyslogFields)
|
|
||||||
core := journalCore.With([]zapcore.Field{
|
|
||||||
zapjournald.SyslogFacility(zapjournald.LogDaemon),
|
|
||||||
zapjournald.SyslogIdentifier(),
|
|
||||||
zapjournald.SyslogPid(),
|
|
||||||
})
|
|
||||||
core = applyZapCoreMiddlewares(core, v, logSettings, tagSettings)
|
|
||||||
l := zap.New(core, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
|
|
||||||
return &Logger{logger: l, lvl: c.Level}
|
|
||||||
}
|
|
||||||
|
|
||||||
func openZapSinks(cfg zap.Config) (zapcore.WriteSyncer, zapcore.WriteSyncer, error) {
|
|
||||||
sink, closeOut, err := zap.Open(cfg.OutputPaths...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
errSink, _, err := zap.Open(cfg.ErrorOutputPaths...)
|
|
||||||
if err != nil {
|
|
||||||
closeOut()
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
return sink, errSink, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ zapcore.Core = (*zapCoreTagFilterWrapper)(nil)
|
|
||||||
|
|
||||||
type zapCoreTagFilterWrapper struct {
|
|
||||||
core zapcore.Core
|
|
||||||
settings TagFilterSettings
|
|
||||||
extra []zap.Field
|
|
||||||
}
|
|
||||||
|
|
||||||
type TagFilterSettings interface {
|
|
||||||
LevelEnabled(tag string, lvl zapcore.Level) bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *zapCoreTagFilterWrapper) Enabled(level zapcore.Level) bool {
|
|
||||||
return c.core.Enabled(level)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *zapCoreTagFilterWrapper) With(fields []zapcore.Field) zapcore.Core {
|
|
||||||
return &zapCoreTagFilterWrapper{
|
|
||||||
core: c.core.With(fields),
|
|
||||||
settings: c.settings,
|
|
||||||
extra: append(c.extra, fields...),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *zapCoreTagFilterWrapper) Check(entry zapcore.Entry, checked *zapcore.CheckedEntry) *zapcore.CheckedEntry {
|
|
||||||
if c.core.Enabled(entry.Level) {
|
|
||||||
return checked.AddCore(entry, c)
|
|
||||||
}
|
|
||||||
return checked
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *zapCoreTagFilterWrapper) Write(entry zapcore.Entry, fields []zapcore.Field) error {
|
|
||||||
if c.shouldSkip(entry, fields) || c.shouldSkip(entry, c.extra) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return c.core.Write(entry, fields)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *zapCoreTagFilterWrapper) shouldSkip(entry zapcore.Entry, fields []zap.Field) bool {
|
|
||||||
for _, field := range fields {
|
|
||||||
if field.Key == logs.TagFieldName && field.Type == zapcore.StringType {
|
|
||||||
if !c.settings.LevelEnabled(field.String, entry.Level) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *zapCoreTagFilterWrapper) Sync() error {
|
|
||||||
return c.core.Sync()
|
|
||||||
}
|
|
||||||
|
|
||||||
func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, appSettings LoggerAppSettings, tagSettings TagFilterSettings) zapcore.Core {
|
|
||||||
core = &zapCoreTagFilterWrapper{
|
|
||||||
core: core,
|
|
||||||
settings: tagSettings,
|
|
||||||
}
|
|
||||||
|
|
||||||
if v.GetBool(cfgLoggerSamplingEnabled) {
|
|
||||||
core = zapcore.NewSamplerWithOptions(core,
|
|
||||||
v.GetDuration(cfgLoggerSamplingInterval),
|
|
||||||
v.GetInt(cfgLoggerSamplingInitial),
|
|
||||||
v.GetInt(cfgLoggerSamplingThereafter),
|
|
||||||
zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) {
|
|
||||||
if dec&zapcore.LogDropped > 0 {
|
|
||||||
appSettings.DroppedLogsInc()
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
return core
|
|
||||||
}
|
|
||||||
|
|
||||||
func newZapLogConfig(v *viper.Viper) zap.Config {
|
|
||||||
lvl, err := getLogLevel(v)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
c := zap.Config{
|
|
||||||
Level: zap.NewAtomicLevelAt(lvl),
|
|
||||||
EncoderConfig: zap.NewProductionEncoderConfig(),
|
|
||||||
OutputPaths: []string{"stderr"},
|
|
||||||
ErrorOutputPaths: []string{"stderr"},
|
|
||||||
}
|
|
||||||
c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
|
|
||||||
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
|
|
||||||
func getLogLevel(v *viper.Viper) (zapcore.Level, error) {
|
|
||||||
var lvl zapcore.Level
|
|
||||||
lvlStr := v.GetString(cfgLoggerLevel)
|
|
||||||
err := lvl.UnmarshalText([]byte(lvlStr))
|
|
||||||
if err != nil {
|
|
||||||
return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+
|
|
||||||
"value should be one of %v", lvlStr, err, [...]zapcore.Level{
|
|
||||||
zapcore.DebugLevel,
|
|
||||||
zapcore.InfoLevel,
|
|
||||||
zapcore.WarnLevel,
|
|
||||||
zapcore.ErrorLevel,
|
|
||||||
zapcore.DPanicLevel,
|
|
||||||
zapcore.PanicLevel,
|
|
||||||
zapcore.FatalLevel,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return lvl, nil
|
|
||||||
}
|
|
|
@ -19,24 +19,24 @@ type Service struct {
|
||||||
// Start runs http service with the exposed endpoint on the configured port.
|
// Start runs http service with the exposed endpoint on the configured port.
|
||||||
func (ms *Service) Start() {
|
func (ms *Service) Start() {
|
||||||
if ms.enabled {
|
if ms.enabled {
|
||||||
ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr), logs.TagField(logs.TagApp))
|
ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr))
|
||||||
err := ms.ListenAndServe()
|
err := ms.ListenAndServe()
|
||||||
if err != nil && err != http.ErrServerClosed {
|
if err != nil && err != http.ErrServerClosed {
|
||||||
ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort, logs.TagField(logs.TagApp))
|
ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled, logs.TagField(logs.TagApp))
|
ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ShutDown stops the service.
|
// ShutDown stops the service.
|
||||||
func (ms *Service) ShutDown(ctx context.Context) {
|
func (ms *Service) ShutDown(ctx context.Context) {
|
||||||
ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr), logs.TagField(logs.TagApp))
|
ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr))
|
||||||
err := ms.Shutdown(ctx)
|
err := ms.Shutdown(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err), logs.TagField(logs.TagApp))
|
ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err))
|
||||||
if err = ms.Close(); err != nil {
|
if err = ms.Close(); err != nil {
|
||||||
ms.log.Panic(logs.CantShutDownService, zap.Error(err), logs.TagField(logs.TagApp))
|
ms.log.Panic(logs.CantShutDownService, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -55,11 +55,6 @@ S3_GW_LOGGER_SAMPLING_ENABLED=false
|
||||||
S3_GW_LOGGER_SAMPLING_INITIAL=100
|
S3_GW_LOGGER_SAMPLING_INITIAL=100
|
||||||
S3_GW_LOGGER_SAMPLING_THEREAFTER=100
|
S3_GW_LOGGER_SAMPLING_THEREAFTER=100
|
||||||
S3_GW_LOGGER_SAMPLING_INTERVAL=1s
|
S3_GW_LOGGER_SAMPLING_INTERVAL=1s
|
||||||
S3_GW_LOGGER_TAGS_0_NAME=app
|
|
||||||
S3_GW_LOGGER_TAGS_0_LEVEL=info
|
|
||||||
S3_GW_LOGGER_TAGS_1_NAME=datapath
|
|
||||||
S3_GW_LOGGER_TAGS_1_LEVEL=fatal
|
|
||||||
|
|
||||||
|
|
||||||
# HTTP logger
|
# HTTP logger
|
||||||
S3_GW_HTTP_LOGGING_ENABLED=false
|
S3_GW_HTTP_LOGGING_ENABLED=false
|
||||||
|
@ -193,10 +188,6 @@ S3_GW_KLUDGE_USE_DEFAULT_XMLNS=false
|
||||||
S3_GW_KLUDGE_BYPASS_CONTENT_ENCODING_CHECK_IN_CHUNKS=false
|
S3_GW_KLUDGE_BYPASS_CONTENT_ENCODING_CHECK_IN_CHUNKS=false
|
||||||
# Namespaces that should be handled as default
|
# Namespaces that should be handled as default
|
||||||
S3_GW_KLUDGE_DEFAULT_NAMESPACES="" "root"
|
S3_GW_KLUDGE_DEFAULT_NAMESPACES="" "root"
|
||||||
# During listing the s3 gate may send whitespaces to client to prevent it from cancelling request.
|
|
||||||
# The gate is going to send whitespace every time it receives chunk of data from FrostFS storage.
|
|
||||||
# This parameter enables this feature and limits frequency of whitespace transmissions.
|
|
||||||
S3_GW_KLUDGE_LISTING_KEEPALIVE_THROTTLE=10s
|
|
||||||
# Kludge profiles
|
# Kludge profiles
|
||||||
S3_GW_KLUDGE_PROFILE_0_USER_AGENT=aws-cli
|
S3_GW_KLUDGE_PROFILE_0_USER_AGENT=aws-cli
|
||||||
S3_GW_KLUDGE_PROFILE_0_USE_DEFAULT_XMLNS=true
|
S3_GW_KLUDGE_PROFILE_0_USE_DEFAULT_XMLNS=true
|
||||||
|
|
|
@ -60,12 +60,6 @@ logger:
|
||||||
initial: 100
|
initial: 100
|
||||||
thereafter: 100
|
thereafter: 100
|
||||||
interval: 1s
|
interval: 1s
|
||||||
tags:
|
|
||||||
- name: "app"
|
|
||||||
level: "debug"
|
|
||||||
- name: "datapath"
|
|
||||||
- name: "external_storage"
|
|
||||||
- name: "external_storage_tree"
|
|
||||||
|
|
||||||
# log http request data (URI, headers, query, etc)
|
# log http request data (URI, headers, query, etc)
|
||||||
http_logging:
|
http_logging:
|
||||||
|
@ -234,10 +228,6 @@ kludge:
|
||||||
bypass_content_encoding_check_in_chunks: false
|
bypass_content_encoding_check_in_chunks: false
|
||||||
# Namespaces that should be handled as default
|
# Namespaces that should be handled as default
|
||||||
default_namespaces: [ "", "root" ]
|
default_namespaces: [ "", "root" ]
|
||||||
# During listing the s3 gate may send whitespaces to client to prevent it from cancelling request.
|
|
||||||
# The gate is going to send whitespace every time it receives chunk of data from FrostFS storage.
|
|
||||||
# This parameter enables this feature and limits frequency of whitespace transmissions.
|
|
||||||
listing_keepalive_throttle: 10s
|
|
||||||
# new profile section override defaults based on user agent
|
# new profile section override defaults based on user agent
|
||||||
profile:
|
profile:
|
||||||
- user_agent: aws-cli
|
- user_agent: aws-cli
|
||||||
|
|
|
@ -202,7 +202,7 @@ func (c *cred) checkIfCredentialsAreRemoved(ctx context.Context, cnrID cid.ID, a
|
||||||
|
|
||||||
func (c *cred) putBoxToCache(accessKeyID string, val *cache.AccessBoxCacheValue) {
|
func (c *cred) putBoxToCache(accessKeyID string, val *cache.AccessBoxCacheValue) {
|
||||||
if err := c.cache.Put(accessKeyID, val); err != nil {
|
if err := c.cache.Put(accessKeyID, val); err != nil {
|
||||||
c.log.Warn(logs.CouldntPutAccessBoxIntoCache, zap.String("accessKeyID", accessKeyID), logs.TagField(logs.TagDatapath))
|
c.log.Warn(logs.CouldntPutAccessBoxIntoCache, zap.String("accessKeyID", accessKeyID))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -241,7 +241,7 @@ func (c *cred) getAccessBox(ctx context.Context, cnrID cid.ID, accessKeyID strin
|
||||||
|
|
||||||
func (c *cred) Put(ctx context.Context, prm CredentialsParam) (oid.Address, error) {
|
func (c *cred) Put(ctx context.Context, prm CredentialsParam) (oid.Address, error) {
|
||||||
if prm.AccessKeyID != "" {
|
if prm.AccessKeyID != "" {
|
||||||
c.log.Info(logs.CheckCustomAccessKeyIDUniqueness, zap.String("access_key_id", prm.AccessKeyID), logs.TagField(logs.TagApp))
|
c.log.Info(logs.CheckCustomAccessKeyIDUniqueness, zap.String("access_key_id", prm.AccessKeyID))
|
||||||
credsPrm := PrmGetCredsObject{
|
credsPrm := PrmGetCredsObject{
|
||||||
Container: prm.Container,
|
Container: prm.Container,
|
||||||
AccessKeyID: prm.AccessKeyID,
|
AccessKeyID: prm.AccessKeyID,
|
||||||
|
|
|
@ -381,13 +381,6 @@ logger:
|
||||||
initial: 100
|
initial: 100
|
||||||
thereafter: 100
|
thereafter: 100
|
||||||
interval: 1s
|
interval: 1s
|
||||||
tags:
|
|
||||||
- name: app
|
|
||||||
level: info
|
|
||||||
- name: datapath
|
|
||||||
- name: external_blockchain
|
|
||||||
- name: external_storage_tree
|
|
||||||
- name: external_storage
|
|
||||||
```
|
```
|
||||||
|
|
||||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||||
|
@ -399,32 +392,6 @@ logger:
|
||||||
| `sampling.thereafter` | `int` | no | '100' | Sampling count of entries after an `interval`. |
|
| `sampling.thereafter` | `int` | no | '100' | Sampling count of entries after an `interval`. |
|
||||||
| `sampling.interval` | `duration` | no | '1s' | Sampling interval of messaging similar entries. |
|
| `sampling.interval` | `duration` | no | '1s' | Sampling interval of messaging similar entries. |
|
||||||
|
|
||||||
## Tags
|
|
||||||
|
|
||||||
There are additional log entries that can hurt performance and can be additionally logged by using `logger.tags`
|
|
||||||
parameter.
|
|
||||||
If section `tags` isn't set the default tags (see [Tag values](#tag-values)) be enabled.
|
|
||||||
If section `tags` set but empty no tags be used.
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
tags:
|
|
||||||
- name: "app"
|
|
||||||
level: info
|
|
||||||
```
|
|
||||||
|
|
||||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
|
||||||
|-----------------------|------------|---------------|---------------------------|-------------------------------------------------------------------------------------------------------|
|
|
||||||
| `name` | `string` | yes | | Tag name. Possible values see below in `Tag values` section. |
|
|
||||||
| `level` | `string` | yes | Value from `logger.level` | Logging level for specific tag. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. |
|
|
||||||
|
|
||||||
### Tag values
|
|
||||||
|
|
||||||
* `app` - common application logs (enabled by default).
|
|
||||||
* `datapath` - main logic of application (enabled by default).
|
|
||||||
* `external_blockchain` - external interaction with neo-go blockchain (enabled by default).
|
|
||||||
* `external_storage` - external interaction with storage node (enabled by default).
|
|
||||||
* `external_storage_tree` - external interaction with tree service in storage node (enabled by default).
|
|
||||||
|
|
||||||
|
|
||||||
### `http_logging` section
|
### `http_logging` section
|
||||||
|
|
||||||
|
@ -670,7 +637,6 @@ kludge:
|
||||||
use_default_xmlns: false
|
use_default_xmlns: false
|
||||||
bypass_content_encoding_check_in_chunks: false
|
bypass_content_encoding_check_in_chunks: false
|
||||||
default_namespaces: [ "", "root" ]
|
default_namespaces: [ "", "root" ]
|
||||||
listing_keepalive_throttle: 10s
|
|
||||||
profile:
|
profile:
|
||||||
- user_agent: aws-cli
|
- user_agent: aws-cli
|
||||||
use_default_xmlns: false
|
use_default_xmlns: false
|
||||||
|
@ -680,11 +646,10 @@ kludge:
|
||||||
```
|
```
|
||||||
|
|
||||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||||
|-------------------------------------------|----------------------------------|---------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|-------------------------------------------|----------------------------------|---------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| `use_default_xmlns` | `bool` | yes | `false` | Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies. |
|
| `use_default_xmlns` | `bool` | yes | `false` | Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies. |
|
||||||
| `bypass_content_encoding_check_in_chunks` | `bool` | yes | `false` | Use this flag to be able to use [chunked upload approach](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html) without having `aws-chunked` value in `Content-Encoding` header. |
|
| `bypass_content_encoding_check_in_chunks` | `bool` | yes | `false` | Use this flag to be able to use [chunked upload approach](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html) without having `aws-chunked` value in `Content-Encoding` header. |
|
||||||
| `default_namespaces` | `[]string` | yes | `["","root"]` | Namespaces that should be handled as default. |
|
| `default_namespaces` | `[]string` | yes | `["","root"]` | Namespaces that should be handled as default. |
|
||||||
| `listing_keepalive_throttle` | `duration` | yes | `0` (means disabled) | During listing the s3 gate may send whitespaces to client to prevent it from cancelling request. The gate is going to send whitespace every time it receives chunk of data from FrostFS storage. This parameter enables this feature and limits frequency of whitespace transmissions. |
|
|
||||||
| `profile` | [[]Profile](#profile-subsection) | yes | | An array of configurable profiles. |
|
| `profile` | [[]Profile](#profile-subsection) | yes | | An array of configurable profiles. |
|
||||||
|
|
||||||
#### `profile` subsection
|
#### `profile` subsection
|
||||||
|
|
5
go.mod
5
go.mod
|
@ -4,9 +4,8 @@ go 1.22
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.1-0.20241022094040-5f956751d48b
|
git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.1-0.20241022094040-5f956751d48b
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121
|
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250227072915-25102d1e1aa3
|
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250228093256-2b8329e026c7
|
|
||||||
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972
|
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972
|
||||||
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b
|
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b
|
||||||
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
|
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
|
||||||
|
|
10
go.sum
10
go.sum
|
@ -40,12 +40,10 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.1-0.20241022094040-5f956751
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.1-0.20241022094040-5f956751d48b/go.mod h1:5fSm/l5xSjGWqsPUffSdboiGFUHa7y/1S0fvxzQowN8=
|
git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.1-0.20241022094040-5f956751d48b/go.mod h1:5fSm/l5xSjGWqsPUffSdboiGFUHa7y/1S0fvxzQowN8=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk=
|
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
|
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 h1:/Z8DfbLZXp7exUQWUKoG/9tbFdI9d5lV1qSReaYoG8I=
|
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g=
|
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250227072915-25102d1e1aa3 h1:QnAt5b2R6+hQthMOIn5ECfLAlVD8IAE5JRm1NCCOmuE=
|
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250227072915-25102d1e1aa3/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U=
|
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250228093256-2b8329e026c7 h1:T7r38zZ/aT1xTp+AxhizfukW10Rq3WQ5/m3moLGVnSk=
|
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250228093256-2b8329e026c7/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8=
|
|
||||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
|
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
|
||||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
|
git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
|
||||||
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8=
|
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8=
|
||||||
|
|
|
@ -10,13 +10,11 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/crdt"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/crdt"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
objectv2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
objectv2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
@ -155,7 +153,7 @@ func (x *AuthmateFrostFS) GetCredsObject(ctx context.Context, prm tokens.PrmGetC
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *AuthmateFrostFS) readObject(ctx context.Context, addr oid.Address) (*object.Object, error) {
|
func (x *AuthmateFrostFS) readObject(ctx context.Context, addr oid.Address) (*object.Object, error) {
|
||||||
res, err := x.frostFS.GetObject(qostagging.ContextWithIOTag(ctx, util.InternalIOTag), frostfs.PrmObjectGet{
|
res, err := x.frostFS.GetObject(ctx, frostfs.PrmObjectGet{
|
||||||
Container: addr.Container(),
|
Container: addr.Container(),
|
||||||
Object: addr.Object(),
|
Object: addr.Object(),
|
||||||
})
|
})
|
||||||
|
@ -213,7 +211,7 @@ func (x *AuthmateFrostFS) CreateObject(ctx context.Context, prm tokens.PrmObject
|
||||||
attributes = append(attributes, [2]string{attr.Key(), attr.Value()})
|
attributes = append(attributes, [2]string{attr.Key(), attr.Value()})
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := x.frostFS.CreateObject(qostagging.ContextWithIOTag(ctx, util.InternalIOTag), frostfs.PrmObjectCreate{
|
res, err := x.frostFS.CreateObject(ctx, frostfs.PrmObjectCreate{
|
||||||
Container: prm.Container,
|
Container: prm.Container,
|
||||||
Filepath: prm.Filepath,
|
Filepath: prm.Filepath,
|
||||||
Attributes: attributes,
|
Attributes: attributes,
|
||||||
|
@ -227,7 +225,7 @@ func (x *AuthmateFrostFS) CreateObject(ctx context.Context, prm tokens.PrmObject
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *AuthmateFrostFS) getCredVersions(ctx context.Context, cnrID cid.ID, accessKeyID string) (*crdt.ObjectVersions, error) {
|
func (x *AuthmateFrostFS) getCredVersions(ctx context.Context, cnrID cid.ID, accessKeyID string) (*crdt.ObjectVersions, error) {
|
||||||
credVersions, err := x.frostFS.SearchObjects(qostagging.ContextWithIOTag(ctx, util.InternalIOTag), frostfs.PrmObjectSearch{
|
credVersions, err := x.frostFS.SearchObjects(ctx, frostfs.PrmObjectSearch{
|
||||||
Container: cnrID,
|
Container: cnrID,
|
||||||
ExactAttribute: [2]string{accessBoxCRDTNameAttr, accessKeyID},
|
ExactAttribute: [2]string{accessBoxCRDTNameAttr, accessKeyID},
|
||||||
})
|
})
|
||||||
|
|
|
@ -9,7 +9,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
||||||
frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
|
||||||
|
@ -58,9 +57,6 @@ func NewFrostFS(p *pool.Pool, key *keys.PrivateKey) *FrostFS {
|
||||||
|
|
||||||
// TimeToEpoch implements layer.FrostFS interface method.
|
// TimeToEpoch implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (uint64, uint64, error) {
|
func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (uint64, uint64, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.TimeToEpoch")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
if futureTime.Before(now) {
|
if futureTime.Before(now) {
|
||||||
return 0, 0, fmt.Errorf("time '%s' must be in the future (after %s)",
|
return 0, 0, fmt.Errorf("time '%s' must be in the future (after %s)",
|
||||||
futureTime.Format(time.RFC3339), now.Format(time.RFC3339))
|
futureTime.Format(time.RFC3339), now.Format(time.RFC3339))
|
||||||
|
@ -81,9 +77,6 @@ func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (u
|
||||||
|
|
||||||
// Container implements layer.FrostFS interface method.
|
// Container implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) Container(ctx context.Context, layerPrm frostfs.PrmContainer) (*container.Container, error) {
|
func (x *FrostFS) Container(ctx context.Context, layerPrm frostfs.PrmContainer) (*container.Container, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.Container")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
prm := pool.PrmContainerGet{
|
prm := pool.PrmContainerGet{
|
||||||
ContainerID: layerPrm.ContainerID,
|
ContainerID: layerPrm.ContainerID,
|
||||||
Session: layerPrm.SessionToken,
|
Session: layerPrm.SessionToken,
|
||||||
|
@ -99,9 +92,6 @@ func (x *FrostFS) Container(ctx context.Context, layerPrm frostfs.PrmContainer)
|
||||||
|
|
||||||
// CreateContainer implements layer.FrostFS interface method.
|
// CreateContainer implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) CreateContainer(ctx context.Context, prm frostfs.PrmContainerCreate) (*frostfs.ContainerCreateResult, error) {
|
func (x *FrostFS) CreateContainer(ctx context.Context, prm frostfs.PrmContainerCreate) (*frostfs.ContainerCreateResult, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.CreateContainer")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var cnr container.Container
|
var cnr container.Container
|
||||||
cnr.Init()
|
cnr.Init()
|
||||||
cnr.SetPlacementPolicy(prm.Policy)
|
cnr.SetPlacementPolicy(prm.Policy)
|
||||||
|
@ -149,9 +139,6 @@ func (x *FrostFS) CreateContainer(ctx context.Context, prm frostfs.PrmContainerC
|
||||||
|
|
||||||
// AddContainerPolicyChain implements frostfs.FrostFS interface method.
|
// AddContainerPolicyChain implements frostfs.FrostFS interface method.
|
||||||
func (x *FrostFS) AddContainerPolicyChain(ctx context.Context, prm frostfs.PrmAddContainerPolicyChain) error {
|
func (x *FrostFS) AddContainerPolicyChain(ctx context.Context, prm frostfs.PrmAddContainerPolicyChain) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.AddContainerPolicyChain")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
data, err := prm.Chain.MarshalBinary()
|
data, err := prm.Chain.MarshalBinary()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -171,9 +158,6 @@ func (x *FrostFS) AddContainerPolicyChain(ctx context.Context, prm frostfs.PrmAd
|
||||||
|
|
||||||
// UserContainers implements layer.FrostFS interface method.
|
// UserContainers implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) UserContainers(ctx context.Context, layerPrm frostfs.PrmUserContainers) ([]cid.ID, error) {
|
func (x *FrostFS) UserContainers(ctx context.Context, layerPrm frostfs.PrmUserContainers) ([]cid.ID, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.UserContainers")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
prm := pool.PrmContainerList{
|
prm := pool.PrmContainerList{
|
||||||
OwnerID: layerPrm.UserID,
|
OwnerID: layerPrm.UserID,
|
||||||
Session: layerPrm.SessionToken,
|
Session: layerPrm.SessionToken,
|
||||||
|
@ -185,9 +169,6 @@ func (x *FrostFS) UserContainers(ctx context.Context, layerPrm frostfs.PrmUserCo
|
||||||
|
|
||||||
// DeleteContainer implements layer.FrostFS interface method.
|
// DeleteContainer implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) DeleteContainer(ctx context.Context, id cid.ID, token *session.Container) error {
|
func (x *FrostFS) DeleteContainer(ctx context.Context, id cid.ID, token *session.Container) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.DeleteContainer")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
prm := pool.PrmContainerDelete{ContainerID: id, Session: token, WaitParams: &x.await}
|
prm := pool.PrmContainerDelete{ContainerID: id, Session: token, WaitParams: &x.await}
|
||||||
|
|
||||||
err := x.pool.DeleteContainer(ctx, prm)
|
err := x.pool.DeleteContainer(ctx, prm)
|
||||||
|
@ -196,9 +177,6 @@ func (x *FrostFS) DeleteContainer(ctx context.Context, id cid.ID, token *session
|
||||||
|
|
||||||
// CreateObject implements layer.FrostFS interface method.
|
// CreateObject implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) CreateObject(ctx context.Context, prm frostfs.PrmObjectCreate) (*frostfs.CreateObjectResult, error) {
|
func (x *FrostFS) CreateObject(ctx context.Context, prm frostfs.PrmObjectCreate) (*frostfs.CreateObjectResult, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.CreateObject")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
attrNum := len(prm.Attributes) + 1 // + creation time
|
attrNum := len(prm.Attributes) + 1 // + creation time
|
||||||
|
|
||||||
if prm.Filepath != "" {
|
if prm.Filepath != "" {
|
||||||
|
@ -293,9 +271,6 @@ func (x payloadReader) Read(p []byte) (int, error) {
|
||||||
|
|
||||||
// HeadObject implements layer.FrostFS interface method.
|
// HeadObject implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) HeadObject(ctx context.Context, prm frostfs.PrmObjectHead) (*object.Object, error) {
|
func (x *FrostFS) HeadObject(ctx context.Context, prm frostfs.PrmObjectHead) (*object.Object, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.HeadObject")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(prm.Container)
|
addr.SetContainer(prm.Container)
|
||||||
addr.SetObject(prm.Object)
|
addr.SetObject(prm.Object)
|
||||||
|
@ -319,9 +294,6 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm frostfs.PrmObjectHead) (*o
|
||||||
|
|
||||||
// GetObject implements layer.FrostFS interface method.
|
// GetObject implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) GetObject(ctx context.Context, prm frostfs.PrmObjectGet) (*frostfs.Object, error) {
|
func (x *FrostFS) GetObject(ctx context.Context, prm frostfs.PrmObjectGet) (*frostfs.Object, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.GetObject")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(prm.Container)
|
addr.SetContainer(prm.Container)
|
||||||
addr.SetObject(prm.Object)
|
addr.SetObject(prm.Object)
|
||||||
|
@ -348,9 +320,6 @@ func (x *FrostFS) GetObject(ctx context.Context, prm frostfs.PrmObjectGet) (*fro
|
||||||
|
|
||||||
// RangeObject implements layer.FrostFS interface method.
|
// RangeObject implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) RangeObject(ctx context.Context, prm frostfs.PrmObjectRange) (io.ReadCloser, error) {
|
func (x *FrostFS) RangeObject(ctx context.Context, prm frostfs.PrmObjectRange) (io.ReadCloser, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.RangeObject")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(prm.Container)
|
addr.SetContainer(prm.Container)
|
||||||
addr.SetObject(prm.Object)
|
addr.SetObject(prm.Object)
|
||||||
|
@ -376,9 +345,6 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm frostfs.PrmObjectRange) (
|
||||||
|
|
||||||
// DeleteObject implements layer.FrostFS interface method.
|
// DeleteObject implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) DeleteObject(ctx context.Context, prm frostfs.PrmObjectDelete) error {
|
func (x *FrostFS) DeleteObject(ctx context.Context, prm frostfs.PrmObjectDelete) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.DeleteObject")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(prm.Container)
|
addr.SetContainer(prm.Container)
|
||||||
addr.SetObject(prm.Object)
|
addr.SetObject(prm.Object)
|
||||||
|
@ -398,9 +364,6 @@ func (x *FrostFS) DeleteObject(ctx context.Context, prm frostfs.PrmObjectDelete)
|
||||||
|
|
||||||
// SearchObjects implements layer.FrostFS interface method.
|
// SearchObjects implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) SearchObjects(ctx context.Context, prm frostfs.PrmObjectSearch) ([]oid.ID, error) {
|
func (x *FrostFS) SearchObjects(ctx context.Context, prm frostfs.PrmObjectSearch) ([]oid.ID, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.SearchObjects")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
filters := object.NewSearchFilters()
|
filters := object.NewSearchFilters()
|
||||||
filters.AddRootFilter()
|
filters.AddRootFilter()
|
||||||
|
|
||||||
|
@ -438,9 +401,6 @@ func (x *FrostFS) SearchObjects(ctx context.Context, prm frostfs.PrmObjectSearch
|
||||||
|
|
||||||
// NetworkInfo implements layer.FrostFS interface method.
|
// NetworkInfo implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) NetworkInfo(ctx context.Context) (netmap.NetworkInfo, error) {
|
func (x *FrostFS) NetworkInfo(ctx context.Context) (netmap.NetworkInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.NetworkInfo")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
ni, err := x.pool.NetworkInfo(ctx)
|
ni, err := x.pool.NetworkInfo(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ni, handleObjectError("get network info via connection pool", err)
|
return ni, handleObjectError("get network info via connection pool", err)
|
||||||
|
@ -450,9 +410,6 @@ func (x *FrostFS) NetworkInfo(ctx context.Context) (netmap.NetworkInfo, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *FrostFS) NetmapSnapshot(ctx context.Context) (netmap.NetMap, error) {
|
func (x *FrostFS) NetmapSnapshot(ctx context.Context) (netmap.NetMap, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.NetmapSnapshot")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
netmapSnapshot, err := x.pool.NetMapSnapshot(ctx)
|
netmapSnapshot, err := x.pool.NetMapSnapshot(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return netmapSnapshot, handleObjectError("get netmap via connection pool", err)
|
return netmapSnapshot, handleObjectError("get netmap via connection pool", err)
|
||||||
|
@ -462,9 +419,6 @@ func (x *FrostFS) NetmapSnapshot(ctx context.Context) (netmap.NetMap, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *FrostFS) PatchObject(ctx context.Context, prm frostfs.PrmObjectPatch) (oid.ID, error) {
|
func (x *FrostFS) PatchObject(ctx context.Context, prm frostfs.PrmObjectPatch) (oid.ID, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.PatchObject")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(prm.Container)
|
addr.SetContainer(prm.Container)
|
||||||
addr.SetObject(prm.Object)
|
addr.SetObject(prm.Object)
|
||||||
|
@ -513,9 +467,6 @@ func NewResolverFrostFS(p *pool.Pool) *ResolverFrostFS {
|
||||||
|
|
||||||
// SystemDNS implements resolver.FrostFS interface method.
|
// SystemDNS implements resolver.FrostFS interface method.
|
||||||
func (x *ResolverFrostFS) SystemDNS(ctx context.Context) (string, error) {
|
func (x *ResolverFrostFS) SystemDNS(ctx context.Context) (string, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "frostfs.SystemDNS")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
networkInfo, err := x.pool.NetworkInfo(ctx)
|
networkInfo, err := x.pool.NetworkInfo(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", handleObjectError("read network info via client", err)
|
return "", handleObjectError("read network info via client", err)
|
||||||
|
|
|
@ -61,7 +61,7 @@ func (f *FrostFSID) GetUserGroupIDsAndClaims(userHash util.Uint160) ([]string, m
|
||||||
subj, err := f.getSubject(userHash)
|
subj, err := f.getSubject(userHash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if strings.Contains(err.Error(), "not found") {
|
if strings.Contains(err.Error(), "not found") {
|
||||||
f.log.Debug(logs.UserGroupsListIsEmpty, zap.Error(err), logs.TagField(logs.TagExternalBlockchain))
|
f.log.Debug(logs.UserGroupsListIsEmpty, zap.Error(err))
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
|
@ -86,7 +86,7 @@ func (f *FrostFSID) getSubject(addr util.Uint160) (*client.SubjectExtended, erro
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = f.cache.PutSubject(addr, subj); err != nil {
|
if err = f.cache.PutSubject(addr, subj); err != nil {
|
||||||
f.log.Warn(logs.CouldntCacheSubject, zap.Error(err), logs.TagField(logs.TagDatapath))
|
f.log.Warn(logs.CouldntCacheSubject, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
return subj, nil
|
return subj, nil
|
||||||
|
@ -121,7 +121,7 @@ func (f *FrostFSID) getUserKey(namespace, name string) (*keys.PublicKey, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = f.cache.PutUserKey(namespace, name, userKey); err != nil {
|
if err = f.cache.PutUserKey(namespace, name, userKey); err != nil {
|
||||||
f.log.Warn(logs.CouldntCacheUserKey, zap.Error(err), logs.TagField(logs.TagDatapath))
|
f.log.Warn(logs.CouldntCacheUserKey, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
return userKey, nil
|
return userKey, nil
|
||||||
|
|
|
@ -69,7 +69,7 @@ func (c *MorphRuleChainStorage) ListMorphRuleChains(name chain.Name, target engi
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = c.cache.Put(key, list); err != nil {
|
if err = c.cache.Put(key, list); err != nil {
|
||||||
c.log.Warn(logs.CouldntCacheListPolicyChains, logs.TagField(logs.TagApp))
|
c.log.Warn(logs.CouldntCacheListPolicyChains)
|
||||||
}
|
}
|
||||||
|
|
||||||
return list, nil
|
return list, nil
|
||||||
|
|
|
@ -13,8 +13,6 @@ import (
|
||||||
"github.com/nspcc-dev/neo-go/pkg/util"
|
"github.com/nspcc-dev/neo-go/pkg/util"
|
||||||
)
|
)
|
||||||
|
|
||||||
const InternalIOTag = "internal"
|
|
||||||
|
|
||||||
// ResolveContractHash determine contract hash by resolving NNS name.
|
// ResolveContractHash determine contract hash by resolving NNS name.
|
||||||
func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error) {
|
func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error) {
|
||||||
if hash, err := util.Uint160DecodeStringLE(contractHash); err == nil {
|
if hash, err := util.Uint160DecodeStringLE(contractHash); err == nil {
|
||||||
|
|
|
@ -1,230 +1,189 @@
|
||||||
package logs
|
package logs
|
||||||
|
|
||||||
import "go.uber.org/zap"
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
TagFieldName = "tag"
|
RequestUnmatched = "request unmatched" // Error in ../../api/router.go
|
||||||
|
CheckContainer = "check container" // Info in ../../authmate/authmate.go
|
||||||
TagApp = "app"
|
CreateContainer = "create container" // Info in ../../authmate/authmate.go
|
||||||
TagDatapath = "datapath"
|
StoreBearerTokenIntoFrostFS = "store bearer token into FrostFS" // Info in ../../authmate/authmate.go
|
||||||
TagExternalStorage = "external_storage"
|
UpdateAccessCredObjectIntoFrostFS = "update access cred object into FrostFS" // Info in ../../authmate/authmate.go
|
||||||
TagExternalStorageTree = "external_storage_tree"
|
MetricsAreDisabled = "metrics are disabled" // Warn in ../../metrics/app.go
|
||||||
TagExternalBlockchain = "external_blockchain"
|
FoundMoreThanOneUnversionedNode = "found more than one unversioned node" // Debug in ../../pkg/service/tree/tree.go
|
||||||
)
|
ServiceIsRunning = "service is running" // Info in ../../cmd/s3-gw/service.go
|
||||||
|
ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../cmd/s3-gw/service.go
|
||||||
func TagField(tag string) zap.Field {
|
ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../cmd/s3-gw/service.go
|
||||||
return zap.String(TagFieldName, tag)
|
ShuttingDownService = "shutting down service" // Info in ../../cmd/s3-gw/service.go
|
||||||
}
|
CantGracefullyShutDownService = "can't gracefully shut down service, force stop" // Error in ../../cmd/s3-gw/service.go
|
||||||
|
ContainerResolverWillBeDisabled = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../cmd/s3-gw/app.go
|
||||||
// App.
|
FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../cmd/s3-gw/app.go
|
||||||
const (
|
TracingConfigUpdated = "tracing config updated" // Info in ../../cmd/s3-gw/app.go
|
||||||
ApplicationStarted = "application started"
|
FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../cmd/s3-gw/app.go
|
||||||
ApplicationFinished = "application finished"
|
UsingCredentials = "using credentials" // Info in ../../cmd/s3-gw/app.go
|
||||||
StartingServer = "starting server"
|
ApplicationStarted = "application started" // Info in ../../cmd/s3-gw/app.go
|
||||||
StoppingServer = "stopping server"
|
ApplicationFinished = "application finished" // Info in ../../cmd/s3-gw/app.go
|
||||||
ServiceIsRunning = "service is running"
|
StartingServer = "starting server" // Info in ../../cmd/s3-gw/app.go
|
||||||
ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port"
|
StoppingServer = "stopping server" // Info in ../../cmd/s3-gw/app.go
|
||||||
ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled"
|
SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../cmd/s3-gw/app.go
|
||||||
ShuttingDownService = "shutting down service"
|
FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../cmd/s3-gw/app.go
|
||||||
CantGracefullyShutDownService = "can't gracefully shut down service, force stop"
|
FailedToReloadConfig = "failed to reload config" // Warn in ../../cmd/s3-gw/app.go
|
||||||
FailedToShutdownTracing = "failed to shutdown tracing"
|
FailedToReloadResolvers = "failed to reload resolvers" // Warn in ../../cmd/s3-gw/app.go
|
||||||
UsingCredentials = "using credentials"
|
FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../cmd/s3-gw/app.go
|
||||||
FailedToAddServer = "failed to add server"
|
SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../cmd/s3-gw/app.go
|
||||||
AddServer = "add server"
|
LogLevelWontBeUpdated = "log level won't be updated" // Warn in ../../cmd/s3-gw/app.go
|
||||||
CantShutDownService = "can't shut down service"
|
FailedToAddServer = "failed to add server" // Warn in ../../cmd/s3-gw/app.go
|
||||||
FailedToCreateResolver = "failed to create resolver"
|
AddServer = "add server" // Info in ../../cmd/s3-gw/app.go
|
||||||
CouldntGenerateRandomKey = "couldn't generate random key"
|
ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver 'nns' won't be used since 'rpc_endpoint' isn't provided" // Warn in ../../cmd/s3-gw/app.go
|
||||||
FailedToCreateConnectionPool = "failed to create connection pool"
|
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/s3-gw/app_settings.go
|
||||||
FailedToDialConnectionPool = "failed to dial connection pool"
|
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/s3-gw/app_settings.go
|
||||||
FailedToCreateTreePool = "failed to create tree pool"
|
FailedToParseDefaultLocationConstraint = "failed to parse 'default' location constraint, default one will be used" // Warn in cmd/s3-gw/app_settings.go
|
||||||
FailedToDialTreePool = "failed to dial tree pool"
|
FailedToReadRegionMapFilePolicies = "failed to read region map file, policies will be empty" // Warn in cmd/s3-gw/app_settings.go
|
||||||
ListenAndServe = "listen and serve"
|
DefaultLocationConstraintCantBeOverriden = "'default' location constraint can't be overriden by custom policy, use 'placement_policy.default'" // Warn in cmd/s3-gw/app_settings.go
|
||||||
NoHealthyServers = "no healthy servers"
|
FailedToParseLocationConstraint = "failed to parse location constraint, it cannot be used" // Warn in cmd/s3-gw/app_settings.go
|
||||||
CouldNotInitializeAPIHandler = "could not initialize API handler"
|
FailedToParseDefaultCopiesNumbers = "failed to parse 'default' copies numbers, default one will be used" // Warn in cmd/s3-gw/app_settings.go
|
||||||
InitFrostfsIDFailed = "init frostfsid failed"
|
FailedToParseCopiesNumbers = "failed to parse copies numbers, skip" // Warn in cmd/s3-gw/app_settings.go
|
||||||
InitPolicyContractFailed = "init policy contract failed"
|
DefaultNamespacesCannotBeEmpty = "default namespaces cannot be empty, defaults will be used" // Warn in cmd/s3-gw/app_settings.go
|
||||||
|
FailedToParseNamespacesConfig = "failed to unmarshal namespaces config" // Warn in cmd/s3-gw/app_settings.go
|
||||||
|
DefaultNamespaceConfigValuesBeOverwritten = "default namespace config value be overwritten by values from 'namespaces.config'" // Warn in cmd/s3-gw/app_settings.go
|
||||||
|
MultipleDefaultOverridesFound = "multiple default overrides found, only one will be used" // Warn in cmd/s3-gw/app_settings.go
|
||||||
|
FailedToParseDefaultDefaultLocationConstraint = "failed to parse default 'default' location constraint" // Fatal in cmd/s3-gw/app_settings.go
|
||||||
|
ConstraintAdded = "constraint added" // Info in ../../cmd/s3-gw/app_settings.go
|
||||||
|
SkipEmptyAddress = "skip, empty address" // Warn in ../../cmd/s3-gw/app_settings.go
|
||||||
|
AddedStoragePeer = "added storage peer" // Info in ../../cmd/s3-gw/app_settings.go
|
||||||
|
PrepareConnectionPool = "prepare connection pool" // Debug in ../../cmd/s3-authmate/modules/utils.go
|
||||||
|
PrepareFrostfsIDClient = "prepare frostfsid client" // Debug in ../../cmd/s3-authmate/modules/utils.go
|
||||||
|
PreparePolicyClient = "prepare policy client" // Debug in ../../cmd/s3-authmate/modules/utils.go
|
||||||
|
CreateSubjectInFrostFSID = "create subject in frostfsid" // Debug in ../../cmd/s3-authmate/modules/utils.go
|
||||||
|
SubjectAlreadyExistsInFrostFSID = "subject already exists in frostfsid" // Debug in ../../cmd/s3-authmate/modules/utils.go
|
||||||
|
SetSubjectNameInFrostFSID = "set subject name in frostfsid" // Debug in ../../cmd/s3-authmate/modules/utils.go
|
||||||
|
AddPolicyChainRules = "add policy chain rules" // Debug in ../../cmd/s3-authmate/modules/utils.go
|
||||||
|
InvalidCacheEntryType = "invalid cache entry type" // Warn in ../../api/cache/*
|
||||||
|
InvalidCacheKeyType = "invalid cache key type" // Warn in ../../api/cache/objectslist.go
|
||||||
|
ObjectIsCopied = "object is copied" // Info in ../../api/handler/copy.go
|
||||||
|
RequestFailed = "request failed" // Error in ../../api/handler/util.go
|
||||||
|
GetBucketInfo = "get bucket info" // Warn in ../../api/handler/cors.go
|
||||||
|
GetBucketCors = "get bucket cors" // Warn in ../../api/handler/cors.go
|
||||||
|
CouldntDeleteObject = "couldn't delete object" // Error in ../../api/layer/layer.go
|
||||||
|
BucketIsCreated = "bucket is created" // Info in ../../api/handler/put.go
|
||||||
|
CouldNotParseContainerObjectLockEnabledAttribute = "could not parse container object lock enabled attribute" // Error in ../../api/layer/container.go
|
||||||
|
CouldNotListUserContainers = "could not list user containers" // Error in ../../api/layer/container.go
|
||||||
|
CouldNotFetchContainerInfo = "could not fetch container info" // Error in ../../api/layer/container.go
|
||||||
|
MismatchedObjEncryptionInfo = "mismatched obj encryptionInfo" // Warn in ../../api/layer/multipart_upload.go
|
||||||
|
UploadPart = "upload part" // Debug in ../../api/layer/multipart_upload.go
|
||||||
|
CouldntDeleteOldPartObject = "couldn't delete old part object" // Error in ../../api/layer/multipart_upload.go
|
||||||
|
CouldNotPutCompletedObject = "could not put a completed object (multipart upload)" // Error in ../../api/layer/multipart_upload.go
|
||||||
|
CouldNotDeleteUploadPart = "could not delete upload part" // Warn in ../../api/layer/multipart_upload.go
|
||||||
|
CouldntDeletePart = "couldn't delete part" // Warn in ../../api/layer/multipart_upload.go
|
||||||
|
PartDetails = "part details" // Debug in ../../api/layer/multipart_upload.go
|
||||||
|
GetObject = "get object" // Debug in ../../api/layer/layer.go
|
||||||
|
ResolveBucket = "resolve bucket" // Info in ../../api/layer/layer.go
|
||||||
|
CouldntDeleteCorsObject = "couldn't delete cors object" // Error in ../../api/layer/cors.go
|
||||||
|
PutObject = "put object" // Debug in ../../api/layer/object.go
|
||||||
|
FailedToDeleteObject = "failed to delete object" // Debug in ../../api/layer/object.go
|
||||||
|
FailedToDiscardPutPayloadProbablyGoroutineLeaks = "failed to discard put payload, probably goroutine leaks" // Warn in ../../api/layer/object.go
|
||||||
|
FailedToSubmitTaskToPool = "failed to submit task to pool" // Warn in ../../api/layer/object.go
|
||||||
|
CouldNotFetchObjectMeta = "could not fetch object meta" // Warn in ../../api/layer/object.go
|
||||||
|
GetTreeNode = "get tree node" // Debug in ../../api/layer/tagging.go
|
||||||
|
GetTreeNodeToDelete = "get tree node to delete" // Debug in ../../api/layer/tagging.go
|
||||||
|
CouldntPutBucketInfoIntoCache = "couldn't put bucket info into cache" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntAddObjectToCache = "couldn't add object to cache" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntCacheAccessControlOperation = "couldn't cache access control operation" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntPutObjAddressToNameCache = "couldn't put obj address to name cache" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntCacheListOfObjects = "couldn't cache list of objects" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntCacheListSession = "couldn't cache list session" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntCacheTags = "couldn't cache tags" // Error in ../../api/layer/cache.go
|
||||||
|
CouldntCacheLockInfo = "couldn't cache lock info" // Error in ../../api/layer/cache.go
|
||||||
|
CouldntCacheBucketSettings = "couldn't cache bucket settings" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntCacheCors = "couldn't cache cors" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntCacheListPolicyChains = "couldn't cache list policy chains" // Warn in ../../api/layer/cache.go
|
||||||
|
RequestEnd = "request end" // Info in ../../api/middleware/response.go
|
||||||
|
CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed = "couldn't receive access box for gate key, random key will be used" // Debug in ../../api/middleware/auth.go
|
||||||
|
FailedToPassAuthentication = "failed to pass authentication" // Error in ../../api/middleware/auth.go
|
||||||
|
FailedToResolveCID = "failed to resolve CID" // Debug in ../../api/middleware/metrics.go
|
||||||
|
RequestStart = "request start" // Info in ../../api/middleware/reqinfo.go
|
||||||
|
LogHTTP = "http log" // Info in ../../api/middleware/log_http.go
|
||||||
|
FailedToCloseHTTPBody = "failed to close http body" // Error in ../../api/middleware/log_http.go
|
||||||
|
FailedToInitializeHTTPLogger = "failed to initialize http logger" // Error in ../../api/middleware/log_http.go
|
||||||
|
FailedToReloadHTTPFileLogger = "failed to reload http file logger" // Error in ../../api/middleware/log_http.go
|
||||||
|
FailedToReadHTTPBody = "failed to read http body" // Error in ../../api/middleware/log_http.go
|
||||||
|
FailedToProcessHTTPBody = "failed to process http body" // Error in ../../api/middleware/log_http.go
|
||||||
|
LogHTTPDisabledInThisBuild = "http logging disabled in this build" // Warn in ../../api/middleware/log_http_stub.go
|
||||||
|
FailedToUnescapeObjectName = "failed to unescape object name" // Warn in ../../api/middleware/reqinfo.go
|
||||||
|
InvalidDefaultMaxAge = "invalid defaultMaxAge" // Fatal in ../../cmd/s3-gw/app_settings.go
|
||||||
|
CantShutDownService = "can't shut down service" // Panic in ../../cmd/s3-gw/service.go
|
||||||
|
CouldntGenerateRandomKey = "couldn't generate random key" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToCreateResolver = "failed to create resolver" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
ListenAndServe = "listen and serve" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
NoHealthyServers = "no healthy servers" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
CouldNotInitializeAPIHandler = "could not initialize API handler" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" // Warn in ../../cmd/s3-gw/app.go
|
||||||
|
RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" // Info in ../../cmd/s3-gw/app.go
|
||||||
|
AnonRequestSkipFrostfsIDValidation = "anon request, skip FrostfsID validation" // Debug in ../../api/middleware/auth.go
|
||||||
|
FrostfsIDValidationFailed = "FrostfsID validation failed" // Error in ../../api/middleware/auth.go
|
||||||
|
InitFrostfsIDContractFailed = "init frostfsid contract failed" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
InitPolicyContractFailed = "init policy contract failed" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
PolicyValidationFailed = "policy validation failed"
|
||||||
ServerReconnecting = "reconnecting server..."
|
ServerReconnecting = "reconnecting server..."
|
||||||
ServerReconnectedSuccessfully = "server reconnected successfully"
|
ServerReconnectedSuccessfully = "server reconnected successfully"
|
||||||
ServerReconnectFailed = "failed to reconnect server"
|
ServerReconnectFailed = "failed to reconnect server"
|
||||||
CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info"
|
ParseTreeNode = "parse tree node"
|
||||||
CouldNotFetchAccessBoxContainerInfo = "couldn't fetch AccessBox container info"
|
FailedToGetRealObjectSize = "failed to get real object size"
|
||||||
MultinetDialSuccess = "multinet dial successful"
|
CouldntDeleteObjectFromStorageContinueDeleting = "couldn't delete object from storage, continue deleting from tree"
|
||||||
MultinetDialFail = "multinet dial failed"
|
CouldntPutAccessBoxIntoCache = "couldn't put accessbox into cache"
|
||||||
FailedToCreateWorkerPool = "failed to create worker pool"
|
|
||||||
CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key"
|
|
||||||
AddedStoragePeer = "added storage peer"
|
|
||||||
SIGHUPConfigReloadStarted = "SIGHUP config reload started"
|
|
||||||
MetricsAreDisabled = "metrics are disabled"
|
|
||||||
ConstraintAdded = "constraint added"
|
|
||||||
ContainerResolverWillBeDisabled = "container resolver will be disabled because of resolvers 'resolver_order' is empty"
|
|
||||||
FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed"
|
|
||||||
FailedToReloadConfig = "failed to reload config"
|
|
||||||
TracingConfigUpdated = "tracing config updated"
|
|
||||||
FailedToReloadResolvers = "failed to reload resolvers"
|
|
||||||
FailedToReloadServerParameters = "failed to reload server parameters"
|
|
||||||
SIGHUPConfigReloadCompleted = "SIGHUP config reload completed"
|
|
||||||
LogLevelWontBeUpdated = "log level won't be updated"
|
|
||||||
ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver 'nns' won't be used since 'rpc_endpoint' isn't provided"
|
|
||||||
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)"
|
|
||||||
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value"
|
|
||||||
FailedToParseDefaultLocationConstraint = "failed to parse 'default' location constraint, default one will be used"
|
|
||||||
FailedToReadRegionMapFilePolicies = "failed to read region map file, policies will be empty"
|
|
||||||
DefaultLocationConstraintCantBeOverriden = "'default' location constraint can't be overriden by custom policy, use 'placement_policy.default'"
|
|
||||||
FailedToParseLocationConstraint = "failed to parse location constraint, it cannot be used"
|
|
||||||
FailedToParseDefaultCopiesNumbers = "failed to parse 'default' copies numbers, default one will be used"
|
|
||||||
FailedToParseCopiesNumbers = "failed to parse copies numbers, skip"
|
|
||||||
FailedToInitializeTracing = "failed to initialize tracing"
|
|
||||||
DefaultNamespacesCannotBeEmpty = "default namespaces cannot be empty, defaults will be used"
|
|
||||||
FailedToParseNamespacesConfig = "failed to unmarshal namespaces config"
|
|
||||||
DefaultNamespaceConfigValuesBeOverwritten = "default namespace config value be overwritten by values from 'namespaces.config'"
|
|
||||||
MultipleDefaultOverridesFound = "multiple default overrides found, only one will be used"
|
|
||||||
SkipEmptyAddress = "skip, empty address"
|
|
||||||
FailedToParseDefaultDefaultLocationConstraint = "failed to parse default 'default' location constraint"
|
|
||||||
LogHTTPDisabledInThisBuild = "http logging disabled in this build"
|
|
||||||
InvalidDefaultMaxAge = "invalid defaultMaxAge"
|
|
||||||
RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped"
|
|
||||||
RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated"
|
|
||||||
InvalidAccessBoxCacheRemovingCheckInterval = "invalid accessbox check removing interval, using default value"
|
InvalidAccessBoxCacheRemovingCheckInterval = "invalid accessbox check removing interval, using default value"
|
||||||
WarnDuplicateAddress = "duplicate address"
|
|
||||||
FailedToLoadMultinetConfig = "failed to load multinet config"
|
|
||||||
MultinetConfigWontBeUpdated = "multinet config won't be updated"
|
|
||||||
WarnDomainContainsPort = "the domain contains a port, domain skipped"
|
|
||||||
TagsLogConfigWontBeUpdated = "tags log config won't be updated"
|
|
||||||
CouldNotFetchLifecycleContainerInfo = "couldn't fetch lifecycle container info"
|
|
||||||
WarnDuplicateNamespaceVHS = "duplicate namespace with enabled VHS, config value skipped"
|
|
||||||
WarnValueVHSEnabledFlagWrongType = "the value of the VHS enable flag for the namespace is of the wrong type, config value skipped"
|
|
||||||
WarnDomainContainsInvalidPlaceholder = "the domain contains an invalid placeholder, domain skipped"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Datapath.
|
|
||||||
const (
|
|
||||||
NotSupported = "not supported"
|
|
||||||
RequestUnmatched = "request unmatched"
|
|
||||||
RequestStart = "request start"
|
|
||||||
RequestFailed = "request failed"
|
|
||||||
RequestEnd = "request end"
|
|
||||||
AnonRequestSkipFrostfsIDValidation = "anon request, skip FrostfsID validation"
|
|
||||||
CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed = "couldn't receive access box for gate key, random key will be used"
|
|
||||||
FrostfsIDValidationFailed = "FrostfsID validation failed"
|
|
||||||
FailedToPassAuthentication = "failed to pass authentication"
|
|
||||||
PolicyValidationFailed = "policy validation failed"
|
|
||||||
CouldNotCloseRequestBody = "could not close request body"
|
CouldNotCloseRequestBody = "could not close request body"
|
||||||
BucketOwnerKeyIsMissing = "bucket owner key is missing"
|
BucketOwnerKeyIsMissing = "bucket owner key is missing"
|
||||||
|
SettingsNodeInvalidOwnerKey = "settings node: invalid owner key"
|
||||||
SuccessfulAuth = "successful auth"
|
SuccessfulAuth = "successful auth"
|
||||||
PolicyRequest = "policy request"
|
PolicyRequest = "policy request"
|
||||||
FailedToGenerateRequestID = "failed to generate request id"
|
FailedToGenerateRequestID = "failed to generate request id"
|
||||||
FailedToWriteResponse = "failed to write response"
|
|
||||||
PolicyCouldntBeConvertedToNativeRules = "policy couldn't be converted to native rules, only s3 rules be applied"
|
|
||||||
FailedToParseHTTPTime = "failed to parse http time, header is ignored"
|
|
||||||
WarnInvalidTypeTLSTerminationHeader = "invalid type of value of tls termination header"
|
|
||||||
FailedToUnescapeObjectName = "failed to unescape object name"
|
|
||||||
MismatchedObjEncryptionInfo = "mismatched obj encryptionInfo"
|
|
||||||
CouldNotParseContainerObjectLockEnabledAttribute = "could not parse container object lock enabled attribute"
|
|
||||||
FailedToParsePartInfo = "failed to parse part info"
|
|
||||||
SettingsNodeInvalidOwnerKey = "settings node: invalid owner key"
|
|
||||||
FailedToParseAddressInTreeNode = "failed to parse object addr in tree node"
|
|
||||||
UnexpectedMultiNodeIDsInSubTreeMultiParts = "unexpected multi node ids in sub tree multi parts"
|
|
||||||
FoundSeveralSystemNodes = "found several system nodes"
|
|
||||||
BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids"
|
|
||||||
UploadPart = "upload part"
|
|
||||||
FailedToSubmitTaskToPool = "failed to submit task to pool"
|
|
||||||
FailedToGetRealObjectSize = "failed to get real object size"
|
|
||||||
PartDetails = "part details"
|
|
||||||
BucketSettingsNotFoundUseDefaults = "bucket settings not found, use defaults"
|
|
||||||
ParseTreeNode = "parse tree node"
|
|
||||||
GetObject = "get object"
|
|
||||||
GetTreeNodeToDelete = "get tree node to delete"
|
|
||||||
InvalidBucketObjectLockEnabledHeader = "invalid X-Amz-Bucket-Object-Lock-Enabled header"
|
InvalidBucketObjectLockEnabledHeader = "invalid X-Amz-Bucket-Object-Lock-Enabled header"
|
||||||
InvalidCacheEntryType = "invalid cache entry type"
|
|
||||||
InvalidCacheKeyType = "invalid cache key type"
|
|
||||||
CouldntPutBucketInfoIntoCache = "couldn't put bucket info into cache"
|
|
||||||
CouldntAddObjectToCache = "couldn't add object to cache"
|
|
||||||
CouldntCacheAccessControlOperation = "couldn't cache access control operation"
|
|
||||||
CouldntPutObjAddressToNameCache = "couldn't put obj address to name cache"
|
|
||||||
CouldntPutAccessBoxIntoCache = "couldn't put accessbox into cache"
|
|
||||||
CouldntCacheListOfObjects = "couldn't cache list of objects"
|
|
||||||
CouldntCacheListSession = "couldn't cache list session"
|
|
||||||
CouldntCacheTags = "couldn't cache tags"
|
|
||||||
CouldntCacheLockInfo = "couldn't cache lock info"
|
|
||||||
CouldntCacheNetworkInfo = "couldn't cache network info"
|
|
||||||
CouldntCacheSubject = "couldn't cache subject info"
|
|
||||||
CouldntCacheNetmap = "couldn't cache netmap"
|
|
||||||
CouldntCacheUserKey = "couldn't cache user key"
|
|
||||||
CouldntCacheBucketSettings = "couldn't cache bucket settings"
|
|
||||||
CouldntCacheCors = "couldn't cache cors"
|
|
||||||
CouldntCacheListPolicyChains = "couldn't cache list policy chains"
|
|
||||||
CouldntCacheLifecycleConfiguration = "couldn't cache lifecycle configuration"
|
|
||||||
GetBucketCors = "get bucket cors"
|
|
||||||
GetBucketInfo = "get bucket info"
|
|
||||||
ResolveBucket = "resolve bucket"
|
|
||||||
FailedToResolveCID = "failed to resolve CID"
|
|
||||||
FailedToDiscardPutPayloadProbablyGoroutineLeaks = "failed to discard put payload, probably goroutine leaks"
|
|
||||||
)
|
|
||||||
|
|
||||||
// External storage.
|
|
||||||
const (
|
|
||||||
ObjectIsCopied = "object is copied"
|
|
||||||
CouldntDeleteObject = "couldn't delete object"
|
|
||||||
CouldNotListUserContainers = "could not list user containers"
|
|
||||||
CouldNotFetchContainerInfo = "could not fetch container info"
|
|
||||||
CouldntDeleteObjectFromStorageContinueDeleting = "couldn't delete object from storage, continue deleting from tree"
|
|
||||||
FailedToPutTombstoneObject = "failed to put tombstone object"
|
|
||||||
FailedToListAllObjectRelations = "failed to list all object relations"
|
|
||||||
FailedToPutTombstones = "failed to put tombstones"
|
|
||||||
CouldntDeleteCorsObject = "couldn't delete cors object"
|
|
||||||
BucketIsCreated = "bucket is created"
|
|
||||||
CouldntDeleteOldPartObject = "couldn't delete old part object"
|
|
||||||
CouldNotPutCompletedObject = "could not put a completed object (multipart upload)"
|
|
||||||
CouldNotDeleteUploadPart = "could not delete upload part"
|
|
||||||
PutObject = "put object"
|
|
||||||
CouldNotFetchObjectMeta = "could not fetch object meta"
|
|
||||||
FailedToDeleteObject = "failed to delete object"
|
|
||||||
CouldntDeleteLifecycleObject = "couldn't delete lifecycle configuration object"
|
|
||||||
)
|
|
||||||
|
|
||||||
// External blockchain.
|
|
||||||
const (
|
|
||||||
UserGroupsListIsEmpty = "user groups list is empty, subject not found"
|
|
||||||
)
|
|
||||||
|
|
||||||
// External storage tree.
|
|
||||||
const (
|
|
||||||
FoundMoreThanOneUnversionedNode = "found more than one unversioned node"
|
|
||||||
GetTreeNode = "get tree node"
|
|
||||||
InvalidTreeKV = "invalid tree service meta KV"
|
InvalidTreeKV = "invalid tree service meta KV"
|
||||||
FailedToRemoveOldSystemNode = "failed to remove old system node"
|
FailedToWriteResponse = "failed to write response"
|
||||||
GetBucketLifecycle = "get bucket lifecycle"
|
WarnDuplicateAddress = "duplicate address"
|
||||||
FailedToRemoveOldPartNode = "failed to remove old part node"
|
PolicyCouldntBeConvertedToNativeRules = "policy couldn't be converted to native rules, only s3 rules be applied"
|
||||||
SystemNodeHasMultipleIDs = "system node has multiple ids"
|
CouldntCacheSubject = "couldn't cache subject info"
|
||||||
|
UserGroupsListIsEmpty = "user groups list is empty, subject not found"
|
||||||
|
CouldntCacheUserKey = "couldn't cache user key"
|
||||||
ObjectTaggingNodeHasMultipleIDs = "object tagging node has multiple ids"
|
ObjectTaggingNodeHasMultipleIDs = "object tagging node has multiple ids"
|
||||||
BucketTaggingNodeHasMultipleIDs = "bucket tagging node has multiple ids"
|
BucketTaggingNodeHasMultipleIDs = "bucket tagging node has multiple ids"
|
||||||
BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids"
|
BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids"
|
||||||
BucketCORSNodeHasMultipleIDs = "bucket cors node has multiple ids"
|
BucketCORSNodeHasMultipleIDs = "bucket cors node has multiple ids"
|
||||||
GetBucketCorsFromTree = "get bucket cors from tree"
|
SystemNodeHasMultipleIDs = "system node has multiple ids"
|
||||||
)
|
FailedToRemoveOldSystemNode = "failed to remove old system node"
|
||||||
|
FailedToParseAddressInTreeNode = "failed to parse object addr in tree node"
|
||||||
// Authmate.
|
UnexpectedMultiNodeIDsInSubTreeMultiParts = "unexpected multi node ids in sub tree multi parts"
|
||||||
const (
|
FoundSeveralSystemNodes = "found several system nodes"
|
||||||
CreateContainer = "create container"
|
FailedToParsePartInfo = "failed to parse part info"
|
||||||
CheckContainer = "check container"
|
CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info"
|
||||||
CheckCustomAccessKeyIDUniqueness = "check custom access key id uniqueness"
|
CouldNotFetchAccessBoxContainerInfo = "couldn't fetch AccessBox container info"
|
||||||
StoreBearerTokenIntoFrostFS = "store bearer token into FrostFS"
|
|
||||||
UpdateAccessCredObjectIntoFrostFS = "update access cred object into FrostFS"
|
|
||||||
SubjectAlreadyExistsInFrostFSID = "subject already exists in frostfsid"
|
|
||||||
SetSubjectNameInFrostFSID = "set subject name in frostfsid"
|
|
||||||
AddPolicyChainRules = "add policy chain rules"
|
|
||||||
PrepareConnectionPool = "prepare connection pool"
|
|
||||||
PrepareFrostfsIDClient = "prepare frostfsid client"
|
|
||||||
PreparePolicyClient = "prepare policy client"
|
|
||||||
CreateSubjectInFrostFSID = "create subject in frostfsid"
|
|
||||||
CloseCredsObjectPayload = "close creds object payload"
|
CloseCredsObjectPayload = "close creds object payload"
|
||||||
)
|
CouldntDeleteLifecycleObject = "couldn't delete lifecycle configuration object"
|
||||||
|
CouldntCacheLifecycleConfiguration = "couldn't cache lifecycle configuration"
|
||||||
// Log HTTP.
|
CouldNotFetchLifecycleContainerInfo = "couldn't fetch lifecycle container info"
|
||||||
const (
|
BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids"
|
||||||
LogHTTP = "http log"
|
GetBucketLifecycle = "get bucket lifecycle"
|
||||||
FailedToCloseHTTPBody = "failed to close http body"
|
WarnDuplicateNamespaceVHS = "duplicate namespace with enabled VHS, config value skipped"
|
||||||
FailedToInitializeHTTPLogger = "failed to initialize http logger"
|
WarnValueVHSEnabledFlagWrongType = "the value of the VHS enable flag for the namespace is of the wrong type, config value skipped"
|
||||||
FailedToReadHTTPBody = "failed to read http body"
|
WarnDomainContainsInvalidPlaceholder = "the domain contains an invalid placeholder, domain skipped"
|
||||||
FailedToProcessHTTPBody = "failed to process http body"
|
FailedToRemoveOldPartNode = "failed to remove old part node"
|
||||||
|
CouldntCacheNetworkInfo = "couldn't cache network info"
|
||||||
|
NotSupported = "not supported"
|
||||||
|
CheckCustomAccessKeyIDUniqueness = "check custom access key id uniqueness"
|
||||||
|
FailedToLoadMultinetConfig = "failed to load multinet config"
|
||||||
|
MultinetConfigWontBeUpdated = "multinet config won't be updated"
|
||||||
|
MultinetDialSuccess = "multinet dial successful"
|
||||||
|
MultinetDialFail = "multinet dial failed"
|
||||||
|
FailedToParseHTTPTime = "failed to parse http time, header is ignored"
|
||||||
|
FailedToPutTombstoneObject = "failed to put tombstone object"
|
||||||
|
FailedToCreateWorkerPool = "failed to create worker pool"
|
||||||
|
FailedToListAllObjectRelations = "failed to list all object relations"
|
||||||
|
WarnInvalidTypeTLSTerminationHeader = "invalid type of value of tls termination header"
|
||||||
|
FailedToPutTombstones = "failed to put tombstones"
|
||||||
|
WarnDomainContainsPort = "the domain contains a port, domain skipped"
|
||||||
|
CouldntCacheNetmap = "couldn't cache netmap"
|
||||||
|
BucketSettingsNotFoundUseDefaults = "bucket settings not found, use defaults"
|
||||||
)
|
)
|
||||||
|
|
|
@ -17,16 +17,9 @@ func (l LogEventHandler) DialPerformed(sourceIP net.Addr, _, address string, err
|
||||||
sourceIPString = sourceIP.Network() + "://" + sourceIP.String()
|
sourceIPString = sourceIP.Network() + "://" + sourceIP.String()
|
||||||
}
|
}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
l.logger.Debug(logs.MultinetDialSuccess,
|
l.logger.Debug(logs.MultinetDialSuccess, zap.String("source", sourceIPString), zap.String("destination", address))
|
||||||
zap.String("source", sourceIPString),
|
|
||||||
zap.String("destination", address),
|
|
||||||
logs.TagField(logs.TagApp))
|
|
||||||
} else {
|
} else {
|
||||||
l.logger.Debug(logs.MultinetDialFail,
|
l.logger.Debug(logs.MultinetDialFail, zap.String("source", sourceIPString), zap.String("destination", address), zap.Error(err))
|
||||||
zap.String("source", sourceIPString),
|
|
||||||
zap.String("destination", address),
|
|
||||||
zap.Error(err),
|
|
||||||
logs.TagField(logs.TagApp))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -27,7 +27,7 @@ type AppMetricsConfig struct {
|
||||||
|
|
||||||
func NewAppMetrics(cfg AppMetricsConfig) *AppMetrics {
|
func NewAppMetrics(cfg AppMetricsConfig) *AppMetrics {
|
||||||
if !cfg.Enabled {
|
if !cfg.Enabled {
|
||||||
cfg.Logger.Warn(logs.MetricsAreDisabled, logs.TagField(logs.TagApp))
|
cfg.Logger.Warn(logs.MetricsAreDisabled)
|
||||||
}
|
}
|
||||||
|
|
||||||
registry := cfg.Registerer
|
registry := cfg.Registerer
|
||||||
|
@ -44,7 +44,7 @@ func NewAppMetrics(cfg AppMetricsConfig) *AppMetrics {
|
||||||
|
|
||||||
func (m *AppMetrics) SetEnabled(enabled bool) {
|
func (m *AppMetrics) SetEnabled(enabled bool) {
|
||||||
if !enabled {
|
if !enabled {
|
||||||
m.logger.Warn(logs.MetricsAreDisabled, logs.TagField(logs.TagApp))
|
m.logger.Warn(logs.MetricsAreDisabled)
|
||||||
}
|
}
|
||||||
|
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
|
|
|
@ -11,7 +11,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
|
||||||
|
@ -258,7 +257,7 @@ func newNodeVersionFromTreeNode(log *zap.Logger, filePath string, treeNode *tree
|
||||||
|
|
||||||
if createdStr, ok := treeNode.Get(createdKV); ok {
|
if createdStr, ok := treeNode.Get(createdKV); ok {
|
||||||
if utcMilli, err := strconv.ParseInt(createdStr, 10, 64); err != nil {
|
if utcMilli, err := strconv.ParseInt(createdStr, 10, 64); err != nil {
|
||||||
log.Warn(logs.InvalidTreeKV, zap.String(createdKV, createdStr), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
|
log.Warn(logs.InvalidTreeKV, zap.String(createdKV, createdStr), zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
created := time.UnixMilli(utcMilli)
|
created := time.UnixMilli(utcMilli)
|
||||||
version.Created = &created
|
version.Created = &created
|
||||||
|
@ -268,7 +267,7 @@ func newNodeVersionFromTreeNode(log *zap.Logger, filePath string, treeNode *tree
|
||||||
if ownerStr, ok := treeNode.Get(ownerKV); ok {
|
if ownerStr, ok := treeNode.Get(ownerKV); ok {
|
||||||
var owner user.ID
|
var owner user.ID
|
||||||
if err := owner.DecodeString(ownerStr); err != nil {
|
if err := owner.DecodeString(ownerStr); err != nil {
|
||||||
log.Warn(logs.InvalidTreeKV, zap.String(ownerKV, ownerStr), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
|
log.Warn(logs.InvalidTreeKV, zap.String(ownerKV, ownerStr), zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
version.Owner = &owner
|
version.Owner = &owner
|
||||||
}
|
}
|
||||||
|
@ -276,7 +275,7 @@ func newNodeVersionFromTreeNode(log *zap.Logger, filePath string, treeNode *tree
|
||||||
|
|
||||||
if creationEpoch, ok := treeNode.Get(creationEpochKV); ok {
|
if creationEpoch, ok := treeNode.Get(creationEpochKV); ok {
|
||||||
if epoch, err := strconv.ParseUint(creationEpoch, 10, 64); err != nil {
|
if epoch, err := strconv.ParseUint(creationEpoch, 10, 64); err != nil {
|
||||||
log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, creationEpoch), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
|
log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, creationEpoch), zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
version.CreationEpoch = epoch
|
version.CreationEpoch = epoch
|
||||||
}
|
}
|
||||||
|
@ -343,13 +342,13 @@ func newMultipartInfoFromTreeNode(log *zap.Logger, filePath string, treeNode *tr
|
||||||
|
|
||||||
if ownerID, ok := treeNode.Get(ownerKV); ok {
|
if ownerID, ok := treeNode.Get(ownerKV); ok {
|
||||||
if err := multipartInfo.Owner.DecodeString(ownerID); err != nil {
|
if err := multipartInfo.Owner.DecodeString(ownerID); err != nil {
|
||||||
log.Warn(logs.InvalidTreeKV, zap.String(ownerKV, ownerID), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
|
log.Warn(logs.InvalidTreeKV, zap.String(ownerKV, ownerID), zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if created, ok := treeNode.Get(createdKV); ok {
|
if created, ok := treeNode.Get(createdKV); ok {
|
||||||
if utcMilli, err := strconv.ParseInt(created, 10, 64); err != nil {
|
if utcMilli, err := strconv.ParseInt(created, 10, 64); err != nil {
|
||||||
log.Warn(logs.InvalidTreeKV, zap.String(createdKV, created), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
|
log.Warn(logs.InvalidTreeKV, zap.String(createdKV, created), zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
multipartInfo.Created = time.UnixMilli(utcMilli)
|
multipartInfo.Created = time.UnixMilli(utcMilli)
|
||||||
}
|
}
|
||||||
|
@ -357,7 +356,7 @@ func newMultipartInfoFromTreeNode(log *zap.Logger, filePath string, treeNode *tr
|
||||||
|
|
||||||
if finished, ok := treeNode.Get(finishedKV); ok {
|
if finished, ok := treeNode.Get(finishedKV); ok {
|
||||||
if flag, err := strconv.ParseBool(finished); err != nil {
|
if flag, err := strconv.ParseBool(finished); err != nil {
|
||||||
log.Warn(logs.InvalidTreeKV, zap.String(finishedKV, finished), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
|
log.Warn(logs.InvalidTreeKV, zap.String(finishedKV, finished), zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
multipartInfo.Finished = flag
|
multipartInfo.Finished = flag
|
||||||
}
|
}
|
||||||
|
@ -365,7 +364,7 @@ func newMultipartInfoFromTreeNode(log *zap.Logger, filePath string, treeNode *tr
|
||||||
|
|
||||||
if creationEpoch, ok := treeNode.Get(creationEpochKV); ok {
|
if creationEpoch, ok := treeNode.Get(creationEpochKV); ok {
|
||||||
if epoch, err := strconv.ParseUint(creationEpoch, 10, 64); err != nil {
|
if epoch, err := strconv.ParseUint(creationEpoch, 10, 64); err != nil {
|
||||||
log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, creationEpoch), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
|
log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, creationEpoch), zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
multipartInfo.CreationEpoch = epoch
|
multipartInfo.CreationEpoch = epoch
|
||||||
}
|
}
|
||||||
|
@ -396,23 +395,23 @@ func newMultipartInfo(log *zap.Logger, node NodeResponse) (*data.MultipartInfo,
|
||||||
multipartInfo.Key = string(kv.GetValue())
|
multipartInfo.Key = string(kv.GetValue())
|
||||||
case createdKV:
|
case createdKV:
|
||||||
if utcMilli, err := strconv.ParseInt(string(kv.GetValue()), 10, 64); err != nil {
|
if utcMilli, err := strconv.ParseInt(string(kv.GetValue()), 10, 64); err != nil {
|
||||||
log.Warn(logs.InvalidTreeKV, zap.String(createdKV, string(kv.GetValue())), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
|
log.Warn(logs.InvalidTreeKV, zap.String(createdKV, string(kv.GetValue())), zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
multipartInfo.Created = time.UnixMilli(utcMilli)
|
multipartInfo.Created = time.UnixMilli(utcMilli)
|
||||||
}
|
}
|
||||||
case ownerKV:
|
case ownerKV:
|
||||||
if err := multipartInfo.Owner.DecodeString(string(kv.GetValue())); err != nil {
|
if err := multipartInfo.Owner.DecodeString(string(kv.GetValue())); err != nil {
|
||||||
log.Warn(logs.InvalidTreeKV, zap.String(ownerKV, string(kv.GetValue())), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
|
log.Warn(logs.InvalidTreeKV, zap.String(ownerKV, string(kv.GetValue())), zap.Error(err))
|
||||||
}
|
}
|
||||||
case finishedKV:
|
case finishedKV:
|
||||||
if isFinished, err := strconv.ParseBool(string(kv.GetValue())); err != nil {
|
if isFinished, err := strconv.ParseBool(string(kv.GetValue())); err != nil {
|
||||||
log.Warn(logs.InvalidTreeKV, zap.String(finishedKV, string(kv.GetValue())), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
|
log.Warn(logs.InvalidTreeKV, zap.String(finishedKV, string(kv.GetValue())), zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
multipartInfo.Finished = isFinished
|
multipartInfo.Finished = isFinished
|
||||||
}
|
}
|
||||||
case creationEpochKV:
|
case creationEpochKV:
|
||||||
if epoch, err := strconv.ParseUint(string(kv.GetValue()), 10, 64); err != nil {
|
if epoch, err := strconv.ParseUint(string(kv.GetValue()), 10, 64); err != nil {
|
||||||
log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, string(kv.GetValue())), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
|
log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, string(kv.GetValue())), zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
multipartInfo.CreationEpoch = epoch
|
multipartInfo.CreationEpoch = epoch
|
||||||
}
|
}
|
||||||
|
@ -494,9 +493,6 @@ func newPartInfo(node NodeResponse) (*data.PartInfoExtended, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
|
func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSettingsNode")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
multiNode, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
|
multiNode, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("couldn't get node: %w", err)
|
return nil, fmt.Errorf("couldn't get node: %w", err)
|
||||||
|
@ -519,7 +515,7 @@ func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*
|
||||||
|
|
||||||
if ownerKeyHex, ok := node.Get(ownerKeyKV); ok {
|
if ownerKeyHex, ok := node.Get(ownerKeyKV); ok {
|
||||||
if settings.OwnerKey, err = keys.NewPublicKeyFromString(ownerKeyHex); err != nil {
|
if settings.OwnerKey, err = keys.NewPublicKeyFromString(ownerKeyHex); err != nil {
|
||||||
c.reqLogger(ctx).Error(logs.SettingsNodeInvalidOwnerKey, zap.Error(err), logs.TagField(logs.TagDatapath))
|
c.reqLogger(ctx).Error(logs.SettingsNodeInvalidOwnerKey, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -527,9 +523,6 @@ func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) PutSettingsNode(ctx context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error {
|
func (c *Tree) PutSettingsNode(ctx context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.PutSettingsNode")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
multiNode, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
|
multiNode, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
|
||||||
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
|
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
|
||||||
if err != nil && !isErrNotFound {
|
if err != nil && !isErrNotFound {
|
||||||
|
@ -546,7 +539,7 @@ func (c *Tree) PutSettingsNode(ctx context.Context, bktInfo *data.BucketInfo, se
|
||||||
latest := multiNode.Latest()
|
latest := multiNode.Latest()
|
||||||
ind := latest.GetLatestNodeIndex()
|
ind := latest.GetLatestNodeIndex()
|
||||||
if latest.IsSplit() {
|
if latest.IsSplit() {
|
||||||
c.reqLogger(ctx).Error(logs.BucketSettingsNodeHasMultipleIDs, zap.Uint64s("ids", latest.ID), logs.TagField(logs.TagExternalStorageTree))
|
c.reqLogger(ctx).Error(logs.BucketSettingsNodeHasMultipleIDs, zap.Uint64s("ids", latest.ID))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
|
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
|
||||||
|
@ -559,9 +552,6 @@ func (c *Tree) PutSettingsNode(ctx context.Context, bktInfo *data.BucketInfo, se
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
|
func (c *Tree) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetBucketCORS")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
node, err := c.getSystemNode(ctx, bktInfo, corsFilename)
|
node, err := c.getSystemNode(ctx, bktInfo, corsFilename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return oid.Address{}, err
|
return oid.Address{}, err
|
||||||
|
@ -571,9 +561,6 @@ func (c *Tree) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
|
func (c *Tree) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.PutBucketCORS")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
multiNode, err := c.getSystemNode(ctx, bktInfo, corsFilename)
|
multiNode, err := c.getSystemNode(ctx, bktInfo, corsFilename)
|
||||||
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
|
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
|
||||||
if err != nil && !isErrNotFound {
|
if err != nil && !isErrNotFound {
|
||||||
|
@ -595,7 +582,7 @@ func (c *Tree) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, addr
|
||||||
latest := multiNode.Latest()
|
latest := multiNode.Latest()
|
||||||
ind := latest.GetLatestNodeIndex()
|
ind := latest.GetLatestNodeIndex()
|
||||||
if latest.IsSplit() {
|
if latest.IsSplit() {
|
||||||
c.reqLogger(ctx).Error(logs.BucketCORSNodeHasMultipleIDs, logs.TagField(logs.TagExternalStorageTree))
|
c.reqLogger(ctx).Error(logs.BucketCORSNodeHasMultipleIDs)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
|
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
|
||||||
|
@ -614,9 +601,6 @@ func (c *Tree) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, addr
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
|
func (c *Tree) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.DeleteBucketCORS")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
multiNode, err := c.getSystemNode(ctx, bktInfo, corsFilename)
|
multiNode, err := c.getSystemNode(ctx, bktInfo, corsFilename)
|
||||||
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
|
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
|
||||||
if err != nil && !isErrNotFound {
|
if err != nil && !isErrNotFound {
|
||||||
|
@ -656,14 +640,14 @@ func (c *Tree) cleanOldNodes(ctx context.Context, nodes []*treeNode, bktInfo *da
|
||||||
for _, node := range nodes {
|
for _, node := range nodes {
|
||||||
ind := node.GetLatestNodeIndex()
|
ind := node.GetLatestNodeIndex()
|
||||||
if node.IsSplit() {
|
if node.IsSplit() {
|
||||||
c.reqLogger(ctx).Error(logs.SystemNodeHasMultipleIDs, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64s("ids", node.ID), logs.TagField(logs.TagExternalStorageTree))
|
c.reqLogger(ctx).Error(logs.SystemNodeHasMultipleIDs, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64s("ids", node.ID))
|
||||||
}
|
}
|
||||||
if err := c.service.RemoveNode(ctx, bktInfo, systemTree, node.ID[ind]); err != nil {
|
if err := c.service.RemoveNode(ctx, bktInfo, systemTree, node.ID[ind]); err != nil {
|
||||||
c.reqLogger(ctx).Warn(logs.FailedToRemoveOldSystemNode, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64("id", node.ID[ind]), logs.TagField(logs.TagExternalStorageTree))
|
c.reqLogger(ctx).Warn(logs.FailedToRemoveOldSystemNode, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64("id", node.ID[ind]))
|
||||||
} else {
|
} else {
|
||||||
addr, err := getTreeNodeAddress(node)
|
addr, err := getTreeNodeAddress(node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.log.Warn(logs.FailedToParseAddressInTreeNode, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64("id", node.ID[ind]), logs.TagField(logs.TagDatapath))
|
c.log.Warn(logs.FailedToParseAddressInTreeNode, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64("id", node.ID[ind]))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
res = append(res, addr)
|
res = append(res, addr)
|
||||||
|
@ -674,9 +658,6 @@ func (c *Tree) cleanOldNodes(ctx context.Context, nodes []*treeNode, bktInfo *da
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, error) {
|
func (c *Tree) GetObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetObjectTagging")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
tagNode, err := c.getTreeNode(ctx, bktInfo, objVersion.ID, isTagKV)
|
tagNode, err := c.getTreeNode(ctx, bktInfo, objVersion.ID, isTagKV)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -702,9 +683,6 @@ func getObjectTagging(tagNode *treeNode) map[string]string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion, tagSet map[string]string) error {
|
func (c *Tree) PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion, tagSet map[string]string) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.PutObjectTagging")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
tagNode, err := c.getTreeNode(ctx, bktInfo, objVersion.ID, isTagKV)
|
tagNode, err := c.getTreeNode(ctx, bktInfo, objVersion.ID, isTagKV)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -724,23 +702,17 @@ func (c *Tree) PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, o
|
||||||
|
|
||||||
ind := tagNode.GetLatestNodeIndex()
|
ind := tagNode.GetLatestNodeIndex()
|
||||||
if tagNode.IsSplit() {
|
if tagNode.IsSplit() {
|
||||||
c.reqLogger(ctx).Error(logs.ObjectTaggingNodeHasMultipleIDs, logs.TagField(logs.TagExternalStorageTree))
|
c.reqLogger(ctx).Error(logs.ObjectTaggingNodeHasMultipleIDs)
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.service.MoveNode(ctx, bktInfo, versionTree, tagNode.ID[ind], objVersion.ID, treeTagSet)
|
return c.service.MoveNode(ctx, bktInfo, versionTree, tagNode.ID[ind], objVersion.ID, treeTagSet)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) DeleteObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) error {
|
func (c *Tree) DeleteObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.DeleteObjectTagging")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
return c.PutObjectTagging(ctx, bktInfo, objVersion, nil)
|
return c.PutObjectTagging(ctx, bktInfo, objVersion, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) {
|
func (c *Tree) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetBucketTagging")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketTaggingFilename)
|
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketTaggingFilename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -758,9 +730,6 @@ func (c *Tree) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error {
|
func (c *Tree) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.PutBucketTagging")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketTaggingFilename)
|
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketTaggingFilename)
|
||||||
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
|
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
|
||||||
if err != nil && !isErrNotFound {
|
if err != nil && !isErrNotFound {
|
||||||
|
@ -782,7 +751,7 @@ func (c *Tree) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, t
|
||||||
latest := multiNode.Latest()
|
latest := multiNode.Latest()
|
||||||
ind := latest.GetLatestNodeIndex()
|
ind := latest.GetLatestNodeIndex()
|
||||||
if latest.IsSplit() {
|
if latest.IsSplit() {
|
||||||
c.reqLogger(ctx).Error(logs.BucketTaggingNodeHasMultipleIDs, zap.Uint64s("ids", latest.ID), logs.TagField(logs.TagExternalStorageTree))
|
c.reqLogger(ctx).Error(logs.BucketTaggingNodeHasMultipleIDs, zap.Uint64s("ids", latest.ID))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, treeTagSet); err != nil {
|
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, treeTagSet); err != nil {
|
||||||
|
@ -795,9 +764,6 @@ func (c *Tree) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, t
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error {
|
func (c *Tree) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.DeleteBucketTagging")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
return c.PutBucketTagging(ctx, bktInfo, nil)
|
return c.PutBucketTagging(ctx, bktInfo, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -841,16 +807,10 @@ func (c *Tree) getTreeNodes(ctx context.Context, bktInfo *data.BucketInfo, nodeI
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetVersions(ctx context.Context, bktInfo *data.BucketInfo, filepath string) ([]*data.NodeVersion, error) {
|
func (c *Tree) GetVersions(ctx context.Context, bktInfo *data.BucketInfo, filepath string) ([]*data.NodeVersion, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetVersions")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
return c.getVersions(ctx, bktInfo, versionTree, filepath, false)
|
return c.getVersions(ctx, bktInfo, versionTree, filepath, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetLatestVersion(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) {
|
func (c *Tree) GetLatestVersion(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetLatestVersion")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
meta := []string{oidKV, isCombinedKV, isUnversionedKV, isDeleteMarkerKV, etagKV, sizeKV, md5KV, creationEpochKV}
|
meta := []string{oidKV, isCombinedKV, isUnversionedKV, isDeleteMarkerKV, etagKV, sizeKV, md5KV, creationEpochKV}
|
||||||
path := pathFromName(objectName)
|
path := pathFromName(objectName)
|
||||||
|
|
||||||
|
@ -1089,7 +1049,7 @@ func (s *VersionsByPrefixStreamImpl) parseNodeResponse(node NodeResponse) (res *
|
||||||
trNode, fileName, err := parseTreeNode(node)
|
trNode, fileName, err := parseTreeNode(node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, errNodeDoesntContainFileName) {
|
if !errors.Is(err, errNodeDoesntContainFileName) {
|
||||||
s.log.Debug(logs.ParseTreeNode, zap.Error(err), logs.TagField(logs.TagDatapath))
|
s.log.Debug(logs.ParseTreeNode, zap.Error(err))
|
||||||
}
|
}
|
||||||
return nil, true, nil
|
return nil, true, nil
|
||||||
}
|
}
|
||||||
|
@ -1120,9 +1080,6 @@ func (s *VersionsByPrefixStreamImpl) parseNodeResponse(node NodeResponse) (res *
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) InitVersionsByPrefixStream(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) (data.VersionsStream, error) {
|
func (c *Tree) InitVersionsByPrefixStream(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) (data.VersionsStream, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.InitVersionsByPrefixStream")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
mainStream, tailPrefix, rootID, err := c.getSubTreeByPrefixMainStream(ctx, bktInfo, versionTree, prefix)
|
mainStream, tailPrefix, rootID, err := c.getSubTreeByPrefixMainStream(ctx, bktInfo, versionTree, prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, io.EOF) {
|
if errors.Is(err, io.EOF) {
|
||||||
|
@ -1315,9 +1272,6 @@ func formLatestNodeKey(parentID uint64, fileName string) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetUnversioned(ctx context.Context, bktInfo *data.BucketInfo, filepath string) (*data.NodeVersion, error) {
|
func (c *Tree) GetUnversioned(ctx context.Context, bktInfo *data.BucketInfo, filepath string) (*data.NodeVersion, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetUnversioned")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
return c.getUnversioned(ctx, bktInfo, versionTree, filepath)
|
return c.getUnversioned(ctx, bktInfo, versionTree, filepath)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1333,9 +1287,7 @@ func (c *Tree) getUnversioned(ctx context.Context, bktInfo *data.BucketInfo, tre
|
||||||
|
|
||||||
if len(nodes) > 1 {
|
if len(nodes) > 1 {
|
||||||
c.reqLogger(ctx).Debug(logs.FoundMoreThanOneUnversionedNode,
|
c.reqLogger(ctx).Debug(logs.FoundMoreThanOneUnversionedNode,
|
||||||
zap.String("treeID", treeID),
|
zap.String("treeID", treeID), zap.String("filepath", filepath))
|
||||||
zap.String("filepath", filepath),
|
|
||||||
logs.TagField(logs.TagExternalStorageTree))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Slice(nodes, func(i, j int) bool {
|
sort.Slice(nodes, func(i, j int) bool {
|
||||||
|
@ -1346,23 +1298,14 @@ func (c *Tree) getUnversioned(ctx context.Context, bktInfo *data.BucketInfo, tre
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) AddVersion(ctx context.Context, bktInfo *data.BucketInfo, version *data.NodeVersion) (uint64, error) {
|
func (c *Tree) AddVersion(ctx context.Context, bktInfo *data.BucketInfo, version *data.NodeVersion) (uint64, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.AddVersion")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
return c.addVersion(ctx, bktInfo, versionTree, version)
|
return c.addVersion(ctx, bktInfo, versionTree, version)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) RemoveVersion(ctx context.Context, bktInfo *data.BucketInfo, id uint64) error {
|
func (c *Tree) RemoveVersion(ctx context.Context, bktInfo *data.BucketInfo, id uint64) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.RemoveVersion")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
return c.service.RemoveNode(ctx, bktInfo, versionTree, id)
|
return c.service.RemoveNode(ctx, bktInfo, versionTree, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) error {
|
func (c *Tree) CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.CreateMultipartUpload")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
path := pathFromName(info.Key)
|
path := pathFromName(info.Key)
|
||||||
meta := metaFromMultipart(info, path[len(path)-1])
|
meta := metaFromMultipart(info, path[len(path)-1])
|
||||||
_, err := c.service.AddNodeByPath(ctx, bktInfo, systemTree, path[:len(path)-1], meta)
|
_, err := c.service.AddNodeByPath(ctx, bktInfo, systemTree, path[:len(path)-1], meta)
|
||||||
|
@ -1371,9 +1314,6 @@ func (c *Tree) CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketIn
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.MultipartInfo, error) {
|
func (c *Tree) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.MultipartInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetMultipartUploadsByPrefix")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
subTreeNodes, headPrefix, err := c.getSubTreeByPrefix(ctx, bktInfo, systemTree, prefix, false)
|
subTreeNodes, headPrefix, err := c.getSubTreeByPrefix(ctx, bktInfo, systemTree, prefix, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -1454,9 +1394,6 @@ func (c *Tree) getSubTreeMultipartUploads(ctx context.Context, bktInfo *data.Buc
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error) {
|
func (c *Tree) GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetMultipartUpload")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
path := pathFromName(objectName)
|
path := pathFromName(objectName)
|
||||||
p := &GetNodesParams{
|
p := &GetNodesParams{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
|
@ -1488,9 +1425,6 @@ func (c *Tree) GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDsToDelete []oid.ID, err error) {
|
func (c *Tree) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDsToDelete []oid.ID, err error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.AddPart")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
parts, err := c.service.GetSubTree(ctx, bktInfo, systemTree, []uint64{multipartNodeID}, 2, false)
|
parts, err := c.service.GetSubTree(ctx, bktInfo, systemTree, []uint64{multipartNodeID}, 2, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -1526,8 +1460,7 @@ func (c *Tree) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartN
|
||||||
zap.String("upload id", info.UploadID),
|
zap.String("upload id", info.UploadID),
|
||||||
zap.Uint64("multipart node id ", multipartNodeID),
|
zap.Uint64("multipart node id ", multipartNodeID),
|
||||||
zap.Uint64s("id", part.GetNodeID()),
|
zap.Uint64s("id", part.GetNodeID()),
|
||||||
zap.Error(err),
|
zap.Error(err))
|
||||||
logs.TagField(logs.TagDatapath))
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if partInfo.Number == info.Number {
|
if partInfo.Number == info.Number {
|
||||||
|
@ -1555,8 +1488,7 @@ func (c *Tree) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartN
|
||||||
c.reqLogger(ctx).Warn(logs.FailedToRemoveOldPartNode,
|
c.reqLogger(ctx).Warn(logs.FailedToRemoveOldPartNode,
|
||||||
zap.String("key", info.Key),
|
zap.String("key", info.Key),
|
||||||
zap.String("upload id", info.UploadID),
|
zap.String("upload id", info.UploadID),
|
||||||
zap.Uint64("id", nodeID),
|
zap.Uint64("id", nodeID))
|
||||||
logs.TagField(logs.TagExternalStorageTree))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1571,9 +1503,6 @@ func (c *Tree) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartN
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfoExtended, error) {
|
func (c *Tree) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfoExtended, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetParts")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
parts, err := c.service.GetSubTree(ctx, bktInfo, systemTree, []uint64{multipartNodeID}, 2, false)
|
parts, err := c.service.GetSubTree(ctx, bktInfo, systemTree, []uint64{multipartNodeID}, 2, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -1585,8 +1514,7 @@ func (c *Tree) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipart
|
||||||
// multipart parts nodeID shouldn't have multiple values
|
// multipart parts nodeID shouldn't have multiple values
|
||||||
c.reqLogger(ctx).Warn(logs.UnexpectedMultiNodeIDsInSubTreeMultiParts,
|
c.reqLogger(ctx).Warn(logs.UnexpectedMultiNodeIDsInSubTreeMultiParts,
|
||||||
zap.Uint64("multipart node id ", multipartNodeID),
|
zap.Uint64("multipart node id ", multipartNodeID),
|
||||||
zap.Uint64s("node ids", part.GetNodeID()),
|
zap.Uint64s("node ids", part.GetNodeID()))
|
||||||
logs.TagField(logs.TagDatapath))
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if part.GetNodeID()[0] == multipartNodeID {
|
if part.GetNodeID()[0] == multipartNodeID {
|
||||||
|
@ -1597,8 +1525,7 @@ func (c *Tree) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipart
|
||||||
c.reqLogger(ctx).Warn(logs.FailedToParsePartInfo,
|
c.reqLogger(ctx).Warn(logs.FailedToParsePartInfo,
|
||||||
zap.Uint64("multipart node id ", multipartNodeID),
|
zap.Uint64("multipart node id ", multipartNodeID),
|
||||||
zap.Uint64s("node ids", part.GetNodeID()),
|
zap.Uint64s("node ids", part.GetNodeID()),
|
||||||
zap.Error(err),
|
zap.Error(err))
|
||||||
logs.TagField(logs.TagDatapath))
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
result = append(result, partInfo)
|
result = append(result, partInfo)
|
||||||
|
@ -1608,9 +1535,6 @@ func (c *Tree) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipart
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
|
func (c *Tree) PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.PutBucketLifecycleConfiguration")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
||||||
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
|
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
|
||||||
if err != nil && !isErrNotFound {
|
if err != nil && !isErrNotFound {
|
||||||
|
@ -1632,7 +1556,7 @@ func (c *Tree) PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *dat
|
||||||
latest := multiNode.Latest()
|
latest := multiNode.Latest()
|
||||||
ind := latest.GetLatestNodeIndex()
|
ind := latest.GetLatestNodeIndex()
|
||||||
if latest.IsSplit() {
|
if latest.IsSplit() {
|
||||||
c.reqLogger(ctx).Error(logs.BucketLifecycleNodeHasMultipleIDs, logs.TagField(logs.TagDatapath))
|
c.reqLogger(ctx).Error(logs.BucketLifecycleNodeHasMultipleIDs)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
|
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
|
||||||
|
@ -1651,9 +1575,6 @@ func (c *Tree) PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *dat
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
|
func (c *Tree) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetBucketLifecycleConfiguration")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
node, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
node, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return oid.Address{}, fmt.Errorf("get lifecycle node: %w", err)
|
return oid.Address{}, fmt.Errorf("get lifecycle node: %w", err)
|
||||||
|
@ -1663,9 +1584,6 @@ func (c *Tree) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *dat
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
|
func (c *Tree) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.DeleteBucketLifecycleConfiguration")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
||||||
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
|
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
|
||||||
if err != nil && !isErrNotFound {
|
if err != nil && !isErrNotFound {
|
||||||
|
@ -1685,9 +1603,6 @@ func (c *Tree) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
func (c *Tree) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.DeleteMultipartUpload")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
err := c.service.RemoveNode(ctx, bktInfo, systemTree, multipartInfo.ID)
|
err := c.service.RemoveNode(ctx, bktInfo, systemTree, multipartInfo.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -1699,9 +1614,6 @@ func (c *Tree) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketIn
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) PutLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error {
|
func (c *Tree) PutLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.PutLock")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
meta := map[string]string{isLockKV: "true"}
|
meta := map[string]string{isLockKV: "true"}
|
||||||
|
|
||||||
if lock.IsLegalHoldSet() {
|
if lock.IsLegalHoldSet() {
|
||||||
|
@ -1724,9 +1636,6 @@ func (c *Tree) PutLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uin
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error) {
|
func (c *Tree) GetLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetLock")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
lockNode, err := c.getTreeNode(ctx, bktInfo, nodeID, isLockKV)
|
lockNode, err := c.getTreeNode(ctx, bktInfo, nodeID, isLockKV)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -1766,9 +1675,6 @@ func getLock(lockNode *treeNode) (*data.LockInfo, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
|
func (c *Tree) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetObjectTaggingAndLock")
|
|
||||||
defer span.End()
|
|
||||||
|
|
||||||
nodes, err := c.getTreeNodes(ctx, bktInfo, objVersion.ID, isTagKV, isLockKV)
|
nodes, err := c.getTreeNodes(ctx, bktInfo, objVersion.ID, isTagKV, isLockKV)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
|
@ -1925,7 +1831,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name
|
||||||
return nil, tree.ErrNodeNotFound
|
return nil, tree.ErrNodeNotFound
|
||||||
}
|
}
|
||||||
if len(nodes) != 1 {
|
if len(nodes) != 1 {
|
||||||
c.reqLogger(ctx).Warn(logs.FoundSeveralSystemNodes, zap.String("name", name), logs.TagField(logs.TagDatapath))
|
c.reqLogger(ctx).Warn(logs.FoundSeveralSystemNodes, zap.String("name", name))
|
||||||
}
|
}
|
||||||
|
|
||||||
return newMultiNode(nodes)
|
return newMultiNode(nodes)
|
||||||
|
|
Loading…
Add table
Reference in a new issue