Compare commits

...

31 commits

Author SHA1 Message Date
bfec3e0a5e [#619] Fix content-length invalid check
All checks were successful
/ Vulncheck (push) Successful in 1m8s
/ Builds (push) Successful in 1m2s
/ OCI image (push) Successful in 2m7s
/ Lint (push) Successful in 2m7s
/ Tests (push) Successful in 1m19s
Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2025-02-18 14:15:23 +00:00
711d6b2c71 [#642] Simplify tests
Some checks failed
/ DCO (pull_request) Successful in 35s
/ Vulncheck (pull_request) Successful in 52s
/ Builds (pull_request) Successful in 1m33s
/ OCI image (pull_request) Successful in 2m13s
/ Lint (pull_request) Successful in 2m16s
/ Tests (pull_request) Successful in 1m51s
/ Vulncheck (push) Successful in 1m8s
/ Builds (push) Successful in 1m3s
/ Tests (push) Successful in 1m17s
/ OCI image (push) Successful in 2m9s
/ Lint (push) Has been cancelled
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-02-17 09:44:41 +03:00
092567a5a0 Release v0.32.10
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-02-14 16:11:27 +03:00
e0a54fcbd3 [#642] Fix streaming empty body
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-02-13 16:55:27 +03:00
e184b333e4 [#612] Port changelog from support branch
All checks were successful
/ DCO (pull_request) Successful in 32s
/ Vulncheck (pull_request) Successful in 1m2s
/ Builds (pull_request) Successful in 1m29s
/ OCI image (pull_request) Successful in 1m58s
/ Lint (pull_request) Successful in 2m20s
/ Tests (pull_request) Successful in 1m17s
/ Vulncheck (push) Successful in 56s
/ Builds (push) Successful in 1m10s
/ OCI image (push) Successful in 2m0s
/ Lint (push) Successful in 2m27s
/ Tests (push) Successful in 1m22s
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2025-02-12 14:50:09 +03:00
853036e44e [#612] Make Content-Md5 header check optional
All checks were successful
/ DCO (pull_request) Successful in 34s
/ Vulncheck (pull_request) Successful in 51s
/ Builds (pull_request) Successful in 1m38s
/ OCI image (pull_request) Successful in 2m19s
/ Lint (pull_request) Successful in 2m27s
/ Tests (pull_request) Successful in 1m53s
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2025-02-12 14:29:19 +03:00
ee46382a68 [#606] Reorganize some log tags
Some checks failed
/ DCO (pull_request) Successful in 32s
/ Vulncheck (pull_request) Successful in 1m12s
/ Builds (pull_request) Successful in 58s
/ OCI image (pull_request) Successful in 2m4s
/ Lint (pull_request) Successful in 2m23s
/ Tests (pull_request) Successful in 1m13s
/ Vulncheck (push) Successful in 42s
/ Builds (push) Successful in 1m12s
/ Lint (push) Successful in 2m25s
/ Tests (push) Successful in 1m7s
/ OCI image (push) Failing after 11m37s
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2025-02-11 16:47:42 +03:00
b207eb48d9 [#606] Use all available log tags by default
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2025-02-11 16:47:36 +03:00
b7650e01ac [#606] Make log tags more explicit
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2025-02-11 16:47:30 +03:00
e7f620f137 [#606] Support log tagging
All checks were successful
/ DCO (pull_request) Successful in 28s
/ Vulncheck (pull_request) Successful in 1m6s
/ Builds (pull_request) Successful in 57s
/ OCI image (pull_request) Successful in 2m0s
/ Lint (pull_request) Successful in 3m19s
/ Tests (pull_request) Successful in 1m12s
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
2025-02-11 15:12:20 +03:00
ffac62e8b4 [#606] logs: Delete comments
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
2025-02-11 15:12:20 +03:00
182262ace2 [#636] Bump go version in vulncheck
All checks were successful
/ DCO (pull_request) Successful in 32s
/ Vulncheck (pull_request) Successful in 56s
/ Builds (pull_request) Successful in 1m6s
/ Lint (pull_request) Successful in 2m22s
/ Tests (pull_request) Successful in 1m15s
/ OCI image (pull_request) Successful in 2m31s
/ Builds (push) Successful in 1m3s
/ Vulncheck (push) Successful in 1m4s
/ OCI image (push) Successful in 2m11s
/ Lint (push) Successful in 3m11s
/ Tests (push) Successful in 1m14s
go1.23.5 triggers GO-2025-3447 but this is applicable
only for ppc64le platform.

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2025-02-11 11:48:22 +03:00
893b506c83 [#626] Fix ALREADY REMOVED response status code
Some checks failed
/ DCO (pull_request) Successful in 30s
/ Vulncheck (pull_request) Failing after 1m15s
/ Builds (pull_request) Successful in 1m27s
/ OCI image (pull_request) Successful in 2m14s
/ Lint (pull_request) Successful in 3m7s
/ Tests (pull_request) Successful in 1m16s
/ Vulncheck (push) Failing after 1m25s
/ Builds (push) Successful in 58s
/ OCI image (push) Successful in 2m7s
/ Lint (push) Successful in 2m22s
/ Tests (push) Successful in 1m17s
Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>
2025-02-10 16:10:45 +03:00
beec37797d [#626] Fix ALREADY REMOVED response status code
Some checks failed
/ DCO (pull_request) Successful in 40s
/ Vulncheck (pull_request) Failing after 1m45s
/ Builds (pull_request) Successful in 2m4s
/ OCI image (pull_request) Successful in 3m3s
/ Lint (pull_request) Successful in 3m21s
/ Tests (pull_request) Successful in 1m24s
Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>
2025-02-07 17:37:40 +03:00
5538dce772 [#628] Add tree_stream_timeout config parameter
Some checks failed
/ DCO (pull_request) Successful in 36s
/ Vulncheck (pull_request) Failing after 58s
/ Builds (pull_request) Successful in 1m44s
/ Lint (pull_request) Successful in 2m17s
/ Tests (pull_request) Successful in 2m11s
/ OCI image (pull_request) Successful in 2m40s
/ Vulncheck (push) Failing after 1m7s
/ Builds (push) Successful in 1m9s
/ OCI image (push) Successful in 2m10s
/ Lint (push) Successful in 3m15s
/ Tests (push) Successful in 1m15s
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2025-02-07 11:57:54 +03:00
da9703ab63 [#623] Fix using copy numbers during multipart
All checks were successful
/ DCO (pull_request) Successful in 34s
/ Vulncheck (pull_request) Successful in 1m2s
/ Builds (pull_request) Successful in 1m12s
/ OCI image (pull_request) Successful in 2m1s
/ Lint (pull_request) Successful in 2m24s
/ Tests (pull_request) Successful in 1m22s
/ Vulncheck (push) Successful in 1m3s
/ Builds (push) Successful in 1m17s
/ Lint (push) Successful in 2m17s
/ Tests (push) Successful in 1m22s
/ OCI image (push) Successful in 2m36s
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-02-06 14:41:27 +03:00
a53e50b324 [#607] Support sigV4a streaming with trailers
All checks were successful
/ DCO (pull_request) Successful in 32s
/ Vulncheck (pull_request) Successful in 1m6s
/ Builds (pull_request) Successful in 1m34s
/ OCI image (pull_request) Successful in 2m2s
/ Lint (pull_request) Successful in 2m10s
/ Tests (pull_request) Successful in 1m17s
/ Vulncheck (push) Successful in 1m6s
/ Builds (push) Successful in 1m6s
/ Lint (push) Successful in 1m57s
/ Tests (push) Successful in 1m20s
/ OCI image (push) Successful in 2m10s
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-02-03 18:25:15 +03:00
5e9c562683 [#607] Fix aws example test for trailing with sigv4
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-02-03 18:25:15 +03:00
49bf3c1bce [#607] Support sigV4 streaming with trailers
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-02-03 18:25:15 +03:00
a4d9658fbb [#607] Support unsigned payload streaming with trailers
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-02-03 18:25:15 +03:00
bec63026bd [#607] Support unsigned payload streaming
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-02-03 18:25:15 +03:00
0064e7ab07 [#618] Port changelog from support branch
All checks were successful
/ Vulncheck (push) Successful in 58s
/ Builds (push) Successful in 1m17s
/ OCI image (push) Successful in 2m10s
/ Lint (push) Successful in 2m21s
/ Tests (push) Successful in 1m46s
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2025-02-03 14:27:09 +00:00
41e1f1ad7a [#617] Bump SDK version to the latest master
Contains fixes:
- memory leak in gRPC client,
- panic and deadlock in tree pool.

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2025-02-03 14:27:09 +00:00
4d2e6f8650 [#610] Fix updateServers finding logic
All checks were successful
/ Vulncheck (push) Successful in 1m24s
/ Builds (push) Successful in 1m55s
/ OCI image (push) Successful in 2m36s
/ Lint (push) Successful in 2m41s
/ Tests (push) Successful in 2m2s
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-02-03 13:00:15 +00:00
3d3dd00211 [#615] Use UNSIGNED_PAYLOAD to check sign
All checks were successful
/ Vulncheck (push) Successful in 1m4s
/ Lint (push) Successful in 1m53s
/ Tests (push) Successful in 1m17s
/ OCI image (push) Successful in 2m7s
/ Builds (push) Successful in 1m2s
Use `UNSIGNED_PAYLOAD` to check signature if x-amz-content-sha256 isn't provided as signed header

https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html

" You include the literal string UNSIGNED-PAYLOAD when constructing a canonical request, and set the same value as the x-amz-content-sha256 header value when sending the request to Amazon S3"

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2025-01-30 13:16:40 +00:00
510b0a1005 [#614] govulncheck: Use patch release with security fixes
All checks were successful
/ Vulncheck (push) Successful in 48s
/ Builds (push) Successful in 1m49s
/ OCI image (push) Successful in 2m28s
/ Lint (push) Successful in 3m3s
/ Tests (push) Successful in 1m48s
https://go.dev/doc/devel/release#go1.23.minor

Signed-off-by: Vitaliy Potyarkin <v.potyarkin@yadro.com>
2025-01-29 12:07:28 +00:00
da77e426b6 [#541] Fix setting of tls.enabled flag
Some checks failed
/ Builds (push) Has been cancelled
/ OCI image (push) Has been cancelled
/ Lint (push) Has been cancelled
/ Tests (push) Has been cancelled
/ Vulncheck (push) Has been cancelled
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2025-01-29 12:06:41 +00:00
e7a8d4bdaf [#605] Fix panic when payload discard
Some checks failed
/ DCO (pull_request) Successful in 30s
/ Vulncheck (pull_request) Successful in 1m7s
/ Builds (pull_request) Successful in 1m31s
/ Lint (pull_request) Successful in 2m4s
/ Tests (pull_request) Successful in 1m8s
/ OCI image (pull_request) Successful in 2m10s
/ Builds (push) Has been cancelled
/ OCI image (push) Has been cancelled
/ Lint (push) Has been cancelled
/ Vulncheck (push) Has been cancelled
/ Tests (push) Has been cancelled
Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2025-01-27 17:01:50 +03:00
250538a9b4 [#541] Use default value if config param is unset after SIGHUP
All checks were successful
/ DCO (pull_request) Successful in 31s
/ Vulncheck (pull_request) Successful in 1m2s
/ Builds (pull_request) Successful in 1m19s
/ Lint (pull_request) Successful in 2m6s
/ Tests (pull_request) Successful in 1m17s
/ OCI image (pull_request) Successful in 2m6s
/ Vulncheck (push) Successful in 1m4s
/ Builds (push) Successful in 1m6s
/ OCI image (push) Successful in 2m6s
/ Lint (push) Successful in 2m6s
/ Tests (push) Successful in 1m15s
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2025-01-23 09:52:48 +03:00
619385836d [#585] Add ListBuckets handler test
All checks were successful
/ Vulncheck (push) Successful in 1m2s
/ Builds (push) Successful in 1m5s
/ OCI image (push) Successful in 1m55s
/ Lint (push) Successful in 2m15s
/ Tests (push) Successful in 1m17s
Modify containers field in TestFrostFS in order to get determined order of containers between test runs

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
2025-01-21 07:49:19 +00:00
65fc776dea [#585] Add ListBuckets pagination
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
2025-01-21 07:49:19 +00:00
88 changed files with 3245 additions and 1103 deletions

View file

@ -16,7 +16,7 @@ jobs:
- name: Setup Go - name: Setup Go
uses: actions/setup-go@v3 uses: actions/setup-go@v3
with: with:
go-version: '1.23' go-version: '1.23.6'
- name: Install govulncheck - name: Install govulncheck
run: go install golang.org/x/vuln/cmd/govulncheck@latest run: go install golang.org/x/vuln/cmd/govulncheck@latest

View file

@ -4,6 +4,55 @@ This document outlines major changes between releases.
## [Unreleased] ## [Unreleased]
## [0.32.10] - 2025-02-14
### Fixed
- Chunk streaming empty body (#642)
## [0.32.9] - 2025-02-12
### Fixed
- Make `Content-Md5` header check optional (#612)
## [0.32.8] - 2025-02-11
### Fixed
- Return 404 instead of 500 when object is missing in object storage and available in the tree (#626)
### Added
- `tree_stream_timeout` configuration parameter (#627)
## [0.32.7] - 2025-02-06
### Fixed
- Correct passing copies number during multipart upload (#623)
## [0.32.6] - 2025-02-05
### Fixed
- Connection leak when `feature.tree_pool_netmap_support` is enabled (#622)
## [0.32.5] - 2025-02-04
### Fixed
- Support trailing headers signature during aws-chunk upload (#607)
## [0.32.4] - 2025-02-03
### Fixed
- Possible deadlock in tree pool component (#617)
- Possible memory leak in gRPC client (#617)
## [0.32.3] - 2025-01-29
### Fixed
- Use `UNSIGNED_PAYLOAD` as content hash to check signature if `x-amz-content-sha256` isn't signed header (#616)
## [0.32.2] - 2025-01-27
### Fixed
- Fix panic when payload discard (#605)
## [0.32.1] - 2025-01-17 ## [0.32.1] - 2025-01-17
### Fixed ### Fixed
@ -400,4 +449,13 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs
[0.31.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.2...v0.31.3 [0.31.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.2...v0.31.3
[0.32.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.3...v0.32.0 [0.32.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.3...v0.32.0
[0.32.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.0...v0.32.1 [0.32.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.0...v0.32.1
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.1...master [0.32.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.1...v0.32.2
[0.32.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.2...v0.32.3
[0.32.4]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.3...v0.32.4
[0.32.5]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.4...v0.32.5
[0.32.6]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.5...v0.32.6
[0.32.7]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.6...v0.32.7
[0.32.8]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.7...v0.32.8
[0.32.9]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.8...v0.32.9
[0.32.10]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.9...v0.32.10
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.10...master

View file

@ -1 +1 @@
v0.32.1 v0.32.10

View file

@ -13,6 +13,7 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"regexp" "regexp"
"slices"
"strings" "strings"
"time" "time"
@ -32,8 +33,8 @@ var (
// AuthorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter. // AuthorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
AuthorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`) AuthorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
// authorizationFieldV4aRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter. // AuthorizationFieldV4aRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
authorizationFieldV4aRegexp = regexp.MustCompile(`AWS4-ECDSA-P256-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`) AuthorizationFieldV4aRegexp = regexp.MustCompile(`AWS4-ECDSA-P256-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
// postPolicyCredentialRegexp -- is regexp for credentials when uploading file using POST with policy. // postPolicyCredentialRegexp -- is regexp for credentials when uploading file using POST with policy.
postPolicyCredentialRegexp = regexp.MustCompile(`(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request`) postPolicyCredentialRegexp = regexp.MustCompile(`(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request`)
@ -106,7 +107,7 @@ func New(creds tokens.Credentials, prefixes []string, settings CenterSettings) *
return &Center{ return &Center{
cli: creds, cli: creds,
reg: NewRegexpMatcher(AuthorizationFieldRegexp), reg: NewRegexpMatcher(AuthorizationFieldRegexp),
regV4a: NewRegexpMatcher(authorizationFieldV4aRegexp), regV4a: NewRegexpMatcher(AuthorizationFieldV4aRegexp),
postReg: NewRegexpMatcher(postPolicyCredentialRegexp), postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
allowedAccessKeyIDPrefixes: prefixes, allowedAccessKeyIDPrefixes: prefixes,
settings: settings, settings: settings,
@ -114,8 +115,8 @@ func New(creds tokens.Credentials, prefixes []string, settings CenterSettings) *
} }
const ( const (
signaturePreambleSigV4 = "AWS4-HMAC-SHA256" SignaturePreambleSigV4 = "AWS4-HMAC-SHA256"
signaturePreambleSigV4A = "AWS4-ECDSA-P256-SHA256" SignaturePreambleSigV4A = "AWS4-ECDSA-P256-SHA256"
) )
func (c *Center) parseAuthHeader(authHeader string, headers http.Header) (*AuthHeader, error) { func (c *Center) parseAuthHeader(authHeader string, headers http.Header) (*AuthHeader, error) {
@ -127,13 +128,13 @@ func (c *Center) parseAuthHeader(authHeader string, headers http.Header) (*AuthH
) )
switch preamble { switch preamble {
case signaturePreambleSigV4: case SignaturePreambleSigV4:
submatches = c.reg.GetSubmatches(authHeader) submatches = c.reg.GetSubmatches(authHeader)
if len(submatches) != authHeaderPartsNum { if len(submatches) != authHeaderPartsNum {
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrAuthorizationHeaderMalformed), authHeader) return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrAuthorizationHeaderMalformed), authHeader)
} }
region = submatches["region"] region = submatches["region"]
case signaturePreambleSigV4A: case SignaturePreambleSigV4A:
submatches = c.regV4a.GetSubmatches(authHeader) submatches = c.regV4a.GetSubmatches(authHeader)
if len(submatches) != authHeaderV4aPartsNum { if len(submatches) != authHeaderV4aPartsNum {
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrAuthorizationHeaderMalformed), authHeader) return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrAuthorizationHeaderMalformed), authHeader)
@ -169,7 +170,7 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
) )
queryValues := r.URL.Query() queryValues := r.URL.Query()
if queryValues.Get(AmzAlgorithm) == signaturePreambleSigV4 { if queryValues.Get(AmzAlgorithm) == SignaturePreambleSigV4 {
creds := strings.Split(queryValues.Get(AmzCredential), "/") creds := strings.Split(queryValues.Get(AmzCredential), "/")
if len(creds) != 5 || creds[4] != "aws4_request" { if len(creds) != 5 || creds[4] != "aws4_request" {
return nil, fmt.Errorf("bad X-Amz-Credential") return nil, fmt.Errorf("bad X-Amz-Credential")
@ -182,7 +183,7 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
SignedFields: strings.Split(queryValues.Get(AmzSignedHeaders), ";"), SignedFields: strings.Split(queryValues.Get(AmzSignedHeaders), ";"),
Date: creds[1], Date: creds[1],
IsPresigned: true, IsPresigned: true,
Preamble: signaturePreambleSigV4, Preamble: SignaturePreambleSigV4,
PayloadHash: r.Header.Get(AmzContentSHA256), PayloadHash: r.Header.Get(AmzContentSHA256),
} }
authHdr.Expiration, err = time.ParseDuration(queryValues.Get(AmzExpires) + "s") authHdr.Expiration, err = time.ParseDuration(queryValues.Get(AmzExpires) + "s")
@ -190,7 +191,7 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
return nil, fmt.Errorf("%w: couldn't parse X-Amz-Expires %v", apierr.GetAPIError(apierr.ErrMalformedExpires), err) return nil, fmt.Errorf("%w: couldn't parse X-Amz-Expires %v", apierr.GetAPIError(apierr.ErrMalformedExpires), err)
} }
signatureDateTimeStr = queryValues.Get(AmzDate) signatureDateTimeStr = queryValues.Get(AmzDate)
} else if queryValues.Get(AmzAlgorithm) == signaturePreambleSigV4A { } else if queryValues.Get(AmzAlgorithm) == SignaturePreambleSigV4A {
creds := strings.Split(queryValues.Get(AmzCredential), "/") creds := strings.Split(queryValues.Get(AmzCredential), "/")
if len(creds) != 4 || creds[3] != "aws4_request" { if len(creds) != 4 || creds[3] != "aws4_request" {
return nil, fmt.Errorf("bad X-Amz-Credential") return nil, fmt.Errorf("bad X-Amz-Credential")
@ -203,7 +204,7 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
SignedFields: strings.Split(queryValues.Get(AmzSignedHeaders), ";"), SignedFields: strings.Split(queryValues.Get(AmzSignedHeaders), ";"),
Date: creds[1], Date: creds[1],
IsPresigned: true, IsPresigned: true,
Preamble: signaturePreambleSigV4A, Preamble: SignaturePreambleSigV4A,
PayloadHash: r.Header.Get(AmzContentSHA256), PayloadHash: r.Header.Get(AmzContentSHA256),
} }
authHdr.Expiration, err = time.ParseDuration(queryValues.Get(AmzExpires) + "s") authHdr.Expiration, err = time.ParseDuration(queryValues.Get(AmzExpires) + "s")
@ -396,8 +397,12 @@ func cloneRequest(r *http.Request, authHeader *AuthHeader) *http.Request {
func (c *Center) checkSign(ctx context.Context, authHeader *AuthHeader, box *accessbox.Box, request *http.Request, signatureDateTime time.Time) error { func (c *Center) checkSign(ctx context.Context, authHeader *AuthHeader, box *accessbox.Box, request *http.Request, signatureDateTime time.Time) error {
var signature string var signature string
if !slices.Contains(authHeader.SignedFields, "x-amz-content-sha256") && authHeader.PayloadHash == "" {
authHeader.PayloadHash = UnsignedPayload
}
switch authHeader.Preamble { switch authHeader.Preamble {
case signaturePreambleSigV4: case SignaturePreambleSigV4:
creds := aws.Credentials{ creds := aws.Credentials{
AccessKeyID: authHeader.AccessKeyID, AccessKeyID: authHeader.AccessKeyID,
SecretAccessKey: box.Gate.SecretKey, SecretAccessKey: box.Gate.SecretKey,
@ -432,7 +437,7 @@ func (c *Center) checkSign(ctx context.Context, authHeader *AuthHeader, box *acc
authHeader.Signature, signature, authHeader.SignedFields) authHeader.Signature, signature, authHeader.SignedFields)
} }
case signaturePreambleSigV4A: case SignaturePreambleSigV4A:
signer := v4a.NewSigner(func(options *v4a.SignerOptions) { signer := v4a.NewSigner(func(options *v4a.SignerOptions) {
options.DisableURIPathEscaping = true options.DisableURIPathEscaping = true
}) })

View file

@ -8,6 +8,7 @@ import (
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/url" "net/url"
"os"
"strings" "strings"
"testing" "testing"
"time" "time"
@ -26,6 +27,9 @@ import (
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
smithyauth "github.com/aws/smithy-go/auth"
"github.com/aws/smithy-go/logging"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest" "go.uber.org/zap/zaptest"
@ -65,7 +69,7 @@ func TestAuthHeaderParse(t *testing.T) {
Signature: "2811ccb9e242f41426738fb1f", Signature: "2811ccb9e242f41426738fb1f",
SignedFields: []string{"host", "x-amz-content-sha256", "x-amz-date"}, SignedFields: []string{"host", "x-amz-content-sha256", "x-amz-date"},
Date: "20210809", Date: "20210809",
Preamble: signaturePreambleSigV4, Preamble: SignaturePreambleSigV4,
}, },
}, },
{ {
@ -298,6 +302,7 @@ func TestAuthenticate(t *testing.T) {
for _, tc := range []struct { for _, tc := range []struct {
name string name string
region string
prefixes []string prefixes []string
request *http.Request request *http.Request
err bool err bool
@ -308,10 +313,23 @@ func TestAuthenticate(t *testing.T) {
prefixes: []string{addr.Container().String()}, prefixes: []string{addr.Container().String()},
request: func() *http.Request { request: func() *http.Request {
r := httptest.NewRequest(http.MethodPost, "/", nil) r := httptest.NewRequest(http.MethodPost, "/", nil)
err = defaultSigner.SignHTTP(ctx, awsCreds, r, UnsignedPayload, service, region, time.Now())
require.NoError(t, err)
return r
}(),
region: region,
},
{
name: "valid sign with hash",
prefixes: []string{addr.Container().String()},
request: func() *http.Request {
r := httptest.NewRequest(http.MethodPost, "/", nil)
r.Header.Set(AmzContentSHA256, "")
err = defaultSigner.SignHTTP(ctx, awsCreds, r, "", service, region, time.Now()) err = defaultSigner.SignHTTP(ctx, awsCreds, r, "", service, region, time.Now())
require.NoError(t, err) require.NoError(t, err)
return r return r
}(), }(),
region: region,
}, },
{ {
name: "no authorization header", name: "no authorization header",
@ -418,12 +436,27 @@ func TestAuthenticate(t *testing.T) {
request: func() *http.Request { request: func() *http.Request {
r := httptest.NewRequest(http.MethodPost, "/", nil) r := httptest.NewRequest(http.MethodPost, "/", nil)
r.Header.Set(AmzExpires, "60") r.Header.Set(AmzExpires, "60")
signedURI, _, err := defaultSigner.PresignHTTP(ctx, awsCreds, r, UnsignedPayload, service, region, time.Now())
require.NoError(t, err)
r.URL, err = url.ParseRequestURI(signedURI)
require.NoError(t, err)
return r
}(),
region: region,
},
{
name: "valid presign with hash",
request: func() *http.Request {
r := httptest.NewRequest(http.MethodPost, "/", nil)
r.Header.Set(AmzExpires, "60")
r.Header.Set(AmzContentSHA256, "")
signedURI, _, err := defaultSigner.PresignHTTP(ctx, awsCreds, r, "", service, region, time.Now()) signedURI, _, err := defaultSigner.PresignHTTP(ctx, awsCreds, r, "", service, region, time.Now())
require.NoError(t, err) require.NoError(t, err)
r.URL, err = url.ParseRequestURI(signedURI) r.URL, err = url.ParseRequestURI(signedURI)
require.NoError(t, err) require.NoError(t, err)
return r return r
}(), }(),
region: region,
}, },
{ {
name: "presign, bad X-Amz-Credential", name: "presign, bad X-Amz-Credential",
@ -480,6 +513,56 @@ func TestAuthenticate(t *testing.T) {
err: true, err: true,
errCode: errors.ErrBadRequest, errCode: errors.ErrBadRequest,
}, },
{
name: "presign using original aws sdk",
request: func() *http.Request {
cli := s3.NewPresignClient(s3.New(s3.Options{
Credentials: credentials.NewStaticCredentialsProvider(awsCreds.AccessKeyID, awsCreds.SecretAccessKey, ""),
UsePathStyle: true,
BaseEndpoint: aws.String("http://localhost"),
Region: region,
Logger: logging.NewStandardLogger(os.Stdout),
ClientLogMode: aws.LogSigning,
}))
res, err := cli.PresignGetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("object"),
})
require.NoError(t, err)
r := httptest.NewRequest(http.MethodGet, "http://localhost", nil)
r.URL, err = url.ParseRequestURI(res.URL)
require.NoError(t, err)
return r
}(),
region: region,
},
{
name: "presign sigv4a using original aws sdk",
request: func() *http.Request {
cli := s3.NewPresignClient(s3.New(s3.Options{
Credentials: credentials.NewStaticCredentialsProvider(awsCreds.AccessKeyID, awsCreds.SecretAccessKey, ""),
UsePathStyle: true,
BaseEndpoint: aws.String("http://localhost"),
Region: region,
Logger: logging.NewStandardLogger(os.Stdout),
ClientLogMode: aws.LogSigning,
AuthSchemeResolver: resolver{},
}))
res, err := cli.PresignGetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("object"),
})
require.NoError(t, err)
r := httptest.NewRequest(http.MethodGet, "http://localhost", nil)
r.URL, err = url.ParseRequestURI(res.URL)
require.NoError(t, err)
return r
}(),
},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
creds := tokens.New(bigConfig) creds := tokens.New(bigConfig)
@ -495,13 +578,19 @@ func TestAuthenticate(t *testing.T) {
} else { } else {
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, accessKeyID, box.AuthHeaders.AccessKeyID) require.Equal(t, accessKeyID, box.AuthHeaders.AccessKeyID)
require.Equal(t, region, box.AuthHeaders.Region) require.Equal(t, tc.region, box.AuthHeaders.Region)
require.Equal(t, secret.SecretKey, box.AccessBox.Gate.SecretKey) require.Equal(t, secret.SecretKey, box.AccessBox.Gate.SecretKey)
} }
}) })
} }
} }
type resolver struct{}
func (r resolver) ResolveAuthSchemes(context.Context, *s3.AuthResolverParameters) ([]*smithyauth.Option, error) {
return []*smithyauth.Option{{SchemeID: smithyauth.SchemeIDSigV4A}}, nil
}
func TestHTTPPostAuthenticate(t *testing.T) { func TestHTTPPostAuthenticate(t *testing.T) {
const ( const (
policyBase64 = "eyJleHBpcmF0aW9uIjogIjIwMjUtMTItMDFUMTI6MDA6MDAuMDAwWiIsImNvbmRpdGlvbnMiOiBbCiBbInN0YXJ0cy13aXRoIiwgIiR4LWFtei1jcmVkZW50aWFsIiwgIiJdLAogWyJzdGFydHMtd2l0aCIsICIkeC1hbXotZGF0ZSIsICIiXQpdfQ==" policyBase64 = "eyJleHBpcmF0aW9uIjogIjIwMjUtMTItMDFUMTI6MDA6MDAuMDAwWiIsImNvbmRpdGlvbnMiOiBbCiBbInN0YXJ0cy13aXRoIiwgIiR4LWFtei1jcmVkZW50aWFsIiwgIiJdLAogWyJzdGFydHMtd2l0aCIsICIkeC1hbXotZGF0ZSIsICIiXQpdfQ=="

View file

@ -52,7 +52,12 @@ func PresignRequest(ctx context.Context, creds aws.Credentials, reqData RequestD
options.Logger = log options.Logger = log
}) })
signedURI, _, err := signer.PresignHTTP(ctx, creds, req, presignData.Headers[AmzContentSHA256], presignData.Service, presignData.Region, presignData.SignTime) payloadHash := presignData.Headers[AmzContentSHA256]
if payloadHash == "" {
payloadHash = UnsignedPayload
}
signedURI, _, err := signer.PresignHTTP(ctx, creds, req, payloadHash, presignData.Service, presignData.Region, presignData.SignTime)
if err != nil { if err != nil {
return nil, fmt.Errorf("presign: %w", err) return nil, fmt.Errorf("presign: %w", err)
} }
@ -93,7 +98,13 @@ func PresignRequestV4a(cred aws.Credentials, reqData RequestData, presignData Pr
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to derive assymetric key from credentials: %w", err) return nil, fmt.Errorf("failed to derive assymetric key from credentials: %w", err)
} }
presignedURL, _, err := signer.PresignHTTP(req.Context(), creds, req, presignData.Headers[AmzContentSHA256], presignData.Service, []string{presignData.Region}, presignData.SignTime)
payloadHash := presignData.Headers[AmzContentSHA256]
if payloadHash == "" {
payloadHash = UnsignedPayload
}
presignedURL, _, err := signer.PresignHTTP(req.Context(), creds, req, payloadHash, presignData.Service, []string{presignData.Region}, presignData.SignTime)
if err != nil { if err != nil {
return nil, fmt.Errorf("presign: %w", err) return nil, fmt.Errorf("presign: %w", err)
} }

View file

@ -78,7 +78,6 @@ func TestCheckSign(t *testing.T) {
SignTime: time.Now().UTC(), SignTime: time.Now().UTC(),
Headers: map[string]string{ Headers: map[string]string{
ContentTypeHdr: "text/plain", ContentTypeHdr: "text/plain",
AmzContentSHA256: UnsignedPayload,
}, },
} }
@ -146,7 +145,7 @@ func TestCheckSignV4a(t *testing.T) {
c := &Center{ c := &Center{
cli: mock, cli: mock,
regV4a: NewRegexpMatcher(authorizationFieldV4aRegexp), regV4a: NewRegexpMatcher(AuthorizationFieldV4aRegexp),
postReg: NewRegexpMatcher(postPolicyCredentialRegexp), postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
} }
box, err := c.Authenticate(req) box, err := c.Authenticate(req)

View file

@ -1,4 +1,6 @@
// This file is adopting https://github.com/aws/aws-sdk-go-v2/blob/a2b751d1ba71f59175a41f9cae5f159f1044360f/aws/signer/v4/stream.go for sigv4a. // This file is adopting https://github.com/aws/aws-sdk-go-v2/blob/a2b751d1ba71f59175a41f9cae5f159f1044360f/aws/signer/v4/stream.go for sigv4a.
// with changes
// * add VerifyTrailerSignature
package v4a package v4a
@ -88,6 +90,39 @@ func (s *StreamSigner) buildEventStreamStringToSign(headers, payload, previousSi
}, "\n") }, "\n")
} }
func (s *StreamSigner) VerifyTrailerSignature(payload []byte, signingTime time.Time, signature []byte) error {
prevSignature := s.prevSignature
st := v4Internal.NewSigningTime(signingTime)
scope := buildCredentialScope(st, s.service)
stringToSign := s.buildEventStreamStringToSignTrailer(payload, prevSignature, scope, &st)
ok, err := signerCrypto.VerifySignature(&s.credentials.PrivateKey.PublicKey, makeHash(sha256.New(), []byte(stringToSign)), signature)
if err != nil {
return err
}
if !ok {
return fmt.Errorf("v4a: invalid signature")
}
s.prevSignature = signature
return nil
}
func (s *StreamSigner) buildEventStreamStringToSignTrailer(payload, previousSignature []byte, credentialScope string, signingTime *v4Internal.SigningTime) string {
hash := sha256.New()
return strings.Join([]string{
"AWS4-ECDSA-P256-SHA256-TRAILER",
signingTime.TimeFormat(),
credentialScope,
hex.EncodeToString(previousSignature),
hex.EncodeToString(makeHash(hash, payload)),
}, "\n")
}
func buildCredentialScope(st v4Internal.SigningTime, service string) string { func buildCredentialScope(st v4Internal.SigningTime, service string) string {
return strings.Join([]string{ return strings.Join([]string{
st.Format(shortTimeFormat), st.Format(shortTimeFormat),

View file

@ -1,4 +1,6 @@
// This is https://github.com/aws/aws-sdk-go-v2/blob/a2b751d1ba71f59175a41f9cae5f159f1044360f/aws/signer/v4/stream.go // This is https://github.com/aws/aws-sdk-go-v2/blob/a2b751d1ba71f59175a41f9cae5f159f1044360f/aws/signer/v4/stream.go
// with changes
// * add GetTrailingSignature
package v4 package v4
@ -87,3 +89,32 @@ func (s *StreamSigner) buildEventStreamStringToSign(headers, payload, previousSi
hex.EncodeToString(makeHash(hash, payload)), hex.EncodeToString(makeHash(hash, payload)),
}, "\n") }, "\n")
} }
// GetTrailerSignature signs the provided header and payload bytes.
func (s *StreamSigner) GetTrailerSignature(payload []byte, signingTime time.Time) ([]byte, error) {
prevSignature := s.prevSignature
st := v4Internal.NewSigningTime(signingTime)
sigKey := s.signingKeyDeriver.DeriveKey(s.credentials, s.service, s.region, st)
scope := v4Internal.BuildCredentialScope(st, s.region, s.service)
stringToSign := s.buildEventStreamStringToSignTrailer(payload, prevSignature, scope, &st)
signature := v4Internal.HMACSHA256(sigKey, []byte(stringToSign))
s.prevSignature = signature
return signature, nil
}
func (s *StreamSigner) buildEventStreamStringToSignTrailer(payload, previousSignature []byte, credentialScope string, signingTime *v4Internal.SigningTime) string {
hash := sha256.New()
return strings.Join([]string{
"AWS4-HMAC-SHA256-TRAILER",
signingTime.TimeFormat(),
credentialScope,
hex.EncodeToString(previousSignature),
hex.EncodeToString(makeHash(hash, payload)),
}, "\n")
}

View file

@ -48,7 +48,7 @@ func (o *AccessControlCache) Get(owner user.ID, key string) bool {
result, ok := entry.(bool) result, ok := entry.(bool)
if !ok { if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return false return false
} }

View file

@ -67,7 +67,7 @@ func (o *AccessBoxCache) Get(accessKeyID string) *AccessBoxCacheValue {
result, ok := entry.(*AccessBoxCacheValue) result, ok := entry.(*AccessBoxCacheValue)
if !ok { if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }

View file

@ -65,7 +65,7 @@ func (o *BucketCache) GetByCID(cnrID cid.ID) *data.BucketInfo {
key, ok := entry.(string) key, ok := entry.(string)
if !ok { if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", key))) zap.String("expected", fmt.Sprintf("%T", key)), logs.TagField(logs.TagDatapath))
return nil return nil
} }
@ -81,7 +81,7 @@ func (o *BucketCache) get(key string) *data.BucketInfo {
result, ok := entry.(*data.BucketInfo) result, ok := entry.(*data.BucketInfo)
if !ok { if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }

View file

@ -69,7 +69,7 @@ func get[T any](c *FrostfsIDCache, key any) *T {
result, ok := entry.(*T) result, ok := entry.(*T)
if !ok { if !ok {
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }

View file

@ -52,7 +52,7 @@ func NewListSessionCache(config *Config) *ListSessionCache {
session, ok := val.(*data.ListSession) session, ok := val.(*data.ListSession)
if !ok { if !ok {
config.Logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", val)), config.Logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", val)),
zap.String("expected", fmt.Sprintf("%T", session))) zap.String("expected", fmt.Sprintf("%T", session)), logs.TagField(logs.TagDatapath))
} }
if !session.Acquired.Load() { if !session.Acquired.Load() {
@ -72,7 +72,7 @@ func (l *ListSessionCache) GetListSession(key ListSessionKey) *data.ListSession
result, ok := entry.(*data.ListSession) result, ok := entry.(*data.ListSession)
if !ok { if !ok {
l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }

2
api/cache/names.go vendored
View file

@ -50,7 +50,7 @@ func (o *ObjectsNameCache) Get(key string) *oid.Address {
result, ok := entry.(oid.Address) result, ok := entry.(oid.Address)
if !ok { if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }

View file

@ -54,7 +54,7 @@ func (c *NetworkCache) GetNetworkInfo() *netmap.NetworkInfo {
result, ok := entry.(netmap.NetworkInfo) result, ok := entry.(netmap.NetworkInfo)
if !ok { if !ok {
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }
@ -74,7 +74,7 @@ func (c *NetworkCache) GetNetmap() *netmap.NetMap {
result, ok := entry.(netmap.NetMap) result, ok := entry.(netmap.NetMap)
if !ok { if !ok {
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }

View file

@ -49,7 +49,7 @@ func (o *ObjectsCache) GetObject(address oid.Address) *data.ExtendedObjectInfo {
result, ok := entry.(*data.ExtendedObjectInfo) result, ok := entry.(*data.ExtendedObjectInfo)
if !ok { if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }

View file

@ -77,7 +77,7 @@ func (l *ObjectsListCache) GetVersions(key ObjectsListKey) []*data.NodeVersion {
result, ok := entry.([]*data.NodeVersion) result, ok := entry.([]*data.NodeVersion)
if !ok { if !ok {
l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }
@ -96,7 +96,7 @@ func (l *ObjectsListCache) CleanCacheEntriesContainingObject(objectName string,
k, ok := key.(ObjectsListKey) k, ok := key.(ObjectsListKey)
if !ok { if !ok {
l.logger.Warn(logs.InvalidCacheKeyType, zap.String("actual", fmt.Sprintf("%T", key)), l.logger.Warn(logs.InvalidCacheKeyType, zap.String("actual", fmt.Sprintf("%T", key)),
zap.String("expected", fmt.Sprintf("%T", k))) zap.String("expected", fmt.Sprintf("%T", k)), logs.TagField(logs.TagDatapath))
continue continue
} }
if cnr.Equals(k.cid) && strings.HasPrefix(objectName, k.prefix) { if cnr.Equals(k.cid) && strings.HasPrefix(objectName, k.prefix) {

2
api/cache/policy.go vendored
View file

@ -54,7 +54,7 @@ func (o *MorphPolicyCache) Get(key MorphPolicyCacheKey) []*chain.Chain {
result, ok := entry.([]*chain.Chain) result, ok := entry.([]*chain.Chain)
if !ok { if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }

8
api/cache/system.go vendored
View file

@ -50,7 +50,7 @@ func (o *SystemCache) GetObject(key string) *data.ObjectInfo {
result, ok := entry.(*data.ObjectInfo) result, ok := entry.(*data.ObjectInfo)
if !ok { if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }
@ -81,7 +81,7 @@ func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration {
result, ok := entry.(*data.CORSConfiguration) result, ok := entry.(*data.CORSConfiguration)
if !ok { if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }
@ -97,7 +97,7 @@ func (o *SystemCache) GetLifecycleConfiguration(key string) *data.LifecycleConfi
result, ok := entry.(*data.LifecycleConfiguration) result, ok := entry.(*data.LifecycleConfiguration)
if !ok { if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }
@ -113,7 +113,7 @@ func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
result, ok := entry.(*data.BucketSettings) result, ok := entry.(*data.BucketSettings)
if !ok { if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result))) zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil return nil
} }

View file

@ -109,7 +109,6 @@ type MultipartInfo struct {
Owner user.ID Owner user.ID
Created time.Time Created time.Time
Meta map[string]string Meta map[string]string
CopiesNumbers []uint32
Finished bool Finished bool
CreationEpoch uint64 CreationEpoch uint64
} }

View file

@ -60,7 +60,6 @@ const (
ErrMalformedACL ErrMalformedACL
ErrMalformedXML ErrMalformedXML
ErrMissingContentLength ErrMissingContentLength
ErrMissingContentMD5
ErrMissingRequestBodyError ErrMissingRequestBodyError
ErrMissingSecurityHeader ErrMissingSecurityHeader
ErrNoSuchBucket ErrNoSuchBucket
@ -478,12 +477,6 @@ var errorCodes = errorCodeMap{
Description: "You must provide the Content-Length HTTP header.", Description: "You must provide the Content-Length HTTP header.",
HTTPStatusCode: http.StatusLengthRequired, HTTPStatusCode: http.StatusLengthRequired,
}, },
ErrMissingContentMD5: {
ErrCode: ErrMissingContentMD5,
Code: "MissingContentMD5",
Description: "Missing required header for this request: Content-Md5.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMissingSecurityHeader: { ErrMissingSecurityHeader: {
ErrCode: ErrMissingSecurityHeader, ErrCode: ErrMissingSecurityHeader,
Code: "MissingSecurityHeader", Code: "MissingSecurityHeader",

View file

@ -99,7 +99,7 @@ func (h *handler) encodePrivateCannedACL(ctx context.Context, bktInfo *data.Buck
ownerEncodedID := ownerDisplayName ownerEncodedID := ownerDisplayName
if settings.OwnerKey == nil { if settings.OwnerKey == nil {
h.reqLogger(ctx).Warn(logs.BucketOwnerKeyIsMissing, zap.String("owner", bktInfo.Owner.String())) h.reqLogger(ctx).Warn(logs.BucketOwnerKeyIsMissing, zap.String("owner", bktInfo.Owner.String()), logs.TagField(logs.TagDatapath))
} else { } else {
ownerDisplayName = settings.OwnerKey.Address() ownerDisplayName = settings.OwnerKey.Address()
ownerEncodedID = hex.EncodeToString(settings.OwnerKey.Bytes()) ownerEncodedID = hex.EncodeToString(settings.OwnerKey.Bytes())
@ -150,7 +150,7 @@ func (h *handler) putBucketACLAPEHandler(w http.ResponseWriter, r *http.Request,
defer func() { defer func() {
if errBody := r.Body.Close(); errBody != nil { if errBody := r.Body.Close(); errBody != nil {
h.reqLogger(ctx).Warn(logs.CouldNotCloseRequestBody, zap.Error(errBody)) h.reqLogger(ctx).Warn(logs.CouldNotCloseRequestBody, zap.Error(errBody), logs.TagField(logs.TagDatapath))
} }
}() }()
@ -382,7 +382,7 @@ func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
h.logAndSendError(ctx, w, "could not convert s3 policy to native chain policy", reqInfo, err) h.logAndSendError(ctx, w, "could not convert s3 policy to native chain policy", reqInfo, err)
return return
} else { } else {
h.reqLogger(ctx).Warn(logs.PolicyCouldntBeConvertedToNativeRules) h.reqLogger(ctx).Warn(logs.PolicyCouldntBeConvertedToNativeRules, logs.TagField(logs.TagDatapath))
} }
chainsToSave := []*chain.Chain{s3Chain} chainsToSave := []*chain.Chain{s3Chain}

View file

@ -7,6 +7,7 @@ import (
"encoding/xml" "encoding/xml"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/url"
"testing" "testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
@ -297,10 +298,17 @@ type createBucketInfo struct {
Key *keys.PrivateKey Key *keys.PrivateKey
} }
type bucketPrm struct {
bktName string
query url.Values
box *accessbox.Box
createParams createBucketParams
}
func createBucket(hc *handlerContext, bktName string) *createBucketInfo { func createBucket(hc *handlerContext, bktName string) *createBucketInfo {
box, key := createAccessBox(hc.t) box, key := createAccessBox(hc.t)
w := createBucketBase(hc, bktName, box) w := createBucketBase(hc, bucketPrm{bktName: bktName, box: box})
assertStatus(hc.t, w, http.StatusOK) assertStatus(hc.t, w, http.StatusOK)
bktInfo, err := hc.Layer().GetBucketInfo(hc.Context(), bktName) bktInfo, err := hc.Layer().GetBucketInfo(hc.Context(), bktName)
@ -314,13 +322,32 @@ func createBucket(hc *handlerContext, bktName string) *createBucketInfo {
} }
func createBucketAssertS3Error(hc *handlerContext, bktName string, box *accessbox.Box, code apierr.ErrorCode) { func createBucketAssertS3Error(hc *handlerContext, bktName string, box *accessbox.Box, code apierr.ErrorCode) {
w := createBucketBase(hc, bktName, box) w := createBucketBase(hc, bucketPrm{bktName: bktName, box: box})
assertS3Error(hc.t, w, apierr.GetAPIError(code)) assertS3Error(hc.t, w, apierr.GetAPIError(code))
} }
func createBucketBase(hc *handlerContext, bktName string, box *accessbox.Box) *httptest.ResponseRecorder { func createBucketWithConstraint(hc *handlerContext, bktName, constraint string) *createBucketInfo {
w, r := prepareTestRequest(hc, bktName, "", nil) box, key := createAccessBox(hc.t)
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box}) var prm createBucketParams
if constraint != "" {
prm.LocationConstraint = constraint
}
w := createBucketBase(hc, bucketPrm{bktName: bktName, box: box, createParams: prm})
assertStatus(hc.t, w, http.StatusOK)
bktInfo, err := hc.Layer().GetBucketInfo(hc.Context(), bktName)
require.NoError(hc.t, err)
return &createBucketInfo{
BktInfo: bktInfo,
Box: box,
Key: key,
}
}
func createBucketBase(hc *handlerContext, prm bucketPrm) *httptest.ResponseRecorder {
w, r := prepareTestFullRequest(hc, prm.bktName, "", nil, prm.createParams)
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: prm.box})
r = r.WithContext(ctx) r = r.WithContext(ctx)
hc.Handler().CreateBucketHandler(w, r) hc.Handler().CreateBucketHandler(w, r)
return w return w

View file

@ -0,0 +1,70 @@
package handler
import (
"net/http"
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
)
const maxBucketList = 10000
// ListBucketsHandler handles bucket listing requests.
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
params, err := parseListBucketParams(r)
if err != nil {
h.logAndSendError(ctx, w, "failed to parse params", reqInfo, err)
return
}
resp, err := h.obj.ListBuckets(ctx, params)
if err != nil {
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
return
}
if err = middleware.EncodeToResponse(w, encodeListBuckets(reqInfo.User, resp, params)); err != nil {
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}
func encodeListBuckets(owner string, resp layer.ListBucketsResult, params layer.ListBucketsParams) *ListBucketsResponse {
res := &ListBucketsResponse{
Owner: Owner{
ID: owner,
DisplayName: owner,
},
ContinuationToken: resp.ContinuationToken,
Prefix: params.Prefix,
}
for _, item := range resp.Containers {
res.Buckets.Buckets = append(res.Buckets.Buckets, Bucket{
Name: item.Name,
CreationDate: item.Created.UTC().Format(time.RFC3339),
BucketRegion: item.LocationConstraint,
})
}
return res
}
func parseListBucketParams(r *http.Request) (prm layer.ListBucketsParams, err error) {
prm.MaxBuckets = maxBucketList
strMaxBuckets := r.URL.Query().Get(middleware.QueryMaxBuckets)
if strMaxBuckets != "" {
if prm.MaxBuckets, err = strconv.Atoi(strMaxBuckets); err != nil || prm.MaxBuckets < 0 {
return layer.ListBucketsParams{}, errors.GetAPIError(errors.ErrInvalidMaxKeys)
}
}
prm.Prefix = r.URL.Query().Get(middleware.QueryPrefix)
prm.BucketRegion = r.URL.Query().Get(middleware.QueryBucketRegion)
prm.ContinuationToken = r.URL.Query().Get(middleware.QueryContinuationToken)
return
}

View file

@ -0,0 +1,174 @@
package handler
import (
"encoding/xml"
"net/http"
"net/http/httptest"
"net/url"
"sort"
"testing"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"github.com/stretchr/testify/require"
)
func TestHandler_ListBucketsHandler(t *testing.T) {
const defaultConstraint = "default"
region := "us-west-1"
hc := prepareHandlerContext(t)
hc.config.putLocationConstraint(region)
props := []Bucket{
{Name: "first"},
{Name: "regional", BucketRegion: "us-west-1"},
{Name: "third"},
}
sort.Slice(props, func(i, j int) bool {
return props[i].Name < props[j].Name
})
for _, bkt := range props {
createBucketWithConstraint(hc, bkt.Name, bkt.BucketRegion)
}
for _, tt := range []struct {
title string
token string
prefix string
bucketRegion string
maxBuckets string
expectErr bool
expected []Bucket
expectedToken string
}{
{
title: "no params",
expected: []Bucket{
{Name: "first", BucketRegion: defaultConstraint},
{Name: "regional", BucketRegion: "us-west-1"},
{Name: "third", BucketRegion: defaultConstraint},
},
},
{
title: "negative max-buckets",
maxBuckets: "-1",
expected: []Bucket{},
expectErr: true,
},
{
title: "zero max-buckets",
maxBuckets: "0",
expected: []Bucket{},
},
{
title: "prefix",
prefix: "thi",
expected: []Bucket{{Name: "third", BucketRegion: defaultConstraint}},
},
{
title: "wrong prefix",
prefix: "sdh",
expected: []Bucket{},
},
{
title: "bucket region",
bucketRegion: region,
expected: []Bucket{{Name: "regional", BucketRegion: "us-west-1"}},
},
{
title: "default bucket region",
bucketRegion: defaultConstraint,
expected: []Bucket{
{Name: "first", BucketRegion: defaultConstraint},
{Name: "third", BucketRegion: defaultConstraint},
},
},
{
title: "wrong bucket region",
bucketRegion: "sj dfdlsj",
expected: []Bucket{},
},
} {
t.Run(tt.title, func(t *testing.T) {
if tt.expectErr {
listBucketsErr(hc, tt.prefix, tt.token, tt.bucketRegion, tt.maxBuckets, apierr.GetAPIError(apierr.ErrInvalidMaxKeys))
return
}
resp := listBuckets(hc, tt.prefix, tt.token, tt.bucketRegion, tt.maxBuckets)
require.Len(t, resp.Buckets.Buckets, len(tt.expected))
require.Equal(t, tt.prefix, resp.Prefix)
require.Equal(t, hc.owner.String(), resp.Owner.ID)
if len(resp.Buckets.Buckets) > 0 {
t.Log(resp.Buckets.Buckets[0].Name)
}
for i, bkt := range resp.Buckets.Buckets {
require.Equal(t, tt.expected[i].Name, bkt.Name)
require.Equal(t, tt.expected[i].BucketRegion, bkt.BucketRegion)
}
})
}
t.Run("pagination", func(t *testing.T) {
t.Run("happy path", func(t *testing.T) {
resp := listBuckets(hc, "", "", "", "1")
require.Len(t, resp.Buckets.Buckets, 1)
require.Equal(t, props[0].Name, resp.Buckets.Buckets[0].Name)
require.NotEmpty(t, resp.ContinuationToken)
resp = listBuckets(hc, "", resp.ContinuationToken, "", "1")
require.Len(t, resp.Buckets.Buckets, 1)
require.Equal(t, props[1].Name, resp.Buckets.Buckets[0].Name)
require.NotEmpty(t, resp.ContinuationToken)
resp = listBuckets(hc, "", resp.ContinuationToken, "", "1")
require.Len(t, resp.Buckets.Buckets, 1)
require.Equal(t, props[2].Name, resp.Buckets.Buckets[0].Name)
require.Empty(t, resp.ContinuationToken)
})
t.Run("wrong continuation-token", func(t *testing.T) {
resp := listBuckets(hc, "", "CebuVwfRpdMqi9dvgV2SUNbrkfteGtudchKKhNabXUu9", "", "1")
require.Len(t, resp.Buckets.Buckets, 0)
require.Empty(t, resp.ContinuationToken)
})
})
}
func listBuckets(hc *handlerContext, prefix, token, bucketRegion, maxBuckets string) ListBucketsResponse {
query := url.Values{
middleware.QueryPrefix: []string{prefix},
middleware.QueryContinuationToken: []string{token},
middleware.QueryBucketRegion: []string{bucketRegion},
middleware.QueryMaxBuckets: []string{maxBuckets},
}
w := listBucketsBase(hc, bucketPrm{query: query})
assertStatus(hc.t, w, http.StatusOK)
var resp ListBucketsResponse
err := xml.NewDecoder(w.Body).Decode(&resp)
require.NoError(hc.t, err)
return resp
}
func listBucketsErr(hc *handlerContext, prefix, token, bucketRegion, maxBuckets string, err apierr.Error) {
query := url.Values{
middleware.QueryPrefix: []string{prefix},
middleware.QueryContinuationToken: []string{token},
middleware.QueryBucketRegion: []string{bucketRegion},
middleware.QueryMaxBuckets: []string{maxBuckets},
}
w := listBucketsBase(hc, bucketPrm{query: query})
assertS3Error(hc.t, w, err)
}
func listBucketsBase(hc *handlerContext, prm bucketPrm) *httptest.ResponseRecorder {
box, _ := createAccessBox(hc.t)
w, r := prepareTestFullRequest(hc, "", "", prm.query, nil)
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
r = r.WithContext(ctx)
hc.Handler().ListBucketsHandler(w, r)
return w
}

View file

@ -244,7 +244,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
} }
} }
h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID)) h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID), logs.TagField(logs.TagExternalStorage))
if dstEncryptionParams.Enabled() { if dstEncryptionParams.Enabled() {
addSSECHeaders(w.Header(), r.Header) addSSECHeaders(w.Header(), r.Header)

View file

@ -108,13 +108,13 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
} }
bktInfo, err := h.getBucketInfo(ctx, reqInfo.BucketName) bktInfo, err := h.getBucketInfo(ctx, reqInfo.BucketName)
if err != nil { if err != nil {
h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err)) h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
return return
} }
cors, err := h.obj.GetBucketCORS(ctx, bktInfo, h.cfg.NewXMLDecoder) cors, err := h.obj.GetBucketCORS(ctx, bktInfo, h.cfg.NewXMLDecoder)
if err != nil { if err != nil {
h.reqLogger(ctx).Warn(logs.GetBucketCors, zap.Error(err)) h.reqLogger(ctx).Warn(logs.GetBucketCors, zap.Error(err), logs.TagField(logs.TagDatapath))
return return
} }

View file

@ -131,13 +131,6 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
ctx := r.Context() ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx) reqInfo := middleware.GetReqInfo(ctx)
// Content-Md5 is required and should be set
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
if _, ok := r.Header[api.ContentMD5]; !ok {
h.logAndSendError(ctx, w, "missing Content-MD5", reqInfo, errors.GetAPIError(errors.ErrMissingContentMD5))
return
}
// Content-Length is required and should be non-zero // Content-Length is required and should be non-zero
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html // http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
if r.ContentLength <= 0 { if r.ContentLength <= 0 {

View file

@ -542,7 +542,6 @@ func deleteObjectsBase(hc *handlerContext, bktName string, objVersions [][2]stri
} }
w, r := prepareTestRequest(hc, bktName, "", req) w, r := prepareTestRequest(hc, bktName, "", req)
r.Header.Set(api.ContentMD5, "")
hc.Handler().DeleteMultipleObjectsHandler(w, r) hc.Handler().DeleteMultipleObjectsHandler(w, r)
assertStatus(hc.t, w, http.StatusOK) assertStatus(hc.t, w, http.StatusOK)

View file

@ -296,12 +296,12 @@ func parseConditionalHeaders(headers http.Header, log *zap.Logger) *conditionalA
if httpTime, err := parseHTTPTime(headers.Get(api.IfModifiedSince)); err == nil { if httpTime, err := parseHTTPTime(headers.Get(api.IfModifiedSince)); err == nil {
args.IfModifiedSince = httpTime args.IfModifiedSince = httpTime
} else { } else {
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfModifiedSince, headers.Get(api.IfModifiedSince)), zap.Error(err)) log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfModifiedSince, headers.Get(api.IfModifiedSince)), zap.Error(err), logs.TagField(logs.TagDatapath))
} }
if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil { if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil {
args.IfUnmodifiedSince = httpTime args.IfUnmodifiedSince = httpTime
} else { } else {
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err)) log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err), logs.TagField(logs.TagDatapath))
} }
return args return args

View file

@ -197,6 +197,33 @@ func TestGetObject(t *testing.T) {
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, apierr.ErrNoSuchKey) getObjectAssertS3Error(hc, bktName, objName, emptyVersion, apierr.ErrNoSuchKey)
} }
func TestGetDeletedObject(t *testing.T) {
hc := prepareHandlerContextWithMinCache(t)
bktName, objName := "bucket", "obj"
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
putObject(hc, bktName, objName)
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
checkFound(hc.t, hc, bktName, objName, emptyVersion)
addr := getAddressOfLastVersion(hc, bktInfo, objName)
t.Run("not found error", func(_ *testing.T) {
hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectNotFound{})
getObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), apierr.ErrNoSuchVersion)
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, apierr.ErrNoSuchKey)
})
t.Run("already removed error", func(_ *testing.T) {
hc.tp.SetObjectError(addr, &apistatus.ObjectAlreadyRemoved{})
hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectAlreadyRemoved{})
getObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), apierr.ErrNoSuchVersion)
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, apierr.ErrNoSuchKey)
})
}
func TestGetObjectEnabledMD5(t *testing.T) { func TestGetObjectEnabledMD5(t *testing.T) {
hc := prepareHandlerContext(t) hc := prepareHandlerContext(t)
bktName, objName := "bucket", "obj" bktName, objName := "bucket", "obj"

View file

@ -74,6 +74,7 @@ func (hc *handlerContextBase) Context() context.Context {
type configMock struct { type configMock struct {
defaultPolicy netmap.PlacementPolicy defaultPolicy netmap.PlacementPolicy
placementPolicies map[string]netmap.PlacementPolicy
copiesNumbers map[string][]uint32 copiesNumbers map[string][]uint32
defaultCopiesNumbers []uint32 defaultCopiesNumbers []uint32
bypassContentEncodingInChunks bool bypassContentEncodingInChunks bool
@ -86,8 +87,9 @@ func (c *configMock) DefaultPlacementPolicy(_ string) netmap.PlacementPolicy {
return c.defaultPolicy return c.defaultPolicy
} }
func (c *configMock) PlacementPolicy(_, _ string) (netmap.PlacementPolicy, bool) { func (c *configMock) PlacementPolicy(_, constraint string) (netmap.PlacementPolicy, bool) {
return netmap.PlacementPolicy{}, false policy, ok := c.placementPolicies[constraint]
return policy, ok
} }
func (c *configMock) CopiesNumbers(_, locationConstraint string) ([]uint32, bool) { func (c *configMock) CopiesNumbers(_, locationConstraint string) ([]uint32, bool) {
@ -151,6 +153,10 @@ func (c *configMock) TLSTerminationHeader() string {
return c.tlsTerminationHeader return c.tlsTerminationHeader
} }
func (c *configMock) putLocationConstraint(constraint string) {
c.placementPolicies[constraint] = c.defaultPolicy
}
func prepareHandlerContext(t *testing.T) *handlerContext { func prepareHandlerContext(t *testing.T) *handlerContext {
hc, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(zap.NewExample())) hc, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(zap.NewExample()))
require.NoError(t, err) require.NoError(t, err)
@ -218,6 +224,7 @@ func prepareHandlerContextBase(cacheCfg *layer.CachesConfig) (*handlerContextBas
cfg := &configMock{ cfg := &configMock{
defaultPolicy: pp, defaultPolicy: pp,
placementPolicies: make(map[string]netmap.PlacementPolicy),
} }
h := &handler{ h := &handler{
log: log, log: log,

View file

@ -55,25 +55,19 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
ctx := r.Context() ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx) reqInfo := middleware.GetReqInfo(ctx)
// Content-Md5 is required and should be set cfg := new(data.LifecycleConfiguration)
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html if err := h.cfg.NewXMLDecoder(tee, r.UserAgent()).Decode(cfg); err != nil {
if _, ok := r.Header[api.ContentMD5]; !ok { h.logAndSendError(ctx, w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
h.logAndSendError(ctx, w, "missing Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrMissingContentMD5))
return return
} }
if _, ok := r.Header[api.ContentMD5]; ok {
headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5)) headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5))
if err != nil { if err != nil {
h.logAndSendError(ctx, w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest)) h.logAndSendError(ctx, w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return return
} }
cfg := new(data.LifecycleConfiguration)
if err = h.cfg.NewXMLDecoder(tee, r.UserAgent()).Decode(cfg); err != nil {
h.logAndSendError(ctx, w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
return
}
bodyMD5, err := getContentMD5(&buf) bodyMD5, err := getContentMD5(&buf)
if err != nil { if err != nil {
h.logAndSendError(ctx, w, "could not get content md5", reqInfo, err) h.logAndSendError(ctx, w, "could not get content md5", reqInfo, err)
@ -84,6 +78,7 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
h.logAndSendError(ctx, w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest)) h.logAndSendError(ctx, w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return return
} }
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName) bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil { if err != nil {

View file

@ -29,6 +29,8 @@ func TestPutBucketLifecycleConfiguration(t *testing.T) {
for _, tc := range []struct { for _, tc := range []struct {
name string name string
body *data.LifecycleConfiguration body *data.LifecycleConfiguration
headers map[string]string
addMD5 bool
errorCode apierr.ErrorCode errorCode apierr.ErrorCode
}{ }{
{ {
@ -70,6 +72,22 @@ func TestPutBucketLifecycleConfiguration(t *testing.T) {
}, },
}, },
}, },
{
name: "correct Content-Md5 header",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
ID: "rule",
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
},
},
},
addMD5: true,
},
{ {
name: "too many rules", name: "too many rules",
body: func() *data.LifecycleConfiguration { body: func() *data.LifecycleConfiguration {
@ -407,14 +425,44 @@ func TestPutBucketLifecycleConfiguration(t *testing.T) {
}, },
errorCode: apierr.ErrInvalidRequest, errorCode: apierr.ErrInvalidRequest,
}, },
{
name: "invalid Content-Md5 header",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
},
},
},
headers: map[string]string{api.ContentMD5: "invalid"},
errorCode: apierr.ErrInvalidDigest,
},
{
name: "Content-Md5 header does not match body md5 hash",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
},
},
},
headers: map[string]string{api.ContentMD5: base64.StdEncoding.EncodeToString([]byte("some-hash"))},
errorCode: apierr.ErrInvalidDigest,
},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
if tc.errorCode > 0 { if tc.errorCode > 0 {
putBucketLifecycleConfigurationErr(hc, bktName, tc.body, apierr.GetAPIError(tc.errorCode)) putBucketLifecycleConfigurationErr(hc, bktName, tc.body, tc.headers, apierr.GetAPIError(tc.errorCode))
return return
} }
putBucketLifecycleConfiguration(hc, bktName, tc.body) putBucketLifecycleConfiguration(hc, bktName, tc.body, tc.headers, tc.addMD5)
cfg := getBucketLifecycleConfiguration(hc, bktName) cfg := getBucketLifecycleConfiguration(hc, bktName)
require.Equal(t, tc.body.Rules, cfg.Rules) require.Equal(t, tc.body.Rules, cfg.Rules)
@ -448,45 +496,13 @@ func TestPutBucketLifecycleIDGeneration(t *testing.T) {
}, },
} }
putBucketLifecycleConfiguration(hc, bktName, lifecycle) putBucketLifecycleConfiguration(hc, bktName, lifecycle, nil, false)
cfg := getBucketLifecycleConfiguration(hc, bktName) cfg := getBucketLifecycleConfiguration(hc, bktName)
require.Len(t, cfg.Rules, 2) require.Len(t, cfg.Rules, 2)
require.NotEmpty(t, cfg.Rules[0].ID) require.NotEmpty(t, cfg.Rules[0].ID)
require.NotEmpty(t, cfg.Rules[1].ID) require.NotEmpty(t, cfg.Rules[1].ID)
} }
func TestPutBucketLifecycleInvalidMD5(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-lifecycle-md5"
createBucket(hc, bktName)
lifecycle := &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
},
},
}
w, r := prepareTestRequest(hc, bktName, "", lifecycle)
hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMissingContentMD5))
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
r.Header.Set(api.ContentMD5, "")
hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
r.Header.Set(api.ContentMD5, "some-hash")
hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
}
func TestPutBucketLifecycleInvalidXML(t *testing.T) { func TestPutBucketLifecycleInvalidXML(t *testing.T) {
hc := prepareHandlerContext(t) hc := prepareHandlerContext(t)
@ -505,25 +521,32 @@ func TestPutBucketLifecycleInvalidXML(t *testing.T) {
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMalformedXML)) assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMalformedXML))
} }
func putBucketLifecycleConfiguration(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) { func putBucketLifecycleConfiguration(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, headers map[string]string, addMD5 bool) {
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg) w := putBucketLifecycleConfigurationBase(hc, bktName, cfg, headers, addMD5)
assertStatus(hc.t, w, http.StatusOK) assertStatus(hc.t, w, http.StatusOK)
} }
func putBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, err apierr.Error) { func putBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, headers map[string]string, err apierr.Error) {
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg) w := putBucketLifecycleConfigurationBase(hc, bktName, cfg, headers, false)
assertS3Error(hc.t, w, err) assertS3Error(hc.t, w, err)
} }
func putBucketLifecycleConfigurationBase(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) *httptest.ResponseRecorder { func putBucketLifecycleConfigurationBase(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, headers map[string]string, addMD5 bool) *httptest.ResponseRecorder {
w, r := prepareTestRequest(hc, bktName, "", cfg) w, r := prepareTestRequest(hc, bktName, "", cfg)
for k, v := range headers {
r.Header.Set(k, v)
}
if addMD5 {
rawBody, err := xml.Marshal(cfg) rawBody, err := xml.Marshal(cfg)
require.NoError(hc.t, err) require.NoError(hc.t, err)
hash := md5.New() hash := md5.New()
hash.Write(rawBody) hash.Write(rawBody)
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(hash.Sum(nil))) r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(hash.Sum(nil)))
}
hc.Handler().PutBucketLifecycleHandler(w, r) hc.Handler().PutBucketLifecycleHandler(w, r)
return w return w
} }

View file

@ -1,49 +0,0 @@
package handler
import (
"net/http"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
)
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
// ListBucketsHandler handles bucket listing requests.
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
var (
own user.ID
res *ListBucketsResponse
ctx = r.Context()
reqInfo = middleware.GetReqInfo(ctx)
)
list, err := h.obj.ListBuckets(ctx)
if err != nil {
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
return
}
if len(list) > 0 {
own = list[0].Owner
}
res = &ListBucketsResponse{
Owner: Owner{
ID: own.String(),
DisplayName: own.String(),
},
}
for _, item := range list {
res.Buckets.Buckets = append(res.Buckets.Buckets, Bucket{
Name: item.Name,
CreationDate: item.Created.UTC().Format(time.RFC3339),
})
}
if err = middleware.EncodeToResponse(w, res); err != nil {
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}

View file

@ -152,12 +152,6 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
p.Header[api.ContentLanguage] = contentLanguage p.Header[api.ContentLanguage] = contentLanguage
} }
p.CopiesNumbers, err = h.pickCopiesNumbers(p.Header, reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err, additional...)
return
}
if err = h.obj.CreateMultipartUpload(ctx, p); err != nil { if err = h.obj.CreateMultipartUpload(ctx, p); err != nil {
h.logAndSendError(ctx, w, "could create multipart upload", reqInfo, err, additional...) h.logAndSendError(ctx, w, "could create multipart upload", reqInfo, err, additional...)
return return
@ -229,6 +223,12 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err, additional...)
return
}
hash, err := h.obj.UploadPart(ctx, p) hash, err := h.obj.UploadPart(ctx, p)
if err != nil { if err != nil {
h.logAndSendError(ctx, w, "could not upload a part", reqInfo, err, additional...) h.logAndSendError(ctx, w, "could not upload a part", reqInfo, err, additional...)
@ -354,6 +354,12 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
return return
} }
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err, additional...)
return
}
info, err := h.obj.UploadPartCopy(ctx, p) info, err := h.obj.UploadPartCopy(ctx, p)
if err != nil { if err != nil {
h.logAndSendError(ctx, w, "could not upload part copy", reqInfo, err, additional...) h.logAndSendError(ctx, w, "could not upload part copy", reqInfo, err, additional...)
@ -416,6 +422,12 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
Parts: reqBody.Parts, Parts: reqBody.Parts,
} }
c.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err, additional...)
return
}
// Start complete multipart upload which may take some time to fetch object // Start complete multipart upload which may take some time to fetch object
// and re-upload it part by part. // and re-upload it part by part.
objInfo, err := h.completeMultipartUpload(r, c, bktInfo) objInfo, err := h.completeMultipartUpload(r, c, bktInfo)

View file

@ -81,6 +81,26 @@ func TestDeleteMultipartAllParts(t *testing.T) {
require.Empty(t, hc.tp.Objects()) require.Empty(t, hc.tp.Objects())
} }
func TestMultipartCopiesNumber(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName := "bucket", "object"
createTestBucket(hc, bktName)
copies := []uint32{2, 0}
hc.config.copiesNumbers = map[string][]uint32{"default": copies}
multipartInfo := createMultipartUpload(hc, bktName, objName, nil)
uploadPart(hc, bktName, objName, multipartInfo.UploadID, 1, layer.UploadMinSize)
objs := hc.tp.Objects()
require.Len(t, objs, 1)
require.EqualValues(t, copies, hc.tp.CopiesNumbers(addrFromObject(objs[0]).EncodeToString()))
}
func TestSpecialMultipartName(t *testing.T) { func TestSpecialMultipartName(t *testing.T) {
hc := prepareHandlerContextWithMinCache(t) hc := prepareHandlerContextWithMinCache(t)
@ -792,3 +812,14 @@ func listPartsBase(hc *handlerContext, bktName, objName string, encrypted bool,
return listPartsResponse return listPartsResponse
} }
func addrFromObject(obj *object.Object) oid.Address {
var addr oid.Address
cnrID, _ := obj.ContainerID()
objID, _ := obj.ID()
addr.SetContainer(cnrID)
addr.SetObject(objID)
return addr
}

View file

@ -15,6 +15,8 @@ import (
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
) )
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
// ListObjectsV1Handler handles objects listing requests for API version 1. // ListObjectsV1Handler handles objects listing requests for API version 1.
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context() ctx := r.Context()

View file

@ -142,7 +142,7 @@ func parsePatchConditionalHeaders(headers http.Header, log *zap.Logger) *conditi
if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil { if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil {
args.IfUnmodifiedSince = httpTime args.IfUnmodifiedSince = httpTime
} else { } else {
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err)) log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err), logs.TagField(logs.TagDatapath))
} }
return args return args

View file

@ -54,17 +54,29 @@ func (p *postPolicy) condition(key string) *policyCondition {
return nil return nil
} }
func (p *postPolicy) CheckContentLength(size uint64) bool { func (p *postPolicy) CheckContentLength(size uint64) error {
if p.empty { if p.empty {
return true return nil
} }
for _, condition := range p.Conditions { for _, condition := range p.Conditions {
if condition.Matching == "content-length-range" { if condition.Matching == "content-length-range" {
length := strconv.FormatUint(size, 10) start, err := strconv.ParseUint(condition.Key, 10, 64)
return condition.Key <= length && length <= condition.Value if err != nil {
return errInvalidCondition
}
end, err := strconv.ParseUint(condition.Value, 10, 64)
if err != nil {
return errInvalidCondition
}
if start <= size && size <= end {
return nil
}
return fmt.Errorf("length of the content did not fall within the range specified in the condition")
} }
} }
return true return nil
} }
func (p *policyCondition) match(value string) bool { func (p *policyCondition) match(value string) bool {
@ -311,10 +323,23 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
} }
} }
func (h *handler) getBodyReader(r *http.Request) (io.ReadCloser, error) { type BodyReader interface {
io.ReadCloser
TrailerHeaders() map[string]string
}
type noTrailerBodyReader struct {
io.ReadCloser
}
func (r *noTrailerBodyReader) TrailerHeaders() map[string]string {
return nil
}
func (h *handler) getBodyReader(r *http.Request) (BodyReader, error) {
shaType, streaming := api.IsSignedStreamingV4(r) shaType, streaming := api.IsSignedStreamingV4(r)
if !streaming { if !streaming {
return r.Body, nil return &noTrailerBodyReader{r.Body}, nil
} }
encodings := r.Header.Values(api.ContentEncoding) encodings := r.Header.Values(api.ContentEncoding)
@ -350,12 +375,15 @@ func (h *handler) getBodyReader(r *http.Request) (io.ReadCloser, error) {
var ( var (
err error err error
chunkReader io.ReadCloser chunkReader BodyReader
) )
if shaType == api.StreamingContentV4aSHA256 { switch shaType {
chunkReader, err = newSignV4aChunkedReader(r) case api.StreamingContentSHA256, api.StreamingContentSHA256Trailer:
} else {
chunkReader, err = newSignV4ChunkedReader(r) chunkReader, err = newSignV4ChunkedReader(r)
case api.StreamingContentV4aSHA256, api.StreamingContentV4aSHA256Trailer:
chunkReader, err = newSignV4aChunkedReader(r)
default:
chunkReader, err = newUnsignedChunkedReader(r.Body)
} }
if err != nil { if err != nil {
@ -450,7 +478,7 @@ func (h *handler) isTLSCheckRequired(r *http.Request) bool {
tlsTermination, err := strconv.ParseBool(tlsTerminationStr) tlsTermination, err := strconv.ParseBool(tlsTerminationStr)
if err != nil { if err != nil {
h.reqLogger(r.Context()).Warn(logs.WarnInvalidTypeTLSTerminationHeader, zap.String("header", tlsTerminationStr), zap.Error(err)) h.reqLogger(r.Context()).Warn(logs.WarnInvalidTypeTLSTerminationHeader, zap.String("header", tlsTerminationStr), zap.Error(err), logs.TagField(logs.TagDatapath))
return true return true
} }
@ -544,8 +572,8 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
return return
} }
if !policy.CheckContentLength(size) { if err := policy.CheckContentLength(size); err != nil {
h.logAndSendError(ctx, w, "invalid content-length", reqInfo, apierr.GetAPIError(apierr.ErrInvalidArgument)) h.logAndSendError(ctx, w, err.Error(), reqInfo, apierr.GetAPIError(apierr.ErrPostPolicyConditionInvalidFormat))
return return
} }
@ -812,7 +840,7 @@ func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Reque
h.logAndSendError(ctx, w, "could not create bucket", reqInfo, err) h.logAndSendError(ctx, w, "could not create bucket", reqInfo, err)
return return
} }
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID)) h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID), logs.TagField(logs.TagExternalStorage))
chains := bucketCannedACLToAPERules(cannedACL, reqInfo, bktInfo.CID) chains := bucketCannedACLToAPERules(cannedACL, reqInfo, bktInfo.CID)
if err = h.ape.SaveACLChains(bktInfo.CID.EncodeToString(), chains); err != nil { if err = h.ape.SaveACLChains(bktInfo.CID.EncodeToString(), chains); err != nil {
@ -1025,7 +1053,7 @@ func isLockEnabled(log *zap.Logger, header http.Header) bool {
lockEnabled, err := strconv.ParseBool(lockEnabledStr) lockEnabled, err := strconv.ParseBool(lockEnabledStr)
if err != nil { if err != nil {
log.Warn(logs.InvalidBucketObjectLockEnabledHeader, zap.String("header", lockEnabledStr), zap.Error(err)) log.Warn(logs.InvalidBucketObjectLockEnabledHeader, zap.String("header", lockEnabledStr), zap.Error(err), logs.TagField(logs.TagDatapath))
} }
return lockEnabled return lockEnabled

View file

@ -10,6 +10,8 @@ import (
"encoding/hex" "encoding/hex"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt"
"hash/crc32"
"io" "io"
"mime/multipart" "mime/multipart"
"net/http" "net/http"
@ -377,13 +379,34 @@ func TestPutObjectCheckContentSHA256(t *testing.T) {
} }
} }
func TestPutObjectWithStreamBodyAWSExample(t *testing.T) { func TestPutObjectWithStreamUnsignedBodySmall(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName := "test2", "tmp.txt"
createTestBucket(hc, bktName)
w, req, chunk := getChunkedRequestUnsignedTrailingSmall(hc.context, t, bktName, objName)
hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
w, req = prepareTestRequest(hc, bktName, objName, nil)
hc.Handler().HeadObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
require.Equal(t, "5", w.Header().Get(api.ContentLength))
data := getObjectRange(t, hc, bktName, objName, 0, 5)
for i := range chunk {
require.Equal(t, chunk[i], data[i])
}
}
func TestPutObjectWithStreamUnsignedBody(t *testing.T) {
hc := prepareHandlerContext(t) hc := prepareHandlerContext(t)
bktName, objName := "examplebucket", "chunkObject.txt" bktName, objName := "examplebucket", "chunkObject.txt"
createTestBucket(hc, bktName) createTestBucket(hc, bktName)
w, req, chunk := getChunkedRequest(hc.context, t, bktName, objName) w, req, chunk := getChunkedRequestUnsignedTrailing(hc.context, t, bktName, objName)
hc.Handler().PutObjectHandler(w, req) hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusOK) assertStatus(t, w, http.StatusOK)
@ -398,13 +421,67 @@ func TestPutObjectWithStreamBodyAWSExample(t *testing.T) {
} }
} }
func TestPutObjectWithStreamEmptyBodyAWSExample(t *testing.T) { func TestPutObjectWithStreamBodyAWSExampleTrailing(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName := "examplebucket", "chunkObject.txt"
createTestBucket(hc, bktName)
t.Run("valid trailer signature", func(t *testing.T) {
w, req, chunk := getChunkedRequestAWSExampleTrailing(t, bktName, objName)
hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
w, req = prepareTestRequest(hc, bktName, objName, nil)
hc.Handler().HeadObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
require.Equal(t, strconv.Itoa(awsChunkedRequestExampleDecodedContentLength), w.Header().Get(api.ContentLength))
data := getObjectRange(t, hc, bktName, objName, 0, awsChunkedRequestExampleDecodedContentLength)
equalDataSlices(t, chunk, data)
})
t.Run("invalid trailer signature", func(t *testing.T) {
w, req, _ := getChunkedRequestAWSExampleTrailing(t, bktName, objName)
body := req.Body.(*customNopCloser)
body.Bytes()[body.Len()-2] = 'a'
hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusForbidden)
})
}
func TestPutObjectWithStreamBodyAWSExample(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName := "examplebucket", "chunkObject.txt"
createTestBucket(hc, bktName)
w, req, chunk := getChunkedRequestAWSExample(t, bktName, objName)
hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
w, req = prepareTestRequest(hc, bktName, objName, nil)
hc.Handler().HeadObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
require.Equal(t, strconv.Itoa(awsChunkedRequestExampleDecodedContentLength), w.Header().Get(api.ContentLength))
data := getObjectRange(t, hc, bktName, objName, 0, awsChunkedRequestExampleDecodedContentLength)
for i := range chunk {
require.Equal(t, chunk[i], data[i])
}
}
func TestPutObjectWithStreamEmptyBodyAWSExampleWithContentType(t *testing.T) {
hc := prepareHandlerContext(t) hc := prepareHandlerContext(t)
bktName, objName := "dkirillov", "tmp" bktName, objName := "dkirillov", "tmp"
createTestBucket(hc, bktName) createTestBucket(hc, bktName)
w, req := getEmptyChunkedRequest(hc.context, t, bktName, objName) signTime, err := time.Parse("20060102T150405Z", "20241003T100055Z")
require.NoError(t, err)
extra := [2]string{api.ContentType, "text/plain; charset=UTF-8"}
w, req := getChunkedRequestBase(t, bktName, objName, nil, api.StreamingContentSHA256, signTime, extra)
hc.Handler().PutObjectHandler(w, req) hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusOK) assertStatus(t, w, http.StatusOK)
@ -418,13 +495,94 @@ func TestPutObjectWithStreamEmptyBodyAWSExample(t *testing.T) {
require.Empty(t, res.Contents[0].Size) require.Empty(t, res.Contents[0].Size)
} }
func TestPutObjectWithStreamEmptyBody(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket"
createTestBucket(hc, bktName)
signTime, err := time.Parse("20060102T150405Z", "20241003T100055Z")
require.NoError(t, err)
t.Run("unsigned", func(t *testing.T) {
t.Run("trailer", func(t *testing.T) {
objName := "unsigned trailer"
w, req := getEmptyChunkedRequestUnsigned(t, bktName, objName)
req.Header.Del(api.ContentType)
hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
d, h := getObject(hc, bktName, objName)
require.Empty(t, d)
require.Equal(t, "0", h.Get(api.ContentLength))
})
})
t.Run("sigv4", func(t *testing.T) {
t.Run("trailer", func(t *testing.T) {
objName := "sigv4 trailer"
w, req := getChunkedRequestBase(t, bktName, objName, nil, api.StreamingContentSHA256Trailer, signTime)
req.Header.Del(api.ContentType)
hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
d, h := getObject(hc, bktName, objName)
require.Empty(t, d)
require.Equal(t, "0", h.Get(api.ContentLength))
})
t.Run("no trailer", func(t *testing.T) {
objName := "sigv4 no trailer"
w, req := getChunkedRequestBase(t, bktName, objName, nil, api.StreamingContentSHA256, signTime)
req.Header.Del(api.ContentType)
hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
d, h := getObject(hc, bktName, objName)
require.Empty(t, d)
require.Equal(t, "0", h.Get(api.ContentLength))
})
})
t.Run("sigv4a", func(t *testing.T) {
t.Run("trailer", func(t *testing.T) {
objName := "sigv4a trailer"
w, req := getEmptyChunkedRequestSigv4aWithTrailers(t, bktName, objName)
req.Header.Del(api.ContentType)
hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
d, h := getObject(hc, bktName, objName)
require.Empty(t, d)
require.Equal(t, "0", h.Get(api.ContentLength))
})
t.Run("no trailer", func(t *testing.T) {
objName := "sigv4a no trailer"
w, req := getEmptyChunkedRequestSigv4a(t, bktName, objName)
req.Header.Del(api.ContentType)
hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
d, h := getObject(hc, bktName, objName)
require.Empty(t, d)
require.Equal(t, "0", h.Get(api.ContentLength))
})
})
}
func TestPutChunkedTestContentEncoding(t *testing.T) { func TestPutChunkedTestContentEncoding(t *testing.T) {
hc := prepareHandlerContext(t) hc := prepareHandlerContext(t)
bktName, objName := "examplebucket", "chunkObject.txt" bktName, objName := "examplebucket", "chunkObject.txt"
createTestBucket(hc, bktName) createTestBucket(hc, bktName)
w, req, _ := getChunkedRequest(hc.context, t, bktName, objName) w, req, _ := getChunkedRequestAWSExample(t, bktName, objName)
req.Header.Set(api.ContentEncoding, api.AwsChunked+",gzip") req.Header.Set(api.ContentEncoding, api.AwsChunked+",gzip")
hc.Handler().PutObjectHandler(w, req) hc.Handler().PutObjectHandler(w, req)
@ -433,13 +591,13 @@ func TestPutChunkedTestContentEncoding(t *testing.T) {
resp := headObjectBase(hc, bktName, objName, emptyVersion) resp := headObjectBase(hc, bktName, objName, emptyVersion)
require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding)) require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding))
w, req, _ = getChunkedRequest(hc.context, t, bktName, objName) w, req, _ = getChunkedRequestAWSExample(t, bktName, objName)
req.Header.Set(api.ContentEncoding, "gzip") req.Header.Set(api.ContentEncoding, "gzip")
hc.Handler().PutObjectHandler(w, req) hc.Handler().PutObjectHandler(w, req)
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidEncodingMethod)) assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidEncodingMethod))
hc.config.bypassContentEncodingInChunks = true hc.config.bypassContentEncodingInChunks = true
w, req, _ = getChunkedRequest(hc.context, t, bktName, objName) w, req, _ = getChunkedRequestAWSExample(t, bktName, objName)
req.Header.Set(api.ContentEncoding, "gzip") req.Header.Set(api.ContentEncoding, "gzip")
hc.Handler().PutObjectHandler(w, req) hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusOK) assertStatus(t, w, http.StatusOK)
@ -448,9 +606,9 @@ func TestPutChunkedTestContentEncoding(t *testing.T) {
require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding)) require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding))
} }
// getChunkedRequest implements request example from // getChunkedRequestAWSExample implements request example from
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) { func getChunkedRequestAWSExample(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
chunk := make([]byte, 65*1024) chunk := make([]byte, 65*1024)
for i := range chunk { for i := range chunk {
chunk[i] = 'a' chunk[i] = 'a'
@ -458,12 +616,8 @@ func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName strin
chunk1 := chunk[:64*1024] chunk1 := chunk[:64*1024]
chunk2 := chunk[64*1024:] chunk2 := chunk[64*1024:]
AWSAccessKeyID := "AKIAIOSFODNN7EXAMPLE"
AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey}
signer := v4.NewSigner()
reqBody := bytes.NewBufferString("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n") reqBody := bytes.NewBufferString("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n")
_, err := reqBody.Write(chunk1) _, err := reqBody.Write(chunk1)
require.NoError(t, err) require.NoError(t, err)
@ -476,79 +630,334 @@ func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName strin
req, err := http.NewRequest("PUT", "https://s3.amazonaws.com/"+bktName+"/"+objName, nil) req, err := http.NewRequest("PUT", "https://s3.amazonaws.com/"+bktName+"/"+objName, nil)
require.NoError(t, err) require.NoError(t, err)
req.Header.Set("content-encoding", "aws-chunked") req.Header.Set("content-encoding", api.AwsChunked)
req.Header.Set("content-length", strconv.Itoa(awsChunkedRequestExampleContentLength)) req.Header.Set("content-length", strconv.Itoa(awsChunkedRequestExampleContentLength))
req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD") req.Header.Set("x-amz-content-sha256", api.StreamingContentSHA256)
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength)) req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9")
signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z") signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z")
require.NoError(t, err) require.NoError(t, err)
err = signer.SignHTTP(ctx, awsCreds, req, auth.UnsignedPayload, "s3", "us-east-1", signTime) req.Body = io.NopCloser(reqBody)
w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
return w, req, chunk
}
type customNopCloser struct {
*bytes.Buffer
}
func (c *customNopCloser) Close() error {
return nil
}
// getChunkedRequestAWSExampleTrailing implements request example from
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html
func getChunkedRequestAWSExampleTrailing(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
chunk := make([]byte, 65*1024)
for i := range chunk {
chunk[i] = 'a'
}
chunk1 := chunk[:64*1024]
chunk2 := chunk[64*1024:]
AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
reqBody := bytes.NewBufferString("10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n")
_, err := reqBody.Write(chunk1)
require.NoError(t, err)
_, err = reqBody.WriteString("\r\n400;chunk-signature=1c1344b170168f8e65b41376b44b20fe354e373826ccbbe2c1d40a8cae51e5c7\r\n")
require.NoError(t, err)
_, err = reqBody.Write(chunk2)
require.NoError(t, err)
_, err = reqBody.WriteString("\r\n0;chunk-signature=2ca2aba2005185cf7159c6277faf83795951dd77a3a99e6e65d5c9f85863f992\r\n")
require.NoError(t, err)
_, err = reqBody.WriteString("x-amz-checksum-crc32c:sOO8/Q==\n")
require.NoError(t, err)
// original signature is 63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f,
// but we use d81f82fc3505edab99d459891051a732e8730629a2e4a59689829ca17fe2e435
// because original signature is incorrect
// it was calculated using the`AWS4-HMAC-SHA256-PAYLOAD` constant in canonical string instead of
// `AWS4-HMAC-SHA256-TRAILER` that actually must be used by spec
// (java sdk use correct `AWS4-HMAC-SHA256-TRAILER` string).
_, err = reqBody.WriteString("x-amz-trailer-signature:d81f82fc3505edab99d459891051a732e8730629a2e4a59689829ca17fe2e435")
require.NoError(t, err)
req, err := http.NewRequest("PUT", "https://s3.amazonaws.com/"+bktName+"/"+objName, nil)
require.NoError(t, err)
req.Header.Set("content-encoding", api.AwsChunked)
req.Header.Set("content-length", strconv.Itoa(awsChunkedRequestExampleContentLength))
req.Header.Set("x-amz-content-sha256", api.StreamingContentSHA256Trailer)
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32c")
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e")
signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z")
require.NoError(t, err)
req.Body = &customNopCloser{Buffer: reqBody}
w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
return w, req, chunk
}
func getChunkedRequestUnsignedTrailing(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
chunk := make([]byte, 65*1024)
for i := range chunk {
chunk[i] = 'a'
}
AWSAccessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6"
AWSSecretAccessKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e"
awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey}
signer := v4.NewSigner()
reqBody := bytes.NewBufferString("10400\r\n")
_, err := reqBody.Write(chunk)
require.NoError(t, err)
_, err = reqBody.WriteString("\r\n0\r\n")
require.NoError(t, err)
_, err = reqBody.WriteString("\r\nx-amz-checksum-crc64nvme:pRf+emrnL+A=\r\n\r\n")
require.NoError(t, err)
req, err := http.NewRequest("PUT", "https://localhost:8184/"+bktName+"/"+objName, nil)
require.NoError(t, err)
req.Header.Set("x-amz-sdk-checksum-algorithm", "CRC64NVME")
req.Header.Set("content-encoding", api.AwsChunked)
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc64nvme")
req.Header.Set("x-amz-content-sha256", api.StreamingUnsignedPayloadTrailer)
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
signTime, err := time.Parse("20060102T150405Z", "20250131T140527Z")
require.NoError(t, err)
err = signer.SignHTTP(ctx, awsCreds, req, api.StreamingContentSHA256Trailer, "s3", "ru", signTime)
require.NoError(t, err) require.NoError(t, err)
req.Body = io.NopCloser(reqBody) req.Body = io.NopCloser(reqBody)
w := httptest.NewRecorder() w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{
ClientTime: signTime,
AuthHeaders: &middleware.AuthHeader{
AccessKeyID: AWSAccessKeyID,
SignatureV4: "4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9",
Region: "us-east-1",
},
AccessBox: &accessbox.Box{
Gate: &accessbox.GateData{
SecretKey: AWSSecretAccessKey,
},
},
}))
return w, req, chunk return w, req, chunk
} }
func getEmptyChunkedRequest(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) { func getChunkedRequestUnsignedTrailingSmall(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
AWSAccessKeyID := "48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh" AWSAccessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6"
AWSSecretAccessKey := "09260955b4eb0279dc017ba20a1ddac909cbd226c86cbb2d868e55534c8e64b0" AWSSecretAccessKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e"
reqBody := bytes.NewBufferString("0;chunk-signature=311a7142c8f3a07972c3aca65c36484b513a8fee48ab7178c7225388f2ae9894\r\n\r\n") awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey}
signer := v4.NewSigner()
chunk := "tmp2\n"
reqBody := bytes.NewBufferString("5\r\n")
_, err := reqBody.WriteString(chunk)
require.NoError(t, err)
_, err = reqBody.WriteString("\r\n0\r\n")
require.NoError(t, err)
_, err = reqBody.WriteString("x-amz-checksum-crc64nvme:q1EYl4rI0TU=\r\n\r\n")
require.NoError(t, err)
req, err := http.NewRequest("PUT", "https://localhost:8184/"+bktName+"/"+objName, nil)
require.NoError(t, err)
req.Header.Set("x-amz-sdk-checksum-algorithm", "CRC64NVME")
req.Header.Set("content-encoding", api.AwsChunked)
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc64nvme")
req.Header.Set("x-amz-content-sha256", api.StreamingUnsignedPayloadTrailer)
req.Header.Set("x-amz-decoded-content-length", "5")
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
signTime, err := time.Parse("20060102T150405Z", "20250203T063745Z")
require.NoError(t, err)
err = signer.SignHTTP(ctx, awsCreds, req, api.StreamingContentSHA256Trailer, "s3", "ru", signTime)
require.NoError(t, err)
req.Body = io.NopCloser(reqBody)
w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
return w, req, []byte(chunk)
}
func getChunkedRequestBase(t *testing.T, bktName, objName string, chunks [][]byte, shaType string, signTime time.Time, extraHeaders ...[2]string) (*httptest.ResponseRecorder, *http.Request) {
creds := aws.Credentials{
AccessKeyID: "48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh",
SecretAccessKey: "09260955b4eb0279dc017ba20a1ddac909cbd226c86cbb2d868e55534c8e64b0",
}
region := "us-east-1"
service := "s3"
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, nil)
require.NoError(t, err)
payloadLength := 0
for _, chunk := range chunks {
payloadLength += len(chunk)
}
for _, kv := range extraHeaders {
req.Header.Set(kv[0], kv[1])
}
req.Header.Set(api.ContentEncoding, api.AwsChunked)
req.Header.Set(api.AmzDecodedContentLength, strconv.Itoa(payloadLength))
req.Header.Set(api.AmzDate, signTime.Format("20060102T150405Z"))
req.Header.Set(api.AmzContentSha256, shaType)
if shaType == api.StreamingContentSHA256Trailer {
req.Header.Set(api.AmzTrailer, "x-amz-checksum-crc32")
}
signer := v4.NewSigner()
err = signer.SignHTTP(req.Context(), creds, req, shaType, service, region, signTime)
require.NoError(t, err)
seedSignature := strings.Split(req.Header.Get(api.Authorization), "Signature=")[1]
seed, err := hex.DecodeString(seedSignature)
require.NoError(t, err)
var reqBody bytes.Buffer
hash := crc32.NewIEEE()
newStreamSigner := v4.NewStreamSigner(creds, service, region, seed)
for _, chunk := range chunks {
_, err = hash.Write(chunk)
require.NoError(t, err)
signature, err := newStreamSigner.GetSignature(req.Context(), nil, chunk, signTime)
require.NoError(t, err)
reqBody.WriteString(fmt.Sprintf("%x;chunk-signature=%x\r\n", len(chunk), signature))
reqBody.Write(chunk)
reqBody.WriteString("\r\n")
}
signature, err := newStreamSigner.GetSignature(req.Context(), nil, nil, signTime)
require.NoError(t, err)
reqBody.WriteString(fmt.Sprintf("0;chunk-signature=%x\r\n", signature))
if shaType == api.StreamingContentSHA256Trailer {
crc32Res := hash.Sum(nil)
checksumStr := "x-amz-checksum-crc32:" + base64.StdEncoding.EncodeToString(crc32Res)
reqBody.WriteString(fmt.Sprintf("%s\r\n", checksumStr))
trailerSignature, err := newStreamSigner.GetTrailerSignature([]byte(checksumStr+"\n"), signTime)
require.NoError(t, err)
reqBody.WriteString(fmt.Sprintf("x-amz-trailer-signature:%x\r\n", trailerSignature))
}
reqBody.WriteString("\r\n")
req.Body = io.NopCloser(&reqBody)
return prepareReqMiddlewares(req, signTime, creds.SecretAccessKey)
}
func prepareReqMiddlewares(req *http.Request, signTime time.Time, secretAccessKey string) (*httptest.ResponseRecorder, *http.Request) {
authHeader := req.Header.Get(api.Authorization)
var parsed map[string]string
var region string
if strings.HasPrefix(authHeader, auth.SignaturePreambleSigV4) {
parsed = auth.NewRegexpMatcher(auth.AuthorizationFieldRegexp).GetSubmatches(authHeader)
region = parsed["region"]
} else {
parsed = auth.NewRegexpMatcher(auth.AuthorizationFieldV4aRegexp).GetSubmatches(authHeader)
region = req.Header.Get("X-Amz-Region-Set")
}
bktObj := strings.Split(req.URL.Path, "/")
box := &middleware.Box{
ClientTime: signTime,
AuthHeaders: &middleware.AuthHeader{
AccessKeyID: parsed["access_key_id"],
SignatureV4: parsed["v4_signature"],
Region: region,
},
AccessBox: &accessbox.Box{Gate: &accessbox.GateData{SecretKey: secretAccessKey}},
}
w := httptest.NewRecorder()
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktObj[1], Object: bktObj[2]}, "")
req = req.WithContext(middleware.SetReqInfo(req.Context(), reqInfo))
req = req.WithContext(middleware.SetBox(req.Context(), box))
return w, req
}
func getEmptyChunkedRequestUnsigned(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) {
AWSSecretAccessKey := "f1a0d650b650149f1a83140418e88a3c5572a0103e912e326492a91c19c4488a"
reqBody := bytes.NewBufferString("0\r\nx-amz-checksum-crc64nvme:AAAAAAAAAAA=\r\n\r\n")
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, reqBody) req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, reqBody)
require.NoError(t, err) require.NoError(t, err)
req.Header.Set("Amz-Sdk-Invocation-Id", "8a8cd4be-aef8-8034-f08d-a6144ade41f9") req.Header.Set(api.Authorization, "AWS4-HMAC-SHA256 Credential=3jNrmDtHtuj1uLcixaSMA4KNUhNYhv1EpUNdFnbTXgUP071pGdSZfHSLtoC8gzjF5HoD6sC3Scq33t1WvvEvjmPnt/20250213/ru/s3/aws4_request, SignedHeaders=content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-sdk-checksum-algorithm;x-amz-trailer, Signature=1231b012c0ac313770c5a95ccf77b95b6c9b1c3760d6aa24cb8309801d56eb4a")
req.Header.Set("Amz-Sdk-Request", "attempt=1; max=2") req.Header.Set(api.ContentEncoding, api.AwsChunked)
req.Header.Set(api.Authorization, "AWS4-HMAC-SHA256 Credential=48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh/20241003/us-east-1/s3/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-encoding;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length, Signature=4b530ab4af2381f214941af591266b209968264a2c94337fa1efc048c7dff352") req.Header.Set(api.AmzDate, "20250213T124858Z")
req.Header.Set(api.ContentEncoding, "aws-chunked") req.Header.Set(api.AmzContentSha256, api.StreamingUnsignedPayloadTrailer)
req.Header.Set(api.ContentLength, "86")
req.Header.Set(api.ContentType, "text/plain; charset=UTF-8")
req.Header.Set(api.AmzDate, "20241003T100055Z")
req.Header.Set(api.AmzContentSha256, "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
req.Header.Set(api.AmzDecodedContentLength, "0") req.Header.Set(api.AmzDecodedContentLength, "0")
req.Header.Set("X-Amz-Trailer", "x-amz-checksum-crc64nvme")
req.Header.Set("X-Amz-Sdk-Checksum-Algorithm", "CRC64NVME")
signTime, err := time.Parse("20060102T150405Z", "20241003T100055Z") signTime, err := time.Parse("20060102T150405Z", req.Header.Get(api.AmzDate))
require.NoError(t, err) require.NoError(t, err)
w := httptest.NewRecorder() return prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "") }
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{
ClientTime: signTime,
AuthHeaders: &middleware.AuthHeader{
AccessKeyID: AWSAccessKeyID,
SignatureV4: "4b530ab4af2381f214941af591266b209968264a2c94337fa1efc048c7dff352",
Region: "us-east-1",
},
AccessBox: &accessbox.Box{
Gate: &accessbox.GateData{
SecretKey: AWSSecretAccessKey,
},
},
}))
return w, req func getEmptyChunkedRequestSigv4aWithTrailers(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) {
AWSSecretAccessKey := "f1a0d650b650149f1a83140418e88a3c5572a0103e912e326492a91c19c4488a"
body := "0;chunk-signature=3046022100ab9229a80d70f4d004768992881821a441a4ad4102e18de567e68216659bf497022100ec47a7a445351683557eedf893e6ed250c97af4b0415814671770b83766d69be\r\n" +
"x-amz-checksum-crc32:AAAAAA==\r\n" +
"x-amz-trailer-signature:3046022100a0a66c1adcee8d99460b4631b23c95fbad9eb4e6c56f1afb9e255715ba141169022100b2cfc8adc8036eb985f1ab0e770b575284c5fc8ca75c226558d3142cbaab83ce\r\n\r\n"
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, bytes.NewBufferString(body))
require.NoError(t, err)
req.Header.Set(api.Authorization, "AWS4-ECDSA-P256-SHA256 Credential=3jNrmDtHtuj1uLcixaSMA4KNUhNYhv1EpUNdFnbTXgUP071pGdSZfHSLtoC8gzjF5HoD6sC3Scq33t1WvvEvjmPnt/20250213/s3/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-region-set;x-amz-sdk-checksum-algorithm;x-amz-trailer, Signature=304402202e1f1efcc56c588d9a94a3d8f20368686df8bfd5e8aad01fc4eff569ff38f1800220215198e3f1ba785492fe6703c4722872909ce8a09e8c9a13da90a9230c7a24b7")
req.Header.Set("Amz-Sdk-Invocation-Id", "d42dc16d-7899-55fb-5b72-a654bd482f4f")
req.Header.Set("Amz-Sdk-Request", "attempt=1; max=2")
req.Header.Set(api.ContentEncoding, api.AwsChunked)
req.Header.Set(api.AmzDate, "20250213T132401Z")
req.Header.Set(api.AmzContentSha256, api.StreamingContentV4aSHA256Trailer)
req.Header.Set(api.AmzDecodedContentLength, "0")
req.Header.Set(api.ContentLength, "367")
req.Header.Set(api.ContentType, "text/plain: charset=UTF-8")
req.Header.Set("X-Amz-Region-Set", "us-east-1")
req.Header.Set("X-Amz-Trailer", "x-amz-checksum-crc32")
req.Header.Set("X-Amz-Sdk-Checksum-Algorithm", "CRC32")
signTime, err := time.Parse("20060102T150405Z", req.Header.Get(api.AmzDate))
require.NoError(t, err)
return prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
}
func getEmptyChunkedRequestSigv4a(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) {
AWSSecretAccessKey := "f1a0d650b650149f1a83140418e88a3c5572a0103e912e326492a91c19c4488a"
body := "0;chunk-signature=304502203f7c598a2e9a6673bf1ca30f5f6bebd0d76a4e9d3c16531448e96c2cda22d16a0221009e7ed578da0a9781366f1461a1484e64f15707f26d4310e59514db6ff9f7e0f1**\r\n\r\n"
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, bytes.NewBufferString(body))
require.NoError(t, err)
req.Header.Set(api.Authorization, "AWS4-ECDSA-P256-SHA256 Credential=3jNrmDtHtuj1uLcixaSMA4KNUhNYhv1EpUNdFnbTXgUP071pGdSZfHSLtoC8gzjF5HoD6sC3Scq33t1WvvEvjmPnt/20250213/s3/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-region-set, Signature=3046022100dc589ea513448b996809db4b314a0b8a4a775c1165c6203c7104b2f1aae1243c0221009bf3a256e7c33415eaad20c1dbfb4e14cb00b362758bc4d2aaf94ca96a5f13f9")
req.Header.Set("Amz-Sdk-Invocation-Id", "f0814a40-0d74-066f-d01f-ed14f28ebfa4")
req.Header.Set("Amz-Sdk-Request", "attempt=1; max=2")
req.Header.Set(api.ContentEncoding, api.AwsChunked)
req.Header.Set(api.AmzDate, "20250213T135717Z")
req.Header.Set(api.AmzContentSha256, api.StreamingContentV4aSHA256)
req.Header.Set(api.AmzDecodedContentLength, "0")
req.Header.Set(api.ContentLength, "166")
req.Header.Set(api.ContentType, "text/plain: charset=UTF-8")
req.Header.Set("X-Amz-Region-Set", "use-east-1")
signTime, err := time.Parse("20060102T150405Z", req.Header.Get(api.AmzDate))
require.NoError(t, err)
return prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
} }
func TestCreateBucket(t *testing.T) { func TestCreateBucket(t *testing.T) {
@ -797,6 +1206,104 @@ func TestFormEncryptionParamsBase(t *testing.T) {
} }
} }
func TestCheckContentLength(t *testing.T) {
contentLength := "content-length-range"
notFallError := "length of the content did not fall within the range specified in the condition"
parseError := "invalid condition"
for _, tc := range []struct {
name string
matching string
key string
value string
size uint64
errMsg string
emptyPolicy bool
}{
{
name: "valid",
matching: contentLength,
key: "0",
value: "1000",
size: 50,
},
{
name: "valid lower limit",
matching: contentLength,
key: "5",
value: "100",
size: 5,
},
{
name: "valid upper limit",
matching: contentLength,
key: "5",
value: "100",
size: 100,
},
{
name: "invalid size value (too small)",
matching: contentLength,
key: "5",
value: "100",
size: 2,
errMsg: notFallError,
},
{
name: "invalid size value (to high)",
matching: contentLength,
key: "5",
value: "100",
size: 200,
errMsg: notFallError,
},
{
name: "no matching",
},
{
name: "invalid key type",
matching: contentLength,
key: "invalid",
value: "100",
size: 10,
errMsg: parseError,
},
{
name: "invalid value type",
matching: contentLength,
key: "5",
value: "invalid",
size: 10,
errMsg: parseError,
},
{
name: "empty policy",
emptyPolicy: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
policy := &postPolicy{
Conditions: []*policyCondition{
{
Matching: tc.matching,
Key: tc.key,
Value: tc.value,
},
},
empty: tc.emptyPolicy,
}
err := policy.CheckContentLength(tc.size)
if tc.errMsg != "" {
require.Error(t, err)
require.Contains(t, err.Error(), tc.errMsg)
return
}
require.NoError(t, err)
})
}
}
func prepareRequestForEncryption(hc *handlerContext, algo, key, md5, tlsTermination string, reqWithoutTLS, reqWithoutSSE, isCopySource bool) *http.Request { func prepareRequestForEncryption(hc *handlerContext, algo, key, md5, tlsTermination string, reqWithoutTLS, reqWithoutSSE, isCopySource bool) *http.Request {
r := httptest.NewRequest(http.MethodPost, "/", nil) r := httptest.NewRequest(http.MethodPost, "/", nil)

View file

@ -15,6 +15,9 @@ type ListBucketsResponse struct {
Buckets struct { Buckets struct {
Buckets []Bucket `xml:"Bucket"` Buckets []Bucket `xml:"Bucket"`
} // Buckets are nested } // Buckets are nested
ContinuationToken string `xml:"ContinuationToken,omitempty"`
Prefix string `xml:"Prefix,omitempty"`
} }
// ListObjectsV1Response -- format for ListObjectsV1 response. // ListObjectsV1Response -- format for ListObjectsV1 response.
@ -51,8 +54,9 @@ type ListObjectsV2Response struct {
// Bucket container for bucket metadata. // Bucket container for bucket metadata.
type Bucket struct { type Bucket struct {
Name string Name string `xml:"Name"`
CreationDate string // time string of format "2006-01-02T15:04:05.000Z" CreationDate string `xml:"CreationDate"` // time string of format "2006-01-02T15:04:05.000Z"
BucketRegion string `xml:"BucketRegion,omitempty"`
} }
// PolicyStatus contains status of bucket policy. // PolicyStatus contains status of bucket policy.

View file

@ -8,6 +8,8 @@ import (
"errors" "errors"
"io" "io"
"net/http" "net/http"
"slices"
"strings"
"time" "time"
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4sdk2/signer/v4" v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4sdk2/signer/v4"
@ -27,6 +29,8 @@ type (
reader *bufio.Reader reader *bufio.Reader
streamSigner *v4.StreamSigner streamSigner *v4.StreamSigner
trailerHeaders []string
trailers map[string]string
requestTime time.Time requestTime time.Time
buffer []byte buffer []byte
offset int offset int
@ -37,6 +41,7 @@ type (
var ( var (
errGiantChunk = errors.New("chunk too big: choose chunk size <= 16MiB") errGiantChunk = errors.New("chunk too big: choose chunk size <= 16MiB")
errMalformedChunkedEncoding = errors.New("malformed chunked encoding") errMalformedChunkedEncoding = errors.New("malformed chunked encoding")
errMalformedTrailerHeaders = errors.New("malformed trailer headers")
) )
func (c *s3ChunkReader) Close() (err error) { func (c *s3ChunkReader) Close() (err error) {
@ -54,6 +59,10 @@ func (c *s3ChunkReader) Read(buf []byte) (num int, err error) {
buf = buf[num:] buf = buf[num:]
} }
if c.err != nil {
return 0, c.err
}
var size int var size int
for { for {
b, err := c.reader.ReadByte() b, err := c.reader.ReadByte()
@ -107,29 +116,9 @@ func (c *s3ChunkReader) Read(buf []byte) (num int, err error) {
c.err = errMalformedChunkedEncoding c.err = errMalformedChunkedEncoding
return num, c.err return num, c.err
} }
b, err := c.reader.ReadByte()
if err == io.EOF { if err = c.readCRLF(); err != nil {
err = io.ErrUnexpectedEOF return num, err
}
if err != nil {
c.err = err
return num, c.err
}
if b != '\r' {
c.err = errMalformedChunkedEncoding
return num, c.err
}
b, err = c.reader.ReadByte()
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
if err != nil {
c.err = err
return num, c.err
}
if b != '\n' {
c.err = errMalformedChunkedEncoding
return num, c.err
} }
if cap(c.buffer) < size { if cap(c.buffer) < size {
@ -147,23 +136,6 @@ func (c *s3ChunkReader) Read(buf []byte) (num int, err error) {
c.err = err c.err = err
return num, c.err return num, c.err
} }
b, err = c.reader.ReadByte()
if b != '\r' || err != nil {
c.err = errMalformedChunkedEncoding
return num, c.err
}
b, err = c.reader.ReadByte()
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
if err != nil {
c.err = err
return num, c.err
}
if b != '\n' {
c.err = errMalformedChunkedEncoding
return num, c.err
}
// Once we have read the entire chunk successfully, we verify // Once we have read the entire chunk successfully, we verify
// that the received signature matches our computed signature. // that the received signature matches our computed signature.
@ -181,16 +153,99 @@ func (c *s3ChunkReader) Read(buf []byte) (num int, err error) {
// If the chunk size is zero we return io.EOF. As specified by AWS, // If the chunk size is zero we return io.EOF. As specified by AWS,
// only the last chunk is zero-sized. // only the last chunk is zero-sized.
if size == 0 { if size == 0 {
if len(c.trailerHeaders) != 0 {
if err = c.readTrailers(); err != nil {
c.err = err
return num, c.err
}
} else if err = c.readCRLF(); err != nil {
return num, err
}
c.err = io.EOF c.err = io.EOF
return num, c.err return num, c.err
} }
if err = c.readCRLF(); err != nil {
return num, err
}
c.offset = copy(buf, c.buffer) c.offset = copy(buf, c.buffer)
num += c.offset num += c.offset
return num, err return num, err
} }
func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, error) { func (c *s3ChunkReader) readCRLF() error {
for _, ch := range [2]byte{'\r', '\n'} {
b, err := c.reader.ReadByte()
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
if err != nil {
c.err = err
return c.err
}
if b != ch {
c.err = errMalformedChunkedEncoding
return c.err
}
}
return nil
}
func (c *s3ChunkReader) readTrailers() error {
var k, v []byte
var err error
for err == nil {
k, err = c.reader.ReadBytes(':')
if err != nil {
if err == io.EOF {
break
}
c.err = errMalformedTrailerHeaders
return c.err
}
v, err = c.reader.ReadBytes('\n')
if err != nil && err != io.EOF {
c.err = errMalformedTrailerHeaders
return c.err
}
if len(v) >= 2 && v[len(v)-2] == '\r' {
v[len(v)-2] = '\n'
v = v[:len(v)-1]
}
switch {
case slices.Contains(c.trailerHeaders, string(k[:len(k)-1])):
c.buffer = append(append(c.buffer, k...), v...) // todo use copy
case string(k) == "x-amz-trailer-signature:":
calculatedSignature, err := c.streamSigner.GetTrailerSignature(c.buffer, c.requestTime)
if err != nil {
c.err = err
return c.err
}
if string(v[:64]) != hex.EncodeToString(calculatedSignature) {
c.err = errs.GetAPIError(errs.ErrSignatureDoesNotMatch)
return c.err
}
default:
c.err = errMalformedTrailerHeaders
return c.err
}
c.trailers[string(k[:len(k)-1])] = string(v[:len(v)-1])
}
return nil
}
func (c *s3ChunkReader) TrailerHeaders() map[string]string {
return c.trailers
}
func newSignV4ChunkedReader(req *http.Request) (*s3ChunkReader, error) {
ctx := req.Context() ctx := req.Context()
box, err := middleware.GetBoxData(ctx) box, err := middleware.GetBoxData(ctx)
if err != nil { if err != nil {
@ -214,11 +269,19 @@ func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, error) {
} }
newStreamSigner := v4.NewStreamSigner(currentCredentials, "s3", authHeaders.Region, seed) newStreamSigner := v4.NewStreamSigner(currentCredentials, "s3", authHeaders.Region, seed)
var trailerHeaders []string
trailer := req.Header.Get("x-amz-trailer")
if trailer != "" {
trailerHeaders = strings.Split(trailer, ";")
}
return &s3ChunkReader{ return &s3ChunkReader{
ctx: ctx, ctx: ctx,
reader: bufio.NewReader(req.Body), reader: bufio.NewReader(req.Body),
streamSigner: newStreamSigner, streamSigner: newStreamSigner,
requestTime: reqTime, requestTime: reqTime,
buffer: make([]byte, 64*1024), buffer: make([]byte, 64*1024),
trailerHeaders: trailerHeaders,
trailers: make(map[string]string, len(trailerHeaders)),
}, nil }, nil
} }

View file

@ -2,8 +2,12 @@ package handler
import ( import (
"bytes" "bytes"
"context"
"fmt"
"io" "io"
"net/http" "net/http"
"strconv"
"strings"
"testing" "testing"
"time" "time"
@ -12,7 +16,53 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestSigV4AStreaming(t *testing.T) { func TestSigV4AChunkedReader(t *testing.T) {
t.Run("with trailers", func(t *testing.T) {
accessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6"
secretKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e"
chunk1 := "Testing with the {sdk-java}"
body := "1b;chunk-signature=3045022100956ca03d2166100b455b532de542892f73925fbcea2f6498674a39a61bb4860902202977c1d47aea548d434540f89640ce97e605d18353cbbd75a619874f02e3dd22**\r\n" +
chunk1 +
"\r\n0;chunk-signature=304502210097dcc1721675469910ef8712fc2af0678eb90c12216dd6228c6b621fb6f805a0022047d27d21ae2af8a8172f2ef83c81ce9d4746aa88fc9ee0ca783eaa5e71aaef6c**\r\n" +
"x-amz-checksum-crc32:Np6zMg==\r\n" +
"x-amz-trailer-signature:304502200ecacd9aa2c432af5a2327c22a2ff9b32f44ab8559de00309219aef105eaaac102210092cbc0e78c4bcd56490a73da8ceed1934be80f3affeffb14d8c743fc292dda4f**\r\n\r\n"
reqBody := bytes.NewBufferString(body)
req, err := http.NewRequest("PUT", "https://localhost:8184/test2/tmp", reqBody)
require.NoError(t, err)
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32")
signature := "3045022100ddbc6ab11785d7f23d299de7db97379116f543377a44e38170a4e43b38b0d62b02201d8dca13c67f04f45491345152db4b704768eb8bb89b5215fd59bb4a4d9d7b61"
signingTime, err := time.Parse("20060102T150405Z", "20250203T144621Z")
require.NoError(t, err)
key, err := keys.NewPrivateKey()
require.NoError(t, err)
accessBox, err := newTestAccessBox(key)
require.NoError(t, err)
accessBox.Gate.SecretKey = secretKey
ctx := middleware.SetBox(req.Context(), &middleware.Box{
AccessBox: accessBox,
AuthHeaders: &middleware.AuthHeader{
AccessKeyID: accessKeyID,
SignatureV4: signature,
},
ClientTime: signingTime,
})
req = req.WithContext(ctx)
r, err := newSignV4aChunkedReader(req)
require.NoError(t, err)
data, err := io.ReadAll(r)
require.NoError(t, err)
require.Equal(t, chunk1, string(data))
})
t.Run("without trailers", func(t *testing.T) {
accessKeyID := "2XEbqH4M3ym7a3E3esxfZ2gRLnMwDXrCN4y1SkQg5fHa09sThVmVL3EE6xeKsyMzaqu5jPi41YCaVbnwbwCTF3bx1" accessKeyID := "2XEbqH4M3ym7a3E3esxfZ2gRLnMwDXrCN4y1SkQg5fHa09sThVmVL3EE6xeKsyMzaqu5jPi41YCaVbnwbwCTF3bx1"
secretKey := "00637f53f842573aaa06c2164c598973cd986880987111416cf71f1619def537" secretKey := "00637f53f842573aaa06c2164c598973cd986880987111416cf71f1619def537"
@ -52,6 +102,136 @@ func TestSigV4AStreaming(t *testing.T) {
data, err := io.ReadAll(r) data, err := io.ReadAll(r)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, chunk1, string(data)) require.Equal(t, chunk1, string(data))
})
}
func TestSigV4ChunkedReader(t *testing.T) {
accessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6"
secretKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e"
signature := "b740b3b2a08c541c3fc4bd155a448e25408b509a29af98a86356b894930b93e8"
signingTime, err := time.Parse("20060102T150405Z", "20250203T134442Z")
require.NoError(t, err)
key, err := keys.NewPrivateKey()
require.NoError(t, err)
accessBox, err := newTestAccessBox(key)
require.NoError(t, err)
accessBox.Gate.SecretKey = secretKey
setBoxFn := func(ctx context.Context) context.Context {
return middleware.SetBox(ctx, &middleware.Box{
AccessBox: accessBox,
AuthHeaders: &middleware.AuthHeader{
AccessKeyID: accessKeyID,
SignatureV4: signature,
Region: "us-east-1",
},
ClientTime: signingTime,
})
}
chunk1 := "Testing with the {sdk-java}"
t.Run("with trailers", func(t *testing.T) {
body := "1b;chunk-signature=a6a9be5fff05db0b542aedb2203d892b4162250885d06b1422b173ee0ea92ba5\r\n" +
chunk1 +
"\r\n0;chunk-signature=31afd083a57c416c46afaf101649d7f0c6c0627cfa60c0f93d1f7ea84396ee42\r\n" +
"x-amz-checksum-crc32:Np6zMg==\r\n" +
"x-amz-trailer-signature:40ec0046ac730fa27a1451d00d849056c49553ee753f5d158306d05671a42125\r\n\r\n"
reqBody := bytes.NewBufferString(body)
req, err := http.NewRequest("PUT", "https://localhost:8184/test2/tmp", reqBody)
require.NoError(t, err)
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32")
req = req.WithContext(setBoxFn(req.Context()))
r, err := newSignV4ChunkedReader(req)
require.NoError(t, err)
data, err := io.ReadAll(r)
require.NoError(t, err)
require.Equal(t, chunk1, string(data))
})
t.Run("without trailers", func(t *testing.T) {
body := "1b;chunk-signature=a6a9be5fff05db0b542aedb2203d892b4162250885d06b1422b173ee0ea92ba5\r\n" +
chunk1 +
"\r\n0;chunk-signature=31afd083a57c416c46afaf101649d7f0c6c0627cfa60c0f93d1f7ea84396ee42\r\n\r\n"
reqBody := bytes.NewBufferString(body)
req, err := http.NewRequest("PUT", "https://localhost:8184/test2/tmp", reqBody)
require.NoError(t, err)
req = req.WithContext(setBoxFn(req.Context()))
r, err := newSignV4ChunkedReader(req)
require.NoError(t, err)
data, err := io.ReadAll(r)
require.NoError(t, err)
require.Equal(t, chunk1, string(data))
})
}
func TestUnsignedChunkReader(t *testing.T) {
chunk1 := "chunk1"
chunk2 := "chunk2"
t.Run("with trailer", func(t *testing.T) {
chunks := []string{chunk1, chunk2}
trailer := map[string]string{"x-amz-checksum-crc64nvme": "q1EYl4rI0TU="}
body, expected := getChunkedBody(t, chunks, trailer)
r, err := newUnsignedChunkedReader(body)
require.NoError(t, err)
data, err := io.ReadAll(r)
require.NoError(t, err)
require.Equal(t, expected, string(data))
require.EqualValues(t, trailer, r.TrailerHeaders())
})
t.Run("without trailer", func(t *testing.T) {
chunks := []string{chunk1, chunk2}
body, expected := getChunkedBody(t, chunks, nil)
r, err := newUnsignedChunkedReader(body)
require.NoError(t, err)
data, err := io.ReadAll(r)
require.NoError(t, err)
require.Equal(t, expected, string(data))
})
}
func getChunkedBody(t *testing.T, chunks []string, trailers map[string]string) (*bytes.Buffer, string) {
res := bytes.NewBufferString("")
for i, chunk := range chunks {
meta := strconv.FormatInt(int64(len(chunk)), 16) + "\r\n"
if i != 0 {
meta = "\r\n" + meta
}
_, err := res.WriteString(meta)
require.NoError(t, err)
_, err = res.WriteString(chunk)
require.NoError(t, err)
}
_, err := res.WriteString("\r\n0\r\n")
require.NoError(t, err)
for k, v := range trailers {
_, err := res.WriteString(fmt.Sprintf("%s:%s\n", k, v))
require.NoError(t, err)
}
_, err = res.WriteString("\r\n")
require.NoError(t, err)
return res, strings.Join(chunks, "")
} }

View file

@ -0,0 +1,165 @@
package handler
import (
"bufio"
"io"
)
type (
s3UnsignedChunkReader struct {
reader *bufio.Reader
trailers map[string]string
buffer []byte
offset int
err error
}
)
func (c *s3UnsignedChunkReader) Close() (err error) {
return nil
}
func (c *s3UnsignedChunkReader) Read(buf []byte) (num int, err error) {
if c.offset > 0 {
num = copy(buf, c.buffer[c.offset:])
if num == len(buf) {
c.offset += num
return num, nil
}
c.offset = 0
buf = buf[num:]
}
if c.err != nil {
return 0, c.err
}
var size int
var b byte
for {
b, err = c.reader.ReadByte()
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
if err != nil {
c.err = err
return num, c.err
}
if b == '\r' {
break
}
// Manually deserialize the size since AWS specified
// the chunk size to be of variable width. In particular,
// a size of 16 is encoded as `10` while a size of 64 KB
// is `10000`.
switch {
case b >= '0' && b <= '9':
size = size<<4 | int(b-'0')
case b >= 'a' && b <= 'f':
size = size<<4 | int(b-('a'-10))
case b >= 'A' && b <= 'F':
size = size<<4 | int(b-('A'-10))
default:
c.err = errMalformedChunkedEncoding
return num, c.err
}
if size > maxChunkSize {
c.err = errGiantChunk
return num, c.err
}
}
if b != '\r' {
c.err = errMalformedChunkedEncoding
return num, c.err
}
b, err = c.reader.ReadByte()
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
if err != nil {
c.err = err
return num, c.err
}
if b != '\n' {
c.err = errMalformedChunkedEncoding
return num, c.err
}
if cap(c.buffer) < size {
c.buffer = make([]byte, size)
} else {
c.buffer = c.buffer[:size]
}
// Now, we read the payload and compute its SHA-256 hash.
_, err = io.ReadFull(c.reader, c.buffer)
if err == io.EOF && size != 0 {
err = io.ErrUnexpectedEOF
}
if err != nil && err != io.EOF {
c.err = err
return num, c.err
}
// If the chunk size is zero we return io.EOF. As specified by AWS,
// only the last chunk is zero-sized.
if size == 0 {
var k, v string
for err == nil {
k, err = c.reader.ReadString(':')
if err != nil {
if err == io.EOF {
break
}
c.err = errMalformedTrailerHeaders
return num, c.err
}
v, err = c.reader.ReadString('\n')
if err != nil {
c.err = errMalformedTrailerHeaders
return num, c.err
}
c.trailers[k[:len(k)-1]] = v[:len(v)-1]
}
c.err = io.EOF
return num, c.err
}
b, err = c.reader.ReadByte()
if b != '\r' || err != nil {
c.err = errMalformedChunkedEncoding
return num, c.err
}
b, err = c.reader.ReadByte()
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
if err != nil {
c.err = err
return num, c.err
}
if b != '\n' {
c.err = errMalformedChunkedEncoding
return num, c.err
}
c.offset = copy(buf, c.buffer)
num += c.offset
return num, err
}
func (c *s3UnsignedChunkReader) TrailerHeaders() map[string]string {
return c.trailers
}
func newUnsignedChunkedReader(body io.Reader) (*s3UnsignedChunkReader, error) {
return &s3UnsignedChunkReader{
reader: bufio.NewReader(body),
trailers: map[string]string{},
buffer: make([]byte, 64*1024),
}, nil
}

View file

@ -7,6 +7,8 @@ import (
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"slices"
"strings"
"time" "time"
v4a "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4asdk2" v4a "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4asdk2"
@ -20,6 +22,8 @@ type (
reader *bufio.Reader reader *bufio.Reader
streamSigner *v4a.StreamSigner streamSigner *v4a.StreamSigner
trailerHeaders []string
trailers map[string]string
requestTime time.Time requestTime time.Time
buffer []byte buffer []byte
offset int offset int
@ -42,6 +46,10 @@ func (c *s3v4aChunkReader) Read(buf []byte) (num int, err error) {
buf = buf[num:] buf = buf[num:]
} }
if c.err != nil {
return 0, c.err
}
var size int var size int
for { for {
b, err := c.reader.ReadByte() b, err := c.reader.ReadByte()
@ -87,21 +95,9 @@ func (c *s3v4aChunkReader) Read(buf []byte) (num int, err error) {
c.err = errMalformedChunkedEncoding c.err = errMalformedChunkedEncoding
return num, c.err return num, c.err
} }
b, err := c.reader.ReadByte()
if err != nil { if err = c.readCRLF(); err != nil {
return c.handleErr(num, err) return num, err
}
if b != '\r' {
c.err = errMalformedChunkedEncoding
return num, c.err
}
b, err = c.reader.ReadByte()
if err != nil {
return c.handleErr(num, err)
}
if b != '\n' {
c.err = errMalformedChunkedEncoding
return num, c.err
} }
if cap(c.buffer) < size { if cap(c.buffer) < size {
@ -119,19 +115,6 @@ func (c *s3v4aChunkReader) Read(buf []byte) (num int, err error) {
c.err = err c.err = err
return num, c.err return num, c.err
} }
b, err = c.reader.ReadByte()
if b != '\r' || err != nil {
c.err = errMalformedChunkedEncoding
return num, c.err
}
b, err = c.reader.ReadByte()
if err != nil {
return c.handleErr(num, err)
}
if b != '\n' {
c.err = errMalformedChunkedEncoding
return num, c.err
}
// Once we have read the entire chunk successfully, we verify // Once we have read the entire chunk successfully, we verify
// that the received signature is valid. // that the received signature is valid.
@ -150,10 +133,23 @@ func (c *s3v4aChunkReader) Read(buf []byte) (num int, err error) {
// If the chunk size is zero we return io.EOF. As specified by AWS, // If the chunk size is zero we return io.EOF. As specified by AWS,
// only the last chunk is zero-sized. // only the last chunk is zero-sized.
if size == 0 { if size == 0 {
if len(c.trailerHeaders) != 0 {
if err = c.readTrailers(); err != nil {
c.err = err
return num, c.err
}
} else if err = c.readCRLF(); err != nil {
return num, err
}
c.err = io.EOF c.err = io.EOF
return num, c.err return num, c.err
} }
if err = c.readCRLF(); err != nil {
return num, err
}
c.offset = copy(buf, c.buffer) c.offset = copy(buf, c.buffer)
num += c.offset num += c.offset
return num, err return num, err
@ -168,7 +164,78 @@ func (c *s3v4aChunkReader) handleErr(num int, err error) (int, error) {
return num, c.err return num, c.err
} }
func newSignV4aChunkedReader(req *http.Request) (io.ReadCloser, error) { func (c *s3v4aChunkReader) readCRLF() error {
for _, ch := range [2]byte{'\r', '\n'} {
b, err := c.reader.ReadByte()
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
if err != nil {
c.err = err
return c.err
}
if b != ch {
c.err = errMalformedChunkedEncoding
return c.err
}
}
return nil
}
func (c *s3v4aChunkReader) readTrailers() error {
var k, v []byte
var err error
for err == nil {
k, err = c.reader.ReadBytes(':')
if err != nil {
if err == io.EOF {
break
}
c.err = errMalformedTrailerHeaders
return c.err
}
v, err = c.reader.ReadBytes('\n')
if err != nil && err != io.EOF {
c.err = errMalformedTrailerHeaders
return c.err
}
if len(v) >= 2 && v[len(v)-2] == '\r' {
v[len(v)-2] = '\n'
v = v[:len(v)-1]
}
switch {
case slices.Contains(c.trailerHeaders, string(k[:len(k)-1])):
c.buffer = append(append(c.buffer, k...), v...) // todo use copy
case string(k) == "x-amz-trailer-signature:":
n, err := hex.Decode(v[:], bytes.TrimRight(v[:], "*\n"))
if err != nil {
c.err = errMalformedChunkedEncoding
return c.err
}
if err = c.streamSigner.VerifyTrailerSignature(c.buffer, c.requestTime, v[:n]); err != nil {
c.err = fmt.Errorf("%w: %s", errs.GetAPIError(errs.ErrSignatureDoesNotMatch), err.Error())
return c.err
}
default:
c.err = errMalformedTrailerHeaders
return c.err
}
c.trailers[string(k[:len(k)-1])] = string(v[:len(v)-1])
}
return nil
}
func (c *s3v4aChunkReader) TrailerHeaders() map[string]string {
return c.trailers
}
func newSignV4aChunkedReader(req *http.Request) (*s3v4aChunkReader, error) {
box, err := middleware.GetBoxData(req.Context()) box, err := middleware.GetBoxData(req.Context())
if err != nil { if err != nil {
return nil, errs.GetAPIError(errs.ErrAuthorizationHeaderMalformed) return nil, errs.GetAPIError(errs.ErrAuthorizationHeaderMalformed)
@ -200,10 +267,18 @@ func newSignV4aChunkedReader(req *http.Request) (io.ReadCloser, error) {
newStreamSigner := v4a.NewStreamSigner(creds, "s3", seed) newStreamSigner := v4a.NewStreamSigner(creds, "s3", seed)
var trailerHeaders []string
trailer := req.Header.Get("x-amz-trailer")
if trailer != "" {
trailerHeaders = strings.Split(trailer, ";")
}
return &s3v4aChunkReader{ return &s3v4aChunkReader{
reader: bufio.NewReader(req.Body), reader: bufio.NewReader(req.Body),
streamSigner: newStreamSigner, streamSigner: newStreamSigner,
requestTime: reqTime, requestTime: reqTime,
buffer: make([]byte, 64*1024), buffer: make([]byte, 64*1024),
trailerHeaders: trailerHeaders,
trailers: make(map[string]string, len(trailerHeaders)),
}, nil }, nil
} }

View file

@ -39,9 +39,10 @@ func (h *handler) logAndSendError(ctx context.Context, w http.ResponseWriter, lo
zap.String("object", reqInfo.ObjectName), zap.String("object", reqInfo.ObjectName),
zap.String("description", logText), zap.String("description", logText),
zap.String("user", reqInfo.User), zap.String("user", reqInfo.User),
zap.Error(err)} zap.Error(err),
}
fields = append(fields, additional...) fields = append(fields, additional...)
h.reqLogger(ctx).Error(logs.RequestFailed, fields...) h.reqLogger(ctx).Error(logs.RequestFailed, append(fields, logs.TagField(logs.TagDatapath))...)
} }
func handleDeleteMarker(w http.ResponseWriter, err error) error { func handleDeleteMarker(w http.ResponseWriter, err error) error {

View file

@ -63,6 +63,7 @@ const (
AmzPartNumberMarker = "X-Amz-Part-Number-Marker" AmzPartNumberMarker = "X-Amz-Part-Number-Marker"
AmzStorageClass = "X-Amz-Storage-Class" AmzStorageClass = "X-Amz-Storage-Class"
AmzForceBucketDelete = "X-Amz-Force-Delete-Bucket" AmzForceBucketDelete = "X-Amz-Force-Delete-Bucket"
AmzTrailer = "X-Amz-Trailer"
AmzServerSideEncryptionCustomerAlgorithm = "x-amz-server-side-encryption-customer-algorithm" AmzServerSideEncryptionCustomerAlgorithm = "x-amz-server-side-encryption-customer-algorithm"
AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key" AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"
@ -95,7 +96,10 @@ const (
DefaultLocationConstraint = "default" DefaultLocationConstraint = "default"
StreamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" StreamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
StreamingContentSHA256Trailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
StreamingContentV4aSHA256 = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD" StreamingContentV4aSHA256 = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"
StreamingContentV4aSHA256Trailer = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"
StreamingUnsignedPayloadTrailer = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
DefaultStorageClass = "STANDARD" DefaultStorageClass = "STANDARD"
) )
@ -129,6 +133,8 @@ var SystemMetadata = map[string]struct{}{
func IsSignedStreamingV4(r *http.Request) (string, bool) { func IsSignedStreamingV4(r *http.Request) (string, bool) {
shaHeader := r.Header.Get(AmzContentSha256) shaHeader := r.Header.Get(AmzContentSha256)
return shaHeader, return shaHeader,
(shaHeader == StreamingContentSHA256 || shaHeader == StreamingContentV4aSHA256) && (shaHeader == StreamingContentSHA256 || shaHeader == StreamingContentSHA256Trailer ||
shaHeader == StreamingContentV4aSHA256 || shaHeader == StreamingContentV4aSHA256Trailer ||
shaHeader == StreamingUnsignedPayloadTrailer) &&
r.Method == http.MethodPut r.Method == http.MethodPut
} }

View file

@ -76,7 +76,8 @@ func (c *Cache) PutBucket(bktInfo *data.BucketInfo) {
zap.String("zone", bktInfo.Zone), zap.String("zone", bktInfo.Zone),
zap.String("bucket name", bktInfo.Name), zap.String("bucket name", bktInfo.Name),
zap.Stringer("bucket cid", bktInfo.CID), zap.Stringer("bucket cid", bktInfo.CID),
zap.Error(err)) zap.Error(err),
logs.TagField(logs.TagDatapath))
} }
} }
@ -118,11 +119,12 @@ func (c *Cache) PutObject(owner user.ID, extObjInfo *data.ExtendedObjectInfo) {
if err := c.objCache.PutObject(extObjInfo); err != nil { if err := c.objCache.PutObject(extObjInfo); err != nil {
c.logger.Warn(logs.CouldntAddObjectToCache, zap.Error(err), c.logger.Warn(logs.CouldntAddObjectToCache, zap.Error(err),
zap.String("object_name", extObjInfo.ObjectInfo.Name), zap.String("bucket_name", extObjInfo.ObjectInfo.Bucket), zap.String("object_name", extObjInfo.ObjectInfo.Name), zap.String("bucket_name", extObjInfo.ObjectInfo.Bucket),
zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString())) zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString()),
logs.TagField(logs.TagDatapath))
} }
if err := c.accessCache.Put(owner, extObjInfo.ObjectInfo.Address().EncodeToString()); err != nil { if err := c.accessCache.Put(owner, extObjInfo.ObjectInfo.Address().EncodeToString()); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err)) c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
} }
@ -132,7 +134,8 @@ func (c *Cache) PutObjectWithName(owner user.ID, extObjInfo *data.ExtendedObject
if err := c.namesCache.Put(extObjInfo.ObjectInfo.NiceName(), extObjInfo.ObjectInfo.Address()); err != nil { if err := c.namesCache.Put(extObjInfo.ObjectInfo.NiceName(), extObjInfo.ObjectInfo.Address()); err != nil {
c.logger.Warn(logs.CouldntPutObjAddressToNameCache, c.logger.Warn(logs.CouldntPutObjAddressToNameCache,
zap.String("obj nice name", extObjInfo.ObjectInfo.NiceName()), zap.String("obj nice name", extObjInfo.ObjectInfo.NiceName()),
zap.Error(err)) zap.Error(err),
logs.TagField(logs.TagDatapath))
} }
} }
@ -146,11 +149,11 @@ func (c *Cache) GetList(owner user.ID, key cache.ObjectsListKey) []*data.NodeVer
func (c *Cache) PutList(owner user.ID, key cache.ObjectsListKey, list []*data.NodeVersion) { func (c *Cache) PutList(owner user.ID, key cache.ObjectsListKey, list []*data.NodeVersion) {
if err := c.listsCache.PutVersions(key, list); err != nil { if err := c.listsCache.PutVersions(key, list); err != nil {
c.logger.Warn(logs.CouldntCacheListOfObjects, zap.Error(err)) c.logger.Warn(logs.CouldntCacheListOfObjects, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
if err := c.accessCache.Put(owner, key.String()); err != nil { if err := c.accessCache.Put(owner, key.String()); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err)) c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
} }
@ -164,11 +167,11 @@ func (c *Cache) GetListSession(owner user.ID, key cache.ListSessionKey) *data.Li
func (c *Cache) PutListSession(owner user.ID, key cache.ListSessionKey, session *data.ListSession) { func (c *Cache) PutListSession(owner user.ID, key cache.ListSessionKey, session *data.ListSession) {
if err := c.sessionListCache.PutListSession(key, session); err != nil { if err := c.sessionListCache.PutListSession(key, session); err != nil {
c.logger.Warn(logs.CouldntCacheListSession, zap.Error(err)) c.logger.Warn(logs.CouldntCacheListSession, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
if err := c.accessCache.Put(owner, key.String()); err != nil { if err := c.accessCache.Put(owner, key.String()); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err)) c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
} }
@ -187,11 +190,11 @@ func (c *Cache) GetTagging(owner user.ID, key string) map[string]string {
func (c *Cache) PutTagging(owner user.ID, key string, tags map[string]string) { func (c *Cache) PutTagging(owner user.ID, key string, tags map[string]string) {
if err := c.systemCache.PutTagging(key, tags); err != nil { if err := c.systemCache.PutTagging(key, tags); err != nil {
c.logger.Error(logs.CouldntCacheTags, zap.Error(err)) c.logger.Error(logs.CouldntCacheTags, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
if err := c.accessCache.Put(owner, key); err != nil { if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err)) c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
} }
@ -209,11 +212,11 @@ func (c *Cache) GetLockInfo(owner user.ID, key string) *data.LockInfo {
func (c *Cache) PutLockInfo(owner user.ID, key string, lockInfo *data.LockInfo) { func (c *Cache) PutLockInfo(owner user.ID, key string, lockInfo *data.LockInfo) {
if err := c.systemCache.PutLockInfo(key, lockInfo); err != nil { if err := c.systemCache.PutLockInfo(key, lockInfo); err != nil {
c.logger.Error(logs.CouldntCacheLockInfo, zap.Error(err)) c.logger.Error(logs.CouldntCacheLockInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
if err := c.accessCache.Put(owner, key); err != nil { if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err)) c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
} }
@ -230,11 +233,11 @@ func (c *Cache) GetSettings(owner user.ID, bktInfo *data.BucketInfo) *data.Bucke
func (c *Cache) PutSettings(owner user.ID, bktInfo *data.BucketInfo, settings *data.BucketSettings) { func (c *Cache) PutSettings(owner user.ID, bktInfo *data.BucketInfo, settings *data.BucketSettings) {
key := bktInfo.Name + bktInfo.SettingsObjectName() key := bktInfo.Name + bktInfo.SettingsObjectName()
if err := c.systemCache.PutSettings(key, settings); err != nil { if err := c.systemCache.PutSettings(key, settings); err != nil {
c.logger.Warn(logs.CouldntCacheBucketSettings, zap.String("bucket", bktInfo.Name), zap.Error(err)) c.logger.Warn(logs.CouldntCacheBucketSettings, zap.String("bucket", bktInfo.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
} }
if err := c.accessCache.Put(owner, key); err != nil { if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err)) c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
} }
@ -252,11 +255,11 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf
key := bkt.CORSObjectName() key := bkt.CORSObjectName()
if err := c.systemCache.PutCORS(key, cors); err != nil { if err := c.systemCache.PutCORS(key, cors); err != nil {
c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err)) c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
} }
if err := c.accessCache.Put(owner, key); err != nil { if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err)) c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
} }
@ -278,11 +281,11 @@ func (c *Cache) PutLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo, c
key := bkt.LifecycleConfigurationObjectName() key := bkt.LifecycleConfigurationObjectName()
if err := c.systemCache.PutLifecycleConfiguration(key, cfg); err != nil { if err := c.systemCache.PutLifecycleConfiguration(key, cfg); err != nil {
c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err)) c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
} }
if err := c.accessCache.Put(owner, key); err != nil { if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err)) c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
} }
@ -296,7 +299,7 @@ func (c *Cache) GetNetworkInfo() *netmap.NetworkInfo {
func (c *Cache) PutNetworkInfo(info netmap.NetworkInfo) { func (c *Cache) PutNetworkInfo(info netmap.NetworkInfo) {
if err := c.networkCache.PutNetworkInfo(info); err != nil { if err := c.networkCache.PutNetworkInfo(info); err != nil {
c.logger.Warn(logs.CouldntCacheNetworkInfo, zap.Error(err)) c.logger.Warn(logs.CouldntCacheNetworkInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
} }
@ -306,7 +309,7 @@ func (c *Cache) GetNetmap() *netmap.NetMap {
func (c *Cache) PutNetmap(nm netmap.NetMap) { func (c *Cache) PutNetmap(nm netmap.NetMap) {
if err := c.networkCache.PutNetmap(nm); err != nil { if err := c.networkCache.PutNetmap(nm); err != nil {
c.logger.Warn(logs.CouldntCacheNetmap, zap.Error(err)) c.logger.Warn(logs.CouldntCacheNetmap, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
} }

View file

@ -3,7 +3,9 @@ package layer
import ( import (
"context" "context"
"fmt" "fmt"
"sort"
"strconv" "strconv"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
@ -62,6 +64,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm frostfs.PrmContainer) (*d
log.Error(logs.CouldNotParseContainerObjectLockEnabledAttribute, log.Error(logs.CouldNotParseContainerObjectLockEnabledAttribute,
zap.String("lock_enabled", attrLockEnabled), zap.String("lock_enabled", attrLockEnabled),
zap.Error(err), zap.Error(err),
logs.TagField(logs.TagDatapath),
) )
} }
} }
@ -76,7 +79,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm frostfs.PrmContainer) (*d
return info, nil return info, nil
} }
func (n *Layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) { func (n *Layer) containerList(ctx context.Context, listParams ListBucketsParams) ([]*data.BucketInfo, error) {
stoken := n.SessionTokenForRead(ctx) stoken := n.SessionTokenForRead(ctx)
prm := frostfs.PrmUserContainers{ prm := frostfs.PrmUserContainers{
@ -86,7 +89,7 @@ func (n *Layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
res, err := n.frostFS.UserContainers(ctx, prm) res, err := n.frostFS.UserContainers(ctx, prm)
if err != nil { if err != nil {
n.reqLogger(ctx).Error(logs.CouldNotListUserContainers, zap.Error(err)) n.reqLogger(ctx).Error(logs.CouldNotListUserContainers, zap.Error(err), logs.TagField(logs.TagExternalStorage))
return nil, err return nil, err
} }
@ -98,14 +101,38 @@ func (n *Layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
} }
info, err := n.containerInfo(ctx, getPrm) info, err := n.containerInfo(ctx, getPrm)
if err != nil { if err != nil {
n.reqLogger(ctx).Error(logs.CouldNotFetchContainerInfo, zap.Error(err)) n.reqLogger(ctx).Error(logs.CouldNotFetchContainerInfo, zap.Error(err), logs.TagField(logs.TagExternalStorage))
continue
}
if shouldSkipBucket(info, listParams) {
continue continue
} }
list = append(list, info) list = append(list, info)
} }
return list, nil sort.Slice(list, func(i, j int) bool {
return list[i].Name < list[j].Name
})
for i, info := range list {
if listParams.ContinuationToken != "" && info.Name != listParams.ContinuationToken {
continue
}
return list[i:], nil
}
return nil, nil
}
func shouldSkipBucket(info *data.BucketInfo, prm ListBucketsParams) bool {
if !strings.HasPrefix(info.Name, prm.Prefix) ||
(prm.BucketRegion != "" && info.LocationConstraint != prm.BucketRegion) {
return true
}
return false
} }
func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) { func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) {

View file

@ -92,7 +92,8 @@ func (n *Layer) deleteCORSObject(ctx context.Context, bktInfo *data.BucketInfo,
if err := n.objectDeleteWithAuth(ctx, corsBkt, addr.Object(), prmAuth); err != nil { if err := n.objectDeleteWithAuth(ctx, corsBkt, addr.Object(), prmAuth); err != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err), n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err),
zap.String("cnrID", corsBkt.CID.EncodeToString()), zap.String("cnrID", corsBkt.CID.EncodeToString()),
zap.String("objID", addr.Object().EncodeToString())) zap.String("objID", addr.Object().EncodeToString()),
logs.TagField(logs.TagExternalStorage))
} }
} }

View file

@ -76,6 +76,7 @@ var _ frostfs.FrostFS = (*TestFrostFS)(nil)
type TestFrostFS struct { type TestFrostFS struct {
objects map[string]*object.Object objects map[string]*object.Object
copiesNumbers map[string][]uint32
objectErrors map[string]error objectErrors map[string]error
objectPutErrors map[string]error objectPutErrors map[string]error
containers map[string]*container.Container containers map[string]*container.Container
@ -88,6 +89,7 @@ type TestFrostFS struct {
func NewTestFrostFS(key *keys.PrivateKey) *TestFrostFS { func NewTestFrostFS(key *keys.PrivateKey) *TestFrostFS {
return &TestFrostFS{ return &TestFrostFS{
objects: make(map[string]*object.Object), objects: make(map[string]*object.Object),
copiesNumbers: make(map[string][]uint32),
objectErrors: make(map[string]error), objectErrors: make(map[string]error),
objectPutErrors: make(map[string]error), objectPutErrors: make(map[string]error),
containers: make(map[string]*container.Container), containers: make(map[string]*container.Container),
@ -126,6 +128,10 @@ func (t *TestFrostFS) Objects() []*object.Object {
return res return res
} }
func (t *TestFrostFS) CopiesNumbers(addr string) []uint32 {
return t.copiesNumbers[addr]
}
func (t *TestFrostFS) ObjectExists(objID oid.ID) bool { func (t *TestFrostFS) ObjectExists(objID oid.ID) bool {
for _, obj := range t.objects { for _, obj := range t.objects {
if id, _ := obj.ID(); id.Equals(objID) { if id, _ := obj.ID(); id.Equals(objID) {
@ -346,6 +352,8 @@ func (t *TestFrostFS) CreateObject(ctx context.Context, prm frostfs.PrmObjectCre
addr := newAddress(cnrID, objID) addr := newAddress(cnrID, objID)
t.objects[addr.EncodeToString()] = obj t.objects[addr.EncodeToString()] = obj
t.copiesNumbers[addr.EncodeToString()] = prm.CopiesNumber
return &frostfs.CreateObjectResult{ return &frostfs.CreateObjectResult{
ObjectID: objID, ObjectID: objID,
CreationEpoch: t.currentEpoch - 1, CreationEpoch: t.currentEpoch - 1,

View file

@ -195,6 +195,18 @@ type (
Encode string Encode string
} }
ListBucketsParams struct {
MaxBuckets int
Prefix string
ContinuationToken string
BucketRegion string
}
ListBucketsResult struct {
Containers []*data.BucketInfo
ContinuationToken string
}
// VersionedObject stores info about objects to delete. // VersionedObject stores info about objects to delete.
VersionedObject struct { VersionedObject struct {
Name string Name string
@ -371,8 +383,24 @@ func (n *Layer) ResolveCID(ctx context.Context, name string) (cid.ID, error) {
// ListBuckets returns all user containers. The name of the bucket is a container // ListBuckets returns all user containers. The name of the bucket is a container
// id. Timestamp is omitted since it is not saved in frostfs container. // id. Timestamp is omitted since it is not saved in frostfs container.
func (n *Layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) { func (n *Layer) ListBuckets(ctx context.Context, params ListBucketsParams) (ListBucketsResult, error) {
return n.containerList(ctx) var result ListBucketsResult
var err error
if params.MaxBuckets == 0 {
return result, nil
}
result.Containers, err = n.containerList(ctx, params)
if err != nil {
return ListBucketsResult{}, err
}
if len(result.Containers) > params.MaxBuckets {
result.ContinuationToken = result.Containers[params.MaxBuckets].Name
result.Containers = result.Containers[:params.MaxBuckets]
}
return result, nil
} }
// GetObject from storage. // GetObject from storage.
@ -515,7 +543,9 @@ func (n *Layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams)
n.reqLogger(ctx).Debug(logs.GetObject, n.reqLogger(ctx).Debug(logs.GetObject,
zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("cid", p.BktInfo.CID),
zap.Stringer("oid", objInfo.ObjectInfo.ID)) zap.Stringer("oid", objInfo.ObjectInfo.ID),
logs.TagField(logs.TagDatapath),
)
return objInfo, nil return objInfo, nil
} }
@ -568,8 +598,8 @@ func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) { if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
return obj return obj
} }
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting, n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting, zap.Stringer("cid", bkt.CID),
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error)) zap.String("oid", obj.VersionID), zap.Error(obj.Error), logs.TagField(logs.TagExternalStorage))
} }
if obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID); obj.Error != nil { if obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID); obj.Error != nil {
@ -607,8 +637,8 @@ func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) { if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
return obj return obj
} }
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting, n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting, zap.Stringer("cid", bkt.CID),
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error)) zap.String("oid", obj.VersionID), zap.Error(obj.Error), logs.TagField(logs.TagExternalStorage))
} }
} }
@ -723,7 +753,7 @@ func (n *Layer) getNodeVersionsToDelete(ctx context.Context, bkt *data.BucketInf
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", apierr.GetAPIError(apierr.ErrNoSuchVersion)) return nil, fmt.Errorf("%w: there isn't tree node with requested version id", apierr.GetAPIError(apierr.ErrNoSuchVersion))
} }
n.reqLogger(ctx).Debug(logs.GetTreeNodeToDelete, zap.Stringer("cid", bkt.CID), zap.Strings("oids", oids)) n.reqLogger(ctx).Debug(logs.GetTreeNodeToDelete, zap.Stringer("cid", bkt.CID), zap.Strings("oids", oids), logs.TagField(logs.TagDatapath))
return versionsToDelete, nil return versionsToDelete, nil
} }
@ -793,7 +823,7 @@ func (n *Layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*Ver
for i, obj := range p.Objects { for i, obj := range p.Objects {
p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.Settings, obj, p.NetworkInfo) p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.Settings, obj, p.NetworkInfo)
if p.IsMultiple && p.Objects[i].Error != nil { if p.IsMultiple && p.Objects[i].Error != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error)) n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error), logs.TagField(logs.TagExternalStorage))
} }
} }
@ -823,7 +853,7 @@ func (n *Layer) ResolveBucket(ctx context.Context, zone, name string) (cid.ID, e
return cid.ID{}, err return cid.ID{}, err
} }
n.reqLogger(ctx).Info(logs.ResolveBucket, zap.Stringer("cid", cnrID)) n.reqLogger(ctx).Info(logs.ResolveBucket, zap.Stringer("cid", cnrID), logs.TagField(logs.TagDatapath))
} }
return cnrID, nil return cnrID, nil
@ -848,12 +878,12 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
corsObj, err := n.treeService.GetBucketCORS(ctx, p.BktInfo) corsObj, err := n.treeService.GetBucketCORS(ctx, p.BktInfo)
if err != nil { if err != nil {
n.reqLogger(ctx).Error(logs.GetBucketCors, zap.Error(err)) n.reqLogger(ctx).Error(logs.GetBucketCorsFromTree, zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
} }
lifecycleObj, treeErr := n.treeService.GetBucketLifecycleConfiguration(ctx, p.BktInfo) lifecycleObj, treeErr := n.treeService.GetBucketLifecycleConfiguration(ctx, p.BktInfo)
if treeErr != nil { if treeErr != nil {
n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr)) n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr), logs.TagField(logs.TagExternalStorageTree))
} }
err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken) err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)

View file

@ -79,7 +79,9 @@ func (n *Layer) deleteLifecycleObject(ctx context.Context, bktInfo *data.BucketI
if err := n.objectDeleteWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth); err != nil { if err := n.objectDeleteWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth); err != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteLifecycleObject, zap.Error(err), n.reqLogger(ctx).Error(logs.CouldntDeleteLifecycleObject, zap.Error(err),
zap.String("cid", lifecycleBkt.CID.EncodeToString()), zap.String("cid", lifecycleBkt.CID.EncodeToString()),
zap.String("oid", addr.Object().EncodeToString())) zap.String("oid", addr.Object().EncodeToString()),
logs.TagField(logs.TagExternalStorage),
)
} }
} }

View file

@ -541,7 +541,7 @@ func (n *Layer) initWorkerPool(ctx context.Context, size int, p commonVersionsLi
realSize, err := GetObjectSize(oi) realSize, err := GetObjectSize(oi)
if err != nil { if err != nil {
reqLog.Debug(logs.FailedToGetRealObjectSize, zap.Error(err)) reqLog.Debug(logs.FailedToGetRealObjectSize, zap.Error(err), logs.TagField(logs.TagDatapath))
realSize = oi.Size realSize = oi.Size
} }
@ -554,7 +554,7 @@ func (n *Layer) initWorkerPool(ctx context.Context, size int, p commonVersionsLi
}) })
if err != nil { if err != nil {
wg.Done() wg.Done()
reqLog.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err)) reqLog.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
}(node) }(node)
} }
@ -645,7 +645,7 @@ func (n *Layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo
meta, err := n.objectHead(ctx, bktInfo, node.OID) meta, err := n.objectHead(ctx, bktInfo, node.OID)
if err != nil { if err != nil {
n.reqLogger(ctx).Warn(logs.CouldNotFetchObjectMeta, zap.Error(err)) n.reqLogger(ctx).Warn(logs.CouldNotFetchObjectMeta, zap.Error(err), logs.TagField(logs.TagExternalStorage))
return nil return nil
} }

View file

@ -61,7 +61,6 @@ type (
Info *UploadInfoParams Info *UploadInfoParams
Header map[string]string Header map[string]string
Data *UploadData Data *UploadData
CopiesNumbers []uint32
} }
UploadData struct { UploadData struct {
@ -75,6 +74,7 @@ type (
Reader io.Reader Reader io.Reader
ContentMD5 string ContentMD5 string
ContentSHA256Hash string ContentSHA256Hash string
CopiesNumbers []uint32
} }
UploadCopyParams struct { UploadCopyParams struct {
@ -85,11 +85,13 @@ type (
SrcEncryption encryption.Params SrcEncryption encryption.Params
PartNumber int PartNumber int
Range *RangeParams Range *RangeParams
CopiesNumbers []uint32
} }
CompleteMultipartParams struct { CompleteMultipartParams struct {
Info *UploadInfoParams Info *UploadInfoParams
Parts []*CompletedPart Parts []*CompletedPart
CopiesNumbers []uint32
} }
CompletedPart struct { CompletedPart struct {
@ -165,7 +167,6 @@ func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
Owner: n.gateOwner, Owner: n.gateOwner,
Created: TimeNow(ctx), Created: TimeNow(ctx),
Meta: make(map[string]string, metaSize), Meta: make(map[string]string, metaSize),
CopiesNumbers: p.CopiesNumbers,
CreationEpoch: networkInfo.CurrentEpoch(), CreationEpoch: networkInfo.CurrentEpoch(),
} }
@ -212,7 +213,7 @@ func (n *Layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, er
func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) { func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) {
encInfo := FormEncryptionInfo(multipartInfo.Meta) encInfo := FormEncryptionInfo(multipartInfo.Meta)
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil { if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err)) n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters) return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters)
} }
@ -222,7 +223,7 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
Attributes: make([][2]string, 2), Attributes: make([][2]string, 2),
Payload: p.Reader, Payload: p.Reader,
CreationTime: TimeNow(ctx), CreationTime: TimeNow(ctx),
CopiesNumber: multipartInfo.CopiesNumbers, CopiesNumber: p.CopiesNumbers,
} }
decSize := p.Size decSize := p.Size
@ -265,7 +266,10 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner) n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
err = n.frostFS.DeleteObject(ctx, prm) err = n.frostFS.DeleteObject(ctx, prm)
if err != nil { if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID)) n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
zap.Stringer("cid", bktInfo.CID),
zap.Stringer("oid", createdObj.ID),
logs.TagField(logs.TagExternalStorage))
} }
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest) return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
} }
@ -282,7 +286,10 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
if !bytes.Equal(contentHashBytes, createdObj.HashSum) { if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
err = n.objectDelete(ctx, bktInfo, createdObj.ID) err = n.objectDelete(ctx, bktInfo, createdObj.ID)
if err != nil { if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID)) n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
zap.Stringer("cid", bktInfo.CID),
zap.Stringer("oid", createdObj.ID),
logs.TagField(logs.TagExternalStorage))
} }
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch) return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
} }
@ -290,7 +297,7 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
n.reqLogger(ctx).Debug(logs.UploadPart, n.reqLogger(ctx).Debug(logs.UploadPart,
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber), zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID)) zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID), logs.TagField(logs.TagDatapath))
partInfo := &data.PartInfo{ partInfo := &data.PartInfo{
Key: p.Info.Key, Key: p.Info.Key,
@ -313,7 +320,8 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil { if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteOldPartObject, zap.Error(err), n.reqLogger(ctx).Error(logs.CouldntDeleteOldPartObject, zap.Error(err),
zap.String("cid", bktInfo.CID.EncodeToString()), zap.String("cid", bktInfo.CID.EncodeToString()),
zap.String("oid", oldPartID.EncodeToString())) zap.String("oid", oldPartID.EncodeToString()),
logs.TagField(logs.TagExternalStorage))
} }
} }
} }
@ -376,6 +384,7 @@ func (n *Layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
PartNumber: p.PartNumber, PartNumber: p.PartNumber,
Size: size, Size: size,
Reader: objPayload, Reader: objPayload,
CopiesNumbers: p.CopiesNumbers,
} }
return n.uploadPart(ctx, multipartInfo, params) return n.uploadPart(ctx, multipartInfo, params)
@ -472,14 +481,15 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
Header: initMetadata, Header: initMetadata,
Size: &multipartObjetSize, Size: &multipartObjetSize,
Encryption: p.Info.Encryption, Encryption: p.Info.Encryption,
CopiesNumbers: multipartInfo.CopiesNumbers, CopiesNumbers: p.CopiesNumbers,
CompleteMD5Hash: hex.EncodeToString(md5Hash.Sum(nil)) + "-" + strconv.Itoa(len(p.Parts)), CompleteMD5Hash: hex.EncodeToString(md5Hash.Sum(nil)) + "-" + strconv.Itoa(len(p.Parts)),
}) })
if err != nil { if err != nil {
n.reqLogger(ctx).Error(logs.CouldNotPutCompletedObject, n.reqLogger(ctx).Error(logs.CouldNotPutCompletedObject,
zap.String("uploadID", p.Info.UploadID), zap.String("uploadID", p.Info.UploadID),
zap.String("uploadKey", p.Info.Key), zap.String("uploadKey", p.Info.Key),
zap.Error(err)) zap.Error(err),
logs.TagField(logs.TagExternalStorage))
return nil, nil, apierr.GetAPIError(apierr.ErrInternalError) return nil, nil, apierr.GetAPIError(apierr.ErrInternalError)
} }
@ -491,7 +501,8 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil { if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil {
n.reqLogger(ctx).Warn(logs.CouldNotDeleteUploadPart, n.reqLogger(ctx).Warn(logs.CouldNotDeleteUploadPart,
zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID), zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID),
zap.Error(err)) zap.Error(err),
logs.TagField(logs.TagExternalStorage))
} }
addr.SetObject(partInfo.OID) addr.SetObject(partInfo.OID)
n.cache.DeleteObject(addr) n.cache.DeleteObject(addr)
@ -585,7 +596,7 @@ func (n *Layer) deleteUploadedParts(ctx context.Context, bkt *data.BucketInfo, p
oids, err := relations.ListAllRelations(ctx, n.frostFS.Relations(), bkt.CID, info.OID, tokens) oids, err := relations.ListAllRelations(ctx, n.frostFS.Relations(), bkt.CID, info.OID, tokens)
if err != nil { if err != nil {
n.reqLogger(ctx).Warn(logs.FailedToListAllObjectRelations, zap.String("cid", bkt.CID.EncodeToString()), n.reqLogger(ctx).Warn(logs.FailedToListAllObjectRelations, zap.String("cid", bkt.CID.EncodeToString()),
zap.String("oid", info.OID.EncodeToString()), zap.Error(err)) zap.String("oid", info.OID.EncodeToString()), zap.Error(err), logs.TagField(logs.TagExternalStorage))
continue continue
} }
members = append(members, append(oids, info.OID)...) members = append(members, append(oids, info.OID)...)
@ -594,7 +605,7 @@ func (n *Layer) deleteUploadedParts(ctx context.Context, bkt *data.BucketInfo, p
err := n.putTombstones(ctx, bkt, networkInfo, members) err := n.putTombstones(ctx, bkt, networkInfo, members)
if err != nil { if err != nil {
n.reqLogger(ctx).Warn(logs.FailedToPutTombstones, zap.Error(err)) n.reqLogger(ctx).Warn(logs.FailedToPutTombstones, zap.Error(err), logs.TagField(logs.TagExternalStorage))
} }
} }
@ -607,7 +618,7 @@ func (n *Layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
encInfo := FormEncryptionInfo(multipartInfo.Meta) encInfo := FormEncryptionInfo(multipartInfo.Meta)
if err = p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil { if err = p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err)) n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters) return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters)
} }
@ -699,7 +710,8 @@ func (n *Layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
zap.Stringer("cid", p.Bkt.CID), zap.Stringer("cid", p.Bkt.CID),
zap.String("upload id", p.UploadID), zap.String("upload id", p.UploadID),
zap.Ints("part numbers", partsNumbers), zap.Ints("part numbers", partsNumbers),
zap.Strings("oids", oids)) zap.Strings("oids", oids),
logs.TagField(logs.TagDatapath))
return multipartInfo, res, nil return multipartInfo, res, nil
} }

View file

@ -295,7 +295,11 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
if !bytes.Equal(headerMd5Hash, createdObj.MD5Sum) { if !bytes.Equal(headerMd5Hash, createdObj.MD5Sum) {
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID) err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
if err != nil { if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID)) n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
zap.Stringer("cid", p.BktInfo.CID),
zap.Stringer("oid", createdObj.ID),
logs.TagField(logs.TagExternalStorage),
)
} }
return nil, apierr.GetAPIError(apierr.ErrBadDigest) return nil, apierr.GetAPIError(apierr.ErrBadDigest)
} }
@ -309,13 +313,17 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
if !bytes.Equal(contentHashBytes, createdObj.HashSum) { if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID) err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
if err != nil { if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID)) n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
zap.Stringer("cid", p.BktInfo.CID),
zap.Stringer("oid", createdObj.ID),
logs.TagField(logs.TagExternalStorage))
} }
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch) return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
} }
} }
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID)) n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID),
zap.Stringer("oid", createdObj.ID), logs.TagField(logs.TagExternalStorage))
now := TimeNow(ctx) now := TimeNow(ctx)
newVersion := &data.NodeVersion{ newVersion := &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{ BaseNodeVersion: data.BaseNodeVersion{
@ -410,7 +418,7 @@ func (n *Layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.Bucke
meta, err := n.objectHead(ctx, bkt, node.OID) meta, err := n.objectHead(ctx, bkt, node.OID)
if err != nil { if err != nil {
if client.IsErrObjectNotFound(err) { if client.IsErrObjectNotFound(err) || client.IsErrObjectAlreadyRemoved(err) {
return nil, fmt.Errorf("%w: %s; %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error(), node.OID.EncodeToString()) return nil, fmt.Errorf("%w: %s; %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error(), node.OID.EncodeToString())
} }
return nil, err return nil, err
@ -467,7 +475,7 @@ func (n *Layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
meta, err := n.objectHead(ctx, bkt, foundVersion.OID) meta, err := n.objectHead(ctx, bkt, foundVersion.OID)
if err != nil { if err != nil {
if client.IsErrObjectNotFound(err) { if client.IsErrObjectNotFound(err) || client.IsErrObjectAlreadyRemoved(err) {
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchVersion), err.Error()) return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchVersion), err.Error())
} }
return nil, err return nil, err
@ -525,10 +533,7 @@ func (n *Layer) objectPutAndHash(ctx context.Context, prm frostfs.PrmObjectCreat
}) })
res, err := n.frostFS.CreateObject(ctx, prm) res, err := n.frostFS.CreateObject(ctx, prm)
if err != nil { if err != nil {
if _, errDiscard := io.Copy(io.Discard, prm.Payload); errDiscard != nil { n.payloadDiscard(ctx, prm.Payload)
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard))
}
return nil, err return nil, err
} }
return &data.CreatedObjectInfo{ return &data.CreatedObjectInfo{
@ -540,12 +545,20 @@ func (n *Layer) objectPutAndHash(ctx context.Context, prm frostfs.PrmObjectCreat
}, nil }, nil
} }
func (n *Layer) payloadDiscard(ctx context.Context, payload io.Reader) {
if payload != nil {
if _, errDiscard := io.Copy(io.Discard, payload); errDiscard != nil {
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard), logs.TagField(logs.TagDatapath))
}
}
}
type logWrapper struct { type logWrapper struct {
log *zap.Logger log *zap.Logger
} }
func (l *logWrapper) Printf(format string, args ...interface{}) { func (l *logWrapper) Printf(format string, args ...interface{}) {
l.log.Info(fmt.Sprintf(format, args...)) l.log.Info(fmt.Sprintf(format, args...), logs.TagField(logs.TagDatapath))
} }
func IsSystemHeader(key string) bool { func IsSystemHeader(key string) bool {

View file

@ -49,3 +49,16 @@ func TestGoroutinesDontLeakInPutAndHash(t *testing.T) {
require.ErrorIs(t, err, expErr) require.ErrorIs(t, err, expErr)
require.Empty(t, payload.Len(), "body must be read out otherwise goroutines can leak in wrapReader") require.Empty(t, payload.Len(), "body must be read out otherwise goroutines can leak in wrapReader")
} }
func TestNilPayloadPutAndHash(t *testing.T) {
tc := prepareContext(t)
prm := frostfs.PrmObjectCreate{
Filepath: tc.obj,
Payload: nil,
}
expErr := errors.New("some error")
tc.testFrostFS.SetObjectPutError(tc.obj, expErr)
_, err := tc.layer.objectPutAndHash(tc.ctx, prm, tc.bktInfo)
require.ErrorIs(t, err, expErr)
}

View file

@ -217,7 +217,7 @@ func (n *Layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo)
return nil, err return nil, err
} }
settings = &data.BucketSettings{Versioning: data.VersioningUnversioned} settings = &data.BucketSettings{Versioning: data.VersioningUnversioned}
n.reqLogger(ctx).Debug(logs.BucketSettingsNotFoundUseDefaults) n.reqLogger(ctx).Debug(logs.BucketSettingsNotFoundUseDefaults, logs.TagField(logs.TagDatapath))
} }
n.cache.PutSettings(owner, bktInfo, settings) n.cache.PutSettings(owner, bktInfo, settings)

View file

@ -168,7 +168,8 @@ func (n *Layer) getNodeVersion(ctx context.Context, objVersion *data.ObjectVersi
if err == nil && version != nil && !version.IsDeleteMarker { if err == nil && version != nil && !version.IsDeleteMarker {
n.reqLogger(ctx).Debug(logs.GetTreeNode, n.reqLogger(ctx).Debug(logs.GetTreeNode,
zap.Stringer("cid", objVersion.BktInfo.CID), zap.Stringer("oid", version.OID)) zap.Stringer("cid", objVersion.BktInfo.CID),
zap.Stringer("oid", version.OID), logs.TagField(logs.TagExternalStorageTree))
} }
return version, err return version, err

View file

@ -65,13 +65,13 @@ func (n *Layer) submitPutTombstone(ctx context.Context, bkt *data.BucketInfo, me
defer wg.Done() defer wg.Done()
if err := n.putTombstoneObject(ctx, tomb, bkt); err != nil { if err := n.putTombstoneObject(ctx, tomb, bkt); err != nil {
n.reqLogger(ctx).Warn(logs.FailedToPutTombstoneObject, zap.String("cid", bkt.CID.EncodeToString()), zap.Error(err)) n.reqLogger(ctx).Warn(logs.FailedToPutTombstoneObject, zap.String("cid", bkt.CID.EncodeToString()), zap.Error(err), logs.TagField(logs.TagExternalStorage))
errCh <- fmt.Errorf("put tombstone object: %w", err) errCh <- fmt.Errorf("put tombstone object: %w", err)
} }
}) })
if err != nil { if err != nil {
wg.Done() wg.Done()
n.reqLogger(ctx).Warn(logs.FailedToSubmitTaskToPool, zap.Error(err)) n.reqLogger(ctx).Warn(logs.FailedToSubmitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath))
errCh <- fmt.Errorf("submit task to pool: %w", err) errCh <- fmt.Errorf("submit task to pool: %w", err)
} }
} }
@ -106,7 +106,7 @@ func (n *Layer) getMembers(ctx context.Context, cnrID cid.ID, objID oid.ID, toke
} }
n.reqLogger(ctx).Warn(logs.FailedToListAllObjectRelations, zap.String("cid", cnrID.EncodeToString()), n.reqLogger(ctx).Warn(logs.FailedToListAllObjectRelations, zap.String("cid", cnrID.EncodeToString()),
zap.String("oid", objID.EncodeToString()), zap.Error(err)) zap.String("oid", objID.EncodeToString()), zap.Error(err), logs.TagField(logs.TagExternalStorage))
return nil, nil return nil, nil
} }
return append(oids, objID), nil return append(oids, objID), nil

View file

@ -110,7 +110,7 @@ func preparePathStyleAddress(reqInfo *ReqInfo, r *http.Request, reqLogger *zap.L
// https://github.com/go-chi/chi/issues/641 // https://github.com/go-chi/chi/issues/641
// https://github.com/go-chi/chi/issues/642 // https://github.com/go-chi/chi/issues/642
if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil { if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil {
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err)) reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err), logs.TagField(logs.TagDatapath))
} else { } else {
reqInfo.ObjectName = obj reqInfo.ObjectName = obj
} }

View file

@ -53,15 +53,15 @@ func Auth(center Center, log *zap.Logger) Func {
box, err := center.Authenticate(r) box, err := center.Authenticate(r)
if err != nil { if err != nil {
if errors.Is(err, ErrNoAuthorizationHeader) { if errors.Is(err, ErrNoAuthorizationHeader) {
reqLogOrDefault(ctx, log).Debug(logs.CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed, zap.Error(err)) reqLogOrDefault(ctx, log).Debug(logs.CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed, zap.Error(err), logs.TagField(logs.TagDatapath))
} else { } else {
reqLogOrDefault(ctx, log).Error(logs.FailedToPassAuthentication, zap.Error(err)) reqLogOrDefault(ctx, log).Error(logs.FailedToPassAuthentication, zap.Error(err), logs.TagField(logs.TagDatapath))
err = apierr.TransformToS3Error(err) err = apierr.TransformToS3Error(err)
if err.(apierr.Error).ErrCode == apierr.ErrInternalError { if err.(apierr.Error).ErrCode == apierr.ErrInternalError {
err = apierr.GetAPIError(apierr.ErrAccessDenied) err = apierr.GetAPIError(apierr.ErrAccessDenied)
} }
if _, wrErr := WriteErrorResponse(w, GetReqInfo(r.Context()), err); wrErr != nil { if _, wrErr := WriteErrorResponse(w, GetReqInfo(r.Context()), err); wrErr != nil {
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr)) reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath))
} }
return return
} }
@ -71,7 +71,7 @@ func Auth(center Center, log *zap.Logger) Func {
if box.AccessBox.Gate.BearerToken != nil { if box.AccessBox.Gate.BearerToken != nil {
reqInfo.User = bearer.ResolveIssuer(*box.AccessBox.Gate.BearerToken).String() reqInfo.User = bearer.ResolveIssuer(*box.AccessBox.Gate.BearerToken).String()
} }
reqLogOrDefault(ctx, log).Debug(logs.SuccessfulAuth, zap.String("accessKeyID", box.AuthHeaders.AccessKeyID)) reqLogOrDefault(ctx, log).Debug(logs.SuccessfulAuth, zap.String("accessKeyID", box.AuthHeaders.AccessKeyID), logs.TagField(logs.TagDatapath))
} }
h.ServeHTTP(w, r.WithContext(ctx)) h.ServeHTTP(w, r.WithContext(ctx))
@ -89,15 +89,15 @@ func FrostfsIDValidation(frostfsID FrostFSIDValidator, log *zap.Logger) Func {
ctx := r.Context() ctx := r.Context()
bd, err := GetBoxData(ctx) bd, err := GetBoxData(ctx)
if err != nil || bd.Gate.BearerToken == nil { if err != nil || bd.Gate.BearerToken == nil {
reqLogOrDefault(ctx, log).Debug(logs.AnonRequestSkipFrostfsIDValidation) reqLogOrDefault(ctx, log).Debug(logs.AnonRequestSkipFrostfsIDValidation, logs.TagField(logs.TagDatapath))
h.ServeHTTP(w, r) h.ServeHTTP(w, r)
return return
} }
if err = validateBearerToken(frostfsID, bd.Gate.BearerToken); err != nil { if err = validateBearerToken(frostfsID, bd.Gate.BearerToken); err != nil {
reqLogOrDefault(ctx, log).Error(logs.FrostfsIDValidationFailed, zap.Error(err)) reqLogOrDefault(ctx, log).Error(logs.FrostfsIDValidationFailed, zap.Error(err), logs.TagField(logs.TagDatapath))
if _, wrErr := WriteErrorResponse(w, GetReqInfo(r.Context()), err); wrErr != nil { if _, wrErr := WriteErrorResponse(w, GetReqInfo(r.Context()), err); wrErr != nil {
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr)) reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath))
} }
return return
} }

View file

@ -23,7 +23,7 @@ type (
) )
func LogHTTP(l *zap.Logger, _ LogHTTPSettings) Func { func LogHTTP(l *zap.Logger, _ LogHTTPSettings) Func {
l.Warn(logs.LogHTTPDisabledInThisBuild) l.Warn(logs.LogHTTPDisabledInThisBuild, logs.TagField(logs.TagApp))
return func(h http.Handler) http.Handler { return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
h.ServeHTTP(w, r) h.ServeHTTP(w, r)

View file

@ -139,7 +139,7 @@ func resolveCID(log *zap.Logger, resolveContainerID ContainerIDResolveFunc) cidR
containerID, err := resolveContainerID(ctx, reqInfo.BucketName) containerID, err := resolveContainerID(ctx, reqInfo.BucketName)
if err != nil { if err != nil {
reqLogOrDefault(ctx, log).Debug(logs.FailedToResolveCID, zap.Error(err)) reqLogOrDefault(ctx, log).Debug(logs.FailedToResolveCID, zap.Error(err), logs.TagField(logs.TagDatapath))
return "" return ""
} }

View file

@ -90,10 +90,10 @@ func PolicyCheck(cfg PolicyConfig) Func {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context() ctx := r.Context()
if err := policyCheck(r, cfg); err != nil { if err := policyCheck(r, cfg); err != nil {
reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err)) reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err), logs.TagField(logs.TagDatapath))
err = apierr.TransformToS3Error(err) err = apierr.TransformToS3Error(err)
if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil { if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil {
reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr)) reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath))
} }
return return
} }
@ -200,7 +200,9 @@ func getPolicyRequest(r *http.Request, cfg PolicyConfig, reqType ReqType, bktNam
reqLogOrDefault(r.Context(), cfg.Log).Debug(logs.PolicyRequest, zap.String("action", op), reqLogOrDefault(r.Context(), cfg.Log).Debug(logs.PolicyRequest, zap.String("action", op),
zap.String("resource", res), zap.Any("request properties", requestProps), zap.String("resource", res), zap.Any("request properties", requestProps),
zap.Any("resource properties", resourceProps)) zap.Any("resource properties", resourceProps),
logs.TagField(logs.TagDatapath),
)
return testutil.NewRequest(op, testutil.NewResource(res, resourceProps), requestProps), pk, groups, nil return testutil.NewRequest(op, testutil.NewResource(res, resourceProps), requestProps), pk, groups, nil
} }

View file

@ -166,7 +166,7 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
// generate random UUIDv4 // generate random UUIDv4
id, err := uuid.NewRandom() id, err := uuid.NewRandom()
if err != nil { if err != nil {
log.Error(logs.FailedToGenerateRequestID, zap.Error(err)) log.Error(logs.FailedToGenerateRequestID, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
// set request id into response header // set request id into response header
@ -198,7 +198,8 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
r = r.WithContext(SetReqLogger(ctx, reqLogger)) r = r.WithContext(SetReqLogger(ctx, reqLogger))
reqLogger.Info(logs.RequestStart, zap.String("host", r.Host), reqLogger.Info(logs.RequestStart, zap.String("host", r.Host),
zap.String("remote_host", reqInfo.RemoteHost), zap.String("namespace", reqInfo.Namespace)) zap.String("remote_host", reqInfo.RemoteHost), zap.String("namespace", reqInfo.Namespace),
logs.TagField(logs.TagDatapath))
// continue execution // continue execution
h.ServeHTTP(lw, r) h.ServeHTTP(lw, r)

View file

@ -331,7 +331,7 @@ func LogSuccessResponse(l *zap.Logger) Func {
fields = append(fields, zap.String("user", reqInfo.User)) fields = append(fields, zap.String("user", reqInfo.User))
} }
reqLogger.Info(logs.RequestEnd, fields...) reqLogger.Info(logs.RequestEnd, append(fields, logs.TagField(logs.TagDatapath))...)
}) })
} }
} }

View file

@ -245,13 +245,14 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
zap.String("method", reqInfo.API), zap.String("method", reqInfo.API),
zap.String("http method", r.Method), zap.String("http method", r.Method),
zap.String("url", r.RequestURI), zap.String("url", r.RequestURI),
logs.TagField(logs.TagDatapath),
} }
if wrErr != nil { if wrErr != nil {
fields = append(fields, zap.NamedError("write_response_error", wrErr)) fields = append(fields, zap.NamedError("write_response_error", wrErr))
} }
log.Error(logs.RequestUnmatched, fields...) log.Error(logs.RequestUnmatched, append(fields, logs.TagField(logs.TagDatapath))...)
} }
} }
@ -266,13 +267,14 @@ func notSupportedHandler() http.HandlerFunc {
fields := []zap.Field{ fields := []zap.Field{
zap.String("http method", r.Method), zap.String("http method", r.Method),
zap.String("url", r.RequestURI), zap.String("url", r.RequestURI),
logs.TagField(logs.TagDatapath),
} }
if wrErr != nil { if wrErr != nil {
fields = append(fields, zap.NamedError("write_response_error", wrErr)) fields = append(fields, zap.NamedError("write_response_error", wrErr))
} }
log.Error(logs.NotSupported, fields...) log.Error(logs.NotSupported, append(fields, logs.TagField(logs.TagDatapath))...)
} }
} }
} }

View file

@ -53,6 +53,7 @@ import (
"github.com/panjf2000/ants/v2" "github.com/panjf2000/ants/v2"
"github.com/spf13/viper" "github.com/spf13/viper"
"go.uber.org/zap" "go.uber.org/zap"
"go.uber.org/zap/zapcore"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
"golang.org/x/text/encoding/ianaindex" "golang.org/x/text/encoding/ianaindex"
"google.golang.org/grpc" "google.golang.org/grpc"
@ -65,7 +66,7 @@ type (
App struct { App struct {
ctr s3middleware.Center ctr s3middleware.Center
log *zap.Logger log *zap.Logger
cfg *viper.Viper cfg *appCfg
pool *pool.Pool pool *pool.Pool
treePool *treepool.Pool treePool *treepool.Pool
key *keys.PrivateKey key *keys.PrivateKey
@ -91,6 +92,10 @@ type (
wrkDone chan struct{} wrkDone chan struct{}
} }
tagsConfig struct {
tagLogs sync.Map
}
loggerSettings struct { loggerSettings struct {
mu sync.RWMutex mu sync.RWMutex
appMetrics *metrics.AppMetrics appMetrics *metrics.AppMetrics
@ -99,6 +104,7 @@ type (
appSettings struct { appSettings struct {
logLevel zap.AtomicLevel logLevel zap.AtomicLevel
httpLogging s3middleware.LogHTTPConfig httpLogging s3middleware.LogHTTPConfig
tagsConfig *tagsConfig
maxClient maxClientsConfig maxClient maxClientsConfig
defaultMaxAge int defaultMaxAge int
reconnectInterval time.Duration reconnectInterval time.Duration
@ -138,13 +144,54 @@ type (
deadline time.Duration deadline time.Duration
count int count int
} }
Logger struct {
logger *zap.Logger
lvl zap.AtomicLevel
}
) )
func (t *tagsConfig) LevelEnabled(tag string, tgtLevel zapcore.Level) bool {
lvl, ok := t.tagLogs.Load(tag)
if !ok {
return false
}
return lvl.(zapcore.Level).Enabled(tgtLevel)
}
func (t *tagsConfig) update(cfg *viper.Viper) error {
tags, err := fetchLogTagsConfig(cfg)
if err != nil {
return err
}
t.tagLogs.Range(func(key, value any) bool {
k := key.(string)
v := value.(zapcore.Level)
if lvl, ok := tags[k]; ok {
if lvl != v {
t.tagLogs.Store(key, lvl)
}
} else {
t.tagLogs.Delete(key)
delete(tags, k)
}
return true
})
for k, v := range tags {
t.tagLogs.Store(k, v)
}
return nil
}
func newTagsConfig(v *viper.Viper) *tagsConfig {
var t tagsConfig
if err := t.update(v); err != nil {
// panic here is analogue of the similar panic during common log level initialization.
panic(err.Error())
}
return &t
}
func (s *loggerSettings) DroppedLogsInc() { func (s *loggerSettings) DroppedLogsInc() {
s.mu.RLock() s.mu.RLock()
defer s.mu.RUnlock() defer s.mu.RUnlock()
@ -161,15 +208,17 @@ func (s *loggerSettings) setMetrics(appMetrics *metrics.AppMetrics) {
s.appMetrics = appMetrics s.appMetrics = appMetrics
} }
func newApp(ctx context.Context, v *viper.Viper) *App { func newApp(ctx context.Context, cfg *appCfg) *App {
logSettings := &loggerSettings{} logSettings := &loggerSettings{}
log := pickLogger(v, logSettings) tagConfig := newTagsConfig(cfg.config())
settings := newAppSettings(log, v) log := pickLogger(cfg.config(), logSettings, tagConfig)
appCache := layer.NewCache(getCacheOptions(v, log.logger)) settings := newAppSettings(log, cfg.config())
settings.tagsConfig = tagConfig
appCache := layer.NewCache(getCacheOptions(cfg.config(), log.logger))
app := &App{ app := &App{
log: log.logger, log: log.logger,
cfg: v, cfg: cfg,
cache: appCache, cache: appCache,
webDone: make(chan struct{}, 1), webDone: make(chan struct{}, 1),
@ -184,6 +233,10 @@ func newApp(ctx context.Context, v *viper.Viper) *App {
return app return app
} }
func (a *App) config() *viper.Viper {
return a.cfg.config()
}
func (a *App) init(ctx context.Context) { func (a *App) init(ctx context.Context) {
a.initPools(ctx) a.initPools(ctx)
a.initResolver() a.initResolver()
@ -198,10 +251,10 @@ func (a *App) init(ctx context.Context) {
} }
func (a *App) initAuthCenter(ctx context.Context) { func (a *App) initAuthCenter(ctx context.Context) {
if a.cfg.IsSet(cfgContainersAccessBox) { if a.config().IsSet(cfgContainersAccessBox) {
cnrID, err := a.resolveContainerID(ctx, cfgContainersAccessBox) cnrID, err := a.resolveContainerID(ctx, cfgContainersAccessBox)
if err != nil { if err != nil {
a.log.Fatal(logs.CouldNotFetchAccessBoxContainerInfo, zap.Error(err)) a.log.Fatal(logs.CouldNotFetchAccessBoxContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
} }
a.settings.accessbox = &cnrID a.settings.accessbox = &cnrID
} }
@ -209,36 +262,36 @@ func (a *App) initAuthCenter(ctx context.Context) {
cfg := tokens.Config{ cfg := tokens.Config{
FrostFS: frostfs.NewAuthmateFrostFS(frostfs.NewFrostFS(a.pool, a.key), a.log), FrostFS: frostfs.NewAuthmateFrostFS(frostfs.NewFrostFS(a.pool, a.key), a.log),
Key: a.key, Key: a.key,
CacheConfig: getAccessBoxCacheConfig(a.cfg, a.log), CacheConfig: getAccessBoxCacheConfig(a.config(), a.log),
RemovingCheckAfterDurations: fetchRemovingCheckInterval(a.cfg, a.log), RemovingCheckAfterDurations: fetchRemovingCheckInterval(a.config(), a.log),
} }
a.ctr = auth.New(tokens.New(cfg), a.cfg.GetStringSlice(cfgAllowedAccessKeyIDPrefixes), a.settings) a.ctr = auth.New(tokens.New(cfg), a.config().GetStringSlice(cfgAllowedAccessKeyIDPrefixes), a.settings)
} }
func (a *App) initLayer(ctx context.Context) { func (a *App) initLayer(ctx context.Context) {
// prepare random key for anonymous requests // prepare random key for anonymous requests
randomKey, err := keys.NewPrivateKey() randomKey, err := keys.NewPrivateKey()
if err != nil { if err != nil {
a.log.Fatal(logs.CouldntGenerateRandomKey, zap.Error(err)) a.log.Fatal(logs.CouldntGenerateRandomKey, zap.Error(err), logs.TagField(logs.TagApp))
} }
var gateOwner user.ID var gateOwner user.ID
user.IDFromKey(&gateOwner, a.key.PrivateKey.PublicKey) user.IDFromKey(&gateOwner, a.key.PrivateKey.PublicKey)
var corsCnrInfo *data.BucketInfo var corsCnrInfo *data.BucketInfo
if a.cfg.IsSet(cfgContainersCORS) { if a.config().IsSet(cfgContainersCORS) {
corsCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersCORS) corsCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersCORS)
if err != nil { if err != nil {
a.log.Fatal(logs.CouldNotFetchCORSContainerInfo, zap.Error(err)) a.log.Fatal(logs.CouldNotFetchCORSContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
} }
} }
var lifecycleCnrInfo *data.BucketInfo var lifecycleCnrInfo *data.BucketInfo
if a.cfg.IsSet(cfgContainersLifecycle) { if a.config().IsSet(cfgContainersLifecycle) {
lifecycleCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersLifecycle) lifecycleCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersLifecycle)
if err != nil { if err != nil {
a.log.Fatal(logs.CouldNotFetchLifecycleContainerInfo, zap.Error(err)) a.log.Fatal(logs.CouldNotFetchLifecycleContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
} }
} }
@ -264,7 +317,7 @@ func (a *App) initLayer(ctx context.Context) {
func (a *App) initWorkerPool() *ants.Pool { func (a *App) initWorkerPool() *ants.Pool {
workerPool, err := ants.NewPool(a.settings.workerPoolSize) workerPool, err := ants.NewPool(a.settings.workerPoolSize)
if err != nil { if err != nil {
a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err)) a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err), logs.TagField(logs.TagApp))
} }
return workerPool return workerPool
} }
@ -273,6 +326,7 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
settings := &appSettings{ settings := &appSettings{
logLevel: log.lvl, logLevel: log.lvl,
httpLogging: s3middleware.LogHTTPConfig{}, httpLogging: s3middleware.LogHTTPConfig{},
tagsConfig: newTagsConfig(v),
maxClient: newMaxClients(v), maxClient: newMaxClients(v),
defaultMaxAge: fetchDefaultMaxAge(v, log.logger), defaultMaxAge: fetchDefaultMaxAge(v, log.logger),
reconnectInterval: fetchReconnectInterval(v), reconnectInterval: fetchReconnectInterval(v),
@ -599,7 +653,7 @@ func (a *App) initMetrics() {
Logger: a.log, Logger: a.log,
PoolStatistics: frostfs.NewPoolStatistic(a.pool), PoolStatistics: frostfs.NewPoolStatistic(a.pool),
TreeStatistic: a.treePool, TreeStatistic: a.treePool,
Enabled: a.cfg.GetBool(cfgPrometheusEnabled), Enabled: a.config().GetBool(cfgPrometheusEnabled),
} }
a.metrics = metrics.NewAppMetrics(cfg) a.metrics = metrics.NewAppMetrics(cfg)
@ -609,9 +663,9 @@ func (a *App) initMetrics() {
func (a *App) initFrostfsID(ctx context.Context) { func (a *App) initFrostfsID(ctx context.Context) {
cli, err := ffidcontract.New(ctx, ffidcontract.Config{ cli, err := ffidcontract.New(ctx, ffidcontract.Config{
RPCAddress: a.cfg.GetString(cfgRPCEndpoint), RPCAddress: a.config().GetString(cfgRPCEndpoint),
Contract: a.cfg.GetString(cfgFrostfsIDContract), Contract: a.config().GetString(cfgFrostfsIDContract),
ProxyContract: a.cfg.GetString(cfgProxyContract), ProxyContract: a.config().GetString(cfgProxyContract),
Key: a.key, Key: a.key,
Waiter: commonclient.WaiterOptions{ Waiter: commonclient.WaiterOptions{
IgnoreAlreadyExistsError: false, IgnoreAlreadyExistsError: false,
@ -619,24 +673,24 @@ func (a *App) initFrostfsID(ctx context.Context) {
}, },
}) })
if err != nil { if err != nil {
a.log.Fatal(logs.InitFrostfsIDContractFailed, zap.Error(err)) a.log.Fatal(logs.InitFrostfsIDFailed, zap.Error(err), logs.TagField(logs.TagApp))
} }
a.frostfsid, err = frostfsid.NewFrostFSID(frostfsid.Config{ a.frostfsid, err = frostfsid.NewFrostFSID(frostfsid.Config{
Cache: cache.NewFrostfsIDCache(getFrostfsIDCacheConfig(a.cfg, a.log)), Cache: cache.NewFrostfsIDCache(getFrostfsIDCacheConfig(a.config(), a.log)),
FrostFSID: cli, FrostFSID: cli,
Logger: a.log, Logger: a.log,
}) })
if err != nil { if err != nil {
a.log.Fatal(logs.InitFrostfsIDContractFailed, zap.Error(err)) a.log.Fatal(logs.InitFrostfsIDFailed, zap.Error(err), logs.TagField(logs.TagApp))
} }
} }
func (a *App) initPolicyStorage(ctx context.Context) { func (a *App) initPolicyStorage(ctx context.Context) {
policyContract, err := contract.New(ctx, contract.Config{ policyContract, err := contract.New(ctx, contract.Config{
RPCAddress: a.cfg.GetString(cfgRPCEndpoint), RPCAddress: a.config().GetString(cfgRPCEndpoint),
Contract: a.cfg.GetString(cfgPolicyContract), Contract: a.config().GetString(cfgPolicyContract),
ProxyContract: a.cfg.GetString(cfgProxyContract), ProxyContract: a.config().GetString(cfgProxyContract),
Key: a.key, Key: a.key,
Waiter: commonclient.WaiterOptions{ Waiter: commonclient.WaiterOptions{
IgnoreAlreadyExistsError: false, IgnoreAlreadyExistsError: false,
@ -644,12 +698,12 @@ func (a *App) initPolicyStorage(ctx context.Context) {
}, },
}) })
if err != nil { if err != nil {
a.log.Fatal(logs.InitPolicyContractFailed, zap.Error(err)) a.log.Fatal(logs.InitPolicyContractFailed, zap.Error(err), logs.TagField(logs.TagApp))
} }
a.policyStorage = policy.NewStorage(policy.StorageConfig{ a.policyStorage = policy.NewStorage(policy.StorageConfig{
Contract: policyContract, Contract: policyContract,
Cache: cache.NewMorphPolicyCache(getMorphPolicyCacheConfig(a.cfg, a.log)), Cache: cache.NewMorphPolicyCache(getMorphPolicyCacheConfig(a.config(), a.log)),
Log: a.log, Log: a.log,
}) })
} }
@ -658,26 +712,26 @@ func (a *App) initResolver() {
var err error var err error
a.bucketResolver, err = resolver.NewBucketResolver(a.getResolverOrder(), a.getResolverConfig()) a.bucketResolver, err = resolver.NewBucketResolver(a.getResolverOrder(), a.getResolverConfig())
if err != nil { if err != nil {
a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err)) a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err), logs.TagField(logs.TagApp))
} }
} }
func (a *App) getResolverConfig() *resolver.Config { func (a *App) getResolverConfig() *resolver.Config {
return &resolver.Config{ return &resolver.Config{
FrostFS: frostfs.NewResolverFrostFS(a.pool), FrostFS: frostfs.NewResolverFrostFS(a.pool),
RPCAddress: a.cfg.GetString(cfgRPCEndpoint), RPCAddress: a.config().GetString(cfgRPCEndpoint),
} }
} }
func (a *App) getResolverOrder() []string { func (a *App) getResolverOrder() []string {
order := a.cfg.GetStringSlice(cfgResolveOrder) order := a.config().GetStringSlice(cfgResolveOrder)
if a.cfg.GetString(cfgRPCEndpoint) == "" { if a.config().GetString(cfgRPCEndpoint) == "" {
order = remove(order, resolver.NNSResolver) order = remove(order, resolver.NNSResolver)
a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided) a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided, logs.TagField(logs.TagApp))
} }
if len(order) == 0 { if len(order) == 0 {
a.log.Info(logs.ContainerResolverWillBeDisabled) a.log.Info(logs.ContainerResolverWillBeDisabled, logs.TagField(logs.TagApp))
} }
return order return order
@ -689,42 +743,42 @@ func (a *App) initTracing(ctx context.Context) {
instanceID = a.servers[0].Address() instanceID = a.servers[0].Address()
} }
cfg := tracing.Config{ cfg := tracing.Config{
Enabled: a.cfg.GetBool(cfgTracingEnabled), Enabled: a.config().GetBool(cfgTracingEnabled),
Exporter: tracing.Exporter(a.cfg.GetString(cfgTracingExporter)), Exporter: tracing.Exporter(a.config().GetString(cfgTracingExporter)),
Endpoint: a.cfg.GetString(cfgTracingEndpoint), Endpoint: a.config().GetString(cfgTracingEndpoint),
Service: "frostfs-s3-gw", Service: "frostfs-s3-gw",
InstanceID: instanceID, InstanceID: instanceID,
Version: version.Version, Version: version.Version,
} }
if trustedCa := a.cfg.GetString(cfgTracingTrustedCa); trustedCa != "" { if trustedCa := a.config().GetString(cfgTracingTrustedCa); trustedCa != "" {
caBytes, err := os.ReadFile(trustedCa) caBytes, err := os.ReadFile(trustedCa)
if err != nil { if err != nil {
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp))
return return
} }
certPool := x509.NewCertPool() certPool := x509.NewCertPool()
ok := certPool.AppendCertsFromPEM(caBytes) ok := certPool.AppendCertsFromPEM(caBytes)
if !ok { if !ok {
a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert")) a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert"), logs.TagField(logs.TagApp))
return return
} }
cfg.ServerCaCertPool = certPool cfg.ServerCaCertPool = certPool
} }
attributes, err := fetchTracingAttributes(a.cfg) attributes, err := fetchTracingAttributes(a.config())
if err != nil { if err != nil {
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp))
return return
} }
cfg.Attributes = attributes cfg.Attributes = attributes
updated, err := tracing.Setup(ctx, cfg) updated, err := tracing.Setup(ctx, cfg)
if err != nil { if err != nil {
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp))
} }
if updated { if updated {
a.log.Info(logs.TracingConfigUpdated) a.log.Info(logs.TracingConfigUpdated, logs.TagField(logs.TagApp))
} }
} }
@ -734,7 +788,7 @@ func (a *App) shutdownTracing() {
defer cancel() defer cancel()
if err := tracing.Shutdown(shdnCtx); err != nil { if err := tracing.Shutdown(shdnCtx); err != nil {
a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err)) a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err), logs.TagField(logs.TagApp))
} }
} }
@ -751,7 +805,7 @@ func newMaxClients(cfg *viper.Viper) maxClientsConfig {
func getDialerSource(logger *zap.Logger, cfg *viper.Viper) *internalnet.DialerSource { func getDialerSource(logger *zap.Logger, cfg *viper.Viper) *internalnet.DialerSource {
source, err := internalnet.NewDialerSource(fetchMultinetConfig(cfg, logger)) source, err := internalnet.NewDialerSource(fetchMultinetConfig(cfg, logger))
if err != nil { if err != nil {
logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err)) logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err), logs.TagField(logs.TagApp))
} }
return source return source
} }
@ -760,46 +814,45 @@ func (a *App) initPools(ctx context.Context) {
var prm pool.InitParameters var prm pool.InitParameters
var prmTree treepool.InitParameters var prmTree treepool.InitParameters
password := wallet.GetPassword(a.cfg, cfgWalletPassphrase) password := wallet.GetPassword(a.config(), cfgWalletPassphrase)
key, err := wallet.GetKeyFromPath(a.cfg.GetString(cfgWalletPath), a.cfg.GetString(cfgWalletAddress), password) key, err := wallet.GetKeyFromPath(a.config().GetString(cfgWalletPath), a.config().GetString(cfgWalletAddress), password)
if err != nil { if err != nil {
a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err), logs.TagField(logs.TagApp))
} }
prm.SetKey(&key.PrivateKey) prm.SetKey(&key.PrivateKey)
prmTree.SetKey(key) prmTree.SetKey(key)
a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())), logs.TagField(logs.TagApp))
for _, peer := range fetchPeers(a.log, a.cfg) { for _, peer := range fetchPeers(a.log, a.config()) {
prm.AddNode(peer) prm.AddNode(peer)
prmTree.AddNode(peer) prmTree.AddNode(peer)
} }
connTimeout := fetchConnectTimeout(a.cfg) connTimeout := fetchConnectTimeout(a.config())
prm.SetNodeDialTimeout(connTimeout) prm.SetNodeDialTimeout(connTimeout)
prmTree.SetNodeDialTimeout(connTimeout) prmTree.SetNodeDialTimeout(connTimeout)
streamTimeout := fetchStreamTimeout(a.cfg) prm.SetNodeStreamTimeout(fetchStreamTimeout(a.config(), cfgStreamTimeout))
prm.SetNodeStreamTimeout(streamTimeout) prmTree.SetNodeStreamTimeout(fetchStreamTimeout(a.config(), cfgTreeStreamTimeout))
prmTree.SetNodeStreamTimeout(streamTimeout)
healthCheckTimeout := fetchHealthCheckTimeout(a.cfg) healthCheckTimeout := fetchHealthCheckTimeout(a.config())
prm.SetHealthcheckTimeout(healthCheckTimeout) prm.SetHealthcheckTimeout(healthCheckTimeout)
prmTree.SetHealthcheckTimeout(healthCheckTimeout) prmTree.SetHealthcheckTimeout(healthCheckTimeout)
rebalanceInterval := fetchRebalanceInterval(a.cfg) rebalanceInterval := fetchRebalanceInterval(a.config())
prm.SetClientRebalanceInterval(rebalanceInterval) prm.SetClientRebalanceInterval(rebalanceInterval)
prmTree.SetClientRebalanceInterval(rebalanceInterval) prmTree.SetClientRebalanceInterval(rebalanceInterval)
errorThreshold := fetchErrorThreshold(a.cfg) errorThreshold := fetchErrorThreshold(a.config())
prm.SetErrorThreshold(errorThreshold) prm.SetErrorThreshold(errorThreshold)
prm.SetGracefulCloseOnSwitchTimeout(fetchSetGracefulCloseOnSwitchTimeout(a.cfg)) prm.SetGracefulCloseOnSwitchTimeout(fetchSetGracefulCloseOnSwitchTimeout(a.config()))
prm.SetLogger(a.log) prm.SetLogger(a.log.With(logs.TagField(logs.TagDatapath)))
prmTree.SetLogger(a.log) prmTree.SetLogger(a.log.With(logs.TagField(logs.TagDatapath)))
prmTree.SetMaxRequestAttempts(a.cfg.GetInt(cfgTreePoolMaxAttempts)) prmTree.SetMaxRequestAttempts(a.config().GetInt(cfgTreePoolMaxAttempts))
interceptors := []grpc.DialOption{ interceptors := []grpc.DialOption{
grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()),
@ -811,23 +864,23 @@ func (a *App) initPools(ctx context.Context) {
p, err := pool.NewPool(prm) p, err := pool.NewPool(prm)
if err != nil { if err != nil {
a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err), logs.TagField(logs.TagApp))
} }
if err = p.Dial(ctx); err != nil { if err = p.Dial(ctx); err != nil {
a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err), logs.TagField(logs.TagApp))
} }
if a.cfg.GetBool(cfgTreePoolNetmapSupport) { if a.config().GetBool(cfgTreePoolNetmapSupport) {
prmTree.SetNetMapInfoSource(frostfs.NewSource(frostfs.NewFrostFS(p, key), a.cache)) prmTree.SetNetMapInfoSource(frostfs.NewSource(frostfs.NewFrostFS(p, key), a.cache))
} }
treePool, err := treepool.NewPool(prmTree) treePool, err := treepool.NewPool(prmTree)
if err != nil { if err != nil {
a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err), logs.TagField(logs.TagApp))
} }
if err = treePool.Dial(ctx); err != nil { if err = treePool.Dial(ctx); err != nil {
a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err)) a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err), logs.TagField(logs.TagApp))
} }
a.treePool = treePool a.treePool = treePool
@ -853,6 +906,7 @@ func (a *App) Wait() {
a.log.Info(logs.ApplicationStarted, a.log.Info(logs.ApplicationStarted,
zap.String("name", "frostfs-s3-gw"), zap.String("name", "frostfs-s3-gw"),
zap.String("version", version.Version), zap.String("version", version.Version),
logs.TagField(logs.TagApp),
) )
a.metrics.State().SetVersion(version.Version) a.metrics.State().SetVersion(version.Version)
@ -860,7 +914,7 @@ func (a *App) Wait() {
<-a.webDone // wait for web-server to be stopped <-a.webDone // wait for web-server to be stopped
a.log.Info(logs.ApplicationFinished) a.log.Info(logs.ApplicationFinished, logs.TagField(logs.TagApp))
} }
func (a *App) setHealthStatus() { func (a *App) setHealthStatus() {
@ -895,10 +949,10 @@ func (a *App) Serve(ctx context.Context) {
srv := new(http.Server) srv := new(http.Server)
srv.Handler = chiRouter srv.Handler = chiRouter
srv.ErrorLog = zap.NewStdLog(a.log) srv.ErrorLog = zap.NewStdLog(a.log)
srv.ReadTimeout = a.cfg.GetDuration(cfgWebReadTimeout) srv.ReadTimeout = a.config().GetDuration(cfgWebReadTimeout)
srv.ReadHeaderTimeout = a.cfg.GetDuration(cfgWebReadHeaderTimeout) srv.ReadHeaderTimeout = a.config().GetDuration(cfgWebReadHeaderTimeout)
srv.WriteTimeout = a.cfg.GetDuration(cfgWebWriteTimeout) srv.WriteTimeout = a.config().GetDuration(cfgWebWriteTimeout)
srv.IdleTimeout = a.cfg.GetDuration(cfgWebIdleTimeout) srv.IdleTimeout = a.config().GetDuration(cfgWebIdleTimeout)
a.startServices() a.startServices()
@ -906,11 +960,11 @@ func (a *App) Serve(ctx context.Context) {
for i := range servs { for i := range servs {
go func(i int) { go func(i int) {
a.log.Info(logs.StartingServer, zap.String("address", servs[i].Address())) a.log.Info(logs.StartingServer, zap.String("address", servs[i].Address()), logs.TagField(logs.TagApp))
if err := srv.Serve(servs[i].Listener()); err != nil && err != http.ErrServerClosed { if err := srv.Serve(servs[i].Listener()); err != nil && err != http.ErrServerClosed {
a.metrics.MarkUnhealthy(servs[i].Address()) a.metrics.MarkUnhealthy(servs[i].Address())
a.log.Fatal(logs.ListenAndServe, zap.Error(err)) a.log.Fatal(logs.ListenAndServe, zap.Error(err), logs.TagField(logs.TagApp))
} }
}(i) }(i)
} }
@ -935,7 +989,7 @@ LOOP:
ctx, cancel := shutdownContext() ctx, cancel := shutdownContext()
defer cancel() defer cancel()
a.log.Info(logs.StoppingServer, zap.Error(srv.Shutdown(ctx))) a.log.Info(logs.StoppingServer, zap.Error(srv.Shutdown(ctx)), logs.TagField(logs.TagApp))
a.metrics.Shutdown() a.metrics.Shutdown()
a.stopServices() a.stopServices()
@ -949,23 +1003,23 @@ func shutdownContext() (context.Context, context.CancelFunc) {
} }
func (a *App) configReload(ctx context.Context) { func (a *App) configReload(ctx context.Context) {
a.log.Info(logs.SIGHUPConfigReloadStarted) a.log.Info(logs.SIGHUPConfigReloadStarted, logs.TagField(logs.TagApp))
if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) { if !a.config().IsSet(cmdConfig) && !a.config().IsSet(cmdConfigDir) {
a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed) a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed, logs.TagField(logs.TagApp))
return return
} }
if err := readInConfig(a.cfg); err != nil { if err := a.cfg.reload(); err != nil {
a.log.Warn(logs.FailedToReloadConfig, zap.Error(err)) a.log.Warn(logs.FailedToReloadConfig, zap.Error(err), logs.TagField(logs.TagApp))
return return
} }
if err := a.bucketResolver.UpdateResolvers(a.getResolverOrder()); err != nil { if err := a.bucketResolver.UpdateResolvers(a.getResolverOrder()); err != nil {
a.log.Warn(logs.FailedToReloadResolvers, zap.Error(err)) a.log.Warn(logs.FailedToReloadResolvers, zap.Error(err), logs.TagField(logs.TagApp))
} }
if err := a.updateServers(); err != nil { if err := a.updateServers(); err != nil {
a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err)) a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err), logs.TagField(logs.TagApp))
} }
a.setRuntimeParameters() a.setRuntimeParameters()
@ -975,41 +1029,45 @@ func (a *App) configReload(ctx context.Context) {
a.updateSettings() a.updateSettings()
a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled)) a.metrics.SetEnabled(a.config().GetBool(cfgPrometheusEnabled))
a.initTracing(ctx) a.initTracing(ctx)
a.setHealthStatus() a.setHealthStatus()
a.log.Info(logs.SIGHUPConfigReloadCompleted) a.log.Info(logs.SIGHUPConfigReloadCompleted, logs.TagField(logs.TagApp))
} }
func (a *App) updateSettings() { func (a *App) updateSettings() {
if lvl, err := getLogLevel(a.cfg); err != nil { if lvl, err := getLogLevel(a.config()); err != nil {
a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err)) a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp))
} else { } else {
a.settings.logLevel.SetLevel(lvl) a.settings.logLevel.SetLevel(lvl)
} }
if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.cfg, a.log)); err != nil { if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.config(), a.log)); err != nil {
a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err)) a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp))
} }
a.settings.update(a.cfg, a.log) if err := a.settings.tagsConfig.update(a.config()); err != nil {
a.log.Warn(logs.TagsLogConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp))
}
a.settings.update(a.config(), a.log)
} }
func (a *App) startServices() { func (a *App) startServices() {
a.services = a.services[:0] a.services = a.services[:0]
pprofService := NewPprofService(a.cfg, a.log) pprofService := NewPprofService(a.config(), a.log)
a.services = append(a.services, pprofService) a.services = append(a.services, pprofService)
go pprofService.Start() go pprofService.Start()
prometheusService := NewPrometheusService(a.cfg, a.log, a.metrics.Handler()) prometheusService := NewPrometheusService(a.config(), a.log, a.metrics.Handler())
a.services = append(a.services, prometheusService) a.services = append(a.services, prometheusService)
go prometheusService.Start() go prometheusService.Start()
} }
func (a *App) initServers(ctx context.Context) { func (a *App) initServers(ctx context.Context) {
serversInfo := fetchServers(a.cfg, a.log) serversInfo := fetchServers(a.config(), a.log)
a.servers = make([]Server, 0, len(serversInfo)) a.servers = make([]Server, 0, len(serversInfo))
for _, serverInfo := range serversInfo { for _, serverInfo := range serversInfo {
@ -1021,22 +1079,22 @@ func (a *App) initServers(ctx context.Context) {
if err != nil { if err != nil {
a.unbindServers = append(a.unbindServers, serverInfo) a.unbindServers = append(a.unbindServers, serverInfo)
a.metrics.MarkUnhealthy(serverInfo.Address) a.metrics.MarkUnhealthy(serverInfo.Address)
a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...) a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err), logs.TagField(logs.TagApp))...)
continue continue
} }
a.metrics.MarkHealthy(serverInfo.Address) a.metrics.MarkHealthy(serverInfo.Address)
a.servers = append(a.servers, srv) a.servers = append(a.servers, srv)
a.log.Info(logs.AddServer, fields...) a.log.Info(logs.AddServer, append(fields, logs.TagField(logs.TagApp))...)
} }
if len(a.servers) == 0 { if len(a.servers) == 0 {
a.log.Fatal(logs.NoHealthyServers) a.log.Fatal(logs.NoHealthyServers, logs.TagField(logs.TagApp))
} }
} }
func (a *App) updateServers() error { func (a *App) updateServers() error {
serversInfo := fetchServers(a.cfg, a.log) serversInfo := fetchServers(a.config(), a.log)
a.mu.Lock() a.mu.Lock()
defer a.mu.Unlock() defer a.mu.Unlock()
@ -1049,8 +1107,8 @@ func (a *App) updateServers() error {
if err := ser.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil { if err := ser.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
return fmt.Errorf("failed to update tls certs: %w", err) return fmt.Errorf("failed to update tls certs: %w", err)
} }
found = true
} }
found = true
} else if unbind := a.updateUnbindServerInfo(serverInfo); unbind { } else if unbind := a.updateUnbindServerInfo(serverInfo); unbind {
found = true found = true
} }
@ -1135,7 +1193,7 @@ func (a *App) initHandler() {
a.api, err = handler.New(a.log, a.obj, a.settings, a.policyStorage, a.frostfsid) a.api, err = handler.New(a.log, a.obj, a.settings, a.policyStorage, a.frostfsid)
if err != nil { if err != nil {
a.log.Fatal(logs.CouldNotInitializeAPIHandler, zap.Error(err)) a.log.Fatal(logs.CouldNotInitializeAPIHandler, zap.Error(err), logs.TagField(logs.TagApp))
} }
} }
@ -1167,16 +1225,18 @@ func (a *App) getServers() []Server {
func (a *App) setRuntimeParameters() { func (a *App) setRuntimeParameters() {
if len(os.Getenv("GOMEMLIMIT")) != 0 { if len(os.Getenv("GOMEMLIMIT")) != 0 {
// default limit < yaml limit < app env limit < GOMEMLIMIT // default limit < yaml limit < app env limit < GOMEMLIMIT
a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT, logs.TagField(logs.TagApp))
return return
} }
softMemoryLimit := fetchSoftMemoryLimit(a.cfg) softMemoryLimit := fetchSoftMemoryLimit(a.config())
previous := debug.SetMemoryLimit(softMemoryLimit) previous := debug.SetMemoryLimit(softMemoryLimit)
if softMemoryLimit != previous { if softMemoryLimit != previous {
a.log.Info(logs.RuntimeSoftMemoryLimitUpdated, a.log.Info(logs.RuntimeSoftMemoryLimitUpdated,
zap.Int64("new_value", softMemoryLimit), zap.Int64("new_value", softMemoryLimit),
zap.Int64("old_value", previous)) zap.Int64("old_value", previous),
logs.TagField(logs.TagApp),
)
} }
} }
@ -1202,7 +1262,7 @@ func (a *App) tryReconnect(ctx context.Context, sr *http.Server) bool {
a.mu.Lock() a.mu.Lock()
defer a.mu.Unlock() defer a.mu.Unlock()
a.log.Info(logs.ServerReconnecting) a.log.Info(logs.ServerReconnecting, logs.TagField(logs.TagApp))
var failedServers []ServerInfo var failedServers []ServerInfo
for _, serverInfo := range a.unbindServers { for _, serverInfo := range a.unbindServers {
@ -1213,23 +1273,23 @@ func (a *App) tryReconnect(ctx context.Context, sr *http.Server) bool {
srv, err := newServer(ctx, serverInfo) srv, err := newServer(ctx, serverInfo)
if err != nil { if err != nil {
a.log.Warn(logs.ServerReconnectFailed, zap.Error(err)) a.log.Warn(logs.ServerReconnectFailed, zap.Error(err), logs.TagField(logs.TagApp))
failedServers = append(failedServers, serverInfo) failedServers = append(failedServers, serverInfo)
a.metrics.MarkUnhealthy(serverInfo.Address) a.metrics.MarkUnhealthy(serverInfo.Address)
continue continue
} }
go func() { go func() {
a.log.Info(logs.StartingServer, zap.String("address", srv.Address())) a.log.Info(logs.StartingServer, zap.String("address", srv.Address()), logs.TagField(logs.TagApp))
a.metrics.MarkHealthy(serverInfo.Address) a.metrics.MarkHealthy(serverInfo.Address)
if err = sr.Serve(srv.Listener()); err != nil && !errors.Is(err, http.ErrServerClosed) { if err = sr.Serve(srv.Listener()); err != nil && !errors.Is(err, http.ErrServerClosed) {
a.log.Warn(logs.ListenAndServe, zap.Error(err)) a.log.Warn(logs.ListenAndServe, zap.Error(err), logs.TagField(logs.TagApp))
a.metrics.MarkUnhealthy(serverInfo.Address) a.metrics.MarkUnhealthy(serverInfo.Address)
} }
}() }()
a.servers = append(a.servers, srv) a.servers = append(a.servers, srv)
a.log.Info(logs.ServerReconnectedSuccessfully, fields...) a.log.Info(logs.ServerReconnectedSuccessfully, append(fields, logs.TagField(logs.TagApp))...)
} }
a.unbindServers = failedServers a.unbindServers = failedServers
@ -1247,7 +1307,7 @@ func (a *App) fetchContainerInfo(ctx context.Context, cfgKey string) (info *data
} }
func (a *App) resolveContainerID(ctx context.Context, cfgKey string) (cid.ID, error) { func (a *App) resolveContainerID(ctx context.Context, cfgKey string) (cid.ID, error) {
containerString := a.cfg.GetString(cfgKey) containerString := a.config().GetString(cfgKey)
var id cid.ID var id cid.ID
if err := id.DecodeString(containerString); err != nil { if err := id.DecodeString(containerString); err != nil {

View file

@ -10,6 +10,7 @@ import (
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
"sync"
"time" "time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
@ -20,20 +21,13 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"git.frostfs.info/TrueCloudLab/zapjournald"
"github.com/spf13/pflag" "github.com/spf13/pflag"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/ssgreg/journald"
"go.uber.org/zap" "go.uber.org/zap"
"go.uber.org/zap/zapcore" "go.uber.org/zap/zapcore"
) )
const ( const wildcardPlaceholder = "<wildcard>"
destinationStdout = "stdout"
destinationJournald = "journald"
wildcardPlaceholder = "<wildcard>"
)
const ( const (
defaultRebalanceInterval = 60 * time.Second defaultRebalanceInterval = 60 * time.Second
@ -88,7 +82,8 @@ var (
defaultDefaultNamespaces = []string{"", "root"} defaultDefaultNamespaces = []string{"", "root"}
) )
const ( // Settings. // Settings.
const (
// Logger. // Logger.
cfgLoggerLevel = "logger.level" cfgLoggerLevel = "logger.level"
cfgLoggerDestination = "logger.destination" cfgLoggerDestination = "logger.destination"
@ -98,6 +93,11 @@ const ( // Settings.
cfgLoggerSamplingThereafter = "logger.sampling.thereafter" cfgLoggerSamplingThereafter = "logger.sampling.thereafter"
cfgLoggerSamplingInterval = "logger.sampling.interval" cfgLoggerSamplingInterval = "logger.sampling.interval"
cfgLoggerTags = "logger.tags"
cfgLoggerTagsPrefixTmpl = cfgLoggerTags + ".%d."
cfgLoggerTagsNameTmpl = cfgLoggerTagsPrefixTmpl + "name"
cfgLoggerTagsLevelTmpl = cfgLoggerTagsPrefixTmpl + "level"
// HttpLogging. // HttpLogging.
cfgHTTPLoggingEnabled = "http_logging.enabled" cfgHTTPLoggingEnabled = "http_logging.enabled"
cfgHTTPLoggingMaxBody = "http_logging.max_body" cfgHTTPLoggingMaxBody = "http_logging.max_body"
@ -122,6 +122,7 @@ const ( // Settings.
// Pool config. // Pool config.
cfgConnectTimeout = "connect_timeout" cfgConnectTimeout = "connect_timeout"
cfgStreamTimeout = "stream_timeout" cfgStreamTimeout = "stream_timeout"
cfgTreeStreamTimeout = "tree_stream_timeout"
cfgHealthcheckTimeout = "healthcheck_timeout" cfgHealthcheckTimeout = "healthcheck_timeout"
cfgRebalanceInterval = "rebalance_interval" cfgRebalanceInterval = "rebalance_interval"
cfgPoolErrorThreshold = "pool_error_threshold" cfgPoolErrorThreshold = "pool_error_threshold"
@ -301,6 +302,49 @@ var ignore = map[string]struct{}{
cmdVersion: {}, cmdVersion: {},
} }
type appCfg struct {
flags *pflag.FlagSet
mu sync.RWMutex
settings *viper.Viper
}
func (a *appCfg) reload() error {
old := a.config()
v, err := newViper(a.flags)
if err != nil {
return err
}
if old.IsSet(cmdConfig) {
v.Set(cmdConfig, old.Get(cmdConfig))
}
if old.IsSet(cmdConfigDir) {
v.Set(cmdConfigDir, old.Get(cmdConfigDir))
}
if err = readInConfig(v); err != nil {
return err
}
a.setConfig(v)
return nil
}
func (a *appCfg) config() *viper.Viper {
a.mu.RLock()
defer a.mu.RUnlock()
return a.settings
}
func (a *appCfg) setConfig(v *viper.Viper) {
a.mu.Lock()
a.settings = v
a.mu.Unlock()
}
func fetchConnectTimeout(cfg *viper.Viper) time.Duration { func fetchConnectTimeout(cfg *viper.Viper) time.Duration {
connTimeout := cfg.GetDuration(cfgConnectTimeout) connTimeout := cfg.GetDuration(cfgConnectTimeout)
if connTimeout <= 0 { if connTimeout <= 0 {
@ -319,8 +363,8 @@ func fetchReconnectInterval(cfg *viper.Viper) time.Duration {
return reconnect return reconnect
} }
func fetchStreamTimeout(cfg *viper.Viper) time.Duration { func fetchStreamTimeout(cfg *viper.Viper, cfgEntry string) time.Duration {
streamTimeout := cfg.GetDuration(cfgStreamTimeout) streamTimeout := cfg.GetDuration(cfgEntry)
if streamTimeout <= 0 { if streamTimeout <= 0 {
streamTimeout = defaultStreamTimeout streamTimeout = defaultStreamTimeout
} }
@ -425,14 +469,18 @@ func fetchDefaultPolicy(l *zap.Logger, cfg *viper.Viper) netmap.PlacementPolicy
policyStr := cfg.GetString(cfgPolicyDefault) policyStr := cfg.GetString(cfgPolicyDefault)
if err := policy.DecodeString(policyStr); err != nil { if err := policy.DecodeString(policyStr); err != nil {
l.Warn(logs.FailedToParseDefaultLocationConstraint, l.Warn(logs.FailedToParseDefaultLocationConstraint,
zap.String("policy", policyStr), zap.String("default", defaultPlacementPolicy), zap.Error(err)) zap.String("policy", policyStr), zap.String("default", defaultPlacementPolicy),
zap.Error(err), logs.TagField(logs.TagApp))
} else { } else {
return policy return policy
} }
} }
if err := policy.DecodeString(defaultPlacementPolicy); err != nil { if err := policy.DecodeString(defaultPlacementPolicy); err != nil {
l.Fatal(logs.FailedToParseDefaultDefaultLocationConstraint, zap.String("policy", defaultPlacementPolicy)) l.Fatal(logs.FailedToParseDefaultDefaultLocationConstraint,
zap.String("policy", defaultPlacementPolicy),
logs.TagField(logs.TagApp),
)
} }
return policy return policy
@ -445,7 +493,9 @@ func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultV
l.Error(logs.InvalidLifetimeUsingDefaultValue, l.Error(logs.InvalidLifetimeUsingDefaultValue,
zap.String("parameter", cfgEntry), zap.String("parameter", cfgEntry),
zap.Duration("value in config", lifetime), zap.Duration("value in config", lifetime),
zap.Duration("default", defaultValue)) zap.Duration("default", defaultValue),
logs.TagField(logs.TagApp),
)
} else { } else {
return lifetime return lifetime
} }
@ -461,7 +511,9 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue
l.Error(logs.InvalidCacheSizeUsingDefaultValue, l.Error(logs.InvalidCacheSizeUsingDefaultValue,
zap.String("parameter", cfgEntry), zap.String("parameter", cfgEntry),
zap.Int("value in config", size), zap.Int("value in config", size),
zap.Int("default", defaultValue)) zap.Int("default", defaultValue),
logs.TagField(logs.TagApp),
)
} else { } else {
return size return size
} }
@ -483,7 +535,8 @@ func fetchRemovingCheckInterval(v *viper.Viper, l *zap.Logger) time.Duration {
l.Error(logs.InvalidAccessBoxCacheRemovingCheckInterval, l.Error(logs.InvalidAccessBoxCacheRemovingCheckInterval,
zap.String("parameter", cfgAccessBoxCacheRemovingCheckInterval), zap.String("parameter", cfgAccessBoxCacheRemovingCheckInterval),
zap.Duration("value in config", duration), zap.Duration("value in config", duration),
zap.Duration("default", defaultAccessBoxCacheRemovingCheckInterval)) zap.Duration("default", defaultAccessBoxCacheRemovingCheckInterval),
logs.TagField(logs.TagApp))
return defaultAccessBoxCacheRemovingCheckInterval return defaultAccessBoxCacheRemovingCheckInterval
} }
@ -497,7 +550,9 @@ func fetchDefaultMaxAge(cfg *viper.Viper, l *zap.Logger) int {
if defaultMaxAge <= 0 && defaultMaxAge != -1 { if defaultMaxAge <= 0 && defaultMaxAge != -1 {
l.Fatal(logs.InvalidDefaultMaxAge, l.Fatal(logs.InvalidDefaultMaxAge,
zap.String("parameter", cfgDefaultMaxAge), zap.String("parameter", cfgDefaultMaxAge),
zap.String("value in config", strconv.Itoa(defaultMaxAge))) zap.String("value in config", strconv.Itoa(defaultMaxAge)),
logs.TagField(logs.TagApp),
)
} }
} }
@ -508,14 +563,19 @@ func fetchRegionMappingPolicies(l *zap.Logger, cfg *viper.Viper) map[string]netm
filepath := cfg.GetString(cfgPolicyRegionMapFile) filepath := cfg.GetString(cfgPolicyRegionMapFile)
regionPolicyMap, err := readRegionMap(filepath) regionPolicyMap, err := readRegionMap(filepath)
if err != nil { if err != nil {
l.Warn(logs.FailedToReadRegionMapFilePolicies, zap.String("file", filepath), zap.Error(err)) l.Warn(logs.FailedToReadRegionMapFilePolicies,
zap.String("file", filepath),
zap.Error(err),
logs.TagField(logs.TagApp))
return make(map[string]netmap.PlacementPolicy) return make(map[string]netmap.PlacementPolicy)
} }
regionMap := make(map[string]netmap.PlacementPolicy, len(regionPolicyMap)) regionMap := make(map[string]netmap.PlacementPolicy, len(regionPolicyMap))
for region, policy := range regionPolicyMap { for region, policy := range regionPolicyMap {
if region == api.DefaultLocationConstraint { if region == api.DefaultLocationConstraint {
l.Warn(logs.DefaultLocationConstraintCantBeOverriden, zap.String("policy", policy)) l.Warn(logs.DefaultLocationConstraintCantBeOverriden,
zap.String("policy", policy),
logs.TagField(logs.TagApp))
continue continue
} }
@ -530,7 +590,10 @@ func fetchRegionMappingPolicies(l *zap.Logger, cfg *viper.Viper) map[string]netm
continue continue
} }
l.Warn(logs.FailedToParseLocationConstraint, zap.String("region", region), zap.String("policy", policy)) l.Warn(logs.FailedToParseLocationConstraint,
zap.String("region", region),
zap.String("policy", policy),
logs.TagField(logs.TagApp))
} }
return regionMap return regionMap
@ -562,7 +625,11 @@ func fetchDefaultCopiesNumbers(l *zap.Logger, v *viper.Viper) []uint32 {
parsedValue, err := strconv.ParseUint(unparsed[i], 10, 32) parsedValue, err := strconv.ParseUint(unparsed[i], 10, 32)
if err != nil { if err != nil {
l.Warn(logs.FailedToParseDefaultCopiesNumbers, l.Warn(logs.FailedToParseDefaultCopiesNumbers,
zap.Strings("copies numbers", unparsed), zap.Uint32s("default", defaultCopiesNumbers), zap.Error(err)) zap.Strings("copies numbers", unparsed),
zap.Uint32s("default", defaultCopiesNumbers),
zap.Error(err),
logs.TagField(logs.TagApp),
)
return defaultCopiesNumbers return defaultCopiesNumbers
} }
result[i] = uint32(parsedValue) result[i] = uint32(parsedValue)
@ -618,15 +685,17 @@ func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) map[string][]uint32 {
for j := range vector { for j := range vector {
parsedValue, err := strconv.ParseUint(vector[j], 10, 32) parsedValue, err := strconv.ParseUint(vector[j], 10, 32)
if err != nil { if err != nil {
l.Warn(logs.FailedToParseCopiesNumbers, zap.String("location", constraint), l.Warn(logs.FailedToParseCopiesNumbers,
zap.Strings("copies numbers", vector), zap.Error(err)) zap.String("location", constraint),
zap.Strings("copies numbers", vector), zap.Error(err),
logs.TagField(logs.TagApp))
continue continue
} }
vector32[j] = uint32(parsedValue) vector32[j] = uint32(parsedValue)
} }
copiesNums[constraint] = vector32 copiesNums[constraint] = vector32
l.Info(logs.ConstraintAdded, zap.String("location", constraint), zap.Strings("copies numbers", vector)) l.Info(logs.ConstraintAdded, zap.String("location", constraint), zap.Strings("copies numbers", vector), logs.TagField(logs.TagApp))
} }
return copiesNums return copiesNums
} }
@ -635,7 +704,9 @@ func fetchDefaultNamespaces(l *zap.Logger, v *viper.Viper) []string {
defaultNamespaces := v.GetStringSlice(cfgKludgeDefaultNamespaces) defaultNamespaces := v.GetStringSlice(cfgKludgeDefaultNamespaces)
if len(defaultNamespaces) == 0 { if len(defaultNamespaces) == 0 {
defaultNamespaces = defaultDefaultNamespaces defaultNamespaces = defaultDefaultNamespaces
l.Warn(logs.DefaultNamespacesCannotBeEmpty, zap.Strings("namespaces", defaultNamespaces)) l.Warn(logs.DefaultNamespacesCannotBeEmpty,
zap.Strings("namespaces", defaultNamespaces),
logs.TagField(logs.TagApp))
} }
for i := range defaultNamespaces { // to be set namespaces in env variable as `S3_GW_KLUDGE_DEFAULT_NAMESPACES="" 'root'` for i := range defaultNamespaces { // to be set namespaces in env variable as `S3_GW_KLUDGE_DEFAULT_NAMESPACES="" 'root'`
@ -659,7 +730,7 @@ func fetchNamespacesConfig(l *zap.Logger, v *viper.Viper) (NamespacesConfig, []s
nsConfig, err := readNamespacesConfig(v.GetString(cfgNamespacesConfig)) nsConfig, err := readNamespacesConfig(v.GetString(cfgNamespacesConfig))
if err != nil { if err != nil {
l.Warn(logs.FailedToParseNamespacesConfig, zap.Error(err)) l.Warn(logs.FailedToParseNamespacesConfig, zap.Error(err), logs.TagField(logs.TagApp))
} }
defaultNamespacesNames := fetchDefaultNamespaces(l, v) defaultNamespacesNames := fetchDefaultNamespaces(l, v)
@ -673,11 +744,13 @@ func fetchNamespacesConfig(l *zap.Logger, v *viper.Viper) (NamespacesConfig, []s
} }
if len(overrideDefaults) > 0 { if len(overrideDefaults) > 0 {
l.Warn(logs.DefaultNamespaceConfigValuesBeOverwritten) l.Warn(logs.DefaultNamespaceConfigValuesBeOverwritten, logs.TagField(logs.TagApp))
defaultNSValue.LocationConstraints = overrideDefaults[0].LocationConstraints defaultNSValue.LocationConstraints = overrideDefaults[0].LocationConstraints
defaultNSValue.CopiesNumbers = overrideDefaults[0].CopiesNumbers defaultNSValue.CopiesNumbers = overrideDefaults[0].CopiesNumbers
if len(overrideDefaults) > 1 { if len(overrideDefaults) > 1 {
l.Warn(logs.MultipleDefaultOverridesFound, zap.String("name", overrideDefaults[0].Name)) l.Warn(logs.MultipleDefaultOverridesFound,
zap.String("name", overrideDefaults[0].Name),
logs.TagField(logs.TagApp))
} }
} }
@ -720,7 +793,7 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
priority := v.GetInt(key + "priority") priority := v.GetInt(key + "priority")
if address == "" { if address == "" {
l.Warn(logs.SkipEmptyAddress) l.Warn(logs.SkipEmptyAddress, logs.TagField(logs.TagApp))
break break
} }
if weight <= 0 { // unspecified or wrong if weight <= 0 { // unspecified or wrong
@ -735,7 +808,9 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
l.Info(logs.AddedStoragePeer, l.Info(logs.AddedStoragePeer,
zap.Int("priority", priority), zap.Int("priority", priority),
zap.String("address", address), zap.String("address", address),
zap.Float64("weight", weight)) zap.Float64("weight", weight),
logs.TagField(logs.TagApp),
)
} }
return nodes return nodes
@ -759,7 +834,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo {
} }
if _, ok := seen[serverInfo.Address]; ok { if _, ok := seen[serverInfo.Address]; ok {
log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address)) log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address), logs.TagField(logs.TagApp))
continue continue
} }
seen[serverInfo.Address] = struct{}{} seen[serverInfo.Address] = struct{}{}
@ -788,13 +863,13 @@ func fetchVHSNamespaces(v *viper.Viper, log *zap.Logger) map[string]bool {
nsMap := v.GetStringMap(cfgVHSNamespaces) nsMap := v.GetStringMap(cfgVHSNamespaces)
for ns, val := range nsMap { for ns, val := range nsMap {
if _, ok := vhsNamespacesEnabled[ns]; ok { if _, ok := vhsNamespacesEnabled[ns]; ok {
log.Warn(logs.WarnDuplicateNamespaceVHS, zap.String("namespace", ns)) log.Warn(logs.WarnDuplicateNamespaceVHS, zap.String("namespace", ns), logs.TagField(logs.TagApp))
continue continue
} }
enabledFlag, ok := val.(bool) enabledFlag, ok := val.(bool)
if !ok { if !ok {
log.Warn(logs.WarnValueVHSEnabledFlagWrongType, zap.String("namespace", ns)) log.Warn(logs.WarnValueVHSEnabledFlagWrongType, zap.String("namespace", ns), logs.TagField(logs.TagApp))
continue continue
} }
@ -878,7 +953,42 @@ func fetchTombstoneWorkerPoolSize(v *viper.Viper) int {
return tombstoneWorkerPoolSize return tombstoneWorkerPoolSize
} }
func newSettings() *viper.Viper { func fetchLogTagsConfig(v *viper.Viper) (map[string]zapcore.Level, error) {
res := make(map[string]zapcore.Level)
defaultLevel := v.GetString(cfgLoggerLevel)
var defaultLvl zapcore.Level
if err := defaultLvl.Set(defaultLevel); err != nil {
return nil, fmt.Errorf("failed to parse log level, unknown level: '%s'", defaultLevel)
}
for i := 0; ; i++ {
name := v.GetString(fmt.Sprintf(cfgLoggerTagsNameTmpl, i))
if name == "" {
break
}
lvl := defaultLvl
level := v.GetString(fmt.Sprintf(cfgLoggerTagsLevelTmpl, i))
if level != "" {
if err := lvl.Set(level); err != nil {
return nil, fmt.Errorf("failed to parse log tags config, unknown level: '%s'", level)
}
}
res[name] = lvl
}
if len(res) == 0 && !v.IsSet(cfgLoggerTags) {
for _, tag := range defaultTags {
res[tag] = defaultLvl
}
}
return res, nil
}
func newViper(flags *pflag.FlagSet) (*viper.Viper, error) {
v := viper.New() v := viper.New()
v.AutomaticEnv() v.AutomaticEnv()
@ -887,6 +997,20 @@ func newSettings() *viper.Viper {
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
v.AllowEmptyEnv(true) v.AllowEmptyEnv(true)
if err := bindFlags(v, flags); err != nil {
return nil, err
}
setDefaults(v, flags)
if v.IsSet(cfgServer+".0."+cfgTLSKeyFile) && v.IsSet(cfgServer+".0."+cfgTLSCertFile) {
v.Set(cfgServer+".0."+cfgTLSEnabled, true)
}
return v, nil
}
func newSettings() *appCfg {
// flags setup: // flags setup:
flags := pflag.NewFlagSet("commandline", pflag.ExitOnError) flags := pflag.NewFlagSet("commandline", pflag.ExitOnError)
flags.SetOutput(os.Stdout) flags.SetOutput(os.Stdout)
@ -914,15 +1038,71 @@ func newSettings() *viper.Viper {
flags.String(cfgTLSCertFile, "", "TLS certificate file to use") flags.String(cfgTLSCertFile, "", "TLS certificate file to use")
flags.String(cfgTLSKeyFile, "", "TLS key file to use") flags.String(cfgTLSKeyFile, "", "TLS key file to use")
peers := flags.StringArrayP(cfgPeers, "p", nil, "set FrostFS nodes") flags.StringArrayP(cfgPeers, "p", nil, "set FrostFS nodes")
flags.StringP(cfgRPCEndpoint, "r", "", "set RPC endpoint") flags.StringP(cfgRPCEndpoint, "r", "", "set RPC endpoint")
resolveMethods := flags.StringSlice(cfgResolveOrder, []string{resolver.DNSResolver}, "set bucket name resolve order") flags.StringSlice(cfgResolveOrder, []string{resolver.DNSResolver}, "set bucket name resolve order")
domains := flags.StringSliceP(cfgListenDomains, "d", nil, "set domains to be listened") flags.StringSliceP(cfgListenDomains, "d", nil, "set domains to be listened")
// set defaults: if err := flags.Parse(os.Args); err != nil {
panic(err)
}
v, err := newViper(flags)
if err != nil {
panic(fmt.Errorf("bind flags: %w", err))
}
switch {
case help != nil && *help:
fmt.Printf("FrostFS S3 gateway %s\n", version.Version)
flags.PrintDefaults()
fmt.Println()
fmt.Println("Default environments:")
fmt.Println()
keys := v.AllKeys()
sort.Strings(keys)
for i := range keys {
if _, ok := ignore[keys[i]]; ok {
continue
}
defaultValue := v.GetString(keys[i])
if len(defaultValue) == 0 {
continue
}
k := strings.Replace(keys[i], ".", "_", -1)
fmt.Printf("%s_%s = %s\n", envPrefix, strings.ToUpper(k), defaultValue)
}
fmt.Println()
fmt.Println("Peers preset:")
fmt.Println()
fmt.Printf("%s_%s_[N]_ADDRESS = string\n", envPrefix, strings.ToUpper(cfgPeers))
fmt.Printf("%s_%s_[N]_WEIGHT = 0..1 (float)\n", envPrefix, strings.ToUpper(cfgPeers))
os.Exit(0)
case versionFlag != nil && *versionFlag:
fmt.Printf("FrostFS S3 Gateway\nVersion: %s\nGoVersion: %s\n", version.Version, runtime.Version())
os.Exit(0)
}
if err = readInConfig(v); err != nil {
panic(err)
}
return &appCfg{
flags: flags,
settings: v,
}
}
func setDefaults(v *viper.Viper, flags *pflag.FlagSet) {
v.SetDefault(cfgAccessBoxCacheRemovingCheckInterval, defaultAccessBoxCacheRemovingCheckInterval) v.SetDefault(cfgAccessBoxCacheRemovingCheckInterval, defaultAccessBoxCacheRemovingCheckInterval)
// logger: // logger:
@ -986,78 +1166,21 @@ func newSettings() *viper.Viper {
// multinet // multinet
v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay) v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay)
// Bind flags if resolveMethods, err := flags.GetStringSlice(cfgResolveOrder); err == nil {
if err := bindFlags(v, flags); err != nil { v.SetDefault(cfgResolveOrder, resolveMethods)
panic(fmt.Errorf("bind flags: %w", err))
} }
if err := flags.Parse(os.Args); err != nil { if peers, err := flags.GetStringArray(cfgPeers); err == nil {
panic(err) for i := range peers {
} v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".address", peers[i])
if v.IsSet(cfgServer+".0."+cfgTLSKeyFile) && v.IsSet(cfgServer+".0."+cfgTLSCertFile) {
v.Set(cfgServer+".0."+cfgTLSEnabled, true)
}
if resolveMethods != nil {
v.SetDefault(cfgResolveOrder, *resolveMethods)
}
if peers != nil && len(*peers) > 0 {
for i := range *peers {
v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".address", (*peers)[i])
v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".weight", 1) v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".weight", 1)
v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".priority", 1) v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".priority", 1)
} }
} }
if domains != nil && len(*domains) > 0 { if domains, err := flags.GetStringSlice(cfgListenDomains); err == nil && len(domains) > 0 {
v.SetDefault(cfgListenDomains, *domains) v.SetDefault(cfgListenDomains, domains)
} }
switch {
case help != nil && *help:
fmt.Printf("FrostFS S3 gateway %s\n", version.Version)
flags.PrintDefaults()
fmt.Println()
fmt.Println("Default environments:")
fmt.Println()
keys := v.AllKeys()
sort.Strings(keys)
for i := range keys {
if _, ok := ignore[keys[i]]; ok {
continue
}
defaultValue := v.GetString(keys[i])
if len(defaultValue) == 0 {
continue
}
k := strings.Replace(keys[i], ".", "_", -1)
fmt.Printf("%s_%s = %s\n", envPrefix, strings.ToUpper(k), defaultValue)
}
fmt.Println()
fmt.Println("Peers preset:")
fmt.Println()
fmt.Printf("%s_%s_[N]_ADDRESS = string\n", envPrefix, strings.ToUpper(cfgPeers))
fmt.Printf("%s_%s_[N]_WEIGHT = 0..1 (float)\n", envPrefix, strings.ToUpper(cfgPeers))
os.Exit(0)
case versionFlag != nil && *versionFlag:
fmt.Printf("FrostFS S3 Gateway\nVersion: %s\nGoVersion: %s\n", version.Version, runtime.Version())
os.Exit(0)
}
if err := readInConfig(v); err != nil {
panic(err)
}
return v
} }
func bindFlags(v *viper.Viper, flags *pflag.FlagSet) error { func bindFlags(v *viper.Viper, flags *pflag.FlagSet) error {
@ -1175,129 +1298,19 @@ type LoggerAppSettings interface {
DroppedLogsInc() DroppedLogsInc()
} }
func pickLogger(v *viper.Viper, settings LoggerAppSettings) *Logger {
lvl, err := getLogLevel(v)
if err != nil {
panic(err)
}
dest := v.GetString(cfgLoggerDestination)
switch dest {
case destinationStdout:
return newStdoutLogger(v, lvl, settings)
case destinationJournald:
return newJournaldLogger(v, lvl, settings)
default:
panic(fmt.Sprintf("wrong destination for logger: %s", dest))
}
}
// newStdoutLogger constructs a Logger instance for the current application.
// Panics on failure.
//
// Logger contains a logger is built from zap's production logging configuration with:
// - parameterized level (debug by default)
// - console encoding
// - ISO8601 time encoding
// - sampling intervals
//
// and atomic log level to dynamically change it.
//
// Logger records a stack trace for all messages at or above fatal level.
//
// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace.
func newStdoutLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger {
stdout := zapcore.AddSync(os.Stderr)
level := zap.NewAtomicLevelAt(lvl)
consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level)
consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, settings)
return &Logger{
logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))),
lvl: level,
}
}
func newJournaldLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger {
level := zap.NewAtomicLevelAt(lvl)
encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields)
core := zapjournald.NewCore(level, encoder, &journald.Journal{}, zapjournald.SyslogFields)
coreWithContext := core.With([]zapcore.Field{
zapjournald.SyslogFacility(zapjournald.LogDaemon),
zapjournald.SyslogIdentifier(),
zapjournald.SyslogPid(),
})
coreWithContext = applyZapCoreMiddlewares(coreWithContext, v, settings)
l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
return &Logger{
logger: l,
lvl: level,
}
}
func newLogEncoder() zapcore.Encoder {
c := zap.NewProductionEncoderConfig()
c.EncodeTime = zapcore.ISO8601TimeEncoder
return zapcore.NewConsoleEncoder(c)
}
func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, settings LoggerAppSettings) zapcore.Core {
if v.GetBool(cfgLoggerSamplingEnabled) {
core = zapcore.NewSamplerWithOptions(core,
v.GetDuration(cfgLoggerSamplingInterval),
v.GetInt(cfgLoggerSamplingInitial),
v.GetInt(cfgLoggerSamplingThereafter),
zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) {
if dec&zapcore.LogDropped > 0 {
settings.DroppedLogsInc()
}
}))
}
return core
}
func getLogLevel(v *viper.Viper) (zapcore.Level, error) {
var lvl zapcore.Level
lvlStr := v.GetString(cfgLoggerLevel)
err := lvl.UnmarshalText([]byte(lvlStr))
if err != nil {
return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+
"value should be one of %v", lvlStr, err, [...]zapcore.Level{
zapcore.DebugLevel,
zapcore.InfoLevel,
zapcore.WarnLevel,
zapcore.ErrorLevel,
zapcore.DPanicLevel,
zapcore.PanicLevel,
zapcore.FatalLevel,
})
}
return lvl, nil
}
func validateDomains(domains []string, log *zap.Logger) []string { func validateDomains(domains []string, log *zap.Logger) []string {
validDomains := make([]string, 0, len(domains)) validDomains := make([]string, 0, len(domains))
LOOP: LOOP:
for _, domain := range domains { for _, domain := range domains {
if strings.Contains(domain, ":") { if strings.Contains(domain, ":") {
log.Warn(logs.WarnDomainContainsPort, zap.String("domain", domain)) log.Warn(logs.WarnDomainContainsPort, zap.String("domain", domain), logs.TagField(logs.TagApp))
continue continue
} }
domainParts := strings.Split(domain, ".") domainParts := strings.Split(domain, ".")
for _, part := range domainParts { for _, part := range domainParts {
if strings.ContainsAny(part, "<>") && part != wildcardPlaceholder { if strings.ContainsAny(part, "<>") && part != wildcardPlaceholder {
log.Warn(logs.WarnDomainContainsInvalidPlaceholder, zap.String("domain", domain)) log.Warn(logs.WarnDomainContainsInvalidPlaceholder, zap.String("domain", domain), logs.TagField(logs.TagApp))
continue LOOP continue LOOP
} }
} }

View file

@ -0,0 +1,59 @@
package main
import (
"os"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
"github.com/stretchr/testify/require"
)
func TestConfigReload(t *testing.T) {
f, err := os.CreateTemp("", "conf")
require.NoError(t, err)
defer func() {
require.NoError(t, os.Remove(f.Name()))
}()
confData := `
pprof:
enabled: true
frostfsid:
contract: name.nns
resolve_order:
- nns
`
_, err = f.WriteString(confData)
require.NoError(t, err)
require.NoError(t, f.Close())
cfg := newSettings()
require.NoError(t, cfg.flags.Parse([]string{"--config", f.Name(), "--max_clients_count", "10"}))
require.NoError(t, cfg.reload())
require.True(t, cfg.config().GetBool(cfgPProfEnabled))
require.Equal(t, "name.nns", cfg.config().GetString(cfgFrostfsIDContract))
require.Equal(t, []string{resolver.NNSResolver}, cfg.config().GetStringSlice(cfgResolveOrder))
require.Equal(t, 10, cfg.config().GetInt(cfgMaxClientsCount))
require.NoError(t, os.Truncate(f.Name(), 0))
require.NoError(t, cfg.reload())
require.False(t, cfg.config().GetBool(cfgPProfEnabled))
require.Equal(t, "frostfsid.frostfs", cfg.config().GetString(cfgFrostfsIDContract))
require.Equal(t, []string{resolver.DNSResolver}, cfg.config().GetStringSlice(cfgResolveOrder))
require.Equal(t, 10, cfg.config().GetInt(cfgMaxClientsCount))
}
func TestSetTLSEnabled(t *testing.T) {
cfg := newSettings()
require.NoError(t, cfg.flags.Parse([]string{"--" + cfgTLSCertFile, "tls.crt", "--" + cfgTLSKeyFile, "tls.key"}))
require.NoError(t, cfg.reload())
require.True(t, cfg.config().GetBool(cfgServer+".0."+cfgTLSEnabled))
}

210
cmd/s3-gw/logger.go Normal file
View file

@ -0,0 +1,210 @@
package main
import (
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/zapjournald"
"github.com/spf13/viper"
"github.com/ssgreg/journald"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
const (
destinationStdout string = "stdout"
destinationJournald string = "journald"
)
var defaultTags = []string{logs.TagApp, logs.TagDatapath, logs.TagExternalStorage, logs.TagExternalStorageTree, logs.TagExternalBlockchain}
type Logger struct {
logger *zap.Logger
lvl zap.AtomicLevel
}
func pickLogger(v *viper.Viper, loggerSettings LoggerAppSettings, tagSettings TagFilterSettings) *Logger {
dest := v.GetString(cfgLoggerDestination)
switch dest {
case destinationStdout:
return newStdoutLogger(v, loggerSettings, tagSettings)
case destinationJournald:
return newJournaldLogger(v, loggerSettings, tagSettings)
default:
panic(fmt.Sprintf("wrong destination for logger: %s", dest))
}
}
// newStdoutLogger constructs a Logger instance for the current application.
// Panics on failure.
//
// Logger contains a logger is built from zap's production logging configuration with:
// - parameterized level (debug by default)
// - console encoding
// - ISO8601 time encoding
// - sampling intervals
//
// and atomic log level to dynamically change it.
//
// Logger records a stack trace for all messages at or above fatal level.
//
// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace.
func newStdoutLogger(v *viper.Viper, loggerSettings LoggerAppSettings, tagSettings TagFilterSettings) *Logger {
c := newZapLogConfig(v)
out, errSink, err := openZapSinks(c)
if err != nil {
panic(fmt.Sprintf("open zap sinks: %v", err.Error()))
}
core := zapcore.NewCore(zapcore.NewConsoleEncoder(c.EncoderConfig), out, c.Level)
core = applyZapCoreMiddlewares(core, v, loggerSettings, tagSettings)
l := zap.New(core, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), zap.ErrorOutput(errSink))
return &Logger{logger: l, lvl: c.Level}
}
func newJournaldLogger(v *viper.Viper, logSettings LoggerAppSettings, tagSettings TagFilterSettings) *Logger {
c := newZapLogConfig(v)
// We can use NewJSONEncoder instead if, say, frontend
// would like to access journald logs and parse them easily.
encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields)
journalCore := zapjournald.NewCore(c.Level, encoder, &journald.Journal{}, zapjournald.SyslogFields)
core := journalCore.With([]zapcore.Field{
zapjournald.SyslogFacility(zapjournald.LogDaemon),
zapjournald.SyslogIdentifier(),
zapjournald.SyslogPid(),
})
core = applyZapCoreMiddlewares(core, v, logSettings, tagSettings)
l := zap.New(core, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
return &Logger{logger: l, lvl: c.Level}
}
func openZapSinks(cfg zap.Config) (zapcore.WriteSyncer, zapcore.WriteSyncer, error) {
sink, closeOut, err := zap.Open(cfg.OutputPaths...)
if err != nil {
return nil, nil, err
}
errSink, _, err := zap.Open(cfg.ErrorOutputPaths...)
if err != nil {
closeOut()
return nil, nil, err
}
return sink, errSink, nil
}
var _ zapcore.Core = (*zapCoreTagFilterWrapper)(nil)
type zapCoreTagFilterWrapper struct {
core zapcore.Core
settings TagFilterSettings
extra []zap.Field
}
type TagFilterSettings interface {
LevelEnabled(tag string, lvl zapcore.Level) bool
}
func (c *zapCoreTagFilterWrapper) Enabled(level zapcore.Level) bool {
return c.core.Enabled(level)
}
func (c *zapCoreTagFilterWrapper) With(fields []zapcore.Field) zapcore.Core {
return &zapCoreTagFilterWrapper{
core: c.core.With(fields),
settings: c.settings,
extra: append(c.extra, fields...),
}
}
func (c *zapCoreTagFilterWrapper) Check(entry zapcore.Entry, checked *zapcore.CheckedEntry) *zapcore.CheckedEntry {
if c.core.Enabled(entry.Level) {
return checked.AddCore(entry, c)
}
return checked
}
func (c *zapCoreTagFilterWrapper) Write(entry zapcore.Entry, fields []zapcore.Field) error {
if c.shouldSkip(entry, fields) || c.shouldSkip(entry, c.extra) {
return nil
}
return c.core.Write(entry, fields)
}
func (c *zapCoreTagFilterWrapper) shouldSkip(entry zapcore.Entry, fields []zap.Field) bool {
for _, field := range fields {
if field.Key == logs.TagFieldName && field.Type == zapcore.StringType {
if !c.settings.LevelEnabled(field.String, entry.Level) {
return true
}
break
}
}
return false
}
func (c *zapCoreTagFilterWrapper) Sync() error {
return c.core.Sync()
}
func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, appSettings LoggerAppSettings, tagSettings TagFilterSettings) zapcore.Core {
core = &zapCoreTagFilterWrapper{
core: core,
settings: tagSettings,
}
if v.GetBool(cfgLoggerSamplingEnabled) {
core = zapcore.NewSamplerWithOptions(core,
v.GetDuration(cfgLoggerSamplingInterval),
v.GetInt(cfgLoggerSamplingInitial),
v.GetInt(cfgLoggerSamplingThereafter),
zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) {
if dec&zapcore.LogDropped > 0 {
appSettings.DroppedLogsInc()
}
}))
}
return core
}
func newZapLogConfig(v *viper.Viper) zap.Config {
lvl, err := getLogLevel(v)
if err != nil {
panic(err)
}
c := zap.Config{
Level: zap.NewAtomicLevelAt(lvl),
EncoderConfig: zap.NewProductionEncoderConfig(),
OutputPaths: []string{"stderr"},
ErrorOutputPaths: []string{"stderr"},
}
c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
return c
}
func getLogLevel(v *viper.Viper) (zapcore.Level, error) {
var lvl zapcore.Level
lvlStr := v.GetString(cfgLoggerLevel)
err := lvl.UnmarshalText([]byte(lvlStr))
if err != nil {
return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+
"value should be one of %v", lvlStr, err, [...]zapcore.Level{
zapcore.DebugLevel,
zapcore.InfoLevel,
zapcore.WarnLevel,
zapcore.ErrorLevel,
zapcore.DPanicLevel,
zapcore.PanicLevel,
zapcore.FatalLevel,
})
}
return lvl, nil
}

View file

@ -8,9 +8,9 @@ import (
func main() { func main() {
g, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) g, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
v := newSettings() cfg := newSettings()
a := newApp(g, v) a := newApp(g, cfg)
go a.Serve(g) go a.Serve(g)

View file

@ -19,24 +19,24 @@ type Service struct {
// Start runs http service with the exposed endpoint on the configured port. // Start runs http service with the exposed endpoint on the configured port.
func (ms *Service) Start() { func (ms *Service) Start() {
if ms.enabled { if ms.enabled {
ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr)) ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr), logs.TagField(logs.TagApp))
err := ms.ListenAndServe() err := ms.ListenAndServe()
if err != nil && err != http.ErrServerClosed { if err != nil && err != http.ErrServerClosed {
ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort) ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort, logs.TagField(logs.TagApp))
} }
} else { } else {
ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled) ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled, logs.TagField(logs.TagApp))
} }
} }
// ShutDown stops the service. // ShutDown stops the service.
func (ms *Service) ShutDown(ctx context.Context) { func (ms *Service) ShutDown(ctx context.Context) {
ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr)) ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr), logs.TagField(logs.TagApp))
err := ms.Shutdown(ctx) err := ms.Shutdown(ctx)
if err != nil { if err != nil {
ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err)) ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err), logs.TagField(logs.TagApp))
if err = ms.Close(); err != nil { if err = ms.Close(); err != nil {
ms.log.Panic(logs.CantShutDownService, zap.Error(err)) ms.log.Panic(logs.CantShutDownService, zap.Error(err), logs.TagField(logs.TagApp))
} }
} }
} }

View file

@ -55,6 +55,11 @@ S3_GW_LOGGER_SAMPLING_ENABLED=false
S3_GW_LOGGER_SAMPLING_INITIAL=100 S3_GW_LOGGER_SAMPLING_INITIAL=100
S3_GW_LOGGER_SAMPLING_THEREAFTER=100 S3_GW_LOGGER_SAMPLING_THEREAFTER=100
S3_GW_LOGGER_SAMPLING_INTERVAL=1s S3_GW_LOGGER_SAMPLING_INTERVAL=1s
S3_GW_LOGGER_TAGS_0_NAME=app
S3_GW_LOGGER_TAGS_0_LEVEL=info
S3_GW_LOGGER_TAGS_1_NAME=datapath
S3_GW_LOGGER_TAGS_1_LEVEL=fatal
# HTTP logger # HTTP logger
S3_GW_HTTP_LOGGING_ENABLED=false S3_GW_HTTP_LOGGING_ENABLED=false
@ -80,8 +85,10 @@ S3_GW_PROMETHEUS_ADDRESS=localhost:8086
# Timeout to connect to a node # Timeout to connect to a node
S3_GW_CONNECT_TIMEOUT=10s S3_GW_CONNECT_TIMEOUT=10s
# Timeout for individual operations in streaming RPC. # Timeout for individual operations in object pool streaming RPC.
S3_GW_STREAM_TIMEOUT=10s S3_GW_STREAM_TIMEOUT=10s
# Timeout for individual operations in tree pool streaming RPC.
S3_GW_TREE_STREAM_TIMEOUT=10s
# Timeout to check node health during rebalance. # Timeout to check node health during rebalance.
S3_GW_HEALTHCHECK_TIMEOUT=15s S3_GW_HEALTHCHECK_TIMEOUT=15s
# Interval to check node health # Interval to check node health

View file

@ -60,6 +60,12 @@ logger:
initial: 100 initial: 100
thereafter: 100 thereafter: 100
interval: 1s interval: 1s
tags:
- name: "app"
level: "debug"
- name: "datapath"
- name: "external_storage"
- name: "external_storage_tree"
# log http request data (URI, headers, query, etc) # log http request data (URI, headers, query, etc)
http_logging: http_logging:
@ -100,8 +106,10 @@ tracing:
# Timeout to connect to a node # Timeout to connect to a node
connect_timeout: 10s connect_timeout: 10s
# Timeout for individual operations in streaming RPC. # Timeout for individual operations in object pool streaming RPC.
stream_timeout: 10s stream_timeout: 10s
# Timeout for individual operations in tree pool streaming RPC.
tree_stream_timeout: 10s
# Timeout to check node health during rebalance # Timeout to check node health during rebalance
healthcheck_timeout: 15s healthcheck_timeout: 15s
# Interval to check node health # Interval to check node health

View file

@ -202,7 +202,7 @@ func (c *cred) checkIfCredentialsAreRemoved(ctx context.Context, cnrID cid.ID, a
func (c *cred) putBoxToCache(accessKeyID string, val *cache.AccessBoxCacheValue) { func (c *cred) putBoxToCache(accessKeyID string, val *cache.AccessBoxCacheValue) {
if err := c.cache.Put(accessKeyID, val); err != nil { if err := c.cache.Put(accessKeyID, val); err != nil {
c.log.Warn(logs.CouldntPutAccessBoxIntoCache, zap.String("accessKeyID", accessKeyID)) c.log.Warn(logs.CouldntPutAccessBoxIntoCache, zap.String("accessKeyID", accessKeyID), logs.TagField(logs.TagDatapath))
} }
} }
@ -241,7 +241,7 @@ func (c *cred) getAccessBox(ctx context.Context, cnrID cid.ID, accessKeyID strin
func (c *cred) Put(ctx context.Context, prm CredentialsParam) (oid.Address, error) { func (c *cred) Put(ctx context.Context, prm CredentialsParam) (oid.Address, error) {
if prm.AccessKeyID != "" { if prm.AccessKeyID != "" {
c.log.Info(logs.CheckCustomAccessKeyIDUniqueness, zap.String("access_key_id", prm.AccessKeyID)) c.log.Info(logs.CheckCustomAccessKeyIDUniqueness, zap.String("access_key_id", prm.AccessKeyID), logs.TagField(logs.TagApp))
credsPrm := PrmGetCredsObject{ credsPrm := PrmGetCredsObject{
Container: prm.Container, Container: prm.Container,
AccessKeyID: prm.AccessKeyID, AccessKeyID: prm.AccessKeyID,

View file

@ -64,6 +64,8 @@ $ frostfs-s3-gw --listen_address 192.168.130.130:443 \
Using these flag you can configure only one address. To set multiple addresses use yaml config. Using these flag you can configure only one address. To set multiple addresses use yaml config.
**Note:** It's not recommended to configure addresses via flags and yaml config at the same time.
### RPC endpoint and resolving of bucket names ### RPC endpoint and resolving of bucket names
To set RPC endpoint specify a value of parameter `-r` or `--rpc_endpoint`. The parameter is **required if** another To set RPC endpoint specify a value of parameter `-r` or `--rpc_endpoint`. The parameter is **required if** another
@ -213,6 +215,7 @@ resolve_order:
connect_timeout: 10s connect_timeout: 10s
stream_timeout: 10s stream_timeout: 10s
tree_stream_timeout: 10s
healthcheck_timeout: 15s healthcheck_timeout: 15s
rebalance_interval: 60s rebalance_interval: 60s
pool_error_threshold: 100 pool_error_threshold: 100
@ -235,7 +238,8 @@ source_ip_header: "Source-Ip"
| `rpc_endpoint` | `string` | no | | The address of the RPC host to which the gateway connects to resolve bucket names and interact with frostfs contracts (required to use the `nns` resolver and `frostfsid` contract). | | `rpc_endpoint` | `string` | no | | The address of the RPC host to which the gateway connects to resolve bucket names and interact with frostfs contracts (required to use the `nns` resolver and `frostfsid` contract). |
| `resolve_order` | `[]string` | yes | `[dns]` | Order of bucket name resolvers to use. Available resolvers: `dns`, `nns`. | | `resolve_order` | `[]string` | yes | `[dns]` | Order of bucket name resolvers to use. Available resolvers: `dns`, `nns`. |
| `connect_timeout` | `duration` | no | `10s` | Timeout to connect to a node. | | `connect_timeout` | `duration` | no | `10s` | Timeout to connect to a node. |
| `stream_timeout` | `duration` | no | `10s` | Timeout for individual operations in streaming RPC. | | `stream_timeout` | `duration` | no | `10s` | Timeout for individual operations in object pool streaming RPC. |
| `tree_stream_timeout` | `duration` | no | `10s` | Timeout for individual operations in tree pool streaming RPC. |
| `healthcheck_timeout` | `duration` | no | `15s` | Timeout to check node health during rebalance. | | `healthcheck_timeout` | `duration` | no | `15s` | Timeout to check node health during rebalance. |
| `rebalance_interval` | `duration` | no | `60s` | Interval to check node health. | | `rebalance_interval` | `duration` | no | `60s` | Interval to check node health. |
| `pool_error_threshold` | `uint32` | no | `100` | The number of errors on connection after which node is considered as unhealthy. | | `pool_error_threshold` | `uint32` | no | `100` | The number of errors on connection after which node is considered as unhealthy. |
@ -377,6 +381,13 @@ logger:
initial: 100 initial: 100
thereafter: 100 thereafter: 100
interval: 1s interval: 1s
tags:
- name: app
level: info
- name: datapath
- name: external_blockchain
- name: external_storage_tree
- name: external_storage
``` ```
| Parameter | Type | SIGHUP reload | Default value | Description | | Parameter | Type | SIGHUP reload | Default value | Description |
@ -388,6 +399,32 @@ logger:
| `sampling.thereafter` | `int` | no | '100' | Sampling count of entries after an `interval`. | | `sampling.thereafter` | `int` | no | '100' | Sampling count of entries after an `interval`. |
| `sampling.interval` | `duration` | no | '1s' | Sampling interval of messaging similar entries. | | `sampling.interval` | `duration` | no | '1s' | Sampling interval of messaging similar entries. |
## Tags
There are additional log entries that can hurt performance and can be additionally logged by using `logger.tags`
parameter.
If section `tags` isn't set the default tags (see [Tag values](#tag-values)) be enabled.
If section `tags` set but empty no tags be used.
```yaml
tags:
- name: "app"
level: info
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|-----------------------|------------|---------------|---------------------------|-------------------------------------------------------------------------------------------------------|
| `name` | `string` | yes | | Tag name. Possible values see below in `Tag values` section. |
| `level` | `string` | yes | Value from `logger.level` | Logging level for specific tag. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. |
### Tag values
* `app` - common application logs (enabled by default).
* `datapath` - main logic of application (enabled by default).
* `external_blockchain` - external interaction with neo-go blockchain (enabled by default).
* `external_storage` - external interaction with storage node (enabled by default).
* `external_storage_tree` - external interaction with tree service in storage node (enabled by default).
### `http_logging` section ### `http_logging` section

47
go.mod
View file

@ -5,13 +5,15 @@ go 1.22
require ( require (
git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.1-0.20241022094040-5f956751d48b git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.1-0.20241022094040-5f956751d48b
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
github.com/aws/aws-sdk-go-v2 v1.30.5 github.com/aws/aws-sdk-go-v2 v1.34.0
github.com/aws/aws-sdk-go-v2/config v1.27.32 github.com/aws/aws-sdk-go-v2/config v1.27.32
github.com/aws/aws-sdk-go-v2/credentials v1.17.31 github.com/aws/aws-sdk-go-v2/credentials v1.17.31
github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1
github.com/aws/smithy-go v1.22.2
github.com/bluele/gcache v0.0.2 github.com/bluele/gcache v0.0.2
github.com/go-chi/chi/v5 v5.0.8 github.com/go-chi/chi/v5 v5.0.8
github.com/google/uuid v1.6.0 github.com/google/uuid v1.6.0
@ -28,16 +30,16 @@ require (
github.com/stretchr/testify v1.9.0 github.com/stretchr/testify v1.9.0
github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4
github.com/urfave/cli/v2 v2.27.2 github.com/urfave/cli/v2 v2.27.2
go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel v1.31.0
go.opentelemetry.io/otel/trace v1.28.0 go.opentelemetry.io/otel/trace v1.31.0
go.uber.org/zap v1.27.0 go.uber.org/zap v1.27.0
golang.org/x/crypto v0.24.0 golang.org/x/crypto v0.31.0
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
golang.org/x/net v0.26.0 golang.org/x/net v0.30.0
golang.org/x/sys v0.22.0 golang.org/x/sys v0.28.0
golang.org/x/text v0.16.0 golang.org/x/text v0.21.0
google.golang.org/grpc v1.66.2 google.golang.org/grpc v1.69.2
google.golang.org/protobuf v1.34.2 google.golang.org/protobuf v1.36.1
gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1
) )
@ -48,16 +50,19 @@ require (
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect
github.com/VictoriaMetrics/easyproto v0.1.4 // indirect github.com/VictoriaMetrics/easyproto v0.1.4 // indirect
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.22.6 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.22.6 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.6 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.30.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.30.6 // indirect
github.com/aws/smithy-go v1.20.4 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect
@ -108,14 +113,14 @@ require (
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/otel/metric v1.31.0 // indirect
go.opentelemetry.io/otel/sdk v1.28.0 // indirect go.opentelemetry.io/otel/sdk v1.31.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect
go.uber.org/multierr v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect
golang.org/x/sync v0.7.0 // indirect golang.org/x/sync v0.10.0 // indirect
golang.org/x/term v0.21.0 // indirect golang.org/x/term v0.27.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.2.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect

98
go.sum
View file

@ -42,8 +42,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI=
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae h1:7gvuOTmS3oaOM79JkHWWlsvGqIRqsum5KnOI1TYqfn0= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae/go.mod h1:dbWUc5jOBTXVvssCLCYxkkSTL9jgLr1KruGP2FMAfiM= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8=
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8=
@ -62,32 +62,42 @@ github.com/VictoriaMetrics/easyproto v0.1.4 h1:r8cNvo8o6sR4QShBXQd1bKw/VVLSQma/V
github.com/VictoriaMetrics/easyproto v0.1.4/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo3/GThPs2KH23mv710= github.com/VictoriaMetrics/easyproto v0.1.4/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo3/GThPs2KH23mv710=
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g= github.com/aws/aws-sdk-go-v2 v1.34.0 h1:9iyL+cjifckRGEVpRKZP3eIxVlL06Qk1Tk13vreaVQU=
github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= github.com/aws/aws-sdk-go-v2 v1.34.0/go.mod h1:JgstGg0JjWU1KpVJjD5H0y0yyAIpSdKEq556EI6yOOM=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg=
github.com/aws/aws-sdk-go-v2/config v1.27.32 h1:jnAMVTJTpAQlePCUUlnXnllHEMGVWmvUJOiGjgtS9S0= github.com/aws/aws-sdk-go-v2/config v1.27.32 h1:jnAMVTJTpAQlePCUUlnXnllHEMGVWmvUJOiGjgtS9S0=
github.com/aws/aws-sdk-go-v2/config v1.27.32/go.mod h1:JibtzKJoXT0M/MhoYL6qfCk7nm/MppwukDFZtdgVRoY= github.com/aws/aws-sdk-go-v2/config v1.27.32/go.mod h1:JibtzKJoXT0M/MhoYL6qfCk7nm/MppwukDFZtdgVRoY=
github.com/aws/aws-sdk-go-v2/credentials v1.17.31 h1:jtyfcOfgoqWA2hW/E8sFbwdfgwD3APnF9CLCKE8dTyw= github.com/aws/aws-sdk-go-v2/credentials v1.17.31 h1:jtyfcOfgoqWA2hW/E8sFbwdfgwD3APnF9CLCKE8dTyw=
github.com/aws/aws-sdk-go-v2/credentials v1.17.31/go.mod h1:RSgY5lfCfw+FoyKWtOpLolPlfQVdDBQWTUniAaE+NKY= github.com/aws/aws-sdk-go-v2/credentials v1.17.31/go.mod h1:RSgY5lfCfw+FoyKWtOpLolPlfQVdDBQWTUniAaE+NKY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 h1:Ej0Rf3GMv50Qh4G4852j2djtoDb7AzQ7MuQeFHa3D70=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29/go.mod h1:oeNTC7PwJNoM5AznVr23wxhLnuJv0ZDe5v7w0wqIs9M=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 h1:6e8a71X+9GfghragVevC5bZqvATtc3mAMgxpSNbgzF0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29/go.mod h1:c4jkZiQ+BWpNqq7VtrxjwISrLrt/VvPq3XiopkUIolI=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 h1:g9OUETuxA8i/Www5Cby0R3WSTe7ppFTZXHVLNskNS4w=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29/go.mod h1:CQk+koLR1QeY1+vm7lqNfFii07DEderKq6T3F1L2pyc=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 h1:EP1ITDgYVPM2dL1bBBntJ7AW5yTjuWGz9XO+CZwpALU=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3/go.mod h1:5lWNWeAgWenJ/BZ/CP9k9DjLbC0pjnM045WjXRPPi14=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 h1:hN4yJBGswmFTOVYqmbz1GBs9ZMtQe8SrYxPwrkrlRv8=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10/go.mod h1:TsxON4fEZXyrKY+D+3d2gSTyJkGORexIYab9PTf56DA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 h1:fXoWC2gi7tdJYNTPnnlSGzEVwewUchOi8xVq/dkg8Qs=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10/go.mod h1:cvzBApD5dVazHU8C2rbBQzzzsKc8m5+wNJ9mCRZLKPc=
github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1 h1:9LawY3cDJ3HE+v2GMd5SOkNLDwgN4K7TsCjyVBYu/L4=
github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1/go.mod h1:hHnELVnIHltd8EOF3YzahVX6F6y2C6dNqpRj1IMkS5I=
github.com/aws/aws-sdk-go-v2/service/sso v1.22.6 h1:o++HUDXlbrTl4PSal3YHtdErQxB8mDGAtkKNXBWPfIU= github.com/aws/aws-sdk-go-v2/service/sso v1.22.6 h1:o++HUDXlbrTl4PSal3YHtdErQxB8mDGAtkKNXBWPfIU=
github.com/aws/aws-sdk-go-v2/service/sso v1.22.6/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ= github.com/aws/aws-sdk-go-v2/service/sso v1.22.6/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.6 h1:yCHcQCOwTfIsc8DoEhM3qXPxD+j8CbI6t1K3dNzsWV0= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.6 h1:yCHcQCOwTfIsc8DoEhM3qXPxD+j8CbI6t1K3dNzsWV0=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.6/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.6/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.6 h1:TrQadF7GcqvQ63kgwEcjlrVc2Fa0wpgLT0xtc73uAd8= github.com/aws/aws-sdk-go-v2/service/sts v1.30.6 h1:TrQadF7GcqvQ63kgwEcjlrVc2Fa0wpgLT0xtc73uAd8=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.6/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o= github.com/aws/aws-sdk-go-v2/service/sts v1.30.6/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o=
github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c=
@ -164,6 +174,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@ -365,20 +377,22 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y=
go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@ -396,8 +410,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -469,8 +483,8 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -490,8 +504,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -535,19 +549,19 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@ -668,10 +682,10 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U=
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@ -688,8 +702,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU=
google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@ -700,8 +714,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

View file

@ -61,7 +61,7 @@ func (f *FrostFSID) GetUserGroupIDsAndClaims(userHash util.Uint160) ([]string, m
subj, err := f.getSubject(userHash) subj, err := f.getSubject(userHash)
if err != nil { if err != nil {
if strings.Contains(err.Error(), "not found") { if strings.Contains(err.Error(), "not found") {
f.log.Debug(logs.UserGroupsListIsEmpty, zap.Error(err)) f.log.Debug(logs.UserGroupsListIsEmpty, zap.Error(err), logs.TagField(logs.TagExternalBlockchain))
return nil, nil, nil return nil, nil, nil
} }
return nil, nil, err return nil, nil, err
@ -86,7 +86,7 @@ func (f *FrostFSID) getSubject(addr util.Uint160) (*client.SubjectExtended, erro
} }
if err = f.cache.PutSubject(addr, subj); err != nil { if err = f.cache.PutSubject(addr, subj); err != nil {
f.log.Warn(logs.CouldntCacheSubject, zap.Error(err)) f.log.Warn(logs.CouldntCacheSubject, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
return subj, nil return subj, nil
@ -121,7 +121,7 @@ func (f *FrostFSID) getUserKey(namespace, name string) (*keys.PublicKey, error)
} }
if err = f.cache.PutUserKey(namespace, name, userKey); err != nil { if err = f.cache.PutUserKey(namespace, name, userKey); err != nil {
f.log.Warn(logs.CouldntCacheUserKey, zap.Error(err)) f.log.Warn(logs.CouldntCacheUserKey, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
return userKey, nil return userKey, nil

View file

@ -69,7 +69,7 @@ func (c *MorphRuleChainStorage) ListMorphRuleChains(name chain.Name, target engi
} }
if err = c.cache.Put(key, list); err != nil { if err = c.cache.Put(key, list); err != nil {
c.log.Warn(logs.CouldntCacheListPolicyChains) c.log.Warn(logs.CouldntCacheListPolicyChains, logs.TagField(logs.TagApp))
} }
return list, nil return list, nil

View file

@ -1,189 +1,230 @@
package logs package logs
import "go.uber.org/zap"
const ( const (
RequestUnmatched = "request unmatched" // Error in ../../api/router.go TagFieldName = "tag"
CheckContainer = "check container" // Info in ../../authmate/authmate.go
CreateContainer = "create container" // Info in ../../authmate/authmate.go TagApp = "app"
StoreBearerTokenIntoFrostFS = "store bearer token into FrostFS" // Info in ../../authmate/authmate.go TagDatapath = "datapath"
UpdateAccessCredObjectIntoFrostFS = "update access cred object into FrostFS" // Info in ../../authmate/authmate.go TagExternalStorage = "external_storage"
MetricsAreDisabled = "metrics are disabled" // Warn in ../../metrics/app.go TagExternalStorageTree = "external_storage_tree"
FoundMoreThanOneUnversionedNode = "found more than one unversioned node" // Debug in ../../pkg/service/tree/tree.go TagExternalBlockchain = "external_blockchain"
ServiceIsRunning = "service is running" // Info in ../../cmd/s3-gw/service.go )
ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../cmd/s3-gw/service.go
ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../cmd/s3-gw/service.go func TagField(tag string) zap.Field {
ShuttingDownService = "shutting down service" // Info in ../../cmd/s3-gw/service.go return zap.String(TagFieldName, tag)
CantGracefullyShutDownService = "can't gracefully shut down service, force stop" // Error in ../../cmd/s3-gw/service.go }
ContainerResolverWillBeDisabled = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../cmd/s3-gw/app.go
FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../cmd/s3-gw/app.go // App.
TracingConfigUpdated = "tracing config updated" // Info in ../../cmd/s3-gw/app.go const (
FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../cmd/s3-gw/app.go ApplicationStarted = "application started"
UsingCredentials = "using credentials" // Info in ../../cmd/s3-gw/app.go ApplicationFinished = "application finished"
ApplicationStarted = "application started" // Info in ../../cmd/s3-gw/app.go StartingServer = "starting server"
ApplicationFinished = "application finished" // Info in ../../cmd/s3-gw/app.go StoppingServer = "stopping server"
StartingServer = "starting server" // Info in ../../cmd/s3-gw/app.go ServiceIsRunning = "service is running"
StoppingServer = "stopping server" // Info in ../../cmd/s3-gw/app.go ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port"
SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../cmd/s3-gw/app.go ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled"
FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../cmd/s3-gw/app.go ShuttingDownService = "shutting down service"
FailedToReloadConfig = "failed to reload config" // Warn in ../../cmd/s3-gw/app.go CantGracefullyShutDownService = "can't gracefully shut down service, force stop"
FailedToReloadResolvers = "failed to reload resolvers" // Warn in ../../cmd/s3-gw/app.go FailedToShutdownTracing = "failed to shutdown tracing"
FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../cmd/s3-gw/app.go UsingCredentials = "using credentials"
SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../cmd/s3-gw/app.go FailedToAddServer = "failed to add server"
LogLevelWontBeUpdated = "log level won't be updated" // Warn in ../../cmd/s3-gw/app.go AddServer = "add server"
FailedToAddServer = "failed to add server" // Warn in ../../cmd/s3-gw/app.go CantShutDownService = "can't shut down service"
AddServer = "add server" // Info in ../../cmd/s3-gw/app.go FailedToCreateResolver = "failed to create resolver"
ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver 'nns' won't be used since 'rpc_endpoint' isn't provided" // Warn in ../../cmd/s3-gw/app.go CouldntGenerateRandomKey = "couldn't generate random key"
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/s3-gw/app_settings.go FailedToCreateConnectionPool = "failed to create connection pool"
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/s3-gw/app_settings.go FailedToDialConnectionPool = "failed to dial connection pool"
FailedToParseDefaultLocationConstraint = "failed to parse 'default' location constraint, default one will be used" // Warn in cmd/s3-gw/app_settings.go FailedToCreateTreePool = "failed to create tree pool"
FailedToReadRegionMapFilePolicies = "failed to read region map file, policies will be empty" // Warn in cmd/s3-gw/app_settings.go FailedToDialTreePool = "failed to dial tree pool"
DefaultLocationConstraintCantBeOverriden = "'default' location constraint can't be overriden by custom policy, use 'placement_policy.default'" // Warn in cmd/s3-gw/app_settings.go ListenAndServe = "listen and serve"
FailedToParseLocationConstraint = "failed to parse location constraint, it cannot be used" // Warn in cmd/s3-gw/app_settings.go NoHealthyServers = "no healthy servers"
FailedToParseDefaultCopiesNumbers = "failed to parse 'default' copies numbers, default one will be used" // Warn in cmd/s3-gw/app_settings.go CouldNotInitializeAPIHandler = "could not initialize API handler"
FailedToParseCopiesNumbers = "failed to parse copies numbers, skip" // Warn in cmd/s3-gw/app_settings.go InitFrostfsIDFailed = "init frostfsid failed"
DefaultNamespacesCannotBeEmpty = "default namespaces cannot be empty, defaults will be used" // Warn in cmd/s3-gw/app_settings.go InitPolicyContractFailed = "init policy contract failed"
FailedToParseNamespacesConfig = "failed to unmarshal namespaces config" // Warn in cmd/s3-gw/app_settings.go
DefaultNamespaceConfigValuesBeOverwritten = "default namespace config value be overwritten by values from 'namespaces.config'" // Warn in cmd/s3-gw/app_settings.go
MultipleDefaultOverridesFound = "multiple default overrides found, only one will be used" // Warn in cmd/s3-gw/app_settings.go
FailedToParseDefaultDefaultLocationConstraint = "failed to parse default 'default' location constraint" // Fatal in cmd/s3-gw/app_settings.go
ConstraintAdded = "constraint added" // Info in ../../cmd/s3-gw/app_settings.go
SkipEmptyAddress = "skip, empty address" // Warn in ../../cmd/s3-gw/app_settings.go
AddedStoragePeer = "added storage peer" // Info in ../../cmd/s3-gw/app_settings.go
PrepareConnectionPool = "prepare connection pool" // Debug in ../../cmd/s3-authmate/modules/utils.go
PrepareFrostfsIDClient = "prepare frostfsid client" // Debug in ../../cmd/s3-authmate/modules/utils.go
PreparePolicyClient = "prepare policy client" // Debug in ../../cmd/s3-authmate/modules/utils.go
CreateSubjectInFrostFSID = "create subject in frostfsid" // Debug in ../../cmd/s3-authmate/modules/utils.go
SubjectAlreadyExistsInFrostFSID = "subject already exists in frostfsid" // Debug in ../../cmd/s3-authmate/modules/utils.go
SetSubjectNameInFrostFSID = "set subject name in frostfsid" // Debug in ../../cmd/s3-authmate/modules/utils.go
AddPolicyChainRules = "add policy chain rules" // Debug in ../../cmd/s3-authmate/modules/utils.go
InvalidCacheEntryType = "invalid cache entry type" // Warn in ../../api/cache/*
InvalidCacheKeyType = "invalid cache key type" // Warn in ../../api/cache/objectslist.go
ObjectIsCopied = "object is copied" // Info in ../../api/handler/copy.go
RequestFailed = "request failed" // Error in ../../api/handler/util.go
GetBucketInfo = "get bucket info" // Warn in ../../api/handler/cors.go
GetBucketCors = "get bucket cors" // Warn in ../../api/handler/cors.go
CouldntDeleteObject = "couldn't delete object" // Error in ../../api/layer/layer.go
BucketIsCreated = "bucket is created" // Info in ../../api/handler/put.go
CouldNotParseContainerObjectLockEnabledAttribute = "could not parse container object lock enabled attribute" // Error in ../../api/layer/container.go
CouldNotListUserContainers = "could not list user containers" // Error in ../../api/layer/container.go
CouldNotFetchContainerInfo = "could not fetch container info" // Error in ../../api/layer/container.go
MismatchedObjEncryptionInfo = "mismatched obj encryptionInfo" // Warn in ../../api/layer/multipart_upload.go
UploadPart = "upload part" // Debug in ../../api/layer/multipart_upload.go
CouldntDeleteOldPartObject = "couldn't delete old part object" // Error in ../../api/layer/multipart_upload.go
CouldNotPutCompletedObject = "could not put a completed object (multipart upload)" // Error in ../../api/layer/multipart_upload.go
CouldNotDeleteUploadPart = "could not delete upload part" // Warn in ../../api/layer/multipart_upload.go
CouldntDeletePart = "couldn't delete part" // Warn in ../../api/layer/multipart_upload.go
PartDetails = "part details" // Debug in ../../api/layer/multipart_upload.go
GetObject = "get object" // Debug in ../../api/layer/layer.go
ResolveBucket = "resolve bucket" // Info in ../../api/layer/layer.go
CouldntDeleteCorsObject = "couldn't delete cors object" // Error in ../../api/layer/cors.go
PutObject = "put object" // Debug in ../../api/layer/object.go
FailedToDeleteObject = "failed to delete object" // Debug in ../../api/layer/object.go
FailedToDiscardPutPayloadProbablyGoroutineLeaks = "failed to discard put payload, probably goroutine leaks" // Warn in ../../api/layer/object.go
FailedToSubmitTaskToPool = "failed to submit task to pool" // Warn in ../../api/layer/object.go
CouldNotFetchObjectMeta = "could not fetch object meta" // Warn in ../../api/layer/object.go
GetTreeNode = "get tree node" // Debug in ../../api/layer/tagging.go
GetTreeNodeToDelete = "get tree node to delete" // Debug in ../../api/layer/tagging.go
CouldntPutBucketInfoIntoCache = "couldn't put bucket info into cache" // Warn in ../../api/layer/cache.go
CouldntAddObjectToCache = "couldn't add object to cache" // Warn in ../../api/layer/cache.go
CouldntCacheAccessControlOperation = "couldn't cache access control operation" // Warn in ../../api/layer/cache.go
CouldntPutObjAddressToNameCache = "couldn't put obj address to name cache" // Warn in ../../api/layer/cache.go
CouldntCacheListOfObjects = "couldn't cache list of objects" // Warn in ../../api/layer/cache.go
CouldntCacheListSession = "couldn't cache list session" // Warn in ../../api/layer/cache.go
CouldntCacheTags = "couldn't cache tags" // Error in ../../api/layer/cache.go
CouldntCacheLockInfo = "couldn't cache lock info" // Error in ../../api/layer/cache.go
CouldntCacheBucketSettings = "couldn't cache bucket settings" // Warn in ../../api/layer/cache.go
CouldntCacheCors = "couldn't cache cors" // Warn in ../../api/layer/cache.go
CouldntCacheListPolicyChains = "couldn't cache list policy chains" // Warn in ../../api/layer/cache.go
RequestEnd = "request end" // Info in ../../api/middleware/response.go
CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed = "couldn't receive access box for gate key, random key will be used" // Debug in ../../api/middleware/auth.go
FailedToPassAuthentication = "failed to pass authentication" // Error in ../../api/middleware/auth.go
FailedToResolveCID = "failed to resolve CID" // Debug in ../../api/middleware/metrics.go
RequestStart = "request start" // Info in ../../api/middleware/reqinfo.go
LogHTTP = "http log" // Info in ../../api/middleware/log_http.go
FailedToCloseHTTPBody = "failed to close http body" // Error in ../../api/middleware/log_http.go
FailedToInitializeHTTPLogger = "failed to initialize http logger" // Error in ../../api/middleware/log_http.go
FailedToReloadHTTPFileLogger = "failed to reload http file logger" // Error in ../../api/middleware/log_http.go
FailedToReadHTTPBody = "failed to read http body" // Error in ../../api/middleware/log_http.go
FailedToProcessHTTPBody = "failed to process http body" // Error in ../../api/middleware/log_http.go
LogHTTPDisabledInThisBuild = "http logging disabled in this build" // Warn in ../../api/middleware/log_http_stub.go
FailedToUnescapeObjectName = "failed to unescape object name" // Warn in ../../api/middleware/reqinfo.go
InvalidDefaultMaxAge = "invalid defaultMaxAge" // Fatal in ../../cmd/s3-gw/app_settings.go
CantShutDownService = "can't shut down service" // Panic in ../../cmd/s3-gw/service.go
CouldntGenerateRandomKey = "couldn't generate random key" // Fatal in ../../cmd/s3-gw/app.go
FailedToCreateResolver = "failed to create resolver" // Fatal in ../../cmd/s3-gw/app.go
CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../cmd/s3-gw/app.go
FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../cmd/s3-gw/app.go
FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../cmd/s3-gw/app.go
FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../cmd/s3-gw/app.go
FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../cmd/s3-gw/app.go
ListenAndServe = "listen and serve" // Fatal in ../../cmd/s3-gw/app.go
NoHealthyServers = "no healthy servers" // Fatal in ../../cmd/s3-gw/app.go
CouldNotInitializeAPIHandler = "could not initialize API handler" // Fatal in ../../cmd/s3-gw/app.go
RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" // Warn in ../../cmd/s3-gw/app.go
RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" // Info in ../../cmd/s3-gw/app.go
AnonRequestSkipFrostfsIDValidation = "anon request, skip FrostfsID validation" // Debug in ../../api/middleware/auth.go
FrostfsIDValidationFailed = "FrostfsID validation failed" // Error in ../../api/middleware/auth.go
InitFrostfsIDContractFailed = "init frostfsid contract failed" // Fatal in ../../cmd/s3-gw/app.go
InitPolicyContractFailed = "init policy contract failed" // Fatal in ../../cmd/s3-gw/app.go
PolicyValidationFailed = "policy validation failed"
ServerReconnecting = "reconnecting server..." ServerReconnecting = "reconnecting server..."
ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectedSuccessfully = "server reconnected successfully"
ServerReconnectFailed = "failed to reconnect server" ServerReconnectFailed = "failed to reconnect server"
ParseTreeNode = "parse tree node" CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info"
FailedToGetRealObjectSize = "failed to get real object size" CouldNotFetchAccessBoxContainerInfo = "couldn't fetch AccessBox container info"
CouldntDeleteObjectFromStorageContinueDeleting = "couldn't delete object from storage, continue deleting from tree" MultinetDialSuccess = "multinet dial successful"
CouldntPutAccessBoxIntoCache = "couldn't put accessbox into cache" MultinetDialFail = "multinet dial failed"
FailedToCreateWorkerPool = "failed to create worker pool"
CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key"
AddedStoragePeer = "added storage peer"
SIGHUPConfigReloadStarted = "SIGHUP config reload started"
MetricsAreDisabled = "metrics are disabled"
ConstraintAdded = "constraint added"
ContainerResolverWillBeDisabled = "container resolver will be disabled because of resolvers 'resolver_order' is empty"
FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed"
FailedToReloadConfig = "failed to reload config"
TracingConfigUpdated = "tracing config updated"
FailedToReloadResolvers = "failed to reload resolvers"
FailedToReloadServerParameters = "failed to reload server parameters"
SIGHUPConfigReloadCompleted = "SIGHUP config reload completed"
LogLevelWontBeUpdated = "log level won't be updated"
ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver 'nns' won't be used since 'rpc_endpoint' isn't provided"
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)"
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value"
FailedToParseDefaultLocationConstraint = "failed to parse 'default' location constraint, default one will be used"
FailedToReadRegionMapFilePolicies = "failed to read region map file, policies will be empty"
DefaultLocationConstraintCantBeOverriden = "'default' location constraint can't be overriden by custom policy, use 'placement_policy.default'"
FailedToParseLocationConstraint = "failed to parse location constraint, it cannot be used"
FailedToParseDefaultCopiesNumbers = "failed to parse 'default' copies numbers, default one will be used"
FailedToParseCopiesNumbers = "failed to parse copies numbers, skip"
FailedToInitializeTracing = "failed to initialize tracing"
DefaultNamespacesCannotBeEmpty = "default namespaces cannot be empty, defaults will be used"
FailedToParseNamespacesConfig = "failed to unmarshal namespaces config"
DefaultNamespaceConfigValuesBeOverwritten = "default namespace config value be overwritten by values from 'namespaces.config'"
MultipleDefaultOverridesFound = "multiple default overrides found, only one will be used"
SkipEmptyAddress = "skip, empty address"
FailedToParseDefaultDefaultLocationConstraint = "failed to parse default 'default' location constraint"
LogHTTPDisabledInThisBuild = "http logging disabled in this build"
InvalidDefaultMaxAge = "invalid defaultMaxAge"
RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped"
RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated"
InvalidAccessBoxCacheRemovingCheckInterval = "invalid accessbox check removing interval, using default value" InvalidAccessBoxCacheRemovingCheckInterval = "invalid accessbox check removing interval, using default value"
WarnDuplicateAddress = "duplicate address"
FailedToLoadMultinetConfig = "failed to load multinet config"
MultinetConfigWontBeUpdated = "multinet config won't be updated"
WarnDomainContainsPort = "the domain contains a port, domain skipped"
TagsLogConfigWontBeUpdated = "tags log config won't be updated"
CouldNotFetchLifecycleContainerInfo = "couldn't fetch lifecycle container info"
WarnDuplicateNamespaceVHS = "duplicate namespace with enabled VHS, config value skipped"
WarnValueVHSEnabledFlagWrongType = "the value of the VHS enable flag for the namespace is of the wrong type, config value skipped"
WarnDomainContainsInvalidPlaceholder = "the domain contains an invalid placeholder, domain skipped"
)
// Datapath.
const (
NotSupported = "not supported"
RequestUnmatched = "request unmatched"
RequestStart = "request start"
RequestFailed = "request failed"
RequestEnd = "request end"
AnonRequestSkipFrostfsIDValidation = "anon request, skip FrostfsID validation"
CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed = "couldn't receive access box for gate key, random key will be used"
FrostfsIDValidationFailed = "FrostfsID validation failed"
FailedToPassAuthentication = "failed to pass authentication"
PolicyValidationFailed = "policy validation failed"
CouldNotCloseRequestBody = "could not close request body" CouldNotCloseRequestBody = "could not close request body"
BucketOwnerKeyIsMissing = "bucket owner key is missing" BucketOwnerKeyIsMissing = "bucket owner key is missing"
SettingsNodeInvalidOwnerKey = "settings node: invalid owner key"
SuccessfulAuth = "successful auth" SuccessfulAuth = "successful auth"
PolicyRequest = "policy request" PolicyRequest = "policy request"
FailedToGenerateRequestID = "failed to generate request id" FailedToGenerateRequestID = "failed to generate request id"
InvalidBucketObjectLockEnabledHeader = "invalid X-Amz-Bucket-Object-Lock-Enabled header"
InvalidTreeKV = "invalid tree service meta KV"
FailedToWriteResponse = "failed to write response" FailedToWriteResponse = "failed to write response"
WarnDuplicateAddress = "duplicate address"
PolicyCouldntBeConvertedToNativeRules = "policy couldn't be converted to native rules, only s3 rules be applied" PolicyCouldntBeConvertedToNativeRules = "policy couldn't be converted to native rules, only s3 rules be applied"
FailedToParseHTTPTime = "failed to parse http time, header is ignored"
WarnInvalidTypeTLSTerminationHeader = "invalid type of value of tls termination header"
FailedToUnescapeObjectName = "failed to unescape object name"
MismatchedObjEncryptionInfo = "mismatched obj encryptionInfo"
CouldNotParseContainerObjectLockEnabledAttribute = "could not parse container object lock enabled attribute"
FailedToParsePartInfo = "failed to parse part info"
SettingsNodeInvalidOwnerKey = "settings node: invalid owner key"
FailedToParseAddressInTreeNode = "failed to parse object addr in tree node"
UnexpectedMultiNodeIDsInSubTreeMultiParts = "unexpected multi node ids in sub tree multi parts"
FoundSeveralSystemNodes = "found several system nodes"
BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids"
UploadPart = "upload part"
FailedToSubmitTaskToPool = "failed to submit task to pool"
FailedToGetRealObjectSize = "failed to get real object size"
PartDetails = "part details"
BucketSettingsNotFoundUseDefaults = "bucket settings not found, use defaults"
ParseTreeNode = "parse tree node"
GetObject = "get object"
GetTreeNodeToDelete = "get tree node to delete"
InvalidBucketObjectLockEnabledHeader = "invalid X-Amz-Bucket-Object-Lock-Enabled header"
InvalidCacheEntryType = "invalid cache entry type"
InvalidCacheKeyType = "invalid cache key type"
CouldntPutBucketInfoIntoCache = "couldn't put bucket info into cache"
CouldntAddObjectToCache = "couldn't add object to cache"
CouldntCacheAccessControlOperation = "couldn't cache access control operation"
CouldntPutObjAddressToNameCache = "couldn't put obj address to name cache"
CouldntPutAccessBoxIntoCache = "couldn't put accessbox into cache"
CouldntCacheListOfObjects = "couldn't cache list of objects"
CouldntCacheListSession = "couldn't cache list session"
CouldntCacheTags = "couldn't cache tags"
CouldntCacheLockInfo = "couldn't cache lock info"
CouldntCacheNetworkInfo = "couldn't cache network info"
CouldntCacheSubject = "couldn't cache subject info" CouldntCacheSubject = "couldn't cache subject info"
UserGroupsListIsEmpty = "user groups list is empty, subject not found" CouldntCacheNetmap = "couldn't cache netmap"
CouldntCacheUserKey = "couldn't cache user key" CouldntCacheUserKey = "couldn't cache user key"
CouldntCacheBucketSettings = "couldn't cache bucket settings"
CouldntCacheCors = "couldn't cache cors"
CouldntCacheListPolicyChains = "couldn't cache list policy chains"
CouldntCacheLifecycleConfiguration = "couldn't cache lifecycle configuration"
GetBucketCors = "get bucket cors"
GetBucketInfo = "get bucket info"
ResolveBucket = "resolve bucket"
FailedToResolveCID = "failed to resolve CID"
FailedToDiscardPutPayloadProbablyGoroutineLeaks = "failed to discard put payload, probably goroutine leaks"
)
// External storage.
const (
ObjectIsCopied = "object is copied"
CouldntDeleteObject = "couldn't delete object"
CouldNotListUserContainers = "could not list user containers"
CouldNotFetchContainerInfo = "could not fetch container info"
CouldntDeleteObjectFromStorageContinueDeleting = "couldn't delete object from storage, continue deleting from tree"
FailedToPutTombstoneObject = "failed to put tombstone object"
FailedToListAllObjectRelations = "failed to list all object relations"
FailedToPutTombstones = "failed to put tombstones"
CouldntDeleteCorsObject = "couldn't delete cors object"
BucketIsCreated = "bucket is created"
CouldntDeleteOldPartObject = "couldn't delete old part object"
CouldNotPutCompletedObject = "could not put a completed object (multipart upload)"
CouldNotDeleteUploadPart = "could not delete upload part"
PutObject = "put object"
CouldNotFetchObjectMeta = "could not fetch object meta"
FailedToDeleteObject = "failed to delete object"
CouldntDeleteLifecycleObject = "couldn't delete lifecycle configuration object"
)
// External blockchain.
const (
UserGroupsListIsEmpty = "user groups list is empty, subject not found"
)
// External storage tree.
const (
FoundMoreThanOneUnversionedNode = "found more than one unversioned node"
GetTreeNode = "get tree node"
InvalidTreeKV = "invalid tree service meta KV"
FailedToRemoveOldSystemNode = "failed to remove old system node"
GetBucketLifecycle = "get bucket lifecycle"
FailedToRemoveOldPartNode = "failed to remove old part node"
SystemNodeHasMultipleIDs = "system node has multiple ids"
ObjectTaggingNodeHasMultipleIDs = "object tagging node has multiple ids" ObjectTaggingNodeHasMultipleIDs = "object tagging node has multiple ids"
BucketTaggingNodeHasMultipleIDs = "bucket tagging node has multiple ids" BucketTaggingNodeHasMultipleIDs = "bucket tagging node has multiple ids"
BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids" BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids"
BucketCORSNodeHasMultipleIDs = "bucket cors node has multiple ids" BucketCORSNodeHasMultipleIDs = "bucket cors node has multiple ids"
SystemNodeHasMultipleIDs = "system node has multiple ids" GetBucketCorsFromTree = "get bucket cors from tree"
FailedToRemoveOldSystemNode = "failed to remove old system node" )
FailedToParseAddressInTreeNode = "failed to parse object addr in tree node"
UnexpectedMultiNodeIDsInSubTreeMultiParts = "unexpected multi node ids in sub tree multi parts" // Authmate.
FoundSeveralSystemNodes = "found several system nodes" const (
FailedToParsePartInfo = "failed to parse part info" CreateContainer = "create container"
CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info" CheckContainer = "check container"
CouldNotFetchAccessBoxContainerInfo = "couldn't fetch AccessBox container info" CheckCustomAccessKeyIDUniqueness = "check custom access key id uniqueness"
CloseCredsObjectPayload = "close creds object payload" StoreBearerTokenIntoFrostFS = "store bearer token into FrostFS"
CouldntDeleteLifecycleObject = "couldn't delete lifecycle configuration object" UpdateAccessCredObjectIntoFrostFS = "update access cred object into FrostFS"
CouldntCacheLifecycleConfiguration = "couldn't cache lifecycle configuration" SubjectAlreadyExistsInFrostFSID = "subject already exists in frostfsid"
CouldNotFetchLifecycleContainerInfo = "couldn't fetch lifecycle container info" SetSubjectNameInFrostFSID = "set subject name in frostfsid"
BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids" AddPolicyChainRules = "add policy chain rules"
GetBucketLifecycle = "get bucket lifecycle" PrepareConnectionPool = "prepare connection pool"
WarnDuplicateNamespaceVHS = "duplicate namespace with enabled VHS, config value skipped" PrepareFrostfsIDClient = "prepare frostfsid client"
WarnValueVHSEnabledFlagWrongType = "the value of the VHS enable flag for the namespace is of the wrong type, config value skipped" PreparePolicyClient = "prepare policy client"
WarnDomainContainsInvalidPlaceholder = "the domain contains an invalid placeholder, domain skipped" CreateSubjectInFrostFSID = "create subject in frostfsid"
FailedToRemoveOldPartNode = "failed to remove old part node" CloseCredsObjectPayload = "close creds object payload"
CouldntCacheNetworkInfo = "couldn't cache network info" )
NotSupported = "not supported"
CheckCustomAccessKeyIDUniqueness = "check custom access key id uniqueness" // Log HTTP.
FailedToLoadMultinetConfig = "failed to load multinet config" const (
MultinetConfigWontBeUpdated = "multinet config won't be updated" LogHTTP = "http log"
MultinetDialSuccess = "multinet dial successful" FailedToCloseHTTPBody = "failed to close http body"
MultinetDialFail = "multinet dial failed" FailedToInitializeHTTPLogger = "failed to initialize http logger"
FailedToParseHTTPTime = "failed to parse http time, header is ignored" FailedToReadHTTPBody = "failed to read http body"
FailedToPutTombstoneObject = "failed to put tombstone object" FailedToProcessHTTPBody = "failed to process http body"
FailedToCreateWorkerPool = "failed to create worker pool"
FailedToListAllObjectRelations = "failed to list all object relations"
WarnInvalidTypeTLSTerminationHeader = "invalid type of value of tls termination header"
FailedToPutTombstones = "failed to put tombstones"
WarnDomainContainsPort = "the domain contains a port, domain skipped"
CouldntCacheNetmap = "couldn't cache netmap"
BucketSettingsNotFoundUseDefaults = "bucket settings not found, use defaults"
) )

View file

@ -17,9 +17,16 @@ func (l LogEventHandler) DialPerformed(sourceIP net.Addr, _, address string, err
sourceIPString = sourceIP.Network() + "://" + sourceIP.String() sourceIPString = sourceIP.Network() + "://" + sourceIP.String()
} }
if err == nil { if err == nil {
l.logger.Debug(logs.MultinetDialSuccess, zap.String("source", sourceIPString), zap.String("destination", address)) l.logger.Debug(logs.MultinetDialSuccess,
zap.String("source", sourceIPString),
zap.String("destination", address),
logs.TagField(logs.TagApp))
} else { } else {
l.logger.Debug(logs.MultinetDialFail, zap.String("source", sourceIPString), zap.String("destination", address), zap.Error(err)) l.logger.Debug(logs.MultinetDialFail,
zap.String("source", sourceIPString),
zap.String("destination", address),
zap.Error(err),
logs.TagField(logs.TagApp))
} }
} }

View file

@ -27,7 +27,7 @@ type AppMetricsConfig struct {
func NewAppMetrics(cfg AppMetricsConfig) *AppMetrics { func NewAppMetrics(cfg AppMetricsConfig) *AppMetrics {
if !cfg.Enabled { if !cfg.Enabled {
cfg.Logger.Warn(logs.MetricsAreDisabled) cfg.Logger.Warn(logs.MetricsAreDisabled, logs.TagField(logs.TagApp))
} }
registry := cfg.Registerer registry := cfg.Registerer
@ -44,7 +44,7 @@ func NewAppMetrics(cfg AppMetricsConfig) *AppMetrics {
func (m *AppMetrics) SetEnabled(enabled bool) { func (m *AppMetrics) SetEnabled(enabled bool) {
if !enabled { if !enabled {
m.logger.Warn(logs.MetricsAreDisabled) m.logger.Warn(logs.MetricsAreDisabled, logs.TagField(logs.TagApp))
} }
m.mu.Lock() m.mu.Lock()

View file

@ -257,7 +257,7 @@ func newNodeVersionFromTreeNode(log *zap.Logger, filePath string, treeNode *tree
if createdStr, ok := treeNode.Get(createdKV); ok { if createdStr, ok := treeNode.Get(createdKV); ok {
if utcMilli, err := strconv.ParseInt(createdStr, 10, 64); err != nil { if utcMilli, err := strconv.ParseInt(createdStr, 10, 64); err != nil {
log.Warn(logs.InvalidTreeKV, zap.String(createdKV, createdStr), zap.Error(err)) log.Warn(logs.InvalidTreeKV, zap.String(createdKV, createdStr), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
} else { } else {
created := time.UnixMilli(utcMilli) created := time.UnixMilli(utcMilli)
version.Created = &created version.Created = &created
@ -267,7 +267,7 @@ func newNodeVersionFromTreeNode(log *zap.Logger, filePath string, treeNode *tree
if ownerStr, ok := treeNode.Get(ownerKV); ok { if ownerStr, ok := treeNode.Get(ownerKV); ok {
var owner user.ID var owner user.ID
if err := owner.DecodeString(ownerStr); err != nil { if err := owner.DecodeString(ownerStr); err != nil {
log.Warn(logs.InvalidTreeKV, zap.String(ownerKV, ownerStr), zap.Error(err)) log.Warn(logs.InvalidTreeKV, zap.String(ownerKV, ownerStr), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
} else { } else {
version.Owner = &owner version.Owner = &owner
} }
@ -275,7 +275,7 @@ func newNodeVersionFromTreeNode(log *zap.Logger, filePath string, treeNode *tree
if creationEpoch, ok := treeNode.Get(creationEpochKV); ok { if creationEpoch, ok := treeNode.Get(creationEpochKV); ok {
if epoch, err := strconv.ParseUint(creationEpoch, 10, 64); err != nil { if epoch, err := strconv.ParseUint(creationEpoch, 10, 64); err != nil {
log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, creationEpoch), zap.Error(err)) log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, creationEpoch), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
} else { } else {
version.CreationEpoch = epoch version.CreationEpoch = epoch
} }
@ -342,13 +342,13 @@ func newMultipartInfoFromTreeNode(log *zap.Logger, filePath string, treeNode *tr
if ownerID, ok := treeNode.Get(ownerKV); ok { if ownerID, ok := treeNode.Get(ownerKV); ok {
if err := multipartInfo.Owner.DecodeString(ownerID); err != nil { if err := multipartInfo.Owner.DecodeString(ownerID); err != nil {
log.Warn(logs.InvalidTreeKV, zap.String(ownerKV, ownerID), zap.Error(err)) log.Warn(logs.InvalidTreeKV, zap.String(ownerKV, ownerID), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
} }
} }
if created, ok := treeNode.Get(createdKV); ok { if created, ok := treeNode.Get(createdKV); ok {
if utcMilli, err := strconv.ParseInt(created, 10, 64); err != nil { if utcMilli, err := strconv.ParseInt(created, 10, 64); err != nil {
log.Warn(logs.InvalidTreeKV, zap.String(createdKV, created), zap.Error(err)) log.Warn(logs.InvalidTreeKV, zap.String(createdKV, created), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
} else { } else {
multipartInfo.Created = time.UnixMilli(utcMilli) multipartInfo.Created = time.UnixMilli(utcMilli)
} }
@ -356,7 +356,7 @@ func newMultipartInfoFromTreeNode(log *zap.Logger, filePath string, treeNode *tr
if finished, ok := treeNode.Get(finishedKV); ok { if finished, ok := treeNode.Get(finishedKV); ok {
if flag, err := strconv.ParseBool(finished); err != nil { if flag, err := strconv.ParseBool(finished); err != nil {
log.Warn(logs.InvalidTreeKV, zap.String(finishedKV, finished), zap.Error(err)) log.Warn(logs.InvalidTreeKV, zap.String(finishedKV, finished), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
} else { } else {
multipartInfo.Finished = flag multipartInfo.Finished = flag
} }
@ -364,7 +364,7 @@ func newMultipartInfoFromTreeNode(log *zap.Logger, filePath string, treeNode *tr
if creationEpoch, ok := treeNode.Get(creationEpochKV); ok { if creationEpoch, ok := treeNode.Get(creationEpochKV); ok {
if epoch, err := strconv.ParseUint(creationEpoch, 10, 64); err != nil { if epoch, err := strconv.ParseUint(creationEpoch, 10, 64); err != nil {
log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, creationEpoch), zap.Error(err)) log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, creationEpoch), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
} else { } else {
multipartInfo.CreationEpoch = epoch multipartInfo.CreationEpoch = epoch
} }
@ -395,23 +395,23 @@ func newMultipartInfo(log *zap.Logger, node NodeResponse) (*data.MultipartInfo,
multipartInfo.Key = string(kv.GetValue()) multipartInfo.Key = string(kv.GetValue())
case createdKV: case createdKV:
if utcMilli, err := strconv.ParseInt(string(kv.GetValue()), 10, 64); err != nil { if utcMilli, err := strconv.ParseInt(string(kv.GetValue()), 10, 64); err != nil {
log.Warn(logs.InvalidTreeKV, zap.String(createdKV, string(kv.GetValue())), zap.Error(err)) log.Warn(logs.InvalidTreeKV, zap.String(createdKV, string(kv.GetValue())), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
} else { } else {
multipartInfo.Created = time.UnixMilli(utcMilli) multipartInfo.Created = time.UnixMilli(utcMilli)
} }
case ownerKV: case ownerKV:
if err := multipartInfo.Owner.DecodeString(string(kv.GetValue())); err != nil { if err := multipartInfo.Owner.DecodeString(string(kv.GetValue())); err != nil {
log.Warn(logs.InvalidTreeKV, zap.String(ownerKV, string(kv.GetValue())), zap.Error(err)) log.Warn(logs.InvalidTreeKV, zap.String(ownerKV, string(kv.GetValue())), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
} }
case finishedKV: case finishedKV:
if isFinished, err := strconv.ParseBool(string(kv.GetValue())); err != nil { if isFinished, err := strconv.ParseBool(string(kv.GetValue())); err != nil {
log.Warn(logs.InvalidTreeKV, zap.String(finishedKV, string(kv.GetValue())), zap.Error(err)) log.Warn(logs.InvalidTreeKV, zap.String(finishedKV, string(kv.GetValue())), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
} else { } else {
multipartInfo.Finished = isFinished multipartInfo.Finished = isFinished
} }
case creationEpochKV: case creationEpochKV:
if epoch, err := strconv.ParseUint(string(kv.GetValue()), 10, 64); err != nil { if epoch, err := strconv.ParseUint(string(kv.GetValue()), 10, 64); err != nil {
log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, string(kv.GetValue())), zap.Error(err)) log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, string(kv.GetValue())), zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
} else { } else {
multipartInfo.CreationEpoch = epoch multipartInfo.CreationEpoch = epoch
} }
@ -515,7 +515,7 @@ func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*
if ownerKeyHex, ok := node.Get(ownerKeyKV); ok { if ownerKeyHex, ok := node.Get(ownerKeyKV); ok {
if settings.OwnerKey, err = keys.NewPublicKeyFromString(ownerKeyHex); err != nil { if settings.OwnerKey, err = keys.NewPublicKeyFromString(ownerKeyHex); err != nil {
c.reqLogger(ctx).Error(logs.SettingsNodeInvalidOwnerKey, zap.Error(err)) c.reqLogger(ctx).Error(logs.SettingsNodeInvalidOwnerKey, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
} }
@ -539,7 +539,7 @@ func (c *Tree) PutSettingsNode(ctx context.Context, bktInfo *data.BucketInfo, se
latest := multiNode.Latest() latest := multiNode.Latest()
ind := latest.GetLatestNodeIndex() ind := latest.GetLatestNodeIndex()
if latest.IsSplit() { if latest.IsSplit() {
c.reqLogger(ctx).Error(logs.BucketSettingsNodeHasMultipleIDs, zap.Uint64s("ids", latest.ID)) c.reqLogger(ctx).Error(logs.BucketSettingsNodeHasMultipleIDs, zap.Uint64s("ids", latest.ID), logs.TagField(logs.TagExternalStorageTree))
} }
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil { if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
@ -582,7 +582,7 @@ func (c *Tree) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, addr
latest := multiNode.Latest() latest := multiNode.Latest()
ind := latest.GetLatestNodeIndex() ind := latest.GetLatestNodeIndex()
if latest.IsSplit() { if latest.IsSplit() {
c.reqLogger(ctx).Error(logs.BucketCORSNodeHasMultipleIDs) c.reqLogger(ctx).Error(logs.BucketCORSNodeHasMultipleIDs, logs.TagField(logs.TagExternalStorageTree))
} }
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil { if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
@ -640,14 +640,14 @@ func (c *Tree) cleanOldNodes(ctx context.Context, nodes []*treeNode, bktInfo *da
for _, node := range nodes { for _, node := range nodes {
ind := node.GetLatestNodeIndex() ind := node.GetLatestNodeIndex()
if node.IsSplit() { if node.IsSplit() {
c.reqLogger(ctx).Error(logs.SystemNodeHasMultipleIDs, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64s("ids", node.ID)) c.reqLogger(ctx).Error(logs.SystemNodeHasMultipleIDs, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64s("ids", node.ID), logs.TagField(logs.TagExternalStorageTree))
} }
if err := c.service.RemoveNode(ctx, bktInfo, systemTree, node.ID[ind]); err != nil { if err := c.service.RemoveNode(ctx, bktInfo, systemTree, node.ID[ind]); err != nil {
c.reqLogger(ctx).Warn(logs.FailedToRemoveOldSystemNode, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64("id", node.ID[ind])) c.reqLogger(ctx).Warn(logs.FailedToRemoveOldSystemNode, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64("id", node.ID[ind]), logs.TagField(logs.TagExternalStorageTree))
} else { } else {
addr, err := getTreeNodeAddress(node) addr, err := getTreeNodeAddress(node)
if err != nil { if err != nil {
c.log.Warn(logs.FailedToParseAddressInTreeNode, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64("id", node.ID[ind])) c.log.Warn(logs.FailedToParseAddressInTreeNode, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64("id", node.ID[ind]), logs.TagField(logs.TagDatapath))
continue continue
} }
res = append(res, addr) res = append(res, addr)
@ -702,7 +702,7 @@ func (c *Tree) PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, o
ind := tagNode.GetLatestNodeIndex() ind := tagNode.GetLatestNodeIndex()
if tagNode.IsSplit() { if tagNode.IsSplit() {
c.reqLogger(ctx).Error(logs.ObjectTaggingNodeHasMultipleIDs) c.reqLogger(ctx).Error(logs.ObjectTaggingNodeHasMultipleIDs, logs.TagField(logs.TagExternalStorageTree))
} }
return c.service.MoveNode(ctx, bktInfo, versionTree, tagNode.ID[ind], objVersion.ID, treeTagSet) return c.service.MoveNode(ctx, bktInfo, versionTree, tagNode.ID[ind], objVersion.ID, treeTagSet)
@ -751,7 +751,7 @@ func (c *Tree) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, t
latest := multiNode.Latest() latest := multiNode.Latest()
ind := latest.GetLatestNodeIndex() ind := latest.GetLatestNodeIndex()
if latest.IsSplit() { if latest.IsSplit() {
c.reqLogger(ctx).Error(logs.BucketTaggingNodeHasMultipleIDs, zap.Uint64s("ids", latest.ID)) c.reqLogger(ctx).Error(logs.BucketTaggingNodeHasMultipleIDs, zap.Uint64s("ids", latest.ID), logs.TagField(logs.TagExternalStorageTree))
} }
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, treeTagSet); err != nil { if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, treeTagSet); err != nil {
@ -1049,7 +1049,7 @@ func (s *VersionsByPrefixStreamImpl) parseNodeResponse(node NodeResponse) (res *
trNode, fileName, err := parseTreeNode(node) trNode, fileName, err := parseTreeNode(node)
if err != nil { if err != nil {
if !errors.Is(err, errNodeDoesntContainFileName) { if !errors.Is(err, errNodeDoesntContainFileName) {
s.log.Debug(logs.ParseTreeNode, zap.Error(err)) s.log.Debug(logs.ParseTreeNode, zap.Error(err), logs.TagField(logs.TagDatapath))
} }
return nil, true, nil return nil, true, nil
} }
@ -1287,7 +1287,9 @@ func (c *Tree) getUnversioned(ctx context.Context, bktInfo *data.BucketInfo, tre
if len(nodes) > 1 { if len(nodes) > 1 {
c.reqLogger(ctx).Debug(logs.FoundMoreThanOneUnversionedNode, c.reqLogger(ctx).Debug(logs.FoundMoreThanOneUnversionedNode,
zap.String("treeID", treeID), zap.String("filepath", filepath)) zap.String("treeID", treeID),
zap.String("filepath", filepath),
logs.TagField(logs.TagExternalStorageTree))
} }
sort.Slice(nodes, func(i, j int) bool { sort.Slice(nodes, func(i, j int) bool {
@ -1460,7 +1462,8 @@ func (c *Tree) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartN
zap.String("upload id", info.UploadID), zap.String("upload id", info.UploadID),
zap.Uint64("multipart node id ", multipartNodeID), zap.Uint64("multipart node id ", multipartNodeID),
zap.Uint64s("id", part.GetNodeID()), zap.Uint64s("id", part.GetNodeID()),
zap.Error(err)) zap.Error(err),
logs.TagField(logs.TagDatapath))
continue continue
} }
if partInfo.Number == info.Number { if partInfo.Number == info.Number {
@ -1488,7 +1491,8 @@ func (c *Tree) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartN
c.reqLogger(ctx).Warn(logs.FailedToRemoveOldPartNode, c.reqLogger(ctx).Warn(logs.FailedToRemoveOldPartNode,
zap.String("key", info.Key), zap.String("key", info.Key),
zap.String("upload id", info.UploadID), zap.String("upload id", info.UploadID),
zap.Uint64("id", nodeID)) zap.Uint64("id", nodeID),
logs.TagField(logs.TagExternalStorageTree))
} }
} }
@ -1514,7 +1518,8 @@ func (c *Tree) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipart
// multipart parts nodeID shouldn't have multiple values // multipart parts nodeID shouldn't have multiple values
c.reqLogger(ctx).Warn(logs.UnexpectedMultiNodeIDsInSubTreeMultiParts, c.reqLogger(ctx).Warn(logs.UnexpectedMultiNodeIDsInSubTreeMultiParts,
zap.Uint64("multipart node id ", multipartNodeID), zap.Uint64("multipart node id ", multipartNodeID),
zap.Uint64s("node ids", part.GetNodeID())) zap.Uint64s("node ids", part.GetNodeID()),
logs.TagField(logs.TagDatapath))
continue continue
} }
if part.GetNodeID()[0] == multipartNodeID { if part.GetNodeID()[0] == multipartNodeID {
@ -1525,7 +1530,8 @@ func (c *Tree) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipart
c.reqLogger(ctx).Warn(logs.FailedToParsePartInfo, c.reqLogger(ctx).Warn(logs.FailedToParsePartInfo,
zap.Uint64("multipart node id ", multipartNodeID), zap.Uint64("multipart node id ", multipartNodeID),
zap.Uint64s("node ids", part.GetNodeID()), zap.Uint64s("node ids", part.GetNodeID()),
zap.Error(err)) zap.Error(err),
logs.TagField(logs.TagDatapath))
continue continue
} }
result = append(result, partInfo) result = append(result, partInfo)
@ -1556,7 +1562,7 @@ func (c *Tree) PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *dat
latest := multiNode.Latest() latest := multiNode.Latest()
ind := latest.GetLatestNodeIndex() ind := latest.GetLatestNodeIndex()
if latest.IsSplit() { if latest.IsSplit() {
c.reqLogger(ctx).Error(logs.BucketLifecycleNodeHasMultipleIDs) c.reqLogger(ctx).Error(logs.BucketLifecycleNodeHasMultipleIDs, logs.TagField(logs.TagDatapath))
} }
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil { if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
@ -1831,7 +1837,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name
return nil, tree.ErrNodeNotFound return nil, tree.ErrNodeNotFound
} }
if len(nodes) != 1 { if len(nodes) != 1 {
c.reqLogger(ctx).Warn(logs.FoundSeveralSystemNodes, zap.String("name", name)) c.reqLogger(ctx).Warn(logs.FoundSeveralSystemNodes, zap.String("name", name), logs.TagField(logs.TagDatapath))
} }
return newMultiNode(nodes) return newMultiNode(nodes)