forked from TrueCloudLab/frostfs-s3-gw
Compare commits
39 commits
Author | SHA1 | Date | |
---|---|---|---|
e184b333e4 | |||
853036e44e | |||
ee46382a68 | |||
b207eb48d9 | |||
b7650e01ac | |||
e7f620f137 | |||
ffac62e8b4 | |||
182262ace2 | |||
893b506c83 | |||
beec37797d | |||
5538dce772 | |||
da9703ab63 | |||
a53e50b324 | |||
5e9c562683 | |||
49bf3c1bce | |||
a4d9658fbb | |||
bec63026bd | |||
0064e7ab07 | |||
41e1f1ad7a | |||
4d2e6f8650 | |||
3d3dd00211 | |||
510b0a1005 | |||
da77e426b6 | |||
e7a8d4bdaf | |||
250538a9b4 | |||
619385836d | |||
65fc776dea | |||
5842f5bad5 | |||
8b3252cbd0 | |||
|
d150f8ddcb | ||
bc975989de | |||
0cab76d01e | |||
|
e060308318 | ||
a725c68d06 | |||
fb4921826e | |||
d46f1d3bfa | |||
16eb289929 | |||
0ae7c35352 | |||
95d847d611 |
110 changed files with 3668 additions and 1269 deletions
27
.forgejo/workflows/oci-image.yml
Normal file
27
.forgejo/workflows/oci-image.yml
Normal file
|
@ -0,0 +1,27 @@
|
|||
on:
|
||||
pull_request:
|
||||
push:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
image:
|
||||
name: OCI image
|
||||
runs-on: docker
|
||||
container: git.frostfs.info/truecloudlab/env:oci-image-builder-bookworm
|
||||
steps:
|
||||
- name: Clone git repo
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Build OCI image
|
||||
run: make image
|
||||
|
||||
- name: Push image to OCI registry
|
||||
run: |
|
||||
echo "$REGISTRY_PASSWORD" \
|
||||
| docker login --username truecloudlab --password-stdin git.frostfs.info
|
||||
make image-push
|
||||
if: >-
|
||||
startsWith(github.ref, 'refs/tags/v') &&
|
||||
(github.event_name == 'workflow_dispatch' || github.event_name == 'push')
|
||||
env:
|
||||
REGISTRY_PASSWORD: ${{secrets.FORGEJO_OCI_REGISTRY_PUSH_TOKEN}}
|
|
@ -16,7 +16,7 @@ jobs:
|
|||
- name: Setup Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.23'
|
||||
go-version: '1.23.6'
|
||||
|
||||
- name: Install govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
|
|
107
CHANGELOG.md
107
CHANGELOG.md
|
@ -4,6 +4,91 @@ This document outlines major changes between releases.
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.32.9] - 2025-02-12
|
||||
|
||||
### Fixed
|
||||
- Make `Content-Md5` header check optional (#612)
|
||||
|
||||
## [0.32.8] - 2025-02-11
|
||||
|
||||
### Fixed
|
||||
- Return 404 instead of 500 when object is missing in object storage and available in the tree (#626)
|
||||
|
||||
### Added
|
||||
- `tree_stream_timeout` configuration parameter (#627)
|
||||
|
||||
## [0.32.7] - 2025-02-06
|
||||
|
||||
### Fixed
|
||||
- Correct passing copies number during multipart upload (#623)
|
||||
|
||||
## [0.32.6] - 2025-02-05
|
||||
|
||||
### Fixed
|
||||
- Connection leak when `feature.tree_pool_netmap_support` is enabled (#622)
|
||||
|
||||
## [0.32.5] - 2025-02-04
|
||||
|
||||
### Fixed
|
||||
- Support trailing headers signature during aws-chunk upload (#607)
|
||||
|
||||
## [0.32.4] - 2025-02-03
|
||||
|
||||
### Fixed
|
||||
- Possible deadlock in tree pool component (#617)
|
||||
- Possible memory leak in gRPC client (#617)
|
||||
|
||||
## [0.32.3] - 2025-01-29
|
||||
|
||||
### Fixed
|
||||
- Use `UNSIGNED_PAYLOAD` as content hash to check signature if `x-amz-content-sha256` isn't signed header (#616)
|
||||
|
||||
## [0.32.2] - 2025-01-27
|
||||
|
||||
### Fixed
|
||||
- Fix panic when payload discard (#605)
|
||||
|
||||
## [0.32.1] - 2025-01-17
|
||||
|
||||
### Fixed
|
||||
- Response codes when checking ACL format (#531)
|
||||
- CORS unmarshal without xmlns (#594)
|
||||
- Response code for invalid Content-Md5 header (#598)
|
||||
|
||||
### Added
|
||||
- Derive encryption keys for accessbox with salt (#529)
|
||||
- Debug log when bucket settings not found (#595)
|
||||
- Context cancellation during tree node streaming (#569)
|
||||
- Add LimitExceeded error (#589)
|
||||
|
||||
### Changed
|
||||
- Docker image repository (#590, #587)
|
||||
|
||||
## [0.32.0] - Khumbu - 2024-12-20
|
||||
|
||||
### Added
|
||||
- Metric of dropped logs by log sampler (#502)
|
||||
- SigV4A signature algorithm (#339)
|
||||
- TLS Termination header for SSE-C (#562)
|
||||
- Kludge profile support (#147)
|
||||
- Netmap support in tree pool (#577)
|
||||
|
||||
### Changed
|
||||
- Improved multipart removal speed (#559)
|
||||
- Updated tree service pool without api-go dependency (#570)
|
||||
|
||||
## [0.31.3] - 2024-12-17
|
||||
|
||||
### Fixed
|
||||
- Return BucketAlreadyExists when global domain taken (#584)
|
||||
- Fix list-buckets vhs routing (#583)
|
||||
- Skip port when matching listen domains (#586)
|
||||
|
||||
## [0.31.2] - 2024-12-13
|
||||
|
||||
### Fixed
|
||||
- Unable to remove EC object (#576)
|
||||
|
||||
## [0.31.1] - 2024-11-28
|
||||
|
||||
### Fixed
|
||||
|
@ -62,6 +147,11 @@ This document outlines major changes between releases.
|
|||
### Removed
|
||||
- Reduce using mutex when update app settings (#329)
|
||||
|
||||
## [0.30.9] - 2024-12-13
|
||||
|
||||
### Fixed
|
||||
- Unable to remove EC object (#576)
|
||||
|
||||
## [0.30.8] - 2024-10-18
|
||||
|
||||
### Fixed
|
||||
|
@ -347,6 +437,19 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs
|
|||
[0.30.6]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.5...v0.30.6
|
||||
[0.30.7]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.6...v0.30.7
|
||||
[0.30.8]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.7...v0.30.8
|
||||
[0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.8...v0.31.0
|
||||
[0.30.9]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.8...v0.30.9
|
||||
[0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.9...v0.31.0
|
||||
[0.31.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.0...v0.31.1
|
||||
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.1...master
|
||||
[0.31.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.1...v0.31.2
|
||||
[0.31.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.2...v0.31.3
|
||||
[0.32.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.3...v0.32.0
|
||||
[0.32.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.0...v0.32.1
|
||||
[0.32.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.1...v0.32.2
|
||||
[0.32.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.2...v0.32.3
|
||||
[0.32.4]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.3...v0.32.4
|
||||
[0.32.5]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.4...v0.32.5
|
||||
[0.32.6]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.5...v0.32.6
|
||||
[0.32.7]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.6...v0.32.7
|
||||
[0.32.8]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.7...v0.32.8
|
||||
[0.32.9]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.8...v0.32.9
|
||||
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.32.9...master
|
2
Makefile
2
Makefile
|
@ -18,7 +18,7 @@ GOFLAGS ?=
|
|||
|
||||
# Variables for docker
|
||||
REPO_BASENAME = $(shell basename `go list -m`)
|
||||
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
|
||||
HUB_IMAGE ?= "git.frostfs.info/truecloudlab/$(REPO_BASENAME)"
|
||||
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
||||
|
||||
OUTPUT_LINT_DIR ?= $(shell pwd)/bin
|
||||
|
|
2
VERSION
2
VERSION
|
@ -1 +1 @@
|
|||
v0.31.1
|
||||
v0.32.9
|
||||
|
|
|
@ -13,6 +13,7 @@ import (
|
|||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
@ -396,6 +397,10 @@ func cloneRequest(r *http.Request, authHeader *AuthHeader) *http.Request {
|
|||
func (c *Center) checkSign(ctx context.Context, authHeader *AuthHeader, box *accessbox.Box, request *http.Request, signatureDateTime time.Time) error {
|
||||
var signature string
|
||||
|
||||
if !slices.Contains(authHeader.SignedFields, "x-amz-content-sha256") && authHeader.PayloadHash == "" {
|
||||
authHeader.PayloadHash = UnsignedPayload
|
||||
}
|
||||
|
||||
switch authHeader.Preamble {
|
||||
case signaturePreambleSigV4:
|
||||
creds := aws.Credentials{
|
||||
|
|
|
@ -8,6 +8,7 @@ import (
|
|||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
@ -26,6 +27,9 @@ import (
|
|||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
smithyauth "github.com/aws/smithy-go/auth"
|
||||
"github.com/aws/smithy-go/logging"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap/zaptest"
|
||||
|
@ -298,6 +302,7 @@ func TestAuthenticate(t *testing.T) {
|
|||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
region string
|
||||
prefixes []string
|
||||
request *http.Request
|
||||
err bool
|
||||
|
@ -308,10 +313,23 @@ func TestAuthenticate(t *testing.T) {
|
|||
prefixes: []string{addr.Container().String()},
|
||||
request: func() *http.Request {
|
||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
err = defaultSigner.SignHTTP(ctx, awsCreds, r, UnsignedPayload, service, region, time.Now())
|
||||
require.NoError(t, err)
|
||||
return r
|
||||
}(),
|
||||
region: region,
|
||||
},
|
||||
{
|
||||
name: "valid sign with hash",
|
||||
prefixes: []string{addr.Container().String()},
|
||||
request: func() *http.Request {
|
||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
r.Header.Set(AmzContentSHA256, "")
|
||||
err = defaultSigner.SignHTTP(ctx, awsCreds, r, "", service, region, time.Now())
|
||||
require.NoError(t, err)
|
||||
return r
|
||||
}(),
|
||||
region: region,
|
||||
},
|
||||
{
|
||||
name: "no authorization header",
|
||||
|
@ -418,12 +436,27 @@ func TestAuthenticate(t *testing.T) {
|
|||
request: func() *http.Request {
|
||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
r.Header.Set(AmzExpires, "60")
|
||||
signedURI, _, err := defaultSigner.PresignHTTP(ctx, awsCreds, r, UnsignedPayload, service, region, time.Now())
|
||||
require.NoError(t, err)
|
||||
r.URL, err = url.ParseRequestURI(signedURI)
|
||||
require.NoError(t, err)
|
||||
return r
|
||||
}(),
|
||||
region: region,
|
||||
},
|
||||
{
|
||||
name: "valid presign with hash",
|
||||
request: func() *http.Request {
|
||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
r.Header.Set(AmzExpires, "60")
|
||||
r.Header.Set(AmzContentSHA256, "")
|
||||
signedURI, _, err := defaultSigner.PresignHTTP(ctx, awsCreds, r, "", service, region, time.Now())
|
||||
require.NoError(t, err)
|
||||
r.URL, err = url.ParseRequestURI(signedURI)
|
||||
require.NoError(t, err)
|
||||
return r
|
||||
}(),
|
||||
region: region,
|
||||
},
|
||||
{
|
||||
name: "presign, bad X-Amz-Credential",
|
||||
|
@ -480,6 +513,56 @@ func TestAuthenticate(t *testing.T) {
|
|||
err: true,
|
||||
errCode: errors.ErrBadRequest,
|
||||
},
|
||||
{
|
||||
name: "presign using original aws sdk",
|
||||
request: func() *http.Request {
|
||||
cli := s3.NewPresignClient(s3.New(s3.Options{
|
||||
Credentials: credentials.NewStaticCredentialsProvider(awsCreds.AccessKeyID, awsCreds.SecretAccessKey, ""),
|
||||
UsePathStyle: true,
|
||||
BaseEndpoint: aws.String("http://localhost"),
|
||||
Region: region,
|
||||
Logger: logging.NewStandardLogger(os.Stdout),
|
||||
ClientLogMode: aws.LogSigning,
|
||||
}))
|
||||
|
||||
res, err := cli.PresignGetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("object"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
r := httptest.NewRequest(http.MethodGet, "http://localhost", nil)
|
||||
r.URL, err = url.ParseRequestURI(res.URL)
|
||||
require.NoError(t, err)
|
||||
return r
|
||||
}(),
|
||||
region: region,
|
||||
},
|
||||
{
|
||||
name: "presign sigv4a using original aws sdk",
|
||||
request: func() *http.Request {
|
||||
cli := s3.NewPresignClient(s3.New(s3.Options{
|
||||
Credentials: credentials.NewStaticCredentialsProvider(awsCreds.AccessKeyID, awsCreds.SecretAccessKey, ""),
|
||||
UsePathStyle: true,
|
||||
BaseEndpoint: aws.String("http://localhost"),
|
||||
Region: region,
|
||||
Logger: logging.NewStandardLogger(os.Stdout),
|
||||
ClientLogMode: aws.LogSigning,
|
||||
AuthSchemeResolver: resolver{},
|
||||
}))
|
||||
|
||||
res, err := cli.PresignGetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("object"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
r := httptest.NewRequest(http.MethodGet, "http://localhost", nil)
|
||||
r.URL, err = url.ParseRequestURI(res.URL)
|
||||
require.NoError(t, err)
|
||||
return r
|
||||
}(),
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
creds := tokens.New(bigConfig)
|
||||
|
@ -495,13 +578,19 @@ func TestAuthenticate(t *testing.T) {
|
|||
} else {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, accessKeyID, box.AuthHeaders.AccessKeyID)
|
||||
require.Equal(t, region, box.AuthHeaders.Region)
|
||||
require.Equal(t, tc.region, box.AuthHeaders.Region)
|
||||
require.Equal(t, secret.SecretKey, box.AccessBox.Gate.SecretKey)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type resolver struct{}
|
||||
|
||||
func (r resolver) ResolveAuthSchemes(context.Context, *s3.AuthResolverParameters) ([]*smithyauth.Option, error) {
|
||||
return []*smithyauth.Option{{SchemeID: smithyauth.SchemeIDSigV4A}}, nil
|
||||
}
|
||||
|
||||
func TestHTTPPostAuthenticate(t *testing.T) {
|
||||
const (
|
||||
policyBase64 = "eyJleHBpcmF0aW9uIjogIjIwMjUtMTItMDFUMTI6MDA6MDAuMDAwWiIsImNvbmRpdGlvbnMiOiBbCiBbInN0YXJ0cy13aXRoIiwgIiR4LWFtei1jcmVkZW50aWFsIiwgIiJdLAogWyJzdGFydHMtd2l0aCIsICIkeC1hbXotZGF0ZSIsICIiXQpdfQ=="
|
||||
|
|
|
@ -52,7 +52,12 @@ func PresignRequest(ctx context.Context, creds aws.Credentials, reqData RequestD
|
|||
options.Logger = log
|
||||
})
|
||||
|
||||
signedURI, _, err := signer.PresignHTTP(ctx, creds, req, presignData.Headers[AmzContentSHA256], presignData.Service, presignData.Region, presignData.SignTime)
|
||||
payloadHash := presignData.Headers[AmzContentSHA256]
|
||||
if payloadHash == "" {
|
||||
payloadHash = UnsignedPayload
|
||||
}
|
||||
|
||||
signedURI, _, err := signer.PresignHTTP(ctx, creds, req, payloadHash, presignData.Service, presignData.Region, presignData.SignTime)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("presign: %w", err)
|
||||
}
|
||||
|
@ -93,7 +98,13 @@ func PresignRequestV4a(cred aws.Credentials, reqData RequestData, presignData Pr
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to derive assymetric key from credentials: %w", err)
|
||||
}
|
||||
presignedURL, _, err := signer.PresignHTTP(req.Context(), creds, req, presignData.Headers[AmzContentSHA256], presignData.Service, []string{presignData.Region}, presignData.SignTime)
|
||||
|
||||
payloadHash := presignData.Headers[AmzContentSHA256]
|
||||
if payloadHash == "" {
|
||||
payloadHash = UnsignedPayload
|
||||
}
|
||||
|
||||
presignedURL, _, err := signer.PresignHTTP(req.Context(), creds, req, payloadHash, presignData.Service, []string{presignData.Region}, presignData.SignTime)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("presign: %w", err)
|
||||
}
|
||||
|
|
|
@ -77,8 +77,7 @@ func TestCheckSign(t *testing.T) {
|
|||
Lifetime: 10 * time.Minute,
|
||||
SignTime: time.Now().UTC(),
|
||||
Headers: map[string]string{
|
||||
ContentTypeHdr: "text/plain",
|
||||
AmzContentSHA256: UnsignedPayload,
|
||||
ContentTypeHdr: "text/plain",
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
@ -1,4 +1,6 @@
|
|||
// This file is adopting https://github.com/aws/aws-sdk-go-v2/blob/a2b751d1ba71f59175a41f9cae5f159f1044360f/aws/signer/v4/stream.go for sigv4a.
|
||||
// with changes
|
||||
// * add VerifyTrailerSignature
|
||||
|
||||
package v4a
|
||||
|
||||
|
@ -88,6 +90,39 @@ func (s *StreamSigner) buildEventStreamStringToSign(headers, payload, previousSi
|
|||
}, "\n")
|
||||
}
|
||||
|
||||
func (s *StreamSigner) VerifyTrailerSignature(payload []byte, signingTime time.Time, signature []byte) error {
|
||||
prevSignature := s.prevSignature
|
||||
|
||||
st := v4Internal.NewSigningTime(signingTime)
|
||||
|
||||
scope := buildCredentialScope(st, s.service)
|
||||
|
||||
stringToSign := s.buildEventStreamStringToSignTrailer(payload, prevSignature, scope, &st)
|
||||
|
||||
ok, err := signerCrypto.VerifySignature(&s.credentials.PrivateKey.PublicKey, makeHash(sha256.New(), []byte(stringToSign)), signature)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("v4a: invalid signature")
|
||||
}
|
||||
|
||||
s.prevSignature = signature
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *StreamSigner) buildEventStreamStringToSignTrailer(payload, previousSignature []byte, credentialScope string, signingTime *v4Internal.SigningTime) string {
|
||||
hash := sha256.New()
|
||||
return strings.Join([]string{
|
||||
"AWS4-ECDSA-P256-SHA256-TRAILER",
|
||||
signingTime.TimeFormat(),
|
||||
credentialScope,
|
||||
hex.EncodeToString(previousSignature),
|
||||
hex.EncodeToString(makeHash(hash, payload)),
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
func buildCredentialScope(st v4Internal.SigningTime, service string) string {
|
||||
return strings.Join([]string{
|
||||
st.Format(shortTimeFormat),
|
||||
|
|
|
@ -1,4 +1,6 @@
|
|||
// This is https://github.com/aws/aws-sdk-go-v2/blob/a2b751d1ba71f59175a41f9cae5f159f1044360f/aws/signer/v4/stream.go
|
||||
// with changes
|
||||
// * add GetTrailingSignature
|
||||
|
||||
package v4
|
||||
|
||||
|
@ -87,3 +89,32 @@ func (s *StreamSigner) buildEventStreamStringToSign(headers, payload, previousSi
|
|||
hex.EncodeToString(makeHash(hash, payload)),
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
// GetTrailerSignature signs the provided header and payload bytes.
|
||||
func (s *StreamSigner) GetTrailerSignature(payload []byte, signingTime time.Time) ([]byte, error) {
|
||||
prevSignature := s.prevSignature
|
||||
|
||||
st := v4Internal.NewSigningTime(signingTime)
|
||||
|
||||
sigKey := s.signingKeyDeriver.DeriveKey(s.credentials, s.service, s.region, st)
|
||||
|
||||
scope := v4Internal.BuildCredentialScope(st, s.region, s.service)
|
||||
|
||||
stringToSign := s.buildEventStreamStringToSignTrailer(payload, prevSignature, scope, &st)
|
||||
|
||||
signature := v4Internal.HMACSHA256(sigKey, []byte(stringToSign))
|
||||
s.prevSignature = signature
|
||||
|
||||
return signature, nil
|
||||
}
|
||||
|
||||
func (s *StreamSigner) buildEventStreamStringToSignTrailer(payload, previousSignature []byte, credentialScope string, signingTime *v4Internal.SigningTime) string {
|
||||
hash := sha256.New()
|
||||
return strings.Join([]string{
|
||||
"AWS4-HMAC-SHA256-TRAILER",
|
||||
signingTime.TimeFormat(),
|
||||
credentialScope,
|
||||
hex.EncodeToString(previousSignature),
|
||||
hex.EncodeToString(makeHash(hash, payload)),
|
||||
}, "\n")
|
||||
}
|
||||
|
|
2
api/cache/access_control.go
vendored
2
api/cache/access_control.go
vendored
|
@ -48,7 +48,7 @@ func (o *AccessControlCache) Get(owner user.ID, key string) bool {
|
|||
result, ok := entry.(bool)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return false
|
||||
}
|
||||
|
||||
|
|
2
api/cache/accessbox.go
vendored
2
api/cache/accessbox.go
vendored
|
@ -67,7 +67,7 @@ func (o *AccessBoxCache) Get(accessKeyID string) *AccessBoxCacheValue {
|
|||
result, ok := entry.(*AccessBoxCacheValue)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
57
api/cache/buckets.go
vendored
57
api/cache/buckets.go
vendored
|
@ -6,14 +6,16 @@ import (
|
|||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// BucketCache contains cache with objects and the lifetime of cache entries.
|
||||
type BucketCache struct {
|
||||
cache gcache.Cache
|
||||
logger *zap.Logger
|
||||
cache gcache.Cache
|
||||
cidCache gcache.Cache
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
const (
|
||||
|
@ -33,14 +35,45 @@ func DefaultBucketConfig(logger *zap.Logger) *Config {
|
|||
}
|
||||
|
||||
// NewBucketCache creates an object of BucketCache.
|
||||
func NewBucketCache(config *Config) *BucketCache {
|
||||
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
||||
return &BucketCache{cache: gc, logger: config.Logger}
|
||||
func NewBucketCache(config *Config, cidCache bool) *BucketCache {
|
||||
cache := &BucketCache{
|
||||
cache: gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build(),
|
||||
logger: config.Logger,
|
||||
}
|
||||
|
||||
if cidCache {
|
||||
cache.cidCache = gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
|
||||
}
|
||||
return cache
|
||||
}
|
||||
|
||||
// Get returns a cached object.
|
||||
func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo {
|
||||
entry, err := o.cache.Get(formKey(ns, bktName))
|
||||
return o.get(formKey(ns, bktName))
|
||||
}
|
||||
|
||||
func (o *BucketCache) GetByCID(cnrID cid.ID) *data.BucketInfo {
|
||||
if o.cidCache == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
entry, err := o.cidCache.Get(cnrID)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
key, ok := entry.(string)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", key)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
return o.get(key)
|
||||
}
|
||||
|
||||
func (o *BucketCache) get(key string) *data.BucketInfo {
|
||||
entry, err := o.cache.Get(key)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
@ -48,7 +81,7 @@ func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo {
|
|||
result, ok := entry.(*data.BucketInfo)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -57,11 +90,21 @@ func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo {
|
|||
|
||||
// Put puts an object to cache.
|
||||
func (o *BucketCache) Put(bkt *data.BucketInfo) error {
|
||||
if o.cidCache != nil {
|
||||
if err := o.cidCache.Set(bkt.CID, formKey(bkt.Zone, bkt.Name)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return o.cache.Set(formKey(bkt.Zone, bkt.Name), bkt)
|
||||
}
|
||||
|
||||
// Delete deletes an object from cache.
|
||||
func (o *BucketCache) Delete(bkt *data.BucketInfo) bool {
|
||||
if o.cidCache != nil {
|
||||
o.cidCache.Remove(bkt.CID)
|
||||
}
|
||||
|
||||
return o.cache.Remove(formKey(bkt.Zone, bkt.Name))
|
||||
}
|
||||
|
||||
|
|
2
api/cache/cache_test.go
vendored
2
api/cache/cache_test.go
vendored
|
@ -42,7 +42,7 @@ func TestAccessBoxCacheType(t *testing.T) {
|
|||
|
||||
func TestBucketsCacheType(t *testing.T) {
|
||||
logger, observedLog := getObservedLogger()
|
||||
cache := NewBucketCache(DefaultBucketConfig(logger))
|
||||
cache := NewBucketCache(DefaultBucketConfig(logger), false)
|
||||
|
||||
bktInfo := &data.BucketInfo{Name: "bucket"}
|
||||
|
||||
|
|
2
api/cache/frostfsid.go
vendored
2
api/cache/frostfsid.go
vendored
|
@ -69,7 +69,7 @@ func get[T any](c *FrostfsIDCache, key any) *T {
|
|||
result, ok := entry.(*T)
|
||||
if !ok {
|
||||
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
4
api/cache/listsession.go
vendored
4
api/cache/listsession.go
vendored
|
@ -52,7 +52,7 @@ func NewListSessionCache(config *Config) *ListSessionCache {
|
|||
session, ok := val.(*data.ListSession)
|
||||
if !ok {
|
||||
config.Logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", val)),
|
||||
zap.String("expected", fmt.Sprintf("%T", session)))
|
||||
zap.String("expected", fmt.Sprintf("%T", session)), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
if !session.Acquired.Load() {
|
||||
|
@ -72,7 +72,7 @@ func (l *ListSessionCache) GetListSession(key ListSessionKey) *data.ListSession
|
|||
result, ok := entry.(*data.ListSession)
|
||||
if !ok {
|
||||
l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
2
api/cache/names.go
vendored
2
api/cache/names.go
vendored
|
@ -50,7 +50,7 @@ func (o *ObjectsNameCache) Get(key string) *oid.Address {
|
|||
result, ok := entry.(oid.Address)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
86
api/cache/network.go
vendored
Normal file
86
api/cache/network.go
vendored
Normal file
|
@ -0,0 +1,86 @@
|
|||
package cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type (
|
||||
// NetworkCache provides cache for network-related values.
|
||||
NetworkCache struct {
|
||||
cache gcache.Cache
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NetworkCacheConfig stores expiration params for cache.
|
||||
NetworkCacheConfig struct {
|
||||
Lifetime time.Duration
|
||||
Logger *zap.Logger
|
||||
}
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultNetworkCacheLifetime = 1 * time.Minute
|
||||
networkCacheSize = 2
|
||||
networkInfoKey = "network_info"
|
||||
netmapKey = "netmap"
|
||||
)
|
||||
|
||||
// DefaultNetworkConfig returns new default cache expiration values.
|
||||
func DefaultNetworkConfig(logger *zap.Logger) *NetworkCacheConfig {
|
||||
return &NetworkCacheConfig{
|
||||
Lifetime: DefaultNetworkCacheLifetime,
|
||||
Logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// NewNetworkCache creates an object of NetworkCache.
|
||||
func NewNetworkCache(config *NetworkCacheConfig) *NetworkCache {
|
||||
gc := gcache.New(networkCacheSize).LRU().Expiration(config.Lifetime).Build()
|
||||
return &NetworkCache{cache: gc, logger: config.Logger}
|
||||
}
|
||||
|
||||
func (c *NetworkCache) GetNetworkInfo() *netmap.NetworkInfo {
|
||||
entry, err := c.cache.Get(networkInfoKey)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, ok := entry.(netmap.NetworkInfo)
|
||||
if !ok {
|
||||
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
return &result
|
||||
}
|
||||
|
||||
func (c *NetworkCache) PutNetworkInfo(info netmap.NetworkInfo) error {
|
||||
return c.cache.Set(networkInfoKey, info)
|
||||
}
|
||||
|
||||
func (c *NetworkCache) GetNetmap() *netmap.NetMap {
|
||||
entry, err := c.cache.Get(netmapKey)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, ok := entry.(netmap.NetMap)
|
||||
if !ok {
|
||||
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
return &result
|
||||
}
|
||||
|
||||
func (c *NetworkCache) PutNetmap(nm netmap.NetMap) error {
|
||||
return c.cache.Set(netmapKey, nm)
|
||||
}
|
65
api/cache/network_info.go
vendored
65
api/cache/network_info.go
vendored
|
@ -1,65 +0,0 @@
|
|||
package cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type (
|
||||
// NetworkInfoCache provides cache for network info.
|
||||
NetworkInfoCache struct {
|
||||
cache gcache.Cache
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NetworkInfoCacheConfig stores expiration params for cache.
|
||||
NetworkInfoCacheConfig struct {
|
||||
Lifetime time.Duration
|
||||
Logger *zap.Logger
|
||||
}
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultNetworkInfoCacheLifetime = 1 * time.Minute
|
||||
networkInfoCacheSize = 1
|
||||
networkInfoKey = "network_info"
|
||||
)
|
||||
|
||||
// DefaultNetworkInfoConfig returns new default cache expiration values.
|
||||
func DefaultNetworkInfoConfig(logger *zap.Logger) *NetworkInfoCacheConfig {
|
||||
return &NetworkInfoCacheConfig{
|
||||
Lifetime: DefaultNetworkInfoCacheLifetime,
|
||||
Logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// NewNetworkInfoCache creates an object of NetworkInfoCache.
|
||||
func NewNetworkInfoCache(config *NetworkInfoCacheConfig) *NetworkInfoCache {
|
||||
gc := gcache.New(networkInfoCacheSize).LRU().Expiration(config.Lifetime).Build()
|
||||
return &NetworkInfoCache{cache: gc, logger: config.Logger}
|
||||
}
|
||||
|
||||
func (c *NetworkInfoCache) Get() *netmap.NetworkInfo {
|
||||
entry, err := c.cache.Get(networkInfoKey)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, ok := entry.(netmap.NetworkInfo)
|
||||
if !ok {
|
||||
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
||||
return &result
|
||||
}
|
||||
|
||||
func (c *NetworkInfoCache) Put(info netmap.NetworkInfo) error {
|
||||
return c.cache.Set(networkInfoKey, info)
|
||||
}
|
2
api/cache/objects.go
vendored
2
api/cache/objects.go
vendored
|
@ -49,7 +49,7 @@ func (o *ObjectsCache) GetObject(address oid.Address) *data.ExtendedObjectInfo {
|
|||
result, ok := entry.(*data.ExtendedObjectInfo)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
4
api/cache/objectslist.go
vendored
4
api/cache/objectslist.go
vendored
|
@ -77,7 +77,7 @@ func (l *ObjectsListCache) GetVersions(key ObjectsListKey) []*data.NodeVersion {
|
|||
result, ok := entry.([]*data.NodeVersion)
|
||||
if !ok {
|
||||
l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -96,7 +96,7 @@ func (l *ObjectsListCache) CleanCacheEntriesContainingObject(objectName string,
|
|||
k, ok := key.(ObjectsListKey)
|
||||
if !ok {
|
||||
l.logger.Warn(logs.InvalidCacheKeyType, zap.String("actual", fmt.Sprintf("%T", key)),
|
||||
zap.String("expected", fmt.Sprintf("%T", k)))
|
||||
zap.String("expected", fmt.Sprintf("%T", k)), logs.TagField(logs.TagDatapath))
|
||||
continue
|
||||
}
|
||||
if cnr.Equals(k.cid) && strings.HasPrefix(objectName, k.prefix) {
|
||||
|
|
2
api/cache/policy.go
vendored
2
api/cache/policy.go
vendored
|
@ -54,7 +54,7 @@ func (o *MorphPolicyCache) Get(key MorphPolicyCacheKey) []*chain.Chain {
|
|||
result, ok := entry.([]*chain.Chain)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
8
api/cache/system.go
vendored
8
api/cache/system.go
vendored
|
@ -50,7 +50,7 @@ func (o *SystemCache) GetObject(key string) *data.ObjectInfo {
|
|||
result, ok := entry.(*data.ObjectInfo)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -81,7 +81,7 @@ func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration {
|
|||
result, ok := entry.(*data.CORSConfiguration)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -97,7 +97,7 @@ func (o *SystemCache) GetLifecycleConfiguration(key string) *data.LifecycleConfi
|
|||
result, ok := entry.(*data.LifecycleConfiguration)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -113,7 +113,7 @@ func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
|
|||
result, ok := entry.(*data.BucketSettings)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
@ -6,6 +6,7 @@ import (
|
|||
"time"
|
||||
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
|
@ -32,6 +33,7 @@ type (
|
|||
LocationConstraint string
|
||||
ObjectLockEnabled bool
|
||||
HomomorphicHashDisabled bool
|
||||
PlacementPolicy netmap.PlacementPolicy
|
||||
}
|
||||
|
||||
// ObjectInfo holds S3 object data.
|
||||
|
|
|
@ -62,7 +62,7 @@ func (e ExtendedObjectInfo) Version() string {
|
|||
// Basically used for "system" object.
|
||||
type BaseNodeVersion struct {
|
||||
ID uint64
|
||||
ParenID uint64
|
||||
ParentID uint64
|
||||
OID oid.ID
|
||||
Timestamp uint64
|
||||
Size uint64
|
||||
|
@ -109,7 +109,6 @@ type MultipartInfo struct {
|
|||
Owner user.ID
|
||||
Created time.Time
|
||||
Meta map[string]string
|
||||
CopiesNumbers []uint32
|
||||
Finished bool
|
||||
CreationEpoch uint64
|
||||
}
|
||||
|
|
|
@ -57,9 +57,9 @@ const (
|
|||
ErrInvalidCopyDest
|
||||
ErrInvalidPolicyDocument
|
||||
ErrInvalidObjectState
|
||||
ErrMalformedACL
|
||||
ErrMalformedXML
|
||||
ErrMissingContentLength
|
||||
ErrMissingContentMD5
|
||||
ErrMissingRequestBodyError
|
||||
ErrMissingSecurityHeader
|
||||
ErrNoSuchBucket
|
||||
|
@ -289,6 +289,9 @@ const (
|
|||
//CORS configuration errors.
|
||||
ErrCORSUnsupportedMethod
|
||||
ErrCORSWildcardExposeHeaders
|
||||
|
||||
// Limits errors.
|
||||
ErrLimitExceeded
|
||||
)
|
||||
|
||||
// error code to Error structure, these fields carry respective
|
||||
|
@ -456,6 +459,12 @@ var errorCodes = errorCodeMap{
|
|||
Description: "The requested range is not satisfiable",
|
||||
HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
|
||||
},
|
||||
ErrMalformedACL: {
|
||||
ErrCode: ErrMalformedACL,
|
||||
Code: "MalformedACLError",
|
||||
Description: "The ACL that you provided was not well formed or did not validate against our published schema.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrMalformedXML: {
|
||||
ErrCode: ErrMalformedXML,
|
||||
Code: "MalformedXML",
|
||||
|
@ -468,12 +477,6 @@ var errorCodes = errorCodeMap{
|
|||
Description: "You must provide the Content-Length HTTP header.",
|
||||
HTTPStatusCode: http.StatusLengthRequired,
|
||||
},
|
||||
ErrMissingContentMD5: {
|
||||
ErrCode: ErrMissingContentMD5,
|
||||
Code: "MissingContentMD5",
|
||||
Description: "Missing required header for this request: Content-Md5.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrMissingSecurityHeader: {
|
||||
ErrCode: ErrMissingSecurityHeader,
|
||||
Code: "MissingSecurityHeader",
|
||||
|
@ -1763,6 +1766,14 @@ var errorCodes = errorCodeMap{
|
|||
Description: "Content-Range header is mandatory for this type of request",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
// The Conflict status is used because this error was made based on the LimitExceeded error
|
||||
// from aws iam error https://docs.aws.amazon.com/IAM/latest/APIReference/API_CreateUser.html#API_CreateUser_Errors.
|
||||
ErrLimitExceeded: {
|
||||
ErrCode: ErrLimitExceeded,
|
||||
Code: "LimitExceeded",
|
||||
Description: "You have reached the quota limit.",
|
||||
HTTPStatusCode: http.StatusConflict,
|
||||
},
|
||||
// Add your error structure here.
|
||||
}
|
||||
|
||||
|
@ -1826,6 +1837,10 @@ func TransformToS3Error(err error) error {
|
|||
return GetAPIError(ErrBucketAlreadyExists)
|
||||
}
|
||||
|
||||
if errors.Is(err, frostfs.ErrQuotaLimitReached) {
|
||||
return GetAPIError(ErrLimitExceeded)
|
||||
}
|
||||
|
||||
return GetAPIError(ErrInternalError)
|
||||
}
|
||||
|
||||
|
|
|
@ -99,7 +99,7 @@ func (h *handler) encodePrivateCannedACL(ctx context.Context, bktInfo *data.Buck
|
|||
ownerEncodedID := ownerDisplayName
|
||||
|
||||
if settings.OwnerKey == nil {
|
||||
h.reqLogger(ctx).Warn(logs.BucketOwnerKeyIsMissing, zap.String("owner", bktInfo.Owner.String()))
|
||||
h.reqLogger(ctx).Warn(logs.BucketOwnerKeyIsMissing, zap.String("owner", bktInfo.Owner.String()), logs.TagField(logs.TagDatapath))
|
||||
} else {
|
||||
ownerDisplayName = settings.OwnerKey.Address()
|
||||
ownerEncodedID = hex.EncodeToString(settings.OwnerKey.Bytes())
|
||||
|
@ -150,7 +150,7 @@ func (h *handler) putBucketACLAPEHandler(w http.ResponseWriter, r *http.Request,
|
|||
|
||||
defer func() {
|
||||
if errBody := r.Body.Close(); errBody != nil {
|
||||
h.reqLogger(ctx).Warn(logs.CouldNotCloseRequestBody, zap.Error(errBody))
|
||||
h.reqLogger(ctx).Warn(logs.CouldNotCloseRequestBody, zap.Error(errBody), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}()
|
||||
|
||||
|
@ -382,7 +382,7 @@ func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
|
|||
h.logAndSendError(ctx, w, "could not convert s3 policy to native chain policy", reqInfo, err)
|
||||
return
|
||||
} else {
|
||||
h.reqLogger(ctx).Warn(logs.PolicyCouldntBeConvertedToNativeRules)
|
||||
h.reqLogger(ctx).Warn(logs.PolicyCouldntBeConvertedToNativeRules, logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
chainsToSave := []*chain.Chain{s3Chain}
|
||||
|
|
|
@ -7,6 +7,7 @@ import (
|
|||
"encoding/xml"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
|
@ -297,10 +298,17 @@ type createBucketInfo struct {
|
|||
Key *keys.PrivateKey
|
||||
}
|
||||
|
||||
type bucketPrm struct {
|
||||
bktName string
|
||||
query url.Values
|
||||
box *accessbox.Box
|
||||
createParams createBucketParams
|
||||
}
|
||||
|
||||
func createBucket(hc *handlerContext, bktName string) *createBucketInfo {
|
||||
box, key := createAccessBox(hc.t)
|
||||
|
||||
w := createBucketBase(hc, bktName, box)
|
||||
w := createBucketBase(hc, bucketPrm{bktName: bktName, box: box})
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
|
||||
bktInfo, err := hc.Layer().GetBucketInfo(hc.Context(), bktName)
|
||||
|
@ -314,13 +322,32 @@ func createBucket(hc *handlerContext, bktName string) *createBucketInfo {
|
|||
}
|
||||
|
||||
func createBucketAssertS3Error(hc *handlerContext, bktName string, box *accessbox.Box, code apierr.ErrorCode) {
|
||||
w := createBucketBase(hc, bktName, box)
|
||||
w := createBucketBase(hc, bucketPrm{bktName: bktName, box: box})
|
||||
assertS3Error(hc.t, w, apierr.GetAPIError(code))
|
||||
}
|
||||
|
||||
func createBucketBase(hc *handlerContext, bktName string, box *accessbox.Box) *httptest.ResponseRecorder {
|
||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
||||
func createBucketWithConstraint(hc *handlerContext, bktName, constraint string) *createBucketInfo {
|
||||
box, key := createAccessBox(hc.t)
|
||||
var prm createBucketParams
|
||||
if constraint != "" {
|
||||
prm.LocationConstraint = constraint
|
||||
}
|
||||
w := createBucketBase(hc, bucketPrm{bktName: bktName, box: box, createParams: prm})
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
|
||||
bktInfo, err := hc.Layer().GetBucketInfo(hc.Context(), bktName)
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
return &createBucketInfo{
|
||||
BktInfo: bktInfo,
|
||||
Box: box,
|
||||
Key: key,
|
||||
}
|
||||
}
|
||||
|
||||
func createBucketBase(hc *handlerContext, prm bucketPrm) *httptest.ResponseRecorder {
|
||||
w, r := prepareTestFullRequest(hc, prm.bktName, "", nil, prm.createParams)
|
||||
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: prm.box})
|
||||
r = r.WithContext(ctx)
|
||||
hc.Handler().CreateBucketHandler(w, r)
|
||||
return w
|
||||
|
|
70
api/handler/bucket_list.go
Normal file
70
api/handler/bucket_list.go
Normal file
|
@ -0,0 +1,70 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
)
|
||||
|
||||
const maxBucketList = 10000
|
||||
|
||||
// ListBucketsHandler handles bucket listing requests.
|
||||
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
|
||||
params, err := parseListBucketParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "failed to parse params", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := h.obj.ListBuckets(ctx, params)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, encodeListBuckets(reqInfo.User, resp, params)); err != nil {
|
||||
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func encodeListBuckets(owner string, resp layer.ListBucketsResult, params layer.ListBucketsParams) *ListBucketsResponse {
|
||||
res := &ListBucketsResponse{
|
||||
Owner: Owner{
|
||||
ID: owner,
|
||||
DisplayName: owner,
|
||||
},
|
||||
ContinuationToken: resp.ContinuationToken,
|
||||
Prefix: params.Prefix,
|
||||
}
|
||||
|
||||
for _, item := range resp.Containers {
|
||||
res.Buckets.Buckets = append(res.Buckets.Buckets, Bucket{
|
||||
Name: item.Name,
|
||||
CreationDate: item.Created.UTC().Format(time.RFC3339),
|
||||
BucketRegion: item.LocationConstraint,
|
||||
})
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func parseListBucketParams(r *http.Request) (prm layer.ListBucketsParams, err error) {
|
||||
prm.MaxBuckets = maxBucketList
|
||||
strMaxBuckets := r.URL.Query().Get(middleware.QueryMaxBuckets)
|
||||
if strMaxBuckets != "" {
|
||||
if prm.MaxBuckets, err = strconv.Atoi(strMaxBuckets); err != nil || prm.MaxBuckets < 0 {
|
||||
return layer.ListBucketsParams{}, errors.GetAPIError(errors.ErrInvalidMaxKeys)
|
||||
}
|
||||
}
|
||||
prm.Prefix = r.URL.Query().Get(middleware.QueryPrefix)
|
||||
prm.BucketRegion = r.URL.Query().Get(middleware.QueryBucketRegion)
|
||||
prm.ContinuationToken = r.URL.Query().Get(middleware.QueryContinuationToken)
|
||||
|
||||
return
|
||||
}
|
174
api/handler/bucket_list_test.go
Normal file
174
api/handler/bucket_list_test.go
Normal file
|
@ -0,0 +1,174 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHandler_ListBucketsHandler(t *testing.T) {
|
||||
const defaultConstraint = "default"
|
||||
|
||||
region := "us-west-1"
|
||||
hc := prepareHandlerContext(t)
|
||||
hc.config.putLocationConstraint(region)
|
||||
|
||||
props := []Bucket{
|
||||
{Name: "first"},
|
||||
{Name: "regional", BucketRegion: "us-west-1"},
|
||||
{Name: "third"},
|
||||
}
|
||||
sort.Slice(props, func(i, j int) bool {
|
||||
return props[i].Name < props[j].Name
|
||||
})
|
||||
for _, bkt := range props {
|
||||
createBucketWithConstraint(hc, bkt.Name, bkt.BucketRegion)
|
||||
}
|
||||
|
||||
for _, tt := range []struct {
|
||||
title string
|
||||
token string
|
||||
prefix string
|
||||
bucketRegion string
|
||||
maxBuckets string
|
||||
expectErr bool
|
||||
expected []Bucket
|
||||
expectedToken string
|
||||
}{
|
||||
{
|
||||
title: "no params",
|
||||
expected: []Bucket{
|
||||
{Name: "first", BucketRegion: defaultConstraint},
|
||||
{Name: "regional", BucketRegion: "us-west-1"},
|
||||
{Name: "third", BucketRegion: defaultConstraint},
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "negative max-buckets",
|
||||
maxBuckets: "-1",
|
||||
expected: []Bucket{},
|
||||
expectErr: true,
|
||||
},
|
||||
{
|
||||
title: "zero max-buckets",
|
||||
maxBuckets: "0",
|
||||
expected: []Bucket{},
|
||||
},
|
||||
{
|
||||
title: "prefix",
|
||||
prefix: "thi",
|
||||
expected: []Bucket{{Name: "third", BucketRegion: defaultConstraint}},
|
||||
},
|
||||
{
|
||||
title: "wrong prefix",
|
||||
prefix: "sdh",
|
||||
expected: []Bucket{},
|
||||
},
|
||||
{
|
||||
title: "bucket region",
|
||||
bucketRegion: region,
|
||||
expected: []Bucket{{Name: "regional", BucketRegion: "us-west-1"}},
|
||||
},
|
||||
{
|
||||
title: "default bucket region",
|
||||
bucketRegion: defaultConstraint,
|
||||
expected: []Bucket{
|
||||
{Name: "first", BucketRegion: defaultConstraint},
|
||||
{Name: "third", BucketRegion: defaultConstraint},
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "wrong bucket region",
|
||||
bucketRegion: "sj dfdlsj",
|
||||
expected: []Bucket{},
|
||||
},
|
||||
} {
|
||||
t.Run(tt.title, func(t *testing.T) {
|
||||
if tt.expectErr {
|
||||
listBucketsErr(hc, tt.prefix, tt.token, tt.bucketRegion, tt.maxBuckets, apierr.GetAPIError(apierr.ErrInvalidMaxKeys))
|
||||
return
|
||||
}
|
||||
|
||||
resp := listBuckets(hc, tt.prefix, tt.token, tt.bucketRegion, tt.maxBuckets)
|
||||
require.Len(t, resp.Buckets.Buckets, len(tt.expected))
|
||||
require.Equal(t, tt.prefix, resp.Prefix)
|
||||
require.Equal(t, hc.owner.String(), resp.Owner.ID)
|
||||
if len(resp.Buckets.Buckets) > 0 {
|
||||
t.Log(resp.Buckets.Buckets[0].Name)
|
||||
}
|
||||
for i, bkt := range resp.Buckets.Buckets {
|
||||
require.Equal(t, tt.expected[i].Name, bkt.Name)
|
||||
require.Equal(t, tt.expected[i].BucketRegion, bkt.BucketRegion)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("pagination", func(t *testing.T) {
|
||||
t.Run("happy path", func(t *testing.T) {
|
||||
resp := listBuckets(hc, "", "", "", "1")
|
||||
require.Len(t, resp.Buckets.Buckets, 1)
|
||||
require.Equal(t, props[0].Name, resp.Buckets.Buckets[0].Name)
|
||||
require.NotEmpty(t, resp.ContinuationToken)
|
||||
|
||||
resp = listBuckets(hc, "", resp.ContinuationToken, "", "1")
|
||||
require.Len(t, resp.Buckets.Buckets, 1)
|
||||
require.Equal(t, props[1].Name, resp.Buckets.Buckets[0].Name)
|
||||
require.NotEmpty(t, resp.ContinuationToken)
|
||||
|
||||
resp = listBuckets(hc, "", resp.ContinuationToken, "", "1")
|
||||
require.Len(t, resp.Buckets.Buckets, 1)
|
||||
require.Equal(t, props[2].Name, resp.Buckets.Buckets[0].Name)
|
||||
require.Empty(t, resp.ContinuationToken)
|
||||
})
|
||||
|
||||
t.Run("wrong continuation-token", func(t *testing.T) {
|
||||
resp := listBuckets(hc, "", "CebuVwfRpdMqi9dvgV2SUNbrkfteGtudchKKhNabXUu9", "", "1")
|
||||
require.Len(t, resp.Buckets.Buckets, 0)
|
||||
require.Empty(t, resp.ContinuationToken)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func listBuckets(hc *handlerContext, prefix, token, bucketRegion, maxBuckets string) ListBucketsResponse {
|
||||
query := url.Values{
|
||||
middleware.QueryPrefix: []string{prefix},
|
||||
middleware.QueryContinuationToken: []string{token},
|
||||
middleware.QueryBucketRegion: []string{bucketRegion},
|
||||
middleware.QueryMaxBuckets: []string{maxBuckets},
|
||||
}
|
||||
w := listBucketsBase(hc, bucketPrm{query: query})
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
var resp ListBucketsResponse
|
||||
err := xml.NewDecoder(w.Body).Decode(&resp)
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
return resp
|
||||
}
|
||||
|
||||
func listBucketsErr(hc *handlerContext, prefix, token, bucketRegion, maxBuckets string, err apierr.Error) {
|
||||
query := url.Values{
|
||||
middleware.QueryPrefix: []string{prefix},
|
||||
middleware.QueryContinuationToken: []string{token},
|
||||
middleware.QueryBucketRegion: []string{bucketRegion},
|
||||
middleware.QueryMaxBuckets: []string{maxBuckets},
|
||||
}
|
||||
w := listBucketsBase(hc, bucketPrm{query: query})
|
||||
assertS3Error(hc.t, w, err)
|
||||
}
|
||||
|
||||
func listBucketsBase(hc *handlerContext, prm bucketPrm) *httptest.ResponseRecorder {
|
||||
box, _ := createAccessBox(hc.t)
|
||||
w, r := prepareTestFullRequest(hc, "", "", prm.query, nil)
|
||||
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
||||
r = r.WithContext(ctx)
|
||||
hc.Handler().ListBucketsHandler(w, r)
|
||||
|
||||
return w
|
||||
}
|
|
@ -244,7 +244,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
}
|
||||
|
||||
h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID))
|
||||
h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID), logs.TagField(logs.TagExternalStorage))
|
||||
|
||||
if dstEncryptionParams.Enabled() {
|
||||
addSSECHeaders(w.Header(), r.Header)
|
||||
|
|
|
@ -29,7 +29,7 @@ func (h *handler) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
cors, err := h.obj.GetBucketCORS(ctx, bktInfo)
|
||||
cors, err := h.obj.GetBucketCORS(ctx, bktInfo, h.cfg.NewXMLDecoder)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "could not get cors", reqInfo, err)
|
||||
return
|
||||
|
@ -108,13 +108,13 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
bktInfo, err := h.getBucketInfo(ctx, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err))
|
||||
h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
return
|
||||
}
|
||||
|
||||
cors, err := h.obj.GetBucketCORS(ctx, bktInfo)
|
||||
cors, err := h.obj.GetBucketCORS(ctx, bktInfo, h.cfg.NewXMLDecoder)
|
||||
if err != nil {
|
||||
h.reqLogger(ctx).Warn(logs.GetBucketCors, zap.Error(err))
|
||||
h.reqLogger(ctx).Warn(logs.GetBucketCors, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -178,7 +178,7 @@ func (h *handler) Preflight(w http.ResponseWriter, r *http.Request) {
|
|||
headers = strings.Split(requestHeaders, ", ")
|
||||
}
|
||||
|
||||
cors, err := h.obj.GetBucketCORS(ctx, bktInfo)
|
||||
cors, err := h.obj.GetBucketCORS(ctx, bktInfo, h.cfg.NewXMLDecoder)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "could not get cors", reqInfo, err)
|
||||
return
|
||||
|
|
|
@ -19,7 +19,14 @@ func TestCORSOriginWildcard(t *testing.T) {
|
|||
</CORSRule>
|
||||
</CORSConfiguration>
|
||||
`
|
||||
hc := prepareHandlerContext(t)
|
||||
bodyNoXmlns := `
|
||||
<CORSConfiguration>
|
||||
<CORSRule>
|
||||
<AllowedMethod>GET</AllowedMethod>
|
||||
<AllowedOrigin>*</AllowedOrigin>
|
||||
</CORSRule>
|
||||
</CORSConfiguration>`
|
||||
hc := prepareHandlerContextWithMinCache(t)
|
||||
|
||||
bktName := "bucket-for-cors"
|
||||
box, _ := createAccessBox(t)
|
||||
|
@ -39,6 +46,17 @@ func TestCORSOriginWildcard(t *testing.T) {
|
|||
w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
|
||||
hc.Handler().GetBucketCorsHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
hc.config.useDefaultXMLNS = true
|
||||
w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(bodyNoXmlns))
|
||||
ctx = middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
||||
r = r.WithContext(ctx)
|
||||
hc.Handler().PutBucketCorsHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
|
||||
hc.Handler().GetBucketCorsHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func TestPreflight(t *testing.T) {
|
||||
|
|
|
@ -131,13 +131,6 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
|||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
|
||||
// Content-Md5 is required and should be set
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if _, ok := r.Header[api.ContentMD5]; !ok {
|
||||
h.logAndSendError(ctx, w, "missing Content-MD5", reqInfo, errors.GetAPIError(errors.ErrMissingContentMD5))
|
||||
return
|
||||
}
|
||||
|
||||
// Content-Length is required and should be non-zero
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if r.ContentLength <= 0 {
|
||||
|
|
|
@ -542,7 +542,6 @@ func deleteObjectsBase(hc *handlerContext, bktName string, objVersions [][2]stri
|
|||
}
|
||||
|
||||
w, r := prepareTestRequest(hc, bktName, "", req)
|
||||
r.Header.Set(api.ContentMD5, "")
|
||||
hc.Handler().DeleteMultipleObjectsHandler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
|
||||
|
|
|
@ -65,10 +65,16 @@ func TestMD5HeaderBadOrEmpty(t *testing.T) {
|
|||
putEncryptedObjectWithHeadersErr(t, tc, bktName, objName, content, headers, errors.ErrInvalidDigest)
|
||||
|
||||
headers = map[string]string{
|
||||
api.ContentMD5: "YWJjMTIzIT8kKiYoKSctPUB+",
|
||||
api.ContentMD5: "yZRvHQZYwL5V7+k2pcwHLg==",
|
||||
}
|
||||
|
||||
putEncryptedObjectWithHeadersErr(t, tc, bktName, objName, content, headers, errors.ErrBadDigest)
|
||||
|
||||
headers = map[string]string{
|
||||
api.ContentMD5: "dGhlIHF1aWNrIGJyb3dF",
|
||||
}
|
||||
|
||||
putEncryptedObjectWithHeadersErr(t, tc, bktName, objName, content, headers, errors.ErrInvalidDigest)
|
||||
}
|
||||
|
||||
func TestGetEncryptedRange(t *testing.T) {
|
||||
|
|
|
@ -296,12 +296,12 @@ func parseConditionalHeaders(headers http.Header, log *zap.Logger) *conditionalA
|
|||
if httpTime, err := parseHTTPTime(headers.Get(api.IfModifiedSince)); err == nil {
|
||||
args.IfModifiedSince = httpTime
|
||||
} else {
|
||||
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfModifiedSince, headers.Get(api.IfModifiedSince)), zap.Error(err))
|
||||
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfModifiedSince, headers.Get(api.IfModifiedSince)), zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil {
|
||||
args.IfUnmodifiedSince = httpTime
|
||||
} else {
|
||||
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err))
|
||||
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
return args
|
||||
|
|
|
@ -197,6 +197,33 @@ func TestGetObject(t *testing.T) {
|
|||
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, apierr.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
func TestGetDeletedObject(t *testing.T) {
|
||||
hc := prepareHandlerContextWithMinCache(t)
|
||||
bktName, objName := "bucket", "obj"
|
||||
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
|
||||
|
||||
putObject(hc, bktName, objName)
|
||||
|
||||
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
|
||||
checkFound(hc.t, hc, bktName, objName, emptyVersion)
|
||||
|
||||
addr := getAddressOfLastVersion(hc, bktInfo, objName)
|
||||
t.Run("not found error", func(_ *testing.T) {
|
||||
hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
|
||||
hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectNotFound{})
|
||||
|
||||
getObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), apierr.ErrNoSuchVersion)
|
||||
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, apierr.ErrNoSuchKey)
|
||||
})
|
||||
t.Run("already removed error", func(_ *testing.T) {
|
||||
hc.tp.SetObjectError(addr, &apistatus.ObjectAlreadyRemoved{})
|
||||
hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectAlreadyRemoved{})
|
||||
|
||||
getObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), apierr.ErrNoSuchVersion)
|
||||
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, apierr.ErrNoSuchKey)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetObjectEnabledMD5(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName, objName := "bucket", "obj"
|
||||
|
|
|
@ -74,19 +74,22 @@ func (hc *handlerContextBase) Context() context.Context {
|
|||
|
||||
type configMock struct {
|
||||
defaultPolicy netmap.PlacementPolicy
|
||||
placementPolicies map[string]netmap.PlacementPolicy
|
||||
copiesNumbers map[string][]uint32
|
||||
defaultCopiesNumbers []uint32
|
||||
bypassContentEncodingInChunks bool
|
||||
md5Enabled bool
|
||||
tlsTerminationHeader string
|
||||
useDefaultXMLNS bool
|
||||
}
|
||||
|
||||
func (c *configMock) DefaultPlacementPolicy(_ string) netmap.PlacementPolicy {
|
||||
return c.defaultPolicy
|
||||
}
|
||||
|
||||
func (c *configMock) PlacementPolicy(_, _ string) (netmap.PlacementPolicy, bool) {
|
||||
return netmap.PlacementPolicy{}, false
|
||||
func (c *configMock) PlacementPolicy(_, constraint string) (netmap.PlacementPolicy, bool) {
|
||||
policy, ok := c.placementPolicies[constraint]
|
||||
return policy, ok
|
||||
}
|
||||
|
||||
func (c *configMock) CopiesNumbers(_, locationConstraint string) ([]uint32, bool) {
|
||||
|
@ -99,7 +102,11 @@ func (c *configMock) DefaultCopiesNumbers(_ string) []uint32 {
|
|||
}
|
||||
|
||||
func (c *configMock) NewXMLDecoder(r io.Reader, _ string) *xml.Decoder {
|
||||
return xml.NewDecoder(r)
|
||||
dec := xml.NewDecoder(r)
|
||||
if c.useDefaultXMLNS {
|
||||
dec.DefaultSpace = "http://s3.amazonaws.com/doc/2006-03-01/"
|
||||
}
|
||||
return dec
|
||||
}
|
||||
|
||||
func (c *configMock) BypassContentEncodingInChunks(_ string) bool {
|
||||
|
@ -146,6 +153,10 @@ func (c *configMock) TLSTerminationHeader() string {
|
|||
return c.tlsTerminationHeader
|
||||
}
|
||||
|
||||
func (c *configMock) putLocationConstraint(constraint string) {
|
||||
c.placementPolicies[constraint] = c.defaultPolicy
|
||||
}
|
||||
|
||||
func prepareHandlerContext(t *testing.T) *handlerContext {
|
||||
hc, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(zap.NewExample()))
|
||||
require.NoError(t, err)
|
||||
|
@ -212,7 +223,8 @@ func prepareHandlerContextBase(cacheCfg *layer.CachesConfig) (*handlerContextBas
|
|||
}
|
||||
|
||||
cfg := &configMock{
|
||||
defaultPolicy: pp,
|
||||
defaultPolicy: pp,
|
||||
placementPolicies: make(map[string]netmap.PlacementPolicy),
|
||||
}
|
||||
h := &handler{
|
||||
log: log,
|
||||
|
@ -256,7 +268,7 @@ func getMinCacheConfig(logger *zap.Logger) *layer.CachesConfig {
|
|||
Buckets: minCacheCfg,
|
||||
System: minCacheCfg,
|
||||
AccessControl: minCacheCfg,
|
||||
NetworkInfo: &cache.NetworkInfoCacheConfig{Lifetime: minCacheCfg.Lifetime},
|
||||
Network: &cache.NetworkCacheConfig{Lifetime: minCacheCfg.Lifetime},
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -404,6 +416,7 @@ func createTestBucketWithLock(hc *handlerContext, bktName string, conf *data.Obj
|
|||
Creator: hc.owner,
|
||||
Name: bktName,
|
||||
AdditionalAttributes: [][2]string{{layer.AttributeLockEnabled, "true"}},
|
||||
Policy: getPlacementPolicy(),
|
||||
})
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
|
@ -415,6 +428,7 @@ func createTestBucketWithLock(hc *handlerContext, bktName string, conf *data.Obj
|
|||
ObjectLockEnabled: true,
|
||||
Owner: ownerID,
|
||||
HomomorphicHashDisabled: res.HomomorphicHashDisabled,
|
||||
PlacementPolicy: getPlacementPolicy(),
|
||||
}
|
||||
|
||||
key, err := keys.NewPrivateKey()
|
||||
|
@ -534,3 +548,10 @@ func readResponse(t *testing.T, w *httptest.ResponseRecorder, status int, model
|
|||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func getPlacementPolicy() (p netmap.PlacementPolicy) {
|
||||
var r netmap.ReplicaDescriptor
|
||||
r.SetNumberOfObjects(1)
|
||||
p.AddReplicas([]netmap.ReplicaDescriptor{r}...)
|
||||
return p
|
||||
}
|
||||
|
|
|
@ -55,34 +55,29 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
|
|||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
|
||||
// Content-Md5 is required and should be set
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html
|
||||
if _, ok := r.Header[api.ContentMD5]; !ok {
|
||||
h.logAndSendError(ctx, w, "missing Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrMissingContentMD5))
|
||||
return
|
||||
}
|
||||
|
||||
headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5))
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
||||
return
|
||||
}
|
||||
|
||||
cfg := new(data.LifecycleConfiguration)
|
||||
if err = h.cfg.NewXMLDecoder(tee, r.UserAgent()).Decode(cfg); err != nil {
|
||||
if err := h.cfg.NewXMLDecoder(tee, r.UserAgent()).Decode(cfg); err != nil {
|
||||
h.logAndSendError(ctx, w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
bodyMD5, err := getContentMD5(&buf)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "could not get content md5", reqInfo, err)
|
||||
return
|
||||
}
|
||||
if _, ok := r.Header[api.ContentMD5]; ok {
|
||||
headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5))
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
||||
return
|
||||
}
|
||||
|
||||
if !bytes.Equal(headerMD5, bodyMD5) {
|
||||
h.logAndSendError(ctx, w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
||||
return
|
||||
bodyMD5, err := getContentMD5(&buf)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "could not get content md5", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if !bytes.Equal(headerMD5, bodyMD5) {
|
||||
h.logAndSendError(ctx, w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
|
|
|
@ -29,6 +29,8 @@ func TestPutBucketLifecycleConfiguration(t *testing.T) {
|
|||
for _, tc := range []struct {
|
||||
name string
|
||||
body *data.LifecycleConfiguration
|
||||
headers map[string]string
|
||||
addMD5 bool
|
||||
errorCode apierr.ErrorCode
|
||||
}{
|
||||
{
|
||||
|
@ -70,6 +72,22 @@ func TestPutBucketLifecycleConfiguration(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
name: "correct Content-Md5 header",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
ID: "rule",
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(21),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
addMD5: true,
|
||||
},
|
||||
{
|
||||
name: "too many rules",
|
||||
body: func() *data.LifecycleConfiguration {
|
||||
|
@ -407,14 +425,44 @@ func TestPutBucketLifecycleConfiguration(t *testing.T) {
|
|||
},
|
||||
errorCode: apierr.ErrInvalidRequest,
|
||||
},
|
||||
{
|
||||
name: "invalid Content-Md5 header",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(21),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
headers: map[string]string{api.ContentMD5: "invalid"},
|
||||
errorCode: apierr.ErrInvalidDigest,
|
||||
},
|
||||
{
|
||||
name: "Content-Md5 header does not match body md5 hash",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(21),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
headers: map[string]string{api.ContentMD5: base64.StdEncoding.EncodeToString([]byte("some-hash"))},
|
||||
errorCode: apierr.ErrInvalidDigest,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if tc.errorCode > 0 {
|
||||
putBucketLifecycleConfigurationErr(hc, bktName, tc.body, apierr.GetAPIError(tc.errorCode))
|
||||
putBucketLifecycleConfigurationErr(hc, bktName, tc.body, tc.headers, apierr.GetAPIError(tc.errorCode))
|
||||
return
|
||||
}
|
||||
|
||||
putBucketLifecycleConfiguration(hc, bktName, tc.body)
|
||||
putBucketLifecycleConfiguration(hc, bktName, tc.body, tc.headers, tc.addMD5)
|
||||
|
||||
cfg := getBucketLifecycleConfiguration(hc, bktName)
|
||||
require.Equal(t, tc.body.Rules, cfg.Rules)
|
||||
|
@ -448,45 +496,13 @@ func TestPutBucketLifecycleIDGeneration(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
putBucketLifecycleConfiguration(hc, bktName, lifecycle)
|
||||
putBucketLifecycleConfiguration(hc, bktName, lifecycle, nil, false)
|
||||
cfg := getBucketLifecycleConfiguration(hc, bktName)
|
||||
require.Len(t, cfg.Rules, 2)
|
||||
require.NotEmpty(t, cfg.Rules[0].ID)
|
||||
require.NotEmpty(t, cfg.Rules[1].ID)
|
||||
}
|
||||
|
||||
func TestPutBucketLifecycleInvalidMD5(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-lifecycle-md5"
|
||||
createBucket(hc, bktName)
|
||||
|
||||
lifecycle := &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(21),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
w, r := prepareTestRequest(hc, bktName, "", lifecycle)
|
||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMissingContentMD5))
|
||||
|
||||
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
|
||||
r.Header.Set(api.ContentMD5, "")
|
||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
||||
|
||||
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
|
||||
r.Header.Set(api.ContentMD5, "some-hash")
|
||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
||||
}
|
||||
|
||||
func TestPutBucketLifecycleInvalidXML(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
|
@ -505,25 +521,32 @@ func TestPutBucketLifecycleInvalidXML(t *testing.T) {
|
|||
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMalformedXML))
|
||||
}
|
||||
|
||||
func putBucketLifecycleConfiguration(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) {
|
||||
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
|
||||
func putBucketLifecycleConfiguration(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, headers map[string]string, addMD5 bool) {
|
||||
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg, headers, addMD5)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func putBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, err apierr.Error) {
|
||||
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
|
||||
func putBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, headers map[string]string, err apierr.Error) {
|
||||
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg, headers, false)
|
||||
assertS3Error(hc.t, w, err)
|
||||
}
|
||||
|
||||
func putBucketLifecycleConfigurationBase(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) *httptest.ResponseRecorder {
|
||||
func putBucketLifecycleConfigurationBase(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, headers map[string]string, addMD5 bool) *httptest.ResponseRecorder {
|
||||
w, r := prepareTestRequest(hc, bktName, "", cfg)
|
||||
|
||||
rawBody, err := xml.Marshal(cfg)
|
||||
require.NoError(hc.t, err)
|
||||
for k, v := range headers {
|
||||
r.Header.Set(k, v)
|
||||
}
|
||||
|
||||
if addMD5 {
|
||||
rawBody, err := xml.Marshal(cfg)
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
hash := md5.New()
|
||||
hash.Write(rawBody)
|
||||
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(hash.Sum(nil)))
|
||||
}
|
||||
|
||||
hash := md5.New()
|
||||
hash.Write(rawBody)
|
||||
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(hash.Sum(nil)))
|
||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||
return w
|
||||
}
|
||||
|
|
|
@ -1,49 +0,0 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
)
|
||||
|
||||
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||
|
||||
// ListBucketsHandler handles bucket listing requests.
|
||||
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
own user.ID
|
||||
res *ListBucketsResponse
|
||||
ctx = r.Context()
|
||||
reqInfo = middleware.GetReqInfo(ctx)
|
||||
)
|
||||
|
||||
list, err := h.obj.ListBuckets(ctx)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(list) > 0 {
|
||||
own = list[0].Owner
|
||||
}
|
||||
|
||||
res = &ListBucketsResponse{
|
||||
Owner: Owner{
|
||||
ID: own.String(),
|
||||
DisplayName: own.String(),
|
||||
},
|
||||
}
|
||||
|
||||
for _, item := range list {
|
||||
res.Buckets.Buckets = append(res.Buckets.Buckets, Bucket{
|
||||
Name: item.Name,
|
||||
CreationDate: item.Created.UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, res); err != nil {
|
||||
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
|
@ -152,12 +152,6 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
|
|||
p.Header[api.ContentLanguage] = contentLanguage
|
||||
}
|
||||
|
||||
p.CopiesNumbers, err = h.pickCopiesNumbers(p.Header, reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
|
||||
if err = h.obj.CreateMultipartUpload(ctx, p); err != nil {
|
||||
h.logAndSendError(ctx, w, "could create multipart upload", reqInfo, err, additional...)
|
||||
return
|
||||
|
@ -229,6 +223,12 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
|
||||
hash, err := h.obj.UploadPart(ctx, p)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "could not upload a part", reqInfo, err, additional...)
|
||||
|
@ -354,6 +354,12 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
|
||||
info, err := h.obj.UploadPartCopy(ctx, p)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "could not upload part copy", reqInfo, err, additional...)
|
||||
|
@ -416,6 +422,12 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
|||
Parts: reqBody.Parts,
|
||||
}
|
||||
|
||||
c.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
|
||||
// Start complete multipart upload which may take some time to fetch object
|
||||
// and re-upload it part by part.
|
||||
objInfo, err := h.completeMultipartUpload(r, c, bktInfo)
|
||||
|
|
|
@ -81,6 +81,26 @@ func TestDeleteMultipartAllParts(t *testing.T) {
|
|||
require.Empty(t, hc.tp.Objects())
|
||||
}
|
||||
|
||||
func TestMultipartCopiesNumber(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket", "object"
|
||||
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
copies := []uint32{2, 0}
|
||||
|
||||
hc.config.copiesNumbers = map[string][]uint32{"default": copies}
|
||||
|
||||
multipartInfo := createMultipartUpload(hc, bktName, objName, nil)
|
||||
uploadPart(hc, bktName, objName, multipartInfo.UploadID, 1, layer.UploadMinSize)
|
||||
|
||||
objs := hc.tp.Objects()
|
||||
require.Len(t, objs, 1)
|
||||
|
||||
require.EqualValues(t, copies, hc.tp.CopiesNumbers(addrFromObject(objs[0]).EncodeToString()))
|
||||
}
|
||||
|
||||
func TestSpecialMultipartName(t *testing.T) {
|
||||
hc := prepareHandlerContextWithMinCache(t)
|
||||
|
||||
|
@ -792,3 +812,14 @@ func listPartsBase(hc *handlerContext, bktName, objName string, encrypted bool,
|
|||
|
||||
return listPartsResponse
|
||||
}
|
||||
|
||||
func addrFromObject(obj *object.Object) oid.Address {
|
||||
var addr oid.Address
|
||||
cnrID, _ := obj.ContainerID()
|
||||
objID, _ := obj.ID()
|
||||
|
||||
addr.SetContainer(cnrID)
|
||||
addr.SetObject(objID)
|
||||
|
||||
return addr
|
||||
}
|
||||
|
|
|
@ -15,6 +15,8 @@ import (
|
|||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||
|
||||
// ListObjectsV1Handler handles objects listing requests for API version 1.
|
||||
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
|
|
@ -142,7 +142,7 @@ func parsePatchConditionalHeaders(headers http.Header, log *zap.Logger) *conditi
|
|||
if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil {
|
||||
args.IfUnmodifiedSince = httpTime
|
||||
} else {
|
||||
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err))
|
||||
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
return args
|
||||
|
|
|
@ -311,10 +311,23 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
}
|
||||
|
||||
func (h *handler) getBodyReader(r *http.Request) (io.ReadCloser, error) {
|
||||
type BodyReader interface {
|
||||
io.ReadCloser
|
||||
TrailerHeaders() map[string]string
|
||||
}
|
||||
|
||||
type noTrailerBodyReader struct {
|
||||
io.ReadCloser
|
||||
}
|
||||
|
||||
func (r *noTrailerBodyReader) TrailerHeaders() map[string]string {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *handler) getBodyReader(r *http.Request) (BodyReader, error) {
|
||||
shaType, streaming := api.IsSignedStreamingV4(r)
|
||||
if !streaming {
|
||||
return r.Body, nil
|
||||
return &noTrailerBodyReader{r.Body}, nil
|
||||
}
|
||||
|
||||
encodings := r.Header.Values(api.ContentEncoding)
|
||||
|
@ -350,12 +363,15 @@ func (h *handler) getBodyReader(r *http.Request) (io.ReadCloser, error) {
|
|||
|
||||
var (
|
||||
err error
|
||||
chunkReader io.ReadCloser
|
||||
chunkReader BodyReader
|
||||
)
|
||||
if shaType == api.StreamingContentV4aSHA256 {
|
||||
chunkReader, err = newSignV4aChunkedReader(r)
|
||||
} else {
|
||||
switch shaType {
|
||||
case api.StreamingContentSHA256, api.StreamingContentSHA256Trailer:
|
||||
chunkReader, err = newSignV4ChunkedReader(r)
|
||||
case api.StreamingContentV4aSHA256, api.StreamingContentV4aSHA256Trailer:
|
||||
chunkReader, err = newSignV4aChunkedReader(r)
|
||||
default:
|
||||
chunkReader, err = newUnsignedChunkedReader(r.Body)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
@ -450,7 +466,7 @@ func (h *handler) isTLSCheckRequired(r *http.Request) bool {
|
|||
|
||||
tlsTermination, err := strconv.ParseBool(tlsTerminationStr)
|
||||
if err != nil {
|
||||
h.reqLogger(r.Context()).Warn(logs.WarnInvalidTypeTLSTerminationHeader, zap.String("header", tlsTerminationStr), zap.Error(err))
|
||||
h.reqLogger(r.Context()).Warn(logs.WarnInvalidTypeTLSTerminationHeader, zap.String("header", tlsTerminationStr), zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
return true
|
||||
}
|
||||
|
||||
|
@ -744,7 +760,7 @@ func parseCannedACL(header http.Header) (string, error) {
|
|||
return acl, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("unknown acl: %s", acl)
|
||||
return "", apierr.GetAPIErrorWithError(apierr.ErrMalformedACL, fmt.Errorf("unknown acl: %s", acl))
|
||||
}
|
||||
|
||||
func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
@ -812,7 +828,7 @@ func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Reque
|
|||
h.logAndSendError(ctx, w, "could not create bucket", reqInfo, err)
|
||||
return
|
||||
}
|
||||
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
|
||||
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID), logs.TagField(logs.TagExternalStorage))
|
||||
|
||||
chains := bucketCannedACLToAPERules(cannedACL, reqInfo, bktInfo.CID)
|
||||
if err = h.ape.SaveACLChains(bktInfo.CID.EncodeToString(), chains); err != nil {
|
||||
|
@ -1025,7 +1041,7 @@ func isLockEnabled(log *zap.Logger, header http.Header) bool {
|
|||
|
||||
lockEnabled, err := strconv.ParseBool(lockEnabledStr)
|
||||
if err != nil {
|
||||
log.Warn(logs.InvalidBucketObjectLockEnabledHeader, zap.String("header", lockEnabledStr), zap.Error(err))
|
||||
log.Warn(logs.InvalidBucketObjectLockEnabledHeader, zap.String("header", lockEnabledStr), zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
return lockEnabled
|
||||
|
|
|
@ -4,6 +4,7 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
|
@ -282,12 +283,20 @@ func TestPutObjectWithInvalidContentMD5(t *testing.T) {
|
|||
createTestBucket(tc, bktName)
|
||||
|
||||
content := []byte("content")
|
||||
md5HeaderContent := make([]byte, md5.Size)
|
||||
n, err := rand.Read(md5HeaderContent)
|
||||
require.Equal(t, md5.Size, n)
|
||||
require.NoError(t, err)
|
||||
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
||||
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString([]byte("invalid")))
|
||||
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(md5HeaderContent))
|
||||
tc.Handler().PutObjectHandler(w, r)
|
||||
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrBadDigest))
|
||||
|
||||
content = []byte("content")
|
||||
w, r = prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
||||
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString([]byte("invalid")))
|
||||
tc.Handler().PutObjectHandler(w, r)
|
||||
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
||||
|
||||
w, r = prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
||||
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString([]byte("")))
|
||||
tc.Handler().PutObjectHandler(w, r)
|
||||
|
@ -368,6 +377,77 @@ func TestPutObjectCheckContentSHA256(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestPutObjectWithStreamUnsignedBodySmall(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "test2", "tmp.txt"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
w, req, chunk := getChunkedRequestUnsignedTrailingSmall(hc.context, t, bktName, objName)
|
||||
hc.Handler().PutObjectHandler(w, req)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
w, req = prepareTestRequest(hc, bktName, objName, nil)
|
||||
hc.Handler().HeadObjectHandler(w, req)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
require.Equal(t, "5", w.Header().Get(api.ContentLength))
|
||||
|
||||
data := getObjectRange(t, hc, bktName, objName, 0, 5)
|
||||
for i := range chunk {
|
||||
require.Equal(t, chunk[i], data[i])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutObjectWithStreamUnsignedBody(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "examplebucket", "chunkObject.txt"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
w, req, chunk := getChunkedRequestUnsignedTrailing(hc.context, t, bktName, objName)
|
||||
hc.Handler().PutObjectHandler(w, req)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
w, req = prepareTestRequest(hc, bktName, objName, nil)
|
||||
hc.Handler().HeadObjectHandler(w, req)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
require.Equal(t, strconv.Itoa(awsChunkedRequestExampleDecodedContentLength), w.Header().Get(api.ContentLength))
|
||||
|
||||
data := getObjectRange(t, hc, bktName, objName, 0, awsChunkedRequestExampleDecodedContentLength)
|
||||
for i := range chunk {
|
||||
require.Equal(t, chunk[i], data[i])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutObjectWithStreamBodyAWSExampleTrailing(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "examplebucket", "chunkObject.txt"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
t.Run("valid trailer signature", func(t *testing.T) {
|
||||
w, req, chunk := getChunkedRequestTrailing(hc.context, t, bktName, objName)
|
||||
hc.Handler().PutObjectHandler(w, req)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
w, req = prepareTestRequest(hc, bktName, objName, nil)
|
||||
hc.Handler().HeadObjectHandler(w, req)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
require.Equal(t, strconv.Itoa(awsChunkedRequestExampleDecodedContentLength), w.Header().Get(api.ContentLength))
|
||||
|
||||
data := getObjectRange(t, hc, bktName, objName, 0, awsChunkedRequestExampleDecodedContentLength)
|
||||
equalDataSlices(t, chunk, data)
|
||||
})
|
||||
|
||||
t.Run("invalid trailer signature", func(t *testing.T) {
|
||||
w, req, _ := getChunkedRequestTrailing(hc.context, t, bktName, objName)
|
||||
body := req.Body.(*customNopCloser)
|
||||
body.Bytes()[body.Len()-2] = 'a'
|
||||
hc.Handler().PutObjectHandler(w, req)
|
||||
assertStatus(t, w, http.StatusForbidden)
|
||||
})
|
||||
}
|
||||
|
||||
func TestPutObjectWithStreamBodyAWSExample(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
|
@ -467,9 +547,9 @@ func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName strin
|
|||
|
||||
req, err := http.NewRequest("PUT", "https://s3.amazonaws.com/"+bktName+"/"+objName, nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("content-encoding", "aws-chunked")
|
||||
req.Header.Set("content-encoding", api.AwsChunked)
|
||||
req.Header.Set("content-length", strconv.Itoa(awsChunkedRequestExampleContentLength))
|
||||
req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
|
||||
req.Header.Set("x-amz-content-sha256", api.StreamingContentSHA256)
|
||||
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
|
||||
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
||||
|
||||
|
@ -501,6 +581,202 @@ func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName strin
|
|||
return w, req, chunk
|
||||
}
|
||||
|
||||
type customNopCloser struct {
|
||||
*bytes.Buffer
|
||||
}
|
||||
|
||||
func (c *customNopCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// getChunkedRequestTrailing implements request example from
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html
|
||||
func getChunkedRequestTrailing(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
|
||||
chunk := make([]byte, 65*1024)
|
||||
for i := range chunk {
|
||||
chunk[i] = 'a'
|
||||
}
|
||||
chunk1 := chunk[:64*1024]
|
||||
chunk2 := chunk[64*1024:]
|
||||
|
||||
AWSAccessKeyID := "AKIAIOSFODNN7EXAMPLE"
|
||||
AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
|
||||
awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey}
|
||||
signer := v4.NewSigner()
|
||||
|
||||
reqBody := bytes.NewBufferString("10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n")
|
||||
_, err := reqBody.Write(chunk1)
|
||||
require.NoError(t, err)
|
||||
_, err = reqBody.WriteString("\r\n400;chunk-signature=1c1344b170168f8e65b41376b44b20fe354e373826ccbbe2c1d40a8cae51e5c7\r\n")
|
||||
require.NoError(t, err)
|
||||
_, err = reqBody.Write(chunk2)
|
||||
require.NoError(t, err)
|
||||
_, err = reqBody.WriteString("\r\n0;chunk-signature=2ca2aba2005185cf7159c6277faf83795951dd77a3a99e6e65d5c9f85863f992\r\n")
|
||||
require.NoError(t, err)
|
||||
_, err = reqBody.WriteString("x-amz-checksum-crc32c:sOO8/Q==\n")
|
||||
require.NoError(t, err)
|
||||
|
||||
// original signature is 63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f,
|
||||
// but we use d81f82fc3505edab99d459891051a732e8730629a2e4a59689829ca17fe2e435
|
||||
// because original signature is incorrect
|
||||
// it was calculated using the`AWS4-HMAC-SHA256-PAYLOAD` constant in canonical string instead of
|
||||
// `AWS4-HMAC-SHA256-TRAILER` that actually must be used by spec
|
||||
// (java sdk use correct `AWS4-HMAC-SHA256-TRAILER` string).
|
||||
_, err = reqBody.WriteString("x-amz-trailer-signature:d81f82fc3505edab99d459891051a732e8730629a2e4a59689829ca17fe2e435")
|
||||
require.NoError(t, err)
|
||||
|
||||
req, err := http.NewRequest("PUT", "https://s3.amazonaws.com/"+bktName+"/"+objName, nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("content-encoding", api.AwsChunked)
|
||||
req.Header.Set("content-length", strconv.Itoa(awsChunkedRequestExampleContentLength))
|
||||
req.Header.Set("x-amz-content-sha256", api.StreamingContentSHA256Trailer)
|
||||
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
|
||||
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
||||
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32c")
|
||||
|
||||
signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z")
|
||||
require.NoError(t, err)
|
||||
|
||||
err = signer.SignHTTP(ctx, awsCreds, req, api.StreamingContentSHA256Trailer, "s3", "us-east-1", signTime)
|
||||
require.NoError(t, err)
|
||||
|
||||
req.Body = &customNopCloser{Buffer: reqBody}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
||||
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
||||
req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{
|
||||
ClientTime: signTime,
|
||||
AuthHeaders: &middleware.AuthHeader{
|
||||
AccessKeyID: AWSAccessKeyID,
|
||||
SignatureV4: "106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e",
|
||||
Region: "us-east-1",
|
||||
},
|
||||
AccessBox: &accessbox.Box{
|
||||
Gate: &accessbox.GateData{
|
||||
SecretKey: AWSSecretAccessKey,
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
return w, req, chunk
|
||||
}
|
||||
|
||||
func getChunkedRequestUnsignedTrailing(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
|
||||
chunk := make([]byte, 65*1024)
|
||||
for i := range chunk {
|
||||
chunk[i] = 'a'
|
||||
}
|
||||
//chunk1 := chunk[:64*1024]
|
||||
//chunk2 := chunk[64*1024:]
|
||||
|
||||
AWSAccessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6"
|
||||
AWSSecretAccessKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e"
|
||||
|
||||
awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey}
|
||||
signer := v4.NewSigner()
|
||||
|
||||
reqBody := bytes.NewBufferString("10400\r\n")
|
||||
_, err := reqBody.Write(chunk)
|
||||
require.NoError(t, err)
|
||||
_, err = reqBody.WriteString("\r\n0\r\n")
|
||||
require.NoError(t, err)
|
||||
_, err = reqBody.WriteString("\r\nx-amz-checksum-crc64nvme:pRf+emrnL+A=\r\n\r\n")
|
||||
require.NoError(t, err)
|
||||
|
||||
req, err := http.NewRequest("PUT", "https://localhost:8184/"+bktName+"/"+objName, nil)
|
||||
//req, err := http.NewRequest("PUT", "https://localhost:8184/test2/body", nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("x-amz-sdk-checksum-algorithm", "CRC64NVME")
|
||||
req.Header.Set("content-encoding", api.AwsChunked)
|
||||
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc64nvme")
|
||||
req.Header.Set("x-amz-content-sha256", api.StreamingUnsignedPayloadTrailer)
|
||||
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
|
||||
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
||||
|
||||
signTime, err := time.Parse("20060102T150405Z", "20250131T140527Z")
|
||||
require.NoError(t, err)
|
||||
|
||||
err = signer.SignHTTP(ctx, awsCreds, req, api.StreamingContentSHA256Trailer, "s3", "ru", signTime)
|
||||
require.NoError(t, err)
|
||||
|
||||
req.Body = io.NopCloser(reqBody)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
||||
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
||||
req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{
|
||||
ClientTime: signTime,
|
||||
AuthHeaders: &middleware.AuthHeader{
|
||||
AccessKeyID: AWSAccessKeyID,
|
||||
SignatureV4: "a075c83779d1c3c02254fbe4c9eff0a21556d15556fc6a25db69147c4838226b",
|
||||
Region: "ru",
|
||||
},
|
||||
AccessBox: &accessbox.Box{
|
||||
Gate: &accessbox.GateData{
|
||||
SecretKey: AWSSecretAccessKey,
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
return w, req, chunk
|
||||
}
|
||||
|
||||
func getChunkedRequestUnsignedTrailingSmall(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
|
||||
AWSAccessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6"
|
||||
AWSSecretAccessKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e"
|
||||
|
||||
awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey}
|
||||
signer := v4.NewSigner()
|
||||
|
||||
chunk := "tmp2\n"
|
||||
|
||||
reqBody := bytes.NewBufferString("5\r\n")
|
||||
_, err := reqBody.WriteString(chunk)
|
||||
require.NoError(t, err)
|
||||
_, err = reqBody.WriteString("\r\n0\r\n")
|
||||
require.NoError(t, err)
|
||||
_, err = reqBody.WriteString("x-amz-checksum-crc64nvme:q1EYl4rI0TU=\r\n\r\n")
|
||||
require.NoError(t, err)
|
||||
|
||||
req, err := http.NewRequest("PUT", "https://localhost:8184/"+bktName+"/"+objName, nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("x-amz-sdk-checksum-algorithm", "CRC64NVME")
|
||||
req.Header.Set("content-encoding", api.AwsChunked)
|
||||
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc64nvme")
|
||||
req.Header.Set("x-amz-content-sha256", api.StreamingUnsignedPayloadTrailer)
|
||||
req.Header.Set("x-amz-decoded-content-length", "5")
|
||||
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
||||
|
||||
signTime, err := time.Parse("20060102T150405Z", "20250203T063745Z")
|
||||
require.NoError(t, err)
|
||||
|
||||
err = signer.SignHTTP(ctx, awsCreds, req, api.StreamingContentSHA256Trailer, "s3", "ru", signTime)
|
||||
require.NoError(t, err)
|
||||
|
||||
req.Body = io.NopCloser(reqBody)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
||||
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
||||
req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{
|
||||
ClientTime: signTime,
|
||||
AuthHeaders: &middleware.AuthHeader{
|
||||
AccessKeyID: AWSAccessKeyID,
|
||||
SignatureV4: "a075c83779d1c3c02254fbe4c9eff0a21556d15556fc6a25db69147c4838226b",
|
||||
Region: "ru",
|
||||
},
|
||||
AccessBox: &accessbox.Box{
|
||||
Gate: &accessbox.GateData{
|
||||
SecretKey: AWSSecretAccessKey,
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
return w, req, []byte(chunk)
|
||||
}
|
||||
|
||||
func getEmptyChunkedRequest(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) {
|
||||
AWSAccessKeyID := "48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh"
|
||||
AWSSecretAccessKey := "09260955b4eb0279dc017ba20a1ddac909cbd226c86cbb2d868e55534c8e64b0"
|
||||
|
|
|
@ -15,6 +15,9 @@ type ListBucketsResponse struct {
|
|||
Buckets struct {
|
||||
Buckets []Bucket `xml:"Bucket"`
|
||||
} // Buckets are nested
|
||||
|
||||
ContinuationToken string `xml:"ContinuationToken,omitempty"`
|
||||
Prefix string `xml:"Prefix,omitempty"`
|
||||
}
|
||||
|
||||
// ListObjectsV1Response -- format for ListObjectsV1 response.
|
||||
|
@ -51,8 +54,9 @@ type ListObjectsV2Response struct {
|
|||
|
||||
// Bucket container for bucket metadata.
|
||||
type Bucket struct {
|
||||
Name string
|
||||
CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
Name string `xml:"Name"`
|
||||
CreationDate string `xml:"CreationDate"` // time string of format "2006-01-02T15:04:05.000Z"
|
||||
BucketRegion string `xml:"BucketRegion,omitempty"`
|
||||
}
|
||||
|
||||
// PolicyStatus contains status of bucket policy.
|
||||
|
|
|
@ -8,6 +8,8 @@ import (
|
|||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4sdk2/signer/v4"
|
||||
|
@ -27,16 +29,19 @@ type (
|
|||
reader *bufio.Reader
|
||||
streamSigner *v4.StreamSigner
|
||||
|
||||
requestTime time.Time
|
||||
buffer []byte
|
||||
offset int
|
||||
err error
|
||||
trailerHeaders []string
|
||||
trailers map[string]string
|
||||
requestTime time.Time
|
||||
buffer []byte
|
||||
offset int
|
||||
err error
|
||||
}
|
||||
)
|
||||
|
||||
var (
|
||||
errGiantChunk = errors.New("chunk too big: choose chunk size <= 16MiB")
|
||||
errMalformedChunkedEncoding = errors.New("malformed chunked encoding")
|
||||
errMalformedTrailerHeaders = errors.New("malformed trailer headers")
|
||||
)
|
||||
|
||||
func (c *s3ChunkReader) Close() (err error) {
|
||||
|
@ -107,29 +112,9 @@ func (c *s3ChunkReader) Read(buf []byte) (num int, err error) {
|
|||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
b, err := c.reader.ReadByte()
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
if b != '\r' {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
b, err = c.reader.ReadByte()
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
if b != '\n' {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
|
||||
if err = c.readCRLF(); err != nil {
|
||||
return num, err
|
||||
}
|
||||
|
||||
if cap(c.buffer) < size {
|
||||
|
@ -147,23 +132,6 @@ func (c *s3ChunkReader) Read(buf []byte) (num int, err error) {
|
|||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
b, err = c.reader.ReadByte()
|
||||
if b != '\r' || err != nil {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
b, err = c.reader.ReadByte()
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
if b != '\n' {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
|
||||
// Once we have read the entire chunk successfully, we verify
|
||||
// that the received signature matches our computed signature.
|
||||
|
@ -181,16 +149,99 @@ func (c *s3ChunkReader) Read(buf []byte) (num int, err error) {
|
|||
// If the chunk size is zero we return io.EOF. As specified by AWS,
|
||||
// only the last chunk is zero-sized.
|
||||
if size == 0 {
|
||||
if len(c.trailerHeaders) != 0 {
|
||||
if err = c.readTrailers(); err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
} else if err = c.readCRLF(); err != nil {
|
||||
return num, err
|
||||
}
|
||||
|
||||
c.err = io.EOF
|
||||
return num, c.err
|
||||
}
|
||||
|
||||
if err = c.readCRLF(); err != nil {
|
||||
return num, err
|
||||
}
|
||||
|
||||
c.offset = copy(buf, c.buffer)
|
||||
num += c.offset
|
||||
return num, err
|
||||
}
|
||||
|
||||
func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, error) {
|
||||
func (c *s3ChunkReader) readCRLF() error {
|
||||
for _, ch := range [2]byte{'\r', '\n'} {
|
||||
b, err := c.reader.ReadByte()
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return c.err
|
||||
}
|
||||
if b != ch {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return c.err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *s3ChunkReader) readTrailers() error {
|
||||
var k, v []byte
|
||||
var err error
|
||||
for err == nil {
|
||||
k, err = c.reader.ReadBytes(':')
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
c.err = errMalformedTrailerHeaders
|
||||
return c.err
|
||||
}
|
||||
v, err = c.reader.ReadBytes('\n')
|
||||
if err != nil && err != io.EOF {
|
||||
c.err = errMalformedTrailerHeaders
|
||||
return c.err
|
||||
}
|
||||
if len(v) >= 2 && v[len(v)-2] == '\r' {
|
||||
v[len(v)-2] = '\n'
|
||||
v = v[:len(v)-1]
|
||||
}
|
||||
|
||||
switch {
|
||||
case slices.Contains(c.trailerHeaders, string(k[:len(k)-1])):
|
||||
c.buffer = append(append(c.buffer, k...), v...) // todo use copy
|
||||
case string(k) == "x-amz-trailer-signature:":
|
||||
calculatedSignature, err := c.streamSigner.GetTrailerSignature(c.buffer, c.requestTime)
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return c.err
|
||||
}
|
||||
if string(v[:64]) != hex.EncodeToString(calculatedSignature) {
|
||||
c.err = errs.GetAPIError(errs.ErrSignatureDoesNotMatch)
|
||||
return c.err
|
||||
}
|
||||
default:
|
||||
c.err = errMalformedTrailerHeaders
|
||||
return c.err
|
||||
}
|
||||
|
||||
c.trailers[string(k[:len(k)-1])] = string(v[:len(v)-1])
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *s3ChunkReader) TrailerHeaders() map[string]string {
|
||||
return c.trailers
|
||||
}
|
||||
|
||||
func newSignV4ChunkedReader(req *http.Request) (*s3ChunkReader, error) {
|
||||
ctx := req.Context()
|
||||
box, err := middleware.GetBoxData(ctx)
|
||||
if err != nil {
|
||||
|
@ -214,11 +265,19 @@ func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, error) {
|
|||
}
|
||||
newStreamSigner := v4.NewStreamSigner(currentCredentials, "s3", authHeaders.Region, seed)
|
||||
|
||||
var trailerHeaders []string
|
||||
trailer := req.Header.Get("x-amz-trailer")
|
||||
if trailer != "" {
|
||||
trailerHeaders = strings.Split(trailer, ";")
|
||||
}
|
||||
|
||||
return &s3ChunkReader{
|
||||
ctx: ctx,
|
||||
reader: bufio.NewReader(req.Body),
|
||||
streamSigner: newStreamSigner,
|
||||
requestTime: reqTime,
|
||||
buffer: make([]byte, 64*1024),
|
||||
ctx: ctx,
|
||||
reader: bufio.NewReader(req.Body),
|
||||
streamSigner: newStreamSigner,
|
||||
requestTime: reqTime,
|
||||
buffer: make([]byte, 64*1024),
|
||||
trailerHeaders: trailerHeaders,
|
||||
trailers: make(map[string]string, len(trailerHeaders)),
|
||||
}, nil
|
||||
}
|
||||
|
|
|
@ -2,8 +2,12 @@ package handler
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
@ -12,22 +16,102 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSigV4AStreaming(t *testing.T) {
|
||||
accessKeyID := "2XEbqH4M3ym7a3E3esxfZ2gRLnMwDXrCN4y1SkQg5fHa09sThVmVL3EE6xeKsyMzaqu5jPi41YCaVbnwbwCTF3bx1"
|
||||
secretKey := "00637f53f842573aaa06c2164c598973cd986880987111416cf71f1619def537"
|
||||
func TestSigV4AChunkedReader(t *testing.T) {
|
||||
t.Run("with trailers", func(t *testing.T) {
|
||||
accessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6"
|
||||
secretKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e"
|
||||
|
||||
chunk1 := "Testing with the {sdk-java}"
|
||||
reqBody := bytes.NewBufferString("1b;chunk-signature=3045022100b63692a1b20759bdabd342011823427a8952df75c93174d98ad043abca8052e002201695228a91ba986171b8d0ad20856d3d94ca3614d0a90a50a531ba8e52447b9b**\r\n")
|
||||
_, err := reqBody.WriteString(chunk1)
|
||||
require.NoError(t, err)
|
||||
_, err = reqBody.WriteString("\r\n0;chunk-signature=30440220455885a2d4e9f705256ca6b0a5a22f7f784780ccbd1c0a371e5db3059c91745b022073259dd44746cbd63261d628a04d25be5a32a974c077c5c2d83c8157fb323b9f****\r\n\r\n")
|
||||
require.NoError(t, err)
|
||||
chunk1 := "Testing with the {sdk-java}"
|
||||
body := "1b;chunk-signature=3045022100956ca03d2166100b455b532de542892f73925fbcea2f6498674a39a61bb4860902202977c1d47aea548d434540f89640ce97e605d18353cbbd75a619874f02e3dd22**\r\n" +
|
||||
chunk1 +
|
||||
"\r\n0;chunk-signature=304502210097dcc1721675469910ef8712fc2af0678eb90c12216dd6228c6b621fb6f805a0022047d27d21ae2af8a8172f2ef83c81ce9d4746aa88fc9ee0ca783eaa5e71aaef6c**\r\n" +
|
||||
"x-amz-checksum-crc32:Np6zMg==\r\n" +
|
||||
"x-amz-trailer-signature:304502200ecacd9aa2c432af5a2327c22a2ff9b32f44ab8559de00309219aef105eaaac102210092cbc0e78c4bcd56490a73da8ceed1934be80f3affeffb14d8c743fc292dda4f**\r\n\r\n"
|
||||
|
||||
req, err := http.NewRequest("PUT", "http://localhost:8084/test/tmp", reqBody)
|
||||
require.NoError(t, err)
|
||||
reqBody := bytes.NewBufferString(body)
|
||||
req, err := http.NewRequest("PUT", "https://localhost:8184/test2/tmp", reqBody)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32")
|
||||
|
||||
signature := "30440220574244c5ff5deba388c4e3b0541a42113179b6839b3e6b4212d255a118fa9089022056f7b9b72c93f67dbcd25fe9ca67950b5913fc00bb7a62bc276c21e828c0b6c7"
|
||||
signingTime, err := time.Parse("20060102T150405Z", "20240904T133253Z")
|
||||
signature := "3045022100ddbc6ab11785d7f23d299de7db97379116f543377a44e38170a4e43b38b0d62b02201d8dca13c67f04f45491345152db4b704768eb8bb89b5215fd59bb4a4d9d7b61"
|
||||
signingTime, err := time.Parse("20060102T150405Z", "20250203T144621Z")
|
||||
require.NoError(t, err)
|
||||
|
||||
key, err := keys.NewPrivateKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
accessBox, err := newTestAccessBox(key)
|
||||
require.NoError(t, err)
|
||||
accessBox.Gate.SecretKey = secretKey
|
||||
|
||||
ctx := middleware.SetBox(req.Context(), &middleware.Box{
|
||||
AccessBox: accessBox,
|
||||
AuthHeaders: &middleware.AuthHeader{
|
||||
AccessKeyID: accessKeyID,
|
||||
SignatureV4: signature,
|
||||
},
|
||||
ClientTime: signingTime,
|
||||
})
|
||||
req = req.WithContext(ctx)
|
||||
|
||||
r, err := newSignV4aChunkedReader(req)
|
||||
require.NoError(t, err)
|
||||
|
||||
data, err := io.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, chunk1, string(data))
|
||||
})
|
||||
|
||||
t.Run("without trailers", func(t *testing.T) {
|
||||
accessKeyID := "2XEbqH4M3ym7a3E3esxfZ2gRLnMwDXrCN4y1SkQg5fHa09sThVmVL3EE6xeKsyMzaqu5jPi41YCaVbnwbwCTF3bx1"
|
||||
secretKey := "00637f53f842573aaa06c2164c598973cd986880987111416cf71f1619def537"
|
||||
|
||||
chunk1 := "Testing with the {sdk-java}"
|
||||
reqBody := bytes.NewBufferString("1b;chunk-signature=3045022100b63692a1b20759bdabd342011823427a8952df75c93174d98ad043abca8052e002201695228a91ba986171b8d0ad20856d3d94ca3614d0a90a50a531ba8e52447b9b**\r\n")
|
||||
_, err := reqBody.WriteString(chunk1)
|
||||
require.NoError(t, err)
|
||||
_, err = reqBody.WriteString("\r\n0;chunk-signature=30440220455885a2d4e9f705256ca6b0a5a22f7f784780ccbd1c0a371e5db3059c91745b022073259dd44746cbd63261d628a04d25be5a32a974c077c5c2d83c8157fb323b9f****\r\n\r\n")
|
||||
require.NoError(t, err)
|
||||
|
||||
req, err := http.NewRequest("PUT", "http://localhost:8084/test/tmp", reqBody)
|
||||
require.NoError(t, err)
|
||||
|
||||
signature := "30440220574244c5ff5deba388c4e3b0541a42113179b6839b3e6b4212d255a118fa9089022056f7b9b72c93f67dbcd25fe9ca67950b5913fc00bb7a62bc276c21e828c0b6c7"
|
||||
signingTime, err := time.Parse("20060102T150405Z", "20240904T133253Z")
|
||||
require.NoError(t, err)
|
||||
|
||||
key, err := keys.NewPrivateKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
accessBox, err := newTestAccessBox(key)
|
||||
require.NoError(t, err)
|
||||
accessBox.Gate.SecretKey = secretKey
|
||||
|
||||
ctx := middleware.SetBox(req.Context(), &middleware.Box{
|
||||
AccessBox: accessBox,
|
||||
AuthHeaders: &middleware.AuthHeader{
|
||||
AccessKeyID: accessKeyID,
|
||||
SignatureV4: signature,
|
||||
},
|
||||
ClientTime: signingTime,
|
||||
})
|
||||
req = req.WithContext(ctx)
|
||||
|
||||
r, err := newSignV4aChunkedReader(req)
|
||||
require.NoError(t, err)
|
||||
|
||||
data, err := io.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, chunk1, string(data))
|
||||
})
|
||||
}
|
||||
|
||||
func TestSigV4ChunkedReader(t *testing.T) {
|
||||
accessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6"
|
||||
secretKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e"
|
||||
|
||||
signature := "b740b3b2a08c541c3fc4bd155a448e25408b509a29af98a86356b894930b93e8"
|
||||
signingTime, err := time.Parse("20060102T150405Z", "20250203T134442Z")
|
||||
require.NoError(t, err)
|
||||
|
||||
key, err := keys.NewPrivateKey()
|
||||
|
@ -37,21 +121,117 @@ func TestSigV4AStreaming(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
accessBox.Gate.SecretKey = secretKey
|
||||
|
||||
ctx := middleware.SetBox(req.Context(), &middleware.Box{
|
||||
AccessBox: accessBox,
|
||||
AuthHeaders: &middleware.AuthHeader{
|
||||
AccessKeyID: accessKeyID,
|
||||
SignatureV4: signature,
|
||||
},
|
||||
ClientTime: signingTime,
|
||||
setBoxFn := func(ctx context.Context) context.Context {
|
||||
return middleware.SetBox(ctx, &middleware.Box{
|
||||
AccessBox: accessBox,
|
||||
AuthHeaders: &middleware.AuthHeader{
|
||||
AccessKeyID: accessKeyID,
|
||||
SignatureV4: signature,
|
||||
Region: "us-east-1",
|
||||
},
|
||||
ClientTime: signingTime,
|
||||
})
|
||||
}
|
||||
|
||||
chunk1 := "Testing with the {sdk-java}"
|
||||
|
||||
t.Run("with trailers", func(t *testing.T) {
|
||||
body := "1b;chunk-signature=a6a9be5fff05db0b542aedb2203d892b4162250885d06b1422b173ee0ea92ba5\r\n" +
|
||||
chunk1 +
|
||||
"\r\n0;chunk-signature=31afd083a57c416c46afaf101649d7f0c6c0627cfa60c0f93d1f7ea84396ee42\r\n" +
|
||||
"x-amz-checksum-crc32:Np6zMg==\r\n" +
|
||||
"x-amz-trailer-signature:40ec0046ac730fa27a1451d00d849056c49553ee753f5d158306d05671a42125\r\n\r\n"
|
||||
|
||||
reqBody := bytes.NewBufferString(body)
|
||||
req, err := http.NewRequest("PUT", "https://localhost:8184/test2/tmp", reqBody)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32")
|
||||
|
||||
req = req.WithContext(setBoxFn(req.Context()))
|
||||
|
||||
r, err := newSignV4ChunkedReader(req)
|
||||
require.NoError(t, err)
|
||||
|
||||
data, err := io.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, chunk1, string(data))
|
||||
})
|
||||
req = req.WithContext(ctx)
|
||||
|
||||
r, err := newSignV4aChunkedReader(req)
|
||||
require.NoError(t, err)
|
||||
t.Run("without trailers", func(t *testing.T) {
|
||||
body := "1b;chunk-signature=a6a9be5fff05db0b542aedb2203d892b4162250885d06b1422b173ee0ea92ba5\r\n" +
|
||||
chunk1 +
|
||||
"\r\n0;chunk-signature=31afd083a57c416c46afaf101649d7f0c6c0627cfa60c0f93d1f7ea84396ee42\r\n\r\n"
|
||||
reqBody := bytes.NewBufferString(body)
|
||||
req, err := http.NewRequest("PUT", "https://localhost:8184/test2/tmp", reqBody)
|
||||
require.NoError(t, err)
|
||||
|
||||
data, err := io.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
req = req.WithContext(setBoxFn(req.Context()))
|
||||
|
||||
require.Equal(t, chunk1, string(data))
|
||||
r, err := newSignV4ChunkedReader(req)
|
||||
require.NoError(t, err)
|
||||
|
||||
data, err := io.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, chunk1, string(data))
|
||||
})
|
||||
}
|
||||
|
||||
func TestUnsignedChunkReader(t *testing.T) {
|
||||
chunk1 := "chunk1"
|
||||
chunk2 := "chunk2"
|
||||
|
||||
t.Run("with trailer", func(t *testing.T) {
|
||||
chunks := []string{chunk1, chunk2}
|
||||
trailer := map[string]string{"x-amz-checksum-crc64nvme": "q1EYl4rI0TU="}
|
||||
body, expected := getChunkedBody(t, chunks, trailer)
|
||||
|
||||
r, err := newUnsignedChunkedReader(body)
|
||||
require.NoError(t, err)
|
||||
|
||||
data, err := io.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expected, string(data))
|
||||
|
||||
require.EqualValues(t, trailer, r.TrailerHeaders())
|
||||
})
|
||||
|
||||
t.Run("without trailer", func(t *testing.T) {
|
||||
chunks := []string{chunk1, chunk2}
|
||||
body, expected := getChunkedBody(t, chunks, nil)
|
||||
|
||||
r, err := newUnsignedChunkedReader(body)
|
||||
require.NoError(t, err)
|
||||
|
||||
data, err := io.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expected, string(data))
|
||||
})
|
||||
}
|
||||
|
||||
func getChunkedBody(t *testing.T, chunks []string, trailers map[string]string) (*bytes.Buffer, string) {
|
||||
res := bytes.NewBufferString("")
|
||||
|
||||
for i, chunk := range chunks {
|
||||
meta := strconv.FormatInt(int64(len(chunk)), 16) + "\r\n"
|
||||
if i != 0 {
|
||||
meta = "\r\n" + meta
|
||||
}
|
||||
_, err := res.WriteString(meta)
|
||||
require.NoError(t, err)
|
||||
_, err = res.WriteString(chunk)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
_, err := res.WriteString("\r\n0\r\n")
|
||||
require.NoError(t, err)
|
||||
|
||||
for k, v := range trailers {
|
||||
_, err := res.WriteString(fmt.Sprintf("%s:%s\n", k, v))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
_, err = res.WriteString("\r\n")
|
||||
require.NoError(t, err)
|
||||
|
||||
return res, strings.Join(chunks, "")
|
||||
}
|
||||
|
|
161
api/handler/s3unsignedreader.go
Normal file
161
api/handler/s3unsignedreader.go
Normal file
|
@ -0,0 +1,161 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
)
|
||||
|
||||
type (
|
||||
s3UnsignedChunkReader struct {
|
||||
reader *bufio.Reader
|
||||
|
||||
trailers map[string]string
|
||||
buffer []byte
|
||||
offset int
|
||||
err error
|
||||
}
|
||||
)
|
||||
|
||||
func (c *s3UnsignedChunkReader) Close() (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *s3UnsignedChunkReader) Read(buf []byte) (num int, err error) {
|
||||
if c.offset > 0 {
|
||||
num = copy(buf, c.buffer[c.offset:])
|
||||
if num == len(buf) {
|
||||
c.offset += num
|
||||
return num, nil
|
||||
}
|
||||
c.offset = 0
|
||||
buf = buf[num:]
|
||||
}
|
||||
|
||||
var size int
|
||||
var b byte
|
||||
for {
|
||||
b, err = c.reader.ReadByte()
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
if b == '\r' {
|
||||
break
|
||||
}
|
||||
|
||||
// Manually deserialize the size since AWS specified
|
||||
// the chunk size to be of variable width. In particular,
|
||||
// a size of 16 is encoded as `10` while a size of 64 KB
|
||||
// is `10000`.
|
||||
switch {
|
||||
case b >= '0' && b <= '9':
|
||||
size = size<<4 | int(b-'0')
|
||||
case b >= 'a' && b <= 'f':
|
||||
size = size<<4 | int(b-('a'-10))
|
||||
case b >= 'A' && b <= 'F':
|
||||
size = size<<4 | int(b-('A'-10))
|
||||
default:
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
if size > maxChunkSize {
|
||||
c.err = errGiantChunk
|
||||
return num, c.err
|
||||
}
|
||||
}
|
||||
|
||||
if b != '\r' {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
b, err = c.reader.ReadByte()
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
if b != '\n' {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
|
||||
if cap(c.buffer) < size {
|
||||
c.buffer = make([]byte, size)
|
||||
} else {
|
||||
c.buffer = c.buffer[:size]
|
||||
}
|
||||
|
||||
// Now, we read the payload and compute its SHA-256 hash.
|
||||
_, err = io.ReadFull(c.reader, c.buffer)
|
||||
if err == io.EOF && size != 0 {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil && err != io.EOF {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
|
||||
// If the chunk size is zero we return io.EOF. As specified by AWS,
|
||||
// only the last chunk is zero-sized.
|
||||
if size == 0 {
|
||||
var k, v string
|
||||
for err == nil {
|
||||
k, err = c.reader.ReadString(':')
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
c.err = errMalformedTrailerHeaders
|
||||
return num, c.err
|
||||
}
|
||||
v, err = c.reader.ReadString('\n')
|
||||
if err != nil {
|
||||
c.err = errMalformedTrailerHeaders
|
||||
return num, c.err
|
||||
}
|
||||
c.trailers[k[:len(k)-1]] = v[:len(v)-1]
|
||||
}
|
||||
|
||||
c.err = io.EOF
|
||||
return num, c.err
|
||||
}
|
||||
|
||||
b, err = c.reader.ReadByte()
|
||||
if b != '\r' || err != nil {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
b, err = c.reader.ReadByte()
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
if b != '\n' {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
|
||||
c.offset = copy(buf, c.buffer)
|
||||
num += c.offset
|
||||
return num, err
|
||||
}
|
||||
|
||||
func (c *s3UnsignedChunkReader) TrailerHeaders() map[string]string {
|
||||
return c.trailers
|
||||
}
|
||||
|
||||
func newUnsignedChunkedReader(body io.Reader) (*s3UnsignedChunkReader, error) {
|
||||
return &s3UnsignedChunkReader{
|
||||
reader: bufio.NewReader(body),
|
||||
trailers: map[string]string{},
|
||||
buffer: make([]byte, 64*1024),
|
||||
}, nil
|
||||
}
|
|
@ -7,6 +7,8 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
v4a "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4asdk2"
|
||||
|
@ -20,10 +22,12 @@ type (
|
|||
reader *bufio.Reader
|
||||
streamSigner *v4a.StreamSigner
|
||||
|
||||
requestTime time.Time
|
||||
buffer []byte
|
||||
offset int
|
||||
err error
|
||||
trailerHeaders []string
|
||||
trailers map[string]string
|
||||
requestTime time.Time
|
||||
buffer []byte
|
||||
offset int
|
||||
err error
|
||||
}
|
||||
)
|
||||
|
||||
|
@ -87,21 +91,9 @@ func (c *s3v4aChunkReader) Read(buf []byte) (num int, err error) {
|
|||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
b, err := c.reader.ReadByte()
|
||||
if err != nil {
|
||||
return c.handleErr(num, err)
|
||||
}
|
||||
if b != '\r' {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
b, err = c.reader.ReadByte()
|
||||
if err != nil {
|
||||
return c.handleErr(num, err)
|
||||
}
|
||||
if b != '\n' {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
|
||||
if err = c.readCRLF(); err != nil {
|
||||
return num, err
|
||||
}
|
||||
|
||||
if cap(c.buffer) < size {
|
||||
|
@ -119,19 +111,6 @@ func (c *s3v4aChunkReader) Read(buf []byte) (num int, err error) {
|
|||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
b, err = c.reader.ReadByte()
|
||||
if b != '\r' || err != nil {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
b, err = c.reader.ReadByte()
|
||||
if err != nil {
|
||||
return c.handleErr(num, err)
|
||||
}
|
||||
if b != '\n' {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return num, c.err
|
||||
}
|
||||
|
||||
// Once we have read the entire chunk successfully, we verify
|
||||
// that the received signature is valid.
|
||||
|
@ -150,10 +129,23 @@ func (c *s3v4aChunkReader) Read(buf []byte) (num int, err error) {
|
|||
// If the chunk size is zero we return io.EOF. As specified by AWS,
|
||||
// only the last chunk is zero-sized.
|
||||
if size == 0 {
|
||||
if len(c.trailerHeaders) != 0 {
|
||||
if err = c.readTrailers(); err != nil {
|
||||
c.err = err
|
||||
return num, c.err
|
||||
}
|
||||
} else if err = c.readCRLF(); err != nil {
|
||||
return num, err
|
||||
}
|
||||
|
||||
c.err = io.EOF
|
||||
return num, c.err
|
||||
}
|
||||
|
||||
if err = c.readCRLF(); err != nil {
|
||||
return num, err
|
||||
}
|
||||
|
||||
c.offset = copy(buf, c.buffer)
|
||||
num += c.offset
|
||||
return num, err
|
||||
|
@ -168,7 +160,78 @@ func (c *s3v4aChunkReader) handleErr(num int, err error) (int, error) {
|
|||
return num, c.err
|
||||
}
|
||||
|
||||
func newSignV4aChunkedReader(req *http.Request) (io.ReadCloser, error) {
|
||||
func (c *s3v4aChunkReader) readCRLF() error {
|
||||
for _, ch := range [2]byte{'\r', '\n'} {
|
||||
b, err := c.reader.ReadByte()
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
c.err = err
|
||||
return c.err
|
||||
}
|
||||
if b != ch {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return c.err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *s3v4aChunkReader) readTrailers() error {
|
||||
var k, v []byte
|
||||
var err error
|
||||
for err == nil {
|
||||
k, err = c.reader.ReadBytes(':')
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
c.err = errMalformedTrailerHeaders
|
||||
return c.err
|
||||
}
|
||||
v, err = c.reader.ReadBytes('\n')
|
||||
if err != nil && err != io.EOF {
|
||||
c.err = errMalformedTrailerHeaders
|
||||
return c.err
|
||||
}
|
||||
if len(v) >= 2 && v[len(v)-2] == '\r' {
|
||||
v[len(v)-2] = '\n'
|
||||
v = v[:len(v)-1]
|
||||
}
|
||||
|
||||
switch {
|
||||
case slices.Contains(c.trailerHeaders, string(k[:len(k)-1])):
|
||||
c.buffer = append(append(c.buffer, k...), v...) // todo use copy
|
||||
case string(k) == "x-amz-trailer-signature:":
|
||||
n, err := hex.Decode(v[:], bytes.TrimRight(v[:], "*\n"))
|
||||
if err != nil {
|
||||
c.err = errMalformedChunkedEncoding
|
||||
return c.err
|
||||
}
|
||||
|
||||
if err = c.streamSigner.VerifyTrailerSignature(c.buffer, c.requestTime, v[:n]); err != nil {
|
||||
c.err = fmt.Errorf("%w: %s", errs.GetAPIError(errs.ErrSignatureDoesNotMatch), err.Error())
|
||||
return c.err
|
||||
}
|
||||
default:
|
||||
c.err = errMalformedTrailerHeaders
|
||||
return c.err
|
||||
}
|
||||
|
||||
c.trailers[string(k[:len(k)-1])] = string(v[:len(v)-1])
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *s3v4aChunkReader) TrailerHeaders() map[string]string {
|
||||
return c.trailers
|
||||
}
|
||||
|
||||
func newSignV4aChunkedReader(req *http.Request) (*s3v4aChunkReader, error) {
|
||||
box, err := middleware.GetBoxData(req.Context())
|
||||
if err != nil {
|
||||
return nil, errs.GetAPIError(errs.ErrAuthorizationHeaderMalformed)
|
||||
|
@ -200,10 +263,18 @@ func newSignV4aChunkedReader(req *http.Request) (io.ReadCloser, error) {
|
|||
|
||||
newStreamSigner := v4a.NewStreamSigner(creds, "s3", seed)
|
||||
|
||||
var trailerHeaders []string
|
||||
trailer := req.Header.Get("x-amz-trailer")
|
||||
if trailer != "" {
|
||||
trailerHeaders = strings.Split(trailer, ";")
|
||||
}
|
||||
|
||||
return &s3v4aChunkReader{
|
||||
reader: bufio.NewReader(req.Body),
|
||||
streamSigner: newStreamSigner,
|
||||
requestTime: reqTime,
|
||||
buffer: make([]byte, 64*1024),
|
||||
reader: bufio.NewReader(req.Body),
|
||||
streamSigner: newStreamSigner,
|
||||
requestTime: reqTime,
|
||||
buffer: make([]byte, 64*1024),
|
||||
trailerHeaders: trailerHeaders,
|
||||
trailers: make(map[string]string, len(trailerHeaders)),
|
||||
}, nil
|
||||
}
|
||||
|
|
|
@ -39,9 +39,10 @@ func (h *handler) logAndSendError(ctx context.Context, w http.ResponseWriter, lo
|
|||
zap.String("object", reqInfo.ObjectName),
|
||||
zap.String("description", logText),
|
||||
zap.String("user", reqInfo.User),
|
||||
zap.Error(err)}
|
||||
zap.Error(err),
|
||||
}
|
||||
fields = append(fields, additional...)
|
||||
h.reqLogger(ctx).Error(logs.RequestFailed, fields...)
|
||||
h.reqLogger(ctx).Error(logs.RequestFailed, append(fields, logs.TagField(logs.TagDatapath))...)
|
||||
}
|
||||
|
||||
func handleDeleteMarker(w http.ResponseWriter, err error) error {
|
||||
|
|
|
@ -94,8 +94,11 @@ const (
|
|||
|
||||
DefaultLocationConstraint = "default"
|
||||
|
||||
StreamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
|
||||
StreamingContentV4aSHA256 = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"
|
||||
StreamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
|
||||
StreamingContentSHA256Trailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
|
||||
StreamingContentV4aSHA256 = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"
|
||||
StreamingContentV4aSHA256Trailer = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"
|
||||
StreamingUnsignedPayloadTrailer = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
|
||||
|
||||
DefaultStorageClass = "STANDARD"
|
||||
)
|
||||
|
@ -129,6 +132,8 @@ var SystemMetadata = map[string]struct{}{
|
|||
func IsSignedStreamingV4(r *http.Request) (string, bool) {
|
||||
shaHeader := r.Header.Get(AmzContentSha256)
|
||||
return shaHeader,
|
||||
(shaHeader == StreamingContentSHA256 || shaHeader == StreamingContentV4aSHA256) &&
|
||||
(shaHeader == StreamingContentSHA256 || shaHeader == StreamingContentSHA256Trailer ||
|
||||
shaHeader == StreamingContentV4aSHA256 || shaHeader == StreamingContentV4aSHA256Trailer ||
|
||||
shaHeader == StreamingUnsignedPayloadTrailer) &&
|
||||
r.Method == http.MethodPut
|
||||
}
|
||||
|
|
|
@ -20,7 +20,7 @@ type Cache struct {
|
|||
bucketCache *cache.BucketCache
|
||||
systemCache *cache.SystemCache
|
||||
accessCache *cache.AccessControlCache
|
||||
networkInfoCache *cache.NetworkInfoCache
|
||||
networkCache *cache.NetworkCache
|
||||
}
|
||||
|
||||
// CachesConfig contains params for caches.
|
||||
|
@ -33,7 +33,8 @@ type CachesConfig struct {
|
|||
Buckets *cache.Config
|
||||
System *cache.Config
|
||||
AccessControl *cache.Config
|
||||
NetworkInfo *cache.NetworkInfoCacheConfig
|
||||
Network *cache.NetworkCacheConfig
|
||||
CIDCache bool
|
||||
}
|
||||
|
||||
// DefaultCachesConfigs returns filled configs.
|
||||
|
@ -47,7 +48,7 @@ func DefaultCachesConfigs(logger *zap.Logger) *CachesConfig {
|
|||
Buckets: cache.DefaultBucketConfig(logger),
|
||||
System: cache.DefaultSystemConfig(logger),
|
||||
AccessControl: cache.DefaultAccessControlConfig(logger),
|
||||
NetworkInfo: cache.DefaultNetworkInfoConfig(logger),
|
||||
Network: cache.DefaultNetworkConfig(logger),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -58,10 +59,10 @@ func NewCache(cfg *CachesConfig) *Cache {
|
|||
sessionListCache: cache.NewListSessionCache(cfg.SessionList),
|
||||
objCache: cache.New(cfg.Objects),
|
||||
namesCache: cache.NewObjectsNameCache(cfg.Names),
|
||||
bucketCache: cache.NewBucketCache(cfg.Buckets),
|
||||
bucketCache: cache.NewBucketCache(cfg.Buckets, cfg.CIDCache),
|
||||
systemCache: cache.NewSystemCache(cfg.System),
|
||||
accessCache: cache.NewAccessControlCache(cfg.AccessControl),
|
||||
networkInfoCache: cache.NewNetworkInfoCache(cfg.NetworkInfo),
|
||||
networkCache: cache.NewNetworkCache(cfg.Network),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -75,7 +76,8 @@ func (c *Cache) PutBucket(bktInfo *data.BucketInfo) {
|
|||
zap.String("zone", bktInfo.Zone),
|
||||
zap.String("bucket name", bktInfo.Name),
|
||||
zap.Stringer("bucket cid", bktInfo.CID),
|
||||
zap.Error(err))
|
||||
zap.Error(err),
|
||||
logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -117,11 +119,12 @@ func (c *Cache) PutObject(owner user.ID, extObjInfo *data.ExtendedObjectInfo) {
|
|||
if err := c.objCache.PutObject(extObjInfo); err != nil {
|
||||
c.logger.Warn(logs.CouldntAddObjectToCache, zap.Error(err),
|
||||
zap.String("object_name", extObjInfo.ObjectInfo.Name), zap.String("bucket_name", extObjInfo.ObjectInfo.Bucket),
|
||||
zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString()))
|
||||
zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString()),
|
||||
logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, extObjInfo.ObjectInfo.Address().EncodeToString()); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -131,7 +134,8 @@ func (c *Cache) PutObjectWithName(owner user.ID, extObjInfo *data.ExtendedObject
|
|||
if err := c.namesCache.Put(extObjInfo.ObjectInfo.NiceName(), extObjInfo.ObjectInfo.Address()); err != nil {
|
||||
c.logger.Warn(logs.CouldntPutObjAddressToNameCache,
|
||||
zap.String("obj nice name", extObjInfo.ObjectInfo.NiceName()),
|
||||
zap.Error(err))
|
||||
zap.Error(err),
|
||||
logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -145,11 +149,11 @@ func (c *Cache) GetList(owner user.ID, key cache.ObjectsListKey) []*data.NodeVer
|
|||
|
||||
func (c *Cache) PutList(owner user.ID, key cache.ObjectsListKey, list []*data.NodeVersion) {
|
||||
if err := c.listsCache.PutVersions(key, list); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheListOfObjects, zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheListOfObjects, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key.String()); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -163,11 +167,11 @@ func (c *Cache) GetListSession(owner user.ID, key cache.ListSessionKey) *data.Li
|
|||
|
||||
func (c *Cache) PutListSession(owner user.ID, key cache.ListSessionKey, session *data.ListSession) {
|
||||
if err := c.sessionListCache.PutListSession(key, session); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheListSession, zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheListSession, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key.String()); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -186,11 +190,11 @@ func (c *Cache) GetTagging(owner user.ID, key string) map[string]string {
|
|||
|
||||
func (c *Cache) PutTagging(owner user.ID, key string, tags map[string]string) {
|
||||
if err := c.systemCache.PutTagging(key, tags); err != nil {
|
||||
c.logger.Error(logs.CouldntCacheTags, zap.Error(err))
|
||||
c.logger.Error(logs.CouldntCacheTags, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -208,11 +212,11 @@ func (c *Cache) GetLockInfo(owner user.ID, key string) *data.LockInfo {
|
|||
|
||||
func (c *Cache) PutLockInfo(owner user.ID, key string, lockInfo *data.LockInfo) {
|
||||
if err := c.systemCache.PutLockInfo(key, lockInfo); err != nil {
|
||||
c.logger.Error(logs.CouldntCacheLockInfo, zap.Error(err))
|
||||
c.logger.Error(logs.CouldntCacheLockInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -229,11 +233,11 @@ func (c *Cache) GetSettings(owner user.ID, bktInfo *data.BucketInfo) *data.Bucke
|
|||
func (c *Cache) PutSettings(owner user.ID, bktInfo *data.BucketInfo, settings *data.BucketSettings) {
|
||||
key := bktInfo.Name + bktInfo.SettingsObjectName()
|
||||
if err := c.systemCache.PutSettings(key, settings); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheBucketSettings, zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheBucketSettings, zap.String("bucket", bktInfo.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -251,11 +255,11 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf
|
|||
key := bkt.CORSObjectName()
|
||||
|
||||
if err := c.systemCache.PutCORS(key, cors); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -277,11 +281,11 @@ func (c *Cache) PutLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo, c
|
|||
key := bkt.LifecycleConfigurationObjectName()
|
||||
|
||||
if err := c.systemCache.PutLifecycleConfiguration(key, cfg); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -290,11 +294,30 @@ func (c *Cache) DeleteLifecycleConfiguration(bktInfo *data.BucketInfo) {
|
|||
}
|
||||
|
||||
func (c *Cache) GetNetworkInfo() *netmap.NetworkInfo {
|
||||
return c.networkInfoCache.Get()
|
||||
return c.networkCache.GetNetworkInfo()
|
||||
}
|
||||
|
||||
func (c *Cache) PutNetworkInfo(info netmap.NetworkInfo) {
|
||||
if err := c.networkInfoCache.Put(info); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheNetworkInfo, zap.Error(err))
|
||||
if err := c.networkCache.PutNetworkInfo(info); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheNetworkInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache) GetNetmap() *netmap.NetMap {
|
||||
return c.networkCache.GetNetmap()
|
||||
}
|
||||
|
||||
func (c *Cache) PutNetmap(nm netmap.NetMap) {
|
||||
if err := c.networkCache.PutNetmap(nm); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheNetmap, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache) GetPlacementPolicy(cnrID cid.ID) *netmap.PlacementPolicy {
|
||||
res := c.bucketCache.GetByCID(cnrID)
|
||||
if res != nil {
|
||||
return &res.PlacementPolicy
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -3,7 +3,9 @@ package layer
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
|
@ -53,6 +55,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm frostfs.PrmContainer) (*d
|
|||
info.Created = container.CreatedAt(cnr)
|
||||
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
|
||||
info.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(cnr)
|
||||
info.PlacementPolicy = cnr.PlacementPolicy()
|
||||
|
||||
attrLockEnabled := cnr.Attribute(AttributeLockEnabled)
|
||||
if len(attrLockEnabled) > 0 {
|
||||
|
@ -61,6 +64,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm frostfs.PrmContainer) (*d
|
|||
log.Error(logs.CouldNotParseContainerObjectLockEnabledAttribute,
|
||||
zap.String("lock_enabled", attrLockEnabled),
|
||||
zap.Error(err),
|
||||
logs.TagField(logs.TagDatapath),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
@ -75,7 +79,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm frostfs.PrmContainer) (*d
|
|||
return info, nil
|
||||
}
|
||||
|
||||
func (n *Layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||
func (n *Layer) containerList(ctx context.Context, listParams ListBucketsParams) ([]*data.BucketInfo, error) {
|
||||
stoken := n.SessionTokenForRead(ctx)
|
||||
|
||||
prm := frostfs.PrmUserContainers{
|
||||
|
@ -85,7 +89,7 @@ func (n *Layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
|||
|
||||
res, err := n.frostFS.UserContainers(ctx, prm)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Error(logs.CouldNotListUserContainers, zap.Error(err))
|
||||
n.reqLogger(ctx).Error(logs.CouldNotListUserContainers, zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
@ -97,14 +101,38 @@ func (n *Layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
|||
}
|
||||
info, err := n.containerInfo(ctx, getPrm)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Error(logs.CouldNotFetchContainerInfo, zap.Error(err))
|
||||
n.reqLogger(ctx).Error(logs.CouldNotFetchContainerInfo, zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
||||
continue
|
||||
}
|
||||
|
||||
if shouldSkipBucket(info, listParams) {
|
||||
continue
|
||||
}
|
||||
|
||||
list = append(list, info)
|
||||
}
|
||||
|
||||
return list, nil
|
||||
sort.Slice(list, func(i, j int) bool {
|
||||
return list[i].Name < list[j].Name
|
||||
})
|
||||
|
||||
for i, info := range list {
|
||||
if listParams.ContinuationToken != "" && info.Name != listParams.ContinuationToken {
|
||||
continue
|
||||
}
|
||||
return list[i:], nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func shouldSkipBucket(info *data.BucketInfo, prm ListBucketsParams) bool {
|
||||
if !strings.HasPrefix(info.Name, prm.Prefix) ||
|
||||
(prm.BucketRegion != "" && info.LocationConstraint != prm.BucketRegion) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) {
|
||||
|
@ -148,6 +176,7 @@ func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
|||
|
||||
bktInfo.CID = res.ContainerID
|
||||
bktInfo.HomomorphicHashDisabled = res.HomomorphicHashDisabled
|
||||
bktInfo.PlacementPolicy = p.Policy
|
||||
|
||||
n.cache.PutBucket(bktInfo)
|
||||
|
||||
|
|
|
@ -3,6 +3,7 @@ package layer
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
@ -91,12 +92,13 @@ func (n *Layer) deleteCORSObject(ctx context.Context, bktInfo *data.BucketInfo,
|
|||
if err := n.objectDeleteWithAuth(ctx, corsBkt, addr.Object(), prmAuth); err != nil {
|
||||
n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err),
|
||||
zap.String("cnrID", corsBkt.CID.EncodeToString()),
|
||||
zap.String("objID", addr.Object().EncodeToString()))
|
||||
zap.String("objID", addr.Object().EncodeToString()),
|
||||
logs.TagField(logs.TagExternalStorage))
|
||||
}
|
||||
}
|
||||
|
||||
func (n *Layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*data.CORSConfiguration, error) {
|
||||
cors, err := n.getCORS(ctx, bktInfo)
|
||||
func (n *Layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, decoder func(io.Reader, string) *xml.Decoder) (*data.CORSConfiguration, error) {
|
||||
cors, err := n.getCORS(ctx, bktInfo, decoder)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -240,6 +240,9 @@ var (
|
|||
|
||||
// ErrGlobalDomainIsAlreadyTaken is returned from FrostFS in case of global domain is already taken.
|
||||
ErrGlobalDomainIsAlreadyTaken = errors.New("global domain is already taken")
|
||||
|
||||
// ErrQuotaLimitReached is returned from FrostFS in case of quota exceeded.
|
||||
ErrQuotaLimitReached = errors.New("quota limit reached")
|
||||
)
|
||||
|
||||
// FrostFS represents virtual connection to FrostFS network.
|
||||
|
@ -354,4 +357,7 @@ type FrostFS interface {
|
|||
|
||||
// Relations returns implementation of relations.Relations interface.
|
||||
Relations() relations.Relations
|
||||
|
||||
// NetmapSnapshot returns information about FrostFS network map.
|
||||
NetmapSnapshot(context.Context) (netmap.NetMap, error)
|
||||
}
|
||||
|
|
|
@ -76,6 +76,7 @@ var _ frostfs.FrostFS = (*TestFrostFS)(nil)
|
|||
|
||||
type TestFrostFS struct {
|
||||
objects map[string]*object.Object
|
||||
copiesNumbers map[string][]uint32
|
||||
objectErrors map[string]error
|
||||
objectPutErrors map[string]error
|
||||
containers map[string]*container.Container
|
||||
|
@ -88,6 +89,7 @@ type TestFrostFS struct {
|
|||
func NewTestFrostFS(key *keys.PrivateKey) *TestFrostFS {
|
||||
return &TestFrostFS{
|
||||
objects: make(map[string]*object.Object),
|
||||
copiesNumbers: make(map[string][]uint32),
|
||||
objectErrors: make(map[string]error),
|
||||
objectPutErrors: make(map[string]error),
|
||||
containers: make(map[string]*container.Container),
|
||||
|
@ -126,6 +128,10 @@ func (t *TestFrostFS) Objects() []*object.Object {
|
|||
return res
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) CopiesNumbers(addr string) []uint32 {
|
||||
return t.copiesNumbers[addr]
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) ObjectExists(objID oid.ID) bool {
|
||||
for _, obj := range t.objects {
|
||||
if id, _ := obj.ID(); id.Equals(objID) {
|
||||
|
@ -346,6 +352,8 @@ func (t *TestFrostFS) CreateObject(ctx context.Context, prm frostfs.PrmObjectCre
|
|||
|
||||
addr := newAddress(cnrID, objID)
|
||||
t.objects[addr.EncodeToString()] = obj
|
||||
t.copiesNumbers[addr.EncodeToString()] = prm.CopiesNumber
|
||||
|
||||
return &frostfs.CreateObjectResult{
|
||||
ObjectID: objID,
|
||||
CreationEpoch: t.currentEpoch - 1,
|
||||
|
@ -475,6 +483,10 @@ func (t *TestFrostFS) NetworkInfo(context.Context) (netmap.NetworkInfo, error) {
|
|||
return ni, nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) NetmapSnapshot(context.Context) (netmap.NetMap, error) {
|
||||
return netmap.NetMap{}, nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) PatchObject(ctx context.Context, prm frostfs.PrmObjectPatch) (oid.ID, error) {
|
||||
obj, err := t.retrieveObject(ctx, prm.Container, prm.Object)
|
||||
if err != nil {
|
||||
|
|
|
@ -195,6 +195,18 @@ type (
|
|||
Encode string
|
||||
}
|
||||
|
||||
ListBucketsParams struct {
|
||||
MaxBuckets int
|
||||
Prefix string
|
||||
ContinuationToken string
|
||||
BucketRegion string
|
||||
}
|
||||
|
||||
ListBucketsResult struct {
|
||||
Containers []*data.BucketInfo
|
||||
ContinuationToken string
|
||||
}
|
||||
|
||||
// VersionedObject stores info about objects to delete.
|
||||
VersionedObject struct {
|
||||
Name string
|
||||
|
@ -371,8 +383,24 @@ func (n *Layer) ResolveCID(ctx context.Context, name string) (cid.ID, error) {
|
|||
|
||||
// ListBuckets returns all user containers. The name of the bucket is a container
|
||||
// id. Timestamp is omitted since it is not saved in frostfs container.
|
||||
func (n *Layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||
return n.containerList(ctx)
|
||||
func (n *Layer) ListBuckets(ctx context.Context, params ListBucketsParams) (ListBucketsResult, error) {
|
||||
var result ListBucketsResult
|
||||
var err error
|
||||
|
||||
if params.MaxBuckets == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
result.Containers, err = n.containerList(ctx, params)
|
||||
if err != nil {
|
||||
return ListBucketsResult{}, err
|
||||
}
|
||||
if len(result.Containers) > params.MaxBuckets {
|
||||
result.ContinuationToken = result.Containers[params.MaxBuckets].Name
|
||||
result.Containers = result.Containers[:params.MaxBuckets]
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// GetObject from storage.
|
||||
|
@ -515,7 +543,9 @@ func (n *Layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams)
|
|||
|
||||
n.reqLogger(ctx).Debug(logs.GetObject,
|
||||
zap.Stringer("cid", p.BktInfo.CID),
|
||||
zap.Stringer("oid", objInfo.ObjectInfo.ID))
|
||||
zap.Stringer("oid", objInfo.ObjectInfo.ID),
|
||||
logs.TagField(logs.TagDatapath),
|
||||
)
|
||||
|
||||
return objInfo, nil
|
||||
}
|
||||
|
@ -568,8 +598,8 @@ func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
|||
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
|
||||
return obj
|
||||
}
|
||||
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting,
|
||||
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error))
|
||||
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting, zap.Stringer("cid", bkt.CID),
|
||||
zap.String("oid", obj.VersionID), zap.Error(obj.Error), logs.TagField(logs.TagExternalStorage))
|
||||
}
|
||||
|
||||
if obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID); obj.Error != nil {
|
||||
|
@ -607,8 +637,8 @@ func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
|||
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
|
||||
return obj
|
||||
}
|
||||
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting,
|
||||
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error))
|
||||
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting, zap.Stringer("cid", bkt.CID),
|
||||
zap.String("oid", obj.VersionID), zap.Error(obj.Error), logs.TagField(logs.TagExternalStorage))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -723,7 +753,7 @@ func (n *Layer) getNodeVersionsToDelete(ctx context.Context, bkt *data.BucketInf
|
|||
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", apierr.GetAPIError(apierr.ErrNoSuchVersion))
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Debug(logs.GetTreeNodeToDelete, zap.Stringer("cid", bkt.CID), zap.Strings("oids", oids))
|
||||
n.reqLogger(ctx).Debug(logs.GetTreeNodeToDelete, zap.Stringer("cid", bkt.CID), zap.Strings("oids", oids), logs.TagField(logs.TagDatapath))
|
||||
|
||||
return versionsToDelete, nil
|
||||
}
|
||||
|
@ -793,7 +823,7 @@ func (n *Layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*Ver
|
|||
for i, obj := range p.Objects {
|
||||
p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.Settings, obj, p.NetworkInfo)
|
||||
if p.IsMultiple && p.Objects[i].Error != nil {
|
||||
n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error))
|
||||
n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error), logs.TagField(logs.TagExternalStorage))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -823,7 +853,7 @@ func (n *Layer) ResolveBucket(ctx context.Context, zone, name string) (cid.ID, e
|
|||
return cid.ID{}, err
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Info(logs.ResolveBucket, zap.Stringer("cid", cnrID))
|
||||
n.reqLogger(ctx).Info(logs.ResolveBucket, zap.Stringer("cid", cnrID), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
return cnrID, nil
|
||||
|
@ -848,12 +878,12 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
|||
|
||||
corsObj, err := n.treeService.GetBucketCORS(ctx, p.BktInfo)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Error(logs.GetBucketCors, zap.Error(err))
|
||||
n.reqLogger(ctx).Error(logs.GetBucketCorsFromTree, zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
|
||||
}
|
||||
|
||||
lifecycleObj, treeErr := n.treeService.GetBucketLifecycleConfiguration(ctx, p.BktInfo)
|
||||
if treeErr != nil {
|
||||
n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr))
|
||||
n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr), logs.TagField(logs.TagExternalStorageTree))
|
||||
}
|
||||
|
||||
err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
||||
|
|
|
@ -79,7 +79,9 @@ func (n *Layer) deleteLifecycleObject(ctx context.Context, bktInfo *data.BucketI
|
|||
if err := n.objectDeleteWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth); err != nil {
|
||||
n.reqLogger(ctx).Error(logs.CouldntDeleteLifecycleObject, zap.Error(err),
|
||||
zap.String("cid", lifecycleBkt.CID.EncodeToString()),
|
||||
zap.String("oid", addr.Object().EncodeToString()))
|
||||
zap.String("oid", addr.Object().EncodeToString()),
|
||||
logs.TagField(logs.TagExternalStorage),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -541,7 +541,7 @@ func (n *Layer) initWorkerPool(ctx context.Context, size int, p commonVersionsLi
|
|||
|
||||
realSize, err := GetObjectSize(oi)
|
||||
if err != nil {
|
||||
reqLog.Debug(logs.FailedToGetRealObjectSize, zap.Error(err))
|
||||
reqLog.Debug(logs.FailedToGetRealObjectSize, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
realSize = oi.Size
|
||||
}
|
||||
|
||||
|
@ -554,7 +554,7 @@ func (n *Layer) initWorkerPool(ctx context.Context, size int, p commonVersionsLi
|
|||
})
|
||||
if err != nil {
|
||||
wg.Done()
|
||||
reqLog.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err))
|
||||
reqLog.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}(node)
|
||||
}
|
||||
|
@ -645,7 +645,7 @@ func (n *Layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo
|
|||
|
||||
meta, err := n.objectHead(ctx, bktInfo, node.OID)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Warn(logs.CouldNotFetchObjectMeta, zap.Error(err))
|
||||
n.reqLogger(ctx).Warn(logs.CouldNotFetchObjectMeta, zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
@ -58,10 +58,9 @@ type (
|
|||
}
|
||||
|
||||
CreateMultipartParams struct {
|
||||
Info *UploadInfoParams
|
||||
Header map[string]string
|
||||
Data *UploadData
|
||||
CopiesNumbers []uint32
|
||||
Info *UploadInfoParams
|
||||
Header map[string]string
|
||||
Data *UploadData
|
||||
}
|
||||
|
||||
UploadData struct {
|
||||
|
@ -75,6 +74,7 @@ type (
|
|||
Reader io.Reader
|
||||
ContentMD5 string
|
||||
ContentSHA256Hash string
|
||||
CopiesNumbers []uint32
|
||||
}
|
||||
|
||||
UploadCopyParams struct {
|
||||
|
@ -85,11 +85,13 @@ type (
|
|||
SrcEncryption encryption.Params
|
||||
PartNumber int
|
||||
Range *RangeParams
|
||||
CopiesNumbers []uint32
|
||||
}
|
||||
|
||||
CompleteMultipartParams struct {
|
||||
Info *UploadInfoParams
|
||||
Parts []*CompletedPart
|
||||
Info *UploadInfoParams
|
||||
Parts []*CompletedPart
|
||||
CopiesNumbers []uint32
|
||||
}
|
||||
|
||||
CompletedPart struct {
|
||||
|
@ -165,7 +167,6 @@ func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
|
|||
Owner: n.gateOwner,
|
||||
Created: TimeNow(ctx),
|
||||
Meta: make(map[string]string, metaSize),
|
||||
CopiesNumbers: p.CopiesNumbers,
|
||||
CreationEpoch: networkInfo.CurrentEpoch(),
|
||||
}
|
||||
|
||||
|
@ -212,7 +213,7 @@ func (n *Layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, er
|
|||
func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) {
|
||||
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
||||
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
||||
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
|
||||
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters)
|
||||
}
|
||||
|
||||
|
@ -222,7 +223,7 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
|||
Attributes: make([][2]string, 2),
|
||||
Payload: p.Reader,
|
||||
CreationTime: TimeNow(ctx),
|
||||
CopiesNumber: multipartInfo.CopiesNumbers,
|
||||
CopiesNumber: p.CopiesNumbers,
|
||||
}
|
||||
|
||||
decSize := p.Size
|
||||
|
@ -265,7 +266,10 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
|||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||
err = n.frostFS.DeleteObject(ctx, prm)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
|
||||
zap.Stringer("cid", bktInfo.CID),
|
||||
zap.Stringer("oid", createdObj.ID),
|
||||
logs.TagField(logs.TagExternalStorage))
|
||||
}
|
||||
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
|
||||
}
|
||||
|
@ -282,7 +286,10 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
|||
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
|
||||
err = n.objectDelete(ctx, bktInfo, createdObj.ID)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
|
||||
zap.Stringer("cid", bktInfo.CID),
|
||||
zap.Stringer("oid", createdObj.ID),
|
||||
logs.TagField(logs.TagExternalStorage))
|
||||
}
|
||||
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
|
||||
}
|
||||
|
@ -290,7 +297,7 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
|||
|
||||
n.reqLogger(ctx).Debug(logs.UploadPart,
|
||||
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
|
||||
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID), logs.TagField(logs.TagDatapath))
|
||||
|
||||
partInfo := &data.PartInfo{
|
||||
Key: p.Info.Key,
|
||||
|
@ -313,7 +320,8 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
|||
if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil {
|
||||
n.reqLogger(ctx).Error(logs.CouldntDeleteOldPartObject, zap.Error(err),
|
||||
zap.String("cid", bktInfo.CID.EncodeToString()),
|
||||
zap.String("oid", oldPartID.EncodeToString()))
|
||||
zap.String("oid", oldPartID.EncodeToString()),
|
||||
logs.TagField(logs.TagExternalStorage))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -372,10 +380,11 @@ func (n *Layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
|
|||
}
|
||||
|
||||
params := &UploadPartParams{
|
||||
Info: p.Info,
|
||||
PartNumber: p.PartNumber,
|
||||
Size: size,
|
||||
Reader: objPayload,
|
||||
Info: p.Info,
|
||||
PartNumber: p.PartNumber,
|
||||
Size: size,
|
||||
Reader: objPayload,
|
||||
CopiesNumbers: p.CopiesNumbers,
|
||||
}
|
||||
|
||||
return n.uploadPart(ctx, multipartInfo, params)
|
||||
|
@ -472,14 +481,15 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
|||
Header: initMetadata,
|
||||
Size: &multipartObjetSize,
|
||||
Encryption: p.Info.Encryption,
|
||||
CopiesNumbers: multipartInfo.CopiesNumbers,
|
||||
CopiesNumbers: p.CopiesNumbers,
|
||||
CompleteMD5Hash: hex.EncodeToString(md5Hash.Sum(nil)) + "-" + strconv.Itoa(len(p.Parts)),
|
||||
})
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Error(logs.CouldNotPutCompletedObject,
|
||||
zap.String("uploadID", p.Info.UploadID),
|
||||
zap.String("uploadKey", p.Info.Key),
|
||||
zap.Error(err))
|
||||
zap.Error(err),
|
||||
logs.TagField(logs.TagExternalStorage))
|
||||
|
||||
return nil, nil, apierr.GetAPIError(apierr.ErrInternalError)
|
||||
}
|
||||
|
@ -491,7 +501,8 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
|||
if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil {
|
||||
n.reqLogger(ctx).Warn(logs.CouldNotDeleteUploadPart,
|
||||
zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID),
|
||||
zap.Error(err))
|
||||
zap.Error(err),
|
||||
logs.TagField(logs.TagExternalStorage))
|
||||
}
|
||||
addr.SetObject(partInfo.OID)
|
||||
n.cache.DeleteObject(addr)
|
||||
|
@ -585,7 +596,7 @@ func (n *Layer) deleteUploadedParts(ctx context.Context, bkt *data.BucketInfo, p
|
|||
oids, err := relations.ListAllRelations(ctx, n.frostFS.Relations(), bkt.CID, info.OID, tokens)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Warn(logs.FailedToListAllObjectRelations, zap.String("cid", bkt.CID.EncodeToString()),
|
||||
zap.String("oid", info.OID.EncodeToString()), zap.Error(err))
|
||||
zap.String("oid", info.OID.EncodeToString()), zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
||||
continue
|
||||
}
|
||||
members = append(members, append(oids, info.OID)...)
|
||||
|
@ -594,7 +605,7 @@ func (n *Layer) deleteUploadedParts(ctx context.Context, bkt *data.BucketInfo, p
|
|||
|
||||
err := n.putTombstones(ctx, bkt, networkInfo, members)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Warn(logs.FailedToPutTombstones, zap.Error(err))
|
||||
n.reqLogger(ctx).Warn(logs.FailedToPutTombstones, zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -607,7 +618,7 @@ func (n *Layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
|
|||
|
||||
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
||||
if err = p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
||||
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
|
||||
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters)
|
||||
}
|
||||
|
||||
|
@ -699,7 +710,8 @@ func (n *Layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
|
|||
zap.Stringer("cid", p.Bkt.CID),
|
||||
zap.String("upload id", p.UploadID),
|
||||
zap.Ints("part numbers", partsNumbers),
|
||||
zap.Strings("oids", oids))
|
||||
zap.Strings("oids", oids),
|
||||
logs.TagField(logs.TagDatapath))
|
||||
|
||||
return multipartInfo, res, nil
|
||||
}
|
||||
|
|
|
@ -289,13 +289,17 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
|||
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
|
||||
}
|
||||
headerMd5Hash, err := base64.StdEncoding.DecodeString(*p.ContentMD5)
|
||||
if err != nil {
|
||||
if err != nil || len(headerMd5Hash) != md5.Size {
|
||||
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
|
||||
}
|
||||
if !bytes.Equal(headerMd5Hash, createdObj.MD5Sum) {
|
||||
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
|
||||
zap.Stringer("cid", p.BktInfo.CID),
|
||||
zap.Stringer("oid", createdObj.ID),
|
||||
logs.TagField(logs.TagExternalStorage),
|
||||
)
|
||||
}
|
||||
return nil, apierr.GetAPIError(apierr.ErrBadDigest)
|
||||
}
|
||||
|
@ -309,13 +313,17 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
|||
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
|
||||
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
|
||||
zap.Stringer("cid", p.BktInfo.CID),
|
||||
zap.Stringer("oid", createdObj.ID),
|
||||
logs.TagField(logs.TagExternalStorage))
|
||||
}
|
||||
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
|
||||
}
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID),
|
||||
zap.Stringer("oid", createdObj.ID), logs.TagField(logs.TagExternalStorage))
|
||||
now := TimeNow(ctx)
|
||||
newVersion := &data.NodeVersion{
|
||||
BaseNodeVersion: data.BaseNodeVersion{
|
||||
|
@ -410,7 +418,7 @@ func (n *Layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.Bucke
|
|||
|
||||
meta, err := n.objectHead(ctx, bkt, node.OID)
|
||||
if err != nil {
|
||||
if client.IsErrObjectNotFound(err) {
|
||||
if client.IsErrObjectNotFound(err) || client.IsErrObjectAlreadyRemoved(err) {
|
||||
return nil, fmt.Errorf("%w: %s; %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error(), node.OID.EncodeToString())
|
||||
}
|
||||
return nil, err
|
||||
|
@ -467,7 +475,7 @@ func (n *Layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
|
|||
|
||||
meta, err := n.objectHead(ctx, bkt, foundVersion.OID)
|
||||
if err != nil {
|
||||
if client.IsErrObjectNotFound(err) {
|
||||
if client.IsErrObjectNotFound(err) || client.IsErrObjectAlreadyRemoved(err) {
|
||||
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchVersion), err.Error())
|
||||
}
|
||||
return nil, err
|
||||
|
@ -525,10 +533,7 @@ func (n *Layer) objectPutAndHash(ctx context.Context, prm frostfs.PrmObjectCreat
|
|||
})
|
||||
res, err := n.frostFS.CreateObject(ctx, prm)
|
||||
if err != nil {
|
||||
if _, errDiscard := io.Copy(io.Discard, prm.Payload); errDiscard != nil {
|
||||
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard))
|
||||
}
|
||||
|
||||
n.payloadDiscard(ctx, prm.Payload)
|
||||
return nil, err
|
||||
}
|
||||
return &data.CreatedObjectInfo{
|
||||
|
@ -540,12 +545,20 @@ func (n *Layer) objectPutAndHash(ctx context.Context, prm frostfs.PrmObjectCreat
|
|||
}, nil
|
||||
}
|
||||
|
||||
func (n *Layer) payloadDiscard(ctx context.Context, payload io.Reader) {
|
||||
if payload != nil {
|
||||
if _, errDiscard := io.Copy(io.Discard, payload); errDiscard != nil {
|
||||
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type logWrapper struct {
|
||||
log *zap.Logger
|
||||
}
|
||||
|
||||
func (l *logWrapper) Printf(format string, args ...interface{}) {
|
||||
l.log.Info(fmt.Sprintf(format, args...))
|
||||
l.log.Info(fmt.Sprintf(format, args...), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
func IsSystemHeader(key string) bool {
|
||||
|
|
|
@ -49,3 +49,16 @@ func TestGoroutinesDontLeakInPutAndHash(t *testing.T) {
|
|||
require.ErrorIs(t, err, expErr)
|
||||
require.Empty(t, payload.Len(), "body must be read out otherwise goroutines can leak in wrapReader")
|
||||
}
|
||||
|
||||
func TestNilPayloadPutAndHash(t *testing.T) {
|
||||
tc := prepareContext(t)
|
||||
prm := frostfs.PrmObjectCreate{
|
||||
Filepath: tc.obj,
|
||||
Payload: nil,
|
||||
}
|
||||
|
||||
expErr := errors.New("some error")
|
||||
tc.testFrostFS.SetObjectPutError(tc.obj, expErr)
|
||||
_, err := tc.layer.objectPutAndHash(tc.ctx, prm, tc.bktInfo)
|
||||
require.ErrorIs(t, err, expErr)
|
||||
}
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"strconv"
|
||||
"time"
|
||||
|
@ -13,6 +14,7 @@ import (
|
|||
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
|
@ -160,7 +162,7 @@ func (n *Layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion)
|
|||
return lockInfo, nil
|
||||
}
|
||||
|
||||
func (n *Layer) getCORS(ctx context.Context, bkt *data.BucketInfo) (*data.CORSConfiguration, error) {
|
||||
func (n *Layer) getCORS(ctx context.Context, bkt *data.BucketInfo, decoder func(io.Reader, string) *xml.Decoder) (*data.CORSConfiguration, error) {
|
||||
owner := n.BearerOwner(ctx)
|
||||
if cors := n.cache.GetCORS(owner, bkt); cors != nil {
|
||||
return cors, nil
|
||||
|
@ -189,7 +191,7 @@ func (n *Layer) getCORS(ctx context.Context, bkt *data.BucketInfo) (*data.CORSCo
|
|||
}
|
||||
|
||||
cors := &data.CORSConfiguration{}
|
||||
if err = xml.NewDecoder(obj.Payload).Decode(&cors); err != nil {
|
||||
if err = decoder(obj.Payload, "").Decode(&cors); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal cors: %w", err)
|
||||
}
|
||||
|
||||
|
@ -215,6 +217,7 @@ func (n *Layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo)
|
|||
return nil, err
|
||||
}
|
||||
settings = &data.BucketSettings{Versioning: data.VersioningUnversioned}
|
||||
n.reqLogger(ctx).Debug(logs.BucketSettingsNotFoundUseDefaults, logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
n.cache.PutSettings(owner, bktInfo, settings)
|
||||
|
|
|
@ -168,7 +168,8 @@ func (n *Layer) getNodeVersion(ctx context.Context, objVersion *data.ObjectVersi
|
|||
|
||||
if err == nil && version != nil && !version.IsDeleteMarker {
|
||||
n.reqLogger(ctx).Debug(logs.GetTreeNode,
|
||||
zap.Stringer("cid", objVersion.BktInfo.CID), zap.Stringer("oid", version.OID))
|
||||
zap.Stringer("cid", objVersion.BktInfo.CID),
|
||||
zap.Stringer("oid", version.OID), logs.TagField(logs.TagExternalStorageTree))
|
||||
}
|
||||
|
||||
return version, err
|
||||
|
|
|
@ -65,13 +65,13 @@ func (n *Layer) submitPutTombstone(ctx context.Context, bkt *data.BucketInfo, me
|
|||
defer wg.Done()
|
||||
|
||||
if err := n.putTombstoneObject(ctx, tomb, bkt); err != nil {
|
||||
n.reqLogger(ctx).Warn(logs.FailedToPutTombstoneObject, zap.String("cid", bkt.CID.EncodeToString()), zap.Error(err))
|
||||
n.reqLogger(ctx).Warn(logs.FailedToPutTombstoneObject, zap.String("cid", bkt.CID.EncodeToString()), zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
||||
errCh <- fmt.Errorf("put tombstone object: %w", err)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
wg.Done()
|
||||
n.reqLogger(ctx).Warn(logs.FailedToSubmitTaskToPool, zap.Error(err))
|
||||
n.reqLogger(ctx).Warn(logs.FailedToSubmitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
errCh <- fmt.Errorf("submit task to pool: %w", err)
|
||||
}
|
||||
}
|
||||
|
@ -106,7 +106,7 @@ func (n *Layer) getMembers(ctx context.Context, cnrID cid.ID, objID oid.ID, toke
|
|||
}
|
||||
|
||||
n.reqLogger(ctx).Warn(logs.FailedToListAllObjectRelations, zap.String("cid", cnrID.EncodeToString()),
|
||||
zap.String("oid", objID.EncodeToString()), zap.Error(err))
|
||||
zap.String("oid", objID.EncodeToString()), zap.Error(err), logs.TagField(logs.TagExternalStorage))
|
||||
return nil, nil
|
||||
}
|
||||
return append(oids, objID), nil
|
||||
|
|
|
@ -11,6 +11,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
|
@ -156,7 +157,8 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
|||
|
||||
bktName := "testbucket1"
|
||||
res, err := tp.CreateContainer(ctx, frostfs.PrmContainerCreate{
|
||||
Name: bktName,
|
||||
Name: bktName,
|
||||
Policy: getPlacementPolicy(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
|
@ -445,3 +447,10 @@ func TestFilterVersionsByMarker(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func getPlacementPolicy() (p netmap.PlacementPolicy) {
|
||||
var r netmap.ReplicaDescriptor
|
||||
r.SetNumberOfObjects(1)
|
||||
p.AddReplicas([]netmap.ReplicaDescriptor{r}...)
|
||||
return p
|
||||
}
|
||||
|
|
|
@ -110,7 +110,7 @@ func preparePathStyleAddress(reqInfo *ReqInfo, r *http.Request, reqLogger *zap.L
|
|||
// https://github.com/go-chi/chi/issues/641
|
||||
// https://github.com/go-chi/chi/issues/642
|
||||
if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil {
|
||||
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err))
|
||||
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
} else {
|
||||
reqInfo.ObjectName = obj
|
||||
}
|
||||
|
|
|
@ -53,15 +53,15 @@ func Auth(center Center, log *zap.Logger) Func {
|
|||
box, err := center.Authenticate(r)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNoAuthorizationHeader) {
|
||||
reqLogOrDefault(ctx, log).Debug(logs.CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed, zap.Error(err))
|
||||
reqLogOrDefault(ctx, log).Debug(logs.CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
} else {
|
||||
reqLogOrDefault(ctx, log).Error(logs.FailedToPassAuthentication, zap.Error(err))
|
||||
reqLogOrDefault(ctx, log).Error(logs.FailedToPassAuthentication, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
err = apierr.TransformToS3Error(err)
|
||||
if err.(apierr.Error).ErrCode == apierr.ErrInternalError {
|
||||
err = apierr.GetAPIError(apierr.ErrAccessDenied)
|
||||
}
|
||||
if _, wrErr := WriteErrorResponse(w, GetReqInfo(r.Context()), err); wrErr != nil {
|
||||
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
|
||||
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
@ -71,7 +71,7 @@ func Auth(center Center, log *zap.Logger) Func {
|
|||
if box.AccessBox.Gate.BearerToken != nil {
|
||||
reqInfo.User = bearer.ResolveIssuer(*box.AccessBox.Gate.BearerToken).String()
|
||||
}
|
||||
reqLogOrDefault(ctx, log).Debug(logs.SuccessfulAuth, zap.String("accessKeyID", box.AuthHeaders.AccessKeyID))
|
||||
reqLogOrDefault(ctx, log).Debug(logs.SuccessfulAuth, zap.String("accessKeyID", box.AuthHeaders.AccessKeyID), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
h.ServeHTTP(w, r.WithContext(ctx))
|
||||
|
@ -89,15 +89,15 @@ func FrostfsIDValidation(frostfsID FrostFSIDValidator, log *zap.Logger) Func {
|
|||
ctx := r.Context()
|
||||
bd, err := GetBoxData(ctx)
|
||||
if err != nil || bd.Gate.BearerToken == nil {
|
||||
reqLogOrDefault(ctx, log).Debug(logs.AnonRequestSkipFrostfsIDValidation)
|
||||
reqLogOrDefault(ctx, log).Debug(logs.AnonRequestSkipFrostfsIDValidation, logs.TagField(logs.TagDatapath))
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
if err = validateBearerToken(frostfsID, bd.Gate.BearerToken); err != nil {
|
||||
reqLogOrDefault(ctx, log).Error(logs.FrostfsIDValidationFailed, zap.Error(err))
|
||||
reqLogOrDefault(ctx, log).Error(logs.FrostfsIDValidationFailed, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
if _, wrErr := WriteErrorResponse(w, GetReqInfo(r.Context()), err); wrErr != nil {
|
||||
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
|
||||
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
|
@ -23,7 +23,7 @@ type (
|
|||
)
|
||||
|
||||
func LogHTTP(l *zap.Logger, _ LogHTTPSettings) Func {
|
||||
l.Warn(logs.LogHTTPDisabledInThisBuild)
|
||||
l.Warn(logs.LogHTTPDisabledInThisBuild, logs.TagField(logs.TagApp))
|
||||
return func(h http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
h.ServeHTTP(w, r)
|
||||
|
|
|
@ -139,7 +139,7 @@ func resolveCID(log *zap.Logger, resolveContainerID ContainerIDResolveFunc) cidR
|
|||
|
||||
containerID, err := resolveContainerID(ctx, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
reqLogOrDefault(ctx, log).Debug(logs.FailedToResolveCID, zap.Error(err))
|
||||
reqLogOrDefault(ctx, log).Debug(logs.FailedToResolveCID, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
return ""
|
||||
}
|
||||
|
||||
|
|
|
@ -90,10 +90,10 @@ func PolicyCheck(cfg PolicyConfig) Func {
|
|||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
if err := policyCheck(r, cfg); err != nil {
|
||||
reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err))
|
||||
reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
err = apierr.TransformToS3Error(err)
|
||||
if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil {
|
||||
reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
|
||||
reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
@ -200,7 +200,9 @@ func getPolicyRequest(r *http.Request, cfg PolicyConfig, reqType ReqType, bktNam
|
|||
|
||||
reqLogOrDefault(r.Context(), cfg.Log).Debug(logs.PolicyRequest, zap.String("action", op),
|
||||
zap.String("resource", res), zap.Any("request properties", requestProps),
|
||||
zap.Any("resource properties", resourceProps))
|
||||
zap.Any("resource properties", resourceProps),
|
||||
logs.TagField(logs.TagDatapath),
|
||||
)
|
||||
|
||||
return testutil.NewRequest(op, testutil.NewResource(res, resourceProps), requestProps), pk, groups, nil
|
||||
}
|
||||
|
|
|
@ -166,7 +166,7 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
|
|||
// generate random UUIDv4
|
||||
id, err := uuid.NewRandom()
|
||||
if err != nil {
|
||||
log.Error(logs.FailedToGenerateRequestID, zap.Error(err))
|
||||
log.Error(logs.FailedToGenerateRequestID, zap.Error(err), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
|
||||
// set request id into response header
|
||||
|
@ -198,7 +198,8 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
|
|||
r = r.WithContext(SetReqLogger(ctx, reqLogger))
|
||||
|
||||
reqLogger.Info(logs.RequestStart, zap.String("host", r.Host),
|
||||
zap.String("remote_host", reqInfo.RemoteHost), zap.String("namespace", reqInfo.Namespace))
|
||||
zap.String("remote_host", reqInfo.RemoteHost), zap.String("namespace", reqInfo.Namespace),
|
||||
logs.TagField(logs.TagDatapath))
|
||||
|
||||
// continue execution
|
||||
h.ServeHTTP(lw, r)
|
||||
|
|
|
@ -331,7 +331,7 @@ func LogSuccessResponse(l *zap.Logger) Func {
|
|||
fields = append(fields, zap.String("user", reqInfo.User))
|
||||
}
|
||||
|
||||
reqLogger.Info(logs.RequestEnd, fields...)
|
||||
reqLogger.Info(logs.RequestEnd, append(fields, logs.TagField(logs.TagDatapath))...)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -245,13 +245,14 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
|||
zap.String("method", reqInfo.API),
|
||||
zap.String("http method", r.Method),
|
||||
zap.String("url", r.RequestURI),
|
||||
logs.TagField(logs.TagDatapath),
|
||||
}
|
||||
|
||||
if wrErr != nil {
|
||||
fields = append(fields, zap.NamedError("write_response_error", wrErr))
|
||||
}
|
||||
|
||||
log.Error(logs.RequestUnmatched, fields...)
|
||||
log.Error(logs.RequestUnmatched, append(fields, logs.TagField(logs.TagDatapath))...)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -266,13 +267,14 @@ func notSupportedHandler() http.HandlerFunc {
|
|||
fields := []zap.Field{
|
||||
zap.String("http method", r.Method),
|
||||
zap.String("url", r.RequestURI),
|
||||
logs.TagField(logs.TagDatapath),
|
||||
}
|
||||
|
||||
if wrErr != nil {
|
||||
fields = append(fields, zap.NamedError("write_response_error", wrErr))
|
||||
}
|
||||
|
||||
log.Error(logs.NotSupported, fields...)
|
||||
log.Error(logs.NotSupported, append(fields, logs.TagField(logs.TagDatapath))...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
311
cmd/s3-gw/app.go
311
cmd/s3-gw/app.go
|
@ -53,6 +53,7 @@ import (
|
|||
"github.com/panjf2000/ants/v2"
|
||||
"github.com/spf13/viper"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
"golang.org/x/exp/slices"
|
||||
"golang.org/x/text/encoding/ianaindex"
|
||||
"google.golang.org/grpc"
|
||||
|
@ -65,12 +66,13 @@ type (
|
|||
App struct {
|
||||
ctr s3middleware.Center
|
||||
log *zap.Logger
|
||||
cfg *viper.Viper
|
||||
cfg *appCfg
|
||||
pool *pool.Pool
|
||||
treePool *treepool.Pool
|
||||
key *keys.PrivateKey
|
||||
obj *layer.Layer
|
||||
api api.Handler
|
||||
cache *layer.Cache
|
||||
|
||||
frostfsid *frostfsid.FrostFSID
|
||||
|
||||
|
@ -90,6 +92,10 @@ type (
|
|||
wrkDone chan struct{}
|
||||
}
|
||||
|
||||
tagsConfig struct {
|
||||
tagLogs sync.Map
|
||||
}
|
||||
|
||||
loggerSettings struct {
|
||||
mu sync.RWMutex
|
||||
appMetrics *metrics.AppMetrics
|
||||
|
@ -98,6 +104,7 @@ type (
|
|||
appSettings struct {
|
||||
logLevel zap.AtomicLevel
|
||||
httpLogging s3middleware.LogHTTPConfig
|
||||
tagsConfig *tagsConfig
|
||||
maxClient maxClientsConfig
|
||||
defaultMaxAge int
|
||||
reconnectInterval time.Duration
|
||||
|
@ -137,13 +144,54 @@ type (
|
|||
deadline time.Duration
|
||||
count int
|
||||
}
|
||||
|
||||
Logger struct {
|
||||
logger *zap.Logger
|
||||
lvl zap.AtomicLevel
|
||||
}
|
||||
)
|
||||
|
||||
func (t *tagsConfig) LevelEnabled(tag string, tgtLevel zapcore.Level) bool {
|
||||
lvl, ok := t.tagLogs.Load(tag)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
return lvl.(zapcore.Level).Enabled(tgtLevel)
|
||||
}
|
||||
|
||||
func (t *tagsConfig) update(cfg *viper.Viper) error {
|
||||
tags, err := fetchLogTagsConfig(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
t.tagLogs.Range(func(key, value any) bool {
|
||||
k := key.(string)
|
||||
v := value.(zapcore.Level)
|
||||
|
||||
if lvl, ok := tags[k]; ok {
|
||||
if lvl != v {
|
||||
t.tagLogs.Store(key, lvl)
|
||||
}
|
||||
} else {
|
||||
t.tagLogs.Delete(key)
|
||||
delete(tags, k)
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
for k, v := range tags {
|
||||
t.tagLogs.Store(k, v)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func newTagsConfig(v *viper.Viper) *tagsConfig {
|
||||
var t tagsConfig
|
||||
if err := t.update(v); err != nil {
|
||||
// panic here is analogue of the similar panic during common log level initialization.
|
||||
panic(err.Error())
|
||||
}
|
||||
return &t
|
||||
}
|
||||
|
||||
func (s *loggerSettings) DroppedLogsInc() {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
@ -160,19 +208,18 @@ func (s *loggerSettings) setMetrics(appMetrics *metrics.AppMetrics) {
|
|||
s.appMetrics = appMetrics
|
||||
}
|
||||
|
||||
func newApp(ctx context.Context, v *viper.Viper) *App {
|
||||
func newApp(ctx context.Context, cfg *appCfg) *App {
|
||||
logSettings := &loggerSettings{}
|
||||
log := pickLogger(v, logSettings)
|
||||
settings := newAppSettings(log, v)
|
||||
|
||||
objPool, treePool, key := getPools(ctx, log.logger, v, settings.dialerSource)
|
||||
tagConfig := newTagsConfig(cfg.config())
|
||||
log := pickLogger(cfg.config(), logSettings, tagConfig)
|
||||
settings := newAppSettings(log, cfg.config())
|
||||
settings.tagsConfig = tagConfig
|
||||
appCache := layer.NewCache(getCacheOptions(cfg.config(), log.logger))
|
||||
|
||||
app := &App{
|
||||
log: log.logger,
|
||||
cfg: v,
|
||||
pool: objPool,
|
||||
treePool: treePool,
|
||||
key: key,
|
||||
log: log.logger,
|
||||
cfg: cfg,
|
||||
cache: appCache,
|
||||
|
||||
webDone: make(chan struct{}, 1),
|
||||
wrkDone: make(chan struct{}, 1),
|
||||
|
@ -186,7 +233,12 @@ func newApp(ctx context.Context, v *viper.Viper) *App {
|
|||
return app
|
||||
}
|
||||
|
||||
func (a *App) config() *viper.Viper {
|
||||
return a.cfg.config()
|
||||
}
|
||||
|
||||
func (a *App) init(ctx context.Context) {
|
||||
a.initPools(ctx)
|
||||
a.initResolver()
|
||||
a.initAuthCenter(ctx)
|
||||
a.setRuntimeParameters()
|
||||
|
@ -199,10 +251,10 @@ func (a *App) init(ctx context.Context) {
|
|||
}
|
||||
|
||||
func (a *App) initAuthCenter(ctx context.Context) {
|
||||
if a.cfg.IsSet(cfgContainersAccessBox) {
|
||||
if a.config().IsSet(cfgContainersAccessBox) {
|
||||
cnrID, err := a.resolveContainerID(ctx, cfgContainersAccessBox)
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.CouldNotFetchAccessBoxContainerInfo, zap.Error(err))
|
||||
a.log.Fatal(logs.CouldNotFetchAccessBoxContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
a.settings.accessbox = &cnrID
|
||||
}
|
||||
|
@ -210,41 +262,41 @@ func (a *App) initAuthCenter(ctx context.Context) {
|
|||
cfg := tokens.Config{
|
||||
FrostFS: frostfs.NewAuthmateFrostFS(frostfs.NewFrostFS(a.pool, a.key), a.log),
|
||||
Key: a.key,
|
||||
CacheConfig: getAccessBoxCacheConfig(a.cfg, a.log),
|
||||
RemovingCheckAfterDurations: fetchRemovingCheckInterval(a.cfg, a.log),
|
||||
CacheConfig: getAccessBoxCacheConfig(a.config(), a.log),
|
||||
RemovingCheckAfterDurations: fetchRemovingCheckInterval(a.config(), a.log),
|
||||
}
|
||||
|
||||
a.ctr = auth.New(tokens.New(cfg), a.cfg.GetStringSlice(cfgAllowedAccessKeyIDPrefixes), a.settings)
|
||||
a.ctr = auth.New(tokens.New(cfg), a.config().GetStringSlice(cfgAllowedAccessKeyIDPrefixes), a.settings)
|
||||
}
|
||||
|
||||
func (a *App) initLayer(ctx context.Context) {
|
||||
// prepare random key for anonymous requests
|
||||
randomKey, err := keys.NewPrivateKey()
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.CouldntGenerateRandomKey, zap.Error(err))
|
||||
a.log.Fatal(logs.CouldntGenerateRandomKey, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
var gateOwner user.ID
|
||||
user.IDFromKey(&gateOwner, a.key.PrivateKey.PublicKey)
|
||||
|
||||
var corsCnrInfo *data.BucketInfo
|
||||
if a.cfg.IsSet(cfgContainersCORS) {
|
||||
if a.config().IsSet(cfgContainersCORS) {
|
||||
corsCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersCORS)
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.CouldNotFetchCORSContainerInfo, zap.Error(err))
|
||||
a.log.Fatal(logs.CouldNotFetchCORSContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
}
|
||||
|
||||
var lifecycleCnrInfo *data.BucketInfo
|
||||
if a.cfg.IsSet(cfgContainersLifecycle) {
|
||||
if a.config().IsSet(cfgContainersLifecycle) {
|
||||
lifecycleCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersLifecycle)
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.CouldNotFetchLifecycleContainerInfo, zap.Error(err))
|
||||
a.log.Fatal(logs.CouldNotFetchLifecycleContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
}
|
||||
|
||||
layerCfg := &layer.Config{
|
||||
Cache: layer.NewCache(getCacheOptions(a.cfg, a.log)),
|
||||
Cache: a.cache,
|
||||
AnonKey: layer.AnonymousKey{
|
||||
Key: randomKey,
|
||||
},
|
||||
|
@ -265,7 +317,7 @@ func (a *App) initLayer(ctx context.Context) {
|
|||
func (a *App) initWorkerPool() *ants.Pool {
|
||||
workerPool, err := ants.NewPool(a.settings.workerPoolSize)
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err))
|
||||
a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
return workerPool
|
||||
}
|
||||
|
@ -274,6 +326,7 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
|
|||
settings := &appSettings{
|
||||
logLevel: log.lvl,
|
||||
httpLogging: s3middleware.LogHTTPConfig{},
|
||||
tagsConfig: newTagsConfig(v),
|
||||
maxClient: newMaxClients(v),
|
||||
defaultMaxAge: fetchDefaultMaxAge(v, log.logger),
|
||||
reconnectInterval: fetchReconnectInterval(v),
|
||||
|
@ -600,7 +653,7 @@ func (a *App) initMetrics() {
|
|||
Logger: a.log,
|
||||
PoolStatistics: frostfs.NewPoolStatistic(a.pool),
|
||||
TreeStatistic: a.treePool,
|
||||
Enabled: a.cfg.GetBool(cfgPrometheusEnabled),
|
||||
Enabled: a.config().GetBool(cfgPrometheusEnabled),
|
||||
}
|
||||
|
||||
a.metrics = metrics.NewAppMetrics(cfg)
|
||||
|
@ -610,9 +663,9 @@ func (a *App) initMetrics() {
|
|||
|
||||
func (a *App) initFrostfsID(ctx context.Context) {
|
||||
cli, err := ffidcontract.New(ctx, ffidcontract.Config{
|
||||
RPCAddress: a.cfg.GetString(cfgRPCEndpoint),
|
||||
Contract: a.cfg.GetString(cfgFrostfsIDContract),
|
||||
ProxyContract: a.cfg.GetString(cfgProxyContract),
|
||||
RPCAddress: a.config().GetString(cfgRPCEndpoint),
|
||||
Contract: a.config().GetString(cfgFrostfsIDContract),
|
||||
ProxyContract: a.config().GetString(cfgProxyContract),
|
||||
Key: a.key,
|
||||
Waiter: commonclient.WaiterOptions{
|
||||
IgnoreAlreadyExistsError: false,
|
||||
|
@ -620,24 +673,24 @@ func (a *App) initFrostfsID(ctx context.Context) {
|
|||
},
|
||||
})
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.InitFrostfsIDContractFailed, zap.Error(err))
|
||||
a.log.Fatal(logs.InitFrostfsIDFailed, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
a.frostfsid, err = frostfsid.NewFrostFSID(frostfsid.Config{
|
||||
Cache: cache.NewFrostfsIDCache(getFrostfsIDCacheConfig(a.cfg, a.log)),
|
||||
Cache: cache.NewFrostfsIDCache(getFrostfsIDCacheConfig(a.config(), a.log)),
|
||||
FrostFSID: cli,
|
||||
Logger: a.log,
|
||||
})
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.InitFrostfsIDContractFailed, zap.Error(err))
|
||||
a.log.Fatal(logs.InitFrostfsIDFailed, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) initPolicyStorage(ctx context.Context) {
|
||||
policyContract, err := contract.New(ctx, contract.Config{
|
||||
RPCAddress: a.cfg.GetString(cfgRPCEndpoint),
|
||||
Contract: a.cfg.GetString(cfgPolicyContract),
|
||||
ProxyContract: a.cfg.GetString(cfgProxyContract),
|
||||
RPCAddress: a.config().GetString(cfgRPCEndpoint),
|
||||
Contract: a.config().GetString(cfgPolicyContract),
|
||||
ProxyContract: a.config().GetString(cfgProxyContract),
|
||||
Key: a.key,
|
||||
Waiter: commonclient.WaiterOptions{
|
||||
IgnoreAlreadyExistsError: false,
|
||||
|
@ -645,12 +698,12 @@ func (a *App) initPolicyStorage(ctx context.Context) {
|
|||
},
|
||||
})
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.InitPolicyContractFailed, zap.Error(err))
|
||||
a.log.Fatal(logs.InitPolicyContractFailed, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
a.policyStorage = policy.NewStorage(policy.StorageConfig{
|
||||
Contract: policyContract,
|
||||
Cache: cache.NewMorphPolicyCache(getMorphPolicyCacheConfig(a.cfg, a.log)),
|
||||
Cache: cache.NewMorphPolicyCache(getMorphPolicyCacheConfig(a.config(), a.log)),
|
||||
Log: a.log,
|
||||
})
|
||||
}
|
||||
|
@ -659,26 +712,26 @@ func (a *App) initResolver() {
|
|||
var err error
|
||||
a.bucketResolver, err = resolver.NewBucketResolver(a.getResolverOrder(), a.getResolverConfig())
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err))
|
||||
a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) getResolverConfig() *resolver.Config {
|
||||
return &resolver.Config{
|
||||
FrostFS: frostfs.NewResolverFrostFS(a.pool),
|
||||
RPCAddress: a.cfg.GetString(cfgRPCEndpoint),
|
||||
RPCAddress: a.config().GetString(cfgRPCEndpoint),
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) getResolverOrder() []string {
|
||||
order := a.cfg.GetStringSlice(cfgResolveOrder)
|
||||
if a.cfg.GetString(cfgRPCEndpoint) == "" {
|
||||
order := a.config().GetStringSlice(cfgResolveOrder)
|
||||
if a.config().GetString(cfgRPCEndpoint) == "" {
|
||||
order = remove(order, resolver.NNSResolver)
|
||||
a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided)
|
||||
a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided, logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
if len(order) == 0 {
|
||||
a.log.Info(logs.ContainerResolverWillBeDisabled)
|
||||
a.log.Info(logs.ContainerResolverWillBeDisabled, logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
return order
|
||||
|
@ -690,42 +743,42 @@ func (a *App) initTracing(ctx context.Context) {
|
|||
instanceID = a.servers[0].Address()
|
||||
}
|
||||
cfg := tracing.Config{
|
||||
Enabled: a.cfg.GetBool(cfgTracingEnabled),
|
||||
Exporter: tracing.Exporter(a.cfg.GetString(cfgTracingExporter)),
|
||||
Endpoint: a.cfg.GetString(cfgTracingEndpoint),
|
||||
Enabled: a.config().GetBool(cfgTracingEnabled),
|
||||
Exporter: tracing.Exporter(a.config().GetString(cfgTracingExporter)),
|
||||
Endpoint: a.config().GetString(cfgTracingEndpoint),
|
||||
Service: "frostfs-s3-gw",
|
||||
InstanceID: instanceID,
|
||||
Version: version.Version,
|
||||
}
|
||||
|
||||
if trustedCa := a.cfg.GetString(cfgTracingTrustedCa); trustedCa != "" {
|
||||
if trustedCa := a.config().GetString(cfgTracingTrustedCa); trustedCa != "" {
|
||||
caBytes, err := os.ReadFile(trustedCa)
|
||||
if err != nil {
|
||||
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err))
|
||||
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
return
|
||||
}
|
||||
certPool := x509.NewCertPool()
|
||||
ok := certPool.AppendCertsFromPEM(caBytes)
|
||||
if !ok {
|
||||
a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert"))
|
||||
a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert"), logs.TagField(logs.TagApp))
|
||||
return
|
||||
}
|
||||
cfg.ServerCaCertPool = certPool
|
||||
}
|
||||
|
||||
attributes, err := fetchTracingAttributes(a.cfg)
|
||||
attributes, err := fetchTracingAttributes(a.config())
|
||||
if err != nil {
|
||||
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err))
|
||||
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
return
|
||||
}
|
||||
cfg.Attributes = attributes
|
||||
|
||||
updated, err := tracing.Setup(ctx, cfg)
|
||||
if err != nil {
|
||||
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err))
|
||||
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
if updated {
|
||||
a.log.Info(logs.TracingConfigUpdated)
|
||||
a.log.Info(logs.TracingConfigUpdated, logs.TagField(logs.TagApp))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -735,7 +788,7 @@ func (a *App) shutdownTracing() {
|
|||
defer cancel()
|
||||
|
||||
if err := tracing.Shutdown(shdnCtx); err != nil {
|
||||
a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err))
|
||||
a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -752,82 +805,87 @@ func newMaxClients(cfg *viper.Viper) maxClientsConfig {
|
|||
func getDialerSource(logger *zap.Logger, cfg *viper.Viper) *internalnet.DialerSource {
|
||||
source, err := internalnet.NewDialerSource(fetchMultinetConfig(cfg, logger))
|
||||
if err != nil {
|
||||
logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err))
|
||||
logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
return source
|
||||
}
|
||||
|
||||
func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper, dialSource *internalnet.DialerSource) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) {
|
||||
func (a *App) initPools(ctx context.Context) {
|
||||
var prm pool.InitParameters
|
||||
var prmTree treepool.InitParameters
|
||||
|
||||
password := wallet.GetPassword(cfg, cfgWalletPassphrase)
|
||||
key, err := wallet.GetKeyFromPath(cfg.GetString(cfgWalletPath), cfg.GetString(cfgWalletAddress), password)
|
||||
password := wallet.GetPassword(a.config(), cfgWalletPassphrase)
|
||||
key, err := wallet.GetKeyFromPath(a.config().GetString(cfgWalletPath), a.config().GetString(cfgWalletAddress), password)
|
||||
if err != nil {
|
||||
logger.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err))
|
||||
a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
prm.SetKey(&key.PrivateKey)
|
||||
prmTree.SetKey(key)
|
||||
logger.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())))
|
||||
a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())), logs.TagField(logs.TagApp))
|
||||
|
||||
for _, peer := range fetchPeers(logger, cfg) {
|
||||
for _, peer := range fetchPeers(a.log, a.config()) {
|
||||
prm.AddNode(peer)
|
||||
prmTree.AddNode(peer)
|
||||
}
|
||||
|
||||
connTimeout := fetchConnectTimeout(cfg)
|
||||
connTimeout := fetchConnectTimeout(a.config())
|
||||
prm.SetNodeDialTimeout(connTimeout)
|
||||
prmTree.SetNodeDialTimeout(connTimeout)
|
||||
|
||||
streamTimeout := fetchStreamTimeout(cfg)
|
||||
prm.SetNodeStreamTimeout(streamTimeout)
|
||||
prmTree.SetNodeStreamTimeout(streamTimeout)
|
||||
prm.SetNodeStreamTimeout(fetchStreamTimeout(a.config(), cfgStreamTimeout))
|
||||
prmTree.SetNodeStreamTimeout(fetchStreamTimeout(a.config(), cfgTreeStreamTimeout))
|
||||
|
||||
healthCheckTimeout := fetchHealthCheckTimeout(cfg)
|
||||
healthCheckTimeout := fetchHealthCheckTimeout(a.config())
|
||||
prm.SetHealthcheckTimeout(healthCheckTimeout)
|
||||
prmTree.SetHealthcheckTimeout(healthCheckTimeout)
|
||||
|
||||
rebalanceInterval := fetchRebalanceInterval(cfg)
|
||||
rebalanceInterval := fetchRebalanceInterval(a.config())
|
||||
prm.SetClientRebalanceInterval(rebalanceInterval)
|
||||
prmTree.SetClientRebalanceInterval(rebalanceInterval)
|
||||
|
||||
errorThreshold := fetchErrorThreshold(cfg)
|
||||
errorThreshold := fetchErrorThreshold(a.config())
|
||||
prm.SetErrorThreshold(errorThreshold)
|
||||
|
||||
prm.SetGracefulCloseOnSwitchTimeout(fetchSetGracefulCloseOnSwitchTimeout(cfg))
|
||||
prm.SetGracefulCloseOnSwitchTimeout(fetchSetGracefulCloseOnSwitchTimeout(a.config()))
|
||||
|
||||
prm.SetLogger(logger)
|
||||
prmTree.SetLogger(logger)
|
||||
prm.SetLogger(a.log.With(logs.TagField(logs.TagDatapath)))
|
||||
prmTree.SetLogger(a.log.With(logs.TagField(logs.TagDatapath)))
|
||||
|
||||
prmTree.SetMaxRequestAttempts(cfg.GetInt(cfgTreePoolMaxAttempts))
|
||||
prmTree.SetMaxRequestAttempts(a.config().GetInt(cfgTreePoolMaxAttempts))
|
||||
|
||||
interceptors := []grpc.DialOption{
|
||||
grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()),
|
||||
grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()),
|
||||
grpc.WithContextDialer(dialSource.GrpcContextDialer()),
|
||||
grpc.WithContextDialer(a.settings.dialerSource.GrpcContextDialer()),
|
||||
}
|
||||
prm.SetGRPCDialOptions(interceptors...)
|
||||
prmTree.SetGRPCDialOptions(interceptors...)
|
||||
|
||||
p, err := pool.NewPool(prm)
|
||||
if err != nil {
|
||||
logger.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err))
|
||||
a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
if err = p.Dial(ctx); err != nil {
|
||||
logger.Fatal(logs.FailedToDialConnectionPool, zap.Error(err))
|
||||
a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
if a.config().GetBool(cfgTreePoolNetmapSupport) {
|
||||
prmTree.SetNetMapInfoSource(frostfs.NewSource(frostfs.NewFrostFS(p, key), a.cache))
|
||||
}
|
||||
|
||||
treePool, err := treepool.NewPool(prmTree)
|
||||
if err != nil {
|
||||
logger.Fatal(logs.FailedToCreateTreePool, zap.Error(err))
|
||||
a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
if err = treePool.Dial(ctx); err != nil {
|
||||
logger.Fatal(logs.FailedToDialTreePool, zap.Error(err))
|
||||
a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
return p, treePool, key
|
||||
a.treePool = treePool
|
||||
a.pool = p
|
||||
a.key = key
|
||||
}
|
||||
|
||||
func remove(list []string, element string) []string {
|
||||
|
@ -848,6 +906,7 @@ func (a *App) Wait() {
|
|||
a.log.Info(logs.ApplicationStarted,
|
||||
zap.String("name", "frostfs-s3-gw"),
|
||||
zap.String("version", version.Version),
|
||||
logs.TagField(logs.TagApp),
|
||||
)
|
||||
|
||||
a.metrics.State().SetVersion(version.Version)
|
||||
|
@ -855,7 +914,7 @@ func (a *App) Wait() {
|
|||
|
||||
<-a.webDone // wait for web-server to be stopped
|
||||
|
||||
a.log.Info(logs.ApplicationFinished)
|
||||
a.log.Info(logs.ApplicationFinished, logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
func (a *App) setHealthStatus() {
|
||||
|
@ -890,10 +949,10 @@ func (a *App) Serve(ctx context.Context) {
|
|||
srv := new(http.Server)
|
||||
srv.Handler = chiRouter
|
||||
srv.ErrorLog = zap.NewStdLog(a.log)
|
||||
srv.ReadTimeout = a.cfg.GetDuration(cfgWebReadTimeout)
|
||||
srv.ReadHeaderTimeout = a.cfg.GetDuration(cfgWebReadHeaderTimeout)
|
||||
srv.WriteTimeout = a.cfg.GetDuration(cfgWebWriteTimeout)
|
||||
srv.IdleTimeout = a.cfg.GetDuration(cfgWebIdleTimeout)
|
||||
srv.ReadTimeout = a.config().GetDuration(cfgWebReadTimeout)
|
||||
srv.ReadHeaderTimeout = a.config().GetDuration(cfgWebReadHeaderTimeout)
|
||||
srv.WriteTimeout = a.config().GetDuration(cfgWebWriteTimeout)
|
||||
srv.IdleTimeout = a.config().GetDuration(cfgWebIdleTimeout)
|
||||
|
||||
a.startServices()
|
||||
|
||||
|
@ -901,11 +960,11 @@ func (a *App) Serve(ctx context.Context) {
|
|||
|
||||
for i := range servs {
|
||||
go func(i int) {
|
||||
a.log.Info(logs.StartingServer, zap.String("address", servs[i].Address()))
|
||||
a.log.Info(logs.StartingServer, zap.String("address", servs[i].Address()), logs.TagField(logs.TagApp))
|
||||
|
||||
if err := srv.Serve(servs[i].Listener()); err != nil && err != http.ErrServerClosed {
|
||||
a.metrics.MarkUnhealthy(servs[i].Address())
|
||||
a.log.Fatal(logs.ListenAndServe, zap.Error(err))
|
||||
a.log.Fatal(logs.ListenAndServe, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
@ -930,7 +989,7 @@ LOOP:
|
|||
ctx, cancel := shutdownContext()
|
||||
defer cancel()
|
||||
|
||||
a.log.Info(logs.StoppingServer, zap.Error(srv.Shutdown(ctx)))
|
||||
a.log.Info(logs.StoppingServer, zap.Error(srv.Shutdown(ctx)), logs.TagField(logs.TagApp))
|
||||
|
||||
a.metrics.Shutdown()
|
||||
a.stopServices()
|
||||
|
@ -944,23 +1003,23 @@ func shutdownContext() (context.Context, context.CancelFunc) {
|
|||
}
|
||||
|
||||
func (a *App) configReload(ctx context.Context) {
|
||||
a.log.Info(logs.SIGHUPConfigReloadStarted)
|
||||
a.log.Info(logs.SIGHUPConfigReloadStarted, logs.TagField(logs.TagApp))
|
||||
|
||||
if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) {
|
||||
a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed)
|
||||
if !a.config().IsSet(cmdConfig) && !a.config().IsSet(cmdConfigDir) {
|
||||
a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed, logs.TagField(logs.TagApp))
|
||||
return
|
||||
}
|
||||
if err := readInConfig(a.cfg); err != nil {
|
||||
a.log.Warn(logs.FailedToReloadConfig, zap.Error(err))
|
||||
if err := a.cfg.reload(); err != nil {
|
||||
a.log.Warn(logs.FailedToReloadConfig, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
return
|
||||
}
|
||||
|
||||
if err := a.bucketResolver.UpdateResolvers(a.getResolverOrder()); err != nil {
|
||||
a.log.Warn(logs.FailedToReloadResolvers, zap.Error(err))
|
||||
a.log.Warn(logs.FailedToReloadResolvers, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
if err := a.updateServers(); err != nil {
|
||||
a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err))
|
||||
a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
a.setRuntimeParameters()
|
||||
|
@ -970,41 +1029,45 @@ func (a *App) configReload(ctx context.Context) {
|
|||
|
||||
a.updateSettings()
|
||||
|
||||
a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled))
|
||||
a.metrics.SetEnabled(a.config().GetBool(cfgPrometheusEnabled))
|
||||
a.initTracing(ctx)
|
||||
a.setHealthStatus()
|
||||
|
||||
a.log.Info(logs.SIGHUPConfigReloadCompleted)
|
||||
a.log.Info(logs.SIGHUPConfigReloadCompleted, logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
func (a *App) updateSettings() {
|
||||
if lvl, err := getLogLevel(a.cfg); err != nil {
|
||||
a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err))
|
||||
if lvl, err := getLogLevel(a.config()); err != nil {
|
||||
a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
} else {
|
||||
a.settings.logLevel.SetLevel(lvl)
|
||||
}
|
||||
|
||||
if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.cfg, a.log)); err != nil {
|
||||
a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err))
|
||||
if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.config(), a.log)); err != nil {
|
||||
a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
a.settings.update(a.cfg, a.log)
|
||||
if err := a.settings.tagsConfig.update(a.config()); err != nil {
|
||||
a.log.Warn(logs.TagsLogConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
a.settings.update(a.config(), a.log)
|
||||
}
|
||||
|
||||
func (a *App) startServices() {
|
||||
a.services = a.services[:0]
|
||||
|
||||
pprofService := NewPprofService(a.cfg, a.log)
|
||||
pprofService := NewPprofService(a.config(), a.log)
|
||||
a.services = append(a.services, pprofService)
|
||||
go pprofService.Start()
|
||||
|
||||
prometheusService := NewPrometheusService(a.cfg, a.log, a.metrics.Handler())
|
||||
prometheusService := NewPrometheusService(a.config(), a.log, a.metrics.Handler())
|
||||
a.services = append(a.services, prometheusService)
|
||||
go prometheusService.Start()
|
||||
}
|
||||
|
||||
func (a *App) initServers(ctx context.Context) {
|
||||
serversInfo := fetchServers(a.cfg, a.log)
|
||||
serversInfo := fetchServers(a.config(), a.log)
|
||||
|
||||
a.servers = make([]Server, 0, len(serversInfo))
|
||||
for _, serverInfo := range serversInfo {
|
||||
|
@ -1016,22 +1079,22 @@ func (a *App) initServers(ctx context.Context) {
|
|||
if err != nil {
|
||||
a.unbindServers = append(a.unbindServers, serverInfo)
|
||||
a.metrics.MarkUnhealthy(serverInfo.Address)
|
||||
a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...)
|
||||
a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err), logs.TagField(logs.TagApp))...)
|
||||
continue
|
||||
}
|
||||
a.metrics.MarkHealthy(serverInfo.Address)
|
||||
|
||||
a.servers = append(a.servers, srv)
|
||||
a.log.Info(logs.AddServer, fields...)
|
||||
a.log.Info(logs.AddServer, append(fields, logs.TagField(logs.TagApp))...)
|
||||
}
|
||||
|
||||
if len(a.servers) == 0 {
|
||||
a.log.Fatal(logs.NoHealthyServers)
|
||||
a.log.Fatal(logs.NoHealthyServers, logs.TagField(logs.TagApp))
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) updateServers() error {
|
||||
serversInfo := fetchServers(a.cfg, a.log)
|
||||
serversInfo := fetchServers(a.config(), a.log)
|
||||
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
|
@ -1044,8 +1107,8 @@ func (a *App) updateServers() error {
|
|||
if err := ser.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
|
||||
return fmt.Errorf("failed to update tls certs: %w", err)
|
||||
}
|
||||
found = true
|
||||
}
|
||||
found = true
|
||||
} else if unbind := a.updateUnbindServerInfo(serverInfo); unbind {
|
||||
found = true
|
||||
}
|
||||
|
@ -1091,7 +1154,9 @@ func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CachesConfig {
|
|||
cacheCfg.AccessControl.Lifetime = fetchCacheLifetime(v, l, cfgAccessControlCacheLifetime, cacheCfg.AccessControl.Lifetime)
|
||||
cacheCfg.AccessControl.Size = fetchCacheSize(v, l, cfgAccessControlCacheSize, cacheCfg.AccessControl.Size)
|
||||
|
||||
cacheCfg.NetworkInfo.Lifetime = fetchCacheLifetime(v, l, cfgNetworkInfoCacheLifetime, cacheCfg.NetworkInfo.Lifetime)
|
||||
cacheCfg.Network.Lifetime = fetchCacheLifetime(v, l, cfgNetworkCacheLifetime, cacheCfg.Network.Lifetime)
|
||||
|
||||
cacheCfg.CIDCache = v.GetBool(cfgTreePoolNetmapSupport)
|
||||
|
||||
return cacheCfg
|
||||
}
|
||||
|
@ -1128,7 +1193,7 @@ func (a *App) initHandler() {
|
|||
|
||||
a.api, err = handler.New(a.log, a.obj, a.settings, a.policyStorage, a.frostfsid)
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.CouldNotInitializeAPIHandler, zap.Error(err))
|
||||
a.log.Fatal(logs.CouldNotInitializeAPIHandler, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1160,16 +1225,18 @@ func (a *App) getServers() []Server {
|
|||
func (a *App) setRuntimeParameters() {
|
||||
if len(os.Getenv("GOMEMLIMIT")) != 0 {
|
||||
// default limit < yaml limit < app env limit < GOMEMLIMIT
|
||||
a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT)
|
||||
a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT, logs.TagField(logs.TagApp))
|
||||
return
|
||||
}
|
||||
|
||||
softMemoryLimit := fetchSoftMemoryLimit(a.cfg)
|
||||
softMemoryLimit := fetchSoftMemoryLimit(a.config())
|
||||
previous := debug.SetMemoryLimit(softMemoryLimit)
|
||||
if softMemoryLimit != previous {
|
||||
a.log.Info(logs.RuntimeSoftMemoryLimitUpdated,
|
||||
zap.Int64("new_value", softMemoryLimit),
|
||||
zap.Int64("old_value", previous))
|
||||
zap.Int64("old_value", previous),
|
||||
logs.TagField(logs.TagApp),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1195,7 +1262,7 @@ func (a *App) tryReconnect(ctx context.Context, sr *http.Server) bool {
|
|||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
|
||||
a.log.Info(logs.ServerReconnecting)
|
||||
a.log.Info(logs.ServerReconnecting, logs.TagField(logs.TagApp))
|
||||
var failedServers []ServerInfo
|
||||
|
||||
for _, serverInfo := range a.unbindServers {
|
||||
|
@ -1206,23 +1273,23 @@ func (a *App) tryReconnect(ctx context.Context, sr *http.Server) bool {
|
|||
|
||||
srv, err := newServer(ctx, serverInfo)
|
||||
if err != nil {
|
||||
a.log.Warn(logs.ServerReconnectFailed, zap.Error(err))
|
||||
a.log.Warn(logs.ServerReconnectFailed, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
failedServers = append(failedServers, serverInfo)
|
||||
a.metrics.MarkUnhealthy(serverInfo.Address)
|
||||
continue
|
||||
}
|
||||
|
||||
go func() {
|
||||
a.log.Info(logs.StartingServer, zap.String("address", srv.Address()))
|
||||
a.log.Info(logs.StartingServer, zap.String("address", srv.Address()), logs.TagField(logs.TagApp))
|
||||
a.metrics.MarkHealthy(serverInfo.Address)
|
||||
if err = sr.Serve(srv.Listener()); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
a.log.Warn(logs.ListenAndServe, zap.Error(err))
|
||||
a.log.Warn(logs.ListenAndServe, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
a.metrics.MarkUnhealthy(serverInfo.Address)
|
||||
}
|
||||
}()
|
||||
|
||||
a.servers = append(a.servers, srv)
|
||||
a.log.Info(logs.ServerReconnectedSuccessfully, fields...)
|
||||
a.log.Info(logs.ServerReconnectedSuccessfully, append(fields, logs.TagField(logs.TagApp))...)
|
||||
}
|
||||
|
||||
a.unbindServers = failedServers
|
||||
|
@ -1240,7 +1307,7 @@ func (a *App) fetchContainerInfo(ctx context.Context, cfgKey string) (info *data
|
|||
}
|
||||
|
||||
func (a *App) resolveContainerID(ctx context.Context, cfgKey string) (cid.ID, error) {
|
||||
containerString := a.cfg.GetString(cfgKey)
|
||||
containerString := a.config().GetString(cfgKey)
|
||||
|
||||
var id cid.ID
|
||||
if err := id.DecodeString(containerString); err != nil {
|
||||
|
|
|
@ -10,6 +10,7 @@ import (
|
|||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
|
@ -20,20 +21,13 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
"git.frostfs.info/TrueCloudLab/zapjournald"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/ssgreg/journald"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
)
|
||||
|
||||
const (
|
||||
destinationStdout = "stdout"
|
||||
destinationJournald = "journald"
|
||||
|
||||
wildcardPlaceholder = "<wildcard>"
|
||||
)
|
||||
const wildcardPlaceholder = "<wildcard>"
|
||||
|
||||
const (
|
||||
defaultRebalanceInterval = 60 * time.Second
|
||||
|
@ -88,7 +82,8 @@ var (
|
|||
defaultDefaultNamespaces = []string{"", "root"}
|
||||
)
|
||||
|
||||
const ( // Settings.
|
||||
// Settings.
|
||||
const (
|
||||
// Logger.
|
||||
cfgLoggerLevel = "logger.level"
|
||||
cfgLoggerDestination = "logger.destination"
|
||||
|
@ -98,6 +93,11 @@ const ( // Settings.
|
|||
cfgLoggerSamplingThereafter = "logger.sampling.thereafter"
|
||||
cfgLoggerSamplingInterval = "logger.sampling.interval"
|
||||
|
||||
cfgLoggerTags = "logger.tags"
|
||||
cfgLoggerTagsPrefixTmpl = cfgLoggerTags + ".%d."
|
||||
cfgLoggerTagsNameTmpl = cfgLoggerTagsPrefixTmpl + "name"
|
||||
cfgLoggerTagsLevelTmpl = cfgLoggerTagsPrefixTmpl + "level"
|
||||
|
||||
// HttpLogging.
|
||||
cfgHTTPLoggingEnabled = "http_logging.enabled"
|
||||
cfgHTTPLoggingMaxBody = "http_logging.max_body"
|
||||
|
@ -122,6 +122,7 @@ const ( // Settings.
|
|||
// Pool config.
|
||||
cfgConnectTimeout = "connect_timeout"
|
||||
cfgStreamTimeout = "stream_timeout"
|
||||
cfgTreeStreamTimeout = "tree_stream_timeout"
|
||||
cfgHealthcheckTimeout = "healthcheck_timeout"
|
||||
cfgRebalanceInterval = "rebalance_interval"
|
||||
cfgPoolErrorThreshold = "pool_error_threshold"
|
||||
|
@ -147,7 +148,7 @@ const ( // Settings.
|
|||
cfgMorphPolicyCacheSize = "cache.morph_policy.size"
|
||||
cfgFrostfsIDCacheLifetime = "cache.frostfsid.lifetime"
|
||||
cfgFrostfsIDCacheSize = "cache.frostfsid.size"
|
||||
cfgNetworkInfoCacheLifetime = "cache.network_info.lifetime"
|
||||
cfgNetworkCacheLifetime = "cache.network_info.lifetime"
|
||||
|
||||
cfgAccessBoxCacheRemovingCheckInterval = "cache.accessbox.removing_check_interval"
|
||||
|
||||
|
@ -268,8 +269,9 @@ const ( // Settings.
|
|||
cfgSoftMemoryLimit = "runtime.soft_memory_limit"
|
||||
|
||||
// Enable return MD5 checksum in ETag.
|
||||
cfgMD5Enabled = "features.md5.enabled"
|
||||
cfgPolicyDenyByDefault = "features.policy.deny_by_default"
|
||||
cfgMD5Enabled = "features.md5.enabled"
|
||||
cfgPolicyDenyByDefault = "features.policy.deny_by_default"
|
||||
cfgTreePoolNetmapSupport = "features.tree_pool_netmap_support"
|
||||
|
||||
// FrostfsID.
|
||||
cfgFrostfsIDContract = "frostfsid.contract"
|
||||
|
@ -300,6 +302,49 @@ var ignore = map[string]struct{}{
|
|||
cmdVersion: {},
|
||||
}
|
||||
|
||||
type appCfg struct {
|
||||
flags *pflag.FlagSet
|
||||
|
||||
mu sync.RWMutex
|
||||
settings *viper.Viper
|
||||
}
|
||||
|
||||
func (a *appCfg) reload() error {
|
||||
old := a.config()
|
||||
|
||||
v, err := newViper(a.flags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if old.IsSet(cmdConfig) {
|
||||
v.Set(cmdConfig, old.Get(cmdConfig))
|
||||
}
|
||||
if old.IsSet(cmdConfigDir) {
|
||||
v.Set(cmdConfigDir, old.Get(cmdConfigDir))
|
||||
}
|
||||
|
||||
if err = readInConfig(v); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
a.setConfig(v)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *appCfg) config() *viper.Viper {
|
||||
a.mu.RLock()
|
||||
defer a.mu.RUnlock()
|
||||
|
||||
return a.settings
|
||||
}
|
||||
|
||||
func (a *appCfg) setConfig(v *viper.Viper) {
|
||||
a.mu.Lock()
|
||||
a.settings = v
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
func fetchConnectTimeout(cfg *viper.Viper) time.Duration {
|
||||
connTimeout := cfg.GetDuration(cfgConnectTimeout)
|
||||
if connTimeout <= 0 {
|
||||
|
@ -318,8 +363,8 @@ func fetchReconnectInterval(cfg *viper.Viper) time.Duration {
|
|||
return reconnect
|
||||
}
|
||||
|
||||
func fetchStreamTimeout(cfg *viper.Viper) time.Duration {
|
||||
streamTimeout := cfg.GetDuration(cfgStreamTimeout)
|
||||
func fetchStreamTimeout(cfg *viper.Viper, cfgEntry string) time.Duration {
|
||||
streamTimeout := cfg.GetDuration(cfgEntry)
|
||||
if streamTimeout <= 0 {
|
||||
streamTimeout = defaultStreamTimeout
|
||||
}
|
||||
|
@ -424,14 +469,18 @@ func fetchDefaultPolicy(l *zap.Logger, cfg *viper.Viper) netmap.PlacementPolicy
|
|||
policyStr := cfg.GetString(cfgPolicyDefault)
|
||||
if err := policy.DecodeString(policyStr); err != nil {
|
||||
l.Warn(logs.FailedToParseDefaultLocationConstraint,
|
||||
zap.String("policy", policyStr), zap.String("default", defaultPlacementPolicy), zap.Error(err))
|
||||
zap.String("policy", policyStr), zap.String("default", defaultPlacementPolicy),
|
||||
zap.Error(err), logs.TagField(logs.TagApp))
|
||||
} else {
|
||||
return policy
|
||||
}
|
||||
}
|
||||
|
||||
if err := policy.DecodeString(defaultPlacementPolicy); err != nil {
|
||||
l.Fatal(logs.FailedToParseDefaultDefaultLocationConstraint, zap.String("policy", defaultPlacementPolicy))
|
||||
l.Fatal(logs.FailedToParseDefaultDefaultLocationConstraint,
|
||||
zap.String("policy", defaultPlacementPolicy),
|
||||
logs.TagField(logs.TagApp),
|
||||
)
|
||||
}
|
||||
|
||||
return policy
|
||||
|
@ -444,7 +493,9 @@ func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultV
|
|||
l.Error(logs.InvalidLifetimeUsingDefaultValue,
|
||||
zap.String("parameter", cfgEntry),
|
||||
zap.Duration("value in config", lifetime),
|
||||
zap.Duration("default", defaultValue))
|
||||
zap.Duration("default", defaultValue),
|
||||
logs.TagField(logs.TagApp),
|
||||
)
|
||||
} else {
|
||||
return lifetime
|
||||
}
|
||||
|
@ -460,7 +511,9 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue
|
|||
l.Error(logs.InvalidCacheSizeUsingDefaultValue,
|
||||
zap.String("parameter", cfgEntry),
|
||||
zap.Int("value in config", size),
|
||||
zap.Int("default", defaultValue))
|
||||
zap.Int("default", defaultValue),
|
||||
logs.TagField(logs.TagApp),
|
||||
)
|
||||
} else {
|
||||
return size
|
||||
}
|
||||
|
@ -482,7 +535,8 @@ func fetchRemovingCheckInterval(v *viper.Viper, l *zap.Logger) time.Duration {
|
|||
l.Error(logs.InvalidAccessBoxCacheRemovingCheckInterval,
|
||||
zap.String("parameter", cfgAccessBoxCacheRemovingCheckInterval),
|
||||
zap.Duration("value in config", duration),
|
||||
zap.Duration("default", defaultAccessBoxCacheRemovingCheckInterval))
|
||||
zap.Duration("default", defaultAccessBoxCacheRemovingCheckInterval),
|
||||
logs.TagField(logs.TagApp))
|
||||
|
||||
return defaultAccessBoxCacheRemovingCheckInterval
|
||||
}
|
||||
|
@ -496,7 +550,9 @@ func fetchDefaultMaxAge(cfg *viper.Viper, l *zap.Logger) int {
|
|||
if defaultMaxAge <= 0 && defaultMaxAge != -1 {
|
||||
l.Fatal(logs.InvalidDefaultMaxAge,
|
||||
zap.String("parameter", cfgDefaultMaxAge),
|
||||
zap.String("value in config", strconv.Itoa(defaultMaxAge)))
|
||||
zap.String("value in config", strconv.Itoa(defaultMaxAge)),
|
||||
logs.TagField(logs.TagApp),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -507,14 +563,19 @@ func fetchRegionMappingPolicies(l *zap.Logger, cfg *viper.Viper) map[string]netm
|
|||
filepath := cfg.GetString(cfgPolicyRegionMapFile)
|
||||
regionPolicyMap, err := readRegionMap(filepath)
|
||||
if err != nil {
|
||||
l.Warn(logs.FailedToReadRegionMapFilePolicies, zap.String("file", filepath), zap.Error(err))
|
||||
l.Warn(logs.FailedToReadRegionMapFilePolicies,
|
||||
zap.String("file", filepath),
|
||||
zap.Error(err),
|
||||
logs.TagField(logs.TagApp))
|
||||
return make(map[string]netmap.PlacementPolicy)
|
||||
}
|
||||
|
||||
regionMap := make(map[string]netmap.PlacementPolicy, len(regionPolicyMap))
|
||||
for region, policy := range regionPolicyMap {
|
||||
if region == api.DefaultLocationConstraint {
|
||||
l.Warn(logs.DefaultLocationConstraintCantBeOverriden, zap.String("policy", policy))
|
||||
l.Warn(logs.DefaultLocationConstraintCantBeOverriden,
|
||||
zap.String("policy", policy),
|
||||
logs.TagField(logs.TagApp))
|
||||
continue
|
||||
}
|
||||
|
||||
|
@ -529,7 +590,10 @@ func fetchRegionMappingPolicies(l *zap.Logger, cfg *viper.Viper) map[string]netm
|
|||
continue
|
||||
}
|
||||
|
||||
l.Warn(logs.FailedToParseLocationConstraint, zap.String("region", region), zap.String("policy", policy))
|
||||
l.Warn(logs.FailedToParseLocationConstraint,
|
||||
zap.String("region", region),
|
||||
zap.String("policy", policy),
|
||||
logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
return regionMap
|
||||
|
@ -561,7 +625,11 @@ func fetchDefaultCopiesNumbers(l *zap.Logger, v *viper.Viper) []uint32 {
|
|||
parsedValue, err := strconv.ParseUint(unparsed[i], 10, 32)
|
||||
if err != nil {
|
||||
l.Warn(logs.FailedToParseDefaultCopiesNumbers,
|
||||
zap.Strings("copies numbers", unparsed), zap.Uint32s("default", defaultCopiesNumbers), zap.Error(err))
|
||||
zap.Strings("copies numbers", unparsed),
|
||||
zap.Uint32s("default", defaultCopiesNumbers),
|
||||
zap.Error(err),
|
||||
logs.TagField(logs.TagApp),
|
||||
)
|
||||
return defaultCopiesNumbers
|
||||
}
|
||||
result[i] = uint32(parsedValue)
|
||||
|
@ -617,15 +685,17 @@ func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) map[string][]uint32 {
|
|||
for j := range vector {
|
||||
parsedValue, err := strconv.ParseUint(vector[j], 10, 32)
|
||||
if err != nil {
|
||||
l.Warn(logs.FailedToParseCopiesNumbers, zap.String("location", constraint),
|
||||
zap.Strings("copies numbers", vector), zap.Error(err))
|
||||
l.Warn(logs.FailedToParseCopiesNumbers,
|
||||
zap.String("location", constraint),
|
||||
zap.Strings("copies numbers", vector), zap.Error(err),
|
||||
logs.TagField(logs.TagApp))
|
||||
continue
|
||||
}
|
||||
vector32[j] = uint32(parsedValue)
|
||||
}
|
||||
|
||||
copiesNums[constraint] = vector32
|
||||
l.Info(logs.ConstraintAdded, zap.String("location", constraint), zap.Strings("copies numbers", vector))
|
||||
l.Info(logs.ConstraintAdded, zap.String("location", constraint), zap.Strings("copies numbers", vector), logs.TagField(logs.TagApp))
|
||||
}
|
||||
return copiesNums
|
||||
}
|
||||
|
@ -634,7 +704,9 @@ func fetchDefaultNamespaces(l *zap.Logger, v *viper.Viper) []string {
|
|||
defaultNamespaces := v.GetStringSlice(cfgKludgeDefaultNamespaces)
|
||||
if len(defaultNamespaces) == 0 {
|
||||
defaultNamespaces = defaultDefaultNamespaces
|
||||
l.Warn(logs.DefaultNamespacesCannotBeEmpty, zap.Strings("namespaces", defaultNamespaces))
|
||||
l.Warn(logs.DefaultNamespacesCannotBeEmpty,
|
||||
zap.Strings("namespaces", defaultNamespaces),
|
||||
logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
for i := range defaultNamespaces { // to be set namespaces in env variable as `S3_GW_KLUDGE_DEFAULT_NAMESPACES="" 'root'`
|
||||
|
@ -658,7 +730,7 @@ func fetchNamespacesConfig(l *zap.Logger, v *viper.Viper) (NamespacesConfig, []s
|
|||
|
||||
nsConfig, err := readNamespacesConfig(v.GetString(cfgNamespacesConfig))
|
||||
if err != nil {
|
||||
l.Warn(logs.FailedToParseNamespacesConfig, zap.Error(err))
|
||||
l.Warn(logs.FailedToParseNamespacesConfig, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
defaultNamespacesNames := fetchDefaultNamespaces(l, v)
|
||||
|
@ -672,11 +744,13 @@ func fetchNamespacesConfig(l *zap.Logger, v *viper.Viper) (NamespacesConfig, []s
|
|||
}
|
||||
|
||||
if len(overrideDefaults) > 0 {
|
||||
l.Warn(logs.DefaultNamespaceConfigValuesBeOverwritten)
|
||||
l.Warn(logs.DefaultNamespaceConfigValuesBeOverwritten, logs.TagField(logs.TagApp))
|
||||
defaultNSValue.LocationConstraints = overrideDefaults[0].LocationConstraints
|
||||
defaultNSValue.CopiesNumbers = overrideDefaults[0].CopiesNumbers
|
||||
if len(overrideDefaults) > 1 {
|
||||
l.Warn(logs.MultipleDefaultOverridesFound, zap.String("name", overrideDefaults[0].Name))
|
||||
l.Warn(logs.MultipleDefaultOverridesFound,
|
||||
zap.String("name", overrideDefaults[0].Name),
|
||||
logs.TagField(logs.TagApp))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -719,7 +793,7 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
|
|||
priority := v.GetInt(key + "priority")
|
||||
|
||||
if address == "" {
|
||||
l.Warn(logs.SkipEmptyAddress)
|
||||
l.Warn(logs.SkipEmptyAddress, logs.TagField(logs.TagApp))
|
||||
break
|
||||
}
|
||||
if weight <= 0 { // unspecified or wrong
|
||||
|
@ -734,7 +808,9 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
|
|||
l.Info(logs.AddedStoragePeer,
|
||||
zap.Int("priority", priority),
|
||||
zap.String("address", address),
|
||||
zap.Float64("weight", weight))
|
||||
zap.Float64("weight", weight),
|
||||
logs.TagField(logs.TagApp),
|
||||
)
|
||||
}
|
||||
|
||||
return nodes
|
||||
|
@ -758,7 +834,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo {
|
|||
}
|
||||
|
||||
if _, ok := seen[serverInfo.Address]; ok {
|
||||
log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address))
|
||||
log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address), logs.TagField(logs.TagApp))
|
||||
continue
|
||||
}
|
||||
seen[serverInfo.Address] = struct{}{}
|
||||
|
@ -787,13 +863,13 @@ func fetchVHSNamespaces(v *viper.Viper, log *zap.Logger) map[string]bool {
|
|||
nsMap := v.GetStringMap(cfgVHSNamespaces)
|
||||
for ns, val := range nsMap {
|
||||
if _, ok := vhsNamespacesEnabled[ns]; ok {
|
||||
log.Warn(logs.WarnDuplicateNamespaceVHS, zap.String("namespace", ns))
|
||||
log.Warn(logs.WarnDuplicateNamespaceVHS, zap.String("namespace", ns), logs.TagField(logs.TagApp))
|
||||
continue
|
||||
}
|
||||
|
||||
enabledFlag, ok := val.(bool)
|
||||
if !ok {
|
||||
log.Warn(logs.WarnValueVHSEnabledFlagWrongType, zap.String("namespace", ns))
|
||||
log.Warn(logs.WarnValueVHSEnabledFlagWrongType, zap.String("namespace", ns), logs.TagField(logs.TagApp))
|
||||
continue
|
||||
}
|
||||
|
||||
|
@ -877,7 +953,42 @@ func fetchTombstoneWorkerPoolSize(v *viper.Viper) int {
|
|||
return tombstoneWorkerPoolSize
|
||||
}
|
||||
|
||||
func newSettings() *viper.Viper {
|
||||
func fetchLogTagsConfig(v *viper.Viper) (map[string]zapcore.Level, error) {
|
||||
res := make(map[string]zapcore.Level)
|
||||
|
||||
defaultLevel := v.GetString(cfgLoggerLevel)
|
||||
var defaultLvl zapcore.Level
|
||||
if err := defaultLvl.Set(defaultLevel); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse log level, unknown level: '%s'", defaultLevel)
|
||||
}
|
||||
|
||||
for i := 0; ; i++ {
|
||||
name := v.GetString(fmt.Sprintf(cfgLoggerTagsNameTmpl, i))
|
||||
if name == "" {
|
||||
break
|
||||
}
|
||||
|
||||
lvl := defaultLvl
|
||||
level := v.GetString(fmt.Sprintf(cfgLoggerTagsLevelTmpl, i))
|
||||
if level != "" {
|
||||
if err := lvl.Set(level); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse log tags config, unknown level: '%s'", level)
|
||||
}
|
||||
}
|
||||
|
||||
res[name] = lvl
|
||||
}
|
||||
|
||||
if len(res) == 0 && !v.IsSet(cfgLoggerTags) {
|
||||
for _, tag := range defaultTags {
|
||||
res[tag] = defaultLvl
|
||||
}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func newViper(flags *pflag.FlagSet) (*viper.Viper, error) {
|
||||
v := viper.New()
|
||||
|
||||
v.AutomaticEnv()
|
||||
|
@ -886,6 +997,20 @@ func newSettings() *viper.Viper {
|
|||
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
|
||||
v.AllowEmptyEnv(true)
|
||||
|
||||
if err := bindFlags(v, flags); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
setDefaults(v, flags)
|
||||
|
||||
if v.IsSet(cfgServer+".0."+cfgTLSKeyFile) && v.IsSet(cfgServer+".0."+cfgTLSCertFile) {
|
||||
v.Set(cfgServer+".0."+cfgTLSEnabled, true)
|
||||
}
|
||||
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func newSettings() *appCfg {
|
||||
// flags setup:
|
||||
flags := pflag.NewFlagSet("commandline", pflag.ExitOnError)
|
||||
flags.SetOutput(os.Stdout)
|
||||
|
@ -913,15 +1038,71 @@ func newSettings() *viper.Viper {
|
|||
flags.String(cfgTLSCertFile, "", "TLS certificate file to use")
|
||||
flags.String(cfgTLSKeyFile, "", "TLS key file to use")
|
||||
|
||||
peers := flags.StringArrayP(cfgPeers, "p", nil, "set FrostFS nodes")
|
||||
flags.StringArrayP(cfgPeers, "p", nil, "set FrostFS nodes")
|
||||
|
||||
flags.StringP(cfgRPCEndpoint, "r", "", "set RPC endpoint")
|
||||
resolveMethods := flags.StringSlice(cfgResolveOrder, []string{resolver.DNSResolver}, "set bucket name resolve order")
|
||||
flags.StringSlice(cfgResolveOrder, []string{resolver.DNSResolver}, "set bucket name resolve order")
|
||||
|
||||
domains := flags.StringSliceP(cfgListenDomains, "d", nil, "set domains to be listened")
|
||||
flags.StringSliceP(cfgListenDomains, "d", nil, "set domains to be listened")
|
||||
|
||||
// set defaults:
|
||||
if err := flags.Parse(os.Args); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
v, err := newViper(flags)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("bind flags: %w", err))
|
||||
}
|
||||
|
||||
switch {
|
||||
case help != nil && *help:
|
||||
fmt.Printf("FrostFS S3 gateway %s\n", version.Version)
|
||||
flags.PrintDefaults()
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("Default environments:")
|
||||
fmt.Println()
|
||||
keys := v.AllKeys()
|
||||
sort.Strings(keys)
|
||||
|
||||
for i := range keys {
|
||||
if _, ok := ignore[keys[i]]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
defaultValue := v.GetString(keys[i])
|
||||
if len(defaultValue) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
k := strings.Replace(keys[i], ".", "_", -1)
|
||||
fmt.Printf("%s_%s = %s\n", envPrefix, strings.ToUpper(k), defaultValue)
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("Peers preset:")
|
||||
fmt.Println()
|
||||
|
||||
fmt.Printf("%s_%s_[N]_ADDRESS = string\n", envPrefix, strings.ToUpper(cfgPeers))
|
||||
fmt.Printf("%s_%s_[N]_WEIGHT = 0..1 (float)\n", envPrefix, strings.ToUpper(cfgPeers))
|
||||
|
||||
os.Exit(0)
|
||||
case versionFlag != nil && *versionFlag:
|
||||
fmt.Printf("FrostFS S3 Gateway\nVersion: %s\nGoVersion: %s\n", version.Version, runtime.Version())
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
if err = readInConfig(v); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return &appCfg{
|
||||
flags: flags,
|
||||
settings: v,
|
||||
}
|
||||
}
|
||||
|
||||
func setDefaults(v *viper.Viper, flags *pflag.FlagSet) {
|
||||
v.SetDefault(cfgAccessBoxCacheRemovingCheckInterval, defaultAccessBoxCacheRemovingCheckInterval)
|
||||
|
||||
// logger:
|
||||
|
@ -985,78 +1166,21 @@ func newSettings() *viper.Viper {
|
|||
// multinet
|
||||
v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay)
|
||||
|
||||
// Bind flags
|
||||
if err := bindFlags(v, flags); err != nil {
|
||||
panic(fmt.Errorf("bind flags: %w", err))
|
||||
if resolveMethods, err := flags.GetStringSlice(cfgResolveOrder); err == nil {
|
||||
v.SetDefault(cfgResolveOrder, resolveMethods)
|
||||
}
|
||||
|
||||
if err := flags.Parse(os.Args); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if v.IsSet(cfgServer+".0."+cfgTLSKeyFile) && v.IsSet(cfgServer+".0."+cfgTLSCertFile) {
|
||||
v.Set(cfgServer+".0."+cfgTLSEnabled, true)
|
||||
}
|
||||
|
||||
if resolveMethods != nil {
|
||||
v.SetDefault(cfgResolveOrder, *resolveMethods)
|
||||
}
|
||||
|
||||
if peers != nil && len(*peers) > 0 {
|
||||
for i := range *peers {
|
||||
v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".address", (*peers)[i])
|
||||
if peers, err := flags.GetStringArray(cfgPeers); err == nil {
|
||||
for i := range peers {
|
||||
v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".address", peers[i])
|
||||
v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".weight", 1)
|
||||
v.SetDefault(cfgPeers+"."+strconv.Itoa(i)+".priority", 1)
|
||||
}
|
||||
}
|
||||
|
||||
if domains != nil && len(*domains) > 0 {
|
||||
v.SetDefault(cfgListenDomains, *domains)
|
||||
if domains, err := flags.GetStringSlice(cfgListenDomains); err == nil && len(domains) > 0 {
|
||||
v.SetDefault(cfgListenDomains, domains)
|
||||
}
|
||||
|
||||
switch {
|
||||
case help != nil && *help:
|
||||
fmt.Printf("FrostFS S3 gateway %s\n", version.Version)
|
||||
flags.PrintDefaults()
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("Default environments:")
|
||||
fmt.Println()
|
||||
keys := v.AllKeys()
|
||||
sort.Strings(keys)
|
||||
|
||||
for i := range keys {
|
||||
if _, ok := ignore[keys[i]]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
defaultValue := v.GetString(keys[i])
|
||||
if len(defaultValue) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
k := strings.Replace(keys[i], ".", "_", -1)
|
||||
fmt.Printf("%s_%s = %s\n", envPrefix, strings.ToUpper(k), defaultValue)
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("Peers preset:")
|
||||
fmt.Println()
|
||||
|
||||
fmt.Printf("%s_%s_[N]_ADDRESS = string\n", envPrefix, strings.ToUpper(cfgPeers))
|
||||
fmt.Printf("%s_%s_[N]_WEIGHT = 0..1 (float)\n", envPrefix, strings.ToUpper(cfgPeers))
|
||||
|
||||
os.Exit(0)
|
||||
case versionFlag != nil && *versionFlag:
|
||||
fmt.Printf("FrostFS S3 Gateway\nVersion: %s\nGoVersion: %s\n", version.Version, runtime.Version())
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
if err := readInConfig(v); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return v
|
||||
}
|
||||
|
||||
func bindFlags(v *viper.Viper, flags *pflag.FlagSet) error {
|
||||
|
@ -1174,129 +1298,19 @@ type LoggerAppSettings interface {
|
|||
DroppedLogsInc()
|
||||
}
|
||||
|
||||
func pickLogger(v *viper.Viper, settings LoggerAppSettings) *Logger {
|
||||
lvl, err := getLogLevel(v)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
dest := v.GetString(cfgLoggerDestination)
|
||||
|
||||
switch dest {
|
||||
case destinationStdout:
|
||||
return newStdoutLogger(v, lvl, settings)
|
||||
case destinationJournald:
|
||||
return newJournaldLogger(v, lvl, settings)
|
||||
default:
|
||||
panic(fmt.Sprintf("wrong destination for logger: %s", dest))
|
||||
}
|
||||
}
|
||||
|
||||
// newStdoutLogger constructs a Logger instance for the current application.
|
||||
// Panics on failure.
|
||||
//
|
||||
// Logger contains a logger is built from zap's production logging configuration with:
|
||||
// - parameterized level (debug by default)
|
||||
// - console encoding
|
||||
// - ISO8601 time encoding
|
||||
// - sampling intervals
|
||||
//
|
||||
// and atomic log level to dynamically change it.
|
||||
//
|
||||
// Logger records a stack trace for all messages at or above fatal level.
|
||||
//
|
||||
// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace.
|
||||
func newStdoutLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger {
|
||||
stdout := zapcore.AddSync(os.Stderr)
|
||||
level := zap.NewAtomicLevelAt(lvl)
|
||||
|
||||
consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level)
|
||||
|
||||
consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, settings)
|
||||
|
||||
return &Logger{
|
||||
logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))),
|
||||
lvl: level,
|
||||
}
|
||||
}
|
||||
|
||||
func newJournaldLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger {
|
||||
level := zap.NewAtomicLevelAt(lvl)
|
||||
|
||||
encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields)
|
||||
|
||||
core := zapjournald.NewCore(level, encoder, &journald.Journal{}, zapjournald.SyslogFields)
|
||||
coreWithContext := core.With([]zapcore.Field{
|
||||
zapjournald.SyslogFacility(zapjournald.LogDaemon),
|
||||
zapjournald.SyslogIdentifier(),
|
||||
zapjournald.SyslogPid(),
|
||||
})
|
||||
|
||||
coreWithContext = applyZapCoreMiddlewares(coreWithContext, v, settings)
|
||||
|
||||
l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
|
||||
|
||||
return &Logger{
|
||||
logger: l,
|
||||
lvl: level,
|
||||
}
|
||||
}
|
||||
|
||||
func newLogEncoder() zapcore.Encoder {
|
||||
c := zap.NewProductionEncoderConfig()
|
||||
c.EncodeTime = zapcore.ISO8601TimeEncoder
|
||||
|
||||
return zapcore.NewConsoleEncoder(c)
|
||||
}
|
||||
|
||||
func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, settings LoggerAppSettings) zapcore.Core {
|
||||
if v.GetBool(cfgLoggerSamplingEnabled) {
|
||||
core = zapcore.NewSamplerWithOptions(core,
|
||||
v.GetDuration(cfgLoggerSamplingInterval),
|
||||
v.GetInt(cfgLoggerSamplingInitial),
|
||||
v.GetInt(cfgLoggerSamplingThereafter),
|
||||
zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) {
|
||||
if dec&zapcore.LogDropped > 0 {
|
||||
settings.DroppedLogsInc()
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
return core
|
||||
}
|
||||
|
||||
func getLogLevel(v *viper.Viper) (zapcore.Level, error) {
|
||||
var lvl zapcore.Level
|
||||
lvlStr := v.GetString(cfgLoggerLevel)
|
||||
err := lvl.UnmarshalText([]byte(lvlStr))
|
||||
if err != nil {
|
||||
return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+
|
||||
"value should be one of %v", lvlStr, err, [...]zapcore.Level{
|
||||
zapcore.DebugLevel,
|
||||
zapcore.InfoLevel,
|
||||
zapcore.WarnLevel,
|
||||
zapcore.ErrorLevel,
|
||||
zapcore.DPanicLevel,
|
||||
zapcore.PanicLevel,
|
||||
zapcore.FatalLevel,
|
||||
})
|
||||
}
|
||||
return lvl, nil
|
||||
}
|
||||
|
||||
func validateDomains(domains []string, log *zap.Logger) []string {
|
||||
validDomains := make([]string, 0, len(domains))
|
||||
LOOP:
|
||||
for _, domain := range domains {
|
||||
if strings.Contains(domain, ":") {
|
||||
log.Warn(logs.WarnDomainContainsPort, zap.String("domain", domain))
|
||||
log.Warn(logs.WarnDomainContainsPort, zap.String("domain", domain), logs.TagField(logs.TagApp))
|
||||
continue
|
||||
}
|
||||
|
||||
domainParts := strings.Split(domain, ".")
|
||||
for _, part := range domainParts {
|
||||
if strings.ContainsAny(part, "<>") && part != wildcardPlaceholder {
|
||||
log.Warn(logs.WarnDomainContainsInvalidPlaceholder, zap.String("domain", domain))
|
||||
log.Warn(logs.WarnDomainContainsInvalidPlaceholder, zap.String("domain", domain), logs.TagField(logs.TagApp))
|
||||
continue LOOP
|
||||
}
|
||||
}
|
||||
|
|
59
cmd/s3-gw/app_settings_test.go
Normal file
59
cmd/s3-gw/app_settings_test.go
Normal file
|
@ -0,0 +1,59 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestConfigReload(t *testing.T) {
|
||||
f, err := os.CreateTemp("", "conf")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
require.NoError(t, os.Remove(f.Name()))
|
||||
}()
|
||||
|
||||
confData := `
|
||||
pprof:
|
||||
enabled: true
|
||||
|
||||
frostfsid:
|
||||
contract: name.nns
|
||||
|
||||
resolve_order:
|
||||
- nns
|
||||
`
|
||||
|
||||
_, err = f.WriteString(confData)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
cfg := newSettings()
|
||||
|
||||
require.NoError(t, cfg.flags.Parse([]string{"--config", f.Name(), "--max_clients_count", "10"}))
|
||||
require.NoError(t, cfg.reload())
|
||||
|
||||
require.True(t, cfg.config().GetBool(cfgPProfEnabled))
|
||||
require.Equal(t, "name.nns", cfg.config().GetString(cfgFrostfsIDContract))
|
||||
require.Equal(t, []string{resolver.NNSResolver}, cfg.config().GetStringSlice(cfgResolveOrder))
|
||||
require.Equal(t, 10, cfg.config().GetInt(cfgMaxClientsCount))
|
||||
|
||||
require.NoError(t, os.Truncate(f.Name(), 0))
|
||||
require.NoError(t, cfg.reload())
|
||||
|
||||
require.False(t, cfg.config().GetBool(cfgPProfEnabled))
|
||||
require.Equal(t, "frostfsid.frostfs", cfg.config().GetString(cfgFrostfsIDContract))
|
||||
require.Equal(t, []string{resolver.DNSResolver}, cfg.config().GetStringSlice(cfgResolveOrder))
|
||||
require.Equal(t, 10, cfg.config().GetInt(cfgMaxClientsCount))
|
||||
}
|
||||
|
||||
func TestSetTLSEnabled(t *testing.T) {
|
||||
cfg := newSettings()
|
||||
|
||||
require.NoError(t, cfg.flags.Parse([]string{"--" + cfgTLSCertFile, "tls.crt", "--" + cfgTLSKeyFile, "tls.key"}))
|
||||
require.NoError(t, cfg.reload())
|
||||
|
||||
require.True(t, cfg.config().GetBool(cfgServer+".0."+cfgTLSEnabled))
|
||||
}
|
210
cmd/s3-gw/logger.go
Normal file
210
cmd/s3-gw/logger.go
Normal file
|
@ -0,0 +1,210 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/zapjournald"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/ssgreg/journald"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
)
|
||||
|
||||
const (
|
||||
destinationStdout string = "stdout"
|
||||
destinationJournald string = "journald"
|
||||
)
|
||||
|
||||
var defaultTags = []string{logs.TagApp, logs.TagDatapath, logs.TagExternalStorage, logs.TagExternalStorageTree, logs.TagExternalBlockchain}
|
||||
|
||||
type Logger struct {
|
||||
logger *zap.Logger
|
||||
lvl zap.AtomicLevel
|
||||
}
|
||||
|
||||
func pickLogger(v *viper.Viper, loggerSettings LoggerAppSettings, tagSettings TagFilterSettings) *Logger {
|
||||
dest := v.GetString(cfgLoggerDestination)
|
||||
|
||||
switch dest {
|
||||
case destinationStdout:
|
||||
return newStdoutLogger(v, loggerSettings, tagSettings)
|
||||
case destinationJournald:
|
||||
return newJournaldLogger(v, loggerSettings, tagSettings)
|
||||
default:
|
||||
panic(fmt.Sprintf("wrong destination for logger: %s", dest))
|
||||
}
|
||||
}
|
||||
|
||||
// newStdoutLogger constructs a Logger instance for the current application.
|
||||
// Panics on failure.
|
||||
//
|
||||
// Logger contains a logger is built from zap's production logging configuration with:
|
||||
// - parameterized level (debug by default)
|
||||
// - console encoding
|
||||
// - ISO8601 time encoding
|
||||
// - sampling intervals
|
||||
//
|
||||
// and atomic log level to dynamically change it.
|
||||
//
|
||||
// Logger records a stack trace for all messages at or above fatal level.
|
||||
//
|
||||
// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace.
|
||||
func newStdoutLogger(v *viper.Viper, loggerSettings LoggerAppSettings, tagSettings TagFilterSettings) *Logger {
|
||||
c := newZapLogConfig(v)
|
||||
|
||||
out, errSink, err := openZapSinks(c)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("open zap sinks: %v", err.Error()))
|
||||
}
|
||||
|
||||
core := zapcore.NewCore(zapcore.NewConsoleEncoder(c.EncoderConfig), out, c.Level)
|
||||
core = applyZapCoreMiddlewares(core, v, loggerSettings, tagSettings)
|
||||
l := zap.New(core, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), zap.ErrorOutput(errSink))
|
||||
|
||||
return &Logger{logger: l, lvl: c.Level}
|
||||
}
|
||||
|
||||
func newJournaldLogger(v *viper.Viper, logSettings LoggerAppSettings, tagSettings TagFilterSettings) *Logger {
|
||||
c := newZapLogConfig(v)
|
||||
|
||||
// We can use NewJSONEncoder instead if, say, frontend
|
||||
// would like to access journald logs and parse them easily.
|
||||
encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields)
|
||||
|
||||
journalCore := zapjournald.NewCore(c.Level, encoder, &journald.Journal{}, zapjournald.SyslogFields)
|
||||
core := journalCore.With([]zapcore.Field{
|
||||
zapjournald.SyslogFacility(zapjournald.LogDaemon),
|
||||
zapjournald.SyslogIdentifier(),
|
||||
zapjournald.SyslogPid(),
|
||||
})
|
||||
core = applyZapCoreMiddlewares(core, v, logSettings, tagSettings)
|
||||
l := zap.New(core, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
|
||||
return &Logger{logger: l, lvl: c.Level}
|
||||
}
|
||||
|
||||
func openZapSinks(cfg zap.Config) (zapcore.WriteSyncer, zapcore.WriteSyncer, error) {
|
||||
sink, closeOut, err := zap.Open(cfg.OutputPaths...)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
errSink, _, err := zap.Open(cfg.ErrorOutputPaths...)
|
||||
if err != nil {
|
||||
closeOut()
|
||||
return nil, nil, err
|
||||
}
|
||||
return sink, errSink, nil
|
||||
}
|
||||
|
||||
var _ zapcore.Core = (*zapCoreTagFilterWrapper)(nil)
|
||||
|
||||
type zapCoreTagFilterWrapper struct {
|
||||
core zapcore.Core
|
||||
settings TagFilterSettings
|
||||
extra []zap.Field
|
||||
}
|
||||
|
||||
type TagFilterSettings interface {
|
||||
LevelEnabled(tag string, lvl zapcore.Level) bool
|
||||
}
|
||||
|
||||
func (c *zapCoreTagFilterWrapper) Enabled(level zapcore.Level) bool {
|
||||
return c.core.Enabled(level)
|
||||
}
|
||||
|
||||
func (c *zapCoreTagFilterWrapper) With(fields []zapcore.Field) zapcore.Core {
|
||||
return &zapCoreTagFilterWrapper{
|
||||
core: c.core.With(fields),
|
||||
settings: c.settings,
|
||||
extra: append(c.extra, fields...),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *zapCoreTagFilterWrapper) Check(entry zapcore.Entry, checked *zapcore.CheckedEntry) *zapcore.CheckedEntry {
|
||||
if c.core.Enabled(entry.Level) {
|
||||
return checked.AddCore(entry, c)
|
||||
}
|
||||
return checked
|
||||
}
|
||||
|
||||
func (c *zapCoreTagFilterWrapper) Write(entry zapcore.Entry, fields []zapcore.Field) error {
|
||||
if c.shouldSkip(entry, fields) || c.shouldSkip(entry, c.extra) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return c.core.Write(entry, fields)
|
||||
}
|
||||
|
||||
func (c *zapCoreTagFilterWrapper) shouldSkip(entry zapcore.Entry, fields []zap.Field) bool {
|
||||
for _, field := range fields {
|
||||
if field.Key == logs.TagFieldName && field.Type == zapcore.StringType {
|
||||
if !c.settings.LevelEnabled(field.String, entry.Level) {
|
||||
return true
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *zapCoreTagFilterWrapper) Sync() error {
|
||||
return c.core.Sync()
|
||||
}
|
||||
|
||||
func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, appSettings LoggerAppSettings, tagSettings TagFilterSettings) zapcore.Core {
|
||||
core = &zapCoreTagFilterWrapper{
|
||||
core: core,
|
||||
settings: tagSettings,
|
||||
}
|
||||
|
||||
if v.GetBool(cfgLoggerSamplingEnabled) {
|
||||
core = zapcore.NewSamplerWithOptions(core,
|
||||
v.GetDuration(cfgLoggerSamplingInterval),
|
||||
v.GetInt(cfgLoggerSamplingInitial),
|
||||
v.GetInt(cfgLoggerSamplingThereafter),
|
||||
zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) {
|
||||
if dec&zapcore.LogDropped > 0 {
|
||||
appSettings.DroppedLogsInc()
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
return core
|
||||
}
|
||||
|
||||
func newZapLogConfig(v *viper.Viper) zap.Config {
|
||||
lvl, err := getLogLevel(v)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
c := zap.Config{
|
||||
Level: zap.NewAtomicLevelAt(lvl),
|
||||
EncoderConfig: zap.NewProductionEncoderConfig(),
|
||||
OutputPaths: []string{"stderr"},
|
||||
ErrorOutputPaths: []string{"stderr"},
|
||||
}
|
||||
c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func getLogLevel(v *viper.Viper) (zapcore.Level, error) {
|
||||
var lvl zapcore.Level
|
||||
lvlStr := v.GetString(cfgLoggerLevel)
|
||||
err := lvl.UnmarshalText([]byte(lvlStr))
|
||||
if err != nil {
|
||||
return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+
|
||||
"value should be one of %v", lvlStr, err, [...]zapcore.Level{
|
||||
zapcore.DebugLevel,
|
||||
zapcore.InfoLevel,
|
||||
zapcore.WarnLevel,
|
||||
zapcore.ErrorLevel,
|
||||
zapcore.DPanicLevel,
|
||||
zapcore.PanicLevel,
|
||||
zapcore.FatalLevel,
|
||||
})
|
||||
}
|
||||
return lvl, nil
|
||||
}
|
|
@ -8,9 +8,9 @@ import (
|
|||
|
||||
func main() {
|
||||
g, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
v := newSettings()
|
||||
cfg := newSettings()
|
||||
|
||||
a := newApp(g, v)
|
||||
a := newApp(g, cfg)
|
||||
|
||||
go a.Serve(g)
|
||||
|
||||
|
|
|
@ -19,24 +19,24 @@ type Service struct {
|
|||
// Start runs http service with the exposed endpoint on the configured port.
|
||||
func (ms *Service) Start() {
|
||||
if ms.enabled {
|
||||
ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr))
|
||||
ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr), logs.TagField(logs.TagApp))
|
||||
err := ms.ListenAndServe()
|
||||
if err != nil && err != http.ErrServerClosed {
|
||||
ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort)
|
||||
ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort, logs.TagField(logs.TagApp))
|
||||
}
|
||||
} else {
|
||||
ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled)
|
||||
ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled, logs.TagField(logs.TagApp))
|
||||
}
|
||||
}
|
||||
|
||||
// ShutDown stops the service.
|
||||
func (ms *Service) ShutDown(ctx context.Context) {
|
||||
ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr))
|
||||
ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr), logs.TagField(logs.TagApp))
|
||||
err := ms.Shutdown(ctx)
|
||||
if err != nil {
|
||||
ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err))
|
||||
ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
if err = ms.Close(); err != nil {
|
||||
ms.log.Panic(logs.CantShutDownService, zap.Error(err))
|
||||
ms.log.Panic(logs.CantShutDownService, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -55,6 +55,11 @@ S3_GW_LOGGER_SAMPLING_ENABLED=false
|
|||
S3_GW_LOGGER_SAMPLING_INITIAL=100
|
||||
S3_GW_LOGGER_SAMPLING_THEREAFTER=100
|
||||
S3_GW_LOGGER_SAMPLING_INTERVAL=1s
|
||||
S3_GW_LOGGER_TAGS_0_NAME=app
|
||||
S3_GW_LOGGER_TAGS_0_LEVEL=info
|
||||
S3_GW_LOGGER_TAGS_1_NAME=datapath
|
||||
S3_GW_LOGGER_TAGS_1_LEVEL=fatal
|
||||
|
||||
|
||||
# HTTP logger
|
||||
S3_GW_HTTP_LOGGING_ENABLED=false
|
||||
|
@ -80,8 +85,10 @@ S3_GW_PROMETHEUS_ADDRESS=localhost:8086
|
|||
|
||||
# Timeout to connect to a node
|
||||
S3_GW_CONNECT_TIMEOUT=10s
|
||||
# Timeout for individual operations in streaming RPC.
|
||||
# Timeout for individual operations in object pool streaming RPC.
|
||||
S3_GW_STREAM_TIMEOUT=10s
|
||||
# Timeout for individual operations in tree pool streaming RPC.
|
||||
S3_GW_TREE_STREAM_TIMEOUT=10s
|
||||
# Timeout to check node health during rebalance.
|
||||
S3_GW_HEALTHCHECK_TIMEOUT=15s
|
||||
# Interval to check node health
|
||||
|
@ -126,7 +133,7 @@ S3_GW_CACHE_MORPH_POLICY_SIZE=10000
|
|||
# Cache which stores frostfsid subject info
|
||||
S3_GW_CACHE_FROSTFSID_LIFETIME=1m
|
||||
S3_GW_CACHE_FROSTFSID_SIZE=10000
|
||||
# Cache which stores network info
|
||||
# Cache which stores network-related values
|
||||
S3_GW_CACHE_NETWORK_INFO_LIFETIME=1m
|
||||
|
||||
# Default policy of placing containers in FrostFS
|
||||
|
@ -205,6 +212,8 @@ S3_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824
|
|||
S3_GW_FEATURES_MD5_ENABLED=false
|
||||
# Enable denying access for request that doesn't match any policy chain rules.
|
||||
S3_GW_FEATURES_POLICY_DENY_BY_DEFAULT=false
|
||||
# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service
|
||||
S3_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true
|
||||
|
||||
# ReadTimeout is the maximum duration for reading the entire
|
||||
# request, including the body. A zero or negative value means
|
||||
|
|
|
@ -60,6 +60,12 @@ logger:
|
|||
initial: 100
|
||||
thereafter: 100
|
||||
interval: 1s
|
||||
tags:
|
||||
- name: "app"
|
||||
level: "debug"
|
||||
- name: "datapath"
|
||||
- name: "external_storage"
|
||||
- name: "external_storage_tree"
|
||||
|
||||
# log http request data (URI, headers, query, etc)
|
||||
http_logging:
|
||||
|
@ -100,8 +106,10 @@ tracing:
|
|||
|
||||
# Timeout to connect to a node
|
||||
connect_timeout: 10s
|
||||
# Timeout for individual operations in streaming RPC.
|
||||
# Timeout for individual operations in object pool streaming RPC.
|
||||
stream_timeout: 10s
|
||||
# Timeout for individual operations in tree pool streaming RPC.
|
||||
tree_stream_timeout: 10s
|
||||
# Timeout to check node health during rebalance
|
||||
healthcheck_timeout: 15s
|
||||
# Interval to check node health
|
||||
|
@ -158,7 +166,7 @@ cache:
|
|||
frostfsid:
|
||||
lifetime: 1m
|
||||
size: 10000
|
||||
# Cache which stores network info
|
||||
# Cache which stores network-related values
|
||||
network_info:
|
||||
lifetime: 1m
|
||||
|
||||
|
@ -243,6 +251,8 @@ features:
|
|||
deny_by_default: false
|
||||
md5:
|
||||
enabled: false
|
||||
# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service
|
||||
tree_pool_netmap_support: true
|
||||
|
||||
web:
|
||||
# ReadTimeout is the maximum duration for reading the entire
|
||||
|
|
|
@ -19,6 +19,11 @@ import (
|
|||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
const (
|
||||
secretLength = 32
|
||||
saltLength = 16
|
||||
)
|
||||
|
||||
// Box represents friendly AccessBox.
|
||||
type Box struct {
|
||||
Gate *GateData
|
||||
|
@ -109,7 +114,7 @@ func PackTokens(gatesData []*GateData, secret []byte, isCustomSecret bool) (*Acc
|
|||
box.IsCustom = isCustomSecret
|
||||
|
||||
if secret == nil {
|
||||
secret, err = generateSecret()
|
||||
secret, err = generateRandomBytes(secretLength)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to generate accessKey as hex: %w", err)
|
||||
}
|
||||
|
@ -212,7 +217,12 @@ func encodeGate(ephemeralKey *keys.PrivateKey, seedKey *keys.PublicKey, tokens *
|
|||
return nil, fmt.Errorf("encode tokens: %w", err)
|
||||
}
|
||||
|
||||
encrypted, err := encrypt(ephemeralKey, seedKey, data)
|
||||
salt, err := generateRandomBytes(saltLength)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate salt for encryption key: %w", err)
|
||||
}
|
||||
|
||||
encrypted, err := encrypt(ephemeralKey, seedKey, data, salt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ecrypt tokens: %w", err)
|
||||
}
|
||||
|
@ -220,11 +230,12 @@ func encodeGate(ephemeralKey *keys.PrivateKey, seedKey *keys.PublicKey, tokens *
|
|||
gate := new(AccessBox_Gate)
|
||||
gate.GatePublicKey = seedKey.Bytes()
|
||||
gate.Tokens = encrypted
|
||||
gate.EncryptionKeySalt = salt
|
||||
return gate, nil
|
||||
}
|
||||
|
||||
func (x *AccessBox) decodeGate(gate *AccessBox_Gate, owner *keys.PrivateKey, seedKey *keys.PublicKey) (*GateData, error) {
|
||||
data, err := decrypt(owner, seedKey, gate.Tokens)
|
||||
data, err := decrypt(owner, seedKey, gate.Tokens, gate.EncryptionKeySalt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypt tokens: %w", err)
|
||||
}
|
||||
|
@ -273,16 +284,16 @@ func generateShared256(prv *keys.PrivateKey, pub *keys.PublicKey) (sk []byte, er
|
|||
return sk, nil
|
||||
}
|
||||
|
||||
func deriveKey(secret []byte) ([]byte, error) {
|
||||
func deriveKey(secret, salt []byte) ([]byte, error) {
|
||||
hash := sha256.New
|
||||
kdf := hkdf.New(hash, secret, nil, nil)
|
||||
kdf := hkdf.New(hash, secret, salt, nil)
|
||||
key := make([]byte, 32)
|
||||
_, err := io.ReadFull(kdf, key)
|
||||
return key, err
|
||||
}
|
||||
|
||||
func encrypt(owner *keys.PrivateKey, seedKey *keys.PublicKey, data []byte) ([]byte, error) {
|
||||
enc, err := getCipher(owner, seedKey)
|
||||
func encrypt(owner *keys.PrivateKey, seedKey *keys.PublicKey, data, salt []byte) ([]byte, error) {
|
||||
enc, err := getCipher(owner, seedKey, salt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get chiper: %w", err)
|
||||
}
|
||||
|
@ -295,8 +306,8 @@ func encrypt(owner *keys.PrivateKey, seedKey *keys.PublicKey, data []byte) ([]by
|
|||
return enc.Seal(nonce, nonce, data, nil), nil
|
||||
}
|
||||
|
||||
func decrypt(owner *keys.PrivateKey, seedKey *keys.PublicKey, data []byte) ([]byte, error) {
|
||||
dec, err := getCipher(owner, seedKey)
|
||||
func decrypt(owner *keys.PrivateKey, seedKey *keys.PublicKey, data, salt []byte) ([]byte, error) {
|
||||
dec, err := getCipher(owner, seedKey, salt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get chiper: %w", err)
|
||||
}
|
||||
|
@ -309,13 +320,13 @@ func decrypt(owner *keys.PrivateKey, seedKey *keys.PublicKey, data []byte) ([]by
|
|||
return dec.Open(nil, nonce, cypher, nil)
|
||||
}
|
||||
|
||||
func getCipher(owner *keys.PrivateKey, seedKey *keys.PublicKey) (cipher.AEAD, error) {
|
||||
func getCipher(owner *keys.PrivateKey, seedKey *keys.PublicKey, salt []byte) (cipher.AEAD, error) {
|
||||
secret, err := generateShared256(owner, seedKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate shared key: %w", err)
|
||||
}
|
||||
|
||||
key, err := deriveKey(secret)
|
||||
key, err := deriveKey(secret, salt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("derive key: %w", err)
|
||||
}
|
||||
|
@ -323,8 +334,8 @@ func getCipher(owner *keys.PrivateKey, seedKey *keys.PublicKey) (cipher.AEAD, er
|
|||
return chacha20poly1305.NewX(key)
|
||||
}
|
||||
|
||||
func generateSecret() ([]byte, error) {
|
||||
b := make([]byte, 32)
|
||||
_, err := io.ReadFull(rand.Reader, b)
|
||||
func generateRandomBytes(length int) ([]byte, error) {
|
||||
b := make([]byte, length)
|
||||
_, err := rand.Read(b)
|
||||
return b, err
|
||||
}
|
||||
|
|
|
@ -159,8 +159,9 @@ type AccessBox_Gate struct {
|
|||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Tokens []byte `protobuf:"bytes,1,opt,name=tokens,proto3" json:"tokens,omitempty"`
|
||||
GatePublicKey []byte `protobuf:"bytes,2,opt,name=gatePublicKey,proto3" json:"gatePublicKey,omitempty"`
|
||||
Tokens []byte `protobuf:"bytes,1,opt,name=tokens,proto3" json:"tokens,omitempty"`
|
||||
GatePublicKey []byte `protobuf:"bytes,2,opt,name=gatePublicKey,proto3" json:"gatePublicKey,omitempty"`
|
||||
EncryptionKeySalt []byte `protobuf:"bytes,3,opt,name=encryptionKeySalt,proto3" json:"encryptionKeySalt,omitempty"`
|
||||
}
|
||||
|
||||
func (x *AccessBox_Gate) Reset() {
|
||||
|
@ -209,6 +210,13 @@ func (x *AccessBox_Gate) GetGatePublicKey() []byte {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (x *AccessBox_Gate) GetEncryptionKeySalt() []byte {
|
||||
if x != nil {
|
||||
return x.EncryptionKeySalt
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type AccessBox_ContainerPolicy struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
|
@ -269,7 +277,7 @@ var File_creds_accessbox_accessbox_proto protoreflect.FileDescriptor
|
|||
var file_creds_accessbox_accessbox_proto_rawDesc = []byte{
|
||||
0x0a, 0x1f, 0x63, 0x72, 0x65, 0x64, 0x73, 0x2f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x62, 0x6f,
|
||||
0x78, 0x2f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x62, 0x6f, 0x78, 0x2e, 0x70, 0x72, 0x6f, 0x74,
|
||||
0x6f, 0x12, 0x09, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x62, 0x6f, 0x78, 0x22, 0xe3, 0x02, 0x0a,
|
||||
0x6f, 0x12, 0x09, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x62, 0x6f, 0x78, 0x22, 0x91, 0x03, 0x0a,
|
||||
0x09, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x42, 0x6f, 0x78, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65,
|
||||
0x65, 0x64, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x65, 0x65,
|
||||
0x64, 0x4b, 0x65, 0x79, 0x12, 0x2f, 0x0a, 0x05, 0x67, 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20,
|
||||
|
@ -282,29 +290,31 @@ var file_creds_accessbox_accessbox_proto_rawDesc = []byte{
|
|||
0x6c, 0x69, 0x63, 0x79, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x50,
|
||||
0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x73, 0x43, 0x75, 0x73, 0x74, 0x6f,
|
||||
0x6d, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x43, 0x75, 0x73, 0x74, 0x6f,
|
||||
0x6d, 0x1a, 0x44, 0x0a, 0x04, 0x47, 0x61, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x6f, 0x6b,
|
||||
0x6d, 0x1a, 0x72, 0x0a, 0x04, 0x47, 0x61, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x6f, 0x6b,
|
||||
0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e,
|
||||
0x73, 0x12, 0x24, 0x0a, 0x0d, 0x67, 0x61, 0x74, 0x65, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b,
|
||||
0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x67, 0x61, 0x74, 0x65, 0x50, 0x75,
|
||||
0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x1a, 0x59, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x74, 0x61,
|
||||
0x69, 0x6e, 0x65, 0x72, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x2e, 0x0a, 0x12, 0x6c, 0x6f,
|
||||
0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74,
|
||||
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e,
|
||||
0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6f,
|
||||
0x6c, 0x69, 0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x70, 0x6f, 0x6c, 0x69,
|
||||
0x63, 0x79, 0x22, 0x6e, 0x0a, 0x06, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09,
|
||||
0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52,
|
||||
0x09, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x62, 0x65,
|
||||
0x61, 0x72, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52,
|
||||
0x0b, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x24, 0x0a, 0x0d,
|
||||
0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x03, 0x20,
|
||||
0x03, 0x28, 0x0c, 0x52, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65,
|
||||
0x6e, 0x73, 0x42, 0x46, 0x5a, 0x44, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66,
|
||||
0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64,
|
||||
0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x73, 0x33, 0x2d, 0x67,
|
||||
0x77, 0x2f, 0x63, 0x72, 0x65, 0x64, 0x73, 0x2f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x62, 0x6f, 0x78,
|
||||
0x3b, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x62, 0x6f, 0x78, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
|
||||
0x6f, 0x33,
|
||||
0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x11, 0x65, 0x6e, 0x63, 0x72, 0x79,
|
||||
0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x53, 0x61, 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01,
|
||||
0x28, 0x0c, 0x52, 0x11, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65,
|
||||
0x79, 0x53, 0x61, 0x6c, 0x74, 0x1a, 0x59, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e,
|
||||
0x65, 0x72, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x2e, 0x0a, 0x12, 0x6c, 0x6f, 0x63, 0x61,
|
||||
0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x18, 0x01,
|
||||
0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f,
|
||||
0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6f, 0x6c, 0x69,
|
||||
0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79,
|
||||
0x22, 0x6e, 0x0a, 0x06, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65,
|
||||
0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73,
|
||||
0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x62, 0x65, 0x61, 0x72,
|
||||
0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x62,
|
||||
0x65, 0x61, 0x72, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x73, 0x65,
|
||||
0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28,
|
||||
0x0c, 0x52, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73,
|
||||
0x42, 0x46, 0x5a, 0x44, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e,
|
||||
0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61,
|
||||
0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x73, 0x33, 0x2d, 0x67, 0x77, 0x2f,
|
||||
0x63, 0x72, 0x65, 0x64, 0x73, 0x2f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x62, 0x6f, 0x78, 0x3b, 0x61,
|
||||
0x63, 0x63, 0x65, 0x73, 0x73, 0x62, 0x6f, 0x78, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
|
@ -10,6 +10,7 @@ message AccessBox {
|
|||
message Gate {
|
||||
bytes tokens = 1 [json_name = "tokens"];
|
||||
bytes gatePublicKey = 2 [json_name = "gatePublicKey"];
|
||||
bytes encryptionKeySalt = 3 [json_name = "encryptionKeySalt"];
|
||||
}
|
||||
|
||||
message ContainerPolicy {
|
||||
|
|
|
@ -23,6 +23,10 @@ func TestTokensEncryptDecrypt(t *testing.T) {
|
|||
tkn bearer.Token
|
||||
tkn2 bearer.Token
|
||||
)
|
||||
|
||||
salt, err := generateRandomBytes(saltLength)
|
||||
require.NoError(t, err)
|
||||
|
||||
sec, err := keys.NewPrivateKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
|
@ -32,16 +36,47 @@ func TestTokensEncryptDecrypt(t *testing.T) {
|
|||
tkn.SetEACLTable(*eacl.NewTable())
|
||||
require.NoError(t, tkn.Sign(sec.PrivateKey))
|
||||
|
||||
data, err := encrypt(cred, cred.PublicKey(), tkn.Marshal())
|
||||
require.NoError(t, err)
|
||||
t.Run("positive case without salt", func(t *testing.T) {
|
||||
data, err := encrypt(cred, cred.PublicKey(), tkn.Marshal(), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
rawTkn2, err := decrypt(cred, cred.PublicKey(), data)
|
||||
require.NoError(t, err)
|
||||
rawTkn2, err := decrypt(cred, cred.PublicKey(), data, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = tkn2.Unmarshal(rawTkn2)
|
||||
require.NoError(t, err)
|
||||
err = tkn2.Unmarshal(rawTkn2)
|
||||
require.NoError(t, err)
|
||||
|
||||
assertBearerToken(t, tkn, tkn2)
|
||||
assertBearerToken(t, tkn, tkn2)
|
||||
})
|
||||
|
||||
t.Run("positive case with salt", func(t *testing.T) {
|
||||
data, err := encrypt(cred, cred.PublicKey(), tkn.Marshal(), salt)
|
||||
require.NoError(t, err)
|
||||
|
||||
rawTkn2, err := decrypt(cred, cred.PublicKey(), data, salt)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = tkn2.Unmarshal(rawTkn2)
|
||||
require.NoError(t, err)
|
||||
|
||||
assertBearerToken(t, tkn, tkn2)
|
||||
})
|
||||
|
||||
t.Run("wrong salt", func(t *testing.T) {
|
||||
data, err := encrypt(cred, cred.PublicKey(), tkn.Marshal(), salt)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = decrypt(cred, cred.PublicKey(), data, nil)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("wrong private key", func(t *testing.T) {
|
||||
data, err := encrypt(cred, cred.PublicKey(), tkn.Marshal(), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = decrypt(sec, cred.PublicKey(), data, nil)
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestBearerTokenInAccessBox(t *testing.T) {
|
||||
|
|
|
@ -202,7 +202,7 @@ func (c *cred) checkIfCredentialsAreRemoved(ctx context.Context, cnrID cid.ID, a
|
|||
|
||||
func (c *cred) putBoxToCache(accessKeyID string, val *cache.AccessBoxCacheValue) {
|
||||
if err := c.cache.Put(accessKeyID, val); err != nil {
|
||||
c.log.Warn(logs.CouldntPutAccessBoxIntoCache, zap.String("accessKeyID", accessKeyID))
|
||||
c.log.Warn(logs.CouldntPutAccessBoxIntoCache, zap.String("accessKeyID", accessKeyID), logs.TagField(logs.TagDatapath))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -241,7 +241,7 @@ func (c *cred) getAccessBox(ctx context.Context, cnrID cid.ID, accessKeyID strin
|
|||
|
||||
func (c *cred) Put(ctx context.Context, prm CredentialsParam) (oid.Address, error) {
|
||||
if prm.AccessKeyID != "" {
|
||||
c.log.Info(logs.CheckCustomAccessKeyIDUniqueness, zap.String("access_key_id", prm.AccessKeyID))
|
||||
c.log.Info(logs.CheckCustomAccessKeyIDUniqueness, zap.String("access_key_id", prm.AccessKeyID), logs.TagField(logs.TagApp))
|
||||
credsPrm := PrmGetCredsObject{
|
||||
Container: prm.Container,
|
||||
AccessKeyID: prm.AccessKeyID,
|
||||
|
|
|
@ -199,6 +199,7 @@ It contains:
|
|||
in [spec](https://git.frostfs.info/TrueCloudLab/frostfs-api/src/commit/4c68d92468503b10282c8a92af83a56f170c8a3a/acl/types.proto#L189)
|
||||
* Marshaled session token - more detail
|
||||
in [spec](https://git.frostfs.info/TrueCloudLab/frostfs-api/src/commit/4c68d92468503b10282c8a92af83a56f170c8a3a/session/types.proto#L89)
|
||||
* Encryption Key Salt - randomly generated 16 bytes that are used to derivation the encryption key
|
||||
* Container placement policies:
|
||||
* `LocationsConstraint` - name of location constraint that can be used to create bucket/container using s3
|
||||
credentials related to this `AccessBox`
|
||||
|
@ -258,19 +259,20 @@ secp256r1 or prime256v1) is used (unless otherwise stated).
|
|||
* Create ephemeral key (`SeedKey`), it's need to generate shared secret
|
||||
* Generate random 32-byte (that after hex-encoded be `SecretAccessKey`) or use existing secret access key
|
||||
(if `AccessBox` is being updated rather than creating brand new) or use arbitrary user-provided string
|
||||
* Generate random 16-byte salt
|
||||
* Generate shared secret as [ECDH](https://en.wikipedia.org/wiki/Elliptic-curve_Diffie%E2%80%93Hellman)
|
||||
* Derive 32-byte key using shared secret from previous step with key derivation function based on
|
||||
* Derive 32-byte key using shared secret from previous step and the salt with key derivation function based on
|
||||
HMAC with SHA256 [HKDF](https://en.wikipedia.org/wiki/HKDF)
|
||||
* Encrypt marshaled [Tokens](../creds/accessbox) using derived key
|
||||
with [ChaCha20-Poly1305](https://en.wikipedia.org/wiki/ChaCha20-Poly1305) algorithm without additional data.
|
||||
|
||||
**Decryption:**
|
||||
|
||||
* Get public part of `SeedKey` from `AccessBox`
|
||||
* Get public part of `SeedKey` and the salt from `AccessBox`
|
||||
* Generate shared secret as follows:
|
||||
* Make scalar curve multiplication of public part of `SeedKey` and private part of s3-gw key
|
||||
* Use `X` part of multiplication (with zero padding at the beginning to fit 32-byte)
|
||||
* Derive 32-byte key using shared secret from previous step with key derivation function based on
|
||||
* Derive 32-byte key using shared secret from previous step and the salt with key derivation function based on
|
||||
HMAC with SHA256 [HKDF](https://en.wikipedia.org/wiki/HKDF)
|
||||
* Decrypt encrypted marshaled [Tokens](../creds/accessbox) using derived key
|
||||
with [ChaCha20-Poly1305](https://en.wikipedia.org/wiki/ChaCha20-Poly1305) algorithm without additional data.
|
||||
|
|
|
@ -64,6 +64,8 @@ $ frostfs-s3-gw --listen_address 192.168.130.130:443 \
|
|||
|
||||
Using these flag you can configure only one address. To set multiple addresses use yaml config.
|
||||
|
||||
**Note:** It's not recommended to configure addresses via flags and yaml config at the same time.
|
||||
|
||||
### RPC endpoint and resolving of bucket names
|
||||
|
||||
To set RPC endpoint specify a value of parameter `-r` or `--rpc_endpoint`. The parameter is **required if** another
|
||||
|
@ -213,6 +215,7 @@ resolve_order:
|
|||
|
||||
connect_timeout: 10s
|
||||
stream_timeout: 10s
|
||||
tree_stream_timeout: 10s
|
||||
healthcheck_timeout: 15s
|
||||
rebalance_interval: 60s
|
||||
pool_error_threshold: 100
|
||||
|
@ -235,7 +238,8 @@ source_ip_header: "Source-Ip"
|
|||
| `rpc_endpoint` | `string` | no | | The address of the RPC host to which the gateway connects to resolve bucket names and interact with frostfs contracts (required to use the `nns` resolver and `frostfsid` contract). |
|
||||
| `resolve_order` | `[]string` | yes | `[dns]` | Order of bucket name resolvers to use. Available resolvers: `dns`, `nns`. |
|
||||
| `connect_timeout` | `duration` | no | `10s` | Timeout to connect to a node. |
|
||||
| `stream_timeout` | `duration` | no | `10s` | Timeout for individual operations in streaming RPC. |
|
||||
| `stream_timeout` | `duration` | no | `10s` | Timeout for individual operations in object pool streaming RPC. |
|
||||
| `tree_stream_timeout` | `duration` | no | `10s` | Timeout for individual operations in tree pool streaming RPC. |
|
||||
| `healthcheck_timeout` | `duration` | no | `15s` | Timeout to check node health during rebalance. |
|
||||
| `rebalance_interval` | `duration` | no | `60s` | Interval to check node health. |
|
||||
| `pool_error_threshold` | `uint32` | no | `100` | The number of errors on connection after which node is considered as unhealthy. |
|
||||
|
@ -377,6 +381,13 @@ logger:
|
|||
initial: 100
|
||||
thereafter: 100
|
||||
interval: 1s
|
||||
tags:
|
||||
- name: app
|
||||
level: info
|
||||
- name: datapath
|
||||
- name: external_blockchain
|
||||
- name: external_storage_tree
|
||||
- name: external_storage
|
||||
```
|
||||
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|
@ -388,6 +399,32 @@ logger:
|
|||
| `sampling.thereafter` | `int` | no | '100' | Sampling count of entries after an `interval`. |
|
||||
| `sampling.interval` | `duration` | no | '1s' | Sampling interval of messaging similar entries. |
|
||||
|
||||
## Tags
|
||||
|
||||
There are additional log entries that can hurt performance and can be additionally logged by using `logger.tags`
|
||||
parameter.
|
||||
If section `tags` isn't set the default tags (see [Tag values](#tag-values)) be enabled.
|
||||
If section `tags` set but empty no tags be used.
|
||||
|
||||
```yaml
|
||||
tags:
|
||||
- name: "app"
|
||||
level: info
|
||||
```
|
||||
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|-----------------------|------------|---------------|---------------------------|-------------------------------------------------------------------------------------------------------|
|
||||
| `name` | `string` | yes | | Tag name. Possible values see below in `Tag values` section. |
|
||||
| `level` | `string` | yes | Value from `logger.level` | Logging level for specific tag. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. |
|
||||
|
||||
### Tag values
|
||||
|
||||
* `app` - common application logs (enabled by default).
|
||||
* `datapath` - main logic of application (enabled by default).
|
||||
* `external_blockchain` - external interaction with neo-go blockchain (enabled by default).
|
||||
* `external_storage` - external interaction with storage node (enabled by default).
|
||||
* `external_storage_tree` - external interaction with tree service in storage node (enabled by default).
|
||||
|
||||
|
||||
### `http_logging` section
|
||||
|
||||
|
@ -465,7 +502,7 @@ cache:
|
|||
| `accesscontrol` | [Cache config](#cache-subsection) | `lifetime: 1m`<br>`size: 100000` | Cache which stores owner to cache operation mapping. |
|
||||
| `morph_policy` | [Cache config](#cache-subsection) | `lifetime: 1m`<br>`size: 10000` | Cache which stores list of policy chains. |
|
||||
| `frostfsid` | [Cache config](#cache-subsection) | `lifetime: 1m`<br>`size: 10000` | Cache which stores FrostfsID subject info. |
|
||||
| `network_info` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores network info. |
|
||||
| `network_info` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores network-related values. |
|
||||
|
||||
#### `cache` subsection
|
||||
|
||||
|
@ -688,12 +725,14 @@ features:
|
|||
deny_by_default: false
|
||||
md5:
|
||||
enabled: false
|
||||
tree_pool_netmap_support: true
|
||||
```
|
||||
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|--------------------------|--------|---------------|---------------|------------------------------------------------------------------------------|
|
||||
| `md5.enabled` | `bool` | yes | false | Flag to enable return MD5 checksum in ETag headers and fields. |
|
||||
| `policy.deny_by_default` | `bool` | yes | false | Enable denying access for request that doesn't match any policy chain rules. |
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|----------------------------|--------|---------------|---------------|---------------------------------------------------------------------------------------------------------|
|
||||
| `md5.enabled` | `bool` | yes | false | Flag to enable return MD5 checksum in ETag headers and fields. |
|
||||
| `policy.deny_by_default` | `bool` | yes | false | Enable denying access for request that doesn't match any policy chain rules. |
|
||||
| `tree_pool_netmap_support` | `bool` | no | false | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. |
|
||||
|
||||
# `web` section
|
||||
Contains web server configuration parameters.
|
||||
|
|
|
@ -10,6 +10,7 @@ package AccessBox {
|
|||
map Gate {
|
||||
GateKey => Encoded public gate key
|
||||
Encrypted tokens *--> Tokens
|
||||
EncryptionKeySalt => Salt for derivation the encryption key
|
||||
}
|
||||
|
||||
map ContainerPolicy {
|
||||
|
|
File diff suppressed because one or more lines are too long
Before Width: | Height: | Size: 13 KiB After Width: | Height: | Size: 14 KiB |
58
go.mod
58
go.mod
|
@ -5,13 +5,15 @@ go 1.22
|
|||
require (
|
||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.1-0.20241022094040-5f956751d48b
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241212101224-902f32eeabcf
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a
|
||||
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972
|
||||
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b
|
||||
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
|
||||
github.com/aws/aws-sdk-go-v2 v1.30.5
|
||||
github.com/aws/aws-sdk-go-v2 v1.34.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.32
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.31
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1
|
||||
github.com/aws/smithy-go v1.22.2
|
||||
github.com/bluele/gcache v0.0.2
|
||||
github.com/go-chi/chi/v5 v5.0.8
|
||||
github.com/google/uuid v1.6.0
|
||||
|
@ -28,16 +30,16 @@ require (
|
|||
github.com/stretchr/testify v1.9.0
|
||||
github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4
|
||||
github.com/urfave/cli/v2 v2.27.2
|
||||
go.opentelemetry.io/otel v1.28.0
|
||||
go.opentelemetry.io/otel/trace v1.28.0
|
||||
go.opentelemetry.io/otel v1.31.0
|
||||
go.opentelemetry.io/otel/trace v1.31.0
|
||||
go.uber.org/zap v1.27.0
|
||||
golang.org/x/crypto v0.24.0
|
||||
golang.org/x/crypto v0.31.0
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
|
||||
golang.org/x/net v0.26.0
|
||||
golang.org/x/sys v0.22.0
|
||||
golang.org/x/text v0.16.0
|
||||
google.golang.org/grpc v1.66.2
|
||||
google.golang.org/protobuf v1.34.2
|
||||
golang.org/x/net v0.30.0
|
||||
golang.org/x/sys v0.28.0
|
||||
golang.org/x/text v0.21.0
|
||||
google.golang.org/grpc v1.69.2
|
||||
google.golang.org/protobuf v1.36.1
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||
)
|
||||
|
||||
|
@ -48,16 +50,19 @@ require (
|
|||
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect
|
||||
github.com/VictoriaMetrics/easyproto v0.1.4 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.22.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.30.6 // indirect
|
||||
github.com/aws/smithy-go v1.20.4 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
|
@ -74,10 +79,19 @@ require (
|
|||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/holiman/uint256 v1.2.4 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/ipfs/go-cid v0.0.7 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/minio/sha256-simd v1.0.1 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/multiformats/go-base32 v0.1.0 // indirect
|
||||
github.com/multiformats/go-base36 v0.2.0 // indirect
|
||||
github.com/multiformats/go-multiaddr v0.14.0 // indirect
|
||||
github.com/multiformats/go-multibase v0.2.0 // indirect
|
||||
github.com/multiformats/go-multihash v0.2.3 // indirect
|
||||
github.com/multiformats/go-varint v0.0.7 // indirect
|
||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect
|
||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240727093519-1a48f1ce43ec // indirect
|
||||
github.com/nspcc-dev/rfc6979 v0.2.1 // indirect
|
||||
|
@ -87,6 +101,7 @@ require (
|
|||
github.com/prometheus/common v0.48.0 // indirect
|
||||
github.com/prometheus/procfs v0.12.0 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/spf13/afero v1.9.3 // indirect
|
||||
github.com/spf13/cast v1.5.0 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
|
@ -98,14 +113,15 @@ require (
|
|||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.28.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.28.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.31.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.31.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/sync v0.7.0 // indirect
|
||||
golang.org/x/term v0.21.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
|
||||
golang.org/x/sync v0.10.0 // indirect
|
||||
golang.org/x/term v0.27.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
lukechampine.com/blake3 v1.2.1 // indirect
|
||||
)
|
||||
|
|
131
go.sum
131
go.sum
|
@ -42,8 +42,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
|
|||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241212101224-902f32eeabcf h1:PqRKQX+Xlqq4qsAlr7Jcx2kapLdPGZZQ09MEW+ui2c4=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241212101224-902f32eeabcf/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8=
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
|
||||
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8=
|
||||
|
@ -62,32 +62,42 @@ github.com/VictoriaMetrics/easyproto v0.1.4 h1:r8cNvo8o6sR4QShBXQd1bKw/VVLSQma/V
|
|||
github.com/VictoriaMetrics/easyproto v0.1.4/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo3/GThPs2KH23mv710=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
|
||||
github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g=
|
||||
github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
|
||||
github.com/aws/aws-sdk-go-v2 v1.34.0 h1:9iyL+cjifckRGEVpRKZP3eIxVlL06Qk1Tk13vreaVQU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.34.0/go.mod h1:JgstGg0JjWU1KpVJjD5H0y0yyAIpSdKEq556EI6yOOM=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.32 h1:jnAMVTJTpAQlePCUUlnXnllHEMGVWmvUJOiGjgtS9S0=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.32/go.mod h1:JibtzKJoXT0M/MhoYL6qfCk7nm/MppwukDFZtdgVRoY=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.31 h1:jtyfcOfgoqWA2hW/E8sFbwdfgwD3APnF9CLCKE8dTyw=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.31/go.mod h1:RSgY5lfCfw+FoyKWtOpLolPlfQVdDBQWTUniAaE+NKY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 h1:Ej0Rf3GMv50Qh4G4852j2djtoDb7AzQ7MuQeFHa3D70=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29/go.mod h1:oeNTC7PwJNoM5AznVr23wxhLnuJv0ZDe5v7w0wqIs9M=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 h1:6e8a71X+9GfghragVevC5bZqvATtc3mAMgxpSNbgzF0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29/go.mod h1:c4jkZiQ+BWpNqq7VtrxjwISrLrt/VvPq3XiopkUIolI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 h1:g9OUETuxA8i/Www5Cby0R3WSTe7ppFTZXHVLNskNS4w=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29/go.mod h1:CQk+koLR1QeY1+vm7lqNfFii07DEderKq6T3F1L2pyc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 h1:EP1ITDgYVPM2dL1bBBntJ7AW5yTjuWGz9XO+CZwpALU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3/go.mod h1:5lWNWeAgWenJ/BZ/CP9k9DjLbC0pjnM045WjXRPPi14=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 h1:hN4yJBGswmFTOVYqmbz1GBs9ZMtQe8SrYxPwrkrlRv8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10/go.mod h1:TsxON4fEZXyrKY+D+3d2gSTyJkGORexIYab9PTf56DA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 h1:fXoWC2gi7tdJYNTPnnlSGzEVwewUchOi8xVq/dkg8Qs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10/go.mod h1:cvzBApD5dVazHU8C2rbBQzzzsKc8m5+wNJ9mCRZLKPc=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1 h1:9LawY3cDJ3HE+v2GMd5SOkNLDwgN4K7TsCjyVBYu/L4=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1/go.mod h1:hHnELVnIHltd8EOF3YzahVX6F6y2C6dNqpRj1IMkS5I=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.22.6 h1:o++HUDXlbrTl4PSal3YHtdErQxB8mDGAtkKNXBWPfIU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.22.6/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.6 h1:yCHcQCOwTfIsc8DoEhM3qXPxD+j8CbI6t1K3dNzsWV0=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.6/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.30.6 h1:TrQadF7GcqvQ63kgwEcjlrVc2Fa0wpgLT0xtc73uAd8=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.30.6/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o=
|
||||
github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
|
||||
github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
|
||||
github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
|
||||
github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c=
|
||||
|
@ -164,6 +174,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
|
|||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
|
@ -216,11 +228,15 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
|
|||
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY=
|
||||
github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
|
||||
github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
|
@ -233,14 +249,37 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V
|
|||
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
|
||||
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
|
||||
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
|
||||
github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus=
|
||||
github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY=
|
||||
github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU=
|
||||
github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
|
||||
github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
||||
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
|
||||
github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
||||
github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
|
||||
github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE=
|
||||
github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI=
|
||||
github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
|
||||
github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0=
|
||||
github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
|
||||
github.com/multiformats/go-multiaddr v0.14.0 h1:bfrHrJhrRuh/NXH5mCnemjpbGjzRw/b+tJFOD41g2tU=
|
||||
github.com/multiformats/go-multiaddr v0.14.0/go.mod h1:6EkVAxtznq2yC3QT5CM1UTAwG0GTP3EWAIcjHuzQ+r4=
|
||||
github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
|
||||
github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
|
||||
github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk=
|
||||
github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
|
||||
github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
|
||||
github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM=
|
||||
github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
|
||||
github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
|
||||
github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
|
||||
github.com/nspcc-dev/dbft v0.2.0 h1:sDwsQES600OSIMncV176t2SX5OvB14lzeOAyKFOkbMI=
|
||||
github.com/nspcc-dev/dbft v0.2.0/go.mod h1:oFE6paSC/yfFh9mcNU6MheMGOYXK9+sPiRk3YMoz49o=
|
||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk=
|
||||
|
@ -284,6 +323,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU
|
|||
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk=
|
||||
github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y=
|
||||
github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
|
||||
|
@ -336,20 +377,22 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
|||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
|
||||
go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
|
||||
go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
|
||||
go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
|
||||
go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y=
|
||||
go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
|
||||
go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
|
||||
go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE=
|
||||
go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
|
||||
go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
|
||||
go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
|
||||
go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
|
||||
go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
|
||||
go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
|
||||
go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
|
||||
go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
|
||||
go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
|
||||
go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
|
||||
go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
|
@ -362,12 +405,13 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
|||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
|
||||
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
|
||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
|
@ -439,8 +483,8 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY
|
|||
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
|
||||
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
|
||||
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
|
||||
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
@ -460,8 +504,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
@ -504,19 +548,20 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
|
||||
golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
|
||||
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
|
||||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
@ -637,10 +682,10 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D
|
|||
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
|
@ -657,8 +702,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
|
|||
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
|
||||
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
|
||||
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo=
|
||||
google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y=
|
||||
google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU=
|
||||
google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
|
@ -669,8 +714,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
|
|||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
|
||||
google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
|
||||
google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
|
@ -699,6 +744,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
|
|||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI=
|
||||
lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
|
|
|
@ -409,6 +409,15 @@ func (x *FrostFS) NetworkInfo(ctx context.Context) (netmap.NetworkInfo, error) {
|
|||
return ni, nil
|
||||
}
|
||||
|
||||
func (x *FrostFS) NetmapSnapshot(ctx context.Context) (netmap.NetMap, error) {
|
||||
netmapSnapshot, err := x.pool.NetMapSnapshot(ctx)
|
||||
if err != nil {
|
||||
return netmapSnapshot, handleObjectError("get netmap via connection pool", err)
|
||||
}
|
||||
|
||||
return netmapSnapshot, nil
|
||||
}
|
||||
|
||||
func (x *FrostFS) PatchObject(ctx context.Context, prm frostfs.PrmObjectPatch) (oid.ID, error) {
|
||||
var addr oid.Address
|
||||
addr.SetContainer(prm.Container)
|
||||
|
@ -477,6 +486,9 @@ func handleObjectError(msg string, err error) error {
|
|||
}
|
||||
|
||||
if reason, ok := frosterr.IsErrObjectAccessDenied(err); ok {
|
||||
if strings.Contains(reason, "limit reached") {
|
||||
return fmt.Errorf("%s: %w: %s", msg, frostfs.ErrQuotaLimitReached, reason)
|
||||
}
|
||||
return fmt.Errorf("%s: %w: %s", msg, frostfs.ErrAccessDenied, reason)
|
||||
}
|
||||
|
||||
|
|
|
@ -8,31 +8,49 @@ import (
|
|||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
||||
frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
func TestErrorChecking(t *testing.T) {
|
||||
reason := "some reason"
|
||||
err := new(apistatus.ObjectAccessDenied)
|
||||
err.WriteReason(reason)
|
||||
func TestHandleObjectError(t *testing.T) {
|
||||
msg := "some msg"
|
||||
|
||||
var wrappedError error
|
||||
t.Run("nil error", func(t *testing.T) {
|
||||
err := handleObjectError(msg, nil)
|
||||
require.Nil(t, err)
|
||||
})
|
||||
|
||||
if fetchedReason, ok := frosterr.IsErrObjectAccessDenied(err); ok {
|
||||
wrappedError = fmt.Errorf("%w: %s", frostfs.ErrAccessDenied, fetchedReason)
|
||||
}
|
||||
t.Run("simple access denied", func(t *testing.T) {
|
||||
reason := "some reason"
|
||||
inputErr := new(apistatus.ObjectAccessDenied)
|
||||
inputErr.WriteReason(reason)
|
||||
|
||||
require.ErrorIs(t, wrappedError, frostfs.ErrAccessDenied)
|
||||
require.Contains(t, wrappedError.Error(), reason)
|
||||
}
|
||||
err := handleObjectError(msg, inputErr)
|
||||
require.ErrorIs(t, err, frostfs.ErrAccessDenied)
|
||||
require.Contains(t, err.Error(), reason)
|
||||
require.Contains(t, err.Error(), msg)
|
||||
})
|
||||
|
||||
t.Run("access denied - quota reached", func(t *testing.T) {
|
||||
reason := "Quota limit reached"
|
||||
inputErr := new(apistatus.ObjectAccessDenied)
|
||||
inputErr.WriteReason(reason)
|
||||
|
||||
err := handleObjectError(msg, inputErr)
|
||||
require.ErrorIs(t, err, frostfs.ErrQuotaLimitReached)
|
||||
require.Contains(t, err.Error(), reason)
|
||||
require.Contains(t, err.Error(), msg)
|
||||
})
|
||||
|
||||
func TestErrorTimeoutChecking(t *testing.T) {
|
||||
t.Run("simple timeout", func(t *testing.T) {
|
||||
require.True(t, frosterr.IsTimeoutError(errors.New("timeout")))
|
||||
inputErr := errors.New("timeout")
|
||||
|
||||
err := handleObjectError(msg, inputErr)
|
||||
require.ErrorIs(t, err, frostfs.ErrGatewayTimeout)
|
||||
require.Contains(t, err.Error(), inputErr.Error())
|
||||
require.Contains(t, err.Error(), msg)
|
||||
})
|
||||
|
||||
t.Run("deadline exceeded", func(t *testing.T) {
|
||||
|
@ -40,11 +58,35 @@ func TestErrorTimeoutChecking(t *testing.T) {
|
|||
defer cancel()
|
||||
<-ctx.Done()
|
||||
|
||||
require.True(t, frosterr.IsTimeoutError(ctx.Err()))
|
||||
err := handleObjectError(msg, ctx.Err())
|
||||
require.ErrorIs(t, err, frostfs.ErrGatewayTimeout)
|
||||
require.Contains(t, err.Error(), ctx.Err().Error())
|
||||
require.Contains(t, err.Error(), msg)
|
||||
})
|
||||
|
||||
t.Run("grpc deadline exceeded", func(t *testing.T) {
|
||||
err := fmt.Errorf("wrap grpc error: %w", status.Error(codes.DeadlineExceeded, "error"))
|
||||
require.True(t, frosterr.IsTimeoutError(err))
|
||||
inputErr := fmt.Errorf("wrap grpc error: %w", status.Error(codes.DeadlineExceeded, "error"))
|
||||
|
||||
err := handleObjectError(msg, inputErr)
|
||||
require.ErrorIs(t, err, frostfs.ErrGatewayTimeout)
|
||||
require.Contains(t, err.Error(), inputErr.Error())
|
||||
require.Contains(t, err.Error(), msg)
|
||||
})
|
||||
|
||||
t.Run("global domain already", func(t *testing.T) {
|
||||
inputErr := errors.New("global domain is already taken")
|
||||
|
||||
err := handleObjectError(msg, inputErr)
|
||||
require.ErrorIs(t, err, frostfs.ErrGlobalDomainIsAlreadyTaken)
|
||||
require.Contains(t, err.Error(), inputErr.Error())
|
||||
require.Contains(t, err.Error(), msg)
|
||||
})
|
||||
|
||||
t.Run("unknown error", func(t *testing.T) {
|
||||
inputErr := errors.New("unknown error")
|
||||
|
||||
err := handleObjectError(msg, inputErr)
|
||||
require.ErrorIs(t, err, inputErr)
|
||||
require.Contains(t, err.Error(), msg)
|
||||
})
|
||||
}
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue