forked from TrueCloudLab/frostfs-s3-gw
Compare commits
4 commits
feature/fi
...
master
Author | SHA1 | Date | |
---|---|---|---|
a12fea8a5b | |||
9875307c9b | |||
b1775f9478 | |||
4fa45bdac2 |
14 changed files with 336 additions and 44 deletions
111
CHANGELOG.md
111
CHANGELOG.md
|
@ -4,17 +4,105 @@ This document outlines major changes between releases.
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.31.0] - Rongbuk - 2024-11-20
|
||||
|
||||
### Fixed
|
||||
- Docker warnings during image build (#421)
|
||||
- `PartNumberMarker` in ListMultipart response (#451)
|
||||
- PostObject handling (#456)
|
||||
- Tag logging errors (#452)
|
||||
- Removing of duplicated parts in tree service during split brain (#448)
|
||||
- Container resolving (#482)
|
||||
- FrostFS to S3 error transformation (#488)
|
||||
- Default bucket routing (#507)
|
||||
- encoding-type in ListBucketObjectVersions (#404)
|
||||
- SIGHUP support for `tracing.enabled` config parameter (#520)
|
||||
- `trace_id` parameter in logs (#501)
|
||||
- Listing marker processing (#539)
|
||||
- Content-MD5 header check (#540)
|
||||
- Precondition check (#538)
|
||||
- Bucket name check during all S3 operations (#556)
|
||||
|
||||
### Added
|
||||
- Add support for virtual hosted style addressing (#446, #449)
|
||||
- Support new param `frostfs.graceful_close_on_switch_timeout` (#475)
|
||||
- Support patch object method (#479)
|
||||
- Add `sign` command to `frostfs-s3-authmate` (#467)
|
||||
- Support for separate container for all CORS settings (#422)
|
||||
- `X-Amz-Force-Delete-Bucket` header for forced bucket removal (#31)
|
||||
- `Location` support in CompleteMultipart response (#451)
|
||||
- Tree pool request duration metric (#447)
|
||||
- Expiration lifecycle configuration support (#42, #412, #459, #460, #516, #536)
|
||||
- Add support for virtual hosted style addressing (#446, #449, #493)
|
||||
- Support `frostfs.graceful_close_on_switch_timeout` (#475)
|
||||
- Vulnerability report document (#413)
|
||||
- Support patch object method (#462, #473, #466, #479)
|
||||
- Enhanced logging and request reproducer (#369)
|
||||
- Root CA configuration for tracing (#484)
|
||||
- Log sampling policy configuration (#461)
|
||||
- `sign` command to `frostfs-s3-authmate` (#467)
|
||||
- Support custom aws credentials (#509)
|
||||
- Multinet dial support (#521)
|
||||
- Source IP binding configuration for FrostFS requests (#521)
|
||||
- Tracing attributes (#549)
|
||||
|
||||
### Changed
|
||||
- Update go version to go1.19 (#470)
|
||||
- Split `FrostFS` interface into separate read methods (#427)
|
||||
- golangci-lint v1.60 support (#474)
|
||||
- Updated Go version to 1.22 (#470)
|
||||
- Container removal after failed bucket creation (#434)
|
||||
- Explicit check for `.` symbol in bucket name (#506)
|
||||
- Transaction waiter in contract clients (#522)
|
||||
- Avoid maintenance mode storage node during object operations (#524)
|
||||
- Content-Type does not include in Presigned URL of s3-authmate (#505)
|
||||
- Check owner ID before deleting bucket (#528)
|
||||
- S3-Authmate now uses APE instead basic-ACL (#553)
|
||||
|
||||
### Removed
|
||||
- Reduce using mutex when update app settings (#329)
|
||||
|
||||
## [0.30.8] - 2024-10-18
|
||||
|
||||
### Fixed
|
||||
- Error handling for correct connection switch in SDK Pool (#517)
|
||||
|
||||
## [0.30.7] - 2024-10-03
|
||||
|
||||
### Fixed
|
||||
- Correct aws-chunk encoding size handling (#511)
|
||||
|
||||
|
||||
## [0.30.6] - 2024-09-17
|
||||
|
||||
### Fixed
|
||||
- Object size of objects upload with aws-chunked encoding (#450)
|
||||
- Object size of objects upload with negative Content-Length (#486)
|
||||
|
||||
## [0.30.5] - 2024-09-16
|
||||
|
||||
### Fixed
|
||||
- Panic catchers for fuzzing tests (#492)
|
||||
|
||||
## [0.30.4] - 2024-09-03
|
||||
|
||||
### Added
|
||||
- Fuzzing tests (#480)
|
||||
|
||||
## [0.30.3] - 2024-08-27
|
||||
|
||||
### Fixed
|
||||
- Empty listing when multipart upload contains more than 1000 parts (#471)
|
||||
|
||||
## [0.30.2] - 2024-08-20
|
||||
|
||||
### Fixed
|
||||
- Error counting in pool component before connection switch (#468)
|
||||
|
||||
### Added
|
||||
- Log of endpoint address during tree pool errors (#468)
|
||||
|
||||
## [0.30.1] - 2024-07-25
|
||||
|
||||
### Fixed
|
||||
- Redundant system node removal in tree service (#437)
|
||||
|
||||
### Added
|
||||
- Log details on SDK Pool health status change (#439)
|
||||
|
||||
## [0.30.0] - Kangshung -2024-07-19
|
||||
|
||||
|
@ -245,4 +333,13 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs
|
|||
[0.29.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.1...v0.29.2
|
||||
[0.29.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.2...v0.29.3
|
||||
[0.30.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.3...v0.30.0
|
||||
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.0...master
|
||||
[0.30.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.0...v0.30.1
|
||||
[0.30.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.1...v0.30.2
|
||||
[0.30.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.2...v0.30.3
|
||||
[0.30.4]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.3...v0.30.4
|
||||
[0.30.5]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.4...v0.30.5
|
||||
[0.30.6]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.5...v0.30.6
|
||||
[0.30.7]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.6...v0.30.7
|
||||
[0.30.8]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.7...v0.30.8
|
||||
[0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.8...v0.31.0
|
||||
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.0...master
|
2
VERSION
2
VERSION
|
@ -1 +1 @@
|
|||
v0.30.0
|
||||
v0.31.0
|
||||
|
|
|
@ -105,7 +105,7 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
|
|||
if reqInfo.BucketName == "" {
|
||||
return
|
||||
}
|
||||
bktInfo, err := h.obj.GetBucketInfo(ctx, reqInfo.BucketName)
|
||||
bktInfo, err := h.getBucketInfo(ctx, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err))
|
||||
return
|
||||
|
@ -154,7 +154,7 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
|
|||
func (h *handler) Preflight(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
bktInfo, err := h.obj.GetBucketInfo(ctx, reqInfo.BucketName)
|
||||
bktInfo, err := h.getBucketInfo(ctx, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
|
||||
return
|
||||
|
|
|
@ -276,6 +276,7 @@ func TestPutBucketLockConfigurationHandler(t *testing.T) {
|
|||
}{
|
||||
{
|
||||
name: "bkt not found",
|
||||
bucket: "not-found-bucket",
|
||||
expectedError: apierr.GetAPIError(apierr.ErrNoSuchBucket),
|
||||
},
|
||||
{
|
||||
|
@ -365,6 +366,7 @@ func TestGetBucketLockConfigurationHandler(t *testing.T) {
|
|||
}{
|
||||
{
|
||||
name: "bkt not found",
|
||||
bucket: "not-found-bucket",
|
||||
expectedError: apierr.GetAPIError(apierr.ErrNoSuchBucket),
|
||||
},
|
||||
{
|
||||
|
|
|
@ -102,7 +102,7 @@ func TestPatch(t *testing.T) {
|
|||
res := patchObject(t, tc, bktName, objName, tt.rng, patchPayload, tt.headers)
|
||||
require.Equal(t, data.Quote(hash), res.Object.ETag)
|
||||
} else {
|
||||
patchObjectErr(t, tc, bktName, objName, tt.rng, patchPayload, tt.headers, tt.code)
|
||||
patchObjectErr(tc, bktName, objName, tt.rng, patchPayload, tt.headers, tt.code)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
@ -377,7 +377,7 @@ func TestPatchEncryptedObject(t *testing.T) {
|
|||
tc.Handler().PutObjectHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
patchObjectErr(t, tc, bktName, objName, "bytes 2-4/*", []byte("new"), nil, apierr.ErrInternalError)
|
||||
patchObjectErr(tc, bktName, objName, "bytes 2-4/*", []byte("new"), nil, apierr.ErrInternalError)
|
||||
}
|
||||
|
||||
func TestPatchMissingHeaders(t *testing.T) {
|
||||
|
@ -402,6 +402,14 @@ func TestPatchMissingHeaders(t *testing.T) {
|
|||
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrMissingContentLength))
|
||||
}
|
||||
|
||||
func TestPatchInvalidBucketName(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
bktName, objName := "bucket", "object"
|
||||
createTestBucket(tc, bktName)
|
||||
|
||||
patchObjectErr(tc, "bkt_name", objName, "bytes 2-4/*", []byte("new"), nil, apierr.ErrInvalidBucketName)
|
||||
}
|
||||
|
||||
func TestParsePatchByteRange(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
rng string
|
||||
|
@ -501,9 +509,9 @@ func patchObjectVersion(t *testing.T, tc *handlerContext, bktName, objName, vers
|
|||
return result
|
||||
}
|
||||
|
||||
func patchObjectErr(t *testing.T, tc *handlerContext, bktName, objName, rng string, payload []byte, headers map[string]string, code apierr.ErrorCode) {
|
||||
func patchObjectErr(tc *handlerContext, bktName, objName, rng string, payload []byte, headers map[string]string, code apierr.ErrorCode) {
|
||||
w := patchObjectBase(tc, bktName, objName, "", rng, payload, headers)
|
||||
assertS3Error(t, w, apierr.GetAPIError(code))
|
||||
assertS3Error(tc.t, w, apierr.GetAPIError(code))
|
||||
}
|
||||
|
||||
func patchObjectBase(tc *handlerContext, bktName, objName, version, rng string, payload []byte, headers map[string]string) *httptest.ResponseRecorder {
|
||||
|
|
|
@ -55,15 +55,22 @@ func handleDeleteMarker(w http.ResponseWriter, err error) error {
|
|||
}
|
||||
|
||||
func (h *handler) ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error) {
|
||||
return h.obj.GetBucketInfo(ctx, bucket)
|
||||
return h.getBucketInfo(ctx, bucket)
|
||||
}
|
||||
|
||||
func (h *handler) ResolveCID(ctx context.Context, bucket string) (cid.ID, error) {
|
||||
return h.obj.ResolveCID(ctx, bucket)
|
||||
}
|
||||
|
||||
func (h *handler) getBucketInfo(ctx context.Context, bucket string) (*data.BucketInfo, error) {
|
||||
if err := checkBucketName(bucket); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return h.obj.GetBucketInfo(ctx, bucket)
|
||||
}
|
||||
|
||||
func (h *handler) getBucketAndCheckOwner(r *http.Request, bucket string, header ...string) (*data.BucketInfo, error) {
|
||||
bktInfo, err := h.obj.GetBucketInfo(r.Context(), bucket)
|
||||
bktInfo, err := h.getBucketInfo(r.Context(), bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -141,7 +141,6 @@ func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
|||
SessionToken: p.SessionContainerCreation,
|
||||
CreationTime: bktInfo.Created,
|
||||
AdditionalAttributes: attributes,
|
||||
BasicACL: 0, // means APE
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create container: %w", err)
|
||||
|
|
|
@ -9,13 +9,13 @@ import (
|
|||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||
)
|
||||
|
||||
// PrmContainerCreate groups parameters of FrostFS.CreateContainer operation.
|
||||
|
@ -38,13 +38,19 @@ type PrmContainerCreate struct {
|
|||
// Token of the container's creation session. Nil means session absence.
|
||||
SessionToken *session.Container
|
||||
|
||||
// Basic ACL of the container.
|
||||
BasicACL acl.Basic
|
||||
|
||||
// Attributes for optional parameters.
|
||||
AdditionalAttributes [][2]string
|
||||
}
|
||||
|
||||
// PrmAddContainerPolicyChain groups parameter of FrostFS.AddContainerPolicyChain operation.
|
||||
type PrmAddContainerPolicyChain struct {
|
||||
// ContainerID is a container identifier.
|
||||
ContainerID cid.ID
|
||||
|
||||
// Chain is Access Policy Engine chain that contains rules which provide access to specific actions in container.
|
||||
Chain chain.Chain
|
||||
}
|
||||
|
||||
// PrmContainer groups parameters of FrostFS.Container operation.
|
||||
type PrmContainer struct {
|
||||
// Container identifier.
|
||||
|
@ -239,6 +245,10 @@ type FrostFS interface {
|
|||
// prevented the container from being created.
|
||||
CreateContainer(context.Context, PrmContainerCreate) (*ContainerCreateResult, error)
|
||||
|
||||
// AddContainerPolicyChain create new policy chain for container.
|
||||
// Can be invoked only by container owner.
|
||||
AddContainerPolicyChain(context.Context, PrmAddContainerPolicyChain) error
|
||||
|
||||
// Container reads a container from FrostFS by ID.
|
||||
//
|
||||
// It returns exactly one non-nil value. It returns any error encountered which
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
@ -25,6 +26,7 @@ import (
|
|||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
)
|
||||
|
||||
|
@ -61,13 +63,14 @@ func (k *FeatureSettingsMock) FormContainerZone(ns string) string {
|
|||
return ns + ".ns"
|
||||
}
|
||||
|
||||
type TestFrostFS struct {
|
||||
frostfs.FrostFS
|
||||
var _ frostfs.FrostFS = (*TestFrostFS)(nil)
|
||||
|
||||
type TestFrostFS struct {
|
||||
objects map[string]*object.Object
|
||||
objectErrors map[string]error
|
||||
objectPutErrors map[string]error
|
||||
containers map[string]*container.Container
|
||||
chains map[string][]chain.Chain
|
||||
currentEpoch uint64
|
||||
key *keys.PrivateKey
|
||||
}
|
||||
|
@ -78,6 +81,7 @@ func NewTestFrostFS(key *keys.PrivateKey) *TestFrostFS {
|
|||
objectErrors: make(map[string]error),
|
||||
objectPutErrors: make(map[string]error),
|
||||
containers: make(map[string]*container.Container),
|
||||
chains: make(map[string][]chain.Chain),
|
||||
key: key,
|
||||
}
|
||||
}
|
||||
|
@ -145,7 +149,6 @@ func (t *TestFrostFS) CreateContainer(_ context.Context, prm frostfs.PrmContaine
|
|||
cnr.Init()
|
||||
cnr.SetOwner(prm.Creator)
|
||||
cnr.SetPlacementPolicy(prm.Policy)
|
||||
cnr.SetBasicACL(prm.BasicACL)
|
||||
|
||||
creationTime := prm.CreationTime
|
||||
if creationTime.IsZero() {
|
||||
|
@ -174,6 +177,7 @@ func (t *TestFrostFS) CreateContainer(_ context.Context, prm frostfs.PrmContaine
|
|||
var id cid.ID
|
||||
id.SetSHA256(sha256.Sum256(b))
|
||||
t.containers[id.EncodeToString()] = &cnr
|
||||
t.chains[id.EncodeToString()] = []chain.Chain{}
|
||||
|
||||
return &frostfs.ContainerCreateResult{ContainerID: id}, nil
|
||||
}
|
||||
|
@ -455,6 +459,17 @@ func (t *TestFrostFS) PatchObject(ctx context.Context, prm frostfs.PrmObjectPatc
|
|||
return newID, nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) AddContainerPolicyChain(_ context.Context, prm frostfs.PrmAddContainerPolicyChain) error {
|
||||
list, ok := t.chains[prm.ContainerID.EncodeToString()]
|
||||
if !ok {
|
||||
return errors.New("container not found")
|
||||
}
|
||||
|
||||
t.chains[prm.ContainerID.EncodeToString()] = append(list, prm.Chain)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) checkAccess(cnrID cid.ID, owner user.ID) bool {
|
||||
cnr, ok := t.containers[cnrID.EncodeToString()]
|
||||
if !ok {
|
||||
|
|
|
@ -14,9 +14,12 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl"
|
||||
sessionv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/retryer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
|
||||
|
@ -25,6 +28,8 @@ import (
|
|||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/retry"
|
||||
"github.com/google/uuid"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"go.uber.org/zap"
|
||||
|
@ -33,7 +38,7 @@ import (
|
|||
// PrmContainerCreate groups parameters of containers created by authmate.
|
||||
type PrmContainerCreate struct {
|
||||
// FrostFS identifier of the container creator.
|
||||
Owner user.ID
|
||||
Owner *keys.PublicKey
|
||||
|
||||
// Container placement policy.
|
||||
Policy netmap.PlacementPolicy
|
||||
|
@ -85,11 +90,56 @@ type FrostFS interface {
|
|||
type Agent struct {
|
||||
frostFS FrostFS
|
||||
log *zap.Logger
|
||||
cfg *config
|
||||
}
|
||||
|
||||
type config struct {
|
||||
RetryMaxAttempts int
|
||||
RetryMaxBackoff time.Duration
|
||||
RetryStrategy handler.RetryStrategy
|
||||
}
|
||||
|
||||
func defaultConfig() *config {
|
||||
return &config{
|
||||
RetryMaxAttempts: 4,
|
||||
RetryMaxBackoff: 30 * time.Second,
|
||||
RetryStrategy: handler.RetryStrategyExponential,
|
||||
}
|
||||
}
|
||||
|
||||
type Option func(cfg *config)
|
||||
|
||||
func WithRetryMaxAttempts(attempts int) func(*config) {
|
||||
return func(cfg *config) {
|
||||
cfg.RetryMaxAttempts = attempts
|
||||
}
|
||||
}
|
||||
|
||||
func WithRetryMaxBackoff(backoff time.Duration) func(*config) {
|
||||
return func(cfg *config) {
|
||||
cfg.RetryMaxBackoff = backoff
|
||||
}
|
||||
}
|
||||
|
||||
func WithRetryStrategy(strategy handler.RetryStrategy) func(*config) {
|
||||
return func(cfg *config) {
|
||||
cfg.RetryStrategy = strategy
|
||||
}
|
||||
}
|
||||
|
||||
// New creates an object of type Agent that consists of Client and logger.
|
||||
func New(log *zap.Logger, frostFS FrostFS) *Agent {
|
||||
return &Agent{log: log, frostFS: frostFS}
|
||||
func New(log *zap.Logger, frostFS FrostFS, options ...Option) *Agent {
|
||||
cfg := defaultConfig()
|
||||
|
||||
for _, opt := range options {
|
||||
opt(cfg)
|
||||
}
|
||||
|
||||
return &Agent{
|
||||
log: log,
|
||||
frostFS: frostFS,
|
||||
cfg: cfg,
|
||||
}
|
||||
}
|
||||
|
||||
type (
|
||||
|
@ -275,7 +325,13 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
|||
CustomAttributes: options.CustomAttributes,
|
||||
}
|
||||
|
||||
addr, err := creds.Put(ctx, prm)
|
||||
var addr oid.Address
|
||||
err = retryer.MakeWithRetry(ctx, func() error {
|
||||
var inErr error
|
||||
addr, inErr = creds.Put(ctx, prm)
|
||||
return inErr
|
||||
}, a.credsPutRetryer())
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to put creds: %w", err)
|
||||
}
|
||||
|
@ -431,6 +487,27 @@ func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSe
|
|||
return enc.Encode(or)
|
||||
}
|
||||
|
||||
func (a *Agent) credsPutRetryer() aws.RetryerV2 {
|
||||
return retry.NewStandard(func(options *retry.StandardOptions) {
|
||||
options.MaxAttempts = a.cfg.RetryMaxAttempts
|
||||
options.MaxBackoff = a.cfg.RetryMaxBackoff
|
||||
if a.cfg.RetryStrategy == handler.RetryStrategyExponential {
|
||||
options.Backoff = retry.NewExponentialJitterBackoff(options.MaxBackoff)
|
||||
} else {
|
||||
options.Backoff = retry.BackoffDelayerFunc(func(int, error) (time.Duration, error) {
|
||||
return options.MaxBackoff, nil
|
||||
})
|
||||
}
|
||||
|
||||
options.Retryables = []retry.IsErrorRetryable{retry.IsErrorRetryableFunc(func(err error) aws.Ternary {
|
||||
if errors.Is(err, frostfs.ErrAccessDenied) {
|
||||
return aws.TrueTernary
|
||||
}
|
||||
return aws.FalseTernary
|
||||
})}
|
||||
})
|
||||
}
|
||||
|
||||
func buildBearerToken(key *keys.PrivateKey, impersonate bool, lifetime lifetimeOptions, gateKey *keys.PublicKey) (*bearer.Token, error) {
|
||||
var ownerID user.ID
|
||||
user.IDFromKey(&ownerID, (ecdsa.PublicKey)(*gateKey))
|
||||
|
|
|
@ -7,6 +7,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
|
||||
|
@ -14,7 +15,6 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
|
@ -48,6 +48,9 @@ const (
|
|||
containerPolicyFlag = "container-policy"
|
||||
awsCLICredentialFlag = "aws-cli-credentials"
|
||||
attributesFlag = "attributes"
|
||||
retryMaxAttemptsFlag = "retry-max-attempts"
|
||||
retryMaxBackoffFlag = "retry-max-backoff"
|
||||
retryStrategyFlag = "retry-strategy"
|
||||
)
|
||||
|
||||
const walletPassphraseCfg = "wallet.passphrase"
|
||||
|
@ -59,6 +62,10 @@ const (
|
|||
defaultPoolHealthcheckTimeout = 5 * time.Second
|
||||
defaultPoolRebalanceInterval = 30 * time.Second
|
||||
defaultPoolStreamTimeout = 10 * time.Second
|
||||
|
||||
defaultRetryMaxAttempts = 4
|
||||
defaultRetryMaxBackoff = 30 * time.Second
|
||||
defaultRetryStrategy = handler.RetryStrategyExponential
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -91,6 +98,9 @@ func initIssueSecretCmd() {
|
|||
issueSecretCmd.Flags().String(accessKeyIDFlag, "", "Access key id of s3 credential that must be created")
|
||||
issueSecretCmd.Flags().String(secretAccessKeyFlag, "", "Secret access key of s3 credential that must be used")
|
||||
issueSecretCmd.Flags().String(rpcEndpointFlag, "", "NEO node RPC address (must be provided if container-id is nns name)")
|
||||
issueSecretCmd.Flags().Int(retryMaxAttemptsFlag, defaultRetryMaxAttempts, "Max amount of request attempts")
|
||||
issueSecretCmd.Flags().Duration(retryMaxBackoffFlag, defaultRetryMaxBackoff, "Max delay before next attempt")
|
||||
issueSecretCmd.Flags().String(retryStrategyFlag, defaultRetryStrategy, "Backoff strategy. `exponential` and `constant` are allowed")
|
||||
|
||||
_ = issueSecretCmd.MarkFlagRequired(walletFlag)
|
||||
_ = issueSecretCmd.MarkFlagRequired(peerFlag)
|
||||
|
@ -181,7 +191,13 @@ func runIssueSecretCmd(cmd *cobra.Command, _ []string) error {
|
|||
CustomAttributes: customAttrs,
|
||||
}
|
||||
|
||||
if err = authmate.New(log, frostFS).IssueSecret(ctx, os.Stdout, issueSecretOptions); err != nil {
|
||||
options := []authmate.Option{
|
||||
authmate.WithRetryMaxAttempts(viper.GetInt(retryMaxAttemptsFlag)),
|
||||
authmate.WithRetryMaxBackoff(viper.GetDuration(retryMaxBackoffFlag)),
|
||||
authmate.WithRetryStrategy(handler.RetryStrategy(viper.GetString(retryStrategyFlag))),
|
||||
}
|
||||
|
||||
if err = authmate.New(log, frostFS, options...).IssueSecret(ctx, os.Stdout, issueSecretOptions); err != nil {
|
||||
return wrapBusinessLogicError(fmt.Errorf("failed to issue secret: %s", err))
|
||||
}
|
||||
return nil
|
||||
|
@ -227,10 +243,9 @@ func createAccessBox(ctx context.Context, frostFS *frostfs.AuthmateFrostFS, key
|
|||
|
||||
prm := authmate.PrmContainerCreate{
|
||||
FriendlyName: friendlyName,
|
||||
Owner: key.PublicKey(),
|
||||
}
|
||||
|
||||
user.IDFromKey(&prm.Owner, key.PrivateKey.PublicKey)
|
||||
|
||||
if err := prm.Policy.DecodeString(placementPolicy); err != nil {
|
||||
return cid.ID{}, fmt.Errorf("failed to build placement policy: %w", err)
|
||||
}
|
||||
|
|
|
@ -3,6 +3,7 @@ package frostfs
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
|
@ -16,10 +17,12 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/crdt"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||
"git.frostfs.info/TrueCloudLab/policy-engine/schema/native"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -55,21 +58,61 @@ func (x *AuthmateFrostFS) TimeToEpoch(ctx context.Context, futureTime time.Time)
|
|||
|
||||
// CreateContainer implements authmate.FrostFS interface method.
|
||||
func (x *AuthmateFrostFS) CreateContainer(ctx context.Context, prm authmate.PrmContainerCreate) (cid.ID, error) {
|
||||
basicACL := acl.Private
|
||||
// allow reading objects to OTHERS in order to provide read access to S3 gateways
|
||||
basicACL.AllowOp(acl.OpObjectGet, acl.RoleOthers)
|
||||
basicACL.AllowOp(acl.OpObjectHead, acl.RoleOthers)
|
||||
basicACL.AllowOp(acl.OpObjectSearch, acl.RoleOthers)
|
||||
var owner user.ID
|
||||
owner.SetScriptHash(prm.Owner.GetScriptHash())
|
||||
|
||||
res, err := x.frostFS.CreateContainer(ctx, frostfs.PrmContainerCreate{
|
||||
Creator: prm.Owner,
|
||||
Policy: prm.Policy,
|
||||
Name: prm.FriendlyName,
|
||||
BasicACL: basicACL,
|
||||
Creator: owner,
|
||||
Policy: prm.Policy,
|
||||
Name: prm.FriendlyName,
|
||||
})
|
||||
if err != nil {
|
||||
return cid.ID{}, err
|
||||
}
|
||||
|
||||
ch := chain.Chain{
|
||||
ID: chain.ID("authmate/" + owner.String()),
|
||||
Rules: []chain.Rule{
|
||||
{
|
||||
Status: chain.Allow,
|
||||
Actions: chain.Actions{Names: []string{"*"}},
|
||||
Resources: chain.Resources{Names: []string{
|
||||
fmt.Sprintf(native.ResourceFormatRootContainer, res.ContainerID),
|
||||
fmt.Sprintf(native.ResourceFormatRootContainerObjects, res.ContainerID),
|
||||
}},
|
||||
Condition: []chain.Condition{{
|
||||
Op: chain.CondStringEquals,
|
||||
Kind: chain.KindRequest,
|
||||
Key: native.PropertyKeyActorPublicKey,
|
||||
Value: hex.EncodeToString(prm.Owner.Bytes()),
|
||||
}},
|
||||
},
|
||||
{
|
||||
Status: chain.Allow,
|
||||
Actions: chain.Actions{Names: []string{
|
||||
native.MethodGetContainer,
|
||||
native.MethodGetObject,
|
||||
native.MethodHeadObject,
|
||||
native.MethodSearchObject,
|
||||
native.MethodRangeObject,
|
||||
native.MethodHashObject,
|
||||
}},
|
||||
Resources: chain.Resources{Names: []string{
|
||||
fmt.Sprintf(native.ResourceFormatRootContainer, res.ContainerID),
|
||||
fmt.Sprintf(native.ResourceFormatRootContainerObjects, res.ContainerID),
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err = x.frostFS.AddContainerPolicyChain(ctx, frostfs.PrmAddContainerPolicyChain{
|
||||
ContainerID: res.ContainerID,
|
||||
Chain: ch,
|
||||
})
|
||||
if err != nil {
|
||||
return cid.ID{}, err
|
||||
}
|
||||
|
||||
return res.ContainerID, nil
|
||||
}
|
||||
|
||||
|
|
|
@ -47,7 +47,7 @@ func TestCredsObject(t *testing.T) {
|
|||
|
||||
cnrID, err := frostfs.CreateContainer(ctx, authmate.PrmContainerCreate{
|
||||
FriendlyName: bktName,
|
||||
Owner: userID,
|
||||
Owner: key.PublicKey(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
|
|
|
@ -12,6 +12,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
|
||||
frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
|
@ -93,7 +94,6 @@ func (x *FrostFS) CreateContainer(ctx context.Context, prm frostfs.PrmContainerC
|
|||
cnr.Init()
|
||||
cnr.SetPlacementPolicy(prm.Policy)
|
||||
cnr.SetOwner(prm.Creator)
|
||||
cnr.SetBasicACL(prm.BasicACL)
|
||||
|
||||
creationTime := prm.CreationTime
|
||||
if creationTime.IsZero() {
|
||||
|
@ -135,6 +135,25 @@ func (x *FrostFS) CreateContainer(ctx context.Context, prm frostfs.PrmContainerC
|
|||
}, handleObjectError("save container via connection pool", err)
|
||||
}
|
||||
|
||||
// AddContainerPolicyChain implements frostfs.FrostFS interface method.
|
||||
func (x *FrostFS) AddContainerPolicyChain(ctx context.Context, prm frostfs.PrmAddContainerPolicyChain) error {
|
||||
data, err := prm.Chain.MarshalBinary()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
prmAddAPEChain := pool.PrmAddAPEChain{
|
||||
Target: ape.ChainTarget{
|
||||
TargetType: ape.TargetTypeContainer,
|
||||
Name: prm.ContainerID.EncodeToString(),
|
||||
},
|
||||
Chain: ape.Chain{Raw: data},
|
||||
}
|
||||
|
||||
err = x.pool.AddAPEChain(ctx, prmAddAPEChain)
|
||||
return handleObjectError("add ape chain to container", err)
|
||||
}
|
||||
|
||||
// UserContainers implements layer.FrostFS interface method.
|
||||
func (x *FrostFS) UserContainers(ctx context.Context, layerPrm frostfs.PrmUserContainers) ([]cid.ID, error) {
|
||||
prm := pool.PrmContainerList{
|
||||
|
|
Loading…
Reference in a new issue