forked from TrueCloudLab/frostfs-s3-gw
Compare commits
25 commits
master
...
feature/96
Author | SHA1 | Date | |
---|---|---|---|
10bc3884af | |||
dea79631bb | |||
618ea318ba | |||
7ae4323116 | |||
c0f959ece7 | |||
0cf19d24ee | |||
188e0cfd01 | |||
50aeba6a4e | |||
c6d0e5fc44 | |||
377fa127b5 | |||
f02bad65a8 | |||
13d00dd7ce | |||
02122892ca | |||
272e3a0f03 | |||
65a8e2dadc | |||
b7e15402a1 | |||
0cc6b41b2d | |||
51be9d9778 | |||
1cad101609 | |||
d631ee55c9 | |||
85c8210ffd | |||
bcfbcdc82f | |||
fd310f5e9f | |||
1f5f2bd3d5 | |||
a32b41716f |
168 changed files with 10785 additions and 12840 deletions
|
@ -1,4 +1,4 @@
|
||||||
FROM golang:1.22 AS builder
|
FROM golang:1.21 as builder
|
||||||
|
|
||||||
ARG BUILD=now
|
ARG BUILD=now
|
||||||
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
|
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
|
||||||
|
|
|
@ -6,7 +6,7 @@ jobs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go_versions: [ '1.22', '1.23' ]
|
go_versions: [ '1.20', '1.21' ]
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
|
|
|
@ -12,7 +12,7 @@ jobs:
|
||||||
- name: Setup Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@v3
|
uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: '1.23'
|
go-version: '1.21'
|
||||||
|
|
||||||
- name: Run commit format checker
|
- name: Run commit format checker
|
||||||
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3
|
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3
|
||||||
|
|
|
@ -10,7 +10,7 @@ jobs:
|
||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
uses: actions/setup-go@v3
|
uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: '1.23'
|
go-version: '1.21'
|
||||||
cache: true
|
cache: true
|
||||||
|
|
||||||
- name: Install linters
|
- name: Install linters
|
||||||
|
@ -24,7 +24,7 @@ jobs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go_versions: [ '1.22', '1.23' ]
|
go_versions: [ '1.20', '1.21' ]
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
|
|
|
@ -12,7 +12,7 @@ jobs:
|
||||||
- name: Setup Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@v3
|
uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: '1.23'
|
go-version: '1.21'
|
||||||
|
|
||||||
- name: Install govulncheck
|
- name: Install govulncheck
|
||||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||||
|
|
|
@ -12,8 +12,7 @@ run:
|
||||||
# output configuration options
|
# output configuration options
|
||||||
output:
|
output:
|
||||||
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
|
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
|
||||||
formats:
|
format: tab
|
||||||
- format: tab
|
|
||||||
|
|
||||||
# all available settings of specific linters
|
# all available settings of specific linters
|
||||||
linters-settings:
|
linters-settings:
|
||||||
|
|
68
CHANGELOG.md
68
CHANGELOG.md
|
@ -4,68 +4,6 @@ This document outlines major changes between releases.
|
||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
### Added
|
|
||||||
- Add support for virtual hosted style addressing (#446, #449)
|
|
||||||
- Support new param `frostfs.graceful_close_on_switch_timeout` (#475)
|
|
||||||
- Support patch object method (#479)
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
- Update go version to go1.19 (#470)
|
|
||||||
|
|
||||||
## [0.30.0] - Kangshung -2024-07-19
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Fix HTTP/2 requests (#341)
|
|
||||||
- Fix Decoder.CharsetReader is nil (#379)
|
|
||||||
- Fix flaky ACL encode test (#340)
|
|
||||||
- Docs grammar (#432)
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- Add new `reconnect_interval` config param for server rebinding (#291)
|
|
||||||
- Support `GetBucketPolicyStatus` (#301)
|
|
||||||
- Support request IP filter with policy (#371, #377)
|
|
||||||
- Support tag checks in policies (#357, #365, #392, #403, #411)
|
|
||||||
- Support IAM-MFA checks (#367)
|
|
||||||
- More docs (#334, #353)
|
|
||||||
- Add `register-user` command to `authmate` (#414)
|
|
||||||
- `User` field in request log (#396)
|
|
||||||
- Erasure coding support in placement policy (#400)
|
|
||||||
- Improved test coverage (#402)
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
- Update dependencies noted by govulncheck (#368)
|
|
||||||
- Improve test coverage (#380, #387)
|
|
||||||
- Support updated naming in native policy JSON (#385)
|
|
||||||
- Improve determining AccessBox latest version (#335)
|
|
||||||
- Don't set full_control policy for bucket owner (#407)
|
|
||||||
|
|
||||||
### Removed
|
|
||||||
- Remove control api (#406)
|
|
||||||
- Remove notifications (#401)
|
|
||||||
- Remove `layer.Client` interface (#410)
|
|
||||||
- Remove extended ACL related code (#372)
|
|
||||||
|
|
||||||
## [0.29.3] - 2024-07-19
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Support tree split environment when multiple nodes
|
|
||||||
may be part of the same sub path (#430)
|
|
||||||
- Collision of multipart name and system data in the tree (#430)
|
|
||||||
- Workaround for removal of multiple null versions in unversioned bucket (#430)
|
|
||||||
|
|
||||||
## [0.29.2] - 2024-07-03
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Parsing of put-bucket-setting retry configuration (#398)
|
|
||||||
|
|
||||||
## [0.29.1] - 2024-06-20
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- OPTIONS request processing for object operations (#399)
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- Retries of put-bucket-setting operation during container creation (#398)
|
|
||||||
|
|
||||||
## [0.29.0] - Zemu - 2024-05-27
|
## [0.29.0] - Zemu - 2024-05-27
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
@ -237,8 +175,4 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs
|
||||||
[0.28.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.28.0...v0.28.1
|
[0.28.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.28.0...v0.28.1
|
||||||
[0.28.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.28.1...v0.28.2
|
[0.28.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.28.1...v0.28.2
|
||||||
[0.29.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.28.2...v0.29.0
|
[0.29.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.28.2...v0.29.0
|
||||||
[0.29.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.0...v0.29.1
|
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.0...master
|
||||||
[0.29.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.1...v0.29.2
|
|
||||||
[0.29.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.2...v0.29.3
|
|
||||||
[0.30.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.3...v0.30.0
|
|
||||||
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.0...master
|
|
||||||
|
|
40
Makefile
40
Makefile
|
@ -3,9 +3,9 @@
|
||||||
# Common variables
|
# Common variables
|
||||||
REPO ?= $(shell go list -m)
|
REPO ?= $(shell go list -m)
|
||||||
VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")
|
VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")
|
||||||
GO_VERSION ?= 1.22
|
GO_VERSION ?= 1.20
|
||||||
LINT_VERSION ?= 1.60.1
|
LINT_VERSION ?= 1.56.1
|
||||||
TRUECLOUDLAB_LINT_VERSION ?= 0.0.6
|
TRUECLOUDLAB_LINT_VERSION ?= 0.0.5
|
||||||
BINDIR = bin
|
BINDIR = bin
|
||||||
|
|
||||||
METRICS_DUMP_OUT ?= ./metrics-dump.json
|
METRICS_DUMP_OUT ?= ./metrics-dump.json
|
||||||
|
@ -23,12 +23,6 @@ OUTPUT_LINT_DIR ?= $(shell pwd)/bin
|
||||||
LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
|
LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
|
||||||
TMP_DIR := .cache
|
TMP_DIR := .cache
|
||||||
|
|
||||||
# Variables for fuzzing
|
|
||||||
FUZZ_NGFUZZ_DIR ?= ""
|
|
||||||
FUZZ_TIMEOUT ?= 30
|
|
||||||
FUZZ_FUNCTIONS ?= "all"
|
|
||||||
FUZZ_AUX ?= ""
|
|
||||||
|
|
||||||
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
|
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
|
||||||
|
|
||||||
# .deb package versioning
|
# .deb package versioning
|
||||||
|
@ -82,34 +76,6 @@ cover:
|
||||||
@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
|
@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
|
||||||
@go tool cover -html=coverage.txt -o coverage.html
|
@go tool cover -html=coverage.txt -o coverage.html
|
||||||
|
|
||||||
# Run fuzzing
|
|
||||||
CLANG := $(shell which clang-17 2>/dev/null)
|
|
||||||
.PHONY: check-clang all
|
|
||||||
check-clang:
|
|
||||||
ifeq ($(CLANG),)
|
|
||||||
@echo "clang-17 is not installed. Please install it before proceeding - https://apt.llvm.org/llvm.sh "
|
|
||||||
@exit 1
|
|
||||||
endif
|
|
||||||
|
|
||||||
.PHONY: check-ngfuzz all
|
|
||||||
check-ngfuzz:
|
|
||||||
@if [ -z "$(FUZZ_NGFUZZ_DIR)" ]; then \
|
|
||||||
echo "Please set a variable FUZZ_NGFUZZ_DIR to specify path to the ngfuzz"; \
|
|
||||||
exit 1; \
|
|
||||||
fi
|
|
||||||
|
|
||||||
.PHONY: install-fuzzing-deps
|
|
||||||
install-fuzzing-deps: check-clang check-ngfuzz
|
|
||||||
|
|
||||||
.PHONY: fuzz
|
|
||||||
fuzz: install-fuzzing-deps
|
|
||||||
@START_PATH=$$(pwd); \
|
|
||||||
ROOT_PATH=$$(realpath --relative-to=$(FUZZ_NGFUZZ_DIR) $$START_PATH) ; \
|
|
||||||
cd $(FUZZ_NGFUZZ_DIR) && \
|
|
||||||
./ngfuzz -clean && \
|
|
||||||
./ngfuzz -fuzz $(FUZZ_FUNCTIONS) -rootdir $$ROOT_PATH -timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \
|
|
||||||
./ngfuzz -report
|
|
||||||
|
|
||||||
# Reformat code
|
# Reformat code
|
||||||
format:
|
format:
|
||||||
@echo "⇒ Processing gofmt check"
|
@echo "⇒ Processing gofmt check"
|
||||||
|
|
18
README.md
18
README.md
|
@ -93,24 +93,6 @@ HTTP/1.1 200 OK
|
||||||
|
|
||||||
Also, you can configure domains using `.env` variables or `yaml` file.
|
Also, you can configure domains using `.env` variables or `yaml` file.
|
||||||
|
|
||||||
## Fuzzing
|
|
||||||
To run fuzzing tests use the following command:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
$ make fuzz
|
|
||||||
```
|
|
||||||
|
|
||||||
This command will install dependencies for the fuzzing process and run existing fuzzing tests.
|
|
||||||
|
|
||||||
You can also use the following arguments:
|
|
||||||
|
|
||||||
```
|
|
||||||
FUZZ_TIMEOUT - time to run each fuzzing test (default 30)
|
|
||||||
FUZZ_FUNCTIONS - fuzzing tests that will be started (default "all")
|
|
||||||
FUZZ_AUX - additional parameters for the fuzzer (for example, "-debug")
|
|
||||||
FUZZ_NGFUZZ_DIR - path to ngfuzz tool
|
|
||||||
````
|
|
||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
- [Configuration](./docs/configuration.md)
|
- [Configuration](./docs/configuration.md)
|
||||||
|
|
26
SECURITY.md
26
SECURITY.md
|
@ -1,26 +0,0 @@
|
||||||
# Security Policy
|
|
||||||
|
|
||||||
|
|
||||||
## How To Report a Vulnerability
|
|
||||||
|
|
||||||
If you think you have found a vulnerability in this repository, please report it to us through coordinated disclosure.
|
|
||||||
|
|
||||||
**Please do not report security vulnerabilities through public issues, discussions, or change requests.**
|
|
||||||
|
|
||||||
Instead, you can report it using one of the following ways:
|
|
||||||
|
|
||||||
* Contact the [TrueCloudLab Security Team](mailto:security@frostfs.info) via email
|
|
||||||
|
|
||||||
Please include as much of the information listed below as you can to help us better understand and resolve the issue:
|
|
||||||
|
|
||||||
* The type of issue (e.g., buffer overflow, or cross-site scripting)
|
|
||||||
* Affected version(s)
|
|
||||||
* Impact of the issue, including how an attacker might exploit the issue
|
|
||||||
* Step-by-step instructions to reproduce the issue
|
|
||||||
* The location of the affected source code (tag/branch/commit or direct URL)
|
|
||||||
* Full paths of source file(s) related to the manifestation of the issue
|
|
||||||
* Any special configuration required to reproduce the issue
|
|
||||||
* Any log files that are related to this issue (if possible)
|
|
||||||
* Proof-of-concept or exploit code (if possible)
|
|
||||||
|
|
||||||
This information will help us triage your report more quickly.
|
|
2
VERSION
2
VERSION
|
@ -1 +1 @@
|
||||||
v0.30.0
|
v0.29.0
|
||||||
|
|
|
@ -186,7 +186,7 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
box, attrs, err := c.cli.GetBox(r.Context(), addr)
|
box, err := c.cli.GetBox(r.Context(), addr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("get box '%s': %w", addr, err)
|
return nil, fmt.Errorf("get box '%s': %w", addr, err)
|
||||||
}
|
}
|
||||||
|
@ -207,7 +207,6 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
|
||||||
Region: authHdr.Region,
|
Region: authHdr.Region,
|
||||||
SignatureV4: authHdr.SignatureV4,
|
SignatureV4: authHdr.SignatureV4,
|
||||||
},
|
},
|
||||||
Attributes: attrs,
|
|
||||||
}
|
}
|
||||||
if needClientTime {
|
if needClientTime {
|
||||||
result.ClientTime = signatureDateTime
|
result.ClientTime = signatureDateTime
|
||||||
|
@ -270,14 +269,12 @@ func (c *Center) checkFormData(r *http.Request) (*middleware.Box, error) {
|
||||||
return nil, fmt.Errorf("failed to parse x-amz-date field: %w", err)
|
return nil, fmt.Errorf("failed to parse x-amz-date field: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
accessKeyID := submatches["access_key_id"]
|
addr, err := getAddress(submatches["access_key_id"])
|
||||||
|
|
||||||
addr, err := getAddress(accessKeyID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
box, attrs, err := c.cli.GetBox(r.Context(), addr)
|
box, err := c.cli.GetBox(r.Context(), addr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("get box '%s': %w", addr, err)
|
return nil, fmt.Errorf("get box '%s': %w", addr, err)
|
||||||
}
|
}
|
||||||
|
@ -285,22 +282,14 @@ func (c *Center) checkFormData(r *http.Request) (*middleware.Box, error) {
|
||||||
secret := box.Gate.SecretKey
|
secret := box.Gate.SecretKey
|
||||||
service, region := submatches["service"], submatches["region"]
|
service, region := submatches["service"], submatches["region"]
|
||||||
|
|
||||||
signature := SignStr(secret, service, region, signatureDateTime, policy)
|
signature := signStr(secret, service, region, signatureDateTime, policy)
|
||||||
reqSignature := MultipartFormValue(r, "x-amz-signature")
|
reqSignature := MultipartFormValue(r, "x-amz-signature")
|
||||||
if signature != reqSignature {
|
if signature != reqSignature {
|
||||||
return nil, fmt.Errorf("%w: %s != %s", apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch),
|
return nil, fmt.Errorf("%w: %s != %s", apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch),
|
||||||
reqSignature, signature)
|
reqSignature, signature)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &middleware.Box{
|
return &middleware.Box{AccessBox: box}, nil
|
||||||
AccessBox: box,
|
|
||||||
AuthHeaders: &middleware.AuthHeader{
|
|
||||||
AccessKeyID: accessKeyID,
|
|
||||||
Region: region,
|
|
||||||
SignatureV4: signature,
|
|
||||||
},
|
|
||||||
Attributes: attrs,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneRequest(r *http.Request, authHeader *AuthHeader) *http.Request {
|
func cloneRequest(r *http.Request, authHeader *AuthHeader) *http.Request {
|
||||||
|
@ -359,7 +348,7 @@ func (c *Center) checkSign(authHeader *AuthHeader, box *accessbox.Box, request *
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func SignStr(secret, service, region string, t time.Time, strToSign string) string {
|
func signStr(secret, service, region string, t time.Time, strToSign string) string {
|
||||||
creds := deriveKey(secret, service, region, t)
|
creds := deriveKey(secret, service, region, t)
|
||||||
signature := hmacSHA256(creds, []byte(strToSign))
|
signature := hmacSHA256(creds, []byte(strToSign))
|
||||||
return hex.EncodeToString(signature)
|
return hex.EncodeToString(signature)
|
||||||
|
|
|
@ -1,88 +0,0 @@
|
||||||
//go:build gofuzz
|
|
||||||
// +build gofuzz
|
|
||||||
|
|
||||||
package auth
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
||||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
|
||||||
utils "github.com/trailofbits/go-fuzz-utils"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
fuzzSuccessExitCode = 0
|
|
||||||
fuzzFailExitCode = -1
|
|
||||||
)
|
|
||||||
|
|
||||||
func InitFuzzAuthenticate() {
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzAuthenticate(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
var accessKeyAddr oid.Address
|
|
||||||
err = tp.Fill(accessKeyAddr)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
accessKeyID := strings.ReplaceAll(accessKeyAddr.String(), "/", "0")
|
|
||||||
secretKey, err := tp.GetString()
|
|
||||||
awsCreds := credentials.NewStaticCredentials(accessKeyID, secretKey, "")
|
|
||||||
|
|
||||||
reqData := RequestData{
|
|
||||||
Method: "GET",
|
|
||||||
Endpoint: "http://localhost:8084",
|
|
||||||
Bucket: "my-bucket",
|
|
||||||
Object: "@obj/name",
|
|
||||||
}
|
|
||||||
presignData := PresignData{
|
|
||||||
Service: "s3",
|
|
||||||
Region: "spb",
|
|
||||||
Lifetime: 10 * time.Minute,
|
|
||||||
SignTime: time.Now().UTC(),
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := PresignRequest(awsCreds, reqData, presignData)
|
|
||||||
if req == nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
expBox := &accessbox.Box{
|
|
||||||
Gate: &accessbox.GateData{
|
|
||||||
SecretKey: secretKey,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
mock := newTokensFrostfsMock()
|
|
||||||
mock.addBox(accessKeyAddr, expBox)
|
|
||||||
|
|
||||||
c := &Center{
|
|
||||||
cli: mock,
|
|
||||||
reg: NewRegexpMatcher(authorizationFieldRegexp),
|
|
||||||
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
|
|
||||||
}
|
|
||||||
|
|
||||||
_, _ = c.Authenticate(req)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzAuthenticate(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzAuthenticate(data)
|
|
||||||
})
|
|
||||||
}
|
|
|
@ -1,31 +1,12 @@
|
||||||
package auth
|
package auth
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"mime/multipart"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
|
||||||
frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
||||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
|
||||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap/zaptest"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestAuthHeaderParse(t *testing.T) {
|
func TestAuthHeaderParse(t *testing.T) {
|
||||||
|
@ -115,7 +96,7 @@ func TestSignature(t *testing.T) {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
signature := SignStr(secret, "s3", "us-east-1", signTime, strToSign)
|
signature := signStr(secret, "s3", "us-east-1", signTime, strToSign)
|
||||||
require.Equal(t, "dfbe886241d9e369cf4b329ca0f15eb27306c97aa1022cc0bb5a914c4ef87634", signature)
|
require.Equal(t, "dfbe886241d9e369cf4b329ca0f15eb27306c97aa1022cc0bb5a914c4ef87634", signature)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -142,11 +123,6 @@ func TestCheckFormatContentSHA256(t *testing.T) {
|
||||||
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f7s",
|
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f7s",
|
||||||
error: defaultErr,
|
error: defaultErr,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "invalid hash format: hash size",
|
|
||||||
hash: "5aadb45520dcd8726b2822a7a78bb53d794f557199d5d4abdedd2c55a4bd6ca73607605c558de3db80c8e86c3196484566163ed1327e82e8b6757d1932113cb8",
|
|
||||||
error: defaultErr,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "unsigned payload",
|
name: "unsigned payload",
|
||||||
hash: "UNSIGNED-PAYLOAD",
|
hash: "UNSIGNED-PAYLOAD",
|
||||||
|
@ -169,467 +145,3 @@ func TestCheckFormatContentSHA256(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type frostFSMock struct {
|
|
||||||
objects map[oid.Address]*object.Object
|
|
||||||
}
|
|
||||||
|
|
||||||
func newFrostFSMock() *frostFSMock {
|
|
||||||
return &frostFSMock{
|
|
||||||
objects: map[oid.Address]*object.Object{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *frostFSMock) GetCredsObject(_ context.Context, address oid.Address) (*object.Object, error) {
|
|
||||||
obj, ok := f.objects[address]
|
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
return obj, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *frostFSMock) CreateObject(context.Context, tokens.PrmObjectCreate) (oid.ID, error) {
|
|
||||||
return oid.ID{}, fmt.Errorf("the mock method is not implemented")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAuthenticate(t *testing.T) {
|
|
||||||
key, err := keys.NewPrivateKey()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
cfg := &cache.Config{
|
|
||||||
Size: 10,
|
|
||||||
Lifetime: 24 * time.Hour,
|
|
||||||
Logger: zaptest.NewLogger(t),
|
|
||||||
}
|
|
||||||
|
|
||||||
gateData := []*accessbox.GateData{{
|
|
||||||
BearerToken: &bearer.Token{},
|
|
||||||
GateKey: key.PublicKey(),
|
|
||||||
}}
|
|
||||||
|
|
||||||
accessBox, secret, err := accessbox.PackTokens(gateData, []byte("secret"))
|
|
||||||
require.NoError(t, err)
|
|
||||||
data, err := accessBox.Marshal()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var obj object.Object
|
|
||||||
obj.SetPayload(data)
|
|
||||||
addr := oidtest.Address()
|
|
||||||
obj.SetContainerID(addr.Container())
|
|
||||||
obj.SetID(addr.Object())
|
|
||||||
|
|
||||||
frostfs := newFrostFSMock()
|
|
||||||
frostfs.objects[addr] = &obj
|
|
||||||
|
|
||||||
accessKeyID := addr.Container().String() + "0" + addr.Object().String()
|
|
||||||
|
|
||||||
awsCreds := credentials.NewStaticCredentials(accessKeyID, secret.SecretKey, "")
|
|
||||||
defaultSigner := v4.NewSigner(awsCreds)
|
|
||||||
|
|
||||||
service, region := "s3", "default"
|
|
||||||
invalidValue := "invalid-value"
|
|
||||||
|
|
||||||
bigConfig := tokens.Config{
|
|
||||||
FrostFS: frostfs,
|
|
||||||
Key: key,
|
|
||||||
CacheConfig: cfg,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
prefixes []string
|
|
||||||
request *http.Request
|
|
||||||
err bool
|
|
||||||
errCode errors.ErrorCode
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "valid sign",
|
|
||||||
prefixes: []string{addr.Container().String()},
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
_, err = defaultSigner.Sign(r, nil, service, region, time.Now())
|
|
||||||
require.NoError(t, err)
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no authorization header",
|
|
||||||
request: func() *http.Request {
|
|
||||||
return httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid authorization header",
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
r.Header.Set(AuthorizationHdr, invalidValue)
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
errCode: errors.ErrAuthorizationHeaderMalformed,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid access key id format",
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
signer := v4.NewSigner(credentials.NewStaticCredentials(addr.Object().String(), secret.SecretKey, ""))
|
|
||||||
_, err = signer.Sign(r, nil, service, region, time.Now())
|
|
||||||
require.NoError(t, err)
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
errCode: errors.ErrInvalidAccessKeyID,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "not allowed access key id",
|
|
||||||
prefixes: []string{addr.Object().String()},
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
_, err = defaultSigner.Sign(r, nil, service, region, time.Now())
|
|
||||||
require.NoError(t, err)
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
errCode: errors.ErrAccessDenied,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid access key id value",
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
signer := v4.NewSigner(credentials.NewStaticCredentials(accessKeyID[:len(accessKeyID)-4], secret.SecretKey, ""))
|
|
||||||
_, err = signer.Sign(r, nil, service, region, time.Now())
|
|
||||||
require.NoError(t, err)
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
errCode: errors.ErrInvalidAccessKeyID,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unknown access key id",
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
signer := v4.NewSigner(credentials.NewStaticCredentials(addr.Object().String()+"0"+addr.Container().String(), secret.SecretKey, ""))
|
|
||||||
_, err = signer.Sign(r, nil, service, region, time.Now())
|
|
||||||
require.NoError(t, err)
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid signature",
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
signer := v4.NewSigner(credentials.NewStaticCredentials(accessKeyID, "secret", ""))
|
|
||||||
_, err = signer.Sign(r, nil, service, region, time.Now())
|
|
||||||
require.NoError(t, err)
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
errCode: errors.ErrSignatureDoesNotMatch,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid signature - AmzDate",
|
|
||||||
prefixes: []string{addr.Container().String()},
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
_, err = defaultSigner.Sign(r, nil, service, region, time.Now())
|
|
||||||
r.Header.Set(AmzDate, invalidValue)
|
|
||||||
require.NoError(t, err)
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid AmzContentSHA256",
|
|
||||||
prefixes: []string{addr.Container().String()},
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
_, err = defaultSigner.Sign(r, nil, service, region, time.Now())
|
|
||||||
r.Header.Set(AmzContentSHA256, invalidValue)
|
|
||||||
require.NoError(t, err)
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid presign",
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
_, err = defaultSigner.Presign(r, nil, service, region, time.Minute, time.Now())
|
|
||||||
require.NoError(t, err)
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "presign, bad X-Amz-Credential",
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
query := url.Values{
|
|
||||||
AmzAlgorithm: []string{"AWS4-HMAC-SHA256"},
|
|
||||||
AmzCredential: []string{invalidValue},
|
|
||||||
}
|
|
||||||
r.URL.RawQuery = query.Encode()
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "presign, bad X-Amz-Expires",
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
_, err = defaultSigner.Presign(r, nil, service, region, time.Minute, time.Now())
|
|
||||||
queryParams := r.URL.Query()
|
|
||||||
queryParams.Set("X-Amz-Expires", invalidValue)
|
|
||||||
r.URL.RawQuery = queryParams.Encode()
|
|
||||||
require.NoError(t, err)
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "presign, expired",
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
_, err = defaultSigner.Presign(r, nil, service, region, time.Minute, time.Now().Add(-time.Minute))
|
|
||||||
require.NoError(t, err)
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
errCode: errors.ErrExpiredPresignRequest,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "presign, signature from future",
|
|
||||||
request: func() *http.Request {
|
|
||||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
_, err = defaultSigner.Presign(r, nil, service, region, time.Minute, time.Now().Add(time.Minute))
|
|
||||||
require.NoError(t, err)
|
|
||||||
return r
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
errCode: errors.ErrBadRequest,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
creds := tokens.New(bigConfig)
|
|
||||||
cntr := New(creds, tc.prefixes)
|
|
||||||
box, err := cntr.Authenticate(tc.request)
|
|
||||||
|
|
||||||
if tc.err {
|
|
||||||
require.Error(t, err)
|
|
||||||
if tc.errCode > 0 {
|
|
||||||
err = frostfsErrors.UnwrapErr(err)
|
|
||||||
require.Equal(t, errors.GetAPIError(tc.errCode), err)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, accessKeyID, box.AuthHeaders.AccessKeyID)
|
|
||||||
require.Equal(t, region, box.AuthHeaders.Region)
|
|
||||||
require.Equal(t, secret.SecretKey, box.AccessBox.Gate.SecretKey)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHTTPPostAuthenticate(t *testing.T) {
|
|
||||||
const (
|
|
||||||
policyBase64 = "eyJleHBpcmF0aW9uIjogIjIwMjUtMTItMDFUMTI6MDA6MDAuMDAwWiIsImNvbmRpdGlvbnMiOiBbCiBbInN0YXJ0cy13aXRoIiwgIiR4LWFtei1jcmVkZW50aWFsIiwgIiJdLAogWyJzdGFydHMtd2l0aCIsICIkeC1hbXotZGF0ZSIsICIiXQpdfQ=="
|
|
||||||
invalidValue = "invalid-value"
|
|
||||||
defaultFieldName = "file"
|
|
||||||
service = "s3"
|
|
||||||
region = "default"
|
|
||||||
)
|
|
||||||
|
|
||||||
key, err := keys.NewPrivateKey()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
cfg := &cache.Config{
|
|
||||||
Size: 10,
|
|
||||||
Lifetime: 24 * time.Hour,
|
|
||||||
Logger: zaptest.NewLogger(t),
|
|
||||||
}
|
|
||||||
|
|
||||||
gateData := []*accessbox.GateData{{
|
|
||||||
BearerToken: &bearer.Token{},
|
|
||||||
GateKey: key.PublicKey(),
|
|
||||||
}}
|
|
||||||
|
|
||||||
accessBox, secret, err := accessbox.PackTokens(gateData, []byte("secret"))
|
|
||||||
require.NoError(t, err)
|
|
||||||
data, err := accessBox.Marshal()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var obj object.Object
|
|
||||||
obj.SetPayload(data)
|
|
||||||
addr := oidtest.Address()
|
|
||||||
obj.SetContainerID(addr.Container())
|
|
||||||
obj.SetID(addr.Object())
|
|
||||||
|
|
||||||
frostfs := newFrostFSMock()
|
|
||||||
frostfs.objects[addr] = &obj
|
|
||||||
|
|
||||||
accessKeyID := addr.Container().String() + "0" + addr.Object().String()
|
|
||||||
invalidAccessKeyID := oidtest.Address().String() + "0" + oidtest.Address().Object().String()
|
|
||||||
|
|
||||||
timeToSign := time.Now()
|
|
||||||
timeToSignStr := timeToSign.Format("20060102T150405Z")
|
|
||||||
|
|
||||||
bigConfig := tokens.Config{
|
|
||||||
FrostFS: frostfs,
|
|
||||||
Key: key,
|
|
||||||
CacheConfig: cfg,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
prefixes []string
|
|
||||||
request *http.Request
|
|
||||||
err bool
|
|
||||||
errCode errors.ErrorCode
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "HTTP POST valid",
|
|
||||||
request: func() *http.Request {
|
|
||||||
creds := getCredsStr(accessKeyID, timeToSignStr, region, service)
|
|
||||||
sign := SignStr(secret.SecretKey, service, region, timeToSign, policyBase64)
|
|
||||||
|
|
||||||
return getRequestWithMultipartForm(t, policyBase64, creds, timeToSignStr, sign, defaultFieldName)
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "HTTP POST valid with custom field name",
|
|
||||||
request: func() *http.Request {
|
|
||||||
creds := getCredsStr(accessKeyID, timeToSignStr, region, service)
|
|
||||||
sign := SignStr(secret.SecretKey, service, region, timeToSign, policyBase64)
|
|
||||||
|
|
||||||
return getRequestWithMultipartForm(t, policyBase64, creds, timeToSignStr, sign, "files")
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "HTTP POST valid with field name with a capital letter",
|
|
||||||
request: func() *http.Request {
|
|
||||||
creds := getCredsStr(accessKeyID, timeToSignStr, region, service)
|
|
||||||
sign := SignStr(secret.SecretKey, service, region, timeToSign, policyBase64)
|
|
||||||
|
|
||||||
return getRequestWithMultipartForm(t, policyBase64, creds, timeToSignStr, sign, "File")
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "HTTP POST invalid multipart form",
|
|
||||||
request: func() *http.Request {
|
|
||||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
||||||
req.Header.Set(ContentTypeHdr, "multipart/form-data")
|
|
||||||
|
|
||||||
return req
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
errCode: errors.ErrInvalidArgument,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "HTTP POST invalid signature date time",
|
|
||||||
request: func() *http.Request {
|
|
||||||
creds := getCredsStr(accessKeyID, timeToSignStr, region, service)
|
|
||||||
sign := SignStr(secret.SecretKey, service, region, timeToSign, policyBase64)
|
|
||||||
|
|
||||||
return getRequestWithMultipartForm(t, policyBase64, creds, invalidValue, sign, defaultFieldName)
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "HTTP POST invalid creds",
|
|
||||||
request: func() *http.Request {
|
|
||||||
sign := SignStr(secret.SecretKey, service, region, timeToSign, policyBase64)
|
|
||||||
|
|
||||||
return getRequestWithMultipartForm(t, policyBase64, invalidValue, timeToSignStr, sign, defaultFieldName)
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
errCode: errors.ErrAuthorizationHeaderMalformed,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "HTTP POST missing policy",
|
|
||||||
request: func() *http.Request {
|
|
||||||
creds := getCredsStr(accessKeyID, timeToSignStr, region, service)
|
|
||||||
sign := SignStr(secret.SecretKey, service, region, timeToSign, policyBase64)
|
|
||||||
|
|
||||||
return getRequestWithMultipartForm(t, "", creds, timeToSignStr, sign, defaultFieldName)
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "HTTP POST invalid accessKeyId",
|
|
||||||
request: func() *http.Request {
|
|
||||||
creds := getCredsStr(invalidValue, timeToSignStr, region, service)
|
|
||||||
sign := SignStr(secret.SecretKey, service, region, timeToSign, policyBase64)
|
|
||||||
|
|
||||||
return getRequestWithMultipartForm(t, policyBase64, creds, timeToSignStr, sign, defaultFieldName)
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "HTTP POST invalid accessKeyId - a non-existent box",
|
|
||||||
request: func() *http.Request {
|
|
||||||
creds := getCredsStr(invalidAccessKeyID, timeToSignStr, region, service)
|
|
||||||
sign := SignStr(secret.SecretKey, service, region, timeToSign, policyBase64)
|
|
||||||
|
|
||||||
return getRequestWithMultipartForm(t, policyBase64, creds, timeToSignStr, sign, defaultFieldName)
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "HTTP POST invalid signature",
|
|
||||||
request: func() *http.Request {
|
|
||||||
creds := getCredsStr(accessKeyID, timeToSignStr, region, service)
|
|
||||||
sign := SignStr(secret.SecretKey, service, region, timeToSign, invalidValue)
|
|
||||||
|
|
||||||
return getRequestWithMultipartForm(t, policyBase64, creds, timeToSignStr, sign, defaultFieldName)
|
|
||||||
}(),
|
|
||||||
err: true,
|
|
||||||
errCode: errors.ErrSignatureDoesNotMatch,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
creds := tokens.New(bigConfig)
|
|
||||||
cntr := New(creds, tc.prefixes)
|
|
||||||
box, err := cntr.Authenticate(tc.request)
|
|
||||||
|
|
||||||
if tc.err {
|
|
||||||
require.Error(t, err)
|
|
||||||
if tc.errCode > 0 {
|
|
||||||
err = frostfsErrors.UnwrapErr(err)
|
|
||||||
require.Equal(t, errors.GetAPIError(tc.errCode), err)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, secret.SecretKey, box.AccessBox.Gate.SecretKey)
|
|
||||||
require.Equal(t, accessKeyID, box.AuthHeaders.AccessKeyID)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func getCredsStr(accessKeyID, timeToSign, region, service string) string {
|
|
||||||
return accessKeyID + "/" + timeToSign + "/" + region + "/" + service + "/aws4_request"
|
|
||||||
}
|
|
||||||
|
|
||||||
func getRequestWithMultipartForm(t *testing.T, policy, creds, date, sign, fieldName string) *http.Request {
|
|
||||||
body := &bytes.Buffer{}
|
|
||||||
writer := multipart.NewWriter(body)
|
|
||||||
defer writer.Close()
|
|
||||||
|
|
||||||
err := writer.WriteField("policy", policy)
|
|
||||||
require.NoError(t, err)
|
|
||||||
err = writer.WriteField(AmzCredential, creds)
|
|
||||||
require.NoError(t, err)
|
|
||||||
err = writer.WriteField(AmzDate, date)
|
|
||||||
require.NoError(t, err)
|
|
||||||
err = writer.WriteField(AmzSignature, sign)
|
|
||||||
require.NoError(t, err)
|
|
||||||
_, err = writer.CreateFormFile(fieldName, "test.txt")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
req := httptest.NewRequest(http.MethodPost, "/", body)
|
|
||||||
req.Header.Set(ContentTypeHdr, writer.FormDataContentType())
|
|
||||||
|
|
||||||
return req
|
|
||||||
}
|
|
||||||
|
|
|
@ -10,7 +10,6 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
@ -32,13 +31,13 @@ func (m credentialsMock) addBox(addr oid.Address, box *accessbox.Box) {
|
||||||
m.boxes[addr.String()] = box
|
m.boxes[addr.String()] = box
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m credentialsMock) GetBox(_ context.Context, addr oid.Address) (*accessbox.Box, []object.Attribute, error) {
|
func (m credentialsMock) GetBox(_ context.Context, addr oid.Address) (*accessbox.Box, error) {
|
||||||
box, ok := m.boxes[addr.String()]
|
box, ok := m.boxes[addr.String()]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, nil, &apistatus.ObjectNotFound{}
|
return nil, &apistatus.ObjectNotFound{}
|
||||||
}
|
}
|
||||||
|
|
||||||
return box, nil, nil
|
return box, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m credentialsMock) Put(context.Context, cid.ID, tokens.CredentialsParam) (oid.Address, error) {
|
func (m credentialsMock) Put(context.Context, cid.ID, tokens.CredentialsParam) (oid.Address, error) {
|
||||||
|
|
5
api/cache/accessbox.go
vendored
5
api/cache/accessbox.go
vendored
|
@ -6,7 +6,6 @@ import (
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -28,7 +27,6 @@ type (
|
||||||
|
|
||||||
AccessBoxCacheValue struct {
|
AccessBoxCacheValue struct {
|
||||||
Box *accessbox.Box
|
Box *accessbox.Box
|
||||||
Attributes []object.Attribute
|
|
||||||
PutTime time.Time
|
PutTime time.Time
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
@ -74,10 +72,9 @@ func (o *AccessBoxCache) Get(address oid.Address) *AccessBoxCacheValue {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put stores an accessbox to cache.
|
// Put stores an accessbox to cache.
|
||||||
func (o *AccessBoxCache) Put(address oid.Address, box *accessbox.Box, attrs []object.Attribute) error {
|
func (o *AccessBoxCache) Put(address oid.Address, box *accessbox.Box) error {
|
||||||
val := &AccessBoxCacheValue{
|
val := &AccessBoxCacheValue{
|
||||||
Box: box,
|
Box: box,
|
||||||
Attributes: attrs,
|
|
||||||
PutTime: time.Now(),
|
PutTime: time.Now(),
|
||||||
}
|
}
|
||||||
return o.cache.Set(address, val)
|
return o.cache.Set(address, val)
|
||||||
|
|
23
api/cache/cache_test.go
vendored
23
api/cache/cache_test.go
vendored
|
@ -7,7 +7,6 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
||||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/util"
|
"github.com/nspcc-dev/neo-go/pkg/util"
|
||||||
|
@ -22,13 +21,11 @@ func TestAccessBoxCacheType(t *testing.T) {
|
||||||
|
|
||||||
addr := oidtest.Address()
|
addr := oidtest.Address()
|
||||||
box := &accessbox.Box{}
|
box := &accessbox.Box{}
|
||||||
var attrs []object.Attribute
|
|
||||||
|
|
||||||
err := cache.Put(addr, box, attrs)
|
err := cache.Put(addr, box)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
val := cache.Get(addr)
|
val := cache.Get(addr)
|
||||||
require.Equal(t, box, val.Box)
|
require.Equal(t, box, val.Box)
|
||||||
require.Equal(t, attrs, val.Attributes)
|
|
||||||
require.Equal(t, 0, observedLog.Len())
|
require.Equal(t, 0, observedLog.Len())
|
||||||
|
|
||||||
err = cache.cache.Set(addr, "tmp")
|
err = cache.cache.Set(addr, "tmp")
|
||||||
|
@ -182,6 +179,24 @@ func TestSettingsCacheType(t *testing.T) {
|
||||||
assertInvalidCacheEntry(t, cache.GetSettings(key), observedLog)
|
assertInvalidCacheEntry(t, cache.GetSettings(key), observedLog)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNotificationConfigurationCacheType(t *testing.T) {
|
||||||
|
logger, observedLog := getObservedLogger()
|
||||||
|
cache := NewSystemCache(DefaultSystemConfig(logger))
|
||||||
|
|
||||||
|
key := "key"
|
||||||
|
notificationConfig := &data.NotificationConfiguration{}
|
||||||
|
|
||||||
|
err := cache.PutNotificationConfiguration(key, notificationConfig)
|
||||||
|
require.NoError(t, err)
|
||||||
|
val := cache.GetNotificationConfiguration(key)
|
||||||
|
require.Equal(t, notificationConfig, val)
|
||||||
|
require.Equal(t, 0, observedLog.Len())
|
||||||
|
|
||||||
|
err = cache.cache.Set(key, "tmp")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assertInvalidCacheEntry(t, cache.GetNotificationConfiguration(key), observedLog)
|
||||||
|
}
|
||||||
|
|
||||||
func TestFrostFSIDSubjectCacheType(t *testing.T) {
|
func TestFrostFSIDSubjectCacheType(t *testing.T) {
|
||||||
logger, observedLog := getObservedLogger()
|
logger, observedLog := getObservedLogger()
|
||||||
cache := NewFrostfsIDCache(DefaultFrostfsIDConfig(logger))
|
cache := NewFrostfsIDCache(DefaultFrostfsIDConfig(logger))
|
||||||
|
|
16
api/cache/system.go
vendored
16
api/cache/system.go
vendored
|
@ -88,13 +88,13 @@ func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func (o *SystemCache) GetLifecycleConfiguration(key string) *data.LifecycleConfiguration {
|
func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
|
||||||
entry, err := o.cache.Get(key)
|
entry, err := o.cache.Get(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
result, ok := entry.(*data.LifecycleConfiguration)
|
result, ok := entry.(*data.BucketSettings)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
|
@ -104,13 +104,13 @@ func (o *SystemCache) GetLifecycleConfiguration(key string) *data.LifecycleConfi
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
|
func (o *SystemCache) GetNotificationConfiguration(key string) *data.NotificationConfiguration {
|
||||||
entry, err := o.cache.Get(key)
|
entry, err := o.cache.Get(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
result, ok := entry.(*data.BucketSettings)
|
result, ok := entry.(*data.NotificationConfiguration)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
|
@ -149,14 +149,14 @@ func (o *SystemCache) PutCORS(key string, obj *data.CORSConfiguration) error {
|
||||||
return o.cache.Set(key, obj)
|
return o.cache.Set(key, obj)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (o *SystemCache) PutLifecycleConfiguration(key string, obj *data.LifecycleConfiguration) error {
|
|
||||||
return o.cache.Set(key, obj)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *SystemCache) PutSettings(key string, settings *data.BucketSettings) error {
|
func (o *SystemCache) PutSettings(key string, settings *data.BucketSettings) error {
|
||||||
return o.cache.Set(key, settings)
|
return o.cache.Set(key, settings)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (o *SystemCache) PutNotificationConfiguration(key string, obj *data.NotificationConfiguration) error {
|
||||||
|
return o.cache.Set(key, obj)
|
||||||
|
}
|
||||||
|
|
||||||
// PutTagging puts tags of a bucket or an object.
|
// PutTagging puts tags of a bucket or an object.
|
||||||
func (o *SystemCache) PutTagging(key string, tagSet map[string]string) error {
|
func (o *SystemCache) PutTagging(key string, tagSet map[string]string) error {
|
||||||
return o.cache.Set(key, tagSet)
|
return o.cache.Set(key, tagSet)
|
||||||
|
|
|
@ -14,7 +14,7 @@ import (
|
||||||
const (
|
const (
|
||||||
bktSettingsObject = ".s3-settings"
|
bktSettingsObject = ".s3-settings"
|
||||||
bktCORSConfigurationObject = ".s3-cors"
|
bktCORSConfigurationObject = ".s3-cors"
|
||||||
bktLifecycleConfigurationObject = ".s3-lifecycle"
|
bktNotificationConfigurationObject = ".s3-notifications"
|
||||||
|
|
||||||
VersioningUnversioned = "Unversioned"
|
VersioningUnversioned = "Unversioned"
|
||||||
VersioningEnabled = "Enabled"
|
VersioningEnabled = "Enabled"
|
||||||
|
@ -32,6 +32,7 @@ type (
|
||||||
LocationConstraint string
|
LocationConstraint string
|
||||||
ObjectLockEnabled bool
|
ObjectLockEnabled bool
|
||||||
HomomorphicHashDisabled bool
|
HomomorphicHashDisabled bool
|
||||||
|
APEEnabled bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// ObjectInfo holds S3 object data.
|
// ObjectInfo holds S3 object data.
|
||||||
|
@ -51,6 +52,14 @@ type (
|
||||||
Headers map[string]string
|
Headers map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NotificationInfo store info to send s3 notification.
|
||||||
|
NotificationInfo struct {
|
||||||
|
Name string
|
||||||
|
Version string
|
||||||
|
Size uint64
|
||||||
|
HashSum string
|
||||||
|
}
|
||||||
|
|
||||||
// BucketSettings stores settings such as versioning.
|
// BucketSettings stores settings such as versioning.
|
||||||
BucketSettings struct {
|
BucketSettings struct {
|
||||||
Versioning string
|
Versioning string
|
||||||
|
@ -74,35 +83,26 @@ type (
|
||||||
ExposeHeaders []string `xml:"ExposeHeader" json:"ExposeHeaders"`
|
ExposeHeaders []string `xml:"ExposeHeader" json:"ExposeHeaders"`
|
||||||
MaxAgeSeconds int `xml:"MaxAgeSeconds,omitempty" json:"MaxAgeSeconds,omitempty"`
|
MaxAgeSeconds int `xml:"MaxAgeSeconds,omitempty" json:"MaxAgeSeconds,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ObjectVersion stores object version info.
|
|
||||||
ObjectVersion struct {
|
|
||||||
BktInfo *BucketInfo
|
|
||||||
ObjectName string
|
|
||||||
VersionID string
|
|
||||||
NoErrorOnDeleteMarker bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// CreatedObjectInfo stores created object info.
|
|
||||||
CreatedObjectInfo struct {
|
|
||||||
ID oid.ID
|
|
||||||
Size uint64
|
|
||||||
HashSum []byte
|
|
||||||
MD5Sum []byte
|
|
||||||
CreationEpoch uint64
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// NotificationInfoFromObject creates new NotificationInfo from ObjectInfo.
|
||||||
|
func NotificationInfoFromObject(objInfo *ObjectInfo, md5Enabled bool) *NotificationInfo {
|
||||||
|
return &NotificationInfo{
|
||||||
|
Name: objInfo.Name,
|
||||||
|
Version: objInfo.VersionID(),
|
||||||
|
Size: objInfo.Size,
|
||||||
|
HashSum: Quote(objInfo.ETag(md5Enabled)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// SettingsObjectName is a system name for a bucket settings file.
|
// SettingsObjectName is a system name for a bucket settings file.
|
||||||
func (b *BucketInfo) SettingsObjectName() string { return bktSettingsObject }
|
func (b *BucketInfo) SettingsObjectName() string { return bktSettingsObject }
|
||||||
|
|
||||||
// CORSObjectName returns a system name for a bucket CORS configuration file.
|
// CORSObjectName returns a system name for a bucket CORS configuration file.
|
||||||
func (b *BucketInfo) CORSObjectName() string {
|
func (b *BucketInfo) CORSObjectName() string { return bktCORSConfigurationObject }
|
||||||
return b.CID.EncodeToString() + bktCORSConfigurationObject
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *BucketInfo) LifecycleConfigurationObjectName() string {
|
func (b *BucketInfo) NotificationConfigurationObjectName() string {
|
||||||
return b.CID.EncodeToString() + bktLifecycleConfigurationObject
|
return bktNotificationConfigurationObject
|
||||||
}
|
}
|
||||||
|
|
||||||
// VersionID returns object version from ObjectInfo.
|
// VersionID returns object version from ObjectInfo.
|
||||||
|
|
|
@ -1,54 +0,0 @@
|
||||||
package data
|
|
||||||
|
|
||||||
import "encoding/xml"
|
|
||||||
|
|
||||||
const (
|
|
||||||
LifecycleStatusEnabled = "Enabled"
|
|
||||||
LifecycleStatusDisabled = "Disabled"
|
|
||||||
)
|
|
||||||
|
|
||||||
type (
|
|
||||||
LifecycleConfiguration struct {
|
|
||||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LifecycleConfiguration" json:"-"`
|
|
||||||
Rules []LifecycleRule `xml:"Rule"`
|
|
||||||
}
|
|
||||||
|
|
||||||
LifecycleRule struct {
|
|
||||||
Status string `xml:"Status,omitempty"`
|
|
||||||
AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
|
|
||||||
Expiration *LifecycleExpiration `xml:"Expiration,omitempty"`
|
|
||||||
Filter *LifecycleRuleFilter `xml:"Filter,omitempty"`
|
|
||||||
ID string `xml:"ID,omitempty"`
|
|
||||||
NonCurrentVersionExpiration *NonCurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
AbortIncompleteMultipartUpload struct {
|
|
||||||
DaysAfterInitiation *int `xml:"DaysAfterInitiation,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
LifecycleExpiration struct {
|
|
||||||
Date string `xml:"Date,omitempty"`
|
|
||||||
Days *int `xml:"Days,omitempty"`
|
|
||||||
ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
LifecycleRuleFilter struct {
|
|
||||||
And *LifecycleRuleAndOperator `xml:"And,omitempty"`
|
|
||||||
ObjectSizeGreaterThan *uint64 `xml:"ObjectSizeGreaterThan,omitempty"`
|
|
||||||
ObjectSizeLessThan *uint64 `xml:"ObjectSizeLessThan,omitempty"`
|
|
||||||
Prefix string `xml:"Prefix,omitempty"`
|
|
||||||
Tag *Tag `xml:"Tag,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
LifecycleRuleAndOperator struct {
|
|
||||||
ObjectSizeGreaterThan *uint64 `xml:"ObjectSizeGreaterThan,omitempty"`
|
|
||||||
ObjectSizeLessThan *uint64 `xml:"ObjectSizeLessThan,omitempty"`
|
|
||||||
Prefix string `xml:"Prefix,omitempty"`
|
|
||||||
Tags []Tag `xml:"Tag"`
|
|
||||||
}
|
|
||||||
|
|
||||||
NonCurrentVersionExpiration struct {
|
|
||||||
NewerNonCurrentVersions *int `xml:"NewerNoncurrentVersions,omitempty"`
|
|
||||||
NonCurrentDays *int `xml:"NoncurrentDays,omitempty"`
|
|
||||||
}
|
|
||||||
)
|
|
42
api/data/notifications.go
Normal file
42
api/data/notifications.go
Normal file
|
@ -0,0 +1,42 @@
|
||||||
|
package data
|
||||||
|
|
||||||
|
import "encoding/xml"
|
||||||
|
|
||||||
|
type (
|
||||||
|
NotificationConfiguration struct {
|
||||||
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NotificationConfiguration" json:"-"`
|
||||||
|
QueueConfigurations []QueueConfiguration `xml:"QueueConfiguration" json:"QueueConfigurations"`
|
||||||
|
// Not supported topics
|
||||||
|
TopicConfigurations []TopicConfiguration `xml:"TopicConfiguration" json:"TopicConfigurations"`
|
||||||
|
LambdaFunctionConfigurations []LambdaFunctionConfiguration `xml:"CloudFunctionConfiguration" json:"CloudFunctionConfigurations"`
|
||||||
|
}
|
||||||
|
|
||||||
|
QueueConfiguration struct {
|
||||||
|
ID string `xml:"Id" json:"Id"`
|
||||||
|
QueueArn string `xml:"Queue" json:"Queue"`
|
||||||
|
Events []string `xml:"Event" json:"Events"`
|
||||||
|
Filter Filter `xml:"Filter" json:"Filter"`
|
||||||
|
}
|
||||||
|
|
||||||
|
Filter struct {
|
||||||
|
Key Key `xml:"S3Key" json:"S3Key"`
|
||||||
|
}
|
||||||
|
|
||||||
|
Key struct {
|
||||||
|
FilterRules []FilterRule `xml:"FilterRule" json:"FilterRules"`
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterRule struct {
|
||||||
|
Name string `xml:"Name" json:"Name"`
|
||||||
|
Value string `xml:"Value" json:"Value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TopicConfiguration and LambdaFunctionConfiguration -- we don't support these configurations,
|
||||||
|
// but we need them to detect in notification configurations in requests.
|
||||||
|
TopicConfiguration struct{}
|
||||||
|
LambdaFunctionConfiguration struct{}
|
||||||
|
)
|
||||||
|
|
||||||
|
func (n NotificationConfiguration) IsEmpty() bool {
|
||||||
|
return len(n.QueueConfigurations) == 0 && len(n.TopicConfigurations) == 0 && len(n.LambdaFunctionConfigurations) == 0
|
||||||
|
}
|
|
@ -1,30 +0,0 @@
|
||||||
package data
|
|
||||||
|
|
||||||
import "encoding/xml"
|
|
||||||
|
|
||||||
// Tagging contains tag set.
|
|
||||||
type Tagging struct {
|
|
||||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"`
|
|
||||||
TagSet []Tag `xml:"TagSet>Tag"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Tag is an AWS key-value tag.
|
|
||||||
type Tag struct {
|
|
||||||
Key string
|
|
||||||
Value string
|
|
||||||
}
|
|
||||||
|
|
||||||
type GetObjectTaggingParams struct {
|
|
||||||
ObjectVersion *ObjectVersion
|
|
||||||
|
|
||||||
// NodeVersion can be nil. If not nil we save one request to tree service.
|
|
||||||
NodeVersion *NodeVersion // optional
|
|
||||||
}
|
|
||||||
|
|
||||||
type PutObjectTaggingParams struct {
|
|
||||||
ObjectVersion *ObjectVersion
|
|
||||||
TagSet map[string]string
|
|
||||||
|
|
||||||
// NodeVersion can be nil. If not nil we save one request to tree service.
|
|
||||||
NodeVersion *NodeVersion // optional
|
|
||||||
}
|
|
|
@ -72,7 +72,6 @@ type BaseNodeVersion struct {
|
||||||
Created *time.Time
|
Created *time.Time
|
||||||
Owner *user.ID
|
Owner *user.ID
|
||||||
IsDeleteMarker bool
|
IsDeleteMarker bool
|
||||||
CreationEpoch uint64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *BaseNodeVersion) GetETag(md5Enabled bool) string {
|
func (v *BaseNodeVersion) GetETag(md5Enabled bool) string {
|
||||||
|
@ -111,7 +110,6 @@ type MultipartInfo struct {
|
||||||
Meta map[string]string
|
Meta map[string]string
|
||||||
CopiesNumbers []uint32
|
CopiesNumbers []uint32
|
||||||
Finished bool
|
Finished bool
|
||||||
CreationEpoch uint64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PartInfo is upload information about part.
|
// PartInfo is upload information about part.
|
||||||
|
@ -126,14 +124,6 @@ type PartInfo struct {
|
||||||
Created time.Time `json:"created"`
|
Created time.Time `json:"created"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type PartInfoExtended struct {
|
|
||||||
PartInfo
|
|
||||||
|
|
||||||
// Timestamp is used to find the latest version of part info in case of tree split
|
|
||||||
// when there are multiple nodes for the same part.
|
|
||||||
Timestamp uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// ToHeaderString form short part representation to use in S3-Completed-Parts header.
|
// ToHeaderString form short part representation to use in S3-Completed-Parts header.
|
||||||
func (p *PartInfo) ToHeaderString() string {
|
func (p *PartInfo) ToHeaderString() string {
|
||||||
// ETag value contains SHA256 checksum which is used while getting object parts attributes.
|
// ETag value contains SHA256 checksum which is used while getting object parts attributes.
|
||||||
|
|
|
@ -91,7 +91,6 @@ const (
|
||||||
ErrBucketNotEmpty
|
ErrBucketNotEmpty
|
||||||
ErrAllAccessDisabled
|
ErrAllAccessDisabled
|
||||||
ErrMalformedPolicy
|
ErrMalformedPolicy
|
||||||
ErrMalformedPolicyNotPrincipal
|
|
||||||
ErrMissingFields
|
ErrMissingFields
|
||||||
ErrMissingCredTag
|
ErrMissingCredTag
|
||||||
ErrCredMalformed
|
ErrCredMalformed
|
||||||
|
@ -187,9 +186,6 @@ const (
|
||||||
ErrInvalidRequestLargeCopy
|
ErrInvalidRequestLargeCopy
|
||||||
ErrInvalidStorageClass
|
ErrInvalidStorageClass
|
||||||
VersionIDMarkerWithoutKeyMarker
|
VersionIDMarkerWithoutKeyMarker
|
||||||
ErrInvalidRangeLength
|
|
||||||
ErrRangeOutOfBounds
|
|
||||||
ErrMissingContentRange
|
|
||||||
|
|
||||||
ErrMalformedJSON
|
ErrMalformedJSON
|
||||||
ErrInsecureClientRequest
|
ErrInsecureClientRequest
|
||||||
|
@ -669,12 +665,6 @@ var errorCodes = errorCodeMap{
|
||||||
Description: "Policy has invalid resource.",
|
Description: "Policy has invalid resource.",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
ErrMalformedPolicyNotPrincipal: {
|
|
||||||
ErrCode: ErrMalformedPolicyNotPrincipal,
|
|
||||||
Code: "MalformedPolicy",
|
|
||||||
Description: "Allow with NotPrincipal is not allowed.",
|
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
|
||||||
},
|
|
||||||
ErrMissingFields: {
|
ErrMissingFields: {
|
||||||
ErrCode: ErrMissingFields,
|
ErrCode: ErrMissingFields,
|
||||||
Code: "MissingFields",
|
Code: "MissingFields",
|
||||||
|
@ -1742,24 +1732,6 @@ var errorCodes = errorCodeMap{
|
||||||
Description: "Part number must be an integer between 1 and 10000, inclusive",
|
Description: "Part number must be an integer between 1 and 10000, inclusive",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
ErrInvalidRangeLength: {
|
|
||||||
ErrCode: ErrInvalidRangeLength,
|
|
||||||
Code: "InvalidRange",
|
|
||||||
Description: "Provided range length must be equal to content length",
|
|
||||||
HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
|
|
||||||
},
|
|
||||||
ErrRangeOutOfBounds: {
|
|
||||||
ErrCode: ErrRangeOutOfBounds,
|
|
||||||
Code: "InvalidRange",
|
|
||||||
Description: "Provided range is outside of object bounds",
|
|
||||||
HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
|
|
||||||
},
|
|
||||||
ErrMissingContentRange: {
|
|
||||||
ErrCode: ErrMissingContentRange,
|
|
||||||
Code: "MissingContentRange",
|
|
||||||
Description: "Content-Range header is mandatory for this type of request",
|
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
|
||||||
},
|
|
||||||
// Add your error structure here.
|
// Add your error structure here.
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
1409
api/handler/acl.go
1409
api/handler/acl.go
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
@ -11,6 +11,7 @@ import (
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||||
|
@ -20,12 +21,18 @@ import (
|
||||||
type (
|
type (
|
||||||
handler struct {
|
handler struct {
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
obj *layer.Layer
|
obj layer.Client
|
||||||
|
notificator Notificator
|
||||||
cfg Config
|
cfg Config
|
||||||
ape APE
|
ape APE
|
||||||
frostfsid FrostFSID
|
frostfsid FrostFSID
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Notificator interface {
|
||||||
|
SendNotifications(topics map[string]string, p *SendNotificationParams) error
|
||||||
|
SendTestNotification(topic, bucketName, requestID, HostID string, now time.Time) error
|
||||||
|
}
|
||||||
|
|
||||||
// Config contains data which handler needs to keep.
|
// Config contains data which handler needs to keep.
|
||||||
Config interface {
|
Config interface {
|
||||||
DefaultPlacementPolicy(namespace string) netmap.PlacementPolicy
|
DefaultPlacementPolicy(namespace string) netmap.PlacementPolicy
|
||||||
|
@ -34,13 +41,12 @@ type (
|
||||||
DefaultCopiesNumbers(namespace string) []uint32
|
DefaultCopiesNumbers(namespace string) []uint32
|
||||||
NewXMLDecoder(io.Reader) *xml.Decoder
|
NewXMLDecoder(io.Reader) *xml.Decoder
|
||||||
DefaultMaxAge() int
|
DefaultMaxAge() int
|
||||||
|
NotificatorEnabled() bool
|
||||||
ResolveZoneList() []string
|
ResolveZoneList() []string
|
||||||
IsResolveListAllow() bool
|
IsResolveListAllow() bool
|
||||||
BypassContentEncodingInChunks() bool
|
BypassContentEncodingInChunks() bool
|
||||||
MD5Enabled() bool
|
MD5Enabled() bool
|
||||||
RetryMaxAttempts() int
|
ACLEnabled() bool
|
||||||
RetryMaxBackoff() time.Duration
|
|
||||||
RetryStrategy() RetryStrategy
|
|
||||||
}
|
}
|
||||||
|
|
||||||
FrostFSID interface {
|
FrostFSID interface {
|
||||||
|
@ -57,17 +63,10 @@ type (
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
type RetryStrategy string
|
|
||||||
|
|
||||||
const (
|
|
||||||
RetryStrategyExponential = "exponential"
|
|
||||||
RetryStrategyConstant = "constant"
|
|
||||||
)
|
|
||||||
|
|
||||||
var _ api.Handler = (*handler)(nil)
|
var _ api.Handler = (*handler)(nil)
|
||||||
|
|
||||||
// New creates new api.Handler using given logger and client.
|
// New creates new api.Handler using given logger and client.
|
||||||
func New(log *zap.Logger, obj *layer.Layer, cfg Config, storage APE, ffsid FrostFSID) (api.Handler, error) {
|
func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg Config, storage APE, ffsid FrostFSID) (api.Handler, error) {
|
||||||
switch {
|
switch {
|
||||||
case obj == nil:
|
case obj == nil:
|
||||||
return nil, errors.New("empty FrostFS Object Layer")
|
return nil, errors.New("empty FrostFS Object Layer")
|
||||||
|
@ -79,11 +78,18 @@ func New(log *zap.Logger, obj *layer.Layer, cfg Config, storage APE, ffsid Frost
|
||||||
return nil, errors.New("empty frostfsid")
|
return nil, errors.New("empty frostfsid")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !cfg.NotificatorEnabled() {
|
||||||
|
log.Warn(logs.NotificatorIsDisabledS3WontProduceNotificationEvents)
|
||||||
|
} else if notificator == nil {
|
||||||
|
return nil, errors.New("empty notificator")
|
||||||
|
}
|
||||||
|
|
||||||
return &handler{
|
return &handler{
|
||||||
log: log,
|
log: log,
|
||||||
obj: obj,
|
obj: obj,
|
||||||
cfg: cfg,
|
cfg: cfg,
|
||||||
ape: storage,
|
ape: storage,
|
||||||
|
notificator: notificator,
|
||||||
frostfsid: ffsid,
|
frostfsid: ffsid,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,6 +13,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -45,6 +46,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
versionID string
|
versionID string
|
||||||
metadata map[string]string
|
metadata map[string]string
|
||||||
tagSet map[string]string
|
tagSet map[string]string
|
||||||
|
sessionTokenEACL *session.Container
|
||||||
|
|
||||||
ctx = r.Context()
|
ctx = r.Context()
|
||||||
reqInfo = middleware.GetReqInfo(ctx)
|
reqInfo = middleware.GetReqInfo(ctx)
|
||||||
|
@ -91,11 +93,20 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if cannedACLStatus == aclStatusYes {
|
apeEnabled := dstBktInfo.APEEnabled || settings.CannedACL != ""
|
||||||
|
if apeEnabled && cannedACLStatus == aclStatusYes {
|
||||||
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
needUpdateEACLTable := !(apeEnabled || cannedACLStatus == aclStatusNo)
|
||||||
|
if needUpdateEACLTable {
|
||||||
|
if sessionTokenEACL, err = getSessionTokenSetEACL(ctx); err != nil {
|
||||||
|
h.logAndSendError(w, "could not get eacl session token from a box", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
extendedSrcObjInfo, err := h.obj.GetExtendedObjectInfo(ctx, srcObjPrm)
|
extendedSrcObjInfo, err := h.obj.GetExtendedObjectInfo(ctx, srcObjPrm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not find object", reqInfo, err)
|
h.logAndSendError(w, "could not find object", reqInfo, err)
|
||||||
|
@ -157,8 +168,8 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
tagPrm := &data.GetObjectTaggingParams{
|
tagPrm := &layer.GetObjectTaggingParams{
|
||||||
ObjectVersion: &data.ObjectVersion{
|
ObjectVersion: &layer.ObjectVersion{
|
||||||
BktInfo: srcObjPrm.BktInfo,
|
BktInfo: srcObjPrm.BktInfo,
|
||||||
ObjectName: srcObject,
|
ObjectName: srcObject,
|
||||||
VersionID: srcObjInfo.VersionID(),
|
VersionID: srcObjInfo.VersionID(),
|
||||||
|
@ -228,9 +239,28 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if needUpdateEACLTable {
|
||||||
|
newEaclTable, err := h.getNewEAclTable(r, dstBktInfo, dstObjInfo)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get new eacl table", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p := &layer.PutBucketACLParams{
|
||||||
|
BktInfo: dstBktInfo,
|
||||||
|
EACL: newEaclTable,
|
||||||
|
SessionToken: sessionTokenEACL,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = h.obj.PutBucketACL(ctx, p); err != nil {
|
||||||
|
h.logAndSendError(w, "could not put bucket acl", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if tagSet != nil {
|
if tagSet != nil {
|
||||||
tagPrm := &data.PutObjectTaggingParams{
|
tagPrm := &layer.PutObjectTaggingParams{
|
||||||
ObjectVersion: &data.ObjectVersion{
|
ObjectVersion: &layer.ObjectVersion{
|
||||||
BktInfo: dstBktInfo,
|
BktInfo: dstBktInfo,
|
||||||
ObjectName: reqInfo.ObjectName,
|
ObjectName: reqInfo.ObjectName,
|
||||||
VersionID: dstObjInfo.VersionID(),
|
VersionID: dstObjInfo.VersionID(),
|
||||||
|
@ -238,7 +268,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
TagSet: tagSet,
|
TagSet: tagSet,
|
||||||
NodeVersion: extendedDstObjInfo.NodeVersion,
|
NodeVersion: extendedDstObjInfo.NodeVersion,
|
||||||
}
|
}
|
||||||
if err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
|
if _, err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
|
||||||
h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
|
h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -246,6 +276,16 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID))
|
h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID))
|
||||||
|
|
||||||
|
s := &SendNotificationParams{
|
||||||
|
Event: EventObjectCreatedCopy,
|
||||||
|
NotificationInfo: data.NotificationInfoFromObject(dstObjInfo, h.cfg.MD5Enabled()),
|
||||||
|
BktInfo: dstBktInfo,
|
||||||
|
ReqInfo: reqInfo,
|
||||||
|
}
|
||||||
|
if err = h.sendNotifications(ctx, s); err != nil {
|
||||||
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
if dstEncryptionParams.Enabled() {
|
if dstEncryptionParams.Enabled() {
|
||||||
addSSECHeaders(w.Header(), r.Header)
|
addSSECHeaders(w.Header(), r.Header)
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,11 +11,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -291,24 +289,23 @@ func copyObject(hc *handlerContext, bktName, fromObject, toObject string, copyMe
|
||||||
}
|
}
|
||||||
|
|
||||||
func putObjectTagging(t *testing.T, tc *handlerContext, bktName, objName string, tags map[string]string) {
|
func putObjectTagging(t *testing.T, tc *handlerContext, bktName, objName string, tags map[string]string) {
|
||||||
body := &data.Tagging{
|
body := &Tagging{
|
||||||
TagSet: make([]data.Tag, 0, len(tags)),
|
TagSet: make([]Tag, 0, len(tags)),
|
||||||
}
|
}
|
||||||
|
|
||||||
for key, val := range tags {
|
for key, val := range tags {
|
||||||
body.TagSet = append(body.TagSet, data.Tag{
|
body.TagSet = append(body.TagSet, Tag{
|
||||||
Key: key,
|
Key: key,
|
||||||
Value: val,
|
Value: val,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
w, r := prepareTestRequest(tc, bktName, objName, body)
|
w, r := prepareTestRequest(tc, bktName, objName, body)
|
||||||
middleware.GetReqInfo(r.Context()).Tagging = body
|
|
||||||
tc.Handler().PutObjectTaggingHandler(w, r)
|
tc.Handler().PutObjectTaggingHandler(w, r)
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(t, w, http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, version string) *data.Tagging {
|
func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, version string) *Tagging {
|
||||||
query := make(url.Values)
|
query := make(url.Values)
|
||||||
query.Add(api.QueryVersionID, version)
|
query.Add(api.QueryVersionID, version)
|
||||||
|
|
||||||
|
@ -316,7 +313,7 @@ func getObjectTagging(t *testing.T, tc *handlerContext, bktName, objName, versio
|
||||||
tc.Handler().GetObjectTaggingHandler(w, r)
|
tc.Handler().GetObjectTaggingHandler(w, r)
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
|
||||||
tagging := &data.Tagging{}
|
tagging := &Tagging{}
|
||||||
err := xml.NewDecoder(w.Result().Body).Decode(tagging)
|
err := xml.NewDecoder(w.Result().Body).Decode(tagging)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
return tagging
|
return tagging
|
||||||
|
|
|
@ -187,8 +187,8 @@ func (h *handler) Preflight(w http.ResponseWriter, r *http.Request) {
|
||||||
if !checkSubslice(rule.AllowedHeaders, headers) {
|
if !checkSubslice(rule.AllowedHeaders, headers) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
w.Header().Set(api.AccessControlAllowOrigin, origin)
|
w.Header().Set(api.AccessControlAllowOrigin, o)
|
||||||
w.Header().Set(api.AccessControlAllowMethods, method)
|
w.Header().Set(api.AccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", "))
|
||||||
if headers != nil {
|
if headers != nil {
|
||||||
w.Header().Set(api.AccessControlAllowHeaders, requestHeaders)
|
w.Header().Set(api.AccessControlAllowHeaders, requestHeaders)
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,7 +7,6 @@ import (
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCORSOriginWildcard(t *testing.T) {
|
func TestCORSOriginWildcard(t *testing.T) {
|
||||||
|
@ -24,14 +23,14 @@ func TestCORSOriginWildcard(t *testing.T) {
|
||||||
bktName := "bucket-for-cors"
|
bktName := "bucket-for-cors"
|
||||||
box, _ := createAccessBox(t)
|
box, _ := createAccessBox(t)
|
||||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||||
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
ctx := middleware.SetBoxData(r.Context(), box)
|
||||||
r = r.WithContext(ctx)
|
r = r.WithContext(ctx)
|
||||||
r.Header.Add(api.AmzACL, "public-read")
|
r.Header.Add(api.AmzACL, "public-read")
|
||||||
hc.Handler().CreateBucketHandler(w, r)
|
hc.Handler().CreateBucketHandler(w, r)
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
|
||||||
w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
|
w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
|
||||||
ctx = middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
ctx = middleware.SetBoxData(r.Context(), box)
|
||||||
r = r.WithContext(ctx)
|
r = r.WithContext(ctx)
|
||||||
hc.Handler().PutBucketCorsHandler(w, r)
|
hc.Handler().PutBucketCorsHandler(w, r)
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
@ -40,181 +39,3 @@ func TestCORSOriginWildcard(t *testing.T) {
|
||||||
hc.Handler().GetBucketCorsHandler(w, r)
|
hc.Handler().GetBucketCorsHandler(w, r)
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(t, w, http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPreflight(t *testing.T) {
|
|
||||||
body := `
|
|
||||||
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
|
||||||
<CORSRule>
|
|
||||||
<AllowedMethod>GET</AllowedMethod>
|
|
||||||
<AllowedOrigin>http://www.example.com</AllowedOrigin>
|
|
||||||
<AllowedHeader>Authorization</AllowedHeader>
|
|
||||||
<ExposeHeader>x-amz-*</ExposeHeader>
|
|
||||||
<ExposeHeader>X-Amz-*</ExposeHeader>
|
|
||||||
<MaxAgeSeconds>600</MaxAgeSeconds>
|
|
||||||
</CORSRule>
|
|
||||||
</CORSConfiguration>
|
|
||||||
`
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
|
|
||||||
bktName := "bucket-preflight-test"
|
|
||||||
box, _ := createAccessBox(t)
|
|
||||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
|
||||||
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
|
||||||
r = r.WithContext(ctx)
|
|
||||||
hc.Handler().CreateBucketHandler(w, r)
|
|
||||||
assertStatus(t, w, http.StatusOK)
|
|
||||||
|
|
||||||
w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
|
|
||||||
ctx = middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
|
||||||
r = r.WithContext(ctx)
|
|
||||||
hc.Handler().PutBucketCorsHandler(w, r)
|
|
||||||
assertStatus(t, w, http.StatusOK)
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
origin string
|
|
||||||
method string
|
|
||||||
headers string
|
|
||||||
expectedStatus int
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "Valid",
|
|
||||||
origin: "http://www.example.com",
|
|
||||||
method: "GET",
|
|
||||||
headers: "Authorization",
|
|
||||||
expectedStatus: http.StatusOK,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Empty origin",
|
|
||||||
method: "GET",
|
|
||||||
headers: "Authorization",
|
|
||||||
expectedStatus: http.StatusBadRequest,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Empty request method",
|
|
||||||
origin: "http://www.example.com",
|
|
||||||
headers: "Authorization",
|
|
||||||
expectedStatus: http.StatusBadRequest,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Not allowed method",
|
|
||||||
origin: "http://www.example.com",
|
|
||||||
method: "PUT",
|
|
||||||
headers: "Authorization",
|
|
||||||
expectedStatus: http.StatusForbidden,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Not allowed headers",
|
|
||||||
origin: "http://www.example.com",
|
|
||||||
method: "GET",
|
|
||||||
headers: "Authorization, Last-Modified",
|
|
||||||
expectedStatus: http.StatusForbidden,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
|
|
||||||
r.Header.Set(api.Origin, tc.origin)
|
|
||||||
r.Header.Set(api.AccessControlRequestMethod, tc.method)
|
|
||||||
r.Header.Set(api.AccessControlRequestHeaders, tc.headers)
|
|
||||||
hc.Handler().Preflight(w, r)
|
|
||||||
assertStatus(t, w, tc.expectedStatus)
|
|
||||||
|
|
||||||
if tc.expectedStatus == http.StatusOK {
|
|
||||||
require.Equal(t, tc.origin, w.Header().Get(api.AccessControlAllowOrigin))
|
|
||||||
require.Equal(t, tc.method, w.Header().Get(api.AccessControlAllowMethods))
|
|
||||||
require.Equal(t, tc.headers, w.Header().Get(api.AccessControlAllowHeaders))
|
|
||||||
require.Equal(t, "x-amz-*, X-Amz-*", w.Header().Get(api.AccessControlExposeHeaders))
|
|
||||||
require.Equal(t, "true", w.Header().Get(api.AccessControlAllowCredentials))
|
|
||||||
require.Equal(t, "600", w.Header().Get(api.AccessControlMaxAge))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPreflightWildcardOrigin(t *testing.T) {
|
|
||||||
body := `
|
|
||||||
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
|
||||||
<CORSRule>
|
|
||||||
<AllowedMethod>GET</AllowedMethod>
|
|
||||||
<AllowedMethod>PUT</AllowedMethod>
|
|
||||||
<AllowedOrigin>*</AllowedOrigin>
|
|
||||||
<AllowedHeader>*</AllowedHeader>
|
|
||||||
</CORSRule>
|
|
||||||
</CORSConfiguration>
|
|
||||||
`
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
|
|
||||||
bktName := "bucket-preflight-wildcard-test"
|
|
||||||
box, _ := createAccessBox(t)
|
|
||||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
|
||||||
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
|
||||||
r = r.WithContext(ctx)
|
|
||||||
hc.Handler().CreateBucketHandler(w, r)
|
|
||||||
assertStatus(t, w, http.StatusOK)
|
|
||||||
|
|
||||||
w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
|
|
||||||
ctx = middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
|
||||||
r = r.WithContext(ctx)
|
|
||||||
hc.Handler().PutBucketCorsHandler(w, r)
|
|
||||||
assertStatus(t, w, http.StatusOK)
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
origin string
|
|
||||||
method string
|
|
||||||
headers string
|
|
||||||
expectedStatus int
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "Valid get",
|
|
||||||
origin: "http://www.example.com",
|
|
||||||
method: "GET",
|
|
||||||
headers: "Authorization, Last-Modified",
|
|
||||||
expectedStatus: http.StatusOK,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Valid put",
|
|
||||||
origin: "http://example.com",
|
|
||||||
method: "PUT",
|
|
||||||
headers: "Authorization, Content-Type",
|
|
||||||
expectedStatus: http.StatusOK,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Empty origin",
|
|
||||||
method: "GET",
|
|
||||||
headers: "Authorization, Last-Modified",
|
|
||||||
expectedStatus: http.StatusBadRequest,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Empty request method",
|
|
||||||
origin: "http://www.example.com",
|
|
||||||
headers: "Authorization, Last-Modified",
|
|
||||||
expectedStatus: http.StatusBadRequest,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Not allowed method",
|
|
||||||
origin: "http://www.example.com",
|
|
||||||
method: "DELETE",
|
|
||||||
headers: "Authorization, Last-Modified",
|
|
||||||
expectedStatus: http.StatusForbidden,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
|
|
||||||
r.Header.Set(api.Origin, tc.origin)
|
|
||||||
r.Header.Set(api.AccessControlRequestMethod, tc.method)
|
|
||||||
r.Header.Set(api.AccessControlRequestHeaders, tc.headers)
|
|
||||||
hc.Handler().Preflight(w, r)
|
|
||||||
assertStatus(t, w, tc.expectedStatus)
|
|
||||||
|
|
||||||
if tc.expectedStatus == http.StatusOK {
|
|
||||||
require.Equal(t, tc.origin, w.Header().Get(api.AccessControlAllowOrigin))
|
|
||||||
require.Equal(t, tc.method, w.Header().Get(api.AccessControlAllowMethods))
|
|
||||||
require.Equal(t, tc.headers, w.Header().Get(api.AccessControlAllowHeaders))
|
|
||||||
require.Empty(t, w.Header().Get(api.AccessControlExposeHeaders))
|
|
||||||
require.Empty(t, w.Header().Get(api.AccessControlAllowCredentials))
|
|
||||||
require.Equal(t, "0", w.Header().Get(api.AccessControlMaxAge))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -2,18 +2,21 @@ package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"fmt"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||||
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
// limitation of AWS https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
|
// limitation of AWS https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
|
||||||
|
@ -81,17 +84,10 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
networkInfo, err := h.obj.GetNetworkInfo(ctx)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not get network info", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
p := &layer.DeleteObjectParams{
|
p := &layer.DeleteObjectParams{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
Objects: versionedObject,
|
Objects: versionedObject,
|
||||||
Settings: bktSettings,
|
Settings: bktSettings,
|
||||||
NetworkInfo: networkInfo,
|
|
||||||
}
|
}
|
||||||
deletedObjects := h.obj.DeleteObjects(ctx, p)
|
deletedObjects := h.obj.DeleteObjects(ctx, p)
|
||||||
deletedObject := deletedObjects[0]
|
deletedObject := deletedObjects[0]
|
||||||
|
@ -104,6 +100,41 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var m *SendNotificationParams
|
||||||
|
|
||||||
|
if bktSettings.VersioningEnabled() && len(versionID) == 0 {
|
||||||
|
m = &SendNotificationParams{
|
||||||
|
Event: EventObjectRemovedDeleteMarkerCreated,
|
||||||
|
NotificationInfo: &data.NotificationInfo{
|
||||||
|
Name: reqInfo.ObjectName,
|
||||||
|
HashSum: deletedObject.DeleteMarkerEtag,
|
||||||
|
},
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
ReqInfo: reqInfo,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
var objID oid.ID
|
||||||
|
if len(versionID) != 0 {
|
||||||
|
if err = objID.DecodeString(versionID); err != nil {
|
||||||
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
m = &SendNotificationParams{
|
||||||
|
Event: EventObjectRemovedDelete,
|
||||||
|
NotificationInfo: &data.NotificationInfo{
|
||||||
|
Name: reqInfo.ObjectName,
|
||||||
|
Version: objID.EncodeToString(),
|
||||||
|
},
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
ReqInfo: reqInfo,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = h.sendNotifications(ctx, m); err != nil {
|
||||||
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
if deletedObject.VersionID != "" {
|
if deletedObject.VersionID != "" {
|
||||||
w.Header().Set(api.AmzVersionID, deletedObject.VersionID)
|
w.Header().Set(api.AmzVersionID, deletedObject.VersionID)
|
||||||
}
|
}
|
||||||
|
@ -148,7 +179,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
||||||
// Unmarshal list of keys to be deleted.
|
// Unmarshal list of keys to be deleted.
|
||||||
requested := &DeleteObjectsRequest{}
|
requested := &DeleteObjectsRequest{}
|
||||||
if err := h.cfg.NewXMLDecoder(r.Body).Decode(requested); err != nil {
|
if err := h.cfg.NewXMLDecoder(r.Body).Decode(requested); err != nil {
|
||||||
h.logAndSendError(w, "couldn't decode body", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()))
|
h.logAndSendError(w, "couldn't decode body", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -157,18 +188,15 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
unique := make(map[string]struct{})
|
removed := make(map[string]*layer.VersionedObject)
|
||||||
toRemove := make([]*layer.VersionedObject, 0, len(requested.Objects))
|
toRemove := make([]*layer.VersionedObject, 0, len(requested.Objects))
|
||||||
for _, obj := range requested.Objects {
|
for _, obj := range requested.Objects {
|
||||||
versionedObj := &layer.VersionedObject{
|
versionedObj := &layer.VersionedObject{
|
||||||
Name: obj.ObjectName,
|
Name: obj.ObjectName,
|
||||||
VersionID: obj.VersionID,
|
VersionID: obj.VersionID,
|
||||||
}
|
}
|
||||||
key := versionedObj.String()
|
|
||||||
if _, ok := unique[key]; !ok {
|
|
||||||
toRemove = append(toRemove, versionedObj)
|
toRemove = append(toRemove, versionedObj)
|
||||||
unique[key] = struct{}{}
|
removed[versionedObj.String()] = versionedObj
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
response := &DeleteObjectsResponse{
|
response := &DeleteObjectsResponse{
|
||||||
|
@ -188,17 +216,10 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
networkInfo, err := h.obj.GetNetworkInfo(ctx)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not get network info", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
p := &layer.DeleteObjectParams{
|
p := &layer.DeleteObjectParams{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
Objects: toRemove,
|
Objects: toRemove,
|
||||||
Settings: bktSettings,
|
Settings: bktSettings,
|
||||||
NetworkInfo: networkInfo,
|
|
||||||
IsMultiple: true,
|
IsMultiple: true,
|
||||||
}
|
}
|
||||||
deletedObjects := h.obj.DeleteObjects(ctx, p)
|
deletedObjects := h.obj.DeleteObjects(ctx, p)
|
||||||
|
@ -251,18 +272,9 @@ func (h *handler) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
sessionToken = boxData.Gate.SessionTokenForDelete()
|
sessionToken = boxData.Gate.SessionTokenForDelete()
|
||||||
}
|
}
|
||||||
|
|
||||||
skipObjCheck := false
|
|
||||||
if value, ok := r.Header[api.AmzForceBucketDelete]; ok {
|
|
||||||
s := value[0]
|
|
||||||
if s == "true" {
|
|
||||||
skipObjCheck = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = h.obj.DeleteBucket(r.Context(), &layer.DeleteBucketParams{
|
if err = h.obj.DeleteBucket(r.Context(), &layer.DeleteBucketParams{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
SessionToken: sessionToken,
|
SessionToken: sessionToken,
|
||||||
SkipCheck: skipObjCheck,
|
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
h.logAndSendError(w, "couldn't delete bucket", reqInfo, err)
|
h.logAndSendError(w, "couldn't delete bucket", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
|
|
@ -85,37 +85,6 @@ func TestDeleteBucketOnNotFoundError(t *testing.T) {
|
||||||
deleteBucket(t, hc, bktName, http.StatusNoContent)
|
deleteBucket(t, hc, bktName, http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestForceDeleteBucket(t *testing.T) {
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
|
|
||||||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
|
||||||
bktInfo := createTestBucket(hc, bktName)
|
|
||||||
|
|
||||||
putObject(hc, bktName, objName)
|
|
||||||
|
|
||||||
nodeVersion, err := hc.tree.GetUnversioned(hc.context, bktInfo, objName)
|
|
||||||
require.NoError(t, err)
|
|
||||||
var addr oid.Address
|
|
||||||
addr.SetContainer(bktInfo.CID)
|
|
||||||
addr.SetObject(nodeVersion.OID)
|
|
||||||
|
|
||||||
deleteBucketForce(t, hc, bktName, http.StatusConflict, "false")
|
|
||||||
deleteBucketForce(t, hc, bktName, http.StatusNoContent, "true")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDeleteMultipleObjectCheckUniqueness(t *testing.T) {
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
|
|
||||||
bktName, objName := "bucket", "object"
|
|
||||||
createTestBucket(hc, bktName)
|
|
||||||
|
|
||||||
putObject(hc, bktName, objName)
|
|
||||||
|
|
||||||
resp := deleteObjects(t, hc, bktName, [][2]string{{objName, emptyVersion}, {objName, emptyVersion}})
|
|
||||||
require.Empty(t, resp.Errors)
|
|
||||||
require.Len(t, resp.DeletedObjects, 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDeleteObjectsError(t *testing.T) {
|
func TestDeleteObjectsError(t *testing.T) {
|
||||||
hc := prepareHandlerContext(t)
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
@ -489,16 +458,6 @@ func putBucketVersioning(t *testing.T, tc *handlerContext, bktName string, enabl
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(t, w, http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
func getBucketVersioning(hc *handlerContext, bktName string) *VersioningConfiguration {
|
|
||||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
|
||||||
hc.Handler().GetBucketVersioningHandler(w, r)
|
|
||||||
assertStatus(hc.t, w, http.StatusOK)
|
|
||||||
|
|
||||||
res := &VersioningConfiguration{}
|
|
||||||
parseTestResponse(hc.t, w, res)
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteObject(t *testing.T, tc *handlerContext, bktName, objName, version string) (string, bool) {
|
func deleteObject(t *testing.T, tc *handlerContext, bktName, objName, version string) (string, bool) {
|
||||||
query := make(url.Values)
|
query := make(url.Values)
|
||||||
query.Add(api.QueryVersionID, version)
|
query.Add(api.QueryVersionID, version)
|
||||||
|
@ -535,13 +494,6 @@ func deleteObjectsBase(hc *handlerContext, bktName string, objVersions [][2]stri
|
||||||
return w
|
return w
|
||||||
}
|
}
|
||||||
|
|
||||||
func deleteBucketForce(t *testing.T, tc *handlerContext, bktName string, code int, value string) {
|
|
||||||
w, r := prepareTestRequest(tc, bktName, "", nil)
|
|
||||||
r.Header.Set(api.AmzForceBucketDelete, value)
|
|
||||||
tc.Handler().DeleteBucketHandler(w, r)
|
|
||||||
assertStatus(t, w, code)
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteBucket(t *testing.T, tc *handlerContext, bktName string, code int) {
|
func deleteBucket(t *testing.T, tc *handlerContext, bktName string, code int) {
|
||||||
w, r := prepareTestRequest(tc, bktName, "", nil)
|
w, r := prepareTestRequest(tc, bktName, "", nil)
|
||||||
tc.Handler().DeleteBucketHandler(w, r)
|
tc.Handler().DeleteBucketHandler(w, r)
|
||||||
|
|
|
@ -37,7 +37,7 @@ func TestSimpleGetEncrypted(t *testing.T) {
|
||||||
|
|
||||||
objInfo, err := tc.Layer().GetObjectInfo(tc.Context(), &layer.HeadObjectParams{BktInfo: bktInfo, Object: objName})
|
objInfo, err := tc.Layer().GetObjectInfo(tc.Context(), &layer.HeadObjectParams{BktInfo: bktInfo, Object: objName})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
obj, err := tc.MockedPool().GetObject(tc.Context(), layer.PrmObjectGet{Container: bktInfo.CID, Object: objInfo.ID})
|
obj, err := tc.MockedPool().ReadObject(tc.Context(), layer.PrmObjectRead{Container: bktInfo.CID, Object: objInfo.ID})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
encryptedContent, err := io.ReadAll(obj.Payload)
|
encryptedContent, err := io.ReadAll(obj.Payload)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -288,21 +288,6 @@ func completeMultipartUploadBase(hc *handlerContext, bktName, objName, uploadID
|
||||||
return w
|
return w
|
||||||
}
|
}
|
||||||
|
|
||||||
func abortMultipartUpload(hc *handlerContext, bktName, objName, uploadID string) {
|
|
||||||
w := abortMultipartUploadBase(hc, bktName, objName, uploadID)
|
|
||||||
assertStatus(hc.t, w, http.StatusNoContent)
|
|
||||||
}
|
|
||||||
|
|
||||||
func abortMultipartUploadBase(hc *handlerContext, bktName, objName, uploadID string) *httptest.ResponseRecorder {
|
|
||||||
query := make(url.Values)
|
|
||||||
query.Set(uploadIDQuery, uploadID)
|
|
||||||
|
|
||||||
w, r := prepareTestFullRequest(hc, bktName, objName, query, nil)
|
|
||||||
hc.Handler().AbortMultipartUploadHandler(w, r)
|
|
||||||
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
func uploadPartEncrypted(hc *handlerContext, bktName, objName, uploadID string, num, size int) (string, []byte) {
|
func uploadPartEncrypted(hc *handlerContext, bktName, objName, uploadID string, num, size int) (string, []byte) {
|
||||||
return uploadPartBase(hc, bktName, objName, true, uploadID, num, size)
|
return uploadPartBase(hc, bktName, objName, true, uploadID, num, size)
|
||||||
}
|
}
|
||||||
|
|
|
@ -184,7 +184,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
t := &data.ObjectVersion{
|
t := &layer.ObjectVersion{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ObjectName: info.Name,
|
ObjectName: info.Name,
|
||||||
VersionID: info.VersionID(),
|
VersionID: info.VersionID(),
|
||||||
|
|
|
@ -228,14 +228,6 @@ func getObjectRange(t *testing.T, tc *handlerContext, bktName, objName string, s
|
||||||
return content
|
return content
|
||||||
}
|
}
|
||||||
|
|
||||||
func getObjectVersion(tc *handlerContext, bktName, objName, version string) []byte {
|
|
||||||
w := getObjectBaseResponse(tc, bktName, objName, version)
|
|
||||||
assertStatus(tc.t, w, http.StatusOK)
|
|
||||||
content, err := io.ReadAll(w.Result().Body)
|
|
||||||
require.NoError(tc.t, err)
|
|
||||||
return content
|
|
||||||
}
|
|
||||||
|
|
||||||
func getObjectAssertS3Error(hc *handlerContext, bktName, objName, version string, code errors.ErrorCode) {
|
func getObjectAssertS3Error(hc *handlerContext, bktName, objName, version string, code errors.ErrorCode) {
|
||||||
w := getObjectBaseResponse(hc, bktName, objName, version)
|
w := getObjectBaseResponse(hc, bktName, objName, version)
|
||||||
assertS3Error(hc.t, w, errors.GetAPIError(code))
|
assertS3Error(hc.t, w, errors.GetAPIError(code))
|
||||||
|
|
|
@ -1,998 +0,0 @@
|
||||||
//go:build gofuzz
|
|
||||||
// +build gofuzz
|
|
||||||
|
|
||||||
package handler
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"crypto/md5"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/xml"
|
|
||||||
"mime/multipart"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
tt "testing" // read https://github.com/AdamKorcz/go-118-fuzz-build?tab=readme-ov-file#workflow
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
|
||||||
engineiam "git.frostfs.info/TrueCloudLab/policy-engine/iam"
|
|
||||||
utils "github.com/trailofbits/go-fuzz-utils"
|
|
||||||
"go.uber.org/zap/zaptest"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
fuzzBktName string
|
|
||||||
fuzzBox *accessbox.Box
|
|
||||||
fuzzHc *handlerContextBase
|
|
||||||
fuzzt *tt.T
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
fuzzSuccessExitCode = 0
|
|
||||||
fuzzFailExitCode = -1
|
|
||||||
)
|
|
||||||
|
|
||||||
func createTestBucketAndInitContext() {
|
|
||||||
fuzzt = new(tt.T)
|
|
||||||
|
|
||||||
log := zaptest.NewLogger(fuzzt)
|
|
||||||
var err error
|
|
||||||
fuzzHc, err = prepareHandlerContextBase(layer.DefaultCachesConfigs(log))
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzBktName = "bucket"
|
|
||||||
fuzzBox, _ = createAccessBox(fuzzt)
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := httptest.NewRequest(http.MethodPut, defaultURL, nil)
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
fuzzHc.Handler().CreateBucketHandler(w, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
func prepareStrings(tp *utils.TypeProvider, count int) ([]string, error) {
|
|
||||||
array := make([]string, count)
|
|
||||||
var err error
|
|
||||||
|
|
||||||
for i := 0; i < count; i++ {
|
|
||||||
err = tp.Reset()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
array[i], err = tp.GetString()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return array, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func addMD5Header(tp *utils.TypeProvider, r *http.Request, rawBody []byte) error {
|
|
||||||
if len(rawBody) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
rand, err := tp.GetBool()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if rand == true {
|
|
||||||
var dst []byte
|
|
||||||
base64.StdEncoding.Encode(dst, rawBody)
|
|
||||||
hash := md5.Sum(dst)
|
|
||||||
r.Header.Set("Content-Md5", hex.EncodeToString(hash[:]))
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func generateParams(tp *utils.TypeProvider, input string, params []string) (string, error) {
|
|
||||||
input += "?"
|
|
||||||
|
|
||||||
count, err := tp.GetInt()
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
count = count % len(params)
|
|
||||||
if count < 0 {
|
|
||||||
count += len(params)
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := 0; i < count; i++ {
|
|
||||||
position, err := tp.GetInt()
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
position = position % len(params)
|
|
||||||
if position < 0 {
|
|
||||||
position += len(params)
|
|
||||||
}
|
|
||||||
|
|
||||||
v, err := tp.GetString()
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
input += params[position] + "=" + v + "&"
|
|
||||||
}
|
|
||||||
|
|
||||||
return input, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func generateHeaders(tp *utils.TypeProvider, r *http.Request, params []string) error {
|
|
||||||
count, err := tp.GetInt()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
count = count % len(params)
|
|
||||||
if count < 0 {
|
|
||||||
count += len(params)
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := 0; i < count; i++ {
|
|
||||||
position, err := tp.GetInt()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
position = position % len(params)
|
|
||||||
if position < 0 {
|
|
||||||
position += len(params)
|
|
||||||
}
|
|
||||||
|
|
||||||
v, err := tp.GetString()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
r.Header.Set(params[position], v)
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzCreateBucketHandler() {
|
|
||||||
fuzzt = new(tt.T)
|
|
||||||
|
|
||||||
log := zaptest.NewLogger(fuzzt)
|
|
||||||
var err error
|
|
||||||
fuzzHc, err = prepareHandlerContextBase(layer.DefaultCachesConfigs(log))
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzBox, _ = createAccessBox(fuzzt)
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzCreateBucketHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
strings, err := prepareStrings(tp, 4)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
bktName := strings[0]
|
|
||||||
body := strings[1]
|
|
||||||
|
|
||||||
bodyXml, err := xml.Marshal(body)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := httptest.NewRequest(http.MethodPut, defaultURL, bytes.NewReader(bodyXml))
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: bktName, Object: ""}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-acl", "x-amz-bucket-object-lock-enabled", "x-amz-grant-full-control", "x-amz-grant-read", "x-amz-grant-read-acp", "x-amz-grant-write", "x-amz-grant-write-acp", "x-amz-object-ownership"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().CreateBucketHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzCreateBucketHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzCreateBucketHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzPutBucketCorsHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzPutBucketCorsHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
var cors data.CORSConfiguration
|
|
||||||
err = tp.Fill(&cors)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
bodyXml, err := xml.Marshal(cors)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := httptest.NewRequest(http.MethodPut, defaultURL+"?cors", bytes.NewReader(bodyXml))
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().PutBucketCorsHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzPutBucketCorsHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzPutBucketCorsHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzPutBucketPolicyHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzPutBucketPolicyHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzPutBucketPolicyHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzPutBucketPolicyHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
var policy engineiam.Policy
|
|
||||||
err = tp.Fill(&policy)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
bodyXml, err := xml.Marshal(policy)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := httptest.NewRequest(http.MethodPut, defaultURL+"?policy", bytes.NewReader(bodyXml))
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-confirm-remove-self-bucket-access"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
err = addMD5Header(tp, r, bodyXml)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().PutBucketPolicyHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzDeleteMultipleObjectsHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzDeleteMultipleObjectsHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzDeleteMultipleObjectsHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzDeleteMultipleObjectsHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
var body DeleteObjectsRequest
|
|
||||||
err = tp.Fill(&body)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
bodyXml, err := xml.Marshal(body)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := httptest.NewRequest(http.MethodPost, defaultURL+"?delete", bytes.NewReader(bodyXml))
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-bypass-governance-retention", "x-amz-mfa"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
err = addMD5Header(tp, r, bodyXml)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().DeleteMultipleObjectsHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzPostObject() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzPostObject(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzPostObject(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func postObject(tp *utils.TypeProvider) ([]byte, string, error) {
|
|
||||||
strings, err := prepareStrings(tp, 2)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
bodyXml, err := xml.Marshal(strings[0])
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
objName := strings[1]
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := httptest.NewRequest(http.MethodPost, defaultURL, bytes.NewReader(bodyXml))
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"X-Amz-Grant-Read", "X-Amz-Grant-Full-Control", "X-Amz-Grant-Write", "X-Amz-Acl", "x-amz-expected-bucket-owner"})
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
var file multipart.Form
|
|
||||||
err = tp.Fill(&file)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
r.MultipartForm = &file
|
|
||||||
|
|
||||||
fuzzHc.Handler().PostObject(w, r)
|
|
||||||
|
|
||||||
return bodyXml, objName, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzPostObject(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
_, _, err = postObject(tp)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzDeleteBucketHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzDeleteBucketHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzDeleteBucketHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzDeleteBucketHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := httptest.NewRequest(http.MethodDelete, defaultURL, nil)
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().DeleteBucketHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzDeleteBucketCorsHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzDeleteBucketCorsHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzDeleteBucketCorsHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzDeleteBucketCorsHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := httptest.NewRequest(http.MethodDelete, defaultURL+"?cors", nil)
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().DeleteBucketCorsHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzDeleteBucketPolicyHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzDeleteBucketPolicyHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzDeleteBucketPolicyHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzDeleteBucketPolicyHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := httptest.NewRequest(http.MethodDelete, defaultURL+"?policy", nil)
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().DeleteBucketPolicyHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzCopyObjectHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzCopyObjectHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzCopyObjectHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzCopyObjectHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
var r *http.Request
|
|
||||||
|
|
||||||
key, err := tp.GetString()
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
params, err := generateParams(tp, key, []string{"versionId"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
r = httptest.NewRequest(http.MethodPut, defaultURL+params, nil)
|
|
||||||
if r != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-acl", "x-amz-checksum-algorithm", "x-amz-copy-source", "x-amz-copy-source-if-match", "x-amz-copy-source-if-match", "x-amz-copy-source-if-unmodified-since", "x-amz-copy-source-if-modified-since", "x-amz-copy-source-if-none-match", "x-amz-copy-source-if-modified-since", "x-amz-copy-source-if-none-match", "x-amz-copy-source-if-none-match", "x-amz-copy-source-if-modified-since", "x-amz-copy-source-if-unmodified-since", "x-amz-copy-source-if-match", "x-amz-copy-source-if-unmodified-since", "x-amz-copy-source-server-side-encryption-customer-algorithm", "x-amz-copy-source-server-side-encryption-customer-key", "x-amz-copy-source-server-side-encryption-customer-key-MD5", "x-amz-expected-bucket-owner", "x-amz-grant-full-control", "x-amz-grant-read", "x-amz-grant-read-acp", "x-amz-grant-write-acp", "x-amz-metadata-directive", "x-amz-website-redirect-location", "x-amz-object-lock-legal-hold", "x-amz-object-lock-mode", "x-amz-object-lock-retain-until-date", "x-amz-request-payer", "x-amz-server-side-encryption", "x-amz-server-side-encryption-aws-kms-key-id", "x-amz-server-side-encryption-bucket-key-enabled", "x-amz-server-side-encryption-context", "x-amz-server-side-encryption-customer-algorithm", "x-amz-server-side-encryption-customer-key", "x-amz-server-side-encryption-customer-key-MD5", "x-amz-source-expected-bucket-owner", "x-amz-storage-class", "x-amz-tagging", "x-amz-tagging-directive", "x-amz-website-redirect-location"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().CopyObjectHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzDeleteObjectHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzDeleteObjectHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzDeleteObjectHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzDeleteObjectHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
_, objName, err := postObject(tp)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
var r *http.Request
|
|
||||||
|
|
||||||
params, err := generateParams(tp, objName, []string{"versionId"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
r = httptest.NewRequest(http.MethodDelete, defaultURL+params, nil)
|
|
||||||
if r != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-bypass-governance-retention", "x-amz-mfa"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().DeleteObjectHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzGetObjectHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzGetObjectHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzGetObjectHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzGetObjectHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
_, objName, err := postObject(tp)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
params, err := generateParams(tp, objName, []string{"versionId", "partNumber", "Range", "response-content-type", "response-content-language", "response-expires", "response-cache-control", "response-content-disposition", "response-content-encoding"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
|
|
||||||
r := httptest.NewRequest(http.MethodGet, defaultURL+params, nil)
|
|
||||||
if r != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "If-Match", "If-None-Match", "If-Modified-Since", "If-Unmodified-Since", "x-amz-server-side-encryption-customer-algorithm", "x-amz-server-side-encryption-customer-key", "x-amz-server-side-encryption-customer-key-MD5", "Range"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().GetObjectHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzPutObjectHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzPutObjectHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
objName, err := tp.GetString()
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := tp.GetBytes()
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
|
|
||||||
r := httptest.NewRequest(http.MethodPut, defaultURL+objName, bytes.NewReader(body))
|
|
||||||
if r != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "X-Amz-Grant-Read", "X-Amz-Grant-Full-Control", "X-Amz-Grant-Write", "X-Amz-Acl", "X-Amz-Tagging", "Content-Type", "Cache-Control", "Expires", "Content-Language", "Content-Encoding", "x-amz-server-side-encryption-customer-algorithm", "x-amz-server-side-encryption-customer-key", "x-amz-server-side-encryption-customer-key-MD5", "X-Amz-Content-Sha256", "X-Amz-Object-Lock-Legal-Hold", "X-Amz-Object-Lock-Mode", "X-Amz-Object-Lock-Retain-Until-Date", "X-Amz-Bypass-Governance-Retention", "X-Amz-Meta-*"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
err = addMD5Header(tp, r, body)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().PutObjectHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzPutObjectHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzPutObjectHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzPutObjectLegalHoldHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzPutObjectLegalHoldHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
_, objName, err := postObject(tp)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
var hold data.LegalHold
|
|
||||||
err = tp.Fill(&hold)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
rawBody, err := xml.Marshal(hold)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
|
|
||||||
r := httptest.NewRequest(http.MethodPut, defaultURL+objName+"?legal-hold", bytes.NewReader(rawBody))
|
|
||||||
if r != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = addMD5Header(tp, r, rawBody)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().PutObjectLegalHoldHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzPutObjectLegalHoldHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzPutObjectLegalHoldHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzPutBucketObjectLockConfigHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzPutBucketObjectLockConfigHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
var hold data.ObjectLockConfiguration
|
|
||||||
err = tp.Fill(&hold)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
rawBody, err := xml.Marshal(&hold)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
|
|
||||||
r := httptest.NewRequest(http.MethodPut, defaultURL+"?object-lock", bytes.NewReader(rawBody))
|
|
||||||
if r != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = addMD5Header(tp, r, rawBody)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-bucket-object-lock-token"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().PutBucketObjectLockConfigHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzPutBucketObjectLockConfigHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzPutBucketObjectLockConfigHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzPutObjectRetentionHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzPutObjectRetentionHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
_, objName, err := postObject(tp)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
var retention data.Retention
|
|
||||||
err = tp.Fill(&retention)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
rawBody, err := xml.Marshal(retention)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
|
|
||||||
r := httptest.NewRequest(http.MethodPut, defaultURL+objName+"?retention", bytes.NewReader(rawBody))
|
|
||||||
if r != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = addMD5Header(tp, r, rawBody)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-bypass-governance-retention"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().PutObjectRetentionHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzPutObjectRetentionHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzPutObjectRetentionHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func InitFuzzPutBucketAclHandler() {
|
|
||||||
createTestBucketAndInitContext()
|
|
||||||
}
|
|
||||||
|
|
||||||
func DoFuzzPutBucketAclHandler(input []byte) int {
|
|
||||||
// FUZZER INIT
|
|
||||||
if len(input) < 100 {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
tp, err := utils.NewTypeProvider(input)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
var policy AccessControlPolicy
|
|
||||||
err = tp.Fill(&policy)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
rawBody, err := xml.Marshal(policy)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := httptest.NewRequest(http.MethodPut, defaultURL+"?acl", bytes.NewReader(rawBody))
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
|
|
||||||
r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
|
|
||||||
|
|
||||||
err = addMD5Header(tp, r, rawBody)
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-acl", "x-amz-expected-bucket-owner", "x-amz-grant-full-control", "x-amz-grant-read", "x-amz-grant-read-acp", "x-amz-grant-write", "x-amz-grant-write-acp"})
|
|
||||||
if err != nil {
|
|
||||||
return fuzzFailExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
fuzzHc.Handler().PutBucketACLHandler(w, r)
|
|
||||||
|
|
||||||
return fuzzSuccessExitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func FuzzPutBucketAclHandler(f *testing.F) {
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
DoFuzzPutBucketAclHandler(data)
|
|
||||||
})
|
|
||||||
}
|
|
|
@ -4,10 +4,8 @@ import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"encoding/hex"
|
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
@ -37,12 +35,8 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type handlerContext struct {
|
type handlerContext struct {
|
||||||
*handlerContextBase
|
|
||||||
t *testing.T
|
|
||||||
}
|
|
||||||
|
|
||||||
type handlerContextBase struct {
|
|
||||||
owner user.ID
|
owner user.ID
|
||||||
|
t *testing.T
|
||||||
h *handler
|
h *handler
|
||||||
tp *layer.TestFrostFS
|
tp *layer.TestFrostFS
|
||||||
tree *tree.Tree
|
tree *tree.Tree
|
||||||
|
@ -54,19 +48,19 @@ type handlerContextBase struct {
|
||||||
cache *layer.Cache
|
cache *layer.Cache
|
||||||
}
|
}
|
||||||
|
|
||||||
func (hc *handlerContextBase) Handler() *handler {
|
func (hc *handlerContext) Handler() *handler {
|
||||||
return hc.h
|
return hc.h
|
||||||
}
|
}
|
||||||
|
|
||||||
func (hc *handlerContextBase) MockedPool() *layer.TestFrostFS {
|
func (hc *handlerContext) MockedPool() *layer.TestFrostFS {
|
||||||
return hc.tp
|
return hc.tp
|
||||||
}
|
}
|
||||||
|
|
||||||
func (hc *handlerContextBase) Layer() *layer.Layer {
|
func (hc *handlerContext) Layer() layer.Client {
|
||||||
return hc.h.obj
|
return hc.h.obj
|
||||||
}
|
}
|
||||||
|
|
||||||
func (hc *handlerContextBase) Context() context.Context {
|
func (hc *handlerContext) Context() context.Context {
|
||||||
return hc.context
|
return hc.context
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -76,6 +70,7 @@ type configMock struct {
|
||||||
defaultCopiesNumbers []uint32
|
defaultCopiesNumbers []uint32
|
||||||
bypassContentEncodingInChunks bool
|
bypassContentEncodingInChunks bool
|
||||||
md5Enabled bool
|
md5Enabled bool
|
||||||
|
aclEnabled bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *configMock) DefaultPlacementPolicy(_ string) netmap.PlacementPolicy {
|
func (c *configMock) DefaultPlacementPolicy(_ string) netmap.PlacementPolicy {
|
||||||
|
@ -107,6 +102,10 @@ func (c *configMock) DefaultMaxAge() int {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *configMock) NotificatorEnabled() bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func (c *configMock) ResolveZoneList() []string {
|
func (c *configMock) ResolveZoneList() []string {
|
||||||
return []string{}
|
return []string{}
|
||||||
}
|
}
|
||||||
|
@ -123,51 +122,31 @@ func (c *configMock) MD5Enabled() bool {
|
||||||
return c.md5Enabled
|
return c.md5Enabled
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *configMock) ACLEnabled() bool {
|
||||||
|
return c.aclEnabled
|
||||||
|
}
|
||||||
|
|
||||||
func (c *configMock) ResolveNamespaceAlias(ns string) string {
|
func (c *configMock) ResolveNamespaceAlias(ns string) string {
|
||||||
return ns
|
return ns
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *configMock) RetryMaxAttempts() int {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *configMock) RetryMaxBackoff() time.Duration {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *configMock) RetryStrategy() RetryStrategy {
|
|
||||||
return RetryStrategyConstant
|
|
||||||
}
|
|
||||||
|
|
||||||
func prepareHandlerContext(t *testing.T) *handlerContext {
|
func prepareHandlerContext(t *testing.T) *handlerContext {
|
||||||
hc, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(zap.NewExample()))
|
return prepareHandlerContextBase(t, layer.DefaultCachesConfigs(zap.NewExample()))
|
||||||
require.NoError(t, err)
|
|
||||||
return &handlerContext{
|
|
||||||
handlerContextBase: hc,
|
|
||||||
t: t,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext {
|
func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext {
|
||||||
hc, err := prepareHandlerContextBase(getMinCacheConfig(zap.NewExample()))
|
return prepareHandlerContextBase(t, getMinCacheConfig(zap.NewExample()))
|
||||||
require.NoError(t, err)
|
|
||||||
return &handlerContext{
|
|
||||||
handlerContextBase: hc,
|
|
||||||
t: t,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func prepareHandlerContextBase(cacheCfg *layer.CachesConfig) (*handlerContextBase, error) {
|
func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *handlerContext {
|
||||||
key, err := keys.NewPrivateKey()
|
key, err := keys.NewPrivateKey()
|
||||||
if err != nil {
|
require.NoError(t, err)
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
log := zap.NewExample()
|
l := zap.NewExample()
|
||||||
tp := layer.NewTestFrostFS(key)
|
tp := layer.NewTestFrostFS(key)
|
||||||
|
|
||||||
testResolver := &resolver.Resolver{Name: "test_resolver"}
|
testResolver := &resolver.Resolver{Name: "test_resolver"}
|
||||||
testResolver.SetResolveFunc(func(_ context.Context, _, name string) (cid.ID, error) {
|
testResolver.SetResolveFunc(func(_ context.Context, name string) (cid.ID, error) {
|
||||||
return tp.ContainerID(name)
|
return tp.ContainerID(name)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -175,9 +154,7 @@ func prepareHandlerContextBase(cacheCfg *layer.CachesConfig) (*handlerContextBas
|
||||||
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
|
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
|
||||||
|
|
||||||
memCli, err := tree.NewTreeServiceClientMemory()
|
memCli, err := tree.NewTreeServiceClientMemory()
|
||||||
if err != nil {
|
require.NoError(t, err)
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
treeMock := tree.NewTree(memCli, zap.NewExample())
|
treeMock := tree.NewTree(memCli, zap.NewExample())
|
||||||
|
|
||||||
|
@ -194,38 +171,31 @@ func prepareHandlerContextBase(cacheCfg *layer.CachesConfig) (*handlerContextBas
|
||||||
|
|
||||||
var pp netmap.PlacementPolicy
|
var pp netmap.PlacementPolicy
|
||||||
err = pp.DecodeString("REP 1")
|
err = pp.DecodeString("REP 1")
|
||||||
if err != nil {
|
require.NoError(t, err)
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg := &configMock{
|
cfg := &configMock{
|
||||||
defaultPolicy: pp,
|
defaultPolicy: pp,
|
||||||
}
|
}
|
||||||
h := &handler{
|
h := &handler{
|
||||||
log: log,
|
log: l,
|
||||||
obj: layer.NewLayer(log, tp, layerCfg),
|
obj: layer.NewLayer(l, tp, layerCfg),
|
||||||
cfg: cfg,
|
cfg: cfg,
|
||||||
ape: newAPEMock(),
|
ape: newAPEMock(),
|
||||||
frostfsid: newFrostfsIDMock(),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
accessBox, err := newTestAccessBox(key)
|
return &handlerContext{
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &handlerContextBase{
|
|
||||||
owner: owner,
|
owner: owner,
|
||||||
|
t: t,
|
||||||
h: h,
|
h: h,
|
||||||
tp: tp,
|
tp: tp,
|
||||||
tree: treeMock,
|
tree: treeMock,
|
||||||
context: middleware.SetBox(context.Background(), &middleware.Box{AccessBox: accessBox}),
|
context: middleware.SetBoxData(context.Background(), newTestAccessBox(t, key)),
|
||||||
config: cfg,
|
config: cfg,
|
||||||
|
|
||||||
layerFeatures: features,
|
layerFeatures: features,
|
||||||
treeMock: memCli,
|
treeMock: memCli,
|
||||||
cache: layerCfg.Cache,
|
cache: layerCfg.Cache,
|
||||||
}, nil
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func getMinCacheConfig(logger *zap.Logger) *layer.CachesConfig {
|
func getMinCacheConfig(logger *zap.Logger) *layer.CachesConfig {
|
||||||
|
@ -337,32 +307,6 @@ func (a *apeMock) SaveACLChains(cid string, chains []*chain.Chain) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type frostfsidMock struct {
|
|
||||||
data map[string]*keys.PublicKey
|
|
||||||
}
|
|
||||||
|
|
||||||
func newFrostfsIDMock() *frostfsidMock {
|
|
||||||
return &frostfsidMock{data: map[string]*keys.PublicKey{}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *frostfsidMock) GetUserAddress(account, user string) (string, error) {
|
|
||||||
res, ok := f.data[account+user]
|
|
||||||
if !ok {
|
|
||||||
return "", fmt.Errorf("not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
return res.Address(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *frostfsidMock) GetUserKey(account, user string) (string, error) {
|
|
||||||
res, ok := f.data[account+user]
|
|
||||||
if !ok {
|
|
||||||
return "", fmt.Errorf("not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
return hex.EncodeToString(res.Bytes()), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func createTestBucket(hc *handlerContext, bktName string) *data.BucketInfo {
|
func createTestBucket(hc *handlerContext, bktName string) *data.BucketInfo {
|
||||||
info := createBucket(hc, bktName)
|
info := createBucket(hc, bktName)
|
||||||
return info.BktInfo
|
return info.BktInfo
|
||||||
|
@ -442,7 +386,7 @@ func prepareTestRequestWithQuery(hc *handlerContext, bktName, objName string, qu
|
||||||
r := httptest.NewRequest(http.MethodPut, defaultURL, bytes.NewReader(body))
|
r := httptest.NewRequest(http.MethodPut, defaultURL, bytes.NewReader(body))
|
||||||
r.URL.RawQuery = query.Encode()
|
r.URL.RawQuery = query.Encode()
|
||||||
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: bktName, Object: objName})
|
||||||
r = r.WithContext(middleware.SetReqInfo(hc.Context(), reqInfo))
|
r = r.WithContext(middleware.SetReqInfo(hc.Context(), reqInfo))
|
||||||
|
|
||||||
return w, r
|
return w, r
|
||||||
|
@ -452,7 +396,7 @@ func prepareTestPayloadRequest(hc *handlerContext, bktName, objName string, payl
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
r := httptest.NewRequest(http.MethodPut, defaultURL, payload)
|
r := httptest.NewRequest(http.MethodPut, defaultURL, payload)
|
||||||
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: bktName, Object: objName})
|
||||||
r = r.WithContext(middleware.SetReqInfo(hc.Context(), reqInfo))
|
r = r.WithContext(middleware.SetReqInfo(hc.Context(), reqInfo))
|
||||||
|
|
||||||
return w, r
|
return w, r
|
||||||
|
|
|
@ -70,7 +70,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
t := &data.ObjectVersion{
|
t := &layer.ObjectVersion{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ObjectName: info.Name,
|
ObjectName: info.Name,
|
||||||
VersionID: info.VersionID(),
|
VersionID: info.VersionID(),
|
||||||
|
|
|
@ -7,9 +7,12 @@ import (
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||||
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
@ -81,6 +84,31 @@ func headObject(t *testing.T, tc *handlerContext, bktName, objName string, heade
|
||||||
assertStatus(t, w, status)
|
assertStatus(t, w, status)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInvalidAccessThroughCache(t *testing.T) {
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
bktName, objName := "bucket-for-cache", "obj-for-cache"
|
||||||
|
bktInfo, _ := createBucketAndObject(hc, bktName, objName)
|
||||||
|
setContainerEACL(hc, bktInfo.CID)
|
||||||
|
|
||||||
|
headObject(t, hc, bktName, objName, nil, http.StatusOK)
|
||||||
|
|
||||||
|
w, r := prepareTestRequest(hc, bktName, objName, nil)
|
||||||
|
hc.Handler().HeadObjectHandler(w, r.WithContext(middleware.SetBoxData(r.Context(), newTestAccessBox(t, nil))))
|
||||||
|
assertStatus(t, w, http.StatusForbidden)
|
||||||
|
}
|
||||||
|
|
||||||
|
func setContainerEACL(hc *handlerContext, cnrID cid.ID) {
|
||||||
|
table := eacl.NewTable()
|
||||||
|
table.SetCID(cnrID)
|
||||||
|
for _, op := range fullOps {
|
||||||
|
table.AddRecord(getOthersRecord(op, eacl.ActionDeny))
|
||||||
|
}
|
||||||
|
|
||||||
|
err := hc.MockedPool().SetContainerEACL(hc.Context(), *table, nil)
|
||||||
|
require.NoError(hc.t, err)
|
||||||
|
}
|
||||||
|
|
||||||
func TestHeadObject(t *testing.T) {
|
func TestHeadObject(t *testing.T) {
|
||||||
hc := prepareHandlerContextWithMinCache(t)
|
hc := prepareHandlerContextWithMinCache(t)
|
||||||
bktName, objName := "bucket", "obj"
|
bktName, objName := "bucket", "obj"
|
||||||
|
@ -119,25 +147,21 @@ func TestIsAvailableToResolve(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newTestAccessBox(key *keys.PrivateKey) (*accessbox.Box, error) {
|
func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box {
|
||||||
var err error
|
var err error
|
||||||
if key == nil {
|
if key == nil {
|
||||||
key, err = keys.NewPrivateKey()
|
key, err = keys.NewPrivateKey()
|
||||||
if err != nil {
|
require.NoError(t, err)
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var btoken bearer.Token
|
var btoken bearer.Token
|
||||||
btoken.SetImpersonate(true)
|
btoken.SetEACLTable(*eacl.NewTable())
|
||||||
err = btoken.Sign(key.PrivateKey)
|
err = btoken.Sign(key.PrivateKey)
|
||||||
if err != nil {
|
require.NoError(t, err)
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &accessbox.Box{
|
return &accessbox.Box{
|
||||||
Gate: &accessbox.GateData{
|
Gate: &accessbox.GateData{
|
||||||
BearerToken: &btoken,
|
BearerToken: &btoken,
|
||||||
},
|
},
|
||||||
}, nil
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,235 +0,0 @@
|
||||||
package handler
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
maxRules = 1000
|
|
||||||
maxRuleIDLen = 255
|
|
||||||
maxNewerNoncurrentVersions = 100
|
|
||||||
)
|
|
||||||
|
|
||||||
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
ctx := r.Context()
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := h.obj.GetBucketLifecycleConfiguration(ctx, bktInfo)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not get bucket lifecycle configuration", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = middleware.EncodeToResponse(w, cfg); err != nil {
|
|
||||||
h.logAndSendError(w, "could not encode GetBucketLifecycle response", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
tee := io.TeeReader(r.Body, &buf)
|
|
||||||
ctx := r.Context()
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
|
||||||
|
|
||||||
// Content-Md5 is required and should be set
|
|
||||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html
|
|
||||||
if _, ok := r.Header[api.ContentMD5]; !ok {
|
|
||||||
h.logAndSendError(w, "missing Content-MD5", reqInfo, apiErr.GetAPIError(apiErr.ErrMissingContentMD5))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg := new(data.LifecycleConfiguration)
|
|
||||||
if err = h.cfg.NewXMLDecoder(tee).Decode(cfg); err != nil {
|
|
||||||
h.logAndSendError(w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrMalformedXML), err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = checkLifecycleConfiguration(cfg); err != nil {
|
|
||||||
h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrMalformedXML), err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
params := &layer.PutBucketLifecycleParams{
|
|
||||||
BktInfo: bktInfo,
|
|
||||||
LifecycleCfg: cfg,
|
|
||||||
LifecycleReader: &buf,
|
|
||||||
MD5Hash: r.Header.Get(api.ContentMD5),
|
|
||||||
}
|
|
||||||
|
|
||||||
params.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = h.obj.PutBucketLifecycleConfiguration(ctx, params); err != nil {
|
|
||||||
h.logAndSendError(w, "could not put bucket lifecycle configuration", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
ctx := r.Context()
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = h.obj.DeleteBucketLifecycleConfiguration(ctx, bktInfo); err != nil {
|
|
||||||
h.logAndSendError(w, "could not delete bucket lifecycle configuration", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration) error {
|
|
||||||
if len(cfg.Rules) > maxRules {
|
|
||||||
return fmt.Errorf("number of rules cannot be greater than %d", maxRules)
|
|
||||||
}
|
|
||||||
|
|
||||||
ids := make(map[string]struct{}, len(cfg.Rules))
|
|
||||||
for _, rule := range cfg.Rules {
|
|
||||||
if _, ok := ids[rule.ID]; ok && rule.ID != "" {
|
|
||||||
return fmt.Errorf("duplicate 'ID': %s", rule.ID)
|
|
||||||
}
|
|
||||||
ids[rule.ID] = struct{}{}
|
|
||||||
|
|
||||||
if len(rule.ID) > maxRuleIDLen {
|
|
||||||
return fmt.Errorf("'ID' value cannot be longer than %d characters", maxRuleIDLen)
|
|
||||||
}
|
|
||||||
|
|
||||||
if rule.Status != data.LifecycleStatusEnabled && rule.Status != data.LifecycleStatusDisabled {
|
|
||||||
return fmt.Errorf("invalid lifecycle status: %s", rule.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
if rule.AbortIncompleteMultipartUpload == nil && rule.Expiration == nil && rule.NonCurrentVersionExpiration == nil {
|
|
||||||
return fmt.Errorf("at least one action needs to be specified in a rule")
|
|
||||||
}
|
|
||||||
|
|
||||||
if rule.AbortIncompleteMultipartUpload != nil {
|
|
||||||
if rule.AbortIncompleteMultipartUpload.DaysAfterInitiation != nil &&
|
|
||||||
*rule.AbortIncompleteMultipartUpload.DaysAfterInitiation <= 0 {
|
|
||||||
return fmt.Errorf("days after initiation must be a positive integer: %d", *rule.AbortIncompleteMultipartUpload.DaysAfterInitiation)
|
|
||||||
}
|
|
||||||
|
|
||||||
if rule.Filter != nil && (rule.Filter.Tag != nil || (rule.Filter.And != nil && len(rule.Filter.And.Tags) > 0)) {
|
|
||||||
return fmt.Errorf("abort incomplete multipart upload cannot be specified with tags")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if rule.Expiration != nil {
|
|
||||||
if rule.Expiration.ExpiredObjectDeleteMarker != nil {
|
|
||||||
if rule.Expiration.Days != nil || rule.Expiration.Date != "" {
|
|
||||||
return fmt.Errorf("expired object delete marker cannot be specified with days or date")
|
|
||||||
}
|
|
||||||
|
|
||||||
if rule.Filter != nil && (rule.Filter.Tag != nil || (rule.Filter.And != nil && len(rule.Filter.And.Tags) > 0)) {
|
|
||||||
return fmt.Errorf("expired object delete marker cannot be specified with tags")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if rule.Expiration.Days != nil && *rule.Expiration.Days <= 0 {
|
|
||||||
return fmt.Errorf("expiration days must be a positive integer: %d", *rule.Expiration.Days)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := time.Parse("2006-01-02T15:04:05Z", rule.Expiration.Date); rule.Expiration.Date != "" && err != nil {
|
|
||||||
return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if rule.NonCurrentVersionExpiration != nil {
|
|
||||||
if rule.NonCurrentVersionExpiration.NewerNonCurrentVersions != nil &&
|
|
||||||
(*rule.NonCurrentVersionExpiration.NewerNonCurrentVersions > maxNewerNoncurrentVersions ||
|
|
||||||
*rule.NonCurrentVersionExpiration.NewerNonCurrentVersions <= 0) {
|
|
||||||
return fmt.Errorf("invalid value of newer noncurrent versions: %d", *rule.NonCurrentVersionExpiration.NewerNonCurrentVersions)
|
|
||||||
}
|
|
||||||
|
|
||||||
if rule.NonCurrentVersionExpiration.NonCurrentDays != nil && *rule.NonCurrentVersionExpiration.NonCurrentDays <= 0 {
|
|
||||||
return fmt.Errorf("invalid value of noncurrent days: %d", *rule.NonCurrentVersionExpiration.NonCurrentDays)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := checkLifecycleRuleFilter(rule.Filter); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error {
|
|
||||||
if filter == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var fields int
|
|
||||||
|
|
||||||
if filter.And != nil {
|
|
||||||
fields++
|
|
||||||
for _, tag := range filter.And.Tags {
|
|
||||||
err := checkTag(tag)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if filter.And.ObjectSizeGreaterThan != nil && filter.And.ObjectSizeLessThan != nil &&
|
|
||||||
*filter.And.ObjectSizeLessThan <= *filter.And.ObjectSizeGreaterThan {
|
|
||||||
return fmt.Errorf("the maximum object size must be larger than the minimum object size")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if filter.ObjectSizeGreaterThan != nil {
|
|
||||||
fields++
|
|
||||||
}
|
|
||||||
|
|
||||||
if filter.ObjectSizeLessThan != nil {
|
|
||||||
fields++
|
|
||||||
}
|
|
||||||
|
|
||||||
if filter.Prefix != "" {
|
|
||||||
fields++
|
|
||||||
}
|
|
||||||
|
|
||||||
if filter.Tag != nil {
|
|
||||||
fields++
|
|
||||||
err := checkTag(*filter.Tag)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if fields > 1 {
|
|
||||||
return fmt.Errorf("filter cannot have more than one field")
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
|
@ -1,457 +0,0 @@
|
||||||
package handler
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/md5"
|
|
||||||
"crypto/rand"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/xml"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
|
||||||
"github.com/mr-tron/base58"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPutBucketLifecycleConfiguration(t *testing.T) {
|
|
||||||
hc := prepareHandlerContextWithMinCache(t)
|
|
||||||
|
|
||||||
bktName := "bucket-lifecycle"
|
|
||||||
createBucket(hc, bktName)
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
body *data.LifecycleConfiguration
|
|
||||||
error bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "correct configuration",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
XMLName: xml.Name{
|
|
||||||
Space: `http://s3.amazonaws.com/doc/2006-03-01/`,
|
|
||||||
Local: "LifecycleConfiguration",
|
|
||||||
},
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Days: ptr(21),
|
|
||||||
Date: time.Now().Format("2006-01-02T15:04:05.000Z"),
|
|
||||||
},
|
|
||||||
Filter: &data.LifecycleRuleFilter{
|
|
||||||
And: &data.LifecycleRuleAndOperator{
|
|
||||||
Prefix: "prefix/",
|
|
||||||
Tags: []data.Tag{{Key: "key", Value: "value"}, {Key: "tag", Value: ""}},
|
|
||||||
ObjectSizeGreaterThan: ptr(uint64(100)),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
|
|
||||||
DaysAfterInitiation: ptr(14),
|
|
||||||
},
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
ExpiredObjectDeleteMarker: ptr(true),
|
|
||||||
},
|
|
||||||
Filter: &data.LifecycleRuleFilter{
|
|
||||||
ObjectSizeLessThan: ptr(uint64(100)),
|
|
||||||
},
|
|
||||||
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
|
||||||
NewerNonCurrentVersions: ptr(1),
|
|
||||||
NonCurrentDays: ptr(21),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "too many rules",
|
|
||||||
body: func() *data.LifecycleConfiguration {
|
|
||||||
lifecycle := new(data.LifecycleConfiguration)
|
|
||||||
for i := 0; i <= maxRules; i++ {
|
|
||||||
lifecycle.Rules = append(lifecycle.Rules, data.LifecycleRule{
|
|
||||||
ID: "Rule" + strconv.Itoa(i),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return lifecycle
|
|
||||||
}(),
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "duplicate rule ID",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
ID: "Rule",
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Days: ptr(21),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ID: "Rule",
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Days: ptr(21),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "too long rule ID",
|
|
||||||
body: func() *data.LifecycleConfiguration {
|
|
||||||
id := make([]byte, maxRuleIDLen+1)
|
|
||||||
_, err := io.ReadFull(rand.Reader, id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
return &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
ID: base58.Encode(id)[:maxRuleIDLen+1],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}(),
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid status",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: "invalid",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no actions",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Filter: &data.LifecycleRuleFilter{
|
|
||||||
Prefix: "prefix/",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid days after initiation",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
|
|
||||||
DaysAfterInitiation: ptr(0),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid expired object delete marker declaration",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Days: ptr(21),
|
|
||||||
ExpiredObjectDeleteMarker: ptr(false),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid expiration days",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Days: ptr(0),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid expiration date",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Date: "invalid",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "newer noncurrent versions is too small",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
|
||||||
NewerNonCurrentVersions: ptr(0),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "newer noncurrent versions is too large",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
|
||||||
NewerNonCurrentVersions: ptr(101),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid noncurrent days",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
|
||||||
NonCurrentDays: ptr(0),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "more than one filter field",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Days: ptr(21),
|
|
||||||
},
|
|
||||||
Filter: &data.LifecycleRuleFilter{
|
|
||||||
Prefix: "prefix/",
|
|
||||||
ObjectSizeGreaterThan: ptr(uint64(100)),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid tag in filter",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Days: ptr(21),
|
|
||||||
},
|
|
||||||
Filter: &data.LifecycleRuleFilter{
|
|
||||||
Tag: &data.Tag{},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "abort incomplete multipart upload with tag",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
|
|
||||||
DaysAfterInitiation: ptr(14),
|
|
||||||
},
|
|
||||||
Filter: &data.LifecycleRuleFilter{
|
|
||||||
Tag: &data.Tag{Key: "key", Value: "value"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "expired object delete marker with tag",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
ExpiredObjectDeleteMarker: ptr(true),
|
|
||||||
},
|
|
||||||
Filter: &data.LifecycleRuleFilter{
|
|
||||||
And: &data.LifecycleRuleAndOperator{
|
|
||||||
Tags: []data.Tag{{Key: "key", Value: "value"}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid size range",
|
|
||||||
body: &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Days: ptr(21),
|
|
||||||
},
|
|
||||||
Filter: &data.LifecycleRuleFilter{
|
|
||||||
And: &data.LifecycleRuleAndOperator{
|
|
||||||
ObjectSizeGreaterThan: ptr(uint64(100)),
|
|
||||||
ObjectSizeLessThan: ptr(uint64(100)),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
if tc.error {
|
|
||||||
putBucketLifecycleConfigurationErr(hc, bktName, tc.body, apiErrors.GetAPIError(apiErrors.ErrMalformedXML))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
putBucketLifecycleConfiguration(hc, bktName, tc.body)
|
|
||||||
|
|
||||||
cfg := getBucketLifecycleConfiguration(hc, bktName)
|
|
||||||
require.Equal(t, *tc.body, *cfg)
|
|
||||||
|
|
||||||
deleteBucketLifecycleConfiguration(hc, bktName)
|
|
||||||
getBucketLifecycleConfigurationErr(hc, bktName, apiErrors.GetAPIError(apiErrors.ErrNoSuchLifecycleConfiguration))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPutBucketLifecycleInvalidMD5(t *testing.T) {
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
|
|
||||||
bktName := "bucket-lifecycle-md5"
|
|
||||||
createBucket(hc, bktName)
|
|
||||||
|
|
||||||
lifecycle := &data.LifecycleConfiguration{
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Days: ptr(21),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
w, r := prepareTestRequest(hc, bktName, "", lifecycle)
|
|
||||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
|
||||||
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrMissingContentMD5))
|
|
||||||
|
|
||||||
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
|
|
||||||
r.Header.Set(api.ContentMD5, "")
|
|
||||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
|
||||||
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest))
|
|
||||||
|
|
||||||
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
|
|
||||||
r.Header.Set(api.ContentMD5, "some-hash")
|
|
||||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
|
||||||
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPutBucketLifecycleInvalidXML(t *testing.T) {
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
|
|
||||||
bktName := "bucket-lifecycle-invalid-xml"
|
|
||||||
createBucket(hc, bktName)
|
|
||||||
|
|
||||||
w, r := prepareTestRequest(hc, bktName, "", &data.CORSConfiguration{})
|
|
||||||
r.Header.Set(api.ContentMD5, "")
|
|
||||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
|
||||||
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrMalformedXML))
|
|
||||||
}
|
|
||||||
|
|
||||||
func putBucketLifecycleConfiguration(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) {
|
|
||||||
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
|
|
||||||
assertStatus(hc.t, w, http.StatusOK)
|
|
||||||
}
|
|
||||||
|
|
||||||
func putBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, err apiErrors.Error) {
|
|
||||||
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
|
|
||||||
assertS3Error(hc.t, w, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func putBucketLifecycleConfigurationBase(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) *httptest.ResponseRecorder {
|
|
||||||
w, r := prepareTestRequest(hc, bktName, "", cfg)
|
|
||||||
|
|
||||||
rawBody, err := xml.Marshal(cfg)
|
|
||||||
require.NoError(hc.t, err)
|
|
||||||
|
|
||||||
hash := md5.New()
|
|
||||||
hash.Write(rawBody)
|
|
||||||
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(hash.Sum(nil)))
|
|
||||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
func getBucketLifecycleConfiguration(hc *handlerContext, bktName string) *data.LifecycleConfiguration {
|
|
||||||
w := getBucketLifecycleConfigurationBase(hc, bktName)
|
|
||||||
assertStatus(hc.t, w, http.StatusOK)
|
|
||||||
res := &data.LifecycleConfiguration{}
|
|
||||||
parseTestResponse(hc.t, w, res)
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
|
|
||||||
func getBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, err apiErrors.Error) {
|
|
||||||
w := getBucketLifecycleConfigurationBase(hc, bktName)
|
|
||||||
assertS3Error(hc.t, w, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func getBucketLifecycleConfigurationBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder {
|
|
||||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
|
||||||
hc.Handler().GetBucketLifecycleHandler(w, r)
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteBucketLifecycleConfiguration(hc *handlerContext, bktName string) {
|
|
||||||
w := deleteBucketLifecycleConfigurationBase(hc, bktName)
|
|
||||||
assertStatus(hc.t, w, http.StatusNoContent)
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteBucketLifecycleConfigurationBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder {
|
|
||||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
|
||||||
hc.Handler().DeleteBucketLifecycleHandler(w, r)
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
func ptr[T any](t T) *T {
|
|
||||||
return &t
|
|
||||||
}
|
|
|
@ -133,7 +133,7 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
|
||||||
}
|
}
|
||||||
|
|
||||||
p := &layer.PutLockInfoParams{
|
p := &layer.PutLockInfoParams{
|
||||||
ObjVersion: &data.ObjectVersion{
|
ObjVersion: &layer.ObjectVersion{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ObjectName: reqInfo.ObjectName,
|
ObjectName: reqInfo.ObjectName,
|
||||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||||
|
@ -172,7 +172,7 @@ func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
p := &data.ObjectVersion{
|
p := &layer.ObjectVersion{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ObjectName: reqInfo.ObjectName,
|
ObjectName: reqInfo.ObjectName,
|
||||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||||
|
@ -221,7 +221,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
|
||||||
}
|
}
|
||||||
|
|
||||||
p := &layer.PutLockInfoParams{
|
p := &layer.PutLockInfoParams{
|
||||||
ObjVersion: &data.ObjectVersion{
|
ObjVersion: &layer.ObjectVersion{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ObjectName: reqInfo.ObjectName,
|
ObjectName: reqInfo.ObjectName,
|
||||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||||
|
@ -256,7 +256,7 @@ func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
p := &data.ObjectVersion{
|
p := &layer.ObjectVersion{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ObjectName: reqInfo.ObjectName,
|
ObjectName: reqInfo.ObjectName,
|
||||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||||
|
|
|
@ -315,7 +315,7 @@ func TestPutBucketLockConfigurationHandler(t *testing.T) {
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
r := httptest.NewRequest(http.MethodPut, defaultURL, bytes.NewReader(body))
|
r := httptest.NewRequest(http.MethodPut, defaultURL, bytes.NewReader(body))
|
||||||
r = r.WithContext(middleware.SetReqInfo(r.Context(), middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: tc.bucket}, "")))
|
r = r.WithContext(middleware.SetReqInfo(r.Context(), middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: tc.bucket})))
|
||||||
|
|
||||||
hc.Handler().PutBucketObjectLockConfigHandler(w, r)
|
hc.Handler().PutBucketObjectLockConfigHandler(w, r)
|
||||||
|
|
||||||
|
@ -388,7 +388,7 @@ func TestGetBucketLockConfigurationHandler(t *testing.T) {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
r := httptest.NewRequest(http.MethodPut, defaultURL, bytes.NewReader(nil))
|
r := httptest.NewRequest(http.MethodPut, defaultURL, bytes.NewReader(nil))
|
||||||
r = r.WithContext(middleware.SetReqInfo(r.Context(), middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: tc.bucket}, "")))
|
r = r.WithContext(middleware.SetReqInfo(r.Context(), middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: tc.bucket})))
|
||||||
|
|
||||||
hc.Handler().GetBucketObjectLockConfigHandler(w, r)
|
hc.Handler().GetBucketObjectLockConfigHandler(w, r)
|
||||||
|
|
||||||
|
|
|
@ -5,7 +5,6 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"path"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
@ -14,6 +13,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -31,7 +31,6 @@ type (
|
||||||
Bucket string `xml:"Bucket"`
|
Bucket string `xml:"Bucket"`
|
||||||
Key string `xml:"Key"`
|
Key string `xml:"Key"`
|
||||||
ETag string `xml:"ETag"`
|
ETag string `xml:"ETag"`
|
||||||
Location string `xml:"Location"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ListMultipartUploadsResponse struct {
|
ListMultipartUploadsResponse struct {
|
||||||
|
@ -56,11 +55,11 @@ type (
|
||||||
Initiator Initiator `xml:"Initiator"`
|
Initiator Initiator `xml:"Initiator"`
|
||||||
IsTruncated bool `xml:"IsTruncated"`
|
IsTruncated bool `xml:"IsTruncated"`
|
||||||
Key string `xml:"Key"`
|
Key string `xml:"Key"`
|
||||||
MaxParts int `xml:"MaxParts"`
|
MaxParts int `xml:"MaxParts,omitempty"`
|
||||||
NextPartNumberMarker int `xml:"NextPartNumberMarker"`
|
NextPartNumberMarker int `xml:"NextPartNumberMarker,omitempty"`
|
||||||
Owner Owner `xml:"Owner"`
|
Owner Owner `xml:"Owner"`
|
||||||
Parts []*layer.Part `xml:"Part"`
|
Parts []*layer.Part `xml:"Part"`
|
||||||
PartNumberMarker int `xml:"PartNumberMarker"`
|
PartNumberMarker int `xml:"PartNumberMarker,omitempty"`
|
||||||
StorageClass string `xml:"StorageClass"`
|
StorageClass string `xml:"StorageClass"`
|
||||||
UploadID string `xml:"UploadId"`
|
UploadID string `xml:"UploadId"`
|
||||||
}
|
}
|
||||||
|
@ -114,7 +113,14 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if cannedACLStatus == aclStatusYes {
|
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
apeEnabled := bktInfo.APEEnabled || settings.CannedACL != ""
|
||||||
|
if apeEnabled && cannedACLStatus == aclStatusYes {
|
||||||
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -128,6 +134,20 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
|
||||||
Data: &layer.UploadData{},
|
Data: &layer.UploadData{},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
needUpdateEACLTable := !(apeEnabled || cannedACLStatus == aclStatusNo)
|
||||||
|
if needUpdateEACLTable {
|
||||||
|
key, err := h.bearerTokenIssuerKey(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "couldn't get gate key", reqInfo, err, additional...)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err = parseACLHeaders(r.Header, key); err != nil {
|
||||||
|
h.logAndSendError(w, "could not parse acl", reqInfo, err, additional...)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.Data.ACLHeaders = formACLHeadersForMultipart(r.Header)
|
||||||
|
}
|
||||||
|
|
||||||
if len(r.Header.Get(api.AmzTagging)) > 0 {
|
if len(r.Header.Get(api.AmzTagging)) > 0 {
|
||||||
p.Data.TagSet, err = parseTaggingHeader(r.Header)
|
p.Data.TagSet, err = parseTaggingHeader(r.Header)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -177,6 +197,25 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func formACLHeadersForMultipart(header http.Header) map[string]string {
|
||||||
|
result := make(map[string]string)
|
||||||
|
|
||||||
|
if value := header.Get(api.AmzACL); value != "" {
|
||||||
|
result[api.AmzACL] = value
|
||||||
|
}
|
||||||
|
if value := header.Get(api.AmzGrantRead); value != "" {
|
||||||
|
result[api.AmzGrantRead] = value
|
||||||
|
}
|
||||||
|
if value := header.Get(api.AmzGrantFullControl); value != "" {
|
||||||
|
result[api.AmzGrantFullControl] = value
|
||||||
|
}
|
||||||
|
if value := header.Get(api.AmzGrantWrite); value != "" {
|
||||||
|
result[api.AmzGrantWrite] = value
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
reqInfo := middleware.GetReqInfo(r.Context())
|
reqInfo := middleware.GetReqInfo(r.Context())
|
||||||
|
|
||||||
|
@ -402,7 +441,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
reqBody := new(CompleteMultipartUpload)
|
reqBody := new(CompleteMultipartUpload)
|
||||||
if err = h.cfg.NewXMLDecoder(r.Body).Decode(reqBody); err != nil {
|
if err = h.cfg.NewXMLDecoder(r.Body).Decode(reqBody); err != nil {
|
||||||
h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
|
h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
|
||||||
fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()), additional...)
|
errors.GetAPIError(errors.ErrMalformedXML), additional...)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if len(reqBody.Parts) == 0 {
|
if len(reqBody.Parts) == 0 {
|
||||||
|
@ -417,7 +456,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
|
|
||||||
// Start complete multipart upload which may take some time to fetch object
|
// Start complete multipart upload which may take some time to fetch object
|
||||||
// and re-upload it part by part.
|
// and re-upload it part by part.
|
||||||
objInfo, err := h.completeMultipartUpload(r, c, bktInfo)
|
objInfo, err := h.completeMultipartUpload(r, c, bktInfo, reqInfo)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "complete multipart error", reqInfo, err, additional...)
|
h.logAndSendError(w, "complete multipart error", reqInfo, err, additional...)
|
||||||
|
@ -428,7 +467,6 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
Bucket: objInfo.Bucket,
|
Bucket: objInfo.Bucket,
|
||||||
Key: objInfo.Name,
|
Key: objInfo.Name,
|
||||||
ETag: data.Quote(objInfo.ETag(h.cfg.MD5Enabled())),
|
ETag: data.Quote(objInfo.ETag(h.cfg.MD5Enabled())),
|
||||||
Location: getObjectLocation(r, reqInfo.BucketName, reqInfo.ObjectName, reqInfo.RequestVHSEnabled),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if settings.VersioningEnabled() {
|
if settings.VersioningEnabled() {
|
||||||
|
@ -440,35 +478,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// returns "https" if the tls boolean is true, "http" otherwise.
|
func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMultipartParams, bktInfo *data.BucketInfo, reqInfo *middleware.ReqInfo) (*data.ObjectInfo, error) {
|
||||||
func getURLScheme(r *http.Request) string {
|
|
||||||
if r.TLS != nil {
|
|
||||||
return "https"
|
|
||||||
}
|
|
||||||
return "http"
|
|
||||||
}
|
|
||||||
|
|
||||||
// getObjectLocation gets the fully qualified URL of an object.
|
|
||||||
func getObjectLocation(r *http.Request, bucket, object string, vhsEnabled bool) string {
|
|
||||||
proto := middleware.GetSourceScheme(r)
|
|
||||||
if proto == "" {
|
|
||||||
proto = getURLScheme(r)
|
|
||||||
}
|
|
||||||
u := &url.URL{
|
|
||||||
Host: r.Host,
|
|
||||||
Path: path.Join("/", bucket, object),
|
|
||||||
Scheme: proto,
|
|
||||||
}
|
|
||||||
|
|
||||||
// If vhs enabled then we need to use bucket DNS style.
|
|
||||||
if vhsEnabled {
|
|
||||||
u.Path = path.Join("/", object)
|
|
||||||
}
|
|
||||||
|
|
||||||
return u.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMultipartParams, bktInfo *data.BucketInfo) (*data.ObjectInfo, error) {
|
|
||||||
ctx := r.Context()
|
ctx := r.Context()
|
||||||
uploadData, extendedObjInfo, err := h.obj.CompleteMultipartUpload(ctx, c)
|
uploadData, extendedObjInfo, err := h.obj.CompleteMultipartUpload(ctx, c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -477,8 +487,8 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
|
||||||
objInfo := extendedObjInfo.ObjectInfo
|
objInfo := extendedObjInfo.ObjectInfo
|
||||||
|
|
||||||
if len(uploadData.TagSet) != 0 {
|
if len(uploadData.TagSet) != 0 {
|
||||||
tagPrm := &data.PutObjectTaggingParams{
|
tagPrm := &layer.PutObjectTaggingParams{
|
||||||
ObjectVersion: &data.ObjectVersion{
|
ObjectVersion: &layer.ObjectVersion{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ObjectName: objInfo.Name,
|
ObjectName: objInfo.Name,
|
||||||
VersionID: objInfo.VersionID(),
|
VersionID: objInfo.VersionID(),
|
||||||
|
@ -486,11 +496,48 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
|
||||||
TagSet: uploadData.TagSet,
|
TagSet: uploadData.TagSet,
|
||||||
NodeVersion: extendedObjInfo.NodeVersion,
|
NodeVersion: extendedObjInfo.NodeVersion,
|
||||||
}
|
}
|
||||||
if err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
|
if _, err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
|
||||||
return nil, fmt.Errorf("could not put tagging file of completed multipart upload: %w", err)
|
return nil, fmt.Errorf("could not put tagging file of completed multipart upload: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(uploadData.ACLHeaders) != 0 {
|
||||||
|
sessionTokenSetEACL, err := getSessionTokenSetEACL(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("couldn't get eacl token: %w", err)
|
||||||
|
}
|
||||||
|
key, err := h.bearerTokenIssuerKey(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("couldn't get gate key: %w", err)
|
||||||
|
}
|
||||||
|
acl, err := parseACLHeaders(r.Header, key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("could not parse acl: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resInfo := &resourceInfo{
|
||||||
|
Bucket: objInfo.Bucket,
|
||||||
|
Object: objInfo.Name,
|
||||||
|
}
|
||||||
|
astObject, err := aclToAst(acl, resInfo)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("could not translate acl of completed multipart upload to ast: %w", err)
|
||||||
|
}
|
||||||
|
if _, err = h.updateBucketACL(r, astObject, bktInfo, sessionTokenSetEACL); err != nil {
|
||||||
|
return nil, fmt.Errorf("could not update bucket acl while completing multipart upload: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
s := &SendNotificationParams{
|
||||||
|
Event: EventObjectCreatedCompleteMultipartUpload,
|
||||||
|
NotificationInfo: data.NotificationInfoFromObject(objInfo, h.cfg.MD5Enabled()),
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
ReqInfo: reqInfo,
|
||||||
|
}
|
||||||
|
if err = h.sendNotifications(ctx, s); err != nil {
|
||||||
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
return objInfo, nil
|
return objInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -17,10 +17,6 @@ import (
|
||||||
s3Errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
s3Errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
||||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
|
||||||
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -72,19 +68,6 @@ func TestDeleteMultipartAllParts(t *testing.T) {
|
||||||
require.Empty(t, hc.tp.Objects())
|
require.Empty(t, hc.tp.Objects())
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSpecialMultipartName(t *testing.T) {
|
|
||||||
hc := prepareHandlerContextWithMinCache(t)
|
|
||||||
|
|
||||||
bktName, objName := "bucket", "bucket-settings"
|
|
||||||
|
|
||||||
createTestBucket(hc, bktName)
|
|
||||||
putBucketVersioning(t, hc, bktName, true)
|
|
||||||
|
|
||||||
createMultipartUpload(hc, bktName, objName, nil)
|
|
||||||
res := getBucketVersioning(hc, bktName)
|
|
||||||
require.Equal(t, enabledValue, res.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMultipartReUploadPart(t *testing.T) {
|
func TestMultipartReUploadPart(t *testing.T) {
|
||||||
hc := prepareHandlerContext(t)
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
@ -126,108 +109,6 @@ func TestMultipartReUploadPart(t *testing.T) {
|
||||||
equalDataSlices(t, append(data1, data2...), data)
|
equalDataSlices(t, append(data1, data2...), data)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMultipartRemovePartsSplit(t *testing.T) {
|
|
||||||
bktName, objName := "bucket-to-upload-part", "object-multipart"
|
|
||||||
partSize := 8
|
|
||||||
|
|
||||||
t.Run("reupload part", func(t *testing.T) {
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
bktInfo := createTestBucket(hc, bktName)
|
|
||||||
uploadInfo := createMultipartUpload(hc, bktName, objName, map[string]string{})
|
|
||||||
|
|
||||||
uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSize)
|
|
||||||
|
|
||||||
multipartInfo, err := hc.tree.GetMultipartUpload(hc.Context(), bktInfo, uploadInfo.Key, uploadInfo.UploadID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
objID := oidtest.ID()
|
|
||||||
_, err = hc.treeMock.AddNode(hc.Context(), bktInfo, "system", multipartInfo.ID, map[string]string{
|
|
||||||
"Number": "1",
|
|
||||||
"OID": objID.EncodeToString(),
|
|
||||||
"Owner": usertest.ID().EncodeToString(),
|
|
||||||
"ETag": "etag",
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
hc.tp.AddObject(bktInfo.CID.EncodeToString()+"/"+objID.EncodeToString(), object.New())
|
|
||||||
require.Len(t, hc.tp.Objects(), 2)
|
|
||||||
|
|
||||||
list := listParts(hc, bktName, objName, uploadInfo.UploadID, "0", http.StatusOK)
|
|
||||||
require.Len(t, list.Parts, 1)
|
|
||||||
require.Equal(t, `"etag"`, list.Parts[0].ETag)
|
|
||||||
|
|
||||||
etag1, _ := uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSize)
|
|
||||||
list = listParts(hc, bktName, objName, uploadInfo.UploadID, "0", http.StatusOK)
|
|
||||||
require.Len(t, list.Parts, 1)
|
|
||||||
require.Equal(t, etag1, list.Parts[0].ETag)
|
|
||||||
|
|
||||||
require.Len(t, hc.tp.Objects(), 1)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("abort multipart", func(t *testing.T) {
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
bktInfo := createTestBucket(hc, bktName)
|
|
||||||
uploadInfo := createMultipartUpload(hc, bktName, objName, map[string]string{})
|
|
||||||
|
|
||||||
uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSize)
|
|
||||||
|
|
||||||
multipartInfo, err := hc.tree.GetMultipartUpload(hc.Context(), bktInfo, uploadInfo.Key, uploadInfo.UploadID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
objID := oidtest.ID()
|
|
||||||
_, err = hc.treeMock.AddNode(hc.Context(), bktInfo, "system", multipartInfo.ID, map[string]string{
|
|
||||||
"Number": "1",
|
|
||||||
"OID": objID.EncodeToString(),
|
|
||||||
"Owner": usertest.ID().EncodeToString(),
|
|
||||||
"ETag": "etag",
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
hc.tp.AddObject(bktInfo.CID.EncodeToString()+"/"+objID.EncodeToString(), object.New())
|
|
||||||
require.Len(t, hc.tp.Objects(), 2)
|
|
||||||
|
|
||||||
abortMultipartUpload(hc, bktName, objName, uploadInfo.UploadID)
|
|
||||||
require.Empty(t, hc.tp.Objects())
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("complete multipart", func(t *testing.T) {
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
bktInfo := createTestBucket(hc, bktName)
|
|
||||||
uploadInfo := createMultipartUpload(hc, bktName, objName, map[string]string{})
|
|
||||||
|
|
||||||
etag1, _ := uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSize)
|
|
||||||
|
|
||||||
multipartInfo, err := hc.tree.GetMultipartUpload(hc.Context(), bktInfo, uploadInfo.Key, uploadInfo.UploadID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
objID := oidtest.ID()
|
|
||||||
_, err = hc.treeMock.AddNode(hc.Context(), bktInfo, "system", multipartInfo.ID, map[string]string{
|
|
||||||
"Number": "1",
|
|
||||||
"OID": objID.EncodeToString(),
|
|
||||||
"Owner": usertest.ID().EncodeToString(),
|
|
||||||
"ETag": "etag",
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
hc.tp.AddObject(bktInfo.CID.EncodeToString()+"/"+objID.EncodeToString(), object.New())
|
|
||||||
require.Len(t, hc.tp.Objects(), 2)
|
|
||||||
|
|
||||||
completeMultipartUpload(hc, bktName, objName, uploadInfo.UploadID, []string{etag1})
|
|
||||||
require.Falsef(t, containsOID(hc.tp.Objects(), objID), "frostfs contains '%s' object, but shouldn't", objID)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func containsOID(objects []*object.Object, objID oid.ID) bool {
|
|
||||||
for _, o := range objects {
|
|
||||||
oID, _ := o.ID()
|
|
||||||
if oID.Equals(objID) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestListMultipartUploads(t *testing.T) {
|
func TestListMultipartUploads(t *testing.T) {
|
||||||
hc := prepareHandlerContext(t)
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
@ -398,19 +279,13 @@ func TestListParts(t *testing.T) {
|
||||||
require.Len(t, list.Parts, 2)
|
require.Len(t, list.Parts, 2)
|
||||||
require.Equal(t, etag1, list.Parts[0].ETag)
|
require.Equal(t, etag1, list.Parts[0].ETag)
|
||||||
require.Equal(t, etag2, list.Parts[1].ETag)
|
require.Equal(t, etag2, list.Parts[1].ETag)
|
||||||
require.Zero(t, list.PartNumberMarker)
|
|
||||||
require.Equal(t, 2, list.NextPartNumberMarker)
|
|
||||||
|
|
||||||
list = listParts(hc, bktName, objName, uploadInfo.UploadID, "1", http.StatusOK)
|
list = listParts(hc, bktName, objName, uploadInfo.UploadID, "1", http.StatusOK)
|
||||||
require.Len(t, list.Parts, 1)
|
require.Len(t, list.Parts, 1)
|
||||||
require.Equal(t, etag2, list.Parts[0].ETag)
|
require.Equal(t, etag2, list.Parts[0].ETag)
|
||||||
require.Equal(t, 1, list.PartNumberMarker)
|
|
||||||
require.Equal(t, 2, list.NextPartNumberMarker)
|
|
||||||
|
|
||||||
list = listParts(hc, bktName, objName, uploadInfo.UploadID, "2", http.StatusOK)
|
list = listParts(hc, bktName, objName, uploadInfo.UploadID, "2", http.StatusOK)
|
||||||
require.Len(t, list.Parts, 0)
|
require.Len(t, list.Parts, 0)
|
||||||
require.Equal(t, 2, list.PartNumberMarker)
|
|
||||||
require.Equal(t, 0, list.NextPartNumberMarker)
|
|
||||||
|
|
||||||
list = listParts(hc, bktName, objName, uploadInfo.UploadID, "7", http.StatusOK)
|
list = listParts(hc, bktName, objName, uploadInfo.UploadID, "7", http.StatusOK)
|
||||||
require.Len(t, list.Parts, 0)
|
require.Len(t, list.Parts, 0)
|
||||||
|
@ -426,9 +301,9 @@ func TestMultipartUploadWithContentLanguage(t *testing.T) {
|
||||||
createTestBucket(hc, bktName)
|
createTestBucket(hc, bktName)
|
||||||
|
|
||||||
partSize := 5 * 1024 * 1024
|
partSize := 5 * 1024 * 1024
|
||||||
expectedContentLanguage := "en"
|
exceptedContentLanguage := "en"
|
||||||
headers := map[string]string{
|
headers := map[string]string{
|
||||||
api.ContentLanguage: expectedContentLanguage,
|
api.ContentLanguage: exceptedContentLanguage,
|
||||||
}
|
}
|
||||||
|
|
||||||
multipartUpload := createMultipartUpload(hc, bktName, objName, headers)
|
multipartUpload := createMultipartUpload(hc, bktName, objName, headers)
|
||||||
|
@ -439,7 +314,7 @@ func TestMultipartUploadWithContentLanguage(t *testing.T) {
|
||||||
|
|
||||||
w, r := prepareTestRequest(hc, bktName, objName, nil)
|
w, r := prepareTestRequest(hc, bktName, objName, nil)
|
||||||
hc.Handler().HeadObjectHandler(w, r)
|
hc.Handler().HeadObjectHandler(w, r)
|
||||||
require.Equal(t, expectedContentLanguage, w.Header().Get(api.ContentLanguage))
|
require.Equal(t, exceptedContentLanguage, w.Header().Get(api.ContentLanguage))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMultipartUploadEnabledMD5(t *testing.T) {
|
func TestMultipartUploadEnabledMD5(t *testing.T) {
|
||||||
|
@ -547,80 +422,6 @@ func TestUploadPartCheckContentSHA256(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMultipartObjectLocation(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
req *http.Request
|
|
||||||
bucket string
|
|
||||||
object string
|
|
||||||
vhsEnabled bool
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
req: &http.Request{
|
|
||||||
Host: "127.0.0.1:8084",
|
|
||||||
Header: map[string][]string{"X-Forwarded-Scheme": {"http"}},
|
|
||||||
},
|
|
||||||
bucket: "testbucket1",
|
|
||||||
object: "test/1.txt",
|
|
||||||
expected: "http://127.0.0.1:8084/testbucket1/test/1.txt",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
req: &http.Request{
|
|
||||||
Host: "localhost:8084",
|
|
||||||
Header: map[string][]string{"X-Forwarded-Scheme": {"https"}},
|
|
||||||
},
|
|
||||||
bucket: "testbucket1",
|
|
||||||
object: "test/1.txt",
|
|
||||||
expected: "https://localhost:8084/testbucket1/test/1.txt",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
req: &http.Request{
|
|
||||||
Host: "s3.mybucket.org",
|
|
||||||
Header: map[string][]string{"X-Forwarded-Scheme": {"http"}},
|
|
||||||
},
|
|
||||||
bucket: "mybucket",
|
|
||||||
object: "test/1.txt",
|
|
||||||
expected: "http://s3.mybucket.org/mybucket/test/1.txt",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
req: &http.Request{Host: "mys3.mybucket.org"},
|
|
||||||
bucket: "mybucket",
|
|
||||||
object: "test/1.txt",
|
|
||||||
expected: "http://mys3.mybucket.org/mybucket/test/1.txt",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
req: &http.Request{Host: "s3.bucket.org", TLS: &tls.ConnectionState{}},
|
|
||||||
bucket: "bucket",
|
|
||||||
object: "obj",
|
|
||||||
expected: "https://s3.bucket.org/bucket/obj",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
req: &http.Request{
|
|
||||||
Host: "mybucket.s3dev.frostfs.devenv",
|
|
||||||
},
|
|
||||||
bucket: "mybucket",
|
|
||||||
object: "test/1.txt",
|
|
||||||
vhsEnabled: true,
|
|
||||||
expected: "http://mybucket.s3dev.frostfs.devenv/test/1.txt",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
req: &http.Request{
|
|
||||||
Host: "mybucket.s3dev.frostfs.devenv",
|
|
||||||
Header: map[string][]string{"X-Forwarded-Scheme": {"https"}},
|
|
||||||
},
|
|
||||||
bucket: "mybucket",
|
|
||||||
object: "test/1.txt",
|
|
||||||
vhsEnabled: true,
|
|
||||||
expected: "https://mybucket.s3dev.frostfs.devenv/test/1.txt",
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run("", func(t *testing.T) {
|
|
||||||
location := getObjectLocation(tc.req, tc.bucket, tc.object, tc.vhsEnabled)
|
|
||||||
require.Equal(t, tc.expected, location)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func uploadPartCopy(hc *handlerContext, bktName, objName, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
|
func uploadPartCopy(hc *handlerContext, bktName, objName, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
|
||||||
return uploadPartCopyBase(hc, bktName, objName, false, uploadID, num, srcObj, start, end)
|
return uploadPartCopyBase(hc, bktName, objName, false, uploadID, num, srcObj, start, end)
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,10 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||||
|
}
|
||||||
|
|
||||||
func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||||
}
|
}
|
||||||
|
|
274
api/handler/notifications.go
Normal file
274
api/handler/notifications.go
Normal file
|
@ -0,0 +1,274 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
type (
|
||||||
|
SendNotificationParams struct {
|
||||||
|
Event string
|
||||||
|
NotificationInfo *data.NotificationInfo
|
||||||
|
BktInfo *data.BucketInfo
|
||||||
|
ReqInfo *middleware.ReqInfo
|
||||||
|
User string
|
||||||
|
Time time.Time
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
filterRuleSuffixName = "suffix"
|
||||||
|
filterRulePrefixName = "prefix"
|
||||||
|
|
||||||
|
EventObjectCreated = "s3:ObjectCreated:*"
|
||||||
|
EventObjectCreatedPut = "s3:ObjectCreated:Put"
|
||||||
|
EventObjectCreatedPost = "s3:ObjectCreated:Post"
|
||||||
|
EventObjectCreatedCopy = "s3:ObjectCreated:Copy"
|
||||||
|
EventReducedRedundancyLostObject = "s3:ReducedRedundancyLostObject"
|
||||||
|
EventObjectCreatedCompleteMultipartUpload = "s3:ObjectCreated:CompleteMultipartUpload"
|
||||||
|
EventObjectRemoved = "s3:ObjectRemoved:*"
|
||||||
|
EventObjectRemovedDelete = "s3:ObjectRemoved:Delete"
|
||||||
|
EventObjectRemovedDeleteMarkerCreated = "s3:ObjectRemoved:DeleteMarkerCreated"
|
||||||
|
EventObjectRestore = "s3:ObjectRestore:*"
|
||||||
|
EventObjectRestorePost = "s3:ObjectRestore:Post"
|
||||||
|
EventObjectRestoreCompleted = "s3:ObjectRestore:Completed"
|
||||||
|
EventReplication = "s3:Replication:*"
|
||||||
|
EventReplicationOperationFailedReplication = "s3:Replication:OperationFailedReplication"
|
||||||
|
EventReplicationOperationNotTracked = "s3:Replication:OperationNotTracked"
|
||||||
|
EventReplicationOperationMissedThreshold = "s3:Replication:OperationMissedThreshold"
|
||||||
|
EventReplicationOperationReplicatedAfterThreshold = "s3:Replication:OperationReplicatedAfterThreshold"
|
||||||
|
EventObjectRestoreDelete = "s3:ObjectRestore:Delete"
|
||||||
|
EventLifecycleTransition = "s3:LifecycleTransition"
|
||||||
|
EventIntelligentTiering = "s3:IntelligentTiering"
|
||||||
|
EventObjectACLPut = "s3:ObjectAcl:Put"
|
||||||
|
EventLifecycleExpiration = "s3:LifecycleExpiration:*"
|
||||||
|
EventLifecycleExpirationDelete = "s3:LifecycleExpiration:Delete"
|
||||||
|
EventLifecycleExpirationDeleteMarkerCreated = "s3:LifecycleExpiration:DeleteMarkerCreated"
|
||||||
|
EventObjectTagging = "s3:ObjectTagging:*"
|
||||||
|
EventObjectTaggingPut = "s3:ObjectTagging:Put"
|
||||||
|
EventObjectTaggingDelete = "s3:ObjectTagging:Delete"
|
||||||
|
)
|
||||||
|
|
||||||
|
var validEvents = map[string]struct{}{
|
||||||
|
EventReducedRedundancyLostObject: {},
|
||||||
|
EventObjectCreated: {},
|
||||||
|
EventObjectCreatedPut: {},
|
||||||
|
EventObjectCreatedPost: {},
|
||||||
|
EventObjectCreatedCopy: {},
|
||||||
|
EventObjectCreatedCompleteMultipartUpload: {},
|
||||||
|
EventObjectRemoved: {},
|
||||||
|
EventObjectRemovedDelete: {},
|
||||||
|
EventObjectRemovedDeleteMarkerCreated: {},
|
||||||
|
EventObjectRestore: {},
|
||||||
|
EventObjectRestorePost: {},
|
||||||
|
EventObjectRestoreCompleted: {},
|
||||||
|
EventReplication: {},
|
||||||
|
EventReplicationOperationFailedReplication: {},
|
||||||
|
EventReplicationOperationNotTracked: {},
|
||||||
|
EventReplicationOperationMissedThreshold: {},
|
||||||
|
EventReplicationOperationReplicatedAfterThreshold: {},
|
||||||
|
EventObjectRestoreDelete: {},
|
||||||
|
EventLifecycleTransition: {},
|
||||||
|
EventIntelligentTiering: {},
|
||||||
|
EventObjectACLPut: {},
|
||||||
|
EventLifecycleExpiration: {},
|
||||||
|
EventLifecycleExpirationDelete: {},
|
||||||
|
EventLifecycleExpirationDeleteMarkerCreated: {},
|
||||||
|
EventObjectTagging: {},
|
||||||
|
EventObjectTaggingPut: {},
|
||||||
|
EventObjectTaggingDelete: {},
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
reqInfo := middleware.GetReqInfo(r.Context())
|
||||||
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
conf := &data.NotificationConfiguration{}
|
||||||
|
if err = h.cfg.NewXMLDecoder(r.Body).Decode(conf); err != nil {
|
||||||
|
h.logAndSendError(w, "couldn't decode notification configuration", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err = h.checkBucketConfiguration(r.Context(), conf, reqInfo); err != nil {
|
||||||
|
h.logAndSendError(w, "couldn't check bucket configuration", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p := &layer.PutBucketNotificationConfigurationParams{
|
||||||
|
RequestInfo: reqInfo,
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
Configuration: conf,
|
||||||
|
}
|
||||||
|
|
||||||
|
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = h.obj.PutBucketNotificationConfiguration(r.Context(), p); err != nil {
|
||||||
|
h.logAndSendError(w, "couldn't put bucket configuration", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *handler) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
reqInfo := middleware.GetReqInfo(r.Context())
|
||||||
|
|
||||||
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
conf, err := h.obj.GetBucketNotificationConfiguration(r.Context(), bktInfo)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get bucket notification configuration", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = middleware.EncodeToResponse(w, conf); err != nil {
|
||||||
|
h.logAndSendError(w, "could not encode bucket notification configuration to response", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *handler) sendNotifications(ctx context.Context, p *SendNotificationParams) error {
|
||||||
|
if !h.cfg.NotificatorEnabled() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
conf, err := h.obj.GetBucketNotificationConfiguration(ctx, p.BktInfo)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get notification configuration: %w", err)
|
||||||
|
}
|
||||||
|
if conf.IsEmpty() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
box, err := middleware.GetBoxData(ctx)
|
||||||
|
if err == nil && box.Gate.BearerToken != nil {
|
||||||
|
p.User = bearer.ResolveIssuer(*box.Gate.BearerToken).EncodeToString()
|
||||||
|
}
|
||||||
|
|
||||||
|
p.Time = layer.TimeNow(ctx)
|
||||||
|
|
||||||
|
topics := filterSubjects(conf, p.Event, p.NotificationInfo.Name)
|
||||||
|
|
||||||
|
return h.notificator.SendNotifications(topics, p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkBucketConfiguration checks notification configuration and generates an ID for configurations with empty ids.
|
||||||
|
func (h *handler) checkBucketConfiguration(ctx context.Context, conf *data.NotificationConfiguration, r *middleware.ReqInfo) (completed bool, err error) {
|
||||||
|
if conf == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if conf.TopicConfigurations != nil || conf.LambdaFunctionConfigurations != nil {
|
||||||
|
return completed, errors.GetAPIError(errors.ErrNotificationTopicNotSupported)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, q := range conf.QueueConfigurations {
|
||||||
|
if err = checkEvents(q.Events); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = checkRules(q.Filter.Key.FilterRules); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if h.cfg.NotificatorEnabled() {
|
||||||
|
if err = h.notificator.SendTestNotification(q.QueueArn, r.BucketName, r.RequestID, r.Host, layer.TimeNow(ctx)); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
h.reqLogger(ctx).Warn(logs.FailedToSendTestEventBecauseNotificationsIsDisabled)
|
||||||
|
}
|
||||||
|
|
||||||
|
if q.ID == "" {
|
||||||
|
completed = true
|
||||||
|
conf.QueueConfigurations[i].ID = uuid.NewString()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkRules(rules []data.FilterRule) error {
|
||||||
|
names := make(map[string]struct{})
|
||||||
|
|
||||||
|
for _, r := range rules {
|
||||||
|
if r.Name != filterRuleSuffixName && r.Name != filterRulePrefixName {
|
||||||
|
return errors.GetAPIError(errors.ErrFilterNameInvalid)
|
||||||
|
}
|
||||||
|
if _, ok := names[r.Name]; ok {
|
||||||
|
if r.Name == filterRuleSuffixName {
|
||||||
|
return errors.GetAPIError(errors.ErrFilterNameSuffix)
|
||||||
|
}
|
||||||
|
return errors.GetAPIError(errors.ErrFilterNamePrefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
names[r.Name] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkEvents(events []string) error {
|
||||||
|
for _, e := range events {
|
||||||
|
if _, ok := validEvents[e]; !ok {
|
||||||
|
return errors.GetAPIError(errors.ErrEventNotification)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func filterSubjects(conf *data.NotificationConfiguration, eventType, objName string) map[string]string {
|
||||||
|
topics := make(map[string]string)
|
||||||
|
|
||||||
|
for _, t := range conf.QueueConfigurations {
|
||||||
|
event := false
|
||||||
|
for _, e := range t.Events {
|
||||||
|
// the second condition is comparison with the events ending with *:
|
||||||
|
// s3:ObjectCreated:*, s3:ObjectRemoved:* etc without the last char
|
||||||
|
if eventType == e || strings.HasPrefix(eventType, e[:len(e)-1]) {
|
||||||
|
event = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !event {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
filter := true
|
||||||
|
for _, f := range t.Filter.Key.FilterRules {
|
||||||
|
if f.Name == filterRulePrefixName && !strings.HasPrefix(objName, f.Value) ||
|
||||||
|
f.Name == filterRuleSuffixName && !strings.HasSuffix(objName, f.Value) {
|
||||||
|
filter = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if filter {
|
||||||
|
topics[t.ID] = t.QueueArn
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return topics
|
||||||
|
}
|
115
api/handler/notifications_test.go
Normal file
115
api/handler/notifications_test.go
Normal file
|
@ -0,0 +1,115 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFilterSubjects(t *testing.T) {
|
||||||
|
config := &data.NotificationConfiguration{
|
||||||
|
QueueConfigurations: []data.QueueConfiguration{
|
||||||
|
{
|
||||||
|
ID: "test1",
|
||||||
|
QueueArn: "test1",
|
||||||
|
Events: []string{EventObjectCreated, EventObjectRemovedDelete},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "test2",
|
||||||
|
QueueArn: "test2",
|
||||||
|
Events: []string{EventObjectTagging},
|
||||||
|
Filter: data.Filter{Key: data.Key{FilterRules: []data.FilterRule{
|
||||||
|
{Name: "prefix", Value: "dir/"},
|
||||||
|
{Name: "suffix", Value: ".png"},
|
||||||
|
}}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("no topics because suitable events not found", func(t *testing.T) {
|
||||||
|
topics := filterSubjects(config, EventObjectACLPut, "dir/a.png")
|
||||||
|
require.Empty(t, topics)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("no topics because of not suitable prefix", func(t *testing.T) {
|
||||||
|
topics := filterSubjects(config, EventObjectTaggingPut, "dirw/cat.png")
|
||||||
|
require.Empty(t, topics)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("no topics because of not suitable suffix", func(t *testing.T) {
|
||||||
|
topics := filterSubjects(config, EventObjectTaggingPut, "a.jpg")
|
||||||
|
require.Empty(t, topics)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("filter topics from queue configs without prefix suffix filter and exact event", func(t *testing.T) {
|
||||||
|
topics := filterSubjects(config, EventObjectCreatedPut, "dir/a.png")
|
||||||
|
require.Contains(t, topics, "test1")
|
||||||
|
require.Len(t, topics, 1)
|
||||||
|
require.Equal(t, topics["test1"], "test1")
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("filter topics from queue configs with prefix suffix filter and '*' ending event", func(t *testing.T) {
|
||||||
|
topics := filterSubjects(config, EventObjectTaggingPut, "dir/a.png")
|
||||||
|
require.Contains(t, topics, "test2")
|
||||||
|
require.Len(t, topics, 1)
|
||||||
|
require.Equal(t, topics["test2"], "test2")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckRules(t *testing.T) {
|
||||||
|
t.Run("correct rules with prefix and suffix", func(t *testing.T) {
|
||||||
|
rules := []data.FilterRule{
|
||||||
|
{Name: "prefix", Value: "asd"},
|
||||||
|
{Name: "suffix", Value: "asd"},
|
||||||
|
}
|
||||||
|
err := checkRules(rules)
|
||||||
|
require.NoError(t, err)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("correct rules with prefix", func(t *testing.T) {
|
||||||
|
rules := []data.FilterRule{
|
||||||
|
{Name: "prefix", Value: "asd"},
|
||||||
|
}
|
||||||
|
err := checkRules(rules)
|
||||||
|
require.NoError(t, err)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("correct rules with suffix", func(t *testing.T) {
|
||||||
|
rules := []data.FilterRule{
|
||||||
|
{Name: "suffix", Value: "asd"},
|
||||||
|
}
|
||||||
|
err := checkRules(rules)
|
||||||
|
require.NoError(t, err)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("incorrect rules with wrong name", func(t *testing.T) {
|
||||||
|
rules := []data.FilterRule{
|
||||||
|
{Name: "prefix", Value: "sdf"},
|
||||||
|
{Name: "sfx", Value: "asd"},
|
||||||
|
}
|
||||||
|
err := checkRules(rules)
|
||||||
|
require.ErrorIs(t, err, errors.GetAPIError(errors.ErrFilterNameInvalid))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("incorrect rules with repeating suffix", func(t *testing.T) {
|
||||||
|
rules := []data.FilterRule{
|
||||||
|
{Name: "suffix", Value: "asd"},
|
||||||
|
{Name: "suffix", Value: "asdf"},
|
||||||
|
{Name: "prefix", Value: "jk"},
|
||||||
|
}
|
||||||
|
err := checkRules(rules)
|
||||||
|
require.ErrorIs(t, err, errors.GetAPIError(errors.ErrFilterNameSuffix))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("incorrect rules with repeating prefix", func(t *testing.T) {
|
||||||
|
rules := []data.FilterRule{
|
||||||
|
{Name: "suffix", Value: "ds"},
|
||||||
|
{Name: "prefix", Value: "asd"},
|
||||||
|
{Name: "prefix", Value: "asdf"},
|
||||||
|
}
|
||||||
|
err := checkRules(rules)
|
||||||
|
require.ErrorIs(t, err, errors.GetAPIError(errors.ErrFilterNamePrefix))
|
||||||
|
})
|
||||||
|
}
|
|
@ -232,7 +232,7 @@ func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
response := encodeListObjectVersionsToResponse(p, info, p.BktInfo.Name, h.cfg.MD5Enabled())
|
response := encodeListObjectVersionsToResponse(info, p.BktInfo.Name, h.cfg.MD5Enabled())
|
||||||
if err = middleware.EncodeToResponse(w, response); err != nil {
|
if err = middleware.EncodeToResponse(w, response); err != nil {
|
||||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||||
}
|
}
|
||||||
|
@ -264,28 +264,24 @@ func parseListObjectVersionsRequest(reqInfo *middleware.ReqInfo) (*layer.ListObj
|
||||||
return &res, nil
|
return &res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func encodeListObjectVersionsToResponse(p *layer.ListObjectVersionsParams, info *layer.ListObjectVersionsInfo, bucketName string, md5Enabled bool) *ListObjectsVersionsResponse {
|
func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, bucketName string, md5Enabled bool) *ListObjectsVersionsResponse {
|
||||||
res := ListObjectsVersionsResponse{
|
res := ListObjectsVersionsResponse{
|
||||||
Name: bucketName,
|
Name: bucketName,
|
||||||
IsTruncated: info.IsTruncated,
|
IsTruncated: info.IsTruncated,
|
||||||
KeyMarker: s3PathEncode(info.KeyMarker, p.Encode),
|
KeyMarker: info.KeyMarker,
|
||||||
NextKeyMarker: s3PathEncode(info.NextKeyMarker, p.Encode),
|
NextKeyMarker: info.NextKeyMarker,
|
||||||
NextVersionIDMarker: info.NextVersionIDMarker,
|
NextVersionIDMarker: info.NextVersionIDMarker,
|
||||||
VersionIDMarker: info.VersionIDMarker,
|
VersionIDMarker: info.VersionIDMarker,
|
||||||
Prefix: s3PathEncode(p.Prefix, p.Encode),
|
|
||||||
Delimiter: s3PathEncode(p.Delimiter, p.Encode),
|
|
||||||
EncodingType: p.Encode,
|
|
||||||
MaxKeys: p.MaxKeys,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, prefix := range info.CommonPrefixes {
|
for _, prefix := range info.CommonPrefixes {
|
||||||
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: s3PathEncode(prefix, p.Encode)})
|
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: prefix})
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, ver := range info.Version {
|
for _, ver := range info.Version {
|
||||||
res.Version = append(res.Version, ObjectVersionResponse{
|
res.Version = append(res.Version, ObjectVersionResponse{
|
||||||
IsLatest: ver.IsLatest,
|
IsLatest: ver.IsLatest,
|
||||||
Key: s3PathEncode(ver.NodeVersion.FilePath, p.Encode),
|
Key: ver.NodeVersion.FilePath,
|
||||||
LastModified: ver.NodeVersion.Created.UTC().Format(time.RFC3339),
|
LastModified: ver.NodeVersion.Created.UTC().Format(time.RFC3339),
|
||||||
Owner: Owner{
|
Owner: Owner{
|
||||||
ID: ver.NodeVersion.Owner.String(),
|
ID: ver.NodeVersion.Owner.String(),
|
||||||
|
@ -301,7 +297,7 @@ func encodeListObjectVersionsToResponse(p *layer.ListObjectVersionsParams, info
|
||||||
for _, del := range info.DeleteMarker {
|
for _, del := range info.DeleteMarker {
|
||||||
res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{
|
res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{
|
||||||
IsLatest: del.IsLatest,
|
IsLatest: del.IsLatest,
|
||||||
Key: s3PathEncode(del.NodeVersion.FilePath, p.Encode),
|
Key: del.NodeVersion.FilePath,
|
||||||
LastModified: del.NodeVersion.Created.UTC().Format(time.RFC3339),
|
LastModified: del.NodeVersion.Created.UTC().Format(time.RFC3339),
|
||||||
Owner: Owner{
|
Owner: Owner{
|
||||||
ID: del.NodeVersion.Owner.String(),
|
ID: del.NodeVersion.Owner.String(),
|
||||||
|
|
|
@ -7,7 +7,6 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
@ -16,11 +15,8 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap"
|
|
||||||
"go.uber.org/zap/zaptest"
|
"go.uber.org/zap/zaptest"
|
||||||
"go.uber.org/zap/zaptest/observer"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestParseContinuationToken(t *testing.T) {
|
func TestParseContinuationToken(t *testing.T) {
|
||||||
|
@ -97,39 +93,12 @@ func TestListObjectsWithOldTreeNodes(t *testing.T) {
|
||||||
checkListVersionsOldNodes(hc, listVers.Version, objInfos)
|
checkListVersionsOldNodes(hc, listVers.Version, objInfos)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestListObjectsVersionsSkipLogTaggingNodesError(t *testing.T) {
|
|
||||||
loggerCore, observedLog := observer.New(zap.DebugLevel)
|
|
||||||
log := zap.New(loggerCore)
|
|
||||||
|
|
||||||
hcBase, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(log))
|
|
||||||
require.NoError(t, err)
|
|
||||||
hc := &handlerContext{
|
|
||||||
handlerContextBase: hcBase,
|
|
||||||
t: t,
|
|
||||||
}
|
|
||||||
|
|
||||||
bktName, objName := "bucket-versioning-enabled", "versions/object"
|
|
||||||
bktInfo := createTestBucket(hc, bktName)
|
|
||||||
|
|
||||||
createTestObject(hc, bktInfo, objName, encryption.Params{})
|
|
||||||
createTestObject(hc, bktInfo, objName, encryption.Params{})
|
|
||||||
|
|
||||||
putObjectTagging(hc.t, hc, bktName, objName, map[string]string{"tag1": "val1"})
|
|
||||||
|
|
||||||
listObjectsVersions(hc, bktName, "", "", "", "", -1)
|
|
||||||
|
|
||||||
filtered := observedLog.Filter(func(entry observer.LoggedEntry) bool {
|
|
||||||
return strings.Contains(entry.Message, logs.ParseTreeNode)
|
|
||||||
})
|
|
||||||
require.Empty(t, filtered)
|
|
||||||
}
|
|
||||||
|
|
||||||
func makeAllTreeObjectsOld(hc *handlerContext, bktInfo *data.BucketInfo) {
|
func makeAllTreeObjectsOld(hc *handlerContext, bktInfo *data.BucketInfo) {
|
||||||
nodes, err := hc.treeMock.GetSubTree(hc.Context(), bktInfo, "version", []uint64{0}, 0, true)
|
nodes, err := hc.treeMock.GetSubTree(hc.Context(), bktInfo, "version", 0, 0)
|
||||||
require.NoError(hc.t, err)
|
require.NoError(hc.t, err)
|
||||||
|
|
||||||
for _, node := range nodes {
|
for _, node := range nodes {
|
||||||
if node.GetNodeID()[0] == 0 {
|
if node.GetNodeID() == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
meta := make(map[string]string, len(node.GetMeta()))
|
meta := make(map[string]string, len(node.GetMeta()))
|
||||||
|
@ -139,7 +108,7 @@ func makeAllTreeObjectsOld(hc *handlerContext, bktInfo *data.BucketInfo) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err = hc.treeMock.MoveNode(hc.Context(), bktInfo, "version", node.GetNodeID()[0], node.GetParentID()[0], meta)
|
err = hc.treeMock.MoveNode(hc.Context(), bktInfo, "version", node.GetNodeID(), node.GetParentID(), meta)
|
||||||
require.NoError(hc.t, err)
|
require.NoError(hc.t, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -169,17 +138,11 @@ func checkListVersionsOldNodes(hc *handlerContext, list []ObjectVersionResponse,
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestListObjectsContextCanceled(t *testing.T) {
|
func TestListObjectsContextCanceled(t *testing.T) {
|
||||||
log := zaptest.NewLogger(t)
|
layerCfg := layer.DefaultCachesConfigs(zaptest.NewLogger(t))
|
||||||
layerCfg := layer.DefaultCachesConfigs(log)
|
|
||||||
layerCfg.SessionList.Lifetime = time.Hour
|
layerCfg.SessionList.Lifetime = time.Hour
|
||||||
layerCfg.SessionList.Size = 1
|
layerCfg.SessionList.Size = 1
|
||||||
|
|
||||||
hcBase, err := prepareHandlerContextBase(layerCfg)
|
hc := prepareHandlerContextBase(t, layerCfg)
|
||||||
require.NoError(t, err)
|
|
||||||
hc := &handlerContext{
|
|
||||||
handlerContextBase: hcBase,
|
|
||||||
t: t,
|
|
||||||
}
|
|
||||||
|
|
||||||
bktName := "bucket-versioning-enabled"
|
bktName := "bucket-versioning-enabled"
|
||||||
bktInfo := createTestBucket(hc, bktName)
|
bktInfo := createTestBucket(hc, bktName)
|
||||||
|
@ -712,49 +675,6 @@ func TestMintVersioningListObjectVersionsVersionIDContinuation(t *testing.T) {
|
||||||
require.Equal(t, page1.NextVersionIDMarker, page2.VersionIDMarker)
|
require.Equal(t, page1.NextVersionIDMarker, page2.VersionIDMarker)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestListObjectVersionsEncoding(t *testing.T) {
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
|
|
||||||
bktName := "bucket-for-listing-versions-encoding"
|
|
||||||
bktInfo := createTestBucket(hc, bktName)
|
|
||||||
putBucketVersioning(t, hc, bktName, true)
|
|
||||||
|
|
||||||
objects := []string{"foo()/bar", "foo()/bar/xyzzy", "auux ab/thud", "asdf+b"}
|
|
||||||
for _, objName := range objects {
|
|
||||||
createTestObject(hc, bktInfo, objName, encryption.Params{})
|
|
||||||
}
|
|
||||||
deleteObject(t, hc, bktName, "auux ab/thud", "")
|
|
||||||
|
|
||||||
listResponse := listObjectsVersionsURL(hc, bktName, "foo(", ")", "", "", -1)
|
|
||||||
|
|
||||||
require.Len(t, listResponse.CommonPrefixes, 1)
|
|
||||||
require.Equal(t, "foo%28%29", listResponse.CommonPrefixes[0].Prefix)
|
|
||||||
require.Len(t, listResponse.Version, 0)
|
|
||||||
require.Len(t, listResponse.DeleteMarker, 0)
|
|
||||||
require.Equal(t, "foo%28", listResponse.Prefix)
|
|
||||||
require.Equal(t, "%29", listResponse.Delimiter)
|
|
||||||
require.Equal(t, "url", listResponse.EncodingType)
|
|
||||||
require.Equal(t, maxObjectList, listResponse.MaxKeys)
|
|
||||||
|
|
||||||
listResponse = listObjectsVersions(hc, bktName, "", "", "", "", 1)
|
|
||||||
require.Empty(t, listResponse.EncodingType)
|
|
||||||
|
|
||||||
listResponse = listObjectsVersionsURL(hc, bktName, "", "", listResponse.NextKeyMarker, listResponse.NextVersionIDMarker, 3)
|
|
||||||
|
|
||||||
require.Len(t, listResponse.CommonPrefixes, 0)
|
|
||||||
require.Len(t, listResponse.Version, 2)
|
|
||||||
require.Equal(t, "auux%20ab/thud", listResponse.Version[0].Key)
|
|
||||||
require.False(t, listResponse.Version[0].IsLatest)
|
|
||||||
require.Equal(t, "foo%28%29/bar", listResponse.Version[1].Key)
|
|
||||||
require.Len(t, listResponse.DeleteMarker, 1)
|
|
||||||
require.Equal(t, "auux%20ab/thud", listResponse.DeleteMarker[0].Key)
|
|
||||||
require.True(t, listResponse.DeleteMarker[0].IsLatest)
|
|
||||||
require.Equal(t, "asdf%2Bb", listResponse.KeyMarker)
|
|
||||||
require.Equal(t, "foo%28%29/bar", listResponse.NextKeyMarker)
|
|
||||||
require.Equal(t, "url", listResponse.EncodingType)
|
|
||||||
require.Equal(t, 3, listResponse.MaxKeys)
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkVersionsNames(t *testing.T, versions *ListObjectsVersionsResponse, names []string) {
|
func checkVersionsNames(t *testing.T, versions *ListObjectsVersionsResponse, names []string) {
|
||||||
for i, v := range versions.Version {
|
for i, v := range versions.Version {
|
||||||
require.Equal(t, names[i], v.Key)
|
require.Equal(t, names[i], v.Key)
|
||||||
|
@ -857,14 +777,6 @@ func listObjectsV1(hc *handlerContext, bktName, prefix, delimiter, marker string
|
||||||
}
|
}
|
||||||
|
|
||||||
func listObjectsVersions(hc *handlerContext, bktName, prefix, delimiter, keyMarker, versionIDMarker string, maxKeys int) *ListObjectsVersionsResponse {
|
func listObjectsVersions(hc *handlerContext, bktName, prefix, delimiter, keyMarker, versionIDMarker string, maxKeys int) *ListObjectsVersionsResponse {
|
||||||
return listObjectsVersionsBase(hc, bktName, prefix, delimiter, keyMarker, versionIDMarker, maxKeys, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
func listObjectsVersionsURL(hc *handlerContext, bktName, prefix, delimiter, keyMarker, versionIDMarker string, maxKeys int) *ListObjectsVersionsResponse {
|
|
||||||
return listObjectsVersionsBase(hc, bktName, prefix, delimiter, keyMarker, versionIDMarker, maxKeys, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
func listObjectsVersionsBase(hc *handlerContext, bktName, prefix, delimiter, keyMarker, versionIDMarker string, maxKeys int, encode bool) *ListObjectsVersionsResponse {
|
|
||||||
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
||||||
if len(keyMarker) != 0 {
|
if len(keyMarker) != 0 {
|
||||||
query.Add("key-marker", keyMarker)
|
query.Add("key-marker", keyMarker)
|
||||||
|
@ -872,9 +784,6 @@ func listObjectsVersionsBase(hc *handlerContext, bktName, prefix, delimiter, key
|
||||||
if len(versionIDMarker) != 0 {
|
if len(versionIDMarker) != 0 {
|
||||||
query.Add("version-id-marker", versionIDMarker)
|
query.Add("version-id-marker", versionIDMarker)
|
||||||
}
|
}
|
||||||
if encode {
|
|
||||||
query.Add("encoding-type", "url")
|
|
||||||
}
|
|
||||||
|
|
||||||
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
||||||
hc.Handler().ListBucketObjectVersionsHandler(w, r)
|
hc.Handler().ListBucketObjectVersionsHandler(w, r)
|
||||||
|
|
|
@ -1,195 +0,0 @@
|
||||||
package handler
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
|
||||||
"go.uber.org/zap"
|
|
||||||
)
|
|
||||||
|
|
||||||
const maxPatchSize = 5 * 1024 * 1024 * 1024 // 5GB
|
|
||||||
|
|
||||||
func (h *handler) PatchObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
ctx := r.Context()
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
|
||||||
|
|
||||||
if _, ok := r.Header[api.ContentRange]; !ok {
|
|
||||||
h.logAndSendError(w, "missing Content-Range", reqInfo, errors.GetAPIError(errors.ErrMissingContentRange))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, ok := r.Header[api.ContentLength]; !ok {
|
|
||||||
h.logAndSendError(w, "missing Content-Length", reqInfo, errors.GetAPIError(errors.ErrMissingContentLength))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
conditional, err := parsePatchConditionalHeaders(r.Header)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not parse conditional headers", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
srcObjPrm := &layer.HeadObjectParams{
|
|
||||||
Object: reqInfo.ObjectName,
|
|
||||||
BktInfo: bktInfo,
|
|
||||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
|
||||||
}
|
|
||||||
|
|
||||||
extendedSrcObjInfo, err := h.obj.GetExtendedObjectInfo(ctx, srcObjPrm)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not find object", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
srcObjInfo := extendedSrcObjInfo.ObjectInfo
|
|
||||||
|
|
||||||
if err = checkPreconditions(srcObjInfo, conditional, h.cfg.MD5Enabled()); err != nil {
|
|
||||||
h.logAndSendError(w, "precondition failed", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
srcSize, err := layer.GetObjectSize(srcObjInfo)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "failed to get source object size", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
byteRange, err := parsePatchByteRange(r.Header.Get(api.ContentRange), srcSize)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not parse byte range", reqInfo, errors.GetAPIError(errors.ErrInvalidRange), zap.Error(err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if maxPatchSize < byteRange.End-byteRange.Start+1 {
|
|
||||||
h.logAndSendError(w, "byte range length is longer than allowed", reqInfo, errors.GetAPIError(errors.ErrInvalidRange), zap.Error(err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if uint64(r.ContentLength) != (byteRange.End - byteRange.Start + 1) {
|
|
||||||
h.logAndSendError(w, "content-length must be equal to byte range length", reqInfo, errors.GetAPIError(errors.ErrInvalidRangeLength))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if byteRange.Start > srcSize {
|
|
||||||
h.logAndSendError(w, "start byte is greater than object size", reqInfo, errors.GetAPIError(errors.ErrRangeOutOfBounds))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
params := &layer.PatchObjectParams{
|
|
||||||
Object: extendedSrcObjInfo,
|
|
||||||
BktInfo: bktInfo,
|
|
||||||
NewBytes: r.Body,
|
|
||||||
Range: byteRange,
|
|
||||||
VersioningEnabled: settings.VersioningEnabled(),
|
|
||||||
}
|
|
||||||
|
|
||||||
params.CopiesNumbers, err = h.pickCopiesNumbers(nil, reqInfo.Namespace, bktInfo.LocationConstraint)
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
extendedObjInfo, err := h.obj.PatchObject(ctx, params)
|
|
||||||
if err != nil {
|
|
||||||
if isErrObjectLocked(err) {
|
|
||||||
h.logAndSendError(w, "object is locked", reqInfo, errors.GetAPIError(errors.ErrAccessDenied))
|
|
||||||
} else {
|
|
||||||
h.logAndSendError(w, "could not patch object", reqInfo, err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if settings.VersioningEnabled() {
|
|
||||||
w.Header().Set(api.AmzVersionID, extendedObjInfo.ObjectInfo.VersionID())
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Header().Set(api.ETag, data.Quote(extendedObjInfo.ObjectInfo.ETag(h.cfg.MD5Enabled())))
|
|
||||||
|
|
||||||
resp := PatchObjectResult{
|
|
||||||
Object: PatchObject{
|
|
||||||
LastModified: extendedObjInfo.ObjectInfo.Created.UTC().Format(time.RFC3339),
|
|
||||||
ETag: data.Quote(extendedObjInfo.ObjectInfo.ETag(h.cfg.MD5Enabled())),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = middleware.EncodeToResponse(w, resp); err != nil {
|
|
||||||
h.logAndSendError(w, "could not encode PatchObjectResult to response", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func parsePatchConditionalHeaders(headers http.Header) (*conditionalArgs, error) {
|
|
||||||
var err error
|
|
||||||
args := &conditionalArgs{
|
|
||||||
IfMatch: data.UnQuote(headers.Get(api.IfMatch)),
|
|
||||||
}
|
|
||||||
|
|
||||||
if args.IfUnmodifiedSince, err = parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return args, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parsePatchByteRange(rangeStr string, objSize uint64) (*layer.RangeParams, error) {
|
|
||||||
const prefix = "bytes "
|
|
||||||
|
|
||||||
if rangeStr == "" {
|
|
||||||
return nil, fmt.Errorf("empty range")
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.HasPrefix(rangeStr, prefix) {
|
|
||||||
return nil, fmt.Errorf("unknown unit in range header")
|
|
||||||
}
|
|
||||||
|
|
||||||
rangeStr, _, found := strings.Cut(strings.TrimPrefix(rangeStr, prefix), "/") // value after / is ignored
|
|
||||||
if !found {
|
|
||||||
return nil, fmt.Errorf("invalid range: %s", rangeStr)
|
|
||||||
}
|
|
||||||
|
|
||||||
startStr, endStr, found := strings.Cut(rangeStr, "-")
|
|
||||||
if !found {
|
|
||||||
return nil, fmt.Errorf("invalid range: %s", rangeStr)
|
|
||||||
}
|
|
||||||
|
|
||||||
start, err := strconv.ParseUint(startStr, 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("invalid start byte: %s", startStr)
|
|
||||||
}
|
|
||||||
|
|
||||||
end := objSize - 1
|
|
||||||
if len(endStr) > 0 {
|
|
||||||
end, err = strconv.ParseUint(endStr, 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("invalid end byte: %s", endStr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if start > end {
|
|
||||||
return nil, fmt.Errorf("start byte is greater than end byte")
|
|
||||||
}
|
|
||||||
|
|
||||||
return &layer.RangeParams{
|
|
||||||
Start: start,
|
|
||||||
End: end,
|
|
||||||
}, nil
|
|
||||||
}
|
|
|
@ -1,524 +0,0 @@
|
||||||
package handler
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"crypto/md5"
|
|
||||||
"crypto/rand"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/xml"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPatch(t *testing.T) {
|
|
||||||
tc := prepareHandlerContext(t)
|
|
||||||
tc.config.md5Enabled = true
|
|
||||||
|
|
||||||
bktName, objName := "bucket-for-patch", "object-for-patch"
|
|
||||||
createTestBucket(tc, bktName)
|
|
||||||
|
|
||||||
content := []byte("old object content")
|
|
||||||
md5Hash := md5.New()
|
|
||||||
md5Hash.Write(content)
|
|
||||||
etag := data.Quote(hex.EncodeToString(md5Hash.Sum(nil)))
|
|
||||||
|
|
||||||
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
|
||||||
created := time.Now()
|
|
||||||
tc.Handler().PutObjectHandler(w, r)
|
|
||||||
require.Equal(t, etag, w.Header().Get(api.ETag))
|
|
||||||
|
|
||||||
patchPayload := []byte("new")
|
|
||||||
sha256Hash := sha256.New()
|
|
||||||
sha256Hash.Write(patchPayload)
|
|
||||||
sha256Hash.Write(content[len(patchPayload):])
|
|
||||||
hash := hex.EncodeToString(sha256Hash.Sum(nil))
|
|
||||||
|
|
||||||
for _, tt := range []struct {
|
|
||||||
name string
|
|
||||||
rng string
|
|
||||||
headers map[string]string
|
|
||||||
code s3errors.ErrorCode
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "success",
|
|
||||||
rng: "bytes 0-2/*",
|
|
||||||
headers: map[string]string{
|
|
||||||
api.IfUnmodifiedSince: created.Format(http.TimeFormat),
|
|
||||||
api.IfMatch: etag,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid range syntax",
|
|
||||||
rng: "bytes 0-2",
|
|
||||||
code: s3errors.ErrInvalidRange,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid range length",
|
|
||||||
rng: "bytes 0-5/*",
|
|
||||||
code: s3errors.ErrInvalidRangeLength,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid range start",
|
|
||||||
rng: "bytes 20-22/*",
|
|
||||||
code: s3errors.ErrRangeOutOfBounds,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "range is too long",
|
|
||||||
rng: "bytes 0-5368709120/*",
|
|
||||||
code: s3errors.ErrInvalidRange,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "If-Unmodified-Since precondition are not satisfied",
|
|
||||||
rng: "bytes 0-2/*",
|
|
||||||
headers: map[string]string{
|
|
||||||
api.IfUnmodifiedSince: created.Add(-24 * time.Hour).Format(http.TimeFormat),
|
|
||||||
},
|
|
||||||
code: s3errors.ErrPreconditionFailed,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "If-Match precondition are not satisfied",
|
|
||||||
rng: "bytes 0-2/*",
|
|
||||||
headers: map[string]string{
|
|
||||||
api.IfMatch: "etag",
|
|
||||||
},
|
|
||||||
code: s3errors.ErrPreconditionFailed,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
if tt.code == 0 {
|
|
||||||
res := patchObject(t, tc, bktName, objName, tt.rng, patchPayload, tt.headers)
|
|
||||||
require.Equal(t, data.Quote(hash), res.Object.ETag)
|
|
||||||
} else {
|
|
||||||
patchObjectErr(t, tc, bktName, objName, tt.rng, patchPayload, tt.headers, tt.code)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPatchMultipartObject(t *testing.T) {
|
|
||||||
tc := prepareHandlerContextWithMinCache(t)
|
|
||||||
tc.config.md5Enabled = true
|
|
||||||
|
|
||||||
bktName, objName, partSize := "bucket-for-multipart-patch", "object-for-multipart-patch", 5*1024*1024
|
|
||||||
createTestBucket(tc, bktName)
|
|
||||||
|
|
||||||
t.Run("patch beginning of the first part", func(t *testing.T) {
|
|
||||||
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
|
|
||||||
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
|
|
||||||
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
|
|
||||||
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
|
|
||||||
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
|
|
||||||
|
|
||||||
patchSize := partSize / 2
|
|
||||||
patchBody := make([]byte, patchSize)
|
|
||||||
_, err := rand.Read(patchBody)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
patchObject(t, tc, bktName, objName, "bytes 0-"+strconv.Itoa(patchSize-1)+"/*", patchBody, nil)
|
|
||||||
object, header := getObject(tc, bktName, objName)
|
|
||||||
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
|
|
||||||
require.NoError(t, err)
|
|
||||||
equalDataSlices(t, bytes.Join([][]byte{patchBody, data1[patchSize:], data2, data3}, []byte("")), object)
|
|
||||||
require.Equal(t, partSize*3, contentLen)
|
|
||||||
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("patch middle of the first part", func(t *testing.T) {
|
|
||||||
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
|
|
||||||
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
|
|
||||||
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
|
|
||||||
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
|
|
||||||
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
|
|
||||||
|
|
||||||
patchSize := partSize / 2
|
|
||||||
patchBody := make([]byte, patchSize)
|
|
||||||
_, err := rand.Read(patchBody)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize/4)+"-"+strconv.Itoa(partSize*3/4-1)+"/*", patchBody, nil)
|
|
||||||
object, header := getObject(tc, bktName, objName)
|
|
||||||
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
|
|
||||||
require.NoError(t, err)
|
|
||||||
equalDataSlices(t, bytes.Join([][]byte{data1[:partSize/4], patchBody, data1[partSize*3/4:], data2, data3}, []byte("")), object)
|
|
||||||
require.Equal(t, partSize*3, contentLen)
|
|
||||||
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("patch first and second parts", func(t *testing.T) {
|
|
||||||
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
|
|
||||||
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
|
|
||||||
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
|
|
||||||
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
|
|
||||||
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
|
|
||||||
|
|
||||||
patchSize := partSize / 2
|
|
||||||
patchBody := make([]byte, patchSize)
|
|
||||||
_, err := rand.Read(patchBody)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*3/4)+"-"+strconv.Itoa(partSize*5/4-1)+"/*", patchBody, nil)
|
|
||||||
object, header := getObject(tc, bktName, objName)
|
|
||||||
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
|
|
||||||
require.NoError(t, err)
|
|
||||||
equalDataSlices(t, bytes.Join([][]byte{data1[:partSize*3/4], patchBody, data2[partSize/4:], data3}, []byte("")), object)
|
|
||||||
require.Equal(t, partSize*3, contentLen)
|
|
||||||
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("patch all parts", func(t *testing.T) {
|
|
||||||
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
|
|
||||||
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
|
|
||||||
etag2, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
|
|
||||||
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
|
|
||||||
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
|
|
||||||
|
|
||||||
patchSize := partSize * 2
|
|
||||||
patchBody := make([]byte, patchSize)
|
|
||||||
_, err := rand.Read(patchBody)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize/2-1)+"-"+strconv.Itoa(partSize/2+patchSize-2)+"/*", patchBody, nil)
|
|
||||||
object, header := getObject(tc, bktName, objName)
|
|
||||||
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
|
|
||||||
require.NoError(t, err)
|
|
||||||
equalDataSlices(t, bytes.Join([][]byte{data1[:partSize/2-1], patchBody, data3[partSize/2-1:]}, []byte("")), object)
|
|
||||||
require.Equal(t, partSize*3, contentLen)
|
|
||||||
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("patch all parts and append bytes", func(t *testing.T) {
|
|
||||||
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
|
|
||||||
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
|
|
||||||
etag2, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
|
|
||||||
etag3, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
|
|
||||||
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
|
|
||||||
|
|
||||||
patchSize := partSize * 3
|
|
||||||
patchBody := make([]byte, patchSize)
|
|
||||||
_, err := rand.Read(patchBody)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize/2)+"-"+strconv.Itoa(partSize/2+patchSize-1)+"/*", patchBody, nil)
|
|
||||||
object, header := getObject(tc, bktName, objName)
|
|
||||||
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
|
|
||||||
require.NoError(t, err)
|
|
||||||
equalDataSlices(t, bytes.Join([][]byte{data1[:partSize/2], patchBody}, []byte("")), object)
|
|
||||||
require.Equal(t, partSize*7/2, contentLen)
|
|
||||||
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("patch second part", func(t *testing.T) {
|
|
||||||
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
|
|
||||||
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
|
|
||||||
etag2, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
|
|
||||||
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
|
|
||||||
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
|
|
||||||
|
|
||||||
patchBody := make([]byte, partSize)
|
|
||||||
_, err := rand.Read(patchBody)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize)+"-"+strconv.Itoa(partSize*2-1)+"/*", patchBody, nil)
|
|
||||||
object, header := getObject(tc, bktName, objName)
|
|
||||||
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
|
|
||||||
require.NoError(t, err)
|
|
||||||
equalDataSlices(t, bytes.Join([][]byte{data1, patchBody, data3}, []byte("")), object)
|
|
||||||
require.Equal(t, partSize*3, contentLen)
|
|
||||||
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("patch last part, equal size", func(t *testing.T) {
|
|
||||||
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
|
|
||||||
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
|
|
||||||
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
|
|
||||||
etag3, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
|
|
||||||
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
|
|
||||||
|
|
||||||
patchBody := make([]byte, partSize)
|
|
||||||
_, err := rand.Read(patchBody)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*2)+"-"+strconv.Itoa(partSize*3-1)+"/*", patchBody, nil)
|
|
||||||
object, header := getObject(tc, bktName, objName)
|
|
||||||
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
|
|
||||||
require.NoError(t, err)
|
|
||||||
equalDataSlices(t, bytes.Join([][]byte{data1, data2, patchBody}, []byte("")), object)
|
|
||||||
require.Equal(t, partSize*3, contentLen)
|
|
||||||
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("patch last part, increase size", func(t *testing.T) {
|
|
||||||
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
|
|
||||||
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
|
|
||||||
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
|
|
||||||
etag3, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
|
|
||||||
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
|
|
||||||
|
|
||||||
patchBody := make([]byte, partSize+1)
|
|
||||||
_, err := rand.Read(patchBody)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*2)+"-"+strconv.Itoa(partSize*3)+"/*", patchBody, nil)
|
|
||||||
object, header := getObject(tc, bktName, objName)
|
|
||||||
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
|
|
||||||
require.NoError(t, err)
|
|
||||||
equalDataSlices(t, bytes.Join([][]byte{data1, data2, patchBody}, []byte("")), object)
|
|
||||||
require.Equal(t, partSize*3+1, contentLen)
|
|
||||||
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("patch last part with offset and append bytes", func(t *testing.T) {
|
|
||||||
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
|
|
||||||
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
|
|
||||||
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
|
|
||||||
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
|
|
||||||
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
|
|
||||||
|
|
||||||
patchBody := make([]byte, partSize)
|
|
||||||
_, err := rand.Read(patchBody)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*2+3)+"-"+strconv.Itoa(partSize*3+2)+"/*", patchBody, nil)
|
|
||||||
object, header := getObject(tc, bktName, objName)
|
|
||||||
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
|
|
||||||
require.NoError(t, err)
|
|
||||||
equalDataSlices(t, bytes.Join([][]byte{data1, data2, data3[:3], patchBody}, []byte("")), object)
|
|
||||||
require.Equal(t, partSize*3+3, contentLen)
|
|
||||||
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("append bytes", func(t *testing.T) {
|
|
||||||
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
|
|
||||||
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
|
|
||||||
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
|
|
||||||
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
|
|
||||||
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
|
|
||||||
|
|
||||||
patchBody := make([]byte, partSize)
|
|
||||||
_, err := rand.Read(patchBody)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*3)+"-"+strconv.Itoa(partSize*4-1)+"/*", patchBody, nil)
|
|
||||||
object, header := getObject(tc, bktName, objName)
|
|
||||||
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
|
|
||||||
require.NoError(t, err)
|
|
||||||
equalDataSlices(t, bytes.Join([][]byte{data1, data2, data3, patchBody}, []byte("")), object)
|
|
||||||
require.Equal(t, partSize*4, contentLen)
|
|
||||||
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("patch empty multipart", func(t *testing.T) {
|
|
||||||
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
|
|
||||||
etag, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, 0)
|
|
||||||
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag})
|
|
||||||
|
|
||||||
patchBody := make([]byte, partSize)
|
|
||||||
_, err := rand.Read(patchBody)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
patchObject(t, tc, bktName, objName, "bytes 0-"+strconv.Itoa(partSize-1)+"/*", patchBody, nil)
|
|
||||||
object, header := getObject(tc, bktName, objName)
|
|
||||||
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
|
|
||||||
require.NoError(t, err)
|
|
||||||
equalDataSlices(t, patchBody, object)
|
|
||||||
require.Equal(t, partSize, contentLen)
|
|
||||||
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-1"))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPatchWithVersion(t *testing.T) {
|
|
||||||
hc := prepareHandlerContextWithMinCache(t)
|
|
||||||
bktName, objName := "bucket", "obj"
|
|
||||||
createVersionedBucket(hc, bktName)
|
|
||||||
objHeader := putObjectContent(hc, bktName, objName, "content")
|
|
||||||
|
|
||||||
putObjectContent(hc, bktName, objName, "some content")
|
|
||||||
|
|
||||||
patchObjectVersion(t, hc, bktName, objName, objHeader.Get(api.AmzVersionID), "bytes 7-14/*", []byte(" updated"))
|
|
||||||
|
|
||||||
res := listObjectsVersions(hc, bktName, "", "", "", "", 3)
|
|
||||||
require.False(t, res.IsTruncated)
|
|
||||||
require.Len(t, res.Version, 3)
|
|
||||||
|
|
||||||
for _, version := range res.Version {
|
|
||||||
content := getObjectVersion(hc, bktName, objName, version.VersionID)
|
|
||||||
if version.IsLatest {
|
|
||||||
require.Equal(t, []byte("content updated"), content)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if version.VersionID == objHeader.Get(api.AmzVersionID) {
|
|
||||||
require.Equal(t, []byte("content"), content)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
require.Equal(t, []byte("some content"), content)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPatchEncryptedObject(t *testing.T) {
|
|
||||||
tc := prepareHandlerContext(t)
|
|
||||||
bktName, objName := "bucket-for-patch-encrypted", "object-for-patch-encrypted"
|
|
||||||
createTestBucket(tc, bktName)
|
|
||||||
|
|
||||||
w, r := prepareTestPayloadRequest(tc, bktName, objName, strings.NewReader("object content"))
|
|
||||||
setEncryptHeaders(r)
|
|
||||||
tc.Handler().PutObjectHandler(w, r)
|
|
||||||
assertStatus(t, w, http.StatusOK)
|
|
||||||
|
|
||||||
patchObjectErr(t, tc, bktName, objName, "bytes 2-4/*", []byte("new"), nil, s3errors.ErrInternalError)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPatchMissingHeaders(t *testing.T) {
|
|
||||||
tc := prepareHandlerContext(t)
|
|
||||||
bktName, objName := "bucket-for-patch-missing-headers", "object-for-patch-missing-headers"
|
|
||||||
createTestBucket(tc, bktName)
|
|
||||||
|
|
||||||
w, r := prepareTestPayloadRequest(tc, bktName, objName, strings.NewReader("object content"))
|
|
||||||
setEncryptHeaders(r)
|
|
||||||
tc.Handler().PutObjectHandler(w, r)
|
|
||||||
assertStatus(t, w, http.StatusOK)
|
|
||||||
|
|
||||||
w = httptest.NewRecorder()
|
|
||||||
r = httptest.NewRequest(http.MethodPatch, defaultURL, strings.NewReader("new"))
|
|
||||||
tc.Handler().PatchObjectHandler(w, r)
|
|
||||||
assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrMissingContentRange))
|
|
||||||
|
|
||||||
w = httptest.NewRecorder()
|
|
||||||
r = httptest.NewRequest(http.MethodPatch, defaultURL, strings.NewReader("new"))
|
|
||||||
r.Header.Set(api.ContentRange, "bytes 0-2/*")
|
|
||||||
tc.Handler().PatchObjectHandler(w, r)
|
|
||||||
assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrMissingContentLength))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParsePatchByteRange(t *testing.T) {
|
|
||||||
for _, tt := range []struct {
|
|
||||||
rng string
|
|
||||||
size uint64
|
|
||||||
expected *layer.RangeParams
|
|
||||||
err bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
rng: "bytes 2-7/*",
|
|
||||||
expected: &layer.RangeParams{
|
|
||||||
Start: 2,
|
|
||||||
End: 7,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rng: "bytes 2-7/3",
|
|
||||||
expected: &layer.RangeParams{
|
|
||||||
Start: 2,
|
|
||||||
End: 7,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rng: "bytes 2-/*",
|
|
||||||
size: 9,
|
|
||||||
expected: &layer.RangeParams{
|
|
||||||
Start: 2,
|
|
||||||
End: 8,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rng: "bytes 2-/3",
|
|
||||||
size: 9,
|
|
||||||
expected: &layer.RangeParams{
|
|
||||||
Start: 2,
|
|
||||||
End: 8,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rng: "",
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rng: "2-7/*",
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rng: "bytes 7-2/*",
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rng: "bytes 2-7",
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rng: "bytes 2/*",
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rng: "bytes a-7/*",
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rng: "bytes 2-a/*",
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(fmt.Sprintf("case: %s", tt.rng), func(t *testing.T) {
|
|
||||||
rng, err := parsePatchByteRange(tt.rng, tt.size)
|
|
||||||
if tt.err {
|
|
||||||
require.Error(t, err)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, tt.expected.Start, rng.Start)
|
|
||||||
require.Equal(t, tt.expected.End, rng.End)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func patchObject(t *testing.T, tc *handlerContext, bktName, objName, rng string, payload []byte, headers map[string]string) *PatchObjectResult {
|
|
||||||
w := patchObjectBase(tc, bktName, objName, "", rng, payload, headers)
|
|
||||||
assertStatus(t, w, http.StatusOK)
|
|
||||||
|
|
||||||
result := &PatchObjectResult{}
|
|
||||||
err := xml.NewDecoder(w.Result().Body).Decode(result)
|
|
||||||
require.NoError(t, err)
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
func patchObjectVersion(t *testing.T, tc *handlerContext, bktName, objName, version, rng string, payload []byte) *PatchObjectResult {
|
|
||||||
w := patchObjectBase(tc, bktName, objName, version, rng, payload, nil)
|
|
||||||
assertStatus(t, w, http.StatusOK)
|
|
||||||
|
|
||||||
result := &PatchObjectResult{}
|
|
||||||
err := xml.NewDecoder(w.Result().Body).Decode(result)
|
|
||||||
require.NoError(t, err)
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
func patchObjectErr(t *testing.T, tc *handlerContext, bktName, objName, rng string, payload []byte, headers map[string]string, code s3errors.ErrorCode) {
|
|
||||||
w := patchObjectBase(tc, bktName, objName, "", rng, payload, headers)
|
|
||||||
assertS3Error(t, w, s3errors.GetAPIError(code))
|
|
||||||
}
|
|
||||||
|
|
||||||
func patchObjectBase(tc *handlerContext, bktName, objName, version, rng string, payload []byte, headers map[string]string) *httptest.ResponseRecorder {
|
|
||||||
query := make(url.Values)
|
|
||||||
if len(version) > 0 {
|
|
||||||
query.Add(api.QueryVersionID, version)
|
|
||||||
}
|
|
||||||
|
|
||||||
w, r := prepareTestRequestWithQuery(tc, bktName, objName, query, payload)
|
|
||||||
r.Header.Set(api.ContentRange, rng)
|
|
||||||
r.Header.Set(api.ContentLength, strconv.Itoa(len(payload)))
|
|
||||||
for k, v := range headers {
|
|
||||||
r.Header.Set(k, v)
|
|
||||||
}
|
|
||||||
|
|
||||||
tc.Handler().PatchObjectHandler(w, r)
|
|
||||||
return w
|
|
||||||
}
|
|
|
@ -4,12 +4,12 @@ import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"crypto/md5"
|
"crypto/md5"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
stderrors "errors"
|
stderrors "errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"mime/multipart"
|
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
@ -26,14 +26,12 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/retryer"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||||
"git.frostfs.info/TrueCloudLab/policy-engine/schema/native"
|
"git.frostfs.info/TrueCloudLab/policy-engine/schema/native"
|
||||||
"git.frostfs.info/TrueCloudLab/policy-engine/schema/s3"
|
"git.frostfs.info/TrueCloudLab/policy-engine/schema/s3"
|
||||||
"github.com/aws/aws-sdk-go-v2/aws"
|
|
||||||
"github.com/aws/aws-sdk-go-v2/aws/retry"
|
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -175,6 +173,7 @@ const (
|
||||||
basicACLPrivate = "private"
|
basicACLPrivate = "private"
|
||||||
basicACLReadOnly = "public-read"
|
basicACLReadOnly = "public-read"
|
||||||
basicACLPublic = "public-read-write"
|
basicACLPublic = "public-read-write"
|
||||||
|
cannedACLAuthRead = "authenticated-read"
|
||||||
)
|
)
|
||||||
|
|
||||||
type createBucketParams struct {
|
type createBucketParams struct {
|
||||||
|
@ -185,6 +184,8 @@ type createBucketParams struct {
|
||||||
func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
|
newEaclTable *eacl.Table
|
||||||
|
sessionTokenEACL *session.Container
|
||||||
cannedACLStatus = aclHeadersStatus(r)
|
cannedACLStatus = aclHeadersStatus(r)
|
||||||
ctx = r.Context()
|
ctx = r.Context()
|
||||||
reqInfo = middleware.GetReqInfo(ctx)
|
reqInfo = middleware.GetReqInfo(ctx)
|
||||||
|
@ -202,11 +203,20 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if cannedACLStatus == aclStatusYes {
|
apeEnabled := bktInfo.APEEnabled || settings.CannedACL != ""
|
||||||
|
if apeEnabled && cannedACLStatus == aclStatusYes {
|
||||||
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
needUpdateEACLTable := !(apeEnabled || cannedACLStatus == aclStatusNo)
|
||||||
|
if needUpdateEACLTable {
|
||||||
|
if sessionTokenEACL, err = getSessionTokenSetEACL(r.Context()); err != nil {
|
||||||
|
h.logAndSendError(w, "could not get eacl session token from a box", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
tagSet, err := parseTaggingHeader(r.Header)
|
tagSet, err := parseTaggingHeader(r.Header)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not parse tagging header", reqInfo, err)
|
h.logAndSendError(w, "could not parse tagging header", reqInfo, err)
|
||||||
|
@ -279,9 +289,26 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
objInfo := extendedObjInfo.ObjectInfo
|
objInfo := extendedObjInfo.ObjectInfo
|
||||||
|
|
||||||
|
s := &SendNotificationParams{
|
||||||
|
Event: EventObjectCreatedPut,
|
||||||
|
NotificationInfo: data.NotificationInfoFromObject(objInfo, h.cfg.MD5Enabled()),
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
ReqInfo: reqInfo,
|
||||||
|
}
|
||||||
|
if err = h.sendNotifications(ctx, s); err != nil {
|
||||||
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if needUpdateEACLTable {
|
||||||
|
if newEaclTable, err = h.getNewEAclTable(r, bktInfo, objInfo); err != nil {
|
||||||
|
h.logAndSendError(w, "could not get new eacl table", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if tagSet != nil {
|
if tagSet != nil {
|
||||||
tagPrm := &data.PutObjectTaggingParams{
|
tagPrm := &layer.PutObjectTaggingParams{
|
||||||
ObjectVersion: &data.ObjectVersion{
|
ObjectVersion: &layer.ObjectVersion{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ObjectName: objInfo.Name,
|
ObjectName: objInfo.Name,
|
||||||
VersionID: objInfo.VersionID(),
|
VersionID: objInfo.VersionID(),
|
||||||
|
@ -289,12 +316,25 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
TagSet: tagSet,
|
TagSet: tagSet,
|
||||||
NodeVersion: extendedObjInfo.NodeVersion,
|
NodeVersion: extendedObjInfo.NodeVersion,
|
||||||
}
|
}
|
||||||
if err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
|
if _, err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
|
||||||
h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
|
h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if newEaclTable != nil {
|
||||||
|
p := &layer.PutBucketACLParams{
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
EACL: newEaclTable,
|
||||||
|
SessionToken: sessionTokenEACL,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = h.obj.PutBucketACL(r.Context(), p); err != nil {
|
||||||
|
h.logAndSendError(w, "could not put bucket acl", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if settings.VersioningEnabled() {
|
if settings.VersioningEnabled() {
|
||||||
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
||||||
}
|
}
|
||||||
|
@ -426,10 +466,13 @@ func formEncryptionParamsBase(r *http.Request, isCopySource bool) (enc encryptio
|
||||||
|
|
||||||
func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
var (
|
var (
|
||||||
|
newEaclTable *eacl.Table
|
||||||
tagSet map[string]string
|
tagSet map[string]string
|
||||||
|
sessionTokenEACL *session.Container
|
||||||
ctx = r.Context()
|
ctx = r.Context()
|
||||||
reqInfo = middleware.GetReqInfo(ctx)
|
reqInfo = middleware.GetReqInfo(ctx)
|
||||||
metadata = make(map[string]string)
|
metadata = make(map[string]string)
|
||||||
|
cannedACLStatus = aclHeadersStatus(r)
|
||||||
)
|
)
|
||||||
|
|
||||||
policy, err := checkPostPolicy(r, reqInfo, metadata)
|
policy, err := checkPostPolicy(r, reqInfo, metadata)
|
||||||
|
@ -440,13 +483,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
if tagging := auth.MultipartFormValue(r, "tagging"); tagging != "" {
|
if tagging := auth.MultipartFormValue(r, "tagging"); tagging != "" {
|
||||||
buffer := bytes.NewBufferString(tagging)
|
buffer := bytes.NewBufferString(tagging)
|
||||||
tags := new(data.Tagging)
|
tagSet, err = h.readTagSet(buffer)
|
||||||
if err = h.cfg.NewXMLDecoder(buffer).Decode(tags); err != nil {
|
|
||||||
h.logAndSendError(w, "could not decode tag set", reqInfo,
|
|
||||||
fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
tagSet, err = h.readTagSet(tags)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not read tag set", reqInfo, err)
|
h.logAndSendError(w, "could not read tag set", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -465,52 +502,35 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if acl := auth.MultipartFormValue(r, "acl"); acl != "" && acl != basicACLPrivate {
|
apeEnabled := bktInfo.APEEnabled || settings.CannedACL != ""
|
||||||
|
if apeEnabled && cannedACLStatus == aclStatusYes {
|
||||||
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
reqInfo.ObjectName = auth.MultipartFormValue(r, "key")
|
needUpdateEACLTable := !(apeEnabled || cannedACLStatus == aclStatusNo)
|
||||||
|
if needUpdateEACLTable {
|
||||||
|
if sessionTokenEACL, err = getSessionTokenSetEACL(ctx); err != nil {
|
||||||
|
h.logAndSendError(w, "could not get eacl session token from a box", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var contentReader io.Reader
|
var contentReader io.Reader
|
||||||
var size uint64
|
var size uint64
|
||||||
var filename string
|
|
||||||
|
|
||||||
if content, ok := r.MultipartForm.Value["file"]; ok {
|
if content, ok := r.MultipartForm.Value["file"]; ok {
|
||||||
fullContent := strings.Join(content, "")
|
contentReader = bytes.NewBufferString(content[0])
|
||||||
contentReader = bytes.NewBufferString(fullContent)
|
size = uint64(len(content[0]))
|
||||||
size = uint64(len(fullContent))
|
|
||||||
|
|
||||||
if reqInfo.ObjectName == "" || strings.Contains(reqInfo.ObjectName, "${filename}") {
|
|
||||||
_, head, err := r.FormFile("file")
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not parse file field", reqInfo, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
filename = head.Filename
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
var head *multipart.FileHeader
|
file, head, err := r.FormFile("file")
|
||||||
contentReader, head, err = r.FormFile("file")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not parse file field", reqInfo, err)
|
h.logAndSendError(w, "could get uploading file", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
contentReader = file
|
||||||
size = uint64(head.Size)
|
size = uint64(head.Size)
|
||||||
filename = head.Filename
|
reqInfo.ObjectName = strings.ReplaceAll(reqInfo.ObjectName, "${filename}", head.Filename)
|
||||||
}
|
}
|
||||||
|
|
||||||
if reqInfo.ObjectName == "" {
|
|
||||||
reqInfo.ObjectName = filename
|
|
||||||
} else {
|
|
||||||
reqInfo.ObjectName = strings.ReplaceAll(reqInfo.ObjectName, "${filename}", filename)
|
|
||||||
}
|
|
||||||
|
|
||||||
if reqInfo.ObjectName == "" {
|
|
||||||
h.logAndSendError(w, "missing object name", reqInfo, errors.GetAPIError(errors.ErrInvalidArgument))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if !policy.CheckContentLength(size) {
|
if !policy.CheckContentLength(size) {
|
||||||
h.logAndSendError(w, "invalid content-length", reqInfo, errors.GetAPIError(errors.ErrInvalidArgument))
|
h.logAndSendError(w, "invalid content-length", reqInfo, errors.GetAPIError(errors.ErrInvalidArgument))
|
||||||
return
|
return
|
||||||
|
@ -531,9 +551,31 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
objInfo := extendedObjInfo.ObjectInfo
|
objInfo := extendedObjInfo.ObjectInfo
|
||||||
|
|
||||||
|
s := &SendNotificationParams{
|
||||||
|
Event: EventObjectCreatedPost,
|
||||||
|
NotificationInfo: data.NotificationInfoFromObject(objInfo, h.cfg.MD5Enabled()),
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
ReqInfo: reqInfo,
|
||||||
|
}
|
||||||
|
if err = h.sendNotifications(ctx, s); err != nil {
|
||||||
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if acl := auth.MultipartFormValue(r, "acl"); acl != "" {
|
||||||
|
r.Header.Set(api.AmzACL, acl)
|
||||||
|
r.Header.Set(api.AmzGrantFullControl, "")
|
||||||
|
r.Header.Set(api.AmzGrantWrite, "")
|
||||||
|
r.Header.Set(api.AmzGrantRead, "")
|
||||||
|
|
||||||
|
if newEaclTable, err = h.getNewEAclTable(r, bktInfo, objInfo); err != nil {
|
||||||
|
h.logAndSendError(w, "could not get new eacl table", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if tagSet != nil {
|
if tagSet != nil {
|
||||||
tagPrm := &data.PutObjectTaggingParams{
|
tagPrm := &layer.PutObjectTaggingParams{
|
||||||
ObjectVersion: &data.ObjectVersion{
|
ObjectVersion: &layer.ObjectVersion{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ObjectName: objInfo.Name,
|
ObjectName: objInfo.Name,
|
||||||
VersionID: objInfo.VersionID(),
|
VersionID: objInfo.VersionID(),
|
||||||
|
@ -541,12 +583,25 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
NodeVersion: extendedObjInfo.NodeVersion,
|
NodeVersion: extendedObjInfo.NodeVersion,
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
|
if _, err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
|
||||||
h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
|
h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if newEaclTable != nil {
|
||||||
|
p := &layer.PutBucketACLParams{
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
EACL: newEaclTable,
|
||||||
|
SessionToken: sessionTokenEACL,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = h.obj.PutBucketACL(ctx, p); err != nil {
|
||||||
|
h.logAndSendError(w, "could not put bucket acl", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if settings.VersioningEnabled() {
|
if settings.VersioningEnabled() {
|
||||||
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
||||||
}
|
}
|
||||||
|
@ -626,6 +681,10 @@ func checkPostPolicy(r *http.Request, reqInfo *middleware.ReqInfo, metadata map[
|
||||||
if key == "content-type" {
|
if key == "content-type" {
|
||||||
metadata[api.ContentType] = value
|
metadata[api.ContentType] = value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if key == "key" {
|
||||||
|
reqInfo.ObjectName = value
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, cond := range policy.Conditions {
|
for _, cond := range policy.Conditions {
|
||||||
|
@ -668,6 +727,56 @@ func aclHeadersStatus(r *http.Request) aclStatus {
|
||||||
return aclStatusNo
|
return aclStatusNo
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *handler) getNewEAclTable(r *http.Request, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo) (*eacl.Table, error) {
|
||||||
|
var newEaclTable *eacl.Table
|
||||||
|
key, err := h.bearerTokenIssuerKey(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("get bearer token issuer: %w", err)
|
||||||
|
}
|
||||||
|
objectACL, err := parseACLHeaders(r.Header, key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("could not parse object acl: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resInfo := &resourceInfo{
|
||||||
|
Bucket: objInfo.Bucket,
|
||||||
|
Object: objInfo.Name,
|
||||||
|
Version: objInfo.VersionID(),
|
||||||
|
}
|
||||||
|
|
||||||
|
bktPolicy, err := aclToPolicy(objectACL, resInfo)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("could not translate object acl to bucket policy: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
astChild, err := policyToAst(bktPolicy)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("could not translate policy to ast: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
bacl, err := h.obj.GetBucketACL(r.Context(), bktInfo)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("could not get bucket eacl: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
parentAst := tableToAst(bacl.EACL, objInfo.Bucket)
|
||||||
|
strCID := bacl.Info.CID.EncodeToString()
|
||||||
|
|
||||||
|
for _, resource := range parentAst.Resources {
|
||||||
|
if resource.Bucket == strCID {
|
||||||
|
resource.Bucket = objInfo.Bucket
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if resAst, updated := mergeAst(parentAst, astChild); updated {
|
||||||
|
if newEaclTable, err = astToTable(resAst); err != nil {
|
||||||
|
return nil, fmt.Errorf("could not translate ast to table: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return newEaclTable, nil
|
||||||
|
}
|
||||||
|
|
||||||
func parseTaggingHeader(header http.Header) (map[string]string, error) {
|
func parseTaggingHeader(header http.Header) (map[string]string, error) {
|
||||||
var tagSet map[string]string
|
var tagSet map[string]string
|
||||||
if tagging := header.Get(api.AmzTagging); len(tagging) > 0 {
|
if tagging := header.Get(api.AmzTagging); len(tagging) > 0 {
|
||||||
|
@ -680,7 +789,7 @@ func parseTaggingHeader(header http.Header) (map[string]string, error) {
|
||||||
}
|
}
|
||||||
tagSet = make(map[string]string, len(queries))
|
tagSet = make(map[string]string, len(queries))
|
||||||
for k, v := range queries {
|
for k, v := range queries {
|
||||||
tag := data.Tag{Key: k, Value: v[0]}
|
tag := Tag{Key: k, Value: v[0]}
|
||||||
if err = checkTag(tag); err != nil {
|
if err = checkTag(tag); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -707,7 +816,8 @@ func parseCannedACL(header http.Header) (string, error) {
|
||||||
return basicACLPrivate, nil
|
return basicACLPrivate, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if acl == basicACLPrivate || acl == basicACLPublic || acl == basicACLReadOnly {
|
if acl == basicACLPrivate || acl == basicACLPublic ||
|
||||||
|
acl == cannedACLAuthRead || acl == basicACLReadOnly {
|
||||||
return acl, nil
|
return acl, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -715,6 +825,11 @@ func parseCannedACL(header http.Header) (string, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if h.cfg.ACLEnabled() {
|
||||||
|
h.createBucketHandlerACL(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
h.createBucketHandlerPolicy(w, r)
|
h.createBucketHandlerPolicy(w, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -774,6 +889,7 @@ func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Reque
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
p.APEEnabled = true
|
||||||
bktInfo, err := h.obj.CreateBucket(ctx, p)
|
bktInfo, err := h.obj.CreateBucket(ctx, p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not create bucket", reqInfo, err)
|
h.logAndSendError(w, "could not create bucket", reqInfo, err)
|
||||||
|
@ -781,7 +897,7 @@ func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Reque
|
||||||
}
|
}
|
||||||
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
|
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
|
||||||
|
|
||||||
chains := bucketCannedACLToAPERules(cannedACL, reqInfo, bktInfo.CID)
|
chains := bucketCannedACLToAPERules(cannedACL, reqInfo, key, bktInfo.CID)
|
||||||
if err = h.ape.SaveACLChains(bktInfo.CID.EncodeToString(), chains); err != nil {
|
if err = h.ape.SaveACLChains(bktInfo.CID.EncodeToString(), chains); err != nil {
|
||||||
h.logAndSendError(w, "failed to add morph rule chain", reqInfo, err)
|
h.logAndSendError(w, "failed to add morph rule chain", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -800,10 +916,7 @@ func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Reque
|
||||||
sp.Settings.Versioning = data.VersioningEnabled
|
sp.Settings.Versioning = data.VersioningEnabled
|
||||||
}
|
}
|
||||||
|
|
||||||
err = retryer.MakeWithRetry(ctx, func() error {
|
if err = h.obj.PutBucketSettings(ctx, sp); err != nil {
|
||||||
return h.obj.PutBucketSettings(ctx, sp)
|
|
||||||
}, h.putBucketSettingsRetryer())
|
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "couldn't save bucket settings", reqInfo, err,
|
h.logAndSendError(w, "couldn't save bucket settings", reqInfo, err,
|
||||||
zap.String("container_id", bktInfo.CID.EncodeToString()))
|
zap.String("container_id", bktInfo.CID.EncodeToString()))
|
||||||
return
|
return
|
||||||
|
@ -815,25 +928,76 @@ func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Reque
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) putBucketSettingsRetryer() aws.RetryerV2 {
|
func (h *handler) createBucketHandlerACL(w http.ResponseWriter, r *http.Request) {
|
||||||
return retry.NewStandard(func(options *retry.StandardOptions) {
|
ctx := r.Context()
|
||||||
options.MaxAttempts = h.cfg.RetryMaxAttempts()
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
options.MaxBackoff = h.cfg.RetryMaxBackoff()
|
|
||||||
if h.cfg.RetryStrategy() == RetryStrategyExponential {
|
boxData, err := middleware.GetBoxData(ctx)
|
||||||
options.Backoff = retry.NewExponentialJitterBackoff(options.MaxBackoff)
|
if err != nil {
|
||||||
} else {
|
h.logAndSendError(w, "get access box from request", reqInfo, err)
|
||||||
options.Backoff = retry.BackoffDelayerFunc(func(int, error) (time.Duration, error) {
|
return
|
||||||
return options.MaxBackoff, nil
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
options.Retryables = []retry.IsErrorRetryable{retry.IsErrorRetryableFunc(func(err error) aws.Ternary {
|
key, p, err := h.parseCommonCreateBucketParams(reqInfo, boxData, r)
|
||||||
if stderrors.Is(err, tree.ErrNodeAccessDenied) {
|
if err != nil {
|
||||||
return aws.TrueTernary
|
h.logAndSendError(w, "parse create bucket params", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
aclPrm := &layer.PutBucketACLParams{SessionToken: boxData.Gate.SessionTokenForSetEACL()}
|
||||||
|
if aclPrm.SessionToken == nil {
|
||||||
|
h.logAndSendError(w, "couldn't find session token for setEACL", reqInfo, errors.GetAPIError(errors.ErrAccessDenied))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
bktACL, err := parseACLHeaders(r.Header, key)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not parse bucket acl", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
resInfo := &resourceInfo{Bucket: reqInfo.BucketName}
|
||||||
|
|
||||||
|
aclPrm.EACL, err = bucketACLToTable(bktACL, resInfo)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could translate bucket acl to eacl", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
bktInfo, err := h.obj.CreateBucket(ctx, p)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not create bucket", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
|
||||||
|
|
||||||
|
aclPrm.BktInfo = bktInfo
|
||||||
|
if err = h.obj.PutBucketACL(r.Context(), aclPrm); err != nil {
|
||||||
|
h.logAndSendError(w, "could not put bucket e/ACL", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
sp := &layer.PutSettingsParams{
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
Settings: &data.BucketSettings{
|
||||||
|
OwnerKey: key,
|
||||||
|
Versioning: data.VersioningUnversioned,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.ObjectLockEnabled {
|
||||||
|
sp.Settings.Versioning = data.VersioningEnabled
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = h.obj.PutBucketSettings(ctx, sp); err != nil {
|
||||||
|
h.logAndSendError(w, "couldn't save bucket settings", reqInfo, err,
|
||||||
|
zap.String("container_id", bktInfo.CID.EncodeToString()))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = middleware.WriteSuccessResponseHeadersOnly(w); err != nil {
|
||||||
|
h.logAndSendError(w, "write response", reqInfo, err)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
return aws.FalseTernary
|
|
||||||
})}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const s3ActionPrefix = "s3:"
|
const s3ActionPrefix = "s3:"
|
||||||
|
@ -874,22 +1038,49 @@ var (
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
func bucketCannedACLToAPERules(cannedACL string, reqInfo *middleware.ReqInfo, cnrID cid.ID) []*chain.Chain {
|
func bucketCannedACLToAPERules(cannedACL string, reqInfo *middleware.ReqInfo, key *keys.PublicKey, cnrID cid.ID) []*chain.Chain {
|
||||||
cnrIDStr := cnrID.EncodeToString()
|
cnrIDStr := cnrID.EncodeToString()
|
||||||
|
|
||||||
chains := []*chain.Chain{
|
chains := []*chain.Chain{
|
||||||
{
|
{
|
||||||
ID: getBucketCannedChainID(chain.S3, cnrID),
|
ID: getBucketCannedChainID(chain.S3, cnrID),
|
||||||
Rules: []chain.Rule{},
|
Rules: []chain.Rule{{
|
||||||
},
|
Status: chain.Allow,
|
||||||
|
Actions: chain.Actions{Names: []string{"s3:*"}},
|
||||||
|
Resources: chain.Resources{Names: []string{
|
||||||
|
fmt.Sprintf(s3.ResourceFormatS3Bucket, reqInfo.BucketName),
|
||||||
|
fmt.Sprintf(s3.ResourceFormatS3BucketObjects, reqInfo.BucketName),
|
||||||
|
}},
|
||||||
|
Condition: []chain.Condition{{
|
||||||
|
Op: chain.CondStringEquals,
|
||||||
|
Object: chain.ObjectRequest,
|
||||||
|
Key: s3.PropertyKeyOwner,
|
||||||
|
Value: key.Address(),
|
||||||
|
}},
|
||||||
|
}}},
|
||||||
{
|
{
|
||||||
ID: getBucketCannedChainID(chain.Ingress, cnrID),
|
ID: getBucketCannedChainID(chain.Ingress, cnrID),
|
||||||
Rules: []chain.Rule{},
|
Rules: []chain.Rule{{
|
||||||
|
Status: chain.Allow,
|
||||||
|
Actions: chain.Actions{Names: []string{"*"}},
|
||||||
|
Resources: chain.Resources{Names: []string{
|
||||||
|
fmt.Sprintf(native.ResourceFormatNamespaceContainer, reqInfo.Namespace, cnrIDStr),
|
||||||
|
fmt.Sprintf(native.ResourceFormatNamespaceContainerObjects, reqInfo.Namespace, cnrIDStr),
|
||||||
|
}},
|
||||||
|
Condition: []chain.Condition{{
|
||||||
|
Op: chain.CondStringEquals,
|
||||||
|
Object: chain.ObjectRequest,
|
||||||
|
Key: native.PropertyKeyActorPublicKey,
|
||||||
|
Value: hex.EncodeToString(key.Bytes()),
|
||||||
|
}},
|
||||||
|
}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
switch cannedACL {
|
switch cannedACL {
|
||||||
case basicACLPrivate:
|
case basicACLPrivate:
|
||||||
|
case cannedACLAuthRead:
|
||||||
|
fallthrough
|
||||||
case basicACLReadOnly:
|
case basicACLReadOnly:
|
||||||
chains[0].Rules = append(chains[0].Rules, chain.Rule{
|
chains[0].Rules = append(chains[0].Rules, chain.Rule{
|
||||||
Status: chain.Allow,
|
Status: chain.Allow,
|
||||||
|
@ -1015,7 +1206,7 @@ func (h *handler) parseLocationConstraint(r *http.Request) (*createBucketParams,
|
||||||
|
|
||||||
params := new(createBucketParams)
|
params := new(createBucketParams)
|
||||||
if err := h.cfg.NewXMLDecoder(r.Body).Decode(params); err != nil {
|
if err := h.cfg.NewXMLDecoder(r.Body).Decode(params); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error())
|
return nil, errors.GetAPIError(errors.ErrMalformedXML)
|
||||||
}
|
}
|
||||||
return params, nil
|
return params, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,7 +17,6 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
|
||||||
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
@ -123,92 +122,6 @@ func TestEmptyPostPolicy(t *testing.T) {
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// if content length is greater than this value
|
|
||||||
// data will be writen to file location.
|
|
||||||
const maxContentSizeForFormData = 10
|
|
||||||
|
|
||||||
func TestPostObject(t *testing.T) {
|
|
||||||
hc := prepareHandlerContext(t)
|
|
||||||
|
|
||||||
ns, bktName := "", "bucket"
|
|
||||||
createTestBucket(hc, bktName)
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
key string
|
|
||||||
filename string
|
|
||||||
content string
|
|
||||||
objName string
|
|
||||||
err bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
key: "user/user1/${filename}",
|
|
||||||
filename: "object",
|
|
||||||
content: "content",
|
|
||||||
objName: "user/user1/object",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
key: "user/user1/${filename}",
|
|
||||||
filename: "object",
|
|
||||||
content: "maxContentSizeForFormData",
|
|
||||||
objName: "user/user1/object",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
key: "user/user1/key-object",
|
|
||||||
filename: "object",
|
|
||||||
content: "",
|
|
||||||
objName: "user/user1/key-object",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
key: "user/user1/key-object",
|
|
||||||
filename: "object",
|
|
||||||
content: "maxContentSizeForFormData",
|
|
||||||
objName: "user/user1/key-object",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
key: "",
|
|
||||||
filename: "object",
|
|
||||||
content: "",
|
|
||||||
objName: "object",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
key: "",
|
|
||||||
filename: "object",
|
|
||||||
content: "maxContentSizeForFormData",
|
|
||||||
objName: "object",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// RFC 7578, Section 4.2 requires that if a filename is provided, the
|
|
||||||
// directory path information must not be used.
|
|
||||||
key: "",
|
|
||||||
filename: "dir/object",
|
|
||||||
content: "content",
|
|
||||||
objName: "object",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
key: "object",
|
|
||||||
filename: "",
|
|
||||||
content: "content",
|
|
||||||
objName: "object",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
key: "",
|
|
||||||
filename: "",
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.key+";"+tc.filename, func(t *testing.T) {
|
|
||||||
w := postObjectBase(hc, ns, bktName, tc.key, tc.filename, tc.content)
|
|
||||||
if tc.err {
|
|
||||||
assertS3Error(hc.t, w, s3errors.GetAPIError(s3errors.ErrInternalError))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
assertStatus(hc.t, w, http.StatusNoContent)
|
|
||||||
content, _ := getObject(hc, bktName, tc.objName)
|
|
||||||
require.Equal(t, tc.content, string(content))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPutObjectOverrideCopiesNumber(t *testing.T) {
|
func TestPutObjectOverrideCopiesNumber(t *testing.T) {
|
||||||
tc := prepareHandlerContext(t)
|
tc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
@ -438,20 +351,18 @@ func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName strin
|
||||||
req.Body = io.NopCloser(reqBody)
|
req.Body = io.NopCloser(reqBody)
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
|
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName})
|
||||||
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
||||||
req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{
|
req = req.WithContext(middleware.SetClientTime(req.Context(), signTime))
|
||||||
ClientTime: signTime,
|
req = req.WithContext(middleware.SetAuthHeaders(req.Context(), &middleware.AuthHeader{
|
||||||
AuthHeaders: &middleware.AuthHeader{
|
|
||||||
AccessKeyID: AWSAccessKeyID,
|
AccessKeyID: AWSAccessKeyID,
|
||||||
SignatureV4: "4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9",
|
SignatureV4: "4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9",
|
||||||
Region: "us-east-1",
|
Region: "us-east-1",
|
||||||
},
|
}))
|
||||||
AccessBox: &accessbox.Box{
|
req = req.WithContext(middleware.SetBoxData(req.Context(), &accessbox.Box{
|
||||||
Gate: &accessbox.GateData{
|
Gate: &accessbox.GateData{
|
||||||
SecretKey: AWSSecretAccessKey,
|
SecretKey: AWSSecretAccessKey,
|
||||||
},
|
},
|
||||||
},
|
|
||||||
}))
|
}))
|
||||||
|
|
||||||
return w, req, chunk
|
return w, req, chunk
|
||||||
|
@ -468,6 +379,21 @@ func TestCreateBucket(t *testing.T) {
|
||||||
createBucketAssertS3Error(hc, bktName, box2, s3errors.ErrBucketAlreadyExists)
|
createBucketAssertS3Error(hc, bktName, box2, s3errors.ErrBucketAlreadyExists)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCreateOldBucketPutVersioning(t *testing.T) {
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
hc.config.aclEnabled = true
|
||||||
|
bktName := "bkt-name"
|
||||||
|
|
||||||
|
info := createBucket(hc, bktName)
|
||||||
|
settings, err := hc.tree.GetSettingsNode(hc.Context(), info.BktInfo)
|
||||||
|
require.NoError(t, err)
|
||||||
|
settings.OwnerKey = nil
|
||||||
|
err = hc.tree.PutSettingsNode(hc.Context(), info.BktInfo, settings)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
putBucketVersioning(t, hc, bktName, true)
|
||||||
|
}
|
||||||
|
|
||||||
func TestCreateNamespacedBucket(t *testing.T) {
|
func TestCreateNamespacedBucket(t *testing.T) {
|
||||||
hc := prepareHandlerContext(t)
|
hc := prepareHandlerContext(t)
|
||||||
bktName := "bkt-name"
|
bktName := "bkt-name"
|
||||||
|
@ -475,7 +401,7 @@ func TestCreateNamespacedBucket(t *testing.T) {
|
||||||
|
|
||||||
box, _ := createAccessBox(t)
|
box, _ := createAccessBox(t)
|
||||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||||
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
ctx := middleware.SetBoxData(r.Context(), box)
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
reqInfo.Namespace = namespace
|
reqInfo.Namespace = namespace
|
||||||
r = r.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
r = r.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
||||||
|
@ -525,96 +451,14 @@ func getObjectAttribute(obj *object.Object, attrName string) string {
|
||||||
func TestPutObjectWithContentLanguage(t *testing.T) {
|
func TestPutObjectWithContentLanguage(t *testing.T) {
|
||||||
tc := prepareHandlerContext(t)
|
tc := prepareHandlerContext(t)
|
||||||
|
|
||||||
expectedContentLanguage := "en"
|
exceptedContentLanguage := "en"
|
||||||
bktName, objName := "bucket-1", "object-1"
|
bktName, objName := "bucket-1", "object-1"
|
||||||
createTestBucket(tc, bktName)
|
createTestBucket(tc, bktName)
|
||||||
|
|
||||||
w, r := prepareTestRequest(tc, bktName, objName, nil)
|
w, r := prepareTestRequest(tc, bktName, objName, nil)
|
||||||
r.Header.Set(api.ContentLanguage, expectedContentLanguage)
|
r.Header.Set(api.ContentLanguage, exceptedContentLanguage)
|
||||||
tc.Handler().PutObjectHandler(w, r)
|
tc.Handler().PutObjectHandler(w, r)
|
||||||
|
|
||||||
tc.Handler().HeadObjectHandler(w, r)
|
tc.Handler().HeadObjectHandler(w, r)
|
||||||
require.Equal(t, expectedContentLanguage, w.Header().Get(api.ContentLanguage))
|
require.Equal(t, exceptedContentLanguage, w.Header().Get(api.ContentLanguage))
|
||||||
}
|
|
||||||
|
|
||||||
func postObjectBase(hc *handlerContext, ns, bktName, key, filename, content string) *httptest.ResponseRecorder {
|
|
||||||
policy := "eyJleHBpcmF0aW9uIjogIjIwMjUtMTItMDFUMTI6MDA6MDAuMDAwWiIsImNvbmRpdGlvbnMiOiBbCiBbInN0YXJ0cy13aXRoIiwgIiR4LWFtei1jcmVkZW50aWFsIiwgIiJdLAogWyJzdGFydHMtd2l0aCIsICIkeC1hbXotZGF0ZSIsICIiXSwKIFsic3RhcnRzLXdpdGgiLCAiJGtleSIsICIiXQpdfQ=="
|
|
||||||
|
|
||||||
timeToSign := time.Now()
|
|
||||||
timeToSignStr := timeToSign.Format("20060102T150405Z")
|
|
||||||
region := "default"
|
|
||||||
service := "s3"
|
|
||||||
|
|
||||||
accessKeyID := "5jizSbYu8hX345aqCKDgRWKCJYHxnzxRS8e6SUYHZ8Fw0HiRkf3KbJAWBn5mRzmiyHQ3UHADGyzVXLusn1BrmAfLn"
|
|
||||||
secretKey := "abf066d77c6744cd956a123a0b9612df587f5c14d3350ecb01b363f182dd7279"
|
|
||||||
|
|
||||||
creds := getCredsStr(accessKeyID, timeToSignStr, region, service)
|
|
||||||
sign := auth.SignStr(secretKey, service, region, timeToSign, policy)
|
|
||||||
|
|
||||||
body, contentType, err := getMultipartFormBody(policy, creds, timeToSignStr, sign, key, filename, content)
|
|
||||||
require.NoError(hc.t, err)
|
|
||||||
|
|
||||||
w, r := prepareTestPostRequest(hc, bktName, body)
|
|
||||||
r.Header.Set(auth.ContentTypeHdr, contentType)
|
|
||||||
r.Header.Set("X-Frostfs-Namespace", ns)
|
|
||||||
|
|
||||||
err = r.ParseMultipartForm(50 * 1024 * 1024)
|
|
||||||
require.NoError(hc.t, err)
|
|
||||||
|
|
||||||
hc.Handler().PostObject(w, r)
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
func getCredsStr(accessKeyID, timeToSign, region, service string) string {
|
|
||||||
return accessKeyID + "/" + timeToSign + "/" + region + "/" + service + "/aws4_request"
|
|
||||||
}
|
|
||||||
|
|
||||||
func getMultipartFormBody(policy, creds, date, sign, key, filename, content string) (io.Reader, string, error) {
|
|
||||||
body := &bytes.Buffer{}
|
|
||||||
writer := multipart.NewWriter(body)
|
|
||||||
defer writer.Close()
|
|
||||||
|
|
||||||
if err := writer.WriteField("policy", policy); err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := writer.WriteField("key", key); err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
if err := writer.WriteField(strings.ToLower(auth.AmzCredential), creds); err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
if err := writer.WriteField(strings.ToLower(auth.AmzDate), date); err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
if err := writer.WriteField(strings.ToLower(auth.AmzSignature), sign); err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := writer.CreateFormFile("file", filename)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(content) < maxContentSizeForFormData {
|
|
||||||
if err = writer.WriteField("file", content); err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if _, err = file.Write([]byte(content)); err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return body, writer.FormDataContentType(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func prepareTestPostRequest(hc *handlerContext, bktName string, payload io.Reader) (*httptest.ResponseRecorder, *http.Request) {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := httptest.NewRequest(http.MethodPost, defaultURL+bktName, payload)
|
|
||||||
|
|
||||||
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: bktName}, "")
|
|
||||||
r = r.WithContext(middleware.SetReqInfo(hc.Context(), reqInfo))
|
|
||||||
|
|
||||||
return w, r
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -55,19 +55,6 @@ type Bucket struct {
|
||||||
CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
|
CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
|
||||||
}
|
}
|
||||||
|
|
||||||
// PolicyStatus contains status of bucket policy.
|
|
||||||
type PolicyStatus struct {
|
|
||||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PolicyStatus" json:"-"`
|
|
||||||
IsPublic PolicyStatusIsPublic `xml:"IsPublic"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type PolicyStatusIsPublic string
|
|
||||||
|
|
||||||
const (
|
|
||||||
PolicyStatusIsPublicFalse = "FALSE"
|
|
||||||
PolicyStatusIsPublicTrue = "TRUE"
|
|
||||||
)
|
|
||||||
|
|
||||||
// AccessControlPolicy contains ACL.
|
// AccessControlPolicy contains ACL.
|
||||||
type AccessControlPolicy struct {
|
type AccessControlPolicy struct {
|
||||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"`
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"`
|
||||||
|
@ -176,9 +163,6 @@ type ListObjectsVersionsResponse struct {
|
||||||
DeleteMarker []DeleteMarkerEntry `xml:"DeleteMarker"`
|
DeleteMarker []DeleteMarkerEntry `xml:"DeleteMarker"`
|
||||||
Version []ObjectVersionResponse `xml:"Version"`
|
Version []ObjectVersionResponse `xml:"Version"`
|
||||||
CommonPrefixes []CommonPrefix `xml:"CommonPrefixes"`
|
CommonPrefixes []CommonPrefix `xml:"CommonPrefixes"`
|
||||||
Prefix string `xml:"Prefix"`
|
|
||||||
Delimiter string `xml:"Delimiter,omitempty"`
|
|
||||||
MaxKeys int `xml:"MaxKeys"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// VersioningConfiguration contains VersioningConfiguration XML representation.
|
// VersioningConfiguration contains VersioningConfiguration XML representation.
|
||||||
|
@ -188,6 +172,12 @@ type VersioningConfiguration struct {
|
||||||
MfaDelete string `xml:"MfaDelete,omitempty"`
|
MfaDelete string `xml:"MfaDelete,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Tagging contains tag set.
|
||||||
|
type Tagging struct {
|
||||||
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"`
|
||||||
|
TagSet []Tag `xml:"TagSet>Tag"`
|
||||||
|
}
|
||||||
|
|
||||||
// PostResponse contains result of posting object.
|
// PostResponse contains result of posting object.
|
||||||
type PostResponse struct {
|
type PostResponse struct {
|
||||||
Bucket string `xml:"Bucket"`
|
Bucket string `xml:"Bucket"`
|
||||||
|
@ -195,13 +185,10 @@ type PostResponse struct {
|
||||||
ETag string `xml:"Etag"`
|
ETag string `xml:"Etag"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type PatchObjectResult struct {
|
// Tag is an AWS key-value tag.
|
||||||
Object PatchObject `xml:"Object"`
|
type Tag struct {
|
||||||
}
|
Key string
|
||||||
|
Value string
|
||||||
type PatchObject struct {
|
|
||||||
LastModified string `xml:"LastModified"`
|
|
||||||
ETag string `xml:"ETag"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// MarshalXML -- StringMap marshals into XML.
|
// MarshalXML -- StringMap marshals into XML.
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
@ -9,7 +10,10 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -24,7 +28,7 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
|
||||||
ctx := r.Context()
|
ctx := r.Context()
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
tagSet, err := h.readTagSet(reqInfo.Tagging)
|
tagSet, err := h.readTagSet(r.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not read tag set", reqInfo, err)
|
h.logAndSendError(w, "could not read tag set", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -36,20 +40,35 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
tagPrm := &data.PutObjectTaggingParams{
|
tagPrm := &layer.PutObjectTaggingParams{
|
||||||
ObjectVersion: &data.ObjectVersion{
|
ObjectVersion: &layer.ObjectVersion{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ObjectName: reqInfo.ObjectName,
|
ObjectName: reqInfo.ObjectName,
|
||||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||||
},
|
},
|
||||||
TagSet: tagSet,
|
TagSet: tagSet,
|
||||||
}
|
}
|
||||||
|
nodeVersion, err := h.obj.PutObjectTagging(ctx, tagPrm)
|
||||||
if err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not put object tagging", reqInfo, err)
|
h.logAndSendError(w, "could not put object tagging", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
s := &SendNotificationParams{
|
||||||
|
Event: EventObjectTaggingPut,
|
||||||
|
NotificationInfo: &data.NotificationInfo{
|
||||||
|
Name: nodeVersion.FilePath,
|
||||||
|
Size: nodeVersion.Size,
|
||||||
|
Version: nodeVersion.OID.EncodeToString(),
|
||||||
|
HashSum: nodeVersion.ETag,
|
||||||
|
},
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
ReqInfo: reqInfo,
|
||||||
|
}
|
||||||
|
if err = h.sendNotifications(ctx, s); err != nil {
|
||||||
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -68,8 +87,8 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
tagPrm := &data.GetObjectTaggingParams{
|
tagPrm := &layer.GetObjectTaggingParams{
|
||||||
ObjectVersion: &data.ObjectVersion{
|
ObjectVersion: &layer.ObjectVersion{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ObjectName: reqInfo.ObjectName,
|
ObjectName: reqInfo.ObjectName,
|
||||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||||
|
@ -100,24 +119,40 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
p := &data.ObjectVersion{
|
p := &layer.ObjectVersion{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ObjectName: reqInfo.ObjectName,
|
ObjectName: reqInfo.ObjectName,
|
||||||
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = h.obj.DeleteObjectTagging(ctx, p); err != nil {
|
nodeVersion, err := h.obj.DeleteObjectTagging(ctx, p)
|
||||||
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not delete object tagging", reqInfo, err)
|
h.logAndSendError(w, "could not delete object tagging", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
s := &SendNotificationParams{
|
||||||
|
Event: EventObjectTaggingDelete,
|
||||||
|
NotificationInfo: &data.NotificationInfo{
|
||||||
|
Name: nodeVersion.FilePath,
|
||||||
|
Size: nodeVersion.Size,
|
||||||
|
Version: nodeVersion.OID.EncodeToString(),
|
||||||
|
HashSum: nodeVersion.ETag,
|
||||||
|
},
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
ReqInfo: reqInfo,
|
||||||
|
}
|
||||||
|
if err = h.sendNotifications(ctx, s); err != nil {
|
||||||
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
reqInfo := middleware.GetReqInfo(r.Context())
|
reqInfo := middleware.GetReqInfo(r.Context())
|
||||||
|
|
||||||
tagSet, err := h.readTagSet(reqInfo.Tagging)
|
tagSet, err := h.readTagSet(r.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not read tag set", reqInfo, err)
|
h.logAndSendError(w, "could not read tag set", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -172,7 +207,12 @@ func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Requ
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) readTagSet(tagging *data.Tagging) (map[string]string, error) {
|
func (h *handler) readTagSet(reader io.Reader) (map[string]string, error) {
|
||||||
|
tagging := new(Tagging)
|
||||||
|
if err := h.cfg.NewXMLDecoder(reader).Decode(tagging); err != nil {
|
||||||
|
return nil, errors.GetAPIError(errors.ErrMalformedXML)
|
||||||
|
}
|
||||||
|
|
||||||
if err := checkTagSet(tagging.TagSet); err != nil {
|
if err := checkTagSet(tagging.TagSet); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -188,10 +228,10 @@ func (h *handler) readTagSet(tagging *data.Tagging) (map[string]string, error) {
|
||||||
return tagSet, nil
|
return tagSet, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func encodeTagging(tagSet map[string]string) *data.Tagging {
|
func encodeTagging(tagSet map[string]string) *Tagging {
|
||||||
tagging := &data.Tagging{}
|
tagging := &Tagging{}
|
||||||
for k, v := range tagSet {
|
for k, v := range tagSet {
|
||||||
tagging.TagSet = append(tagging.TagSet, data.Tag{Key: k, Value: v})
|
tagging.TagSet = append(tagging.TagSet, Tag{Key: k, Value: v})
|
||||||
}
|
}
|
||||||
sort.Slice(tagging.TagSet, func(i, j int) bool {
|
sort.Slice(tagging.TagSet, func(i, j int) bool {
|
||||||
return tagging.TagSet[i].Key < tagging.TagSet[j].Key
|
return tagging.TagSet[i].Key < tagging.TagSet[j].Key
|
||||||
|
@ -200,7 +240,7 @@ func encodeTagging(tagSet map[string]string) *data.Tagging {
|
||||||
return tagging
|
return tagging
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkTagSet(tagSet []data.Tag) error {
|
func checkTagSet(tagSet []Tag) error {
|
||||||
if len(tagSet) > maxTags {
|
if len(tagSet) > maxTags {
|
||||||
return errors.GetAPIError(errors.ErrInvalidTagsSizeExceed)
|
return errors.GetAPIError(errors.ErrInvalidTagsSizeExceed)
|
||||||
}
|
}
|
||||||
|
@ -214,7 +254,7 @@ func checkTagSet(tagSet []data.Tag) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkTag(tag data.Tag) error {
|
func checkTag(tag Tag) error {
|
||||||
if len(tag.Key) < 1 || len(tag.Key) > keyTagMaxLength {
|
if len(tag.Key) < 1 || len(tag.Key) > keyTagMaxLength {
|
||||||
return errors.GetAPIError(errors.ErrInvalidTagKey)
|
return errors.GetAPIError(errors.ErrInvalidTagKey)
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,9 +5,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -22,23 +20,23 @@ func TestTagsValidity(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
tag data.Tag
|
tag Tag
|
||||||
valid bool
|
valid bool
|
||||||
}{
|
}{
|
||||||
{tag: data.Tag{}, valid: false},
|
{tag: Tag{}, valid: false},
|
||||||
{tag: data.Tag{Key: "", Value: "1"}, valid: false},
|
{tag: Tag{Key: "", Value: "1"}, valid: false},
|
||||||
{tag: data.Tag{Key: "aws:key", Value: "val"}, valid: false},
|
{tag: Tag{Key: "aws:key", Value: "val"}, valid: false},
|
||||||
{tag: data.Tag{Key: "key~", Value: "val"}, valid: false},
|
{tag: Tag{Key: "key~", Value: "val"}, valid: false},
|
||||||
{tag: data.Tag{Key: "key\\", Value: "val"}, valid: false},
|
{tag: Tag{Key: "key\\", Value: "val"}, valid: false},
|
||||||
{tag: data.Tag{Key: "key?", Value: "val"}, valid: false},
|
{tag: Tag{Key: "key?", Value: "val"}, valid: false},
|
||||||
{tag: data.Tag{Key: sbKey.String() + "b", Value: "val"}, valid: false},
|
{tag: Tag{Key: sbKey.String() + "b", Value: "val"}, valid: false},
|
||||||
{tag: data.Tag{Key: "key", Value: sbValue.String() + "b"}, valid: false},
|
{tag: Tag{Key: "key", Value: sbValue.String() + "b"}, valid: false},
|
||||||
|
|
||||||
{tag: data.Tag{Key: sbKey.String(), Value: "val"}, valid: true},
|
{tag: Tag{Key: sbKey.String(), Value: "val"}, valid: true},
|
||||||
{tag: data.Tag{Key: "key", Value: sbValue.String()}, valid: true},
|
{tag: Tag{Key: "key", Value: sbValue.String()}, valid: true},
|
||||||
{tag: data.Tag{Key: "k e y", Value: "v a l"}, valid: true},
|
{tag: Tag{Key: "k e y", Value: "v a l"}, valid: true},
|
||||||
{tag: data.Tag{Key: "12345", Value: "1234"}, valid: true},
|
{tag: Tag{Key: "12345", Value: "1234"}, valid: true},
|
||||||
{tag: data.Tag{Key: allowedTagChars, Value: allowedTagChars}, valid: true},
|
{tag: Tag{Key: allowedTagChars, Value: allowedTagChars}, valid: true},
|
||||||
} {
|
} {
|
||||||
err := checkTag(tc.tag)
|
err := checkTag(tc.tag)
|
||||||
if tc.valid {
|
if tc.valid {
|
||||||
|
@ -57,13 +55,13 @@ func TestPutObjectTaggingCheckUniqueness(t *testing.T) {
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
body *data.Tagging
|
body *Tagging
|
||||||
error bool
|
error bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "Two tags with unique keys",
|
name: "Two tags with unique keys",
|
||||||
body: &data.Tagging{
|
body: &Tagging{
|
||||||
TagSet: []data.Tag{
|
TagSet: []Tag{
|
||||||
{
|
{
|
||||||
Key: "key-1",
|
Key: "key-1",
|
||||||
Value: "val-1",
|
Value: "val-1",
|
||||||
|
@ -78,8 +76,8 @@ func TestPutObjectTaggingCheckUniqueness(t *testing.T) {
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "Two tags with the same keys",
|
name: "Two tags with the same keys",
|
||||||
body: &data.Tagging{
|
body: &Tagging{
|
||||||
TagSet: []data.Tag{
|
TagSet: []Tag{
|
||||||
{
|
{
|
||||||
Key: "key-1",
|
Key: "key-1",
|
||||||
Value: "val-1",
|
Value: "val-1",
|
||||||
|
@ -95,7 +93,6 @@ func TestPutObjectTaggingCheckUniqueness(t *testing.T) {
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
w, r := prepareTestRequest(hc, bktName, objName, tc.body)
|
w, r := prepareTestRequest(hc, bktName, objName, tc.body)
|
||||||
middleware.GetReqInfo(r.Context()).Tagging = tc.body
|
|
||||||
hc.Handler().PutObjectTaggingHandler(w, r)
|
hc.Handler().PutObjectTaggingHandler(w, r)
|
||||||
if tc.error {
|
if tc.error {
|
||||||
assertS3Error(t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidTagKeyUniqueness))
|
assertS3Error(t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidTagKeyUniqueness))
|
||||||
|
|
|
@ -11,6 +11,10 @@ func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Requ
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
|
}
|
||||||
|
|
||||||
func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
@ -47,14 +51,10 @@ func (h *handler) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request)
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
|
}
|
||||||
|
|
||||||
func (h *handler) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
|
||||||
}
|
|
||||||
|
|
|
@ -16,6 +16,7 @@ import (
|
||||||
frosterrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
frosterrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.opentelemetry.io/otel/trace"
|
"go.opentelemetry.io/otel/trace"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -41,7 +42,6 @@ func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo
|
||||||
zap.String("bucket", reqInfo.BucketName),
|
zap.String("bucket", reqInfo.BucketName),
|
||||||
zap.String("object", reqInfo.ObjectName),
|
zap.String("object", reqInfo.ObjectName),
|
||||||
zap.String("description", logText),
|
zap.String("description", logText),
|
||||||
zap.String("user", reqInfo.User),
|
|
||||||
zap.Error(err)}
|
zap.Error(err)}
|
||||||
fields = append(fields, additional...)
|
fields = append(fields, additional...)
|
||||||
if traceID, err := trace.TraceIDFromHex(reqInfo.TraceID); err == nil && traceID.IsValid() {
|
if traceID, err := trace.TraceIDFromHex(reqInfo.TraceID); err == nil && traceID.IsValid() {
|
||||||
|
@ -141,3 +141,16 @@ func parseRange(s string) (*layer.RangeParams, error) {
|
||||||
End: values[1],
|
End: values[1],
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getSessionTokenSetEACL(ctx context.Context) (*session.Container, error) {
|
||||||
|
boxData, err := middleware.GetBoxData(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sessionToken := boxData.Gate.SessionTokenForSetEACL()
|
||||||
|
if sessionToken == nil {
|
||||||
|
return nil, s3errors.GetAPIError(s3errors.ErrAccessDenied)
|
||||||
|
}
|
||||||
|
|
||||||
|
return sessionToken, nil
|
||||||
|
}
|
||||||
|
|
|
@ -62,7 +62,6 @@ const (
|
||||||
AmzMaxParts = "X-Amz-Max-Parts"
|
AmzMaxParts = "X-Amz-Max-Parts"
|
||||||
AmzPartNumberMarker = "X-Amz-Part-Number-Marker"
|
AmzPartNumberMarker = "X-Amz-Part-Number-Marker"
|
||||||
AmzStorageClass = "X-Amz-Storage-Class"
|
AmzStorageClass = "X-Amz-Storage-Class"
|
||||||
AmzForceBucketDelete = "X-Amz-Force-Delete-Bucket"
|
|
||||||
|
|
||||||
AmzServerSideEncryptionCustomerAlgorithm = "x-amz-server-side-encryption-customer-algorithm"
|
AmzServerSideEncryptionCustomerAlgorithm = "x-amz-server-side-encryption-customer-algorithm"
|
||||||
AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"
|
AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"
|
||||||
|
|
|
@ -233,7 +233,7 @@ func (c *Cache) PutSettings(owner user.ID, bktInfo *data.BucketInfo, settings *d
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Cache) GetCORS(owner user.ID, bkt *data.BucketInfo) *data.CORSConfiguration {
|
func (c *Cache) GetCORS(owner user.ID, bkt *data.BucketInfo) *data.CORSConfiguration {
|
||||||
key := bkt.CORSObjectName()
|
key := bkt.Name + bkt.CORSObjectName()
|
||||||
|
|
||||||
if !c.accessCache.Get(owner, key) {
|
if !c.accessCache.Get(owner, key) {
|
||||||
return nil
|
return nil
|
||||||
|
@ -243,7 +243,7 @@ func (c *Cache) GetCORS(owner user.ID, bkt *data.BucketInfo) *data.CORSConfigura
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConfiguration) {
|
func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConfiguration) {
|
||||||
key := bkt.CORSObjectName()
|
key := bkt.Name + bkt.CORSObjectName()
|
||||||
|
|
||||||
if err := c.systemCache.PutCORS(key, cors); err != nil {
|
if err := c.systemCache.PutCORS(key, cors); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err))
|
c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err))
|
||||||
|
@ -255,31 +255,26 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Cache) DeleteCORS(bktInfo *data.BucketInfo) {
|
func (c *Cache) DeleteCORS(bktInfo *data.BucketInfo) {
|
||||||
c.systemCache.Delete(bktInfo.CORSObjectName())
|
c.systemCache.Delete(bktInfo.Name + bktInfo.CORSObjectName())
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Cache) GetLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo) *data.LifecycleConfiguration {
|
func (c *Cache) GetNotificationConfiguration(owner user.ID, bktInfo *data.BucketInfo) *data.NotificationConfiguration {
|
||||||
key := bkt.LifecycleConfigurationObjectName()
|
key := bktInfo.Name + bktInfo.NotificationConfigurationObjectName()
|
||||||
|
|
||||||
if !c.accessCache.Get(owner, key) {
|
if !c.accessCache.Get(owner, key) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.systemCache.GetLifecycleConfiguration(key)
|
return c.systemCache.GetNotificationConfiguration(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Cache) PutLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo, cfg *data.LifecycleConfiguration) {
|
func (c *Cache) PutNotificationConfiguration(owner user.ID, bktInfo *data.BucketInfo, configuration *data.NotificationConfiguration) {
|
||||||
key := bkt.LifecycleConfigurationObjectName()
|
key := bktInfo.Name + bktInfo.NotificationConfigurationObjectName()
|
||||||
|
if err := c.systemCache.PutNotificationConfiguration(key, configuration); err != nil {
|
||||||
if err := c.systemCache.PutLifecycleConfiguration(key, cfg); err != nil {
|
c.logger.Warn(logs.CouldntCacheNotificationConfiguration, zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||||
c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key); err != nil {
|
if err := c.accessCache.Put(owner, key); err != nil {
|
||||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Cache) DeleteLifecycleConfiguration(bktInfo *data.BucketInfo) {
|
|
||||||
c.systemCache.Delete(bktInfo.LifecycleConfigurationObjectName())
|
|
||||||
}
|
|
||||||
|
|
|
@ -9,7 +9,7 @@ import (
|
||||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *Layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) {
|
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) {
|
||||||
var err error
|
var err error
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
|
|
||||||
|
|
|
@ -12,15 +12,27 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||||
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type (
|
||||||
|
// BucketACL extends BucketInfo by eacl.Table.
|
||||||
|
BucketACL struct {
|
||||||
|
Info *data.BucketInfo
|
||||||
|
EACL *eacl.Table
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
attributeLocationConstraint = ".s3-location-constraint"
|
attributeLocationConstraint = ".s3-location-constraint"
|
||||||
AttributeLockEnabled = "LockEnabled"
|
AttributeLockEnabled = "LockEnabled"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *Layer) containerInfo(ctx context.Context, prm PrmContainer) (*data.BucketInfo, error) {
|
func (n *layer) containerInfo(ctx context.Context, prm PrmContainer) (*data.BucketInfo, error) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
res *container.Container
|
res *container.Container
|
||||||
|
@ -52,6 +64,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm PrmContainer) (*data.Buck
|
||||||
info.Created = container.CreatedAt(cnr)
|
info.Created = container.CreatedAt(cnr)
|
||||||
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
|
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
|
||||||
info.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(cnr)
|
info.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(cnr)
|
||||||
|
info.APEEnabled = cnr.BasicACL().Bits() == 0
|
||||||
|
|
||||||
attrLockEnabled := cnr.Attribute(AttributeLockEnabled)
|
attrLockEnabled := cnr.Attribute(AttributeLockEnabled)
|
||||||
if len(attrLockEnabled) > 0 {
|
if len(attrLockEnabled) > 0 {
|
||||||
|
@ -64,7 +77,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm PrmContainer) (*data.Buck
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
zone := n.features.FormContainerZone(reqInfo.Namespace)
|
zone, _ := n.features.FormContainerZone(reqInfo.Namespace)
|
||||||
if zone != info.Zone {
|
if zone != info.Zone {
|
||||||
return nil, fmt.Errorf("ns '%s' and zone '%s' are mismatched for container '%s'", zone, info.Zone, prm.ContainerID)
|
return nil, fmt.Errorf("ns '%s' and zone '%s' are mismatched for container '%s'", zone, info.Zone, prm.ContainerID)
|
||||||
}
|
}
|
||||||
|
@ -74,7 +87,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm PrmContainer) (*data.Buck
|
||||||
return info, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||||
stoken := n.SessionTokenForRead(ctx)
|
stoken := n.SessionTokenForRead(ctx)
|
||||||
|
|
||||||
prm := PrmUserContainers{
|
prm := PrmUserContainers{
|
||||||
|
@ -106,12 +119,12 @@ func (n *Layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||||
return list, nil
|
return list, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) {
|
func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) {
|
||||||
if p.LocationConstraint == "" {
|
if p.LocationConstraint == "" {
|
||||||
p.LocationConstraint = api.DefaultLocationConstraint // s3tests_boto3.functional.test_s3:test_bucket_get_location
|
p.LocationConstraint = api.DefaultLocationConstraint // s3tests_boto3.functional.test_s3:test_bucket_get_location
|
||||||
}
|
}
|
||||||
|
|
||||||
zone := n.features.FormContainerZone(p.Namespace)
|
zone, _ := n.features.FormContainerZone(p.Namespace)
|
||||||
|
|
||||||
bktInfo := &data.BucketInfo{
|
bktInfo := &data.BucketInfo{
|
||||||
Name: p.Name,
|
Name: p.Name,
|
||||||
|
@ -120,6 +133,7 @@ func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
||||||
Created: TimeNow(ctx),
|
Created: TimeNow(ctx),
|
||||||
LocationConstraint: p.LocationConstraint,
|
LocationConstraint: p.LocationConstraint,
|
||||||
ObjectLockEnabled: p.ObjectLockEnabled,
|
ObjectLockEnabled: p.ObjectLockEnabled,
|
||||||
|
APEEnabled: p.APEEnabled,
|
||||||
}
|
}
|
||||||
|
|
||||||
attributes := [][2]string{
|
attributes := [][2]string{
|
||||||
|
@ -132,6 +146,11 @@ func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
basicACL := acl.PublicRWExtended
|
||||||
|
if p.APEEnabled {
|
||||||
|
basicACL = 0
|
||||||
|
}
|
||||||
|
|
||||||
res, err := n.frostFS.CreateContainer(ctx, PrmContainerCreate{
|
res, err := n.frostFS.CreateContainer(ctx, PrmContainerCreate{
|
||||||
Creator: bktInfo.Owner,
|
Creator: bktInfo.Owner,
|
||||||
Policy: p.Policy,
|
Policy: p.Policy,
|
||||||
|
@ -140,7 +159,7 @@ func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
||||||
SessionToken: p.SessionContainerCreation,
|
SessionToken: p.SessionContainerCreation,
|
||||||
CreationTime: bktInfo.Created,
|
CreationTime: bktInfo.Created,
|
||||||
AdditionalAttributes: attributes,
|
AdditionalAttributes: attributes,
|
||||||
BasicACL: 0, // means APE
|
BasicACL: basicACL,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create container: %w", err)
|
return nil, fmt.Errorf("create container: %w", err)
|
||||||
|
@ -153,3 +172,17 @@ func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
||||||
|
|
||||||
return bktInfo, nil
|
return bktInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (n *layer) setContainerEACLTable(ctx context.Context, idCnr cid.ID, table *eacl.Table, sessionToken *session.Container) error {
|
||||||
|
table.SetCID(idCnr)
|
||||||
|
|
||||||
|
return n.frostFS.SetContainerEACL(ctx, *table, sessionToken)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *layer) GetContainerEACL(ctx context.Context, cnrID cid.ID) (*eacl.Table, error) {
|
||||||
|
prm := PrmContainerEACL{
|
||||||
|
ContainerID: cnrID,
|
||||||
|
SessionToken: n.SessionTokenForRead(ctx),
|
||||||
|
}
|
||||||
|
return n.frostFS.ContainerEACL(ctx, prm)
|
||||||
|
}
|
||||||
|
|
|
@ -10,8 +10,6 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -19,7 +17,7 @@ const wildcard = "*"
|
||||||
|
|
||||||
var supportedMethods = map[string]struct{}{"GET": {}, "HEAD": {}, "POST": {}, "PUT": {}, "DELETE": {}}
|
var supportedMethods = map[string]struct{}{"GET": {}, "HEAD": {}, "POST": {}, "PUT": {}, "DELETE": {}}
|
||||||
|
|
||||||
func (n *Layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
||||||
var (
|
var (
|
||||||
buf bytes.Buffer
|
buf bytes.Buffer
|
||||||
tee = io.TeeReader(p.Reader, &buf)
|
tee = io.TeeReader(p.Reader, &buf)
|
||||||
|
@ -39,36 +37,29 @@ func (n *Layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
prm := PrmObjectCreate{
|
prm := PrmObjectCreate{
|
||||||
|
Container: p.BktInfo.CID,
|
||||||
Payload: &buf,
|
Payload: &buf,
|
||||||
Filepath: p.BktInfo.CORSObjectName(),
|
Filepath: p.BktInfo.CORSObjectName(),
|
||||||
CreationTime: TimeNow(ctx),
|
CreationTime: TimeNow(ctx),
|
||||||
|
CopiesNumber: p.CopiesNumbers,
|
||||||
}
|
}
|
||||||
|
|
||||||
var corsBkt *data.BucketInfo
|
_, objID, _, _, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
|
||||||
if n.corsCnrInfo == nil {
|
|
||||||
corsBkt = p.BktInfo
|
|
||||||
prm.CopiesNumber = p.CopiesNumbers
|
|
||||||
} else {
|
|
||||||
corsBkt = n.corsCnrInfo
|
|
||||||
prm.PrmAuth.PrivateKey = &n.gateKey.PrivateKey
|
|
||||||
}
|
|
||||||
|
|
||||||
prm.Container = corsBkt.CID
|
|
||||||
|
|
||||||
createdObj, err := n.objectPutAndHash(ctx, prm, corsBkt)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("put cors object: %w", err)
|
return fmt.Errorf("put system object: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
objsToDelete, err := n.treeService.PutBucketCORS(ctx, p.BktInfo, newAddress(corsBkt.CID, createdObj.ID))
|
objIDToDelete, err := n.treeService.PutBucketCORS(ctx, p.BktInfo, objID)
|
||||||
objToDeleteNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
|
objIDToDeleteNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
|
||||||
if err != nil && !objToDeleteNotFound {
|
if err != nil && !objIDToDeleteNotFound {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if !objToDeleteNotFound {
|
if !objIDToDeleteNotFound {
|
||||||
for _, addr := range objsToDelete {
|
if err = n.objectDelete(ctx, p.BktInfo, objIDToDelete); err != nil {
|
||||||
n.deleteCORSObject(ctx, p.BktInfo, addr)
|
n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err),
|
||||||
|
zap.String("cnrID", p.BktInfo.CID.EncodeToString()),
|
||||||
|
zap.String("objID", objIDToDelete.EncodeToString()))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -77,41 +68,27 @@ func (n *Layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// deleteCORSObject removes object and logs in case of error.
|
func (n *layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*data.CORSConfiguration, error) {
|
||||||
func (n *Layer) deleteCORSObject(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) {
|
|
||||||
var prmAuth PrmAuth
|
|
||||||
corsBkt := bktInfo
|
|
||||||
if !addr.Container().Equals(bktInfo.CID) && !addr.Container().Equals(cid.ID{}) {
|
|
||||||
corsBkt = &data.BucketInfo{CID: addr.Container()}
|
|
||||||
prmAuth.PrivateKey = &n.gateKey.PrivateKey
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := n.objectDeleteWithAuth(ctx, corsBkt, addr.Object(), prmAuth); err != nil {
|
|
||||||
n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err),
|
|
||||||
zap.String("cnrID", corsBkt.CID.EncodeToString()),
|
|
||||||
zap.String("objID", addr.Object().EncodeToString()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*data.CORSConfiguration, error) {
|
|
||||||
cors, err := n.getCORS(ctx, bktInfo)
|
cors, err := n.getCORS(ctx, bktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errorsStd.Is(err, ErrNodeNotFound) {
|
||||||
|
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchCORSConfiguration), err.Error())
|
||||||
|
}
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return cors, nil
|
return cors, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) error {
|
func (n *layer) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||||
objs, err := n.treeService.DeleteBucketCORS(ctx, bktInfo)
|
objID, err := n.treeService.DeleteBucketCORS(ctx, bktInfo)
|
||||||
objNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
|
objIDNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
|
||||||
if err != nil && !objNotFound {
|
if err != nil && !objIDNotFound {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if !objIDNotFound {
|
||||||
if !objNotFound {
|
if err = n.objectDelete(ctx, bktInfo, objID); err != nil {
|
||||||
for _, addr := range objs {
|
return err
|
||||||
n.deleteCORSObject(ctx, bktInfo, addr)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -11,6 +11,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
@ -63,6 +64,15 @@ type PrmUserContainers struct {
|
||||||
SessionToken *session.Container
|
SessionToken *session.Container
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PrmContainerEACL groups parameters of FrostFS.ContainerEACL operation.
|
||||||
|
type PrmContainerEACL struct {
|
||||||
|
// Container identifier.
|
||||||
|
ContainerID cid.ID
|
||||||
|
|
||||||
|
// Token of the container's creation session. Nil means session absence.
|
||||||
|
SessionToken *session.Container
|
||||||
|
}
|
||||||
|
|
||||||
// ContainerCreateResult is a result parameter of FrostFS.CreateContainer operation.
|
// ContainerCreateResult is a result parameter of FrostFS.CreateContainer operation.
|
||||||
type ContainerCreateResult struct {
|
type ContainerCreateResult struct {
|
||||||
ContainerID cid.ID
|
ContainerID cid.ID
|
||||||
|
@ -78,8 +88,8 @@ type PrmAuth struct {
|
||||||
PrivateKey *ecdsa.PrivateKey
|
PrivateKey *ecdsa.PrivateKey
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrmObjectHead groups parameters of FrostFS.HeadObject operation.
|
// PrmObjectRead groups parameters of FrostFS.ReadObject operation.
|
||||||
type PrmObjectHead struct {
|
type PrmObjectRead struct {
|
||||||
// Authentication parameters.
|
// Authentication parameters.
|
||||||
PrmAuth
|
PrmAuth
|
||||||
|
|
||||||
|
@ -88,39 +98,21 @@ type PrmObjectHead struct {
|
||||||
|
|
||||||
// ID of the object for which to read the header.
|
// ID of the object for which to read the header.
|
||||||
Object oid.ID
|
Object oid.ID
|
||||||
}
|
|
||||||
|
|
||||||
// PrmObjectGet groups parameters of FrostFS.GetObject operation.
|
// Flag to read object header.
|
||||||
type PrmObjectGet struct {
|
WithHeader bool
|
||||||
// Authentication parameters.
|
|
||||||
PrmAuth
|
|
||||||
|
|
||||||
// Container to read the object header from.
|
// Flag to read object payload. False overlaps payload range.
|
||||||
Container cid.ID
|
WithPayload bool
|
||||||
|
|
||||||
// ID of the object for which to read the header.
|
|
||||||
Object oid.ID
|
|
||||||
}
|
|
||||||
|
|
||||||
// PrmObjectRange groups parameters of FrostFS.RangeObject operation.
|
|
||||||
type PrmObjectRange struct {
|
|
||||||
// Authentication parameters.
|
|
||||||
PrmAuth
|
|
||||||
|
|
||||||
// Container to read the object header from.
|
|
||||||
Container cid.ID
|
|
||||||
|
|
||||||
// ID of the object for which to read the header.
|
|
||||||
Object oid.ID
|
|
||||||
|
|
||||||
// Offset-length range of the object payload to be read.
|
// Offset-length range of the object payload to be read.
|
||||||
PayloadRange [2]uint64
|
PayloadRange [2]uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
// Object represents full read FrostFS object.
|
// ObjectPart represents partially read FrostFS object.
|
||||||
type Object struct {
|
type ObjectPart struct {
|
||||||
// Object header (doesn't contain payload).
|
// Object header with optional in-memory payload part.
|
||||||
Header object.Object
|
Head *object.Object
|
||||||
|
|
||||||
// Object payload part encapsulated in io.Reader primitive.
|
// Object payload part encapsulated in io.Reader primitive.
|
||||||
// Returns ErrAccessDenied on read access violation.
|
// Returns ErrAccessDenied on read access violation.
|
||||||
|
@ -166,12 +158,6 @@ type PrmObjectCreate struct {
|
||||||
BufferMaxSize uint64
|
BufferMaxSize uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateObjectResult is a result parameter of FrostFS.CreateObject operation.
|
|
||||||
type CreateObjectResult struct {
|
|
||||||
ObjectID oid.ID
|
|
||||||
CreationEpoch uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
|
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
|
||||||
type PrmObjectDelete struct {
|
type PrmObjectDelete struct {
|
||||||
// Authentication parameters.
|
// Authentication parameters.
|
||||||
|
@ -200,27 +186,6 @@ type PrmObjectSearch struct {
|
||||||
FilePrefix string
|
FilePrefix string
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrmObjectPatch groups parameters of FrostFS.PatchObject operation.
|
|
||||||
type PrmObjectPatch struct {
|
|
||||||
// Authentication parameters.
|
|
||||||
PrmAuth
|
|
||||||
|
|
||||||
// Container of the patched object.
|
|
||||||
Container cid.ID
|
|
||||||
|
|
||||||
// Identifier of the patched object.
|
|
||||||
Object oid.ID
|
|
||||||
|
|
||||||
// Object patch payload encapsulated in io.Reader primitive.
|
|
||||||
Payload io.Reader
|
|
||||||
|
|
||||||
// Object range to patch.
|
|
||||||
Offset, Length uint64
|
|
||||||
|
|
||||||
// Size of original object payload.
|
|
||||||
ObjectSize uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
// ErrAccessDenied is returned from FrostFS in case of access violation.
|
// ErrAccessDenied is returned from FrostFS in case of access violation.
|
||||||
ErrAccessDenied = errors.New("access denied")
|
ErrAccessDenied = errors.New("access denied")
|
||||||
|
@ -251,6 +216,18 @@ type FrostFS interface {
|
||||||
// prevented the containers from being listed.
|
// prevented the containers from being listed.
|
||||||
UserContainers(context.Context, PrmUserContainers) ([]cid.ID, error)
|
UserContainers(context.Context, PrmUserContainers) ([]cid.ID, error)
|
||||||
|
|
||||||
|
// SetContainerEACL saves the eACL table of the container in FrostFS. The
|
||||||
|
// extended ACL is modified within session if session token is not nil.
|
||||||
|
//
|
||||||
|
// It returns any error encountered which prevented the eACL from being saved.
|
||||||
|
SetContainerEACL(context.Context, eacl.Table, *session.Container) error
|
||||||
|
|
||||||
|
// ContainerEACL reads the container eACL from FrostFS by the container ID.
|
||||||
|
//
|
||||||
|
// It returns exactly one non-nil value. It returns any error encountered which
|
||||||
|
// prevented the eACL from being read.
|
||||||
|
ContainerEACL(context.Context, PrmContainerEACL) (*eacl.Table, error)
|
||||||
|
|
||||||
// DeleteContainer marks the container to be removed from FrostFS by ID.
|
// DeleteContainer marks the container to be removed from FrostFS by ID.
|
||||||
// Request is sent within session if the session token is specified.
|
// Request is sent within session if the session token is specified.
|
||||||
// Successful return does not guarantee actual removal.
|
// Successful return does not guarantee actual removal.
|
||||||
|
@ -258,15 +235,13 @@ type FrostFS interface {
|
||||||
// It returns any error encountered which prevented the removal request from being sent.
|
// It returns any error encountered which prevented the removal request from being sent.
|
||||||
DeleteContainer(context.Context, cid.ID, *session.Container) error
|
DeleteContainer(context.Context, cid.ID, *session.Container) error
|
||||||
|
|
||||||
// HeadObject reads an info of the object from the FrostFS container by identifier.
|
// ReadObject reads a part of the object from the FrostFS container by identifier.
|
||||||
|
// Exact part is returned according to the parameters:
|
||||||
|
// * with header only: empty payload (both in-mem and reader parts are nil);
|
||||||
|
// * with payload only: header is nil (zero range means full payload);
|
||||||
|
// * with header and payload: full in-mem object, payload reader is nil.
|
||||||
//
|
//
|
||||||
// It returns ErrAccessDenied on read access violation.
|
// WithHeader or WithPayload is true. Range length is positive if offset is positive.
|
||||||
//
|
|
||||||
// It returns exactly one non-nil value. It returns any error encountered which
|
|
||||||
// prevented the object header from being read.
|
|
||||||
HeadObject(ctx context.Context, prm PrmObjectHead) (*object.Object, error)
|
|
||||||
|
|
||||||
// GetObject reads an object from the FrostFS container by identifier.
|
|
||||||
//
|
//
|
||||||
// Payload reader should be closed if it is no longer needed.
|
// Payload reader should be closed if it is no longer needed.
|
||||||
//
|
//
|
||||||
|
@ -274,29 +249,19 @@ type FrostFS interface {
|
||||||
//
|
//
|
||||||
// It returns exactly one non-nil value. It returns any error encountered which
|
// It returns exactly one non-nil value. It returns any error encountered which
|
||||||
// prevented the object header from being read.
|
// prevented the object header from being read.
|
||||||
GetObject(ctx context.Context, prm PrmObjectGet) (*Object, error)
|
ReadObject(context.Context, PrmObjectRead) (*ObjectPart, error)
|
||||||
|
|
||||||
// RangeObject reads a part of object from the FrostFS container by identifier.
|
|
||||||
//
|
|
||||||
// Payload reader should be closed if it is no longer needed.
|
|
||||||
//
|
|
||||||
// It returns ErrAccessDenied on read access violation.
|
|
||||||
//
|
|
||||||
// It returns exactly one non-nil value. It returns any error encountered which
|
|
||||||
// prevented the object header from being read.
|
|
||||||
RangeObject(ctx context.Context, prm PrmObjectRange) (io.ReadCloser, error)
|
|
||||||
|
|
||||||
// CreateObject creates and saves a parameterized object in the FrostFS container.
|
// CreateObject creates and saves a parameterized object in the FrostFS container.
|
||||||
// It sets 'Timestamp' attribute to the current time.
|
// It sets 'Timestamp' attribute to the current time.
|
||||||
// It returns the ID and creation epoch of the saved object.
|
// It returns the ID of the saved object.
|
||||||
//
|
//
|
||||||
// Creation time should be written into the object (UTC).
|
// Creation time should be written into the object (UTC).
|
||||||
//
|
//
|
||||||
// It returns ErrAccessDenied on write access violation.
|
// It returns ErrAccessDenied on write access violation.
|
||||||
//
|
//
|
||||||
// It returns exactly one non-nil value. It returns any error encountered which
|
// It returns exactly one non-zero value. It returns any error encountered which
|
||||||
// prevented the object from being created.
|
// prevented the container from being created.
|
||||||
CreateObject(context.Context, PrmObjectCreate) (*CreateObjectResult, error)
|
CreateObject(context.Context, PrmObjectCreate) (oid.ID, error)
|
||||||
|
|
||||||
// DeleteObject marks the object to be removed from the FrostFS container by identifier.
|
// DeleteObject marks the object to be removed from the FrostFS container by identifier.
|
||||||
// Successful return does not guarantee actual removal.
|
// Successful return does not guarantee actual removal.
|
||||||
|
@ -315,15 +280,6 @@ type FrostFS interface {
|
||||||
// prevented the objects from being selected.
|
// prevented the objects from being selected.
|
||||||
SearchObjects(context.Context, PrmObjectSearch) ([]oid.ID, error)
|
SearchObjects(context.Context, PrmObjectSearch) ([]oid.ID, error)
|
||||||
|
|
||||||
// PatchObject performs object patch in the FrostFS container.
|
|
||||||
// It returns the ID of the patched object.
|
|
||||||
//
|
|
||||||
// It returns ErrAccessDenied on selection access violation.
|
|
||||||
//
|
|
||||||
// It returns exactly one non-nil value. It returns any error encountered which
|
|
||||||
// prevented the objects from being patched.
|
|
||||||
PatchObject(context.Context, PrmObjectPatch) (oid.ID, error)
|
|
||||||
|
|
||||||
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided now and future time.
|
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided now and future time.
|
||||||
// Note:
|
// Note:
|
||||||
// * future time must be after the now
|
// * future time must be after the now
|
||||||
|
@ -331,7 +287,4 @@ type FrostFS interface {
|
||||||
//
|
//
|
||||||
// It returns any error encountered which prevented computing epochs.
|
// It returns any error encountered which prevented computing epochs.
|
||||||
TimeToEpoch(ctx context.Context, now time.Time, future time.Time) (uint64, uint64, error)
|
TimeToEpoch(ctx context.Context, now time.Time, future time.Time) (uint64, uint64, error)
|
||||||
|
|
||||||
// NetworkInfo returns parameters of FrostFS network.
|
|
||||||
NetworkInfo(context.Context) (netmap.NetworkInfo, error)
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,11 +5,12 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl"
|
||||||
v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
|
v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
|
||||||
objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
@ -18,10 +19,9 @@ import (
|
||||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
|
@ -52,12 +52,12 @@ func (k *FeatureSettingsMock) SetMD5Enabled(md5Enabled bool) {
|
||||||
k.md5Enabled = md5Enabled
|
k.md5Enabled = md5Enabled
|
||||||
}
|
}
|
||||||
|
|
||||||
func (k *FeatureSettingsMock) FormContainerZone(ns string) string {
|
func (k *FeatureSettingsMock) FormContainerZone(ns string) (zone string, isDefault bool) {
|
||||||
if ns == "" {
|
if ns == "" {
|
||||||
return v2container.SysAttributeZoneDefault
|
return v2container.SysAttributeZoneDefault, true
|
||||||
}
|
}
|
||||||
|
|
||||||
return ns + ".ns"
|
return ns + ".ns", false
|
||||||
}
|
}
|
||||||
|
|
||||||
type TestFrostFS struct {
|
type TestFrostFS struct {
|
||||||
|
@ -67,6 +67,7 @@ type TestFrostFS struct {
|
||||||
objectErrors map[string]error
|
objectErrors map[string]error
|
||||||
objectPutErrors map[string]error
|
objectPutErrors map[string]error
|
||||||
containers map[string]*container.Container
|
containers map[string]*container.Container
|
||||||
|
eaclTables map[string]*eacl.Table
|
||||||
currentEpoch uint64
|
currentEpoch uint64
|
||||||
key *keys.PrivateKey
|
key *keys.PrivateKey
|
||||||
}
|
}
|
||||||
|
@ -77,6 +78,7 @@ func NewTestFrostFS(key *keys.PrivateKey) *TestFrostFS {
|
||||||
objectErrors: make(map[string]error),
|
objectErrors: make(map[string]error),
|
||||||
objectPutErrors: make(map[string]error),
|
objectPutErrors: make(map[string]error),
|
||||||
containers: make(map[string]*container.Container),
|
containers: make(map[string]*container.Container),
|
||||||
|
eaclTables: make(map[string]*eacl.Table),
|
||||||
key: key,
|
key: key,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -206,10 +208,10 @@ func (t *TestFrostFS) UserContainers(context.Context, PrmUserContainers) ([]cid.
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) retrieveObject(ctx context.Context, cnrID cid.ID, objID oid.ID) (*object.Object, error) {
|
func (t *TestFrostFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectPart, error) {
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(cnrID)
|
addr.SetContainer(prm.Container)
|
||||||
addr.SetObject(objID)
|
addr.SetObject(prm.Object)
|
||||||
|
|
||||||
sAddr := addr.EncodeToString()
|
sAddr := addr.EncodeToString()
|
||||||
|
|
||||||
|
@ -219,48 +221,30 @@ func (t *TestFrostFS) retrieveObject(ctx context.Context, cnrID cid.ID, objID oi
|
||||||
|
|
||||||
if obj, ok := t.objects[sAddr]; ok {
|
if obj, ok := t.objects[sAddr]; ok {
|
||||||
owner := getBearerOwner(ctx)
|
owner := getBearerOwner(ctx)
|
||||||
if !t.checkAccess(cnrID, owner) {
|
if !t.checkAccess(prm.Container, owner, eacl.OperationGet, obj) {
|
||||||
return nil, ErrAccessDenied
|
return nil, ErrAccessDenied
|
||||||
}
|
}
|
||||||
|
|
||||||
return obj, nil
|
payload := obj.Payload()
|
||||||
|
|
||||||
|
if prm.PayloadRange[0]+prm.PayloadRange[1] > 0 {
|
||||||
|
off := prm.PayloadRange[0]
|
||||||
|
payload = payload[off : off+prm.PayloadRange[1]]
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ObjectPart{
|
||||||
|
Head: obj,
|
||||||
|
Payload: io.NopCloser(bytes.NewReader(payload)),
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, fmt.Errorf("%w: %s", &apistatus.ObjectNotFound{}, addr)
|
return nil, fmt.Errorf("%w: %s", &apistatus.ObjectNotFound{}, addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) HeadObject(ctx context.Context, prm PrmObjectHead) (*object.Object, error) {
|
func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.ID, error) {
|
||||||
return t.retrieveObject(ctx, prm.Container, prm.Object)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TestFrostFS) GetObject(ctx context.Context, prm PrmObjectGet) (*Object, error) {
|
|
||||||
obj, err := t.retrieveObject(ctx, prm.Container, prm.Object)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Object{
|
|
||||||
Header: *obj,
|
|
||||||
Payload: io.NopCloser(bytes.NewReader(obj.Payload())),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TestFrostFS) RangeObject(ctx context.Context, prm PrmObjectRange) (io.ReadCloser, error) {
|
|
||||||
obj, err := t.retrieveObject(ctx, prm.Container, prm.Object)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
off := prm.PayloadRange[0]
|
|
||||||
payload := obj.Payload()[off : off+prm.PayloadRange[1]]
|
|
||||||
|
|
||||||
return io.NopCloser(bytes.NewReader(payload)), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (*CreateObjectResult, error) {
|
|
||||||
b := make([]byte, 32)
|
b := make([]byte, 32)
|
||||||
if _, err := io.ReadFull(rand.Reader, b); err != nil {
|
if _, err := io.ReadFull(rand.Reader, b); err != nil {
|
||||||
return nil, err
|
return oid.ID{}, err
|
||||||
}
|
}
|
||||||
var id oid.ID
|
var id oid.ID
|
||||||
id.SetSHA256(sha256.Sum256(b))
|
id.SetSHA256(sha256.Sum256(b))
|
||||||
|
@ -268,7 +252,7 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (*Cre
|
||||||
attrs := make([]object.Attribute, 0)
|
attrs := make([]object.Attribute, 0)
|
||||||
|
|
||||||
if err := t.objectPutErrors[prm.Filepath]; err != nil {
|
if err := t.objectPutErrors[prm.Filepath]; err != nil {
|
||||||
return nil, err
|
return oid.ID{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if prm.Filepath != "" {
|
if prm.Filepath != "" {
|
||||||
|
@ -313,7 +297,7 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (*Cre
|
||||||
if prm.Payload != nil {
|
if prm.Payload != nil {
|
||||||
all, err := io.ReadAll(prm.Payload)
|
all, err := io.ReadAll(prm.Payload)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return oid.ID{}, err
|
||||||
}
|
}
|
||||||
obj.SetPayload(all)
|
obj.SetPayload(all)
|
||||||
obj.SetPayloadSize(uint64(len(all)))
|
obj.SetPayloadSize(uint64(len(all)))
|
||||||
|
@ -327,10 +311,7 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (*Cre
|
||||||
|
|
||||||
addr := newAddress(cnrID, objID)
|
addr := newAddress(cnrID, objID)
|
||||||
t.objects[addr.EncodeToString()] = obj
|
t.objects[addr.EncodeToString()] = obj
|
||||||
return &CreateObjectResult{
|
return objID, nil
|
||||||
ObjectID: objID,
|
|
||||||
CreationEpoch: t.currentEpoch - 1,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
|
func (t *TestFrostFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
|
||||||
|
@ -342,9 +323,9 @@ func (t *TestFrostFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) err
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := t.objects[addr.EncodeToString()]; ok {
|
if obj, ok := t.objects[addr.EncodeToString()]; ok {
|
||||||
owner := getBearerOwner(ctx)
|
owner := getBearerOwner(ctx)
|
||||||
if !t.checkAccess(prm.Container, owner) {
|
if !t.checkAccess(prm.Container, owner, eacl.OperationDelete, obj) {
|
||||||
return ErrAccessDenied
|
return ErrAccessDenied
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -372,87 +353,31 @@ func (t *TestFrostFS) AllObjects(cnrID cid.ID) []oid.ID {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) ([]oid.ID, error) {
|
func (t *TestFrostFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *session.Container) error {
|
||||||
filters := object.NewSearchFilters()
|
cnrID, ok := table.CID()
|
||||||
filters.AddRootFilter()
|
if !ok {
|
||||||
|
return errors.New("invalid cid")
|
||||||
if prm.ExactAttribute[0] != "" {
|
|
||||||
filters.AddFilter(prm.ExactAttribute[0], prm.ExactAttribute[1], object.MatchStringEqual)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cidStr := prm.Container.EncodeToString()
|
if _, ok = t.containers[cnrID.EncodeToString()]; !ok {
|
||||||
|
return errors.New("not found")
|
||||||
var res []oid.ID
|
|
||||||
|
|
||||||
if len(filters) == 1 {
|
|
||||||
for k, v := range t.objects {
|
|
||||||
if strings.Contains(k, cidStr) {
|
|
||||||
id, _ := v.ID()
|
|
||||||
res = append(res, id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return res, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
filter := filters[1]
|
t.eaclTables[cnrID.EncodeToString()] = &table
|
||||||
if len(filters) != 2 || filter.Operation() != object.MatchStringEqual {
|
|
||||||
return nil, fmt.Errorf("usupported filters")
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
for k, v := range t.objects {
|
func (t *TestFrostFS) ContainerEACL(_ context.Context, prm PrmContainerEACL) (*eacl.Table, error) {
|
||||||
if strings.Contains(k, cidStr) && isMatched(v.Attributes(), filter) {
|
table, ok := t.eaclTables[prm.ContainerID.EncodeToString()]
|
||||||
id, _ := v.ID()
|
if !ok {
|
||||||
res = append(res, id)
|
return nil, errors.New("not found")
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return res, nil
|
return table, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) NetworkInfo(context.Context) (netmap.NetworkInfo, error) {
|
func (t *TestFrostFS) checkAccess(cnrID cid.ID, owner user.ID, op eacl.Operation, obj *object.Object) bool {
|
||||||
ni := netmap.NetworkInfo{}
|
|
||||||
ni.SetCurrentEpoch(t.currentEpoch)
|
|
||||||
|
|
||||||
return ni, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TestFrostFS) PatchObject(ctx context.Context, prm PrmObjectPatch) (oid.ID, error) {
|
|
||||||
obj, err := t.retrieveObject(ctx, prm.Container, prm.Object)
|
|
||||||
if err != nil {
|
|
||||||
return oid.ID{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
newObj := *obj
|
|
||||||
|
|
||||||
patchBytes, err := io.ReadAll(prm.Payload)
|
|
||||||
if err != nil {
|
|
||||||
return oid.ID{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var newPayload []byte
|
|
||||||
if prm.Offset > 0 {
|
|
||||||
newPayload = append(newPayload, obj.Payload()[:prm.Offset]...)
|
|
||||||
}
|
|
||||||
newPayload = append(newPayload, patchBytes...)
|
|
||||||
if prm.Offset+prm.Length < obj.PayloadSize() {
|
|
||||||
newPayload = append(newPayload, obj.Payload()[prm.Offset+prm.Length:]...)
|
|
||||||
}
|
|
||||||
newObj.SetPayload(newPayload)
|
|
||||||
newObj.SetPayloadSize(uint64(len(newPayload)))
|
|
||||||
|
|
||||||
var hash checksum.Checksum
|
|
||||||
checksum.Calculate(&hash, checksum.SHA256, newPayload)
|
|
||||||
newObj.SetPayloadChecksum(hash)
|
|
||||||
|
|
||||||
newID := oidtest.ID()
|
|
||||||
newObj.SetID(newID)
|
|
||||||
|
|
||||||
t.objects[newAddress(prm.Container, newID).EncodeToString()] = &newObj
|
|
||||||
|
|
||||||
return newID, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TestFrostFS) checkAccess(cnrID cid.ID, owner user.ID) bool {
|
|
||||||
cnr, ok := t.containers[cnrID.EncodeToString()]
|
cnr, ok := t.containers[cnrID.EncodeToString()]
|
||||||
if !ok {
|
if !ok {
|
||||||
return false
|
return false
|
||||||
|
@ -462,6 +387,57 @@ func (t *TestFrostFS) checkAccess(cnrID cid.ID, owner user.ID) bool {
|
||||||
return cnr.Owner().Equals(owner)
|
return cnr.Owner().Equals(owner)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
table, ok := t.eaclTables[cnrID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, rec := range table.Records() {
|
||||||
|
if rec.Operation() != op {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !matchTarget(rec, owner) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if matchFilter(rec.Filters(), obj) {
|
||||||
|
return rec.Action() == eacl.ActionAllow
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func matchTarget(rec eacl.Record, owner user.ID) bool {
|
||||||
|
for _, trgt := range rec.Targets() {
|
||||||
|
if trgt.Role() == eacl.RoleOthers {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
var targetOwner user.ID
|
||||||
|
for _, pk := range eacl.TargetECDSAKeys(&trgt) {
|
||||||
|
user.IDFromKey(&targetOwner, *pk)
|
||||||
|
if targetOwner.Equals(owner) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func matchFilter(filters []eacl.Filter, obj *object.Object) bool {
|
||||||
|
objID, _ := obj.ID()
|
||||||
|
for _, f := range filters {
|
||||||
|
fv2 := f.ToV2()
|
||||||
|
if fv2.GetMatchType() != acl.MatchTypeStringEqual ||
|
||||||
|
fv2.GetHeaderType() != acl.HeaderTypeObject ||
|
||||||
|
fv2.GetKey() != acl.FilterObjectID ||
|
||||||
|
fv2.GetValue() != objID.EncodeToString() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -472,12 +448,3 @@ func getBearerOwner(ctx context.Context) user.ID {
|
||||||
|
|
||||||
return user.ID{}
|
return user.ID{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func isMatched(attributes []object.Attribute, filter object.SearchFilter) bool {
|
|
||||||
for _, attr := range attributes {
|
|
||||||
if attr.Key() == filter.Header() && attr.Value() == filter.Value() {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
|
@ -6,11 +6,9 @@ import (
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
stderrors "errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/url"
|
"net/url"
|
||||||
"sort"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
@ -18,45 +16,55 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
EventListener interface {
|
||||||
|
Subscribe(context.Context, string, MsgHandler) error
|
||||||
|
Listen(context.Context)
|
||||||
|
}
|
||||||
|
|
||||||
|
MsgHandler interface {
|
||||||
|
HandleMessage(context.Context, *nats.Msg) error
|
||||||
|
}
|
||||||
|
|
||||||
|
MsgHandlerFunc func(context.Context, *nats.Msg) error
|
||||||
|
|
||||||
BucketResolver interface {
|
BucketResolver interface {
|
||||||
Resolve(ctx context.Context, zone, name string) (cid.ID, error)
|
Resolve(ctx context.Context, name string) (cid.ID, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
FeatureSettings interface {
|
FeatureSettings interface {
|
||||||
ClientCut() bool
|
ClientCut() bool
|
||||||
BufferMaxSizeForPut() uint64
|
BufferMaxSizeForPut() uint64
|
||||||
MD5Enabled() bool
|
MD5Enabled() bool
|
||||||
FormContainerZone(ns string) string
|
FormContainerZone(ns string) (zone string, isDefault bool)
|
||||||
}
|
}
|
||||||
|
|
||||||
Layer struct {
|
layer struct {
|
||||||
frostFS FrostFS
|
frostFS FrostFS
|
||||||
gateOwner user.ID
|
gateOwner user.ID
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
anonKey AnonymousKey
|
anonKey AnonymousKey
|
||||||
resolver BucketResolver
|
resolver BucketResolver
|
||||||
|
ncontroller EventListener
|
||||||
cache *Cache
|
cache *Cache
|
||||||
treeService TreeService
|
treeService TreeService
|
||||||
features FeatureSettings
|
features FeatureSettings
|
||||||
gateKey *keys.PrivateKey
|
|
||||||
corsCnrInfo *data.BucketInfo
|
|
||||||
lifecycleCnrInfo *data.BucketInfo
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Config struct {
|
Config struct {
|
||||||
|
@ -67,9 +75,6 @@ type (
|
||||||
Resolver BucketResolver
|
Resolver BucketResolver
|
||||||
TreeService TreeService
|
TreeService TreeService
|
||||||
Features FeatureSettings
|
Features FeatureSettings
|
||||||
GateKey *keys.PrivateKey
|
|
||||||
CORSCnrInfo *data.BucketInfo
|
|
||||||
LifecycleCnrInfo *data.BucketInfo
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// AnonymousKey contains data for anonymous requests.
|
// AnonymousKey contains data for anonymous requests.
|
||||||
|
@ -93,6 +98,14 @@ type (
|
||||||
VersionID string
|
VersionID string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ObjectVersion stores object version info.
|
||||||
|
ObjectVersion struct {
|
||||||
|
BktInfo *data.BucketInfo
|
||||||
|
ObjectName string
|
||||||
|
VersionID string
|
||||||
|
NoErrorOnDeleteMarker bool
|
||||||
|
}
|
||||||
|
|
||||||
// RangeParams stores range header request parameters.
|
// RangeParams stores range header request parameters.
|
||||||
RangeParams struct {
|
RangeParams struct {
|
||||||
Start uint64
|
Start uint64
|
||||||
|
@ -127,7 +140,6 @@ type (
|
||||||
BktInfo *data.BucketInfo
|
BktInfo *data.BucketInfo
|
||||||
Objects []*VersionedObject
|
Objects []*VersionedObject
|
||||||
Settings *data.BucketSettings
|
Settings *data.BucketSettings
|
||||||
NetworkInfo netmap.NetworkInfo
|
|
||||||
IsMultiple bool
|
IsMultiple bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -160,7 +172,6 @@ type (
|
||||||
DstEncryption encryption.Params
|
DstEncryption encryption.Params
|
||||||
CopiesNumbers []uint32
|
CopiesNumbers []uint32
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateBucketParams stores bucket create request parameters.
|
// CreateBucketParams stores bucket create request parameters.
|
||||||
CreateBucketParams struct {
|
CreateBucketParams struct {
|
||||||
Name string
|
Name string
|
||||||
|
@ -169,12 +180,18 @@ type (
|
||||||
SessionContainerCreation *session.Container
|
SessionContainerCreation *session.Container
|
||||||
LocationConstraint string
|
LocationConstraint string
|
||||||
ObjectLockEnabled bool
|
ObjectLockEnabled bool
|
||||||
|
APEEnabled bool
|
||||||
|
}
|
||||||
|
// PutBucketACLParams stores put bucket acl request parameters.
|
||||||
|
PutBucketACLParams struct {
|
||||||
|
BktInfo *data.BucketInfo
|
||||||
|
EACL *eacl.Table
|
||||||
|
SessionToken *session.Container
|
||||||
}
|
}
|
||||||
// DeleteBucketParams stores delete bucket request parameters.
|
// DeleteBucketParams stores delete bucket request parameters.
|
||||||
DeleteBucketParams struct {
|
DeleteBucketParams struct {
|
||||||
BktInfo *data.BucketInfo
|
BktInfo *data.BucketInfo
|
||||||
SessionToken *session.Container
|
SessionToken *session.Container
|
||||||
SkipCheck bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListObjectVersionsParams stores list objects versions parameters.
|
// ListObjectVersionsParams stores list objects versions parameters.
|
||||||
|
@ -203,6 +220,68 @@ type (
|
||||||
encrypted bool
|
encrypted bool
|
||||||
decryptedLen uint64
|
decryptedLen uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Client provides S3 API client interface.
|
||||||
|
Client interface {
|
||||||
|
Initialize(ctx context.Context, c EventListener) error
|
||||||
|
EphemeralKey() *keys.PublicKey
|
||||||
|
|
||||||
|
GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error)
|
||||||
|
PutBucketSettings(ctx context.Context, p *PutSettingsParams) error
|
||||||
|
|
||||||
|
PutBucketCORS(ctx context.Context, p *PutCORSParams) error
|
||||||
|
GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*data.CORSConfiguration, error)
|
||||||
|
DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) error
|
||||||
|
|
||||||
|
ListBuckets(ctx context.Context) ([]*data.BucketInfo, error)
|
||||||
|
GetBucketInfo(ctx context.Context, name string) (*data.BucketInfo, error)
|
||||||
|
ResolveCID(ctx context.Context, name string) (cid.ID, error)
|
||||||
|
GetBucketACL(ctx context.Context, bktInfo *data.BucketInfo) (*BucketACL, error)
|
||||||
|
PutBucketACL(ctx context.Context, p *PutBucketACLParams) error
|
||||||
|
CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error)
|
||||||
|
DeleteBucket(ctx context.Context, p *DeleteBucketParams) error
|
||||||
|
|
||||||
|
GetObject(ctx context.Context, p *GetObjectParams) (*ObjectPayload, error)
|
||||||
|
GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error)
|
||||||
|
GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error)
|
||||||
|
|
||||||
|
GetLockInfo(ctx context.Context, obj *ObjectVersion) (*data.LockInfo, error)
|
||||||
|
PutLockInfo(ctx context.Context, p *PutLockInfoParams) error
|
||||||
|
|
||||||
|
GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error)
|
||||||
|
PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error
|
||||||
|
DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error
|
||||||
|
|
||||||
|
GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams) (string, map[string]string, error)
|
||||||
|
PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (*data.NodeVersion, error)
|
||||||
|
DeleteObjectTagging(ctx context.Context, p *ObjectVersion) (*data.NodeVersion, error)
|
||||||
|
|
||||||
|
PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error)
|
||||||
|
|
||||||
|
CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ExtendedObjectInfo, error)
|
||||||
|
|
||||||
|
ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error)
|
||||||
|
ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error)
|
||||||
|
ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error)
|
||||||
|
|
||||||
|
DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*VersionedObject
|
||||||
|
|
||||||
|
CreateMultipartUpload(ctx context.Context, p *CreateMultipartParams) error
|
||||||
|
CompleteMultipartUpload(ctx context.Context, p *CompleteMultipartParams) (*UploadData, *data.ExtendedObjectInfo, error)
|
||||||
|
UploadPart(ctx context.Context, p *UploadPartParams) (string, error)
|
||||||
|
UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.ObjectInfo, error)
|
||||||
|
ListMultipartUploads(ctx context.Context, p *ListMultipartUploadsParams) (*ListMultipartUploadsInfo, error)
|
||||||
|
AbortMultipartUpload(ctx context.Context, p *UploadInfoParams) error
|
||||||
|
ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsInfo, error)
|
||||||
|
|
||||||
|
PutBucketNotificationConfiguration(ctx context.Context, p *PutBucketNotificationConfigurationParams) error
|
||||||
|
GetBucketNotificationConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (*data.NotificationConfiguration, error)
|
||||||
|
|
||||||
|
// Compound methods for optimizations
|
||||||
|
|
||||||
|
// GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation.
|
||||||
|
GetObjectTaggingAndLock(ctx context.Context, p *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error)
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -229,14 +308,18 @@ func (t *VersionedObject) String() string {
|
||||||
return t.Name + ":" + t.VersionID
|
return t.Name + ":" + t.VersionID
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f MsgHandlerFunc) HandleMessage(ctx context.Context, msg *nats.Msg) error {
|
||||||
|
return f(ctx, msg)
|
||||||
|
}
|
||||||
|
|
||||||
func (p HeadObjectParams) Versioned() bool {
|
func (p HeadObjectParams) Versioned() bool {
|
||||||
return len(p.VersionID) > 0
|
return len(p.VersionID) > 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewLayer creates an instance of a Layer. It checks credentials
|
// NewLayer creates an instance of a layer. It checks credentials
|
||||||
// and establishes gRPC connection with the node.
|
// and establishes gRPC connection with the node.
|
||||||
func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) *Layer {
|
func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) Client {
|
||||||
return &Layer{
|
return &layer{
|
||||||
frostFS: frostFS,
|
frostFS: frostFS,
|
||||||
log: log,
|
log: log,
|
||||||
gateOwner: config.GateOwner,
|
gateOwner: config.GateOwner,
|
||||||
|
@ -245,16 +328,30 @@ func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) *Layer {
|
||||||
cache: config.Cache,
|
cache: config.Cache,
|
||||||
treeService: config.TreeService,
|
treeService: config.TreeService,
|
||||||
features: config.Features,
|
features: config.Features,
|
||||||
gateKey: config.GateKey,
|
|
||||||
corsCnrInfo: config.CORSCnrInfo,
|
|
||||||
lifecycleCnrInfo: config.LifecycleCnrInfo,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) EphemeralKey() *keys.PublicKey {
|
func (n *layer) EphemeralKey() *keys.PublicKey {
|
||||||
return n.anonKey.Key.PublicKey()
|
return n.anonKey.Key.PublicKey()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (n *layer) Initialize(ctx context.Context, c EventListener) error {
|
||||||
|
if n.IsNotificationEnabled() {
|
||||||
|
return fmt.Errorf("already initialized")
|
||||||
|
}
|
||||||
|
|
||||||
|
// todo add notification handlers (e.g. for lifecycles)
|
||||||
|
|
||||||
|
c.Listen(ctx)
|
||||||
|
|
||||||
|
n.ncontroller = c
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *layer) IsNotificationEnabled() bool {
|
||||||
|
return n.ncontroller != nil
|
||||||
|
}
|
||||||
|
|
||||||
// IsAuthenticatedRequest checks if access box exists in the current request.
|
// IsAuthenticatedRequest checks if access box exists in the current request.
|
||||||
func IsAuthenticatedRequest(ctx context.Context) bool {
|
func IsAuthenticatedRequest(ctx context.Context) bool {
|
||||||
_, err := middleware.GetBoxData(ctx)
|
_, err := middleware.GetBoxData(ctx)
|
||||||
|
@ -271,7 +368,7 @@ func TimeNow(ctx context.Context) time.Time {
|
||||||
}
|
}
|
||||||
|
|
||||||
// BearerOwner returns owner id from BearerToken (context) or from client owner.
|
// BearerOwner returns owner id from BearerToken (context) or from client owner.
|
||||||
func (n *Layer) BearerOwner(ctx context.Context) user.ID {
|
func (n *layer) BearerOwner(ctx context.Context) user.ID {
|
||||||
if bd, err := middleware.GetBoxData(ctx); err == nil && bd.Gate.BearerToken != nil {
|
if bd, err := middleware.GetBoxData(ctx); err == nil && bd.Gate.BearerToken != nil {
|
||||||
return bearer.ResolveIssuer(*bd.Gate.BearerToken)
|
return bearer.ResolveIssuer(*bd.Gate.BearerToken)
|
||||||
}
|
}
|
||||||
|
@ -283,7 +380,7 @@ func (n *Layer) BearerOwner(ctx context.Context) user.ID {
|
||||||
}
|
}
|
||||||
|
|
||||||
// SessionTokenForRead returns session container token.
|
// SessionTokenForRead returns session container token.
|
||||||
func (n *Layer) SessionTokenForRead(ctx context.Context) *session.Container {
|
func (n *layer) SessionTokenForRead(ctx context.Context) *session.Container {
|
||||||
if bd, err := middleware.GetBoxData(ctx); err == nil && bd.Gate != nil {
|
if bd, err := middleware.GetBoxData(ctx); err == nil && bd.Gate != nil {
|
||||||
return bd.Gate.SessionToken()
|
return bd.Gate.SessionToken()
|
||||||
}
|
}
|
||||||
|
@ -291,7 +388,7 @@ func (n *Layer) SessionTokenForRead(ctx context.Context) *session.Container {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) reqLogger(ctx context.Context) *zap.Logger {
|
func (n *layer) reqLogger(ctx context.Context) *zap.Logger {
|
||||||
reqLogger := middleware.GetReqLog(ctx)
|
reqLogger := middleware.GetReqLog(ctx)
|
||||||
if reqLogger != nil {
|
if reqLogger != nil {
|
||||||
return reqLogger
|
return reqLogger
|
||||||
|
@ -299,11 +396,7 @@ func (n *Layer) reqLogger(ctx context.Context) *zap.Logger {
|
||||||
return n.log
|
return n.log
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) prepareAuthParameters(ctx context.Context, prm *PrmAuth, bktOwner user.ID) {
|
func (n *layer) prepareAuthParameters(ctx context.Context, prm *PrmAuth, bktOwner user.ID) {
|
||||||
if prm.BearerToken != nil || prm.PrivateKey != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if bd, err := middleware.GetBoxData(ctx); err == nil && bd.Gate.BearerToken != nil {
|
if bd, err := middleware.GetBoxData(ctx); err == nil && bd.Gate.BearerToken != nil {
|
||||||
if bd.Gate.BearerToken.Impersonate() || bktOwner.Equals(bearer.ResolveIssuer(*bd.Gate.BearerToken)) {
|
if bd.Gate.BearerToken.Impersonate() || bktOwner.Equals(bearer.ResolveIssuer(*bd.Gate.BearerToken)) {
|
||||||
prm.BearerToken = bd.Gate.BearerToken
|
prm.BearerToken = bd.Gate.BearerToken
|
||||||
|
@ -315,20 +408,20 @@ func (n *Layer) prepareAuthParameters(ctx context.Context, prm *PrmAuth, bktOwne
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetBucketInfo returns bucket info by name.
|
// GetBucketInfo returns bucket info by name.
|
||||||
func (n *Layer) GetBucketInfo(ctx context.Context, name string) (*data.BucketInfo, error) {
|
func (n *layer) GetBucketInfo(ctx context.Context, name string) (*data.BucketInfo, error) {
|
||||||
name, err := url.QueryUnescape(name)
|
name, err := url.QueryUnescape(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("unescape bucket name: %w", err)
|
return nil, fmt.Errorf("unescape bucket name: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
zone := n.features.FormContainerZone(reqInfo.Namespace)
|
zone, _ := n.features.FormContainerZone(reqInfo.Namespace)
|
||||||
|
|
||||||
if bktInfo := n.cache.GetBucket(zone, name); bktInfo != nil {
|
if bktInfo := n.cache.GetBucket(zone, name); bktInfo != nil {
|
||||||
return bktInfo, nil
|
return bktInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
containerID, err := n.ResolveBucket(ctx, zone, name)
|
containerID, err := n.ResolveBucket(ctx, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if strings.Contains(err.Error(), "not found") {
|
if strings.Contains(err.Error(), "not found") {
|
||||||
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchBucket), err.Error())
|
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchBucket), err.Error())
|
||||||
|
@ -345,30 +438,48 @@ func (n *Layer) GetBucketInfo(ctx context.Context, name string) (*data.BucketInf
|
||||||
}
|
}
|
||||||
|
|
||||||
// ResolveCID returns container id by name.
|
// ResolveCID returns container id by name.
|
||||||
func (n *Layer) ResolveCID(ctx context.Context, name string) (cid.ID, error) {
|
func (n *layer) ResolveCID(ctx context.Context, name string) (cid.ID, error) {
|
||||||
name, err := url.QueryUnescape(name)
|
name, err := url.QueryUnescape(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cid.ID{}, fmt.Errorf("unescape bucket name: %w", err)
|
return cid.ID{}, fmt.Errorf("unescape bucket name: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
reqInfo := middleware.GetReqInfo(ctx)
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
zone := n.features.FormContainerZone(reqInfo.Namespace)
|
zone, _ := n.features.FormContainerZone(reqInfo.Namespace)
|
||||||
|
|
||||||
if bktInfo := n.cache.GetBucket(zone, name); bktInfo != nil {
|
if bktInfo := n.cache.GetBucket(zone, name); bktInfo != nil {
|
||||||
return bktInfo.CID, nil
|
return bktInfo.CID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return n.ResolveBucket(ctx, zone, name)
|
return n.ResolveBucket(ctx, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetBucketACL returns bucket acl info by name.
|
||||||
|
func (n *layer) GetBucketACL(ctx context.Context, bktInfo *data.BucketInfo) (*BucketACL, error) {
|
||||||
|
eACL, err := n.GetContainerEACL(ctx, bktInfo.CID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("get container eacl: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &BucketACL{
|
||||||
|
Info: bktInfo,
|
||||||
|
EACL: eACL,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// PutBucketACL puts bucket acl by name.
|
||||||
|
func (n *layer) PutBucketACL(ctx context.Context, param *PutBucketACLParams) error {
|
||||||
|
return n.setContainerEACLTable(ctx, param.BktInfo.CID, param.EACL, param.SessionToken)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListBuckets returns all user containers. The name of the bucket is a container
|
// ListBuckets returns all user containers. The name of the bucket is a container
|
||||||
// id. Timestamp is omitted since it is not saved in frostfs container.
|
// id. Timestamp is omitted since it is not saved in frostfs container.
|
||||||
func (n *Layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) {
|
func (n *layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||||
return n.containerList(ctx)
|
return n.containerList(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetObject from storage.
|
// GetObject from storage.
|
||||||
func (n *Layer) GetObject(ctx context.Context, p *GetObjectParams) (*ObjectPayload, error) {
|
func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) (*ObjectPayload, error) {
|
||||||
var params getParams
|
var params getParams
|
||||||
|
|
||||||
params.objInfo = p.ObjectInfo
|
params.objInfo = p.ObjectInfo
|
||||||
|
@ -482,7 +593,7 @@ func getDecrypter(p *GetObjectParams) (*encryption.Decrypter, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetObjectInfo returns meta information about the object.
|
// GetObjectInfo returns meta information about the object.
|
||||||
func (n *Layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error) {
|
func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ObjectInfo, error) {
|
||||||
extendedObjectInfo, err := n.GetExtendedObjectInfo(ctx, p)
|
extendedObjectInfo, err := n.GetExtendedObjectInfo(ctx, p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -492,7 +603,7 @@ func (n *Layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.O
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetExtendedObjectInfo returns meta information and corresponding info from the tree service about the object.
|
// GetExtendedObjectInfo returns meta information and corresponding info from the tree service about the object.
|
||||||
func (n *Layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) {
|
func (n *layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) {
|
||||||
var objInfo *data.ExtendedObjectInfo
|
var objInfo *data.ExtendedObjectInfo
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
|
@ -513,7 +624,7 @@ func (n *Layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CopyObject from one bucket into another bucket.
|
// CopyObject from one bucket into another bucket.
|
||||||
func (n *Layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ExtendedObjectInfo, error) {
|
func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.ExtendedObjectInfo, error) {
|
||||||
objPayload, err := n.GetObject(ctx, &GetObjectParams{
|
objPayload, err := n.GetObject(ctx, &GetObjectParams{
|
||||||
ObjectInfo: p.SrcObject,
|
ObjectInfo: p.SrcObject,
|
||||||
Versioned: p.SrcVersioned,
|
Versioned: p.SrcVersioned,
|
||||||
|
@ -547,29 +658,19 @@ func getRandomOID() (oid.ID, error) {
|
||||||
return objID, nil
|
return objID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings *data.BucketSettings, obj *VersionedObject,
|
func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings *data.BucketSettings, obj *VersionedObject) *VersionedObject {
|
||||||
networkInfo netmap.NetworkInfo) *VersionedObject {
|
|
||||||
if len(obj.VersionID) != 0 || settings.Unversioned() {
|
if len(obj.VersionID) != 0 || settings.Unversioned() {
|
||||||
var nodeVersions []*data.NodeVersion
|
var nodeVersion *data.NodeVersion
|
||||||
if nodeVersions, obj.Error = n.getNodeVersionsToDelete(ctx, bkt, obj); obj.Error != nil {
|
if nodeVersion, obj.Error = n.getNodeVersionToDelete(ctx, bkt, obj); obj.Error != nil {
|
||||||
return n.handleNotFoundError(bkt, obj)
|
return n.handleNotFoundError(bkt, obj)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, nodeVersion := range nodeVersions {
|
|
||||||
if obj.DeleteMarkVersion, obj.Error = n.removeOldVersion(ctx, bkt, nodeVersion, obj); obj.Error != nil {
|
if obj.DeleteMarkVersion, obj.Error = n.removeOldVersion(ctx, bkt, nodeVersion, obj); obj.Error != nil {
|
||||||
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
|
return n.handleObjectDeleteErrors(ctx, bkt, obj, nodeVersion.ID)
|
||||||
return obj
|
|
||||||
}
|
|
||||||
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting,
|
|
||||||
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID); obj.Error != nil {
|
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID)
|
||||||
return obj
|
n.cache.CleanListCacheEntriesContainingObject(obj.Name, bkt.CID)
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
n.cache.DeleteObjectName(bkt.CID, bkt.Name, obj.Name)
|
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -582,30 +683,20 @@ func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
||||||
if settings.VersioningSuspended() {
|
if settings.VersioningSuspended() {
|
||||||
obj.VersionID = data.UnversionedObjectVersionID
|
obj.VersionID = data.UnversionedObjectVersionID
|
||||||
|
|
||||||
var nodeVersions []*data.NodeVersion
|
var nullVersionToDelete *data.NodeVersion
|
||||||
if nodeVersions, obj.Error = n.getNodeVersionsToDelete(ctx, bkt, obj); obj.Error != nil {
|
if lastVersion.IsUnversioned {
|
||||||
|
if !lastVersion.IsDeleteMarker {
|
||||||
|
nullVersionToDelete = lastVersion
|
||||||
|
}
|
||||||
|
} else if nullVersionToDelete, obj.Error = n.getNodeVersionToDelete(ctx, bkt, obj); obj.Error != nil {
|
||||||
if !isNotFoundError(obj.Error) {
|
if !isNotFoundError(obj.Error) {
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, nodeVersion := range nodeVersions {
|
if nullVersionToDelete != nil {
|
||||||
if nodeVersion.ID == lastVersion.ID && nodeVersion.IsDeleteMarker {
|
if obj.DeleteMarkVersion, obj.Error = n.removeOldVersion(ctx, bkt, nullVersionToDelete, obj); obj.Error != nil {
|
||||||
continue
|
return n.handleObjectDeleteErrors(ctx, bkt, obj, nullVersionToDelete.ID)
|
||||||
}
|
|
||||||
|
|
||||||
if !nodeVersion.IsDeleteMarker {
|
|
||||||
if obj.DeleteMarkVersion, obj.Error = n.removeOldVersion(ctx, bkt, nodeVersion, obj); obj.Error != nil {
|
|
||||||
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
|
|
||||||
return obj
|
|
||||||
}
|
|
||||||
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting,
|
|
||||||
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID); obj.Error != nil {
|
|
||||||
return obj
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -630,7 +721,6 @@ func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
||||||
Created: &now,
|
Created: &now,
|
||||||
Owner: &n.gateOwner,
|
Owner: &n.gateOwner,
|
||||||
IsDeleteMarker: true,
|
IsDeleteMarker: true,
|
||||||
CreationEpoch: networkInfo.CurrentEpoch(),
|
|
||||||
},
|
},
|
||||||
IsUnversioned: settings.VersioningSuspended(),
|
IsUnversioned: settings.VersioningSuspended(),
|
||||||
}
|
}
|
||||||
|
@ -644,7 +734,7 @@ func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) handleNotFoundError(bkt *data.BucketInfo, obj *VersionedObject) *VersionedObject {
|
func (n *layer) handleNotFoundError(bkt *data.BucketInfo, obj *VersionedObject) *VersionedObject {
|
||||||
if isNotFoundError(obj.Error) {
|
if isNotFoundError(obj.Error) {
|
||||||
obj.Error = nil
|
obj.Error = nil
|
||||||
n.cache.CleanListCacheEntriesContainingObject(obj.Name, bkt.CID)
|
n.cache.CleanListCacheEntriesContainingObject(obj.Name, bkt.CID)
|
||||||
|
@ -654,74 +744,40 @@ func (n *Layer) handleNotFoundError(bkt *data.BucketInfo, obj *VersionedObject)
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject, nodeID uint64) *VersionedObject {
|
||||||
|
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
|
||||||
|
return obj
|
||||||
|
}
|
||||||
|
|
||||||
|
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting,
|
||||||
|
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error))
|
||||||
|
|
||||||
|
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeID)
|
||||||
|
if obj.Error == nil {
|
||||||
|
n.cache.DeleteObjectName(bkt.CID, bkt.Name, obj.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
return obj
|
||||||
|
}
|
||||||
|
|
||||||
func isNotFoundError(err error) bool {
|
func isNotFoundError(err error) bool {
|
||||||
return errors.IsS3Error(err, errors.ErrNoSuchKey) ||
|
return errors.IsS3Error(err, errors.ErrNoSuchKey) ||
|
||||||
errors.IsS3Error(err, errors.ErrNoSuchVersion)
|
errors.IsS3Error(err, errors.ErrNoSuchVersion)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) getNodeVersionsToDelete(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) ([]*data.NodeVersion, error) {
|
func (n *layer) getNodeVersionToDelete(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.NodeVersion, error) {
|
||||||
var versionsToDelete []*data.NodeVersion
|
objVersion := &ObjectVersion{
|
||||||
versions, err := n.treeService.GetVersions(ctx, bkt, obj.Name)
|
BktInfo: bkt,
|
||||||
if err != nil {
|
ObjectName: obj.Name,
|
||||||
if stderrors.Is(err, ErrNodeNotFound) {
|
VersionID: obj.VersionID,
|
||||||
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
|
NoErrorOnDeleteMarker: true,
|
||||||
}
|
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(versions) == 0 {
|
return n.getNodeVersion(ctx, objVersion)
|
||||||
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", s3errors.GetAPIError(s3errors.ErrNoSuchVersion))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Slice(versions, func(i, j int) bool {
|
func (n *layer) getLastNodeVersion(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.NodeVersion, error) {
|
||||||
return versions[i].Timestamp < versions[j].Timestamp
|
objVersion := &ObjectVersion{
|
||||||
})
|
|
||||||
|
|
||||||
var matchFn func(nv *data.NodeVersion) bool
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case obj.VersionID == data.UnversionedObjectVersionID:
|
|
||||||
matchFn = func(nv *data.NodeVersion) bool {
|
|
||||||
return nv.IsUnversioned
|
|
||||||
}
|
|
||||||
case len(obj.VersionID) == 0:
|
|
||||||
latest := versions[len(versions)-1]
|
|
||||||
if latest.IsUnversioned {
|
|
||||||
matchFn = func(nv *data.NodeVersion) bool {
|
|
||||||
return nv.IsUnversioned
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
matchFn = func(nv *data.NodeVersion) bool {
|
|
||||||
return nv.ID == latest.ID
|
|
||||||
}
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
matchFn = func(nv *data.NodeVersion) bool {
|
|
||||||
return nv.OID.EncodeToString() == obj.VersionID
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var oids []string
|
|
||||||
for _, v := range versions {
|
|
||||||
if matchFn(v) {
|
|
||||||
versionsToDelete = append(versionsToDelete, v)
|
|
||||||
if !v.IsDeleteMarker {
|
|
||||||
oids = append(oids, v.OID.EncodeToString())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(versionsToDelete) == 0 {
|
|
||||||
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", s3errors.GetAPIError(s3errors.ErrNoSuchVersion))
|
|
||||||
}
|
|
||||||
|
|
||||||
n.reqLogger(ctx).Debug(logs.GetTreeNodeToDelete, zap.Stringer("cid", bkt.CID), zap.Strings("oids", oids))
|
|
||||||
|
|
||||||
return versionsToDelete, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) getLastNodeVersion(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) (*data.NodeVersion, error) {
|
|
||||||
objVersion := &data.ObjectVersion{
|
|
||||||
BktInfo: bkt,
|
BktInfo: bkt,
|
||||||
ObjectName: obj.Name,
|
ObjectName: obj.Name,
|
||||||
VersionID: "",
|
VersionID: "",
|
||||||
|
@ -731,7 +787,7 @@ func (n *Layer) getLastNodeVersion(ctx context.Context, bkt *data.BucketInfo, ob
|
||||||
return n.getNodeVersion(ctx, objVersion)
|
return n.getNodeVersion(ctx, objVersion)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) removeOldVersion(ctx context.Context, bkt *data.BucketInfo, nodeVersion *data.NodeVersion, obj *VersionedObject) (string, error) {
|
func (n *layer) removeOldVersion(ctx context.Context, bkt *data.BucketInfo, nodeVersion *data.NodeVersion, obj *VersionedObject) (string, error) {
|
||||||
if nodeVersion.IsDeleteMarker {
|
if nodeVersion.IsDeleteMarker {
|
||||||
return obj.VersionID, nil
|
return obj.VersionID, nil
|
||||||
}
|
}
|
||||||
|
@ -743,14 +799,14 @@ func (n *Layer) removeOldVersion(ctx context.Context, bkt *data.BucketInfo, node
|
||||||
return "", n.objectDelete(ctx, bkt, nodeVersion.OID)
|
return "", n.objectDelete(ctx, bkt, nodeVersion.OID)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) removeCombinedObject(ctx context.Context, bkt *data.BucketInfo, nodeVersion *data.NodeVersion) error {
|
func (n *layer) removeCombinedObject(ctx context.Context, bkt *data.BucketInfo, nodeVersion *data.NodeVersion) error {
|
||||||
combinedObj, err := n.objectGet(ctx, bkt, nodeVersion.OID)
|
combinedObj, err := n.objectGet(ctx, bkt, nodeVersion.OID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("get combined object '%s': %w", nodeVersion.OID.EncodeToString(), err)
|
return fmt.Errorf("get combined object '%s': %w", nodeVersion.OID.EncodeToString(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var parts []*data.PartInfo
|
var parts []*data.PartInfo
|
||||||
if err = json.NewDecoder(combinedObj.Payload).Decode(&parts); err != nil {
|
if err = json.Unmarshal(combinedObj.Payload(), &parts); err != nil {
|
||||||
return fmt.Errorf("unmarshal combined object parts: %w", err)
|
return fmt.Errorf("unmarshal combined object parts: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -771,9 +827,9 @@ func (n *Layer) removeCombinedObject(ctx context.Context, bkt *data.BucketInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteObjects from the storage.
|
// DeleteObjects from the storage.
|
||||||
func (n *Layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*VersionedObject {
|
func (n *layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*VersionedObject {
|
||||||
for i, obj := range p.Objects {
|
for i, obj := range p.Objects {
|
||||||
p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.Settings, obj, p.NetworkInfo)
|
p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.Settings, obj)
|
||||||
if p.IsMultiple && p.Objects[i].Error != nil {
|
if p.IsMultiple && p.Objects[i].Error != nil {
|
||||||
n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error))
|
n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error))
|
||||||
}
|
}
|
||||||
|
@ -782,7 +838,7 @@ func (n *Layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*Ver
|
||||||
return p.Objects
|
return p.Objects
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) {
|
func (n *layer) CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) {
|
||||||
bktInfo, err := n.GetBucketInfo(ctx, p.Name)
|
bktInfo, err := n.GetBucketInfo(ctx, p.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.IsS3Error(err, errors.ErrNoSuchBucket) {
|
if errors.IsS3Error(err, errors.ErrNoSuchBucket) {
|
||||||
|
@ -798,10 +854,10 @@ func (n *Layer) CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.
|
||||||
return nil, errors.GetAPIError(errors.ErrBucketAlreadyExists)
|
return nil, errors.GetAPIError(errors.ErrBucketAlreadyExists)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) ResolveBucket(ctx context.Context, zone, name string) (cid.ID, error) {
|
func (n *layer) ResolveBucket(ctx context.Context, name string) (cid.ID, error) {
|
||||||
var cnrID cid.ID
|
var cnrID cid.ID
|
||||||
if err := cnrID.DecodeString(name); err != nil {
|
if err := cnrID.DecodeString(name); err != nil {
|
||||||
if cnrID, err = n.resolver.Resolve(ctx, zone, name); err != nil {
|
if cnrID, err = n.resolver.Resolve(ctx, name); err != nil {
|
||||||
return cid.ID{}, err
|
return cid.ID{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -811,8 +867,7 @@ func (n *Layer) ResolveBucket(ctx context.Context, zone, name string) (cid.ID, e
|
||||||
return cnrID, nil
|
return cnrID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
if !p.SkipCheck {
|
|
||||||
res, _, err := n.getAllObjectsVersions(ctx, commonVersionsListingParams{
|
res, _, err := n.getAllObjectsVersions(ctx, commonVersionsListingParams{
|
||||||
BktInfo: p.BktInfo,
|
BktInfo: p.BktInfo,
|
||||||
MaxKeys: 1,
|
MaxKeys: 1,
|
||||||
|
@ -824,41 +879,7 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
if len(res) != 0 {
|
if len(res) != 0 {
|
||||||
return errors.GetAPIError(errors.ErrBucketNotEmpty)
|
return errors.GetAPIError(errors.ErrBucketNotEmpty)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
n.cache.DeleteBucket(p.BktInfo)
|
n.cache.DeleteBucket(p.BktInfo)
|
||||||
|
return n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
||||||
corsObj, err := n.treeService.GetBucketCORS(ctx, p.BktInfo)
|
|
||||||
if err != nil {
|
|
||||||
n.reqLogger(ctx).Error(logs.GetBucketCors, zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
lifecycleObj, treeErr := n.treeService.GetBucketLifecycleConfiguration(ctx, p.BktInfo)
|
|
||||||
if treeErr != nil {
|
|
||||||
n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr))
|
|
||||||
}
|
|
||||||
|
|
||||||
err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("delete container: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !corsObj.Container().Equals(p.BktInfo.CID) && !corsObj.Container().Equals(cid.ID{}) {
|
|
||||||
n.deleteCORSObject(ctx, p.BktInfo, corsObj)
|
|
||||||
}
|
|
||||||
|
|
||||||
if treeErr == nil && !lifecycleObj.Container().Equals(p.BktInfo.CID) {
|
|
||||||
n.deleteLifecycleObject(ctx, p.BktInfo, lifecycleObj)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) GetNetworkInfo(ctx context.Context) (netmap.NetworkInfo, error) {
|
|
||||||
networkInfo, err := n.frostFS.NetworkInfo(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return networkInfo, fmt.Errorf("get network info: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return networkInfo, nil
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,148 +0,0 @@
|
||||||
package layer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/xml"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
||||||
"go.uber.org/zap"
|
|
||||||
)
|
|
||||||
|
|
||||||
type PutBucketLifecycleParams struct {
|
|
||||||
BktInfo *data.BucketInfo
|
|
||||||
LifecycleCfg *data.LifecycleConfiguration
|
|
||||||
LifecycleReader io.Reader
|
|
||||||
CopiesNumbers []uint32
|
|
||||||
MD5Hash string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucketLifecycleParams) error {
|
|
||||||
prm := PrmObjectCreate{
|
|
||||||
Payload: p.LifecycleReader,
|
|
||||||
Filepath: p.BktInfo.LifecycleConfigurationObjectName(),
|
|
||||||
CreationTime: TimeNow(ctx),
|
|
||||||
}
|
|
||||||
|
|
||||||
var lifecycleBkt *data.BucketInfo
|
|
||||||
if n.lifecycleCnrInfo == nil {
|
|
||||||
lifecycleBkt = p.BktInfo
|
|
||||||
prm.CopiesNumber = p.CopiesNumbers
|
|
||||||
} else {
|
|
||||||
lifecycleBkt = n.lifecycleCnrInfo
|
|
||||||
prm.PrmAuth.PrivateKey = &n.gateKey.PrivateKey
|
|
||||||
}
|
|
||||||
|
|
||||||
prm.Container = lifecycleBkt.CID
|
|
||||||
|
|
||||||
createdObj, err := n.objectPutAndHash(ctx, prm, lifecycleBkt)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("put lifecycle object: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
hashBytes, err := base64.StdEncoding.DecodeString(p.MD5Hash)
|
|
||||||
if err != nil {
|
|
||||||
return apiErr.GetAPIError(apiErr.ErrInvalidDigest)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !bytes.Equal(hashBytes, createdObj.MD5Sum) {
|
|
||||||
n.deleteLifecycleObject(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, createdObj.ID))
|
|
||||||
|
|
||||||
return apiErr.GetAPIError(apiErr.ErrInvalidDigest)
|
|
||||||
}
|
|
||||||
|
|
||||||
objsToDelete, err := n.treeService.PutBucketLifecycleConfiguration(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, createdObj.ID))
|
|
||||||
objsToDeleteNotFound := errors.Is(err, ErrNoNodeToRemove)
|
|
||||||
if err != nil && !objsToDeleteNotFound {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if !objsToDeleteNotFound {
|
|
||||||
for _, addr := range objsToDelete {
|
|
||||||
n.deleteLifecycleObject(ctx, p.BktInfo, addr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
n.cache.PutLifecycleConfiguration(n.BearerOwner(ctx), p.BktInfo, p.LifecycleCfg)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// deleteLifecycleObject removes object and logs in case of error.
|
|
||||||
func (n *Layer) deleteLifecycleObject(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) {
|
|
||||||
var prmAuth PrmAuth
|
|
||||||
lifecycleBkt := bktInfo
|
|
||||||
if !addr.Container().Equals(bktInfo.CID) {
|
|
||||||
lifecycleBkt = &data.BucketInfo{CID: addr.Container()}
|
|
||||||
prmAuth.PrivateKey = &n.gateKey.PrivateKey
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := n.objectDeleteWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth); err != nil {
|
|
||||||
n.reqLogger(ctx).Error(logs.CouldntDeleteLifecycleObject, zap.Error(err),
|
|
||||||
zap.String("cid", lifecycleBkt.CID.EncodeToString()),
|
|
||||||
zap.String("oid", addr.Object().EncodeToString()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (*data.LifecycleConfiguration, error) {
|
|
||||||
owner := n.BearerOwner(ctx)
|
|
||||||
if cfg := n.cache.GetLifecycleConfiguration(owner, bktInfo); cfg != nil {
|
|
||||||
return cfg, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
addr, err := n.treeService.GetBucketLifecycleConfiguration(ctx, bktInfo)
|
|
||||||
objNotFound := errors.Is(err, ErrNodeNotFound)
|
|
||||||
if err != nil && !objNotFound {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if objNotFound {
|
|
||||||
return nil, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
var prmAuth PrmAuth
|
|
||||||
lifecycleBkt := bktInfo
|
|
||||||
if !addr.Container().Equals(bktInfo.CID) {
|
|
||||||
lifecycleBkt = &data.BucketInfo{CID: addr.Container()}
|
|
||||||
prmAuth.PrivateKey = &n.gateKey.PrivateKey
|
|
||||||
}
|
|
||||||
|
|
||||||
obj, err := n.objectGetWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("get lifecycle object: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
lifecycleCfg := &data.LifecycleConfiguration{}
|
|
||||||
|
|
||||||
if err = xml.NewDecoder(obj.Payload).Decode(&lifecycleCfg); err != nil {
|
|
||||||
return nil, fmt.Errorf("unmarshal lifecycle configuration: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
n.cache.PutLifecycleConfiguration(owner, bktInfo, lifecycleCfg)
|
|
||||||
|
|
||||||
return lifecycleCfg, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) error {
|
|
||||||
objs, err := n.treeService.DeleteBucketLifecycleConfiguration(ctx, bktInfo)
|
|
||||||
objsNotFound := errors.Is(err, ErrNoNodeToRemove)
|
|
||||||
if err != nil && !objsNotFound {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !objsNotFound {
|
|
||||||
for _, addr := range objs {
|
|
||||||
n.deleteLifecycleObject(ctx, bktInfo, addr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
n.cache.DeleteLifecycleConfiguration(bktInfo)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
|
@ -1,65 +0,0 @@
|
||||||
package layer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"crypto/md5"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/xml"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
|
||||||
frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBucketLifecycle(t *testing.T) {
|
|
||||||
tc := prepareContext(t)
|
|
||||||
|
|
||||||
lifecycle := &data.LifecycleConfiguration{
|
|
||||||
XMLName: xml.Name{
|
|
||||||
Space: `http://s3.amazonaws.com/doc/2006-03-01/`,
|
|
||||||
Local: "LifecycleConfiguration",
|
|
||||||
},
|
|
||||||
Rules: []data.LifecycleRule{
|
|
||||||
{
|
|
||||||
Status: data.LifecycleStatusEnabled,
|
|
||||||
Expiration: &data.LifecycleExpiration{
|
|
||||||
Days: ptr(21),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
lifecycleBytes, err := xml.Marshal(lifecycle)
|
|
||||||
require.NoError(t, err)
|
|
||||||
hash := md5.New()
|
|
||||||
hash.Write(lifecycleBytes)
|
|
||||||
|
|
||||||
_, err = tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
|
||||||
require.Equal(t, apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), frostfsErrors.UnwrapErr(err))
|
|
||||||
|
|
||||||
err = tc.layer.DeleteBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
err = tc.layer.PutBucketLifecycleConfiguration(tc.ctx, &PutBucketLifecycleParams{
|
|
||||||
BktInfo: tc.bktInfo,
|
|
||||||
LifecycleCfg: lifecycle,
|
|
||||||
LifecycleReader: bytes.NewReader(lifecycleBytes),
|
|
||||||
MD5Hash: base64.StdEncoding.EncodeToString(hash.Sum(nil)),
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
cfg, err := tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, *lifecycle, *cfg)
|
|
||||||
|
|
||||||
err = tc.layer.DeleteBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
_, err = tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
|
||||||
require.Equal(t, apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), frostfsErrors.UnwrapErr(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
func ptr[T any](t T) *T {
|
|
||||||
return &t
|
|
||||||
}
|
|
|
@ -96,7 +96,7 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
// ListObjectsV1 returns objects in a bucket for requests of Version 1.
|
// ListObjectsV1 returns objects in a bucket for requests of Version 1.
|
||||||
func (n *Layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error) {
|
func (n *layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error) {
|
||||||
var result ListObjectsInfoV1
|
var result ListObjectsInfoV1
|
||||||
|
|
||||||
prm := commonLatestVersionsListingParams{
|
prm := commonLatestVersionsListingParams{
|
||||||
|
@ -127,7 +127,7 @@ func (n *Layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*Lis
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListObjectsV2 returns objects in a bucket for requests of Version 2.
|
// ListObjectsV2 returns objects in a bucket for requests of Version 2.
|
||||||
func (n *Layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error) {
|
func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error) {
|
||||||
var result ListObjectsInfoV2
|
var result ListObjectsInfoV2
|
||||||
|
|
||||||
prm := commonLatestVersionsListingParams{
|
prm := commonLatestVersionsListingParams{
|
||||||
|
@ -157,7 +157,7 @@ func (n *Layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
|
||||||
return &result, nil
|
return &result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
||||||
prm := commonVersionsListingParams{
|
prm := commonVersionsListingParams{
|
||||||
BktInfo: p.BktInfo,
|
BktInfo: p.BktInfo,
|
||||||
Delimiter: p.Delimiter,
|
Delimiter: p.Delimiter,
|
||||||
|
@ -188,7 +188,7 @@ func (n *Layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) getLatestObjectsVersions(ctx context.Context, p commonLatestVersionsListingParams) (objects []*data.ExtendedNodeVersion, next *data.ExtendedNodeVersion, err error) {
|
func (n *layer) getLatestObjectsVersions(ctx context.Context, p commonLatestVersionsListingParams) (objects []*data.ExtendedNodeVersion, next *data.ExtendedNodeVersion, err error) {
|
||||||
if p.MaxKeys == 0 {
|
if p.MaxKeys == 0 {
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
|
@ -225,7 +225,7 @@ func (n *Layer) getLatestObjectsVersions(ctx context.Context, p commonLatestVers
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) getAllObjectsVersions(ctx context.Context, p commonVersionsListingParams) ([]*data.ExtendedNodeVersion, bool, error) {
|
func (n *layer) getAllObjectsVersions(ctx context.Context, p commonVersionsListingParams) ([]*data.ExtendedNodeVersion, bool, error) {
|
||||||
if p.MaxKeys == 0 {
|
if p.MaxKeys == 0 {
|
||||||
return nil, false, nil
|
return nil, false, nil
|
||||||
}
|
}
|
||||||
|
@ -301,15 +301,15 @@ func formVersionsListRow(objects []*data.ExtendedNodeVersion, rowStartIndex int,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) getListLatestVersionsSession(ctx context.Context, p commonLatestVersionsListingParams) (*data.ListSession, error) {
|
func (n *layer) getListLatestVersionsSession(ctx context.Context, p commonLatestVersionsListingParams) (*data.ListSession, error) {
|
||||||
return n.getListVersionsSession(ctx, p.commonVersionsListingParams, true)
|
return n.getListVersionsSession(ctx, p.commonVersionsListingParams, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) getListAllVersionsSession(ctx context.Context, p commonVersionsListingParams) (*data.ListSession, error) {
|
func (n *layer) getListAllVersionsSession(ctx context.Context, p commonVersionsListingParams) (*data.ListSession, error) {
|
||||||
return n.getListVersionsSession(ctx, p, false)
|
return n.getListVersionsSession(ctx, p, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) getListVersionsSession(ctx context.Context, p commonVersionsListingParams, latestOnly bool) (*data.ListSession, error) {
|
func (n *layer) getListVersionsSession(ctx context.Context, p commonVersionsListingParams, latestOnly bool) (*data.ListSession, error) {
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
|
|
||||||
cacheKey := cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, p.Bookmark)
|
cacheKey := cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, p.Bookmark)
|
||||||
|
@ -329,12 +329,12 @@ func (n *Layer) getListVersionsSession(ctx context.Context, p commonVersionsList
|
||||||
return session, nil
|
return session, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) initNewVersionsByPrefixSession(ctx context.Context, p commonVersionsListingParams, latestOnly bool) (session *data.ListSession, err error) {
|
func (n *layer) initNewVersionsByPrefixSession(ctx context.Context, p commonVersionsListingParams, latestOnly bool) (session *data.ListSession, err error) {
|
||||||
session = &data.ListSession{NamesMap: make(map[string]struct{})}
|
session = &data.ListSession{NamesMap: make(map[string]struct{})}
|
||||||
session.Context, session.Cancel = context.WithCancel(context.Background())
|
session.Context, session.Cancel = context.WithCancel(context.Background())
|
||||||
|
|
||||||
if bd, err := middleware.GetBoxData(ctx); err == nil {
|
if bd, err := middleware.GetBoxData(ctx); err == nil {
|
||||||
session.Context = middleware.SetBox(session.Context, &middleware.Box{AccessBox: bd})
|
session.Context = middleware.SetBoxData(session.Context, bd)
|
||||||
}
|
}
|
||||||
|
|
||||||
session.Stream, err = n.treeService.InitVersionsByPrefixStream(session.Context, p.BktInfo, p.Prefix, latestOnly)
|
session.Stream, err = n.treeService.InitVersionsByPrefixStream(session.Context, p.BktInfo, p.Prefix, latestOnly)
|
||||||
|
@ -345,7 +345,7 @@ func (n *Layer) initNewVersionsByPrefixSession(ctx context.Context, p commonVers
|
||||||
return session, nil
|
return session, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) putListLatestVersionsSession(ctx context.Context, p commonLatestVersionsListingParams, session *data.ListSession, allObjects []*data.ExtendedNodeVersion) {
|
func (n *layer) putListLatestVersionsSession(ctx context.Context, p commonLatestVersionsListingParams, session *data.ListSession, allObjects []*data.ExtendedNodeVersion) {
|
||||||
if len(allObjects) <= p.MaxKeys {
|
if len(allObjects) <= p.MaxKeys {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -366,7 +366,7 @@ func (n *Layer) putListLatestVersionsSession(ctx context.Context, p commonLatest
|
||||||
n.cache.PutListSession(n.BearerOwner(ctx), cacheKey, session)
|
n.cache.PutListSession(n.BearerOwner(ctx), cacheKey, session)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) putListAllVersionsSession(ctx context.Context, p commonVersionsListingParams, session *data.ListSession, allObjects []*data.ExtendedNodeVersion) {
|
func (n *layer) putListAllVersionsSession(ctx context.Context, p commonVersionsListingParams, session *data.ListSession, allObjects []*data.ExtendedNodeVersion) {
|
||||||
if len(allObjects) <= p.MaxKeys {
|
if len(allObjects) <= p.MaxKeys {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -498,7 +498,7 @@ func nodesGeneratorVersions(ctx context.Context, p commonVersionsListingParams,
|
||||||
return nodeCh, errCh
|
return nodeCh, errCh
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) initWorkerPool(ctx context.Context, size int, p commonVersionsListingParams, input <-chan *data.ExtendedNodeVersion) (<-chan *data.ExtendedNodeVersion, error) {
|
func (n *layer) initWorkerPool(ctx context.Context, size int, p commonVersionsListingParams, input <-chan *data.ExtendedNodeVersion) (<-chan *data.ExtendedNodeVersion, error) {
|
||||||
reqLog := n.reqLogger(ctx)
|
reqLog := n.reqLogger(ctx)
|
||||||
pool, err := ants.NewPool(size, ants.WithLogger(&logWrapper{reqLog}))
|
pool, err := ants.NewPool(size, ants.WithLogger(&logWrapper{reqLog}))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -637,7 +637,7 @@ func triageExtendedObjects(allObjects []*data.ExtendedNodeVersion) (prefixes []s
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo *data.BucketInfo, node *data.NodeVersion) (oi *data.ObjectInfo) {
|
func (n *layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo *data.BucketInfo, node *data.NodeVersion) (oi *data.ObjectInfo) {
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
if extInfo := n.cache.GetObject(owner, newAddress(bktInfo.CID, node.OID)); extInfo != nil {
|
if extInfo := n.cache.GetObject(owner, newAddress(bktInfo.CID, node.OID)); extInfo != nil {
|
||||||
return extInfo.ObjectInfo
|
return extInfo.ObjectInfo
|
||||||
|
|
|
@ -20,7 +20,7 @@ func TestObjectLockAttributes(t *testing.T) {
|
||||||
obj := tc.putObject([]byte("content obj1 v1"))
|
obj := tc.putObject([]byte("content obj1 v1"))
|
||||||
|
|
||||||
p := &PutLockInfoParams{
|
p := &PutLockInfoParams{
|
||||||
ObjVersion: &data.ObjectVersion{
|
ObjVersion: &ObjectVersion{
|
||||||
BktInfo: tc.bktInfo,
|
BktInfo: tc.bktInfo,
|
||||||
ObjectName: obj.Name,
|
ObjectName: obj.Name,
|
||||||
VersionID: obj.VersionID(),
|
VersionID: obj.VersionID(),
|
||||||
|
|
|
@ -36,6 +36,7 @@ const (
|
||||||
MultipartObjectSize = "S3-Multipart-Object-Size"
|
MultipartObjectSize = "S3-Multipart-Object-Size"
|
||||||
|
|
||||||
metaPrefix = "meta-"
|
metaPrefix = "meta-"
|
||||||
|
aclPrefix = "acl-"
|
||||||
|
|
||||||
MaxSizeUploadsList = 1000
|
MaxSizeUploadsList = 1000
|
||||||
MaxSizePartsList = 1000
|
MaxSizePartsList = 1000
|
||||||
|
@ -62,6 +63,7 @@ type (
|
||||||
|
|
||||||
UploadData struct {
|
UploadData struct {
|
||||||
TagSet map[string]string
|
TagSet map[string]string
|
||||||
|
ACLHeaders map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
UploadPartParams struct {
|
UploadPartParams struct {
|
||||||
|
@ -144,17 +146,13 @@ type (
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartParams) error {
|
func (n *layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartParams) error {
|
||||||
metaSize := len(p.Header)
|
metaSize := len(p.Header)
|
||||||
if p.Data != nil {
|
if p.Data != nil {
|
||||||
|
metaSize += len(p.Data.ACLHeaders)
|
||||||
metaSize += len(p.Data.TagSet)
|
metaSize += len(p.Data.TagSet)
|
||||||
}
|
}
|
||||||
|
|
||||||
networkInfo, err := n.frostFS.NetworkInfo(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("get network info: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
info := &data.MultipartInfo{
|
info := &data.MultipartInfo{
|
||||||
Key: p.Info.Key,
|
Key: p.Info.Key,
|
||||||
UploadID: p.Info.UploadID,
|
UploadID: p.Info.UploadID,
|
||||||
|
@ -162,7 +160,6 @@ func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
|
||||||
Created: TimeNow(ctx),
|
Created: TimeNow(ctx),
|
||||||
Meta: make(map[string]string, metaSize),
|
Meta: make(map[string]string, metaSize),
|
||||||
CopiesNumbers: p.CopiesNumbers,
|
CopiesNumbers: p.CopiesNumbers,
|
||||||
CreationEpoch: networkInfo.CurrentEpoch(),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for key, val := range p.Header {
|
for key, val := range p.Header {
|
||||||
|
@ -170,6 +167,10 @@ func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
|
||||||
}
|
}
|
||||||
|
|
||||||
if p.Data != nil {
|
if p.Data != nil {
|
||||||
|
for key, val := range p.Data.ACLHeaders {
|
||||||
|
info.Meta[aclPrefix+key] = val
|
||||||
|
}
|
||||||
|
|
||||||
for key, val := range p.Data.TagSet {
|
for key, val := range p.Data.TagSet {
|
||||||
info.Meta[tagPrefix+key] = val
|
info.Meta[tagPrefix+key] = val
|
||||||
}
|
}
|
||||||
|
@ -184,7 +185,7 @@ func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
|
||||||
return n.treeService.CreateMultipartUpload(ctx, p.Info.Bkt, info)
|
return n.treeService.CreateMultipartUpload(ctx, p.Info.Bkt, info)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, error) {
|
func (n *layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, error) {
|
||||||
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
|
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, ErrNodeNotFound) {
|
if errors.Is(err, ErrNodeNotFound) {
|
||||||
|
@ -205,7 +206,7 @@ func (n *Layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, er
|
||||||
return objInfo.ETag(n.features.MD5Enabled()), nil
|
return objInfo.ETag(n.features.MD5Enabled()), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) {
|
func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) {
|
||||||
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
||||||
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
||||||
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
|
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
|
||||||
|
@ -235,7 +236,7 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
prm.Attributes[0][0], prm.Attributes[0][1] = UploadIDAttributeName, p.Info.UploadID
|
prm.Attributes[0][0], prm.Attributes[0][1] = UploadIDAttributeName, p.Info.UploadID
|
||||||
prm.Attributes[1][0], prm.Attributes[1][1] = UploadPartNumberAttributeName, strconv.Itoa(p.PartNumber)
|
prm.Attributes[1][0], prm.Attributes[1][1] = UploadPartNumberAttributeName, strconv.Itoa(p.PartNumber)
|
||||||
|
|
||||||
createdObj, err := n.objectPutAndHash(ctx, prm, bktInfo)
|
size, id, hash, md5Hash, err := n.objectPutAndHash(ctx, prm, bktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -244,21 +245,21 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidDigest)
|
return nil, s3errors.GetAPIError(s3errors.ErrInvalidDigest)
|
||||||
}
|
}
|
||||||
if hex.EncodeToString(hashBytes) != hex.EncodeToString(createdObj.MD5Sum) {
|
if hex.EncodeToString(hashBytes) != hex.EncodeToString(md5Hash) {
|
||||||
prm := PrmObjectDelete{
|
prm := PrmObjectDelete{
|
||||||
Object: createdObj.ID,
|
Object: id,
|
||||||
Container: bktInfo.CID,
|
Container: bktInfo.CID,
|
||||||
}
|
}
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||||
err = n.frostFS.DeleteObject(ctx, prm)
|
err = n.frostFS.DeleteObject(ctx, prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
|
||||||
}
|
}
|
||||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidDigest)
|
return nil, s3errors.GetAPIError(s3errors.ErrInvalidDigest)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if p.Info.Encryption.Enabled() {
|
if p.Info.Encryption.Enabled() {
|
||||||
createdObj.Size = decSize
|
size = decSize
|
||||||
}
|
}
|
||||||
|
|
||||||
if !p.Info.Encryption.Enabled() && len(p.ContentSHA256Hash) > 0 && !auth.IsStandardContentSHA256(p.ContentSHA256Hash) {
|
if !p.Info.Encryption.Enabled() && len(p.ContentSHA256Hash) > 0 && !auth.IsStandardContentSHA256(p.ContentSHA256Hash) {
|
||||||
|
@ -266,10 +267,10 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
|
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
|
||||||
}
|
}
|
||||||
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
|
if !bytes.Equal(contentHashBytes, hash) {
|
||||||
err = n.objectDelete(ctx, bktInfo, createdObj.ID)
|
err = n.objectDelete(ctx, bktInfo, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
|
||||||
}
|
}
|
||||||
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
|
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
|
||||||
}
|
}
|
||||||
|
@ -277,36 +278,34 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
|
|
||||||
n.reqLogger(ctx).Debug(logs.UploadPart,
|
n.reqLogger(ctx).Debug(logs.UploadPart,
|
||||||
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
|
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
|
||||||
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
|
||||||
|
|
||||||
partInfo := &data.PartInfo{
|
partInfo := &data.PartInfo{
|
||||||
Key: p.Info.Key,
|
Key: p.Info.Key,
|
||||||
UploadID: p.Info.UploadID,
|
UploadID: p.Info.UploadID,
|
||||||
Number: p.PartNumber,
|
Number: p.PartNumber,
|
||||||
OID: createdObj.ID,
|
OID: id,
|
||||||
Size: createdObj.Size,
|
Size: size,
|
||||||
ETag: hex.EncodeToString(createdObj.HashSum),
|
ETag: hex.EncodeToString(hash),
|
||||||
Created: prm.CreationTime,
|
Created: prm.CreationTime,
|
||||||
MD5: hex.EncodeToString(createdObj.MD5Sum),
|
MD5: hex.EncodeToString(md5Hash),
|
||||||
}
|
}
|
||||||
|
|
||||||
oldPartIDs, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)
|
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)
|
||||||
oldPartIDNotFound := errors.Is(err, ErrNoNodeToRemove)
|
oldPartIDNotFound := errors.Is(err, ErrNoNodeToRemove)
|
||||||
if err != nil && !oldPartIDNotFound {
|
if err != nil && !oldPartIDNotFound {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if !oldPartIDNotFound {
|
if !oldPartIDNotFound {
|
||||||
for _, oldPartID := range oldPartIDs {
|
|
||||||
if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil {
|
if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil {
|
||||||
n.reqLogger(ctx).Error(logs.CouldntDeleteOldPartObject, zap.Error(err),
|
n.reqLogger(ctx).Error(logs.CouldntDeleteOldPartObject, zap.Error(err),
|
||||||
zap.String("cid", bktInfo.CID.EncodeToString()),
|
zap.String("cid", bktInfo.CID.EncodeToString()),
|
||||||
zap.String("oid", oldPartID.EncodeToString()))
|
zap.String("oid", oldPartID.EncodeToString()))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
objInfo := &data.ObjectInfo{
|
objInfo := &data.ObjectInfo{
|
||||||
ID: createdObj.ID,
|
ID: id,
|
||||||
CID: bktInfo.CID,
|
CID: bktInfo.CID,
|
||||||
|
|
||||||
Owner: bktInfo.Owner,
|
Owner: bktInfo.Owner,
|
||||||
|
@ -320,7 +319,7 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
return objInfo, nil
|
return objInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.ObjectInfo, error) {
|
func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.ObjectInfo, error) {
|
||||||
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
|
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, ErrNodeNotFound) {
|
if errors.Is(err, ErrNodeNotFound) {
|
||||||
|
@ -368,7 +367,7 @@ func (n *Layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
|
||||||
return n.uploadPart(ctx, multipartInfo, params)
|
return n.uploadPart(ctx, multipartInfo, params)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipartParams) (*UploadData, *data.ExtendedObjectInfo, error) {
|
func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipartParams) (*UploadData, *data.ExtendedObjectInfo, error) {
|
||||||
for i := 1; i < len(p.Parts); i++ {
|
for i := 1; i < len(p.Parts); i++ {
|
||||||
if p.Parts[i].PartNumber <= p.Parts[i-1].PartNumber {
|
if p.Parts[i].PartNumber <= p.Parts[i-1].PartNumber {
|
||||||
return nil, nil, s3errors.GetAPIError(s3errors.ErrInvalidPartOrder)
|
return nil, nil, s3errors.GetAPIError(s3errors.ErrInvalidPartOrder)
|
||||||
|
@ -387,15 +386,16 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
||||||
|
|
||||||
var multipartObjetSize uint64
|
var multipartObjetSize uint64
|
||||||
var encMultipartObjectSize uint64
|
var encMultipartObjectSize uint64
|
||||||
parts := make([]*data.PartInfoExtended, 0, len(p.Parts))
|
parts := make([]*data.PartInfo, 0, len(p.Parts))
|
||||||
|
|
||||||
var completedPartsHeader strings.Builder
|
var completedPartsHeader strings.Builder
|
||||||
md5Hash := md5.New()
|
md5Hash := md5.New()
|
||||||
for i, part := range p.Parts {
|
for i, part := range p.Parts {
|
||||||
partInfo := partsInfo.Extract(part.PartNumber, data.UnQuote(part.ETag), n.features.MD5Enabled())
|
partInfo := partsInfo[part.PartNumber]
|
||||||
if partInfo == nil {
|
if partInfo == nil || data.UnQuote(part.ETag) != partInfo.GetETag(n.features.MD5Enabled()) {
|
||||||
return nil, nil, fmt.Errorf("%w: unknown part %d or etag mismatched", s3errors.GetAPIError(s3errors.ErrInvalidPart), part.PartNumber)
|
return nil, nil, fmt.Errorf("%w: unknown part %d or etag mismatched", s3errors.GetAPIError(s3errors.ErrInvalidPart), part.PartNumber)
|
||||||
}
|
}
|
||||||
|
delete(partsInfo, part.PartNumber)
|
||||||
|
|
||||||
// for the last part we have no minimum size limit
|
// for the last part we have no minimum size limit
|
||||||
if i != len(p.Parts)-1 && partInfo.Size < UploadMinSize {
|
if i != len(p.Parts)-1 && partInfo.Size < UploadMinSize {
|
||||||
|
@ -433,12 +433,15 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
||||||
|
|
||||||
uploadData := &UploadData{
|
uploadData := &UploadData{
|
||||||
TagSet: make(map[string]string),
|
TagSet: make(map[string]string),
|
||||||
|
ACLHeaders: make(map[string]string),
|
||||||
}
|
}
|
||||||
for key, val := range multipartInfo.Meta {
|
for key, val := range multipartInfo.Meta {
|
||||||
if strings.HasPrefix(key, metaPrefix) {
|
if strings.HasPrefix(key, metaPrefix) {
|
||||||
initMetadata[strings.TrimPrefix(key, metaPrefix)] = val
|
initMetadata[strings.TrimPrefix(key, metaPrefix)] = val
|
||||||
} else if strings.HasPrefix(key, tagPrefix) {
|
} else if strings.HasPrefix(key, tagPrefix) {
|
||||||
uploadData.TagSet[strings.TrimPrefix(key, tagPrefix)] = val
|
uploadData.TagSet[strings.TrimPrefix(key, tagPrefix)] = val
|
||||||
|
} else if strings.HasPrefix(key, aclPrefix) {
|
||||||
|
uploadData.ACLHeaders[strings.TrimPrefix(key, aclPrefix)] = val
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -476,8 +479,7 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
||||||
|
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(p.Info.Bkt.CID)
|
addr.SetContainer(p.Info.Bkt.CID)
|
||||||
for _, prts := range partsInfo {
|
for _, partInfo := range partsInfo {
|
||||||
for _, partInfo := range prts {
|
|
||||||
if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil {
|
if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil {
|
||||||
n.reqLogger(ctx).Warn(logs.CouldNotDeleteUploadPart,
|
n.reqLogger(ctx).Warn(logs.CouldNotDeleteUploadPart,
|
||||||
zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID),
|
zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID),
|
||||||
|
@ -486,12 +488,11 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
||||||
addr.SetObject(partInfo.OID)
|
addr.SetObject(partInfo.OID)
|
||||||
n.cache.DeleteObject(addr)
|
n.cache.DeleteObject(addr)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return uploadData, extObjInfo, n.treeService.DeleteMultipartUpload(ctx, p.Info.Bkt, multipartInfo)
|
return uploadData, extObjInfo, n.treeService.DeleteMultipartUpload(ctx, p.Info.Bkt, multipartInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) ListMultipartUploads(ctx context.Context, p *ListMultipartUploadsParams) (*ListMultipartUploadsInfo, error) {
|
func (n *layer) ListMultipartUploads(ctx context.Context, p *ListMultipartUploadsParams) (*ListMultipartUploadsInfo, error) {
|
||||||
var result ListMultipartUploadsInfo
|
var result ListMultipartUploadsInfo
|
||||||
if p.MaxUploads == 0 {
|
if p.MaxUploads == 0 {
|
||||||
return &result, nil
|
return &result, nil
|
||||||
|
@ -551,25 +552,23 @@ func (n *Layer) ListMultipartUploads(ctx context.Context, p *ListMultipartUpload
|
||||||
return &result, nil
|
return &result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) AbortMultipartUpload(ctx context.Context, p *UploadInfoParams) error {
|
func (n *layer) AbortMultipartUpload(ctx context.Context, p *UploadInfoParams) error {
|
||||||
multipartInfo, parts, err := n.getUploadParts(ctx, p)
|
multipartInfo, parts, err := n.getUploadParts(ctx, p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, infos := range parts {
|
for _, info := range parts {
|
||||||
for _, info := range infos {
|
|
||||||
if err = n.objectDelete(ctx, p.Bkt, info.OID); err != nil {
|
if err = n.objectDelete(ctx, p.Bkt, info.OID); err != nil {
|
||||||
n.reqLogger(ctx).Warn(logs.CouldntDeletePart, zap.String("cid", p.Bkt.CID.EncodeToString()),
|
n.reqLogger(ctx).Warn(logs.CouldntDeletePart, zap.String("cid", p.Bkt.CID.EncodeToString()),
|
||||||
zap.String("oid", info.OID.EncodeToString()), zap.Int("part number", info.Number), zap.Error(err))
|
zap.String("oid", info.OID.EncodeToString()), zap.Int("part number", info.Number), zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return n.treeService.DeleteMultipartUpload(ctx, p.Bkt, multipartInfo)
|
return n.treeService.DeleteMultipartUpload(ctx, p.Bkt, multipartInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsInfo, error) {
|
func (n *layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsInfo, error) {
|
||||||
var res ListPartsInfo
|
var res ListPartsInfo
|
||||||
multipartInfo, partsInfo, err := n.getUploadParts(ctx, p.Info)
|
multipartInfo, partsInfo, err := n.getUploadParts(ctx, p.Info)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -586,12 +585,7 @@ func (n *Layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
|
||||||
|
|
||||||
parts := make([]*Part, 0, len(partsInfo))
|
parts := make([]*Part, 0, len(partsInfo))
|
||||||
|
|
||||||
for _, infos := range partsInfo {
|
for _, partInfo := range partsInfo {
|
||||||
sort.Slice(infos, func(i, j int) bool {
|
|
||||||
return infos[i].Timestamp < infos[j].Timestamp
|
|
||||||
})
|
|
||||||
|
|
||||||
partInfo := infos[len(infos)-1]
|
|
||||||
parts = append(parts, &Part{
|
parts = append(parts, &Part{
|
||||||
ETag: data.Quote(partInfo.GetETag(n.features.MD5Enabled())),
|
ETag: data.Quote(partInfo.GetETag(n.features.MD5Enabled())),
|
||||||
LastModified: partInfo.Created.UTC().Format(time.RFC3339),
|
LastModified: partInfo.Created.UTC().Format(time.RFC3339),
|
||||||
|
@ -619,31 +613,16 @@ func (n *Layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
|
||||||
|
|
||||||
if len(parts) > p.MaxParts {
|
if len(parts) > p.MaxParts {
|
||||||
res.IsTruncated = true
|
res.IsTruncated = true
|
||||||
|
res.NextPartNumberMarker = parts[p.MaxParts-1].PartNumber
|
||||||
parts = parts[:p.MaxParts]
|
parts = parts[:p.MaxParts]
|
||||||
}
|
}
|
||||||
|
|
||||||
res.NextPartNumberMarker = parts[len(parts)-1].PartNumber
|
|
||||||
res.Parts = parts
|
res.Parts = parts
|
||||||
|
|
||||||
return &res, nil
|
return &res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type PartsInfo map[int][]*data.PartInfoExtended
|
func (n *layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.MultipartInfo, map[int]*data.PartInfo, error) {
|
||||||
|
|
||||||
func (p PartsInfo) Extract(part int, etag string, md5Enabled bool) *data.PartInfoExtended {
|
|
||||||
parts := p[part]
|
|
||||||
|
|
||||||
for i, info := range parts {
|
|
||||||
if info.GetETag(md5Enabled) == etag {
|
|
||||||
p[part] = append(parts[:i], parts[i+1:]...)
|
|
||||||
return info
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.MultipartInfo, PartsInfo, error) {
|
|
||||||
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Bkt, p.Key, p.UploadID)
|
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Bkt, p.Key, p.UploadID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, ErrNodeNotFound) {
|
if errors.Is(err, ErrNodeNotFound) {
|
||||||
|
@ -657,11 +636,11 @@ func (n *Layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
res := make(map[int][]*data.PartInfoExtended, len(parts))
|
res := make(map[int]*data.PartInfo, len(parts))
|
||||||
partsNumbers := make([]int, len(parts))
|
partsNumbers := make([]int, len(parts))
|
||||||
oids := make([]string, len(parts))
|
oids := make([]string, len(parts))
|
||||||
for i, part := range parts {
|
for i, part := range parts {
|
||||||
res[part.Number] = append(res[part.Number], part)
|
res[part.Number] = part
|
||||||
partsNumbers[i] = part.Number
|
partsNumbers[i] = part.Number
|
||||||
oids[i] = part.OID.EncodeToString()
|
oids[i] = part.OID.EncodeToString()
|
||||||
}
|
}
|
||||||
|
|
89
api/layer/notifications.go
Normal file
89
api/layer/notifications.go
Normal file
|
@ -0,0 +1,89 @@
|
||||||
|
package layer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/xml"
|
||||||
|
errorsStd "errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
type PutBucketNotificationConfigurationParams struct {
|
||||||
|
RequestInfo *middleware.ReqInfo
|
||||||
|
BktInfo *data.BucketInfo
|
||||||
|
Configuration *data.NotificationConfiguration
|
||||||
|
CopiesNumbers []uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBucketNotificationConfigurationParams) error {
|
||||||
|
confXML, err := xml.Marshal(p.Configuration)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("marshal notify configuration: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prm := PrmObjectCreate{
|
||||||
|
Container: p.BktInfo.CID,
|
||||||
|
Payload: bytes.NewReader(confXML),
|
||||||
|
Filepath: p.BktInfo.NotificationConfigurationObjectName(),
|
||||||
|
CreationTime: TimeNow(ctx),
|
||||||
|
CopiesNumber: p.CopiesNumbers,
|
||||||
|
}
|
||||||
|
|
||||||
|
_, objID, _, _, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
objIDToDelete, err := n.treeService.PutNotificationConfigurationNode(ctx, p.BktInfo, objID)
|
||||||
|
objIDToDeleteNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
|
||||||
|
if err != nil && !objIDToDeleteNotFound {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !objIDToDeleteNotFound {
|
||||||
|
if err = n.objectDelete(ctx, p.BktInfo, objIDToDelete); err != nil {
|
||||||
|
n.reqLogger(ctx).Error(logs.CouldntDeleteNotificationConfigurationObject, zap.Error(err),
|
||||||
|
zap.String("cid", p.BktInfo.CID.EncodeToString()),
|
||||||
|
zap.String("oid", objIDToDelete.EncodeToString()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
n.cache.PutNotificationConfiguration(n.BearerOwner(ctx), p.BktInfo, p.Configuration)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *layer) GetBucketNotificationConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (*data.NotificationConfiguration, error) {
|
||||||
|
owner := n.BearerOwner(ctx)
|
||||||
|
if conf := n.cache.GetNotificationConfiguration(owner, bktInfo); conf != nil {
|
||||||
|
return conf, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
objID, err := n.treeService.GetNotificationConfigurationNode(ctx, bktInfo)
|
||||||
|
objIDNotFound := errorsStd.Is(err, ErrNodeNotFound)
|
||||||
|
if err != nil && !objIDNotFound {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
conf := &data.NotificationConfiguration{}
|
||||||
|
|
||||||
|
if !objIDNotFound {
|
||||||
|
obj, err := n.objectGet(ctx, bktInfo, objID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = xml.Unmarshal(obj.Payload(), &conf); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal notify configuration: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
n.cache.PutNotificationConfiguration(owner, bktInfo, conf)
|
||||||
|
|
||||||
|
return conf, nil
|
||||||
|
}
|
|
@ -67,18 +67,24 @@ func newAddress(cnr cid.ID, obj oid.ID) oid.Address {
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectHead returns all object's headers.
|
// objectHead returns all object's headers.
|
||||||
func (n *Layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) (*object.Object, error) {
|
func (n *layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) (*object.Object, error) {
|
||||||
prm := PrmObjectHead{
|
prm := PrmObjectRead{
|
||||||
Container: bktInfo.CID,
|
Container: bktInfo.CID,
|
||||||
Object: idObj,
|
Object: idObj,
|
||||||
|
WithHeader: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||||
|
|
||||||
return n.frostFS.HeadObject(ctx, prm)
|
res, err := n.frostFS.ReadObject(ctx, prm)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Reader, error) {
|
return res.Head, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Reader, error) {
|
||||||
if _, isCombined := p.objInfo.Headers[MultipartObjectSize]; !isCombined {
|
if _, isCombined := p.objInfo.Headers[MultipartObjectSize]; !isCombined {
|
||||||
return n.initFrostFSObjectPayloadReader(ctx, getFrostFSParams{
|
return n.initFrostFSObjectPayloadReader(ctx, getFrostFSParams{
|
||||||
off: p.off,
|
off: p.off,
|
||||||
|
@ -94,7 +100,7 @@ func (n *Layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Re
|
||||||
}
|
}
|
||||||
|
|
||||||
var parts []*data.PartInfo
|
var parts []*data.PartInfo
|
||||||
if err = json.NewDecoder(combinedObj.Payload).Decode(&parts); err != nil {
|
if err = json.Unmarshal(combinedObj.Payload(), &parts); err != nil {
|
||||||
return nil, fmt.Errorf("unmarshal combined object parts: %w", err)
|
return nil, fmt.Errorf("unmarshal combined object parts: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -125,28 +131,17 @@ func (n *Layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Re
|
||||||
|
|
||||||
// initializes payload reader of the FrostFS object.
|
// initializes payload reader of the FrostFS object.
|
||||||
// Zero range corresponds to full payload (panics if only offset is set).
|
// Zero range corresponds to full payload (panics if only offset is set).
|
||||||
func (n *Layer) initFrostFSObjectPayloadReader(ctx context.Context, p getFrostFSParams) (io.Reader, error) {
|
func (n *layer) initFrostFSObjectPayloadReader(ctx context.Context, p getFrostFSParams) (io.Reader, error) {
|
||||||
var prmAuth PrmAuth
|
prm := PrmObjectRead{
|
||||||
n.prepareAuthParameters(ctx, &prmAuth, p.bktInfo.Owner)
|
|
||||||
|
|
||||||
if p.off+p.ln != 0 {
|
|
||||||
prm := PrmObjectRange{
|
|
||||||
PrmAuth: prmAuth,
|
|
||||||
Container: p.bktInfo.CID,
|
Container: p.bktInfo.CID,
|
||||||
Object: p.oid,
|
Object: p.oid,
|
||||||
|
WithPayload: true,
|
||||||
PayloadRange: [2]uint64{p.off, p.ln},
|
PayloadRange: [2]uint64{p.off, p.ln},
|
||||||
}
|
}
|
||||||
|
|
||||||
return n.frostFS.RangeObject(ctx, prm)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, p.bktInfo.Owner)
|
||||||
}
|
|
||||||
|
|
||||||
prm := PrmObjectGet{
|
res, err := n.frostFS.ReadObject(ctx, prm)
|
||||||
PrmAuth: prmAuth,
|
|
||||||
Container: p.bktInfo.CID,
|
|
||||||
Object: p.oid,
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := n.frostFS.GetObject(ctx, prm)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -155,25 +150,22 @@ func (n *Layer) initFrostFSObjectPayloadReader(ctx context.Context, p getFrostFS
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectGet returns an object with payload in the object.
|
// objectGet returns an object with payload in the object.
|
||||||
func (n *Layer) objectGet(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (*Object, error) {
|
func (n *layer) objectGet(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (*object.Object, error) {
|
||||||
return n.objectGetBase(ctx, bktInfo, objID, PrmAuth{})
|
prm := PrmObjectRead{
|
||||||
}
|
|
||||||
|
|
||||||
// objectGetWithAuth returns an object with payload in the object. Uses provided PrmAuth.
|
|
||||||
func (n *Layer) objectGetWithAuth(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, auth PrmAuth) (*Object, error) {
|
|
||||||
return n.objectGetBase(ctx, bktInfo, objID, auth)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) objectGetBase(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, auth PrmAuth) (*Object, error) {
|
|
||||||
prm := PrmObjectGet{
|
|
||||||
PrmAuth: auth,
|
|
||||||
Container: bktInfo.CID,
|
Container: bktInfo.CID,
|
||||||
Object: objID,
|
Object: objID,
|
||||||
|
WithHeader: true,
|
||||||
|
WithPayload: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||||
|
|
||||||
return n.frostFS.GetObject(ctx, prm)
|
res, err := n.frostFS.ReadObject(ctx, prm)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.Head, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// MimeByFilePath detect mime type by file path extension.
|
// MimeByFilePath detect mime type by file path extension.
|
||||||
|
@ -222,7 +214,7 @@ func ParseCompletedPartHeader(hdr string) (*Part, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// PutObject stores object into FrostFS, took payload from io.Reader.
|
// PutObject stores object into FrostFS, took payload from io.Reader.
|
||||||
func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error) {
|
func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error) {
|
||||||
bktSettings, err := n.GetBucketSettings(ctx, p.BktInfo)
|
bktSettings, err := n.GetBucketSettings(ctx, p.BktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("couldn't get versioning settings object: %w", err)
|
return nil, fmt.Errorf("couldn't get versioning settings object: %w", err)
|
||||||
|
@ -271,7 +263,7 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
prm.Attributes = append(prm.Attributes, [2]string{k, v})
|
prm.Attributes = append(prm.Attributes, [2]string{k, v})
|
||||||
}
|
}
|
||||||
|
|
||||||
createdObj, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
|
size, id, hash, md5Hash, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -280,10 +272,10 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)
|
||||||
}
|
}
|
||||||
if !bytes.Equal(headerMd5Hash, createdObj.MD5Sum) {
|
if !bytes.Equal(headerMd5Hash, md5Hash) {
|
||||||
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
|
err = n.objectDelete(ctx, p.BktInfo, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
|
||||||
}
|
}
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)
|
||||||
}
|
}
|
||||||
|
@ -294,26 +286,25 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
||||||
}
|
}
|
||||||
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
|
if !bytes.Equal(contentHashBytes, hash) {
|
||||||
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
|
err = n.objectDelete(ctx, p.BktInfo, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
|
||||||
}
|
}
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
|
||||||
now := TimeNow(ctx)
|
now := TimeNow(ctx)
|
||||||
newVersion := &data.NodeVersion{
|
newVersion := &data.NodeVersion{
|
||||||
BaseNodeVersion: data.BaseNodeVersion{
|
BaseNodeVersion: data.BaseNodeVersion{
|
||||||
OID: createdObj.ID,
|
OID: id,
|
||||||
ETag: hex.EncodeToString(createdObj.HashSum),
|
ETag: hex.EncodeToString(hash),
|
||||||
FilePath: p.Object,
|
FilePath: p.Object,
|
||||||
Size: p.Size,
|
Size: p.Size,
|
||||||
Created: &now,
|
Created: &now,
|
||||||
Owner: &n.gateOwner,
|
Owner: &n.gateOwner,
|
||||||
CreationEpoch: createdObj.CreationEpoch,
|
|
||||||
},
|
},
|
||||||
IsUnversioned: !bktSettings.VersioningEnabled(),
|
IsUnversioned: !bktSettings.VersioningEnabled(),
|
||||||
IsCombined: p.Header[MultipartObjectSize] != "",
|
IsCombined: p.Header[MultipartObjectSize] != "",
|
||||||
|
@ -321,7 +312,7 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
if len(p.CompleteMD5Hash) > 0 {
|
if len(p.CompleteMD5Hash) > 0 {
|
||||||
newVersion.MD5 = p.CompleteMD5Hash
|
newVersion.MD5 = p.CompleteMD5Hash
|
||||||
} else {
|
} else {
|
||||||
newVersion.MD5 = hex.EncodeToString(createdObj.MD5Sum)
|
newVersion.MD5 = hex.EncodeToString(md5Hash)
|
||||||
}
|
}
|
||||||
|
|
||||||
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
|
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
|
||||||
|
@ -330,10 +321,10 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
|
|
||||||
if p.Lock != nil && (p.Lock.Retention != nil || p.Lock.LegalHold != nil) {
|
if p.Lock != nil && (p.Lock.Retention != nil || p.Lock.LegalHold != nil) {
|
||||||
putLockInfoPrms := &PutLockInfoParams{
|
putLockInfoPrms := &PutLockInfoParams{
|
||||||
ObjVersion: &data.ObjectVersion{
|
ObjVersion: &ObjectVersion{
|
||||||
BktInfo: p.BktInfo,
|
BktInfo: p.BktInfo,
|
||||||
ObjectName: p.Object,
|
ObjectName: p.Object,
|
||||||
VersionID: createdObj.ID.EncodeToString(),
|
VersionID: id.EncodeToString(),
|
||||||
},
|
},
|
||||||
NewLock: p.Lock,
|
NewLock: p.Lock,
|
||||||
CopiesNumbers: p.CopiesNumbers,
|
CopiesNumbers: p.CopiesNumbers,
|
||||||
|
@ -348,13 +339,13 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
n.cache.CleanListCacheEntriesContainingObject(p.Object, p.BktInfo.CID)
|
n.cache.CleanListCacheEntriesContainingObject(p.Object, p.BktInfo.CID)
|
||||||
|
|
||||||
objInfo := &data.ObjectInfo{
|
objInfo := &data.ObjectInfo{
|
||||||
ID: createdObj.ID,
|
ID: id,
|
||||||
CID: p.BktInfo.CID,
|
CID: p.BktInfo.CID,
|
||||||
|
|
||||||
Owner: n.gateOwner,
|
Owner: n.gateOwner,
|
||||||
Bucket: p.BktInfo.Name,
|
Bucket: p.BktInfo.Name,
|
||||||
Name: p.Object,
|
Name: p.Object,
|
||||||
Size: createdObj.Size,
|
Size: size,
|
||||||
Created: prm.CreationTime,
|
Created: prm.CreationTime,
|
||||||
Headers: p.Header,
|
Headers: p.Header,
|
||||||
ContentType: p.Header[api.ContentType],
|
ContentType: p.Header[api.ContentType],
|
||||||
|
@ -372,7 +363,7 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
return extendedObjInfo, nil
|
return extendedObjInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ExtendedObjectInfo, error) {
|
func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.BucketInfo, objectName string) (*data.ExtendedObjectInfo, error) {
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
if extObjInfo := n.cache.GetLastObject(owner, bkt.Name, objectName); extObjInfo != nil {
|
if extObjInfo := n.cache.GetLastObject(owner, bkt.Name, objectName); extObjInfo != nil {
|
||||||
return extObjInfo, nil
|
return extObjInfo, nil
|
||||||
|
@ -410,7 +401,7 @@ func (n *Layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.Bucke
|
||||||
return extObjInfo, nil
|
return extObjInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) {
|
func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) {
|
||||||
var err error
|
var err error
|
||||||
var foundVersion *data.NodeVersion
|
var foundVersion *data.NodeVersion
|
||||||
if p.VersionID == data.UnversionedObjectVersionID {
|
if p.VersionID == data.UnversionedObjectVersionID {
|
||||||
|
@ -468,18 +459,8 @@ func (n *Layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectDelete puts tombstone object into frostfs.
|
// objectDelete puts tombstone object into frostfs.
|
||||||
func (n *Layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) error {
|
func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) error {
|
||||||
return n.objectDeleteBase(ctx, bktInfo, idObj, PrmAuth{})
|
|
||||||
}
|
|
||||||
|
|
||||||
// objectDeleteWithAuth puts tombstone object into frostfs. Uses provided PrmAuth.
|
|
||||||
func (n *Layer) objectDeleteWithAuth(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID, auth PrmAuth) error {
|
|
||||||
return n.objectDeleteBase(ctx, bktInfo, idObj, auth)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) objectDeleteBase(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID, auth PrmAuth) error {
|
|
||||||
prm := PrmObjectDelete{
|
prm := PrmObjectDelete{
|
||||||
PrmAuth: auth,
|
|
||||||
Container: bktInfo.CID,
|
Container: bktInfo.CID,
|
||||||
Object: idObj,
|
Object: idObj,
|
||||||
}
|
}
|
||||||
|
@ -492,7 +473,8 @@ func (n *Layer) objectDeleteBase(ctx context.Context, bktInfo *data.BucketInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectPutAndHash prepare auth parameters and invoke frostfs.CreateObject.
|
// objectPutAndHash prepare auth parameters and invoke frostfs.CreateObject.
|
||||||
func (n *Layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (*data.CreatedObjectInfo, error) {
|
// Returns object ID and payload sha256 hash.
|
||||||
|
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (uint64, oid.ID, []byte, []byte, error) {
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||||
prm.ClientCut = n.features.ClientCut()
|
prm.ClientCut = n.features.ClientCut()
|
||||||
prm.BufferMaxSize = n.features.BufferMaxSizeForPut()
|
prm.BufferMaxSize = n.features.BufferMaxSizeForPut()
|
||||||
|
@ -505,21 +487,15 @@ func (n *Layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktIn
|
||||||
hash.Write(buf)
|
hash.Write(buf)
|
||||||
md5Hash.Write(buf)
|
md5Hash.Write(buf)
|
||||||
})
|
})
|
||||||
res, err := n.frostFS.CreateObject(ctx, prm)
|
id, err := n.frostFS.CreateObject(ctx, prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if _, errDiscard := io.Copy(io.Discard, prm.Payload); errDiscard != nil {
|
if _, errDiscard := io.Copy(io.Discard, prm.Payload); errDiscard != nil {
|
||||||
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard))
|
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard))
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, err
|
return 0, oid.ID{}, nil, nil, err
|
||||||
}
|
}
|
||||||
return &data.CreatedObjectInfo{
|
return size, id, hash.Sum(nil), md5Hash.Sum(nil), nil
|
||||||
ID: res.ObjectID,
|
|
||||||
Size: size,
|
|
||||||
HashSum: hash.Sum(nil),
|
|
||||||
MD5Sum: md5Hash.Sum(nil),
|
|
||||||
CreationEpoch: res.CreationEpoch,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type logWrapper struct {
|
type logWrapper struct {
|
||||||
|
|
|
@ -31,6 +31,8 @@ func TestWrapReader(t *testing.T) {
|
||||||
|
|
||||||
func TestGoroutinesDontLeakInPutAndHash(t *testing.T) {
|
func TestGoroutinesDontLeakInPutAndHash(t *testing.T) {
|
||||||
tc := prepareContext(t)
|
tc := prepareContext(t)
|
||||||
|
l, ok := tc.layer.(*layer)
|
||||||
|
require.True(t, ok)
|
||||||
|
|
||||||
content := make([]byte, 128*1024)
|
content := make([]byte, 128*1024)
|
||||||
_, err := rand.Read(content)
|
_, err := rand.Read(content)
|
||||||
|
@ -44,7 +46,7 @@ func TestGoroutinesDontLeakInPutAndHash(t *testing.T) {
|
||||||
|
|
||||||
expErr := errors.New("some error")
|
expErr := errors.New("some error")
|
||||||
tc.testFrostFS.SetObjectPutError(tc.obj, expErr)
|
tc.testFrostFS.SetObjectPutError(tc.obj, expErr)
|
||||||
_, err = tc.layer.objectPutAndHash(tc.ctx, prm, tc.bktInfo)
|
_, _, _, _, err = l.objectPutAndHash(tc.ctx, prm, tc.bktInfo)
|
||||||
require.ErrorIs(t, err, expErr)
|
require.ErrorIs(t, err, expErr)
|
||||||
require.Empty(t, payload.Len(), "body must be read out otherwise goroutines can leak in wrapReader")
|
require.Empty(t, payload.Len(), "body must be read out otherwise goroutines can leak in wrapReader")
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,264 +0,0 @@
|
||||||
package layer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
||||||
)
|
|
||||||
|
|
||||||
type PatchObjectParams struct {
|
|
||||||
Object *data.ExtendedObjectInfo
|
|
||||||
BktInfo *data.BucketInfo
|
|
||||||
NewBytes io.Reader
|
|
||||||
Range *RangeParams
|
|
||||||
VersioningEnabled bool
|
|
||||||
CopiesNumbers []uint32
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) PatchObject(ctx context.Context, p *PatchObjectParams) (*data.ExtendedObjectInfo, error) {
|
|
||||||
if p.Object.ObjectInfo.Headers[AttributeDecryptedSize] != "" {
|
|
||||||
return nil, fmt.Errorf("patch encrypted object")
|
|
||||||
}
|
|
||||||
|
|
||||||
if p.Object.ObjectInfo.Headers[MultipartObjectSize] != "" {
|
|
||||||
return n.patchMultipartObject(ctx, p)
|
|
||||||
}
|
|
||||||
|
|
||||||
prmPatch := PrmObjectPatch{
|
|
||||||
Container: p.BktInfo.CID,
|
|
||||||
Object: p.Object.ObjectInfo.ID,
|
|
||||||
Payload: p.NewBytes,
|
|
||||||
Offset: p.Range.Start,
|
|
||||||
Length: p.Range.End - p.Range.Start + 1,
|
|
||||||
ObjectSize: p.Object.ObjectInfo.Size,
|
|
||||||
}
|
|
||||||
n.prepareAuthParameters(ctx, &prmPatch.PrmAuth, p.BktInfo.Owner)
|
|
||||||
|
|
||||||
createdObj, err := n.patchObject(ctx, prmPatch)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("patch object: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
newVersion := &data.NodeVersion{
|
|
||||||
BaseNodeVersion: data.BaseNodeVersion{
|
|
||||||
OID: createdObj.ID,
|
|
||||||
ETag: hex.EncodeToString(createdObj.HashSum),
|
|
||||||
FilePath: p.Object.ObjectInfo.Name,
|
|
||||||
Size: createdObj.Size,
|
|
||||||
Created: &p.Object.ObjectInfo.Created,
|
|
||||||
Owner: &n.gateOwner,
|
|
||||||
CreationEpoch: p.Object.NodeVersion.CreationEpoch,
|
|
||||||
},
|
|
||||||
IsUnversioned: !p.VersioningEnabled,
|
|
||||||
IsCombined: p.Object.ObjectInfo.Headers[MultipartObjectSize] != "",
|
|
||||||
}
|
|
||||||
|
|
||||||
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
|
|
||||||
return nil, fmt.Errorf("couldn't add new version to tree service: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
p.Object.ObjectInfo.ID = createdObj.ID
|
|
||||||
p.Object.ObjectInfo.Size = createdObj.Size
|
|
||||||
p.Object.ObjectInfo.MD5Sum = ""
|
|
||||||
p.Object.ObjectInfo.HashSum = hex.EncodeToString(createdObj.HashSum)
|
|
||||||
p.Object.NodeVersion = newVersion
|
|
||||||
|
|
||||||
return p.Object, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) patchObject(ctx context.Context, p PrmObjectPatch) (*data.CreatedObjectInfo, error) {
|
|
||||||
objID, err := n.frostFS.PatchObject(ctx, p)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("patch object: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
prmHead := PrmObjectHead{
|
|
||||||
PrmAuth: p.PrmAuth,
|
|
||||||
Container: p.Container,
|
|
||||||
Object: objID,
|
|
||||||
}
|
|
||||||
obj, err := n.frostFS.HeadObject(ctx, prmHead)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("head object: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
payloadChecksum, _ := obj.PayloadChecksum()
|
|
||||||
|
|
||||||
return &data.CreatedObjectInfo{
|
|
||||||
ID: objID,
|
|
||||||
Size: obj.PayloadSize(),
|
|
||||||
HashSum: payloadChecksum.Value(),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) patchMultipartObject(ctx context.Context, p *PatchObjectParams) (*data.ExtendedObjectInfo, error) {
|
|
||||||
combinedObj, err := n.objectGet(ctx, p.BktInfo, p.Object.ObjectInfo.ID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("get combined object '%s': %w", p.Object.ObjectInfo.ID.EncodeToString(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var parts []*data.PartInfo
|
|
||||||
if err = json.NewDecoder(combinedObj.Payload).Decode(&parts); err != nil {
|
|
||||||
return nil, fmt.Errorf("unmarshal combined object parts: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
prmPatch := PrmObjectPatch{
|
|
||||||
Container: p.BktInfo.CID,
|
|
||||||
}
|
|
||||||
n.prepareAuthParameters(ctx, &prmPatch.PrmAuth, p.BktInfo.Owner)
|
|
||||||
|
|
||||||
off, ln := p.Range.Start, p.Range.End-p.Range.Start+1
|
|
||||||
var multipartObjectSize uint64
|
|
||||||
for i, part := range parts {
|
|
||||||
if off > part.Size || (off == part.Size && i != len(parts)-1) || ln == 0 {
|
|
||||||
multipartObjectSize += part.Size
|
|
||||||
if ln != 0 {
|
|
||||||
off -= part.Size
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
var createdObj *data.CreatedObjectInfo
|
|
||||||
createdObj, off, ln, err = n.patchPart(ctx, part, p, &prmPatch, off, ln, i == len(parts)-1)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("patch part: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
parts[i].OID = createdObj.ID
|
|
||||||
parts[i].Size = createdObj.Size
|
|
||||||
parts[i].MD5 = ""
|
|
||||||
parts[i].ETag = hex.EncodeToString(createdObj.HashSum)
|
|
||||||
|
|
||||||
multipartObjectSize += createdObj.Size
|
|
||||||
}
|
|
||||||
|
|
||||||
return n.updateCombinedObject(ctx, parts, multipartObjectSize, p)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Returns patched part info, updated offset and length.
|
|
||||||
func (n *Layer) patchPart(ctx context.Context, part *data.PartInfo, p *PatchObjectParams, prmPatch *PrmObjectPatch, off, ln uint64, lastPart bool) (*data.CreatedObjectInfo, uint64, uint64, error) {
|
|
||||||
if off == 0 && ln >= part.Size {
|
|
||||||
curLen := part.Size
|
|
||||||
if lastPart {
|
|
||||||
curLen = ln
|
|
||||||
}
|
|
||||||
prm := PrmObjectCreate{
|
|
||||||
Container: p.BktInfo.CID,
|
|
||||||
Payload: io.LimitReader(p.NewBytes, int64(curLen)),
|
|
||||||
CreationTime: part.Created,
|
|
||||||
CopiesNumber: p.CopiesNumbers,
|
|
||||||
}
|
|
||||||
|
|
||||||
createdObj, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
|
|
||||||
if err != nil {
|
|
||||||
return nil, 0, 0, fmt.Errorf("put new part object '%s': %w", part.OID.EncodeToString(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ln -= curLen
|
|
||||||
|
|
||||||
return createdObj, off, ln, err
|
|
||||||
}
|
|
||||||
|
|
||||||
curLen := ln
|
|
||||||
if off+curLen > part.Size && !lastPart {
|
|
||||||
curLen = part.Size - off
|
|
||||||
}
|
|
||||||
prmPatch.Object = part.OID
|
|
||||||
prmPatch.ObjectSize = part.Size
|
|
||||||
prmPatch.Offset = off
|
|
||||||
prmPatch.Length = curLen
|
|
||||||
|
|
||||||
prmPatch.Payload = io.LimitReader(p.NewBytes, int64(prmPatch.Length))
|
|
||||||
|
|
||||||
createdObj, err := n.patchObject(ctx, *prmPatch)
|
|
||||||
if err != nil {
|
|
||||||
return nil, 0, 0, fmt.Errorf("patch part object '%s': %w", part.OID.EncodeToString(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ln -= curLen
|
|
||||||
off = 0
|
|
||||||
|
|
||||||
return createdObj, off, ln, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) updateCombinedObject(ctx context.Context, parts []*data.PartInfo, fullObjSize uint64, p *PatchObjectParams) (*data.ExtendedObjectInfo, error) {
|
|
||||||
newParts, err := json.Marshal(parts)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("marshal parts for combined object: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var headerParts strings.Builder
|
|
||||||
for i, part := range parts {
|
|
||||||
headerPart := part.ToHeaderString()
|
|
||||||
if i != len(parts)-1 {
|
|
||||||
headerPart += ","
|
|
||||||
}
|
|
||||||
headerParts.WriteString(headerPart)
|
|
||||||
}
|
|
||||||
|
|
||||||
prm := PrmObjectCreate{
|
|
||||||
Container: p.BktInfo.CID,
|
|
||||||
PayloadSize: fullObjSize,
|
|
||||||
Filepath: p.Object.ObjectInfo.Name,
|
|
||||||
Payload: bytes.NewReader(newParts),
|
|
||||||
CreationTime: p.Object.ObjectInfo.Created,
|
|
||||||
CopiesNumber: p.CopiesNumbers,
|
|
||||||
}
|
|
||||||
|
|
||||||
prm.Attributes = make([][2]string, 0, len(p.Object.ObjectInfo.Headers)+1)
|
|
||||||
|
|
||||||
for k, v := range p.Object.ObjectInfo.Headers {
|
|
||||||
switch k {
|
|
||||||
case MultipartObjectSize:
|
|
||||||
prm.Attributes = append(prm.Attributes, [2]string{MultipartObjectSize, strconv.FormatUint(fullObjSize, 10)})
|
|
||||||
case UploadCompletedParts:
|
|
||||||
prm.Attributes = append(prm.Attributes, [2]string{UploadCompletedParts, headerParts.String()})
|
|
||||||
case api.ContentType:
|
|
||||||
default:
|
|
||||||
prm.Attributes = append(prm.Attributes, [2]string{k, v})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
prm.Attributes = append(prm.Attributes, [2]string{api.ContentType, p.Object.ObjectInfo.ContentType})
|
|
||||||
|
|
||||||
createdObj, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("put new combined object: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
newVersion := &data.NodeVersion{
|
|
||||||
BaseNodeVersion: data.BaseNodeVersion{
|
|
||||||
OID: createdObj.ID,
|
|
||||||
ETag: hex.EncodeToString(createdObj.HashSum),
|
|
||||||
MD5: hex.EncodeToString(createdObj.MD5Sum) + "-" + strconv.Itoa(len(parts)),
|
|
||||||
FilePath: p.Object.ObjectInfo.Name,
|
|
||||||
Size: fullObjSize,
|
|
||||||
Created: &p.Object.ObjectInfo.Created,
|
|
||||||
Owner: &n.gateOwner,
|
|
||||||
CreationEpoch: p.Object.NodeVersion.CreationEpoch,
|
|
||||||
},
|
|
||||||
IsUnversioned: !p.VersioningEnabled,
|
|
||||||
IsCombined: p.Object.ObjectInfo.Headers[MultipartObjectSize] != "",
|
|
||||||
}
|
|
||||||
|
|
||||||
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
|
|
||||||
return nil, fmt.Errorf("couldn't add new version to tree service: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
p.Object.ObjectInfo.ID = createdObj.ID
|
|
||||||
p.Object.ObjectInfo.Size = createdObj.Size
|
|
||||||
p.Object.ObjectInfo.MD5Sum = hex.EncodeToString(createdObj.MD5Sum) + "-" + strconv.Itoa(len(parts))
|
|
||||||
p.Object.ObjectInfo.HashSum = hex.EncodeToString(createdObj.HashSum)
|
|
||||||
p.Object.ObjectInfo.Headers[MultipartObjectSize] = strconv.FormatUint(fullObjSize, 10)
|
|
||||||
p.Object.ObjectInfo.Headers[UploadCompletedParts] = headerParts.String()
|
|
||||||
p.Object.NodeVersion = newVersion
|
|
||||||
|
|
||||||
return p.Object, nil
|
|
||||||
}
|
|
|
@ -12,7 +12,6 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -21,13 +20,13 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
type PutLockInfoParams struct {
|
type PutLockInfoParams struct {
|
||||||
ObjVersion *data.ObjectVersion
|
ObjVersion *ObjectVersion
|
||||||
NewLock *data.ObjectLock
|
NewLock *data.ObjectLock
|
||||||
CopiesNumbers []uint32
|
CopiesNumbers []uint32
|
||||||
NodeVersion *data.NodeVersion // optional
|
NodeVersion *data.NodeVersion // optional
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err error) {
|
func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err error) {
|
||||||
newLock := p.NewLock
|
newLock := p.NewLock
|
||||||
versionNode := p.NodeVersion
|
versionNode := p.NodeVersion
|
||||||
// sometimes node version can be provided from executing context
|
// sometimes node version can be provided from executing context
|
||||||
|
@ -101,7 +100,7 @@ func (n *Layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion *data.ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
|
func (n *layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion *ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
|
||||||
// check cache if node version is stored inside extendedObjectVersion
|
// check cache if node version is stored inside extendedObjectVersion
|
||||||
nodeVersion = n.getNodeVersionFromCache(n.BearerOwner(ctx), objVersion)
|
nodeVersion = n.getNodeVersionFromCache(n.BearerOwner(ctx), objVersion)
|
||||||
if nodeVersion == nil {
|
if nodeVersion == nil {
|
||||||
|
@ -112,7 +111,7 @@ func (n *Layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion
|
||||||
return nodeVersion, nil
|
return nodeVersion, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, lock *data.ObjectLock, copiesNumber []uint32) (oid.ID, error) {
|
func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, lock *data.ObjectLock, copiesNumber []uint32) (oid.ID, error) {
|
||||||
prm := PrmObjectCreate{
|
prm := PrmObjectCreate{
|
||||||
Container: bktInfo.CID,
|
Container: bktInfo.CID,
|
||||||
Locks: []oid.ID{objID},
|
Locks: []oid.ID{objID},
|
||||||
|
@ -126,15 +125,11 @@ func (n *Layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
|
||||||
return oid.ID{}, err
|
return oid.ID{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
createdObj, err := n.objectPutAndHash(ctx, prm, bktInfo)
|
_, id, _, _, err := n.objectPutAndHash(ctx, prm, bktInfo)
|
||||||
if err != nil {
|
return id, err
|
||||||
return oid.ID{}, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return createdObj.ID, nil
|
func (n *layer) GetLockInfo(ctx context.Context, objVersion *ObjectVersion) (*data.LockInfo, error) {
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion) (*data.LockInfo, error) {
|
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
if lockInfo := n.cache.GetLockInfo(owner, lockObjectKey(objVersion)); lockInfo != nil {
|
if lockInfo := n.cache.GetLockInfo(owner, lockObjectKey(objVersion)); lockInfo != nil {
|
||||||
return lockInfo, nil
|
return lockInfo, nil
|
||||||
|
@ -158,36 +153,30 @@ func (n *Layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion)
|
||||||
return lockInfo, nil
|
return lockInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) getCORS(ctx context.Context, bkt *data.BucketInfo) (*data.CORSConfiguration, error) {
|
func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo) (*data.CORSConfiguration, error) {
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
if cors := n.cache.GetCORS(owner, bkt); cors != nil {
|
if cors := n.cache.GetCORS(owner, bkt); cors != nil {
|
||||||
return cors, nil
|
return cors, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
addr, err := n.treeService.GetBucketCORS(ctx, bkt)
|
objID, err := n.treeService.GetBucketCORS(ctx, bkt)
|
||||||
objNotFound := errorsStd.Is(err, ErrNodeNotFound)
|
objIDNotFound := errorsStd.Is(err, ErrNodeNotFound)
|
||||||
if err != nil && !objNotFound {
|
if err != nil && !objIDNotFound {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if objNotFound {
|
if objIDNotFound {
|
||||||
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchCORSConfiguration), err.Error())
|
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchCORSConfiguration), err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
var prmAuth PrmAuth
|
obj, err := n.objectGet(ctx, bkt, objID)
|
||||||
corsBkt := bkt
|
|
||||||
if !addr.Container().Equals(bkt.CID) && !addr.Container().Equals(cid.ID{}) {
|
|
||||||
corsBkt = &data.BucketInfo{CID: addr.Container()}
|
|
||||||
prmAuth.PrivateKey = &n.gateKey.PrivateKey
|
|
||||||
}
|
|
||||||
|
|
||||||
obj, err := n.objectGetWithAuth(ctx, corsBkt, addr.Object(), prmAuth)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("get cors object: %w", err)
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
cors := &data.CORSConfiguration{}
|
cors := &data.CORSConfiguration{}
|
||||||
if err = xml.NewDecoder(obj.Payload).Decode(&cors); err != nil {
|
|
||||||
|
if err = xml.Unmarshal(obj.Payload(), &cors); err != nil {
|
||||||
return nil, fmt.Errorf("unmarshal cors: %w", err)
|
return nil, fmt.Errorf("unmarshal cors: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -196,12 +185,12 @@ func (n *Layer) getCORS(ctx context.Context, bkt *data.BucketInfo) (*data.CORSCo
|
||||||
return cors, nil
|
return cors, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func lockObjectKey(objVersion *data.ObjectVersion) string {
|
func lockObjectKey(objVersion *ObjectVersion) string {
|
||||||
// todo reconsider forming name since versionID can be "null" or ""
|
// todo reconsider forming name since versionID can be "null" or ""
|
||||||
return ".lock." + objVersion.BktInfo.CID.EncodeToString() + "." + objVersion.ObjectName + "." + objVersion.VersionID
|
return ".lock." + objVersion.BktInfo.CID.EncodeToString() + "." + objVersion.ObjectName + "." + objVersion.VersionID
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
|
func (n *layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
if settings := n.cache.GetSettings(owner, bktInfo); settings != nil {
|
if settings := n.cache.GetSettings(owner, bktInfo); settings != nil {
|
||||||
return settings, nil
|
return settings, nil
|
||||||
|
@ -220,7 +209,7 @@ func (n *Layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo)
|
||||||
return settings, nil
|
return settings, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) PutBucketSettings(ctx context.Context, p *PutSettingsParams) error {
|
func (n *layer) PutBucketSettings(ctx context.Context, p *PutSettingsParams) error {
|
||||||
if err := n.treeService.PutSettingsNode(ctx, p.BktInfo, p.Settings); err != nil {
|
if err := n.treeService.PutSettingsNode(ctx, p.BktInfo, p.Settings); err != nil {
|
||||||
return fmt.Errorf("failed to get settings node: %w", err)
|
return fmt.Errorf("failed to get settings node: %w", err)
|
||||||
}
|
}
|
||||||
|
@ -230,7 +219,7 @@ func (n *Layer) PutBucketSettings(ctx context.Context, p *PutSettingsParams) err
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) ([][2]string, error) {
|
func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) ([][2]string, error) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
expEpoch uint64
|
expEpoch uint64
|
||||||
|
|
|
@ -14,7 +14,22 @@ import (
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *Layer) GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error) {
|
type GetObjectTaggingParams struct {
|
||||||
|
ObjectVersion *ObjectVersion
|
||||||
|
|
||||||
|
// NodeVersion can be nil. If not nil we save one request to tree service.
|
||||||
|
NodeVersion *data.NodeVersion // optional
|
||||||
|
}
|
||||||
|
|
||||||
|
type PutObjectTaggingParams struct {
|
||||||
|
ObjectVersion *ObjectVersion
|
||||||
|
TagSet map[string]string
|
||||||
|
|
||||||
|
// NodeVersion can be nil. If not nil we save one request to tree service.
|
||||||
|
NodeVersion *data.NodeVersion // optional
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams) (string, map[string]string, error) {
|
||||||
var err error
|
var err error
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
|
|
||||||
|
@ -50,12 +65,12 @@ func (n *Layer) GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingPa
|
||||||
return p.ObjectVersion.VersionID, tags, nil
|
return p.ObjectVersion.VersionID, tags, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingParams) (err error) {
|
func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (nodeVersion *data.NodeVersion, err error) {
|
||||||
nodeVersion := p.NodeVersion
|
nodeVersion = p.NodeVersion
|
||||||
if nodeVersion == nil {
|
if nodeVersion == nil {
|
||||||
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
|
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
p.ObjectVersion.VersionID = nodeVersion.OID.EncodeToString()
|
p.ObjectVersion.VersionID = nodeVersion.OID.EncodeToString()
|
||||||
|
@ -63,38 +78,38 @@ func (n *Layer) PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingPa
|
||||||
err = n.treeService.PutObjectTagging(ctx, p.ObjectVersion.BktInfo, nodeVersion, p.TagSet)
|
err = n.treeService.PutObjectTagging(ctx, p.ObjectVersion.BktInfo, nodeVersion, p.TagSet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, ErrNodeNotFound) {
|
if errors.Is(err, ErrNodeNotFound) {
|
||||||
return fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
|
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
|
||||||
}
|
}
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
n.cache.PutTagging(n.BearerOwner(ctx), objectTaggingCacheKey(p.ObjectVersion), p.TagSet)
|
n.cache.PutTagging(n.BearerOwner(ctx), objectTaggingCacheKey(p.ObjectVersion), p.TagSet)
|
||||||
|
|
||||||
return nil
|
return nodeVersion, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion) error {
|
func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) (*data.NodeVersion, error) {
|
||||||
version, err := n.getNodeVersion(ctx, p)
|
version, err := n.getNodeVersion(ctx, p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = n.treeService.DeleteObjectTagging(ctx, p.BktInfo, version)
|
err = n.treeService.DeleteObjectTagging(ctx, p.BktInfo, version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, ErrNodeNotFound) {
|
if errors.Is(err, ErrNodeNotFound) {
|
||||||
return fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
|
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
|
||||||
}
|
}
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
p.VersionID = version.OID.EncodeToString()
|
p.VersionID = version.OID.EncodeToString()
|
||||||
|
|
||||||
n.cache.DeleteTagging(objectTaggingCacheKey(p))
|
n.cache.DeleteTagging(objectTaggingCacheKey(p))
|
||||||
|
|
||||||
return nil
|
return version, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) {
|
func (n *layer) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) {
|
||||||
owner := n.BearerOwner(ctx)
|
owner := n.BearerOwner(ctx)
|
||||||
|
|
||||||
if tags := n.cache.GetTagging(owner, bucketTaggingCacheKey(bktInfo.CID)); tags != nil {
|
if tags := n.cache.GetTagging(owner, bucketTaggingCacheKey(bktInfo.CID)); tags != nil {
|
||||||
|
@ -111,7 +126,7 @@ func (n *Layer) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo)
|
||||||
return tags, nil
|
return tags, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error {
|
func (n *layer) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error {
|
||||||
if err := n.treeService.PutBucketTagging(ctx, bktInfo, tagSet); err != nil {
|
if err := n.treeService.PutBucketTagging(ctx, bktInfo, tagSet); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -121,13 +136,13 @@ func (n *Layer) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo,
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error {
|
func (n *layer) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||||
n.cache.DeleteTagging(bucketTaggingCacheKey(bktInfo.CID))
|
n.cache.DeleteTagging(bucketTaggingCacheKey(bktInfo.CID))
|
||||||
|
|
||||||
return n.treeService.DeleteBucketTagging(ctx, bktInfo)
|
return n.treeService.DeleteBucketTagging(ctx, bktInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
func objectTaggingCacheKey(p *data.ObjectVersion) string {
|
func objectTaggingCacheKey(p *ObjectVersion) string {
|
||||||
return ".tagset." + p.BktInfo.CID.EncodeToString() + "." + p.ObjectName + "." + p.VersionID
|
return ".tagset." + p.BktInfo.CID.EncodeToString() + "." + p.ObjectName + "." + p.VersionID
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -135,7 +150,7 @@ func bucketTaggingCacheKey(cnrID cid.ID) string {
|
||||||
return ".tagset." + cnrID.EncodeToString()
|
return ".tagset." + cnrID.EncodeToString()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) getNodeVersion(ctx context.Context, objVersion *data.ObjectVersion) (*data.NodeVersion, error) {
|
func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (*data.NodeVersion, error) {
|
||||||
var err error
|
var err error
|
||||||
var version *data.NodeVersion
|
var version *data.NodeVersion
|
||||||
|
|
||||||
|
@ -173,7 +188,7 @@ func (n *Layer) getNodeVersion(ctx context.Context, objVersion *data.ObjectVersi
|
||||||
return version, err
|
return version, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) getNodeVersionFromCache(owner user.ID, o *data.ObjectVersion) *data.NodeVersion {
|
func (n *layer) getNodeVersionFromCache(owner user.ID, o *ObjectVersion) *data.NodeVersion {
|
||||||
if len(o.VersionID) == 0 || o.VersionID == data.UnversionedObjectVersionID {
|
if len(o.VersionID) == 0 || o.VersionID == data.UnversionedObjectVersionID {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,7 +6,6 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
@ -34,7 +33,7 @@ type TreeServiceMock struct {
|
||||||
locks map[string]map[uint64]*data.LockInfo
|
locks map[string]map[uint64]*data.LockInfo
|
||||||
tags map[string]map[uint64]map[string]string
|
tags map[string]map[uint64]map[string]string
|
||||||
multiparts map[string]map[string][]*data.MultipartInfo
|
multiparts map[string]map[string][]*data.MultipartInfo
|
||||||
parts map[string]map[int]*data.PartInfoExtended
|
parts map[string]map[int]*data.PartInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
|
func (t *TreeServiceMock) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
|
||||||
|
@ -93,7 +92,7 @@ func NewTreeService() *TreeServiceMock {
|
||||||
locks: make(map[string]map[uint64]*data.LockInfo),
|
locks: make(map[string]map[uint64]*data.LockInfo),
|
||||||
tags: make(map[string]map[uint64]map[string]string),
|
tags: make(map[string]map[uint64]map[string]string),
|
||||||
multiparts: make(map[string]map[string][]*data.MultipartInfo),
|
multiparts: make(map[string]map[string][]*data.MultipartInfo),
|
||||||
parts: make(map[string]map[int]*data.PartInfoExtended),
|
parts: make(map[string]map[int]*data.PartInfo),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -111,39 +110,44 @@ func (t *TreeServiceMock) GetSettingsNode(_ context.Context, bktInfo *data.Bucke
|
||||||
return settings, nil
|
return settings, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) GetBucketCORS(_ context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
|
func (t *TreeServiceMock) GetNotificationConfigurationNode(context.Context, *data.BucketInfo) (oid.ID, error) {
|
||||||
|
panic("implement me")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *TreeServiceMock) PutNotificationConfigurationNode(context.Context, *data.BucketInfo, oid.ID) (oid.ID, error) {
|
||||||
|
panic("implement me")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *TreeServiceMock) GetBucketCORS(_ context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
|
||||||
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||||
if !ok {
|
if !ok {
|
||||||
return oid.Address{}, nil
|
return oid.ID{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
node, ok := systemMap["cors"]
|
node, ok := systemMap["cors"]
|
||||||
if !ok {
|
if !ok {
|
||||||
return oid.Address{}, nil
|
return oid.ID{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var addr oid.Address
|
return node.OID, nil
|
||||||
addr.SetContainer(bktInfo.CID)
|
|
||||||
addr.SetObject(node.OID)
|
|
||||||
return addr, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) PutBucketCORS(_ context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
|
func (t *TreeServiceMock) PutBucketCORS(_ context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) {
|
||||||
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||||
if !ok {
|
if !ok {
|
||||||
systemMap = make(map[string]*data.BaseNodeVersion)
|
systemMap = make(map[string]*data.BaseNodeVersion)
|
||||||
}
|
}
|
||||||
|
|
||||||
systemMap["cors"] = &data.BaseNodeVersion{
|
systemMap["cors"] = &data.BaseNodeVersion{
|
||||||
OID: addr.Object(),
|
OID: objID,
|
||||||
}
|
}
|
||||||
|
|
||||||
t.system[bktInfo.CID.EncodeToString()] = systemMap
|
t.system[bktInfo.CID.EncodeToString()] = systemMap
|
||||||
|
|
||||||
return nil, ErrNoNodeToRemove
|
return oid.ID{}, ErrNoNodeToRemove
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) DeleteBucketCORS(context.Context, *data.BucketInfo) ([]oid.Address, error) {
|
func (t *TreeServiceMock) DeleteBucketCORS(context.Context, *data.BucketInfo) (oid.ID, error) {
|
||||||
panic("implement me")
|
panic("implement me")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -347,31 +351,28 @@ func (t *TreeServiceMock) GetMultipartUpload(_ context.Context, bktInfo *data.Bu
|
||||||
return nil, ErrNodeNotFound
|
return nil, ErrNodeNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDsToDelete []oid.ID, err error) {
|
func (t *TreeServiceMock) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error) {
|
||||||
multipartInfo, err := t.GetMultipartUpload(ctx, bktInfo, info.Key, info.UploadID)
|
multipartInfo, err := t.GetMultipartUpload(ctx, bktInfo, info.Key, info.UploadID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return oid.ID{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if multipartInfo.ID != multipartNodeID {
|
if multipartInfo.ID != multipartNodeID {
|
||||||
return nil, fmt.Errorf("invalid multipart info id")
|
return oid.ID{}, fmt.Errorf("invalid multipart info id")
|
||||||
}
|
}
|
||||||
|
|
||||||
partsMap, ok := t.parts[info.UploadID]
|
partsMap, ok := t.parts[info.UploadID]
|
||||||
if !ok {
|
if !ok {
|
||||||
partsMap = make(map[int]*data.PartInfoExtended)
|
partsMap = make(map[int]*data.PartInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
partsMap[info.Number] = &data.PartInfoExtended{
|
partsMap[info.Number] = info
|
||||||
PartInfo: *info,
|
|
||||||
Timestamp: uint64(time.Now().UnixMicro()),
|
|
||||||
}
|
|
||||||
|
|
||||||
t.parts[info.UploadID] = partsMap
|
t.parts[info.UploadID] = partsMap
|
||||||
return nil, nil
|
return oid.ID{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) GetParts(_ context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfoExtended, error) {
|
func (t *TreeServiceMock) GetParts(_ context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error) {
|
||||||
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
|
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
|
||||||
|
|
||||||
var foundMultipart *data.MultipartInfo
|
var foundMultipart *data.MultipartInfo
|
||||||
|
@ -391,7 +392,7 @@ LOOP:
|
||||||
}
|
}
|
||||||
|
|
||||||
partsMap := t.parts[foundMultipart.UploadID]
|
partsMap := t.parts[foundMultipart.UploadID]
|
||||||
result := make([]*data.PartInfoExtended, 0, len(partsMap))
|
result := make([]*data.PartInfo, 0, len(partsMap))
|
||||||
for _, part := range partsMap {
|
for _, part := range partsMap {
|
||||||
result = append(result, part)
|
result = append(result, part)
|
||||||
}
|
}
|
||||||
|
@ -399,51 +400,6 @@ LOOP:
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) PutBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
|
|
||||||
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
|
||||||
if !ok {
|
|
||||||
systemMap = make(map[string]*data.BaseNodeVersion)
|
|
||||||
}
|
|
||||||
|
|
||||||
systemMap["lifecycle"] = &data.BaseNodeVersion{
|
|
||||||
OID: addr.Object(),
|
|
||||||
}
|
|
||||||
|
|
||||||
t.system[bktInfo.CID.EncodeToString()] = systemMap
|
|
||||||
|
|
||||||
return nil, ErrNoNodeToRemove
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TreeServiceMock) GetBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
|
|
||||||
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
|
||||||
if !ok {
|
|
||||||
return oid.Address{}, ErrNodeNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
node, ok := systemMap["lifecycle"]
|
|
||||||
if !ok {
|
|
||||||
return oid.Address{}, ErrNodeNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
return newAddress(bktInfo.CID, node.OID), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TreeServiceMock) DeleteBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
|
|
||||||
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
|
||||||
if !ok {
|
|
||||||
return nil, ErrNoNodeToRemove
|
|
||||||
}
|
|
||||||
|
|
||||||
node, ok := systemMap["lifecycle"]
|
|
||||||
if !ok {
|
|
||||||
return nil, ErrNoNodeToRemove
|
|
||||||
}
|
|
||||||
|
|
||||||
delete(systemMap, "lifecycle")
|
|
||||||
|
|
||||||
return []oid.Address{newAddress(bktInfo.CID, node.OID)}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TreeServiceMock) DeleteMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
func (t *TreeServiceMock) DeleteMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
||||||
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
|
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
|
||||||
|
|
||||||
|
|
|
@ -18,20 +18,31 @@ type TreeService interface {
|
||||||
// If tree node is not found returns ErrNodeNotFound error.
|
// If tree node is not found returns ErrNodeNotFound error.
|
||||||
GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error)
|
GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error)
|
||||||
|
|
||||||
|
// GetNotificationConfigurationNode gets an object id that corresponds to object with bucket CORS.
|
||||||
|
//
|
||||||
|
// If tree node is not found returns ErrNodeNotFound error.
|
||||||
|
GetNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
||||||
|
|
||||||
|
// PutNotificationConfigurationNode puts a node to a system tree
|
||||||
|
// and returns objectID of a previous notif config which must be deleted in FrostFS.
|
||||||
|
//
|
||||||
|
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||||
|
PutNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
|
||||||
|
|
||||||
// GetBucketCORS gets an object id that corresponds to object with bucket CORS.
|
// GetBucketCORS gets an object id that corresponds to object with bucket CORS.
|
||||||
//
|
//
|
||||||
// If object id is not found returns ErrNodeNotFound error.
|
// If object id is not found returns ErrNodeNotFound error.
|
||||||
GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error)
|
GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
||||||
|
|
||||||
// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in FrostFS.
|
// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in FrostFS.
|
||||||
//
|
//
|
||||||
// If object ids to remove is not found returns ErrNoNodeToRemove error.
|
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||||
PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error)
|
PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
|
||||||
|
|
||||||
// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in FrostFS.
|
// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in FrostFS.
|
||||||
//
|
//
|
||||||
// If object ids to remove is not found returns ErrNoNodeToRemove error.
|
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||||
DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error)
|
DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
||||||
|
|
||||||
GetObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, error)
|
GetObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, error)
|
||||||
PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion, tagSet map[string]string) error
|
PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion, tagSet map[string]string) error
|
||||||
|
@ -57,15 +68,11 @@ type TreeService interface {
|
||||||
GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error)
|
GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error)
|
||||||
|
|
||||||
// AddPart puts a node to a system tree as a child of appropriate multipart upload
|
// AddPart puts a node to a system tree as a child of appropriate multipart upload
|
||||||
// and returns objectIDs of a previous part/s which must be deleted in FrostFS.
|
// and returns objectID of a previous part which must be deleted in FrostFS.
|
||||||
//
|
//
|
||||||
// If object ids to remove is not found returns ErrNoNodeToRemove error.
|
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||||
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDsToDelete []oid.ID, err error)
|
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)
|
||||||
GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfoExtended, error)
|
GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error)
|
||||||
|
|
||||||
PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error)
|
|
||||||
GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error)
|
|
||||||
DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error)
|
|
||||||
|
|
||||||
// Compound methods for optimizations
|
// Compound methods for optimizations
|
||||||
|
|
||||||
|
|
|
@ -130,7 +130,7 @@ func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
|
||||||
type testContext struct {
|
type testContext struct {
|
||||||
t *testing.T
|
t *testing.T
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
layer *Layer
|
layer Client
|
||||||
bktInfo *data.BucketInfo
|
bktInfo *data.BucketInfo
|
||||||
obj string
|
obj string
|
||||||
testFrostFS *TestFrostFS
|
testFrostFS *TestFrostFS
|
||||||
|
@ -145,12 +145,12 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
||||||
bearerToken := bearertest.Token()
|
bearerToken := bearertest.Token()
|
||||||
require.NoError(t, bearerToken.Sign(key.PrivateKey))
|
require.NoError(t, bearerToken.Sign(key.PrivateKey))
|
||||||
|
|
||||||
ctx := middleware.SetBox(context.Background(), &middleware.Box{AccessBox: &accessbox.Box{
|
ctx := middleware.SetBoxData(context.Background(), &accessbox.Box{
|
||||||
Gate: &accessbox.GateData{
|
Gate: &accessbox.GateData{
|
||||||
BearerToken: &bearerToken,
|
BearerToken: &bearerToken,
|
||||||
GateKey: key.PublicKey(),
|
GateKey: key.PublicKey(),
|
||||||
},
|
},
|
||||||
}})
|
})
|
||||||
tp := NewTestFrostFS(key)
|
tp := NewTestFrostFS(key)
|
||||||
|
|
||||||
bktName := "testbucket1"
|
bktName := "testbucket1"
|
||||||
|
|
|
@ -1,148 +0,0 @@
|
||||||
package middleware
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
|
||||||
"go.uber.org/zap"
|
|
||||||
)
|
|
||||||
|
|
||||||
const wildcardPlaceholder = "<wildcard>"
|
|
||||||
|
|
||||||
type VHSSettings interface {
|
|
||||||
Domains() []string
|
|
||||||
GlobalVHS() bool
|
|
||||||
VHSHeader() string
|
|
||||||
ServernameHeader() string
|
|
||||||
VHSNamespacesEnabled() map[string]bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func PrepareAddressStyle(settings VHSSettings, log *zap.Logger) Func {
|
|
||||||
return func(h http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
ctx := r.Context()
|
|
||||||
reqInfo := GetReqInfo(ctx)
|
|
||||||
reqLogger := reqLogOrDefault(ctx, log)
|
|
||||||
headerVHSEnabled := r.Header.Get(settings.VHSHeader())
|
|
||||||
|
|
||||||
if isVHSAddress(headerVHSEnabled, settings.GlobalVHS(), settings.VHSNamespacesEnabled(), reqInfo.Namespace) {
|
|
||||||
prepareVHSAddress(reqInfo, r, settings)
|
|
||||||
} else {
|
|
||||||
preparePathStyleAddress(reqInfo, r, reqLogger)
|
|
||||||
}
|
|
||||||
|
|
||||||
h.ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func isVHSAddress(headerVHSEnabled string, enabledFlag bool, vhsNamespaces map[string]bool, namespace string) bool {
|
|
||||||
if result, err := strconv.ParseBool(headerVHSEnabled); err == nil {
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
result := enabledFlag
|
|
||||||
if v, ok := vhsNamespaces[namespace]; ok {
|
|
||||||
result = v
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
func prepareVHSAddress(reqInfo *ReqInfo, r *http.Request, settings VHSSettings) {
|
|
||||||
reqInfo.RequestVHSEnabled = true
|
|
||||||
bktName, match := checkDomain(r.Host, getDomains(r, settings))
|
|
||||||
if match {
|
|
||||||
if bktName == "" {
|
|
||||||
reqInfo.RequestType = noneType
|
|
||||||
} else {
|
|
||||||
if objName := strings.TrimPrefix(r.URL.Path, "/"); objName != "" {
|
|
||||||
reqInfo.RequestType = objectType
|
|
||||||
reqInfo.ObjectName = objName
|
|
||||||
reqInfo.BucketName = bktName
|
|
||||||
} else {
|
|
||||||
reqInfo.RequestType = bucketType
|
|
||||||
reqInfo.BucketName = bktName
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
parts := strings.Split(r.Host, ".")
|
|
||||||
reqInfo.BucketName = parts[0]
|
|
||||||
|
|
||||||
if objName := strings.TrimPrefix(r.URL.Path, "/"); objName != "" {
|
|
||||||
reqInfo.RequestType = objectType
|
|
||||||
reqInfo.ObjectName = objName
|
|
||||||
} else {
|
|
||||||
reqInfo.RequestType = bucketType
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func getDomains(r *http.Request, settings VHSSettings) []string {
|
|
||||||
if headerServername := r.Header.Get(settings.ServernameHeader()); headerServername != "" {
|
|
||||||
return []string{headerServername}
|
|
||||||
}
|
|
||||||
|
|
||||||
return settings.Domains()
|
|
||||||
}
|
|
||||||
|
|
||||||
func preparePathStyleAddress(reqInfo *ReqInfo, r *http.Request, reqLogger *zap.Logger) {
|
|
||||||
bktObj := strings.TrimPrefix(r.URL.Path, "/")
|
|
||||||
if bktObj == "" {
|
|
||||||
reqInfo.RequestType = noneType
|
|
||||||
} else if ind := strings.IndexByte(bktObj, '/'); ind != -1 && bktObj[ind+1:] != "" {
|
|
||||||
reqInfo.RequestType = objectType
|
|
||||||
reqInfo.BucketName = bktObj[:ind]
|
|
||||||
reqInfo.ObjectName = bktObj[ind+1:]
|
|
||||||
|
|
||||||
if r.URL.RawPath != "" {
|
|
||||||
// we have to do this because of
|
|
||||||
// https://github.com/go-chi/chi/issues/641
|
|
||||||
// https://github.com/go-chi/chi/issues/642
|
|
||||||
if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil {
|
|
||||||
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err))
|
|
||||||
} else {
|
|
||||||
reqInfo.ObjectName = obj
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
reqInfo.RequestType = bucketType
|
|
||||||
reqInfo.BucketName = strings.TrimSuffix(bktObj, "/")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkDomain(host string, domains []string) (bktName string, match bool) {
|
|
||||||
partsHost := strings.Split(host, ".")
|
|
||||||
for _, pattern := range domains {
|
|
||||||
partsPattern := strings.Split(pattern, ".")
|
|
||||||
bktName, match = compareMatch(partsHost, partsPattern)
|
|
||||||
if match {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func compareMatch(host, pattern []string) (bktName string, match bool) {
|
|
||||||
if len(host) < len(pattern) {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
i, j := len(host)-1, len(pattern)-1
|
|
||||||
for j >= 0 && (pattern[j] == wildcardPlaceholder || host[i] == pattern[j]) {
|
|
||||||
i--
|
|
||||||
j--
|
|
||||||
}
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case i == -1:
|
|
||||||
return "", true
|
|
||||||
case i == 0 && (j != 0 || host[i] == pattern[j]):
|
|
||||||
return host[0], true
|
|
||||||
default:
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,443 +0,0 @@
|
||||||
package middleware
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"go.uber.org/zap/zaptest"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
FrostfsVHSHeader = "X-Frostfs-S3-VHS"
|
|
||||||
FrostfsServernameHeader = "X-Frostfs-Servername"
|
|
||||||
)
|
|
||||||
|
|
||||||
type VHSSettingsMock struct {
|
|
||||||
domains []string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *VHSSettingsMock) Domains() []string {
|
|
||||||
return v.domains
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *VHSSettingsMock) GlobalVHS() bool {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *VHSSettingsMock) VHSHeader() string {
|
|
||||||
return FrostfsVHSHeader
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *VHSSettingsMock) ServernameHeader() string {
|
|
||||||
return FrostfsServernameHeader
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *VHSSettingsMock) VHSNamespacesEnabled() map[string]bool {
|
|
||||||
return make(map[string]bool)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIsVHSAddress(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
headerVHSEnabled string
|
|
||||||
vhsEnabledFlag bool
|
|
||||||
vhsNamespaced map[string]bool
|
|
||||||
namespace string
|
|
||||||
expected bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "vhs disabled",
|
|
||||||
expected: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "vhs disabled for namespace",
|
|
||||||
vhsEnabledFlag: true,
|
|
||||||
vhsNamespaced: map[string]bool{
|
|
||||||
"kapusta": false,
|
|
||||||
},
|
|
||||||
namespace: "kapusta",
|
|
||||||
expected: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "vhs enabled (global vhs flag)",
|
|
||||||
vhsEnabledFlag: true,
|
|
||||||
expected: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "vhs enabled for namespace",
|
|
||||||
vhsNamespaced: map[string]bool{
|
|
||||||
"kapusta": true,
|
|
||||||
},
|
|
||||||
namespace: "kapusta",
|
|
||||||
expected: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "vhs enabled (header)",
|
|
||||||
headerVHSEnabled: "true",
|
|
||||||
vhsEnabledFlag: false,
|
|
||||||
vhsNamespaced: map[string]bool{
|
|
||||||
"kapusta": false,
|
|
||||||
},
|
|
||||||
namespace: "kapusta",
|
|
||||||
expected: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "vhs disabled (header)",
|
|
||||||
headerVHSEnabled: "false",
|
|
||||||
vhsEnabledFlag: true,
|
|
||||||
vhsNamespaced: map[string]bool{
|
|
||||||
"kapusta": true,
|
|
||||||
},
|
|
||||||
namespace: "kapusta",
|
|
||||||
expected: false,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
actual := isVHSAddress(tc.headerVHSEnabled, tc.vhsEnabledFlag, tc.vhsNamespaced, tc.namespace)
|
|
||||||
require.Equal(t, tc.expected, actual)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPreparePathStyleAddress(t *testing.T) {
|
|
||||||
bkt, obj := "test-bucket", "test-object"
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
urlParams string
|
|
||||||
expectedReqType ReqType
|
|
||||||
expectedBktName string
|
|
||||||
expectedObjName string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "bucket request",
|
|
||||||
urlParams: "/" + bkt,
|
|
||||||
expectedReqType: bucketType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "bucket request with slash",
|
|
||||||
urlParams: "/" + bkt + "/",
|
|
||||||
expectedReqType: bucketType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "object request",
|
|
||||||
urlParams: "/" + bkt + "/" + obj,
|
|
||||||
expectedReqType: objectType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
expectedObjName: obj,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "object request with slash",
|
|
||||||
urlParams: "/" + bkt + "/" + obj + "/",
|
|
||||||
expectedReqType: objectType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
expectedObjName: obj + "/",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "none type request",
|
|
||||||
urlParams: "/",
|
|
||||||
expectedReqType: noneType,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
reqInfo := &ReqInfo{}
|
|
||||||
r := httptest.NewRequest(http.MethodGet, tc.urlParams, nil)
|
|
||||||
|
|
||||||
preparePathStyleAddress(reqInfo, r, reqLogOrDefault(r.Context(), zaptest.NewLogger(t)))
|
|
||||||
require.Equal(t, tc.expectedReqType, reqInfo.RequestType)
|
|
||||||
require.Equal(t, tc.expectedBktName, reqInfo.BucketName)
|
|
||||||
require.Equal(t, tc.expectedObjName, reqInfo.ObjectName)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPrepareVHSAddress(t *testing.T) {
|
|
||||||
bkt, obj, domain := "test-bucket", "test-object", "domain.com"
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
domains []string
|
|
||||||
host string
|
|
||||||
urlParams string
|
|
||||||
expectedReqType ReqType
|
|
||||||
expectedBktName string
|
|
||||||
expectedObjName string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "bucket request, the domain matched",
|
|
||||||
domains: []string{domain},
|
|
||||||
host: bkt + "." + domain,
|
|
||||||
urlParams: "/",
|
|
||||||
expectedReqType: bucketType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "object request, the domain matched",
|
|
||||||
domains: []string{domain},
|
|
||||||
host: bkt + "." + domain,
|
|
||||||
urlParams: "/" + obj,
|
|
||||||
expectedReqType: objectType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
expectedObjName: obj,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "object request with slash, the domain matched",
|
|
||||||
domains: []string{domain},
|
|
||||||
host: bkt + "." + domain,
|
|
||||||
urlParams: "/" + obj + "/",
|
|
||||||
expectedReqType: objectType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
expectedObjName: obj + "/",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "list-buckets request, the domain matched",
|
|
||||||
domains: []string{domain},
|
|
||||||
host: domain,
|
|
||||||
urlParams: "/",
|
|
||||||
expectedReqType: noneType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "bucket request, the domain don't match",
|
|
||||||
host: bkt + "." + domain,
|
|
||||||
urlParams: "/",
|
|
||||||
expectedReqType: bucketType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "object request, the domain don't match",
|
|
||||||
host: bkt + "." + domain,
|
|
||||||
urlParams: "/" + obj,
|
|
||||||
expectedReqType: objectType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
expectedObjName: obj,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "object request with slash, the domain don't match",
|
|
||||||
host: bkt + "." + domain,
|
|
||||||
urlParams: "/" + obj + "/",
|
|
||||||
expectedReqType: objectType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
expectedObjName: obj + "/",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "list-buckets request, the domain don't match (list-buckets isn't supported if the domains don't match)",
|
|
||||||
host: domain,
|
|
||||||
urlParams: "/",
|
|
||||||
expectedReqType: bucketType,
|
|
||||||
expectedBktName: strings.Split(domain, ".")[0],
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
reqInfo := &ReqInfo{}
|
|
||||||
vhsSettings := &VHSSettingsMock{domains: tc.domains}
|
|
||||||
r := httptest.NewRequest(http.MethodGet, tc.urlParams, nil)
|
|
||||||
r.Host = tc.host
|
|
||||||
|
|
||||||
prepareVHSAddress(reqInfo, r, vhsSettings)
|
|
||||||
require.Equal(t, tc.expectedReqType, reqInfo.RequestType)
|
|
||||||
require.Equal(t, tc.expectedBktName, reqInfo.BucketName)
|
|
||||||
require.Equal(t, tc.expectedObjName, reqInfo.ObjectName)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckDomains(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
domains []string
|
|
||||||
requestURL string
|
|
||||||
expectedBktName string
|
|
||||||
expectedMatch bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "valid url with bktName and namespace (wildcard after protocol infix)",
|
|
||||||
domains: []string{"s3.<wildcard>.domain.com"},
|
|
||||||
requestURL: "bktA.s3.kapusta.domain.com",
|
|
||||||
expectedBktName: "bktA",
|
|
||||||
expectedMatch: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid url without bktName and namespace (wildcard after protocol infix)",
|
|
||||||
domains: []string{"s3.<wildcard>.domain.com"},
|
|
||||||
requestURL: "s3.kapusta.domain.com",
|
|
||||||
expectedBktName: "",
|
|
||||||
expectedMatch: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid url with invalid bktName (wildcard after protocol infix)",
|
|
||||||
domains: []string{"s3.<wildcard>.domain.com"},
|
|
||||||
requestURL: "bktA.bktB.s3.kapusta.domain.com",
|
|
||||||
expectedMatch: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid url without namespace (wildcard after protocol infix)",
|
|
||||||
domains: []string{"s3.<wildcard>.domain.com"},
|
|
||||||
requestURL: "bktA.s3.domain.com",
|
|
||||||
expectedMatch: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid url with invalid infix (wildcard after protocol infix)",
|
|
||||||
domains: []string{"s3.<wildcard>.domain.com"},
|
|
||||||
requestURL: "bktA.s4.kapusta.domain.com",
|
|
||||||
expectedMatch: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid url with invalid postfix (wildcard after protocol infix)",
|
|
||||||
domains: []string{"s3.<wildcard>.domain.com"},
|
|
||||||
requestURL: "bktA.s3.kapusta.dom.su",
|
|
||||||
expectedMatch: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid url with bktName and namespace (wildcard at the beginning of the domain)",
|
|
||||||
domains: []string{"<wildcard>.domain.com"},
|
|
||||||
requestURL: "bktA.kapusta.domain.com",
|
|
||||||
expectedBktName: "bktA",
|
|
||||||
expectedMatch: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid url without bktName and namespace (wildcard at the beginning of the domain)",
|
|
||||||
domains: []string{"<wildcard>.domain.com"},
|
|
||||||
requestURL: "kapusta.domain.com",
|
|
||||||
expectedBktName: "",
|
|
||||||
expectedMatch: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid url with invalid bktName (wildcard at the beginning of the domain)",
|
|
||||||
domains: []string{"<wildcard>.domain.com"},
|
|
||||||
requestURL: "bktA.bktB.kapusta.domain.com",
|
|
||||||
expectedMatch: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "collision test - true, because we cannot clearly distinguish a namespace from a bucket (wildcard at the beginning of the domain)",
|
|
||||||
domains: []string{"<wildcard>.domain.com"},
|
|
||||||
requestURL: "bktA.domain.com",
|
|
||||||
expectedMatch: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid url (fewer hosts)",
|
|
||||||
domains: []string{"<wildcard>.domain.com"},
|
|
||||||
requestURL: "domain.com",
|
|
||||||
expectedMatch: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid url with invalid postfix (wildcard at the beginning of the domain)",
|
|
||||||
domains: []string{"<wildcard>.domain.com"},
|
|
||||||
requestURL: "bktA.kapusta.dom.su",
|
|
||||||
expectedMatch: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid url with bktName and without wildcard (root namaspace)",
|
|
||||||
domains: []string{"domain.com"},
|
|
||||||
requestURL: "bktA.domain.com",
|
|
||||||
expectedBktName: "bktA",
|
|
||||||
expectedMatch: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid url without bktName and without wildcard (root namaspace)",
|
|
||||||
domains: []string{"domain.com"},
|
|
||||||
requestURL: "domain.com",
|
|
||||||
expectedBktName: "",
|
|
||||||
expectedMatch: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid url with bktName without wildcard (root namaspace)",
|
|
||||||
domains: []string{"domain.com"},
|
|
||||||
requestURL: "bktA.dom.su",
|
|
||||||
expectedMatch: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid url without wildcard (root namaspace)",
|
|
||||||
domains: []string{"domain.com"},
|
|
||||||
requestURL: "dom.su",
|
|
||||||
expectedMatch: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid url, with a sorted list of domains",
|
|
||||||
domains: []string{"s3.<wildcard>.domain.com", "<wildcard>.domain.com", "domain.com"},
|
|
||||||
requestURL: "s3.kapusta.domain.com",
|
|
||||||
expectedBktName: "",
|
|
||||||
expectedMatch: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid url with bktName, multiple wildcards (wildcards at the beginning of the domain)",
|
|
||||||
domains: []string{"<wildcard>.<wildcard>.domain.com"},
|
|
||||||
requestURL: "bktA.s3.kapusta.domain.com",
|
|
||||||
expectedBktName: "bktA",
|
|
||||||
expectedMatch: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid url without bktName, multiple wildcards (wildcards at the beginning of the domain)",
|
|
||||||
domains: []string{"<wildcard>.<wildcard>.domain.com"},
|
|
||||||
requestURL: "s3.kapusta.domain.com",
|
|
||||||
expectedBktName: "",
|
|
||||||
expectedMatch: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid url with bktName, multiply wildcards",
|
|
||||||
domains: []string{"s3.<wildcard>.subdomain.<wildcard>.com"},
|
|
||||||
requestURL: "bktA.s3.kapusta.subdomain.domain.com",
|
|
||||||
expectedBktName: "bktA",
|
|
||||||
expectedMatch: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid url without bktName, multiply wildcards",
|
|
||||||
domains: []string{"s3.<wildcard>.subdomain.<wildcard>.com"},
|
|
||||||
requestURL: "s3.kapusta.subdomain.domain.com",
|
|
||||||
expectedBktName: "",
|
|
||||||
expectedMatch: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid url without one wildcard",
|
|
||||||
domains: []string{"<wildcard>.<wildcard>.domain.com"},
|
|
||||||
requestURL: "kapusta.domain.com",
|
|
||||||
expectedMatch: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid url, multiply wildcards",
|
|
||||||
domains: []string{"<wildcard>.<wildcard>.domain.com"},
|
|
||||||
requestURL: "s3.kapusta.dom.com",
|
|
||||||
expectedMatch: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid url with invalid bktName, multiply wildcards",
|
|
||||||
domains: []string{"<wildcard>.<wildcard>.domain.com"},
|
|
||||||
requestURL: "bktA.bktB.s3.kapusta.domain.com",
|
|
||||||
expectedMatch: false,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
bktName, match := checkDomain(tc.requestURL, tc.domains)
|
|
||||||
require.Equal(t, tc.expectedBktName, bktName)
|
|
||||||
require.Equal(t, tc.expectedMatch, match)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetDomains(t *testing.T) {
|
|
||||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
|
||||||
settings := &VHSSettingsMock{
|
|
||||||
domains: []string{
|
|
||||||
"s3.domain.com",
|
|
||||||
"s3.<wildcard>.domain.com",
|
|
||||||
"domain.com",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("the request does not contain the X-Frostfs-Servername header", func(t *testing.T) {
|
|
||||||
actualDomains := getDomains(req, settings)
|
|
||||||
require.Equal(t, settings.domains, actualDomains)
|
|
||||||
})
|
|
||||||
|
|
||||||
serverName := "domain.com"
|
|
||||||
req.Header.Set(settings.ServernameHeader(), serverName)
|
|
||||||
|
|
||||||
t.Run("the request contains the X-Frostfs-Servername header", func(t *testing.T) {
|
|
||||||
actualDomains := getDomains(req, settings)
|
|
||||||
require.Equal(t, []string{serverName}, actualDomains)
|
|
||||||
})
|
|
||||||
}
|
|
|
@ -13,7 +13,6 @@ import (
|
||||||
frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -24,7 +23,6 @@ type (
|
||||||
AccessBox *accessbox.Box
|
AccessBox *accessbox.Box
|
||||||
ClientTime time.Time
|
ClientTime time.Time
|
||||||
AuthHeaders *AuthHeader
|
AuthHeaders *AuthHeader
|
||||||
Attributes []object.Attribute
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Center is a user authentication interface.
|
// Center is a user authentication interface.
|
||||||
|
@ -67,7 +65,11 @@ func Auth(center Center, log *zap.Logger) Func {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
ctx = SetBox(ctx, box)
|
ctx = SetBoxData(ctx, box.AccessBox)
|
||||||
|
if !box.ClientTime.IsZero() {
|
||||||
|
ctx = SetClientTime(ctx, box.ClientTime)
|
||||||
|
}
|
||||||
|
ctx = SetAuthHeaders(ctx, box.AuthHeaders)
|
||||||
|
|
||||||
if box.AccessBox.Gate.BearerToken != nil {
|
if box.AccessBox.Gate.BearerToken != nil {
|
||||||
reqInfo.User = bearer.ResolveIssuer(*box.AccessBox.Gate.BearerToken).String()
|
reqInfo.User = bearer.ResolveIssuer(*box.AccessBox.Gate.BearerToken).String()
|
||||||
|
|
|
@ -5,11 +5,10 @@ const (
|
||||||
|
|
||||||
// bucket operations.
|
// bucket operations.
|
||||||
|
|
||||||
OptionsBucketOperation = "OptionsBucket"
|
OptionsOperation = "Options"
|
||||||
HeadBucketOperation = "HeadBucket"
|
HeadBucketOperation = "HeadBucket"
|
||||||
ListMultipartUploadsOperation = "ListMultipartUploads"
|
ListMultipartUploadsOperation = "ListMultipartUploads"
|
||||||
GetBucketLocationOperation = "GetBucketLocation"
|
GetBucketLocationOperation = "GetBucketLocation"
|
||||||
GetBucketPolicyStatusOperation = "GetBucketPolicyStatus"
|
|
||||||
GetBucketPolicyOperation = "GetBucketPolicy"
|
GetBucketPolicyOperation = "GetBucketPolicy"
|
||||||
GetBucketLifecycleOperation = "GetBucketLifecycle"
|
GetBucketLifecycleOperation = "GetBucketLifecycle"
|
||||||
GetBucketEncryptionOperation = "GetBucketEncryption"
|
GetBucketEncryptionOperation = "GetBucketEncryption"
|
||||||
|
@ -51,7 +50,6 @@ const (
|
||||||
|
|
||||||
// object operations.
|
// object operations.
|
||||||
|
|
||||||
OptionsObjectOperation = "OptionsObject"
|
|
||||||
HeadObjectOperation = "HeadObject"
|
HeadObjectOperation = "HeadObject"
|
||||||
ListPartsOperation = "ListParts"
|
ListPartsOperation = "ListParts"
|
||||||
GetObjectACLOperation = "GetObjectACL"
|
GetObjectACLOperation = "GetObjectACL"
|
||||||
|
@ -74,13 +72,11 @@ const (
|
||||||
AbortMultipartUploadOperation = "AbortMultipartUpload"
|
AbortMultipartUploadOperation = "AbortMultipartUpload"
|
||||||
DeleteObjectTaggingOperation = "DeleteObjectTagging"
|
DeleteObjectTaggingOperation = "DeleteObjectTagging"
|
||||||
DeleteObjectOperation = "DeleteObject"
|
DeleteObjectOperation = "DeleteObject"
|
||||||
PatchObjectOperation = "PatchObject"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
UploadsQuery = "uploads"
|
UploadsQuery = "uploads"
|
||||||
LocationQuery = "location"
|
LocationQuery = "location"
|
||||||
PolicyStatusQuery = "policyStatus"
|
|
||||||
PolicyQuery = "policy"
|
PolicyQuery = "policy"
|
||||||
LifecycleQuery = "lifecycle"
|
LifecycleQuery = "lifecycle"
|
||||||
EncryptionQuery = "encryption"
|
EncryptionQuery = "encryption"
|
||||||
|
|
60
api/middleware/log_http.go
Normal file
60
api/middleware/log_http.go
Normal file
|
@ -0,0 +1,60 @@
|
||||||
|
package middleware
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
type HttpLogSettings interface {
|
||||||
|
GetHttpLogEnabled() bool
|
||||||
|
GetMaxLogBody() int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// LogHttp logs http parameters from s3 request
|
||||||
|
func LogHttp(l *zap.Logger, settings HttpLogSettings) Func {
|
||||||
|
return func(h http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !settings.GetHttpLogEnabled() {
|
||||||
|
h.ServeHTTP(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var httplog = l.With(
|
||||||
|
zap.String("from", r.RemoteAddr),
|
||||||
|
zap.String("URI", r.RequestURI),
|
||||||
|
zap.String("method", r.Method),
|
||||||
|
)
|
||||||
|
|
||||||
|
httplog = withFieldIfExist(httplog, "query", r.URL.Query())
|
||||||
|
httplog = withFieldIfExist(httplog, "headers", r.Header)
|
||||||
|
if r.ContentLength != 0 && r.ContentLength <= settings.GetMaxLogBody() {
|
||||||
|
var err error
|
||||||
|
httplog, err = withBody(httplog, r)
|
||||||
|
if err != nil {
|
||||||
|
l.Error("read body error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
httplog.Info(logs.RequestHTTP)
|
||||||
|
h.ServeHTTP(w, r)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func withBody(httplog *zap.Logger, r *http.Request) (*zap.Logger, error) {
|
||||||
|
var body = make([]byte, r.ContentLength)
|
||||||
|
_, err := r.Body.Read(body)
|
||||||
|
if err != nil && err != io.EOF {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
httplog = httplog.With(zap.String("body", string(body)))
|
||||||
|
|
||||||
|
return httplog, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func withFieldIfExist(log *zap.Logger, label string, data map[string][]string) *zap.Logger {
|
||||||
|
if len(data) != 0 {
|
||||||
|
log = log.With(zap.Any(label, data))
|
||||||
|
}
|
||||||
|
return log
|
||||||
|
}
|
|
@ -103,7 +103,7 @@ func stats(f http.HandlerFunc, resolveCID cidResolveFunc, appMetrics *metrics.Ap
|
||||||
|
|
||||||
func requestTypeFromAPI(api string) metrics.RequestType {
|
func requestTypeFromAPI(api string) metrics.RequestType {
|
||||||
switch api {
|
switch api {
|
||||||
case OptionsBucketOperation, OptionsObjectOperation, HeadObjectOperation, HeadBucketOperation:
|
case OptionsOperation, HeadObjectOperation, HeadBucketOperation:
|
||||||
return metrics.HEADRequest
|
return metrics.HEADRequest
|
||||||
case CreateMultipartUploadOperation, UploadPartCopyOperation, UploadPartOperation, CompleteMultipartUploadOperation,
|
case CreateMultipartUploadOperation, UploadPartCopyOperation, UploadPartOperation, CompleteMultipartUploadOperation,
|
||||||
PutObjectACLOperation, PutObjectTaggingOperation, CopyObjectOperation, PutObjectRetentionOperation, PutObjectLegalHoldOperation,
|
PutObjectACLOperation, PutObjectTaggingOperation, CopyObjectOperation, PutObjectRetentionOperation, PutObjectLegalHoldOperation,
|
||||||
|
|
|
@ -3,16 +3,12 @@ package middleware
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/elliptic"
|
"crypto/elliptic"
|
||||||
"encoding/xml"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
|
||||||
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
|
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
|
||||||
|
@ -24,46 +20,13 @@ import (
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
QueryVersionID = "versionId"
|
|
||||||
QueryPrefix = "prefix"
|
|
||||||
QueryDelimiter = "delimiter"
|
|
||||||
QueryMaxKeys = "max-keys"
|
|
||||||
amzTagging = "x-amz-tagging"
|
|
||||||
)
|
|
||||||
|
|
||||||
// In these operations we don't check resource tags because
|
|
||||||
// * they haven't been created yet
|
|
||||||
// * resource tags shouldn't be checked by AWS spec.
|
|
||||||
var withoutResourceOps = []string{
|
|
||||||
CreateBucketOperation,
|
|
||||||
CreateMultipartUploadOperation,
|
|
||||||
AbortMultipartUploadOperation,
|
|
||||||
CompleteMultipartUploadOperation,
|
|
||||||
UploadPartOperation,
|
|
||||||
UploadPartCopyOperation,
|
|
||||||
ListPartsOperation,
|
|
||||||
PutObjectOperation,
|
|
||||||
CopyObjectOperation,
|
|
||||||
DeleteObjectOperation,
|
|
||||||
DeleteMultipleObjectsOperation,
|
|
||||||
}
|
|
||||||
|
|
||||||
type PolicySettings interface {
|
type PolicySettings interface {
|
||||||
PolicyDenyByDefault() bool
|
PolicyDenyByDefault() bool
|
||||||
|
ACLEnabled() bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type FrostFSIDInformer interface {
|
type FrostFSIDInformer interface {
|
||||||
GetUserGroupIDsAndClaims(userHash util.Uint160) ([]string, map[string]string, error)
|
GetUserGroupIDs(userHash util.Uint160) ([]string, error)
|
||||||
}
|
|
||||||
|
|
||||||
type XMLDecoder interface {
|
|
||||||
NewXMLDecoder(io.Reader) *xml.Decoder
|
|
||||||
}
|
|
||||||
|
|
||||||
type ResourceTagging interface {
|
|
||||||
GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error)
|
|
||||||
GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingParams) (string, map[string]string, error)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BucketResolveFunc is a func to resolve bucket info by name.
|
// BucketResolveFunc is a func to resolve bucket info by name.
|
||||||
|
@ -73,10 +36,9 @@ type PolicyConfig struct {
|
||||||
Storage engine.ChainRouter
|
Storage engine.ChainRouter
|
||||||
FrostfsID FrostFSIDInformer
|
FrostfsID FrostFSIDInformer
|
||||||
Settings PolicySettings
|
Settings PolicySettings
|
||||||
|
Domains []string
|
||||||
Log *zap.Logger
|
Log *zap.Logger
|
||||||
BucketResolver BucketResolveFunc
|
BucketResolver BucketResolveFunc
|
||||||
Decoder XMLDecoder
|
|
||||||
Tagging ResourceTagging
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func PolicyCheck(cfg PolicyConfig) Func {
|
func PolicyCheck(cfg PolicyConfig) Func {
|
||||||
|
@ -85,7 +47,6 @@ func PolicyCheck(cfg PolicyConfig) Func {
|
||||||
ctx := r.Context()
|
ctx := r.Context()
|
||||||
if err := policyCheck(r, cfg); err != nil {
|
if err := policyCheck(r, cfg); err != nil {
|
||||||
reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err))
|
reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err))
|
||||||
err = frostfsErrors.UnwrapErr(err)
|
|
||||||
if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil {
|
if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil {
|
||||||
reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
|
reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
|
||||||
}
|
}
|
||||||
|
@ -98,21 +59,21 @@ func PolicyCheck(cfg PolicyConfig) Func {
|
||||||
}
|
}
|
||||||
|
|
||||||
func policyCheck(r *http.Request, cfg PolicyConfig) error {
|
func policyCheck(r *http.Request, cfg PolicyConfig) error {
|
||||||
reqInfo := GetReqInfo(r.Context())
|
reqType, bktName, objName := getBucketObject(r, cfg.Domains)
|
||||||
|
req, userKey, userGroups, err := getPolicyRequest(r, cfg.FrostfsID, reqType, bktName, objName, cfg.Log)
|
||||||
req, userKey, userGroups, err := getPolicyRequest(r, cfg, reqInfo.RequestType, reqInfo.BucketName, reqInfo.ObjectName)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var bktInfo *data.BucketInfo
|
var bktInfo *data.BucketInfo
|
||||||
if reqInfo.RequestType != noneType && !strings.HasSuffix(req.Operation(), CreateBucketOperation) {
|
if reqType != noneType && !strings.HasSuffix(req.Operation(), CreateBucketOperation) {
|
||||||
bktInfo, err = cfg.BucketResolver(r.Context(), reqInfo.BucketName)
|
bktInfo, err = cfg.BucketResolver(r.Context(), bktName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
reqInfo := GetReqInfo(r.Context())
|
||||||
target := engine.NewRequestTargetWithNamespace(reqInfo.Namespace)
|
target := engine.NewRequestTargetWithNamespace(reqInfo.Namespace)
|
||||||
if bktInfo != nil {
|
if bktInfo != nil {
|
||||||
cnrTarget := engine.ContainerTarget(bktInfo.CID.EncodeToString())
|
cnrTarget := engine.ContainerTarget(bktInfo.CID.EncodeToString())
|
||||||
|
@ -148,18 +109,22 @@ func policyCheck(r *http.Request, cfg PolicyConfig) error {
|
||||||
return apiErr.GetAPIErrorWithError(apiErr.ErrAccessDenied, fmt.Errorf("policy check: %s", st.String()))
|
return apiErr.GetAPIErrorWithError(apiErr.ErrAccessDenied, fmt.Errorf("policy check: %s", st.String()))
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Settings.PolicyDenyByDefault() {
|
isAPE := !cfg.Settings.ACLEnabled()
|
||||||
|
if bktInfo != nil {
|
||||||
|
isAPE = bktInfo.APEEnabled
|
||||||
|
}
|
||||||
|
|
||||||
|
if isAPE && cfg.Settings.PolicyDenyByDefault() {
|
||||||
return apiErr.GetAPIErrorWithError(apiErr.ErrAccessDenied, fmt.Errorf("policy check: %s", st.String()))
|
return apiErr.GetAPIErrorWithError(apiErr.ErrAccessDenied, fmt.Errorf("policy check: %s", st.String()))
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getPolicyRequest(r *http.Request, cfg PolicyConfig, reqType ReqType, bktName string, objName string) (*testutil.Request, *keys.PublicKey, []string, error) {
|
func getPolicyRequest(r *http.Request, frostfsid FrostFSIDInformer, reqType ReqType, bktName string, objName string, log *zap.Logger) (*testutil.Request, *keys.PublicKey, []string, error) {
|
||||||
var (
|
var (
|
||||||
owner string
|
owner string
|
||||||
groups []string
|
groups []string
|
||||||
tags map[string]string
|
|
||||||
pk *keys.PublicKey
|
pk *keys.PublicKey
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -172,7 +137,7 @@ func getPolicyRequest(r *http.Request, cfg PolicyConfig, reqType ReqType, bktNam
|
||||||
}
|
}
|
||||||
owner = pk.Address()
|
owner = pk.Address()
|
||||||
|
|
||||||
groups, tags, err = cfg.FrostfsID.GetUserGroupIDsAndClaims(pk.GetScriptHash())
|
groups, err = frostfsid.GetUserGroupIDs(pk.GetScriptHash())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, fmt.Errorf("get group ids: %w", err)
|
return nil, nil, nil, fmt.Errorf("get group ids: %w", err)
|
||||||
}
|
}
|
||||||
|
@ -187,16 +152,15 @@ func getPolicyRequest(r *http.Request, cfg PolicyConfig, reqType ReqType, bktNam
|
||||||
res = fmt.Sprintf(s3.ResourceFormatS3Bucket, bktName)
|
res = fmt.Sprintf(s3.ResourceFormatS3Bucket, bktName)
|
||||||
}
|
}
|
||||||
|
|
||||||
requestProps, resourceProps, err := determineProperties(r, cfg.Decoder, cfg.BucketResolver, cfg.Tagging, reqType, op, bktName, objName, owner, groups, tags)
|
reqLogOrDefault(r.Context(), log).Debug(logs.PolicyRequest, zap.String("action", op),
|
||||||
if err != nil {
|
zap.String("resource", res), zap.String("owner", owner))
|
||||||
return nil, nil, nil, fmt.Errorf("determine properties: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
reqLogOrDefault(r.Context(), cfg.Log).Debug(logs.PolicyRequest, zap.String("action", op),
|
return testutil.NewRequest(op, testutil.NewResource(res, nil),
|
||||||
zap.String("resource", res), zap.Any("request properties", requestProps),
|
map[string]string{
|
||||||
zap.Any("resource properties", resourceProps))
|
s3.PropertyKeyOwner: owner,
|
||||||
|
common.PropertyKeyFrostFSIDGroupID: chain.FormCondSliceContainsValue(groups),
|
||||||
return testutil.NewRequest(op, testutil.NewResource(res, resourceProps), requestProps), pk, groups, nil
|
},
|
||||||
|
), pk, groups, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type ReqType int
|
type ReqType int
|
||||||
|
@ -207,6 +171,33 @@ const (
|
||||||
objectType
|
objectType
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func getBucketObject(r *http.Request, domains []string) (reqType ReqType, bktName string, objName string) {
|
||||||
|
for _, domain := range domains {
|
||||||
|
ind := strings.Index(r.Host, "."+domain)
|
||||||
|
if ind == -1 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
bkt := r.Host[:ind]
|
||||||
|
if obj := strings.TrimPrefix(r.URL.Path, "/"); obj != "" {
|
||||||
|
return objectType, bkt, obj
|
||||||
|
}
|
||||||
|
|
||||||
|
return bucketType, bkt, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
bktObj := strings.TrimPrefix(r.URL.Path, "/")
|
||||||
|
if bktObj == "" {
|
||||||
|
return noneType, "", ""
|
||||||
|
}
|
||||||
|
|
||||||
|
if ind := strings.IndexByte(bktObj, '/'); ind != -1 && bktObj[ind+1:] != "" {
|
||||||
|
return objectType, bktObj[:ind], bktObj[ind+1:]
|
||||||
|
}
|
||||||
|
|
||||||
|
return bucketType, strings.TrimSuffix(bktObj, "/"), ""
|
||||||
|
}
|
||||||
|
|
||||||
func determineOperation(r *http.Request, reqType ReqType) (operation string) {
|
func determineOperation(r *http.Request, reqType ReqType) (operation string) {
|
||||||
switch reqType {
|
switch reqType {
|
||||||
case objectType:
|
case objectType:
|
||||||
|
@ -224,7 +215,7 @@ func determineBucketOperation(r *http.Request) string {
|
||||||
query := r.URL.Query()
|
query := r.URL.Query()
|
||||||
switch r.Method {
|
switch r.Method {
|
||||||
case http.MethodOptions:
|
case http.MethodOptions:
|
||||||
return OptionsBucketOperation
|
return OptionsOperation
|
||||||
case http.MethodHead:
|
case http.MethodHead:
|
||||||
return HeadBucketOperation
|
return HeadBucketOperation
|
||||||
case http.MethodGet:
|
case http.MethodGet:
|
||||||
|
@ -327,10 +318,6 @@ func determineBucketOperation(r *http.Request) string {
|
||||||
func determineObjectOperation(r *http.Request) string {
|
func determineObjectOperation(r *http.Request) string {
|
||||||
query := r.URL.Query()
|
query := r.URL.Query()
|
||||||
switch r.Method {
|
switch r.Method {
|
||||||
case http.MethodOptions:
|
|
||||||
return OptionsObjectOperation
|
|
||||||
case http.MethodPatch:
|
|
||||||
return PatchObjectOperation
|
|
||||||
case http.MethodHead:
|
case http.MethodHead:
|
||||||
return HeadObjectOperation
|
return HeadObjectOperation
|
||||||
case http.MethodGet:
|
case http.MethodGet:
|
||||||
|
@ -398,135 +385,3 @@ func determineGeneralOperation(r *http.Request) string {
|
||||||
}
|
}
|
||||||
return "UnmatchedOperation"
|
return "UnmatchedOperation"
|
||||||
}
|
}
|
||||||
|
|
||||||
func determineProperties(r *http.Request, decoder XMLDecoder, resolver BucketResolveFunc, tagging ResourceTagging, reqType ReqType,
|
|
||||||
op, bktName, objName, owner string, groups []string, userClaims map[string]string) (requestProperties map[string]string, resourceProperties map[string]string, err error) {
|
|
||||||
requestProperties = map[string]string{
|
|
||||||
s3.PropertyKeyOwner: owner,
|
|
||||||
common.PropertyKeyFrostFSIDGroupID: chain.FormCondSliceContainsValue(groups),
|
|
||||||
common.PropertyKeyFrostFSSourceIP: GetReqInfo(r.Context()).RemoteHost,
|
|
||||||
}
|
|
||||||
queries := GetReqInfo(r.Context()).URL.Query()
|
|
||||||
|
|
||||||
for k, v := range userClaims {
|
|
||||||
requestProperties[fmt.Sprintf(common.PropertyKeyFormatFrostFSIDUserClaim, k)] = v
|
|
||||||
}
|
|
||||||
|
|
||||||
if reqType == objectType {
|
|
||||||
if versionID := queries.Get(QueryVersionID); len(versionID) > 0 {
|
|
||||||
requestProperties[s3.PropertyKeyVersionID] = versionID
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if reqType == bucketType && (strings.HasSuffix(op, ListObjectsV1Operation) || strings.HasSuffix(op, ListObjectsV2Operation) ||
|
|
||||||
strings.HasSuffix(op, ListBucketObjectVersionsOperation) || strings.HasSuffix(op, ListMultipartUploadsOperation)) {
|
|
||||||
if prefix := queries.Get(QueryPrefix); len(prefix) > 0 {
|
|
||||||
requestProperties[s3.PropertyKeyPrefix] = prefix
|
|
||||||
}
|
|
||||||
if delimiter := queries.Get(QueryDelimiter); len(delimiter) > 0 {
|
|
||||||
requestProperties[s3.PropertyKeyDelimiter] = delimiter
|
|
||||||
}
|
|
||||||
if maxKeys := queries.Get(QueryMaxKeys); len(maxKeys) > 0 {
|
|
||||||
requestProperties[s3.PropertyKeyMaxKeys] = maxKeys
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
requestProperties[s3.PropertyKeyAccessBoxAttrMFA] = "false"
|
|
||||||
attrs, err := GetAccessBoxAttrs(r.Context())
|
|
||||||
if err == nil {
|
|
||||||
for _, attr := range attrs {
|
|
||||||
requestProperties[fmt.Sprintf(s3.PropertyKeyFormatAccessBoxAttr, attr.Key())] = attr.Value()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
reqTags, err := determineRequestTags(r, decoder, op)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("determine request tags: %w", err)
|
|
||||||
}
|
|
||||||
for k, v := range reqTags {
|
|
||||||
requestProperties[k] = v
|
|
||||||
}
|
|
||||||
|
|
||||||
resourceProperties, err = determineResourceTags(r.Context(), reqType, op, bktName, objName, queries.Get(QueryVersionID), resolver, tagging)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("determine resource tags: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return requestProperties, resourceProperties, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func determineRequestTags(r *http.Request, decoder XMLDecoder, op string) (map[string]string, error) {
|
|
||||||
tags := make(map[string]string)
|
|
||||||
|
|
||||||
if strings.HasSuffix(op, PutObjectTaggingOperation) || strings.HasSuffix(op, PutBucketTaggingOperation) {
|
|
||||||
tagging := new(data.Tagging)
|
|
||||||
if err := decoder.NewXMLDecoder(r.Body).Decode(tagging); err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrMalformedXML), err.Error())
|
|
||||||
}
|
|
||||||
GetReqInfo(r.Context()).Tagging = tagging
|
|
||||||
|
|
||||||
for _, tag := range tagging.TagSet {
|
|
||||||
tags[fmt.Sprintf(s3.PropertyKeyFormatRequestTag, tag.Key)] = tag.Value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if tagging := r.Header.Get(amzTagging); len(tagging) > 0 {
|
|
||||||
queries, err := url.ParseQuery(tagging)
|
|
||||||
if err != nil {
|
|
||||||
return nil, apiErr.GetAPIError(apiErr.ErrInvalidArgument)
|
|
||||||
}
|
|
||||||
for key := range queries {
|
|
||||||
tags[fmt.Sprintf(s3.PropertyKeyFormatRequestTag, key)] = queries.Get(key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return tags, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func determineResourceTags(ctx context.Context, reqType ReqType, op, bktName, objName, versionID string, resolver BucketResolveFunc,
|
|
||||||
tagging ResourceTagging) (map[string]string, error) {
|
|
||||||
tags := make(map[string]string)
|
|
||||||
|
|
||||||
if reqType != bucketType && reqType != objectType {
|
|
||||||
return tags, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, withoutResOp := range withoutResourceOps {
|
|
||||||
if strings.HasSuffix(op, withoutResOp) {
|
|
||||||
return tags, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
bktInfo, err := resolver(ctx, bktName)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("get bucket info: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if reqType == bucketType {
|
|
||||||
tags, err = tagging.GetBucketTagging(ctx, bktInfo)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("get bucket tagging: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if reqType == objectType {
|
|
||||||
tagPrm := &data.GetObjectTaggingParams{
|
|
||||||
ObjectVersion: &data.ObjectVersion{
|
|
||||||
BktInfo: bktInfo,
|
|
||||||
ObjectName: objName,
|
|
||||||
VersionID: versionID,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
_, tags, err = tagging.GetObjectTagging(ctx, tagPrm)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("get object tagging: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
res := make(map[string]string, len(tags))
|
|
||||||
for k, v := range tags {
|
|
||||||
res[fmt.Sprintf(s3.PropertyKeyFormatResourceTag, k)] = v
|
|
||||||
}
|
|
||||||
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -8,466 +8,75 @@ import (
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDetermineBucketOperation(t *testing.T) {
|
func TestReqTypeDetermination(t *testing.T) {
|
||||||
const defaultValue = "value"
|
bkt, obj, domain := "test-bucket", "test-object", "domain"
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
method string
|
target string
|
||||||
queryParam map[string]string
|
host string
|
||||||
expected string
|
domains []string
|
||||||
|
expectedType ReqType
|
||||||
|
expectedBktName string
|
||||||
|
expectedObjName string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "OptionsBucketOperation",
|
name: "bucket request, path-style",
|
||||||
method: http.MethodOptions,
|
target: "/" + bkt,
|
||||||
expected: OptionsBucketOperation,
|
expectedType: bucketType,
|
||||||
|
expectedBktName: bkt,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "HeadBucketOperation",
|
name: "bucket request with slash, path-style",
|
||||||
method: http.MethodHead,
|
target: "/" + bkt + "/",
|
||||||
expected: HeadBucketOperation,
|
expectedType: bucketType,
|
||||||
|
expectedBktName: bkt,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "ListMultipartUploadsOperation",
|
name: "object request, path-style",
|
||||||
method: http.MethodGet,
|
target: "/" + bkt + "/" + obj,
|
||||||
queryParam: map[string]string{UploadsQuery: defaultValue},
|
expectedType: objectType,
|
||||||
expected: ListMultipartUploadsOperation,
|
expectedBktName: bkt,
|
||||||
|
expectedObjName: obj,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "GetBucketLocationOperation",
|
name: "object request with slash, path-style",
|
||||||
method: http.MethodGet,
|
target: "/" + bkt + "/" + obj + "/",
|
||||||
queryParam: map[string]string{LocationQuery: defaultValue},
|
expectedType: objectType,
|
||||||
expected: GetBucketLocationOperation,
|
expectedBktName: bkt,
|
||||||
|
expectedObjName: obj + "/",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "GetBucketPolicyOperation",
|
name: "none type request",
|
||||||
method: http.MethodGet,
|
target: "/",
|
||||||
queryParam: map[string]string{PolicyQuery: defaultValue},
|
expectedType: noneType,
|
||||||
expected: GetBucketPolicyOperation,
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "GetBucketLifecycleOperation",
|
name: "bucket request, virtual-hosted style",
|
||||||
method: http.MethodGet,
|
target: "/",
|
||||||
queryParam: map[string]string{LifecycleQuery: defaultValue},
|
host: bkt + "." + domain,
|
||||||
expected: GetBucketLifecycleOperation,
|
domains: []string{"some-domain", domain},
|
||||||
|
expectedType: bucketType,
|
||||||
|
expectedBktName: bkt,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "GetBucketEncryptionOperation",
|
name: "object request, virtual-hosted style",
|
||||||
method: http.MethodGet,
|
target: "/" + obj,
|
||||||
queryParam: map[string]string{EncryptionQuery: defaultValue},
|
host: bkt + "." + domain,
|
||||||
expected: GetBucketEncryptionOperation,
|
domains: []string{"some-domain", domain},
|
||||||
},
|
expectedType: objectType,
|
||||||
{
|
expectedBktName: bkt,
|
||||||
name: "GetBucketCorsOperation",
|
expectedObjName: obj,
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{CorsQuery: defaultValue},
|
|
||||||
expected: GetBucketCorsOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetBucketACLOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{ACLQuery: defaultValue},
|
|
||||||
expected: GetBucketACLOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetBucketWebsiteOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{WebsiteQuery: defaultValue},
|
|
||||||
expected: GetBucketWebsiteOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetBucketAccelerateOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{AccelerateQuery: defaultValue},
|
|
||||||
expected: GetBucketAccelerateOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetBucketRequestPaymentOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{RequestPaymentQuery: defaultValue},
|
|
||||||
expected: GetBucketRequestPaymentOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetBucketLoggingOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{LoggingQuery: defaultValue},
|
|
||||||
expected: GetBucketLoggingOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetBucketReplicationOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{ReplicationQuery: defaultValue},
|
|
||||||
expected: GetBucketReplicationOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetBucketTaggingOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{TaggingQuery: defaultValue},
|
|
||||||
expected: GetBucketTaggingOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetBucketObjectLockConfigOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{ObjectLockQuery: defaultValue},
|
|
||||||
expected: GetBucketObjectLockConfigOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetBucketVersioningOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{VersioningQuery: defaultValue},
|
|
||||||
expected: GetBucketVersioningOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetBucketNotificationOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{NotificationQuery: defaultValue},
|
|
||||||
expected: GetBucketNotificationOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "ListenBucketNotificationOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{EventsQuery: defaultValue},
|
|
||||||
expected: ListenBucketNotificationOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "ListBucketObjectVersionsOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{VersionsQuery: defaultValue},
|
|
||||||
expected: ListBucketObjectVersionsOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "ListObjectsV2MOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{ListTypeQuery: "2", MetadataQuery: "true"},
|
|
||||||
expected: ListObjectsV2MOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "ListObjectsV2Operation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{ListTypeQuery: "2"},
|
|
||||||
expected: ListObjectsV2Operation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "ListObjectsV1Operation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
expected: ListObjectsV1Operation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutBucketCorsOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{CorsQuery: defaultValue},
|
|
||||||
expected: PutBucketCorsOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutBucketACLOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{ACLQuery: defaultValue},
|
|
||||||
expected: PutBucketACLOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutBucketLifecycleOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{LifecycleQuery: defaultValue},
|
|
||||||
expected: PutBucketLifecycleOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutBucketEncryptionOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{EncryptionQuery: defaultValue},
|
|
||||||
expected: PutBucketEncryptionOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutBucketPolicyOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{PolicyQuery: defaultValue},
|
|
||||||
expected: PutBucketPolicyOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutBucketObjectLockConfigOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{ObjectLockQuery: defaultValue},
|
|
||||||
expected: PutBucketObjectLockConfigOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutBucketTaggingOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{TaggingQuery: defaultValue},
|
|
||||||
expected: PutBucketTaggingOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutBucketVersioningOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{VersioningQuery: defaultValue},
|
|
||||||
expected: PutBucketVersioningOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutBucketNotificationOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{NotificationQuery: defaultValue},
|
|
||||||
expected: PutBucketNotificationOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "CreateBucketOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
expected: CreateBucketOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DeleteMultipleObjectsOperation",
|
|
||||||
method: http.MethodPost,
|
|
||||||
queryParam: map[string]string{DeleteQuery: defaultValue},
|
|
||||||
expected: DeleteMultipleObjectsOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PostObjectOperation",
|
|
||||||
method: http.MethodPost,
|
|
||||||
expected: PostObjectOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DeleteBucketCorsOperation",
|
|
||||||
method: http.MethodDelete,
|
|
||||||
queryParam: map[string]string{CorsQuery: defaultValue},
|
|
||||||
expected: DeleteBucketCorsOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DeleteBucketWebsiteOperation",
|
|
||||||
method: http.MethodDelete,
|
|
||||||
queryParam: map[string]string{WebsiteQuery: defaultValue},
|
|
||||||
expected: DeleteBucketWebsiteOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DeleteBucketTaggingOperation",
|
|
||||||
method: http.MethodDelete,
|
|
||||||
queryParam: map[string]string{TaggingQuery: defaultValue},
|
|
||||||
expected: DeleteBucketTaggingOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DeleteBucketPolicyOperation",
|
|
||||||
method: http.MethodDelete,
|
|
||||||
queryParam: map[string]string{PolicyQuery: defaultValue},
|
|
||||||
expected: DeleteBucketPolicyOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DeleteBucketLifecycleOperation",
|
|
||||||
method: http.MethodDelete,
|
|
||||||
queryParam: map[string]string{LifecycleQuery: defaultValue},
|
|
||||||
expected: DeleteBucketLifecycleOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DeleteBucketEncryptionOperation",
|
|
||||||
method: http.MethodDelete,
|
|
||||||
queryParam: map[string]string{EncryptionQuery: defaultValue},
|
|
||||||
expected: DeleteBucketEncryptionOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DeleteBucketOperation",
|
|
||||||
method: http.MethodDelete,
|
|
||||||
expected: DeleteBucketOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "UnmatchedBucketOperation",
|
|
||||||
method: "invalid-method",
|
|
||||||
expected: "UnmatchedBucketOperation",
|
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
req := httptest.NewRequest(tc.method, "/test", nil)
|
r := httptest.NewRequest(http.MethodPut, tc.target, nil)
|
||||||
if tc.queryParam != nil {
|
r.Host = tc.host
|
||||||
addQueryParams(req, tc.queryParam)
|
|
||||||
}
|
|
||||||
|
|
||||||
actual := determineBucketOperation(req)
|
reqType, bktName, objName := getBucketObject(r, tc.domains)
|
||||||
require.Equal(t, tc.expected, actual)
|
require.Equal(t, tc.expectedType, reqType)
|
||||||
|
require.Equal(t, tc.expectedBktName, bktName)
|
||||||
|
require.Equal(t, tc.expectedObjName, objName)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDetermineObjectOperation(t *testing.T) {
|
|
||||||
const (
|
|
||||||
amzCopySource = "X-Amz-Copy-Source"
|
|
||||||
defaultValue = "value"
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
method string
|
|
||||||
queryParam map[string]string
|
|
||||||
headerKeys []string
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "OptionsObjectOperation",
|
|
||||||
method: http.MethodOptions,
|
|
||||||
expected: OptionsObjectOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "HeadObjectOperation",
|
|
||||||
method: http.MethodHead,
|
|
||||||
expected: HeadObjectOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "ListPartsOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{UploadIDQuery: defaultValue},
|
|
||||||
expected: ListPartsOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetObjectACLOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{ACLQuery: defaultValue},
|
|
||||||
expected: GetObjectACLOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetObjectTaggingOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{TaggingQuery: defaultValue},
|
|
||||||
expected: GetObjectTaggingOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetObjectRetentionOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{RetentionQuery: defaultValue},
|
|
||||||
expected: GetObjectRetentionOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetObjectLegalHoldOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{LegalQuery: defaultValue},
|
|
||||||
expected: GetObjectLegalHoldOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetObjectAttributesOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
queryParam: map[string]string{AttributesQuery: defaultValue},
|
|
||||||
expected: GetObjectAttributesOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GetObjectOperation",
|
|
||||||
method: http.MethodGet,
|
|
||||||
expected: GetObjectOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "UploadPartCopyOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{PartNumberQuery: defaultValue, UploadIDQuery: defaultValue},
|
|
||||||
headerKeys: []string{amzCopySource},
|
|
||||||
expected: UploadPartCopyOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "UploadPartOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{PartNumberQuery: defaultValue, UploadIDQuery: defaultValue},
|
|
||||||
expected: UploadPartOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutObjectACLOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{ACLQuery: defaultValue},
|
|
||||||
expected: PutObjectACLOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutObjectTaggingOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{TaggingQuery: defaultValue},
|
|
||||||
expected: PutObjectTaggingOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "CopyObjectOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
headerKeys: []string{amzCopySource},
|
|
||||||
expected: CopyObjectOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutObjectRetentionOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{RetentionQuery: defaultValue},
|
|
||||||
expected: PutObjectRetentionOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutObjectLegalHoldOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
queryParam: map[string]string{LegalHoldQuery: defaultValue},
|
|
||||||
expected: PutObjectLegalHoldOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PutObjectOperation",
|
|
||||||
method: http.MethodPut,
|
|
||||||
expected: PutObjectOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "CompleteMultipartUploadOperation",
|
|
||||||
method: http.MethodPost,
|
|
||||||
queryParam: map[string]string{UploadIDQuery: defaultValue},
|
|
||||||
expected: CompleteMultipartUploadOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "CreateMultipartUploadOperation",
|
|
||||||
method: http.MethodPost,
|
|
||||||
queryParam: map[string]string{UploadsQuery: defaultValue},
|
|
||||||
expected: CreateMultipartUploadOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "SelectObjectContentOperation",
|
|
||||||
method: http.MethodPost,
|
|
||||||
expected: SelectObjectContentOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "AbortMultipartUploadOperation",
|
|
||||||
method: http.MethodDelete,
|
|
||||||
queryParam: map[string]string{UploadIDQuery: defaultValue},
|
|
||||||
expected: AbortMultipartUploadOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DeleteObjectTaggingOperation",
|
|
||||||
method: http.MethodDelete,
|
|
||||||
queryParam: map[string]string{TaggingQuery: defaultValue},
|
|
||||||
expected: DeleteObjectTaggingOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DeleteObjectOperation",
|
|
||||||
method: http.MethodDelete,
|
|
||||||
expected: DeleteObjectOperation,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "UnmatchedObjectOperation",
|
|
||||||
method: "invalid-method",
|
|
||||||
expected: "UnmatchedObjectOperation",
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
req := httptest.NewRequest(tc.method, "/test", nil)
|
|
||||||
if tc.queryParam != nil {
|
|
||||||
addQueryParams(req, tc.queryParam)
|
|
||||||
}
|
|
||||||
if tc.headerKeys != nil {
|
|
||||||
addHeaderParams(req, tc.headerKeys)
|
|
||||||
}
|
|
||||||
|
|
||||||
actual := determineObjectOperation(req)
|
|
||||||
require.Equal(t, tc.expected, actual)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func addQueryParams(req *http.Request, pairs map[string]string) {
|
|
||||||
values := req.URL.Query()
|
|
||||||
for key, val := range pairs {
|
|
||||||
values.Add(key, val)
|
|
||||||
}
|
|
||||||
req.URL.RawQuery = values.Encode()
|
|
||||||
}
|
|
||||||
|
|
||||||
func addHeaderParams(req *http.Request, keys []string) {
|
|
||||||
for _, key := range keys {
|
|
||||||
req.Header.Set(key, "val")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDetermineGeneralOperation(t *testing.T) {
|
|
||||||
req := httptest.NewRequest(http.MethodGet, "/test", nil)
|
|
||||||
actual := determineGeneralOperation(req)
|
|
||||||
require.Equal(t, ListBucketsOperation, actual)
|
|
||||||
|
|
||||||
req = httptest.NewRequest(http.MethodPost, "/test", nil)
|
|
||||||
actual = determineGeneralOperation(req)
|
|
||||||
require.Equal(t, "UnmatchedOperation", actual)
|
|
||||||
}
|
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
"google.golang.org/grpc/metadata"
|
"google.golang.org/grpc/metadata"
|
||||||
|
@ -39,9 +40,7 @@ type (
|
||||||
URL *url.URL // Request url
|
URL *url.URL // Request url
|
||||||
Namespace string
|
Namespace string
|
||||||
User string // User owner id
|
User string // User owner id
|
||||||
Tagging *data.Tagging
|
tags []KeyVal // Any additional info not accommodated by above fields
|
||||||
RequestVHSEnabled bool
|
|
||||||
RequestType ReqType
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ObjectRequest represents object request data.
|
// ObjectRequest represents object request data.
|
||||||
|
@ -62,14 +61,16 @@ const (
|
||||||
|
|
||||||
const HdrAmzRequestID = "x-amz-request-id"
|
const HdrAmzRequestID = "x-amz-request-id"
|
||||||
|
|
||||||
|
const (
|
||||||
|
BucketURLPrm = "bucket"
|
||||||
|
)
|
||||||
|
|
||||||
var deploymentID = uuid.Must(uuid.NewRandom())
|
var deploymentID = uuid.Must(uuid.NewRandom())
|
||||||
|
|
||||||
var (
|
var (
|
||||||
// De-facto standard header keys.
|
// De-facto standard header keys.
|
||||||
xForwardedFor = http.CanonicalHeaderKey("X-Forwarded-For")
|
xForwardedFor = http.CanonicalHeaderKey("X-Forwarded-For")
|
||||||
xRealIP = http.CanonicalHeaderKey("X-Real-IP")
|
xRealIP = http.CanonicalHeaderKey("X-Real-IP")
|
||||||
xForwardedProto = http.CanonicalHeaderKey("X-Forwarded-Proto")
|
|
||||||
xForwardedScheme = http.CanonicalHeaderKey("X-Forwarded-Scheme")
|
|
||||||
|
|
||||||
// RFC7239 defines a new "Forwarded: " header designed to replace the
|
// RFC7239 defines a new "Forwarded: " header designed to replace the
|
||||||
// existing use of X-Forwarded-* headers.
|
// existing use of X-Forwarded-* headers.
|
||||||
|
@ -78,30 +79,64 @@ var (
|
||||||
// Allows for a sub-match of the first value after 'for=' to the next
|
// Allows for a sub-match of the first value after 'for=' to the next
|
||||||
// comma, semi-colon or space. The match is case-insensitive.
|
// comma, semi-colon or space. The match is case-insensitive.
|
||||||
forRegex = regexp.MustCompile(`(?i)(?:for=)([^(;|, )]+)(.*)`)
|
forRegex = regexp.MustCompile(`(?i)(?:for=)([^(;|, )]+)(.*)`)
|
||||||
// Allows for a sub-match for the first instance of scheme (http|https)
|
|
||||||
// prefixed by 'proto='. The match is case-insensitive.
|
|
||||||
protoRegex = regexp.MustCompile(`(?i)^(;|,| )+(?:proto=)(https|http)`)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewReqInfo returns new ReqInfo based on parameters.
|
// NewReqInfo returns new ReqInfo based on parameters.
|
||||||
func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest, sourceIPHeader string) *ReqInfo {
|
func NewReqInfo(w http.ResponseWriter, r *http.Request, req ObjectRequest) *ReqInfo {
|
||||||
reqInfo := &ReqInfo{
|
return &ReqInfo{
|
||||||
API: req.Method,
|
API: req.Method,
|
||||||
BucketName: req.Bucket,
|
BucketName: req.Bucket,
|
||||||
ObjectName: req.Object,
|
ObjectName: req.Object,
|
||||||
UserAgent: r.UserAgent(),
|
UserAgent: r.UserAgent(),
|
||||||
|
RemoteHost: getSourceIP(r),
|
||||||
RequestID: GetRequestID(w),
|
RequestID: GetRequestID(w),
|
||||||
DeploymentID: deploymentID.String(),
|
DeploymentID: deploymentID.String(),
|
||||||
URL: r.URL,
|
URL: r.URL,
|
||||||
}
|
}
|
||||||
|
|
||||||
if sourceIPHeader != "" {
|
|
||||||
reqInfo.RemoteHost = r.Header.Get(sourceIPHeader)
|
|
||||||
} else {
|
|
||||||
reqInfo.RemoteHost = getSourceIP(r)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return reqInfo
|
// AppendTags -- appends key/val to ReqInfo.tags.
|
||||||
|
func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo {
|
||||||
|
if r == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
r.Lock()
|
||||||
|
defer r.Unlock()
|
||||||
|
r.tags = append(r.tags, KeyVal{key, val})
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetTags -- sets key/val to ReqInfo.tags.
|
||||||
|
func (r *ReqInfo) SetTags(key string, val string) *ReqInfo {
|
||||||
|
if r == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
r.Lock()
|
||||||
|
defer r.Unlock()
|
||||||
|
// Search for a tag key already existing in tags
|
||||||
|
var updated bool
|
||||||
|
for _, tag := range r.tags {
|
||||||
|
if tag.Key == key {
|
||||||
|
tag.Val = val
|
||||||
|
updated = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !updated {
|
||||||
|
// Append to the end of tags list
|
||||||
|
r.tags = append(r.tags, KeyVal{key, val})
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTags -- returns the user defined tags.
|
||||||
|
func (r *ReqInfo) GetTags() []KeyVal {
|
||||||
|
if r == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
r.RLock()
|
||||||
|
defer r.RUnlock()
|
||||||
|
return append([]KeyVal(nil), r.tags...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetRequestID returns the request ID from the response writer or the context.
|
// GetRequestID returns the request ID from the response writer or the context.
|
||||||
|
@ -157,7 +192,6 @@ func GetReqLog(ctx context.Context) *zap.Logger {
|
||||||
type RequestSettings interface {
|
type RequestSettings interface {
|
||||||
NamespaceHeader() string
|
NamespaceHeader() string
|
||||||
ResolveNamespaceAlias(string) string
|
ResolveNamespaceAlias(string) string
|
||||||
SourceIPHeader() string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func Request(log *zap.Logger, settings RequestSettings) Func {
|
func Request(log *zap.Logger, settings RequestSettings) Func {
|
||||||
|
@ -176,7 +210,7 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
|
||||||
|
|
||||||
// set request info into context
|
// set request info into context
|
||||||
// bucket name and object will be set in reqInfo later (limitation of go-chi)
|
// bucket name and object will be set in reqInfo later (limitation of go-chi)
|
||||||
reqInfo := NewReqInfo(w, r, ObjectRequest{}, settings.SourceIPHeader())
|
reqInfo := NewReqInfo(w, r, ObjectRequest{})
|
||||||
reqInfo.Namespace = settings.ResolveNamespaceAlias(r.Header.Get(settings.NamespaceHeader()))
|
reqInfo.Namespace = settings.ResolveNamespaceAlias(r.Header.Get(settings.NamespaceHeader()))
|
||||||
r = r.WithContext(SetReqInfo(r.Context(), reqInfo))
|
r = r.WithContext(SetReqInfo(r.Context(), reqInfo))
|
||||||
|
|
||||||
|
@ -199,6 +233,57 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AddBucketName adds bucket name to ReqInfo from context.
|
||||||
|
func AddBucketName(l *zap.Logger) Func {
|
||||||
|
return func(h http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := r.Context()
|
||||||
|
|
||||||
|
reqInfo := GetReqInfo(ctx)
|
||||||
|
reqInfo.BucketName = chi.URLParam(r, BucketURLPrm)
|
||||||
|
|
||||||
|
if reqInfo.BucketName != "" {
|
||||||
|
reqLogger := reqLogOrDefault(ctx, l)
|
||||||
|
r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("bucket", reqInfo.BucketName))))
|
||||||
|
}
|
||||||
|
|
||||||
|
h.ServeHTTP(w, r)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddObjectName adds objects name to ReqInfo from context.
|
||||||
|
func AddObjectName(l *zap.Logger) Func {
|
||||||
|
return func(h http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := r.Context()
|
||||||
|
reqInfo := GetReqInfo(ctx)
|
||||||
|
reqLogger := reqLogOrDefault(ctx, l)
|
||||||
|
|
||||||
|
rctx := chi.RouteContext(ctx)
|
||||||
|
// trim leading slash (always present)
|
||||||
|
reqInfo.ObjectName = rctx.RoutePath[1:]
|
||||||
|
|
||||||
|
if r.URL.RawPath != "" {
|
||||||
|
// we have to do this because of
|
||||||
|
// https://github.com/go-chi/chi/issues/641
|
||||||
|
// https://github.com/go-chi/chi/issues/642
|
||||||
|
if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil {
|
||||||
|
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err))
|
||||||
|
} else {
|
||||||
|
reqInfo.ObjectName = obj
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if reqInfo.ObjectName != "" {
|
||||||
|
r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("object", reqInfo.ObjectName))))
|
||||||
|
}
|
||||||
|
|
||||||
|
h.ServeHTTP(w, r)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// getSourceIP retrieves the IP from the X-Forwarded-For, X-Real-IP and RFC7239
|
// getSourceIP retrieves the IP from the X-Forwarded-For, X-Real-IP and RFC7239
|
||||||
// Forwarded headers (in that order), falls back to r.RemoteAddr when everything
|
// Forwarded headers (in that order), falls back to r.RemoteAddr when everything
|
||||||
// else fails.
|
// else fails.
|
||||||
|
@ -231,42 +316,11 @@ func getSourceIP(r *http.Request) string {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if addr == "" {
|
if addr != "" {
|
||||||
addr = r.RemoteAddr
|
return addr
|
||||||
}
|
}
|
||||||
|
|
||||||
// Default to remote address if headers not set.
|
// Default to remote address if headers not set.
|
||||||
raddr, _, _ := net.SplitHostPort(addr)
|
addr, _, _ = net.SplitHostPort(r.RemoteAddr)
|
||||||
if raddr == "" {
|
|
||||||
return addr
|
return addr
|
||||||
}
|
}
|
||||||
return raddr
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetSourceScheme retrieves the scheme from the X-Forwarded-Proto and RFC7239
|
|
||||||
// Forwarded headers (in that order).
|
|
||||||
func GetSourceScheme(r *http.Request) string {
|
|
||||||
var scheme string
|
|
||||||
|
|
||||||
// Retrieve the scheme from X-Forwarded-Proto.
|
|
||||||
if proto := r.Header.Get(xForwardedProto); proto != "" {
|
|
||||||
scheme = strings.ToLower(proto)
|
|
||||||
} else if proto = r.Header.Get(xForwardedScheme); proto != "" {
|
|
||||||
scheme = strings.ToLower(proto)
|
|
||||||
} else if proto := r.Header.Get(forwarded); proto != "" {
|
|
||||||
// match should contain at least two elements if the protocol was
|
|
||||||
// specified in the Forwarded header. The first element will always be
|
|
||||||
// the 'for=', which we ignore, subsequently we proceed to look for
|
|
||||||
// 'proto=' which should precede right after `for=` if not
|
|
||||||
// we simply ignore the values and return empty. This is in line
|
|
||||||
// with the approach we took for returning first ip from multiple
|
|
||||||
// params.
|
|
||||||
if match := forRegex.FindStringSubmatch(proto); len(match) > 1 {
|
|
||||||
if match = protoRegex.FindStringSubmatch(match[2]); len(match) > 1 {
|
|
||||||
scheme = strings.ToLower(match[2])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return scheme
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,70 +0,0 @@
|
||||||
package middleware
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestGetSourceIP(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
req *http.Request
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "headers not set",
|
|
||||||
req: func() *http.Request {
|
|
||||||
request := httptest.NewRequest(http.MethodGet, "/test", nil)
|
|
||||||
request.RemoteAddr = "192.0.2.1:1234"
|
|
||||||
return request
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "headers not set, and the port is not set",
|
|
||||||
req: func() *http.Request {
|
|
||||||
request := httptest.NewRequest(http.MethodGet, "/test", nil)
|
|
||||||
request.RemoteAddr = "192.0.2.1"
|
|
||||||
return request
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "x-forwarded-for single-host header",
|
|
||||||
req: func() *http.Request {
|
|
||||||
request := httptest.NewRequest(http.MethodGet, "/test", nil)
|
|
||||||
request.Header.Set(xForwardedFor, "192.0.2.1")
|
|
||||||
return request
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "x-forwarded-for header by multiple hosts",
|
|
||||||
req: func() *http.Request {
|
|
||||||
request := httptest.NewRequest(http.MethodGet, "/test", nil)
|
|
||||||
request.Header.Set(xForwardedFor, "192.0.2.1, 10.1.1.1")
|
|
||||||
return request
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "x-real-ip header",
|
|
||||||
req: func() *http.Request {
|
|
||||||
request := httptest.NewRequest(http.MethodGet, "/test", nil)
|
|
||||||
request.Header.Set(xRealIP, "192.0.2.1")
|
|
||||||
return request
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "forwarded header",
|
|
||||||
req: func() *http.Request {
|
|
||||||
request := httptest.NewRequest(http.MethodGet, "/test", nil)
|
|
||||||
request.Header.Set(forwarded, "for=192.0.2.1, 10.1.1.1; proto=https; by=192.0.2.4")
|
|
||||||
return request
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
actual := getSourceIP(tc.req)
|
|
||||||
require.Equal(t, actual, "192.0.2.1")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -201,6 +201,18 @@ func EncodeResponse(response interface{}) ([]byte, error) {
|
||||||
return bytesBuffer.Bytes(), nil
|
return bytesBuffer.Bytes(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EncodeResponseNoHeader encodes response without setting xml.Header.
|
||||||
|
// Should be used with periodicXMLWriter which sends xml.Header to the client
|
||||||
|
// with whitespaces to keep connection alive.
|
||||||
|
func EncodeResponseNoHeader(response interface{}) ([]byte, error) {
|
||||||
|
var bytesBuffer bytes.Buffer
|
||||||
|
if err := xml.NewEncoder(&bytesBuffer).Encode(response); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return bytesBuffer.Bytes(), nil
|
||||||
|
}
|
||||||
|
|
||||||
// EncodeToResponse encodes the response into ResponseWriter.
|
// EncodeToResponse encodes the response into ResponseWriter.
|
||||||
func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
|
func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
|
@ -328,9 +340,6 @@ func LogSuccessResponse(l *zap.Logger) Func {
|
||||||
if reqInfo.ObjectName != "" {
|
if reqInfo.ObjectName != "" {
|
||||||
fields = append(fields, zap.String("object", reqInfo.ObjectName))
|
fields = append(fields, zap.String("object", reqInfo.ObjectName))
|
||||||
}
|
}
|
||||||
if reqInfo.User != "" {
|
|
||||||
fields = append(fields, zap.String("user", reqInfo.User))
|
|
||||||
}
|
|
||||||
|
|
||||||
if traceID, err := trace.TraceIDFromHex(reqInfo.TraceID); err == nil && traceID.IsValid() {
|
if traceID, err := trace.TraceIDFromHex(reqInfo.TraceID); err == nil && traceID.IsValid() {
|
||||||
fields = append(fields, zap.String("trace_id", reqInfo.TraceID))
|
fields = append(fields, zap.String("trace_id", reqInfo.TraceID))
|
||||||
|
|
|
@ -1,43 +0,0 @@
|
||||||
package middleware
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/xml"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
type testXMLData struct {
|
|
||||||
XMLName xml.Name `xml:"data"`
|
|
||||||
Text string `xml:"text"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEncodeResponse(t *testing.T) {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
|
|
||||||
err := EncodeToResponse(w, []byte{})
|
|
||||||
require.Error(t, err)
|
|
||||||
require.Contains(t, err.Error(), "encode xml response")
|
|
||||||
|
|
||||||
err = EncodeToResponse(w, testXMLData{Text: "test"})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
expectedXML := "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<data><text>test</text></data>"
|
|
||||||
require.Equal(t, expectedXML, w.Body.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestErrorResponse(t *testing.T) {
|
|
||||||
errResp := ErrorResponse{Code: "invalid-code"}
|
|
||||||
|
|
||||||
actual := errResp.Error()
|
|
||||||
require.Contains(t, actual, "Error response code")
|
|
||||||
|
|
||||||
errResp.Code = "AccessDenied"
|
|
||||||
actual = errResp.Error()
|
|
||||||
require.Equal(t, "Access Denied.", actual)
|
|
||||||
|
|
||||||
errResp.Message = "Request body is empty."
|
|
||||||
actual = errResp.Error()
|
|
||||||
require.Equal(t, "Request body is empty.", actual)
|
|
||||||
}
|
|
|
@ -1,115 +0,0 @@
|
||||||
package middleware
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestHTTPResponseCarrierSetGet(t *testing.T) {
|
|
||||||
const (
|
|
||||||
testKey1 = "Key"
|
|
||||||
testValue1 = "Value"
|
|
||||||
)
|
|
||||||
|
|
||||||
respCarrier := httpResponseCarrier{}
|
|
||||||
respCarrier.resp = httptest.NewRecorder()
|
|
||||||
|
|
||||||
actual := respCarrier.Get(testKey1)
|
|
||||||
require.Equal(t, "", actual)
|
|
||||||
|
|
||||||
respCarrier.Set(testKey1, testValue1)
|
|
||||||
actual = respCarrier.Get(testKey1)
|
|
||||||
require.Equal(t, testValue1, actual)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHTTPResponseCarrierKeys(t *testing.T) {
|
|
||||||
const (
|
|
||||||
testKey1 = "Key1"
|
|
||||||
testKey2 = "Key2"
|
|
||||||
testKey3 = "Key3"
|
|
||||||
testValue1 = "Value1"
|
|
||||||
testValue2 = "Value2"
|
|
||||||
testValue3 = "Value3"
|
|
||||||
)
|
|
||||||
|
|
||||||
respCarrier := httpResponseCarrier{}
|
|
||||||
respCarrier.resp = httptest.NewRecorder()
|
|
||||||
|
|
||||||
actual := respCarrier.Keys()
|
|
||||||
require.Equal(t, 0, len(actual))
|
|
||||||
|
|
||||||
respCarrier.Set(testKey1, testValue1)
|
|
||||||
respCarrier.Set(testKey2, testValue2)
|
|
||||||
respCarrier.Set(testKey3, testValue3)
|
|
||||||
|
|
||||||
actual = respCarrier.Keys()
|
|
||||||
require.Equal(t, 3, len(actual))
|
|
||||||
require.Contains(t, actual, testKey1)
|
|
||||||
require.Contains(t, actual, testKey2)
|
|
||||||
require.Contains(t, actual, testKey3)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHTTPRequestCarrierSet(t *testing.T) {
|
|
||||||
const (
|
|
||||||
testKey = "Key"
|
|
||||||
testValue = "Value"
|
|
||||||
)
|
|
||||||
|
|
||||||
reqCarrier := httpRequestCarrier{}
|
|
||||||
reqCarrier.req = httptest.NewRequest(http.MethodGet, "/test", nil)
|
|
||||||
reqCarrier.req.Response = httptest.NewRecorder().Result()
|
|
||||||
|
|
||||||
actual := reqCarrier.req.Response.Header.Get(testKey)
|
|
||||||
require.Equal(t, "", actual)
|
|
||||||
|
|
||||||
reqCarrier.Set(testKey, testValue)
|
|
||||||
actual = reqCarrier.req.Response.Header.Get(testKey)
|
|
||||||
require.Contains(t, testValue, actual)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHTTPRequestCarrierGet(t *testing.T) {
|
|
||||||
const (
|
|
||||||
testKey = "Key"
|
|
||||||
testValue = "Value"
|
|
||||||
)
|
|
||||||
|
|
||||||
reqCarrier := httpRequestCarrier{}
|
|
||||||
reqCarrier.req = httptest.NewRequest(http.MethodGet, "/test", nil)
|
|
||||||
|
|
||||||
actual := reqCarrier.Get(testKey)
|
|
||||||
require.Equal(t, "", actual)
|
|
||||||
|
|
||||||
reqCarrier.req.Header.Set(testKey, testValue)
|
|
||||||
actual = reqCarrier.Get(testKey)
|
|
||||||
require.Equal(t, testValue, actual)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHTTPRequestCarrierKeys(t *testing.T) {
|
|
||||||
const (
|
|
||||||
testKey1 = "Key1"
|
|
||||||
testKey2 = "Key2"
|
|
||||||
testKey3 = "Key3"
|
|
||||||
testValue1 = "Value1"
|
|
||||||
testValue2 = "Value2"
|
|
||||||
testValue3 = "Value3"
|
|
||||||
)
|
|
||||||
|
|
||||||
reqCarrier := httpRequestCarrier{}
|
|
||||||
reqCarrier.req = httptest.NewRequest(http.MethodGet, "/test", nil)
|
|
||||||
|
|
||||||
actual := reqCarrier.Keys()
|
|
||||||
require.Equal(t, 0, len(actual))
|
|
||||||
|
|
||||||
reqCarrier.req.Header.Set(testKey1, testValue1)
|
|
||||||
reqCarrier.req.Header.Set(testKey2, testValue2)
|
|
||||||
reqCarrier.req.Header.Set(testKey3, testValue3)
|
|
||||||
|
|
||||||
actual = reqCarrier.Keys()
|
|
||||||
require.Equal(t, 3, len(actual))
|
|
||||||
require.Contains(t, actual, testKey1)
|
|
||||||
require.Contains(t, actual, testKey2)
|
|
||||||
require.Contains(t, actual, testKey3)
|
|
||||||
}
|
|
|
@ -6,27 +6,29 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// keyWrapper is wrapper for context keys.
|
// keyWrapper is wrapper for context keys.
|
||||||
type keyWrapper string
|
type keyWrapper string
|
||||||
|
|
||||||
// boxKey is an ID used to store Box in a context.
|
// authHeaders is a wrapper for authentication headers of a request.
|
||||||
var boxKey = keyWrapper("__context_box_key")
|
var authHeadersKey = keyWrapper("__context_auth_headers_key")
|
||||||
|
|
||||||
|
// boxData is an ID used to store accessbox.Box in a context.
|
||||||
|
var boxDataKey = keyWrapper("__context_box_key")
|
||||||
|
|
||||||
|
// clientTime is an ID used to store client time.Time in a context.
|
||||||
|
var clientTimeKey = keyWrapper("__context_client_time")
|
||||||
|
|
||||||
// GetBoxData extracts accessbox.Box from context.
|
// GetBoxData extracts accessbox.Box from context.
|
||||||
func GetBoxData(ctx context.Context) (*accessbox.Box, error) {
|
func GetBoxData(ctx context.Context) (*accessbox.Box, error) {
|
||||||
data, ok := ctx.Value(boxKey).(*Box)
|
var box *accessbox.Box
|
||||||
|
data, ok := ctx.Value(boxDataKey).(*accessbox.Box)
|
||||||
if !ok || data == nil {
|
if !ok || data == nil {
|
||||||
return nil, fmt.Errorf("couldn't get box from context")
|
|
||||||
}
|
|
||||||
|
|
||||||
if data.AccessBox == nil {
|
|
||||||
return nil, fmt.Errorf("couldn't get box data from context")
|
return nil, fmt.Errorf("couldn't get box data from context")
|
||||||
}
|
}
|
||||||
|
|
||||||
box := data.AccessBox
|
box = data
|
||||||
if box.Gate == nil {
|
if box.Gate == nil {
|
||||||
box.Gate = &accessbox.GateData{}
|
box.Gate = &accessbox.GateData{}
|
||||||
}
|
}
|
||||||
|
@ -35,39 +37,35 @@ func GetBoxData(ctx context.Context) (*accessbox.Box, error) {
|
||||||
|
|
||||||
// GetAuthHeaders extracts auth.AuthHeader from context.
|
// GetAuthHeaders extracts auth.AuthHeader from context.
|
||||||
func GetAuthHeaders(ctx context.Context) (*AuthHeader, error) {
|
func GetAuthHeaders(ctx context.Context) (*AuthHeader, error) {
|
||||||
data, ok := ctx.Value(boxKey).(*Box)
|
authHeaders, ok := ctx.Value(authHeadersKey).(*AuthHeader)
|
||||||
if !ok || data == nil {
|
if !ok {
|
||||||
return nil, fmt.Errorf("couldn't get box from context")
|
return nil, fmt.Errorf("couldn't get auth headers from context")
|
||||||
}
|
}
|
||||||
|
|
||||||
return data.AuthHeaders, nil
|
return authHeaders, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetClientTime extracts time.Time from context.
|
// GetClientTime extracts time.Time from context.
|
||||||
func GetClientTime(ctx context.Context) (time.Time, error) {
|
func GetClientTime(ctx context.Context) (time.Time, error) {
|
||||||
data, ok := ctx.Value(boxKey).(*Box)
|
clientTime, ok := ctx.Value(clientTimeKey).(time.Time)
|
||||||
if !ok || data == nil {
|
if !ok {
|
||||||
return time.Time{}, fmt.Errorf("couldn't get box from context")
|
|
||||||
}
|
|
||||||
|
|
||||||
if data.ClientTime.IsZero() {
|
|
||||||
return time.Time{}, fmt.Errorf("couldn't get client time from context")
|
return time.Time{}, fmt.Errorf("couldn't get client time from context")
|
||||||
}
|
}
|
||||||
|
|
||||||
return data.ClientTime, nil
|
return clientTime, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetAccessBoxAttrs extracts []object.Attribute from context.
|
// SetBoxData sets accessbox.Box in the context.
|
||||||
func GetAccessBoxAttrs(ctx context.Context) ([]object.Attribute, error) {
|
func SetBoxData(ctx context.Context, box *accessbox.Box) context.Context {
|
||||||
data, ok := ctx.Value(boxKey).(*Box)
|
return context.WithValue(ctx, boxDataKey, box)
|
||||||
if !ok || data == nil {
|
|
||||||
return nil, fmt.Errorf("couldn't get box from context")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return data.Attributes, nil
|
// SetAuthHeaders sets auth.AuthHeader in the context.
|
||||||
|
func SetAuthHeaders(ctx context.Context, header *AuthHeader) context.Context {
|
||||||
|
return context.WithValue(ctx, authHeadersKey, header)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetBox sets Box in the context.
|
// SetClientTime sets time.Time in the context.
|
||||||
func SetBox(ctx context.Context, box *Box) context.Context {
|
func SetClientTime(ctx context.Context, newTime time.Time) context.Context {
|
||||||
return context.WithValue(ctx, boxKey, box)
|
return context.WithValue(ctx, clientTimeKey, newTime)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,179 +0,0 @@
|
||||||
package middleware
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestGetBoxData(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
value any
|
|
||||||
error string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "valid",
|
|
||||||
value: &Box{
|
|
||||||
AccessBox: &accessbox.Box{},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid data",
|
|
||||||
value: "invalid-data",
|
|
||||||
error: "couldn't get box from context",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "box does not exist",
|
|
||||||
error: "couldn't get box from context",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "access box is nil",
|
|
||||||
value: &Box{},
|
|
||||||
error: "couldn't get box data from context",
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
ctx := context.WithValue(context.Background(), boxKey, tc.value)
|
|
||||||
actual, err := GetBoxData(ctx)
|
|
||||||
if tc.error != "" {
|
|
||||||
require.Contains(t, err.Error(), tc.error)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, actual)
|
|
||||||
require.NotNil(t, actual.Gate)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetAuthHeaders(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
value any
|
|
||||||
error bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "valid",
|
|
||||||
value: &Box{
|
|
||||||
AuthHeaders: &AuthHeader{
|
|
||||||
AccessKeyID: "valid-key",
|
|
||||||
Region: "valid-region",
|
|
||||||
SignatureV4: "valid-sign",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid data",
|
|
||||||
value: "invalid-data",
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "box does not exist",
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
ctx := context.WithValue(context.Background(), boxKey, tc.value)
|
|
||||||
actual, err := GetAuthHeaders(ctx)
|
|
||||||
if tc.error {
|
|
||||||
require.Contains(t, err.Error(), "couldn't get box from context")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, tc.value.(*Box).AuthHeaders.AccessKeyID, actual.AccessKeyID)
|
|
||||||
require.Equal(t, tc.value.(*Box).AuthHeaders.Region, actual.Region)
|
|
||||||
require.Equal(t, tc.value.(*Box).AuthHeaders.SignatureV4, actual.SignatureV4)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetClientTime(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
value any
|
|
||||||
error string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "valid",
|
|
||||||
value: &Box{
|
|
||||||
ClientTime: time.Now(),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid data",
|
|
||||||
value: "invalid-data",
|
|
||||||
error: "couldn't get box from context",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "box does not exist",
|
|
||||||
error: "couldn't get box from context",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "zero time",
|
|
||||||
value: &Box{
|
|
||||||
ClientTime: time.Time{},
|
|
||||||
},
|
|
||||||
error: "couldn't get client time from context",
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
ctx := context.WithValue(context.Background(), boxKey, tc.value)
|
|
||||||
actual, err := GetClientTime(ctx)
|
|
||||||
if tc.error != "" {
|
|
||||||
require.Contains(t, err.Error(), tc.error)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, tc.value.(*Box).ClientTime, actual)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetAccessBoxAttrs(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
value any
|
|
||||||
error bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "valid",
|
|
||||||
value: func() *Box {
|
|
||||||
var attr object.Attribute
|
|
||||||
attr.SetKey("key")
|
|
||||||
attr.SetValue("value")
|
|
||||||
return &Box{Attributes: []object.Attribute{attr}}
|
|
||||||
}(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid data",
|
|
||||||
value: "invalid-data",
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "box does not exist",
|
|
||||||
error: true,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
ctx := context.WithValue(context.Background(), boxKey, tc.value)
|
|
||||||
actual, err := GetAccessBoxAttrs(ctx)
|
|
||||||
if tc.error {
|
|
||||||
require.Contains(t, err.Error(), "couldn't get box from context")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, len(tc.value.(*Box).Attributes), len(actual))
|
|
||||||
require.Equal(t, tc.value.(*Box).Attributes[0].Key(), actual[0].Key())
|
|
||||||
require.Equal(t, tc.value.(*Box).Attributes[0].Value(), actual[0].Value())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
263
api/notifications/controller.go
Normal file
263
api/notifications/controller.go
Normal file
|
@ -0,0 +1,263 @@
|
||||||
|
package notifications
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
DefaultTimeout = 30 * time.Second
|
||||||
|
|
||||||
|
// EventVersion23 is used for lifecycle, tiering, objectACL, objectTagging, object restoration notifications.
|
||||||
|
EventVersion23 = "2.3"
|
||||||
|
// EventVersion22 is used for replication notifications.
|
||||||
|
EventVersion22 = "2.2"
|
||||||
|
// EventVersion21 is used for all other notification types.
|
||||||
|
EventVersion21 = "2.1"
|
||||||
|
)
|
||||||
|
|
||||||
|
type (
|
||||||
|
Options struct {
|
||||||
|
URL string
|
||||||
|
TLSCertFilepath string
|
||||||
|
TLSAuthPrivateKeyFilePath string
|
||||||
|
Timeout time.Duration
|
||||||
|
RootCAFiles []string
|
||||||
|
}
|
||||||
|
|
||||||
|
Controller struct {
|
||||||
|
logger *zap.Logger
|
||||||
|
taskQueueConnection *nats.Conn
|
||||||
|
jsClient nats.JetStreamContext
|
||||||
|
handlers map[string]Stream
|
||||||
|
mu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
Stream struct {
|
||||||
|
h layer.MsgHandler
|
||||||
|
ch chan *nats.Msg
|
||||||
|
}
|
||||||
|
|
||||||
|
TestEvent struct {
|
||||||
|
Service string
|
||||||
|
Event string
|
||||||
|
Time time.Time
|
||||||
|
Bucket string
|
||||||
|
RequestID string
|
||||||
|
HostID string
|
||||||
|
}
|
||||||
|
|
||||||
|
Event struct {
|
||||||
|
Records []EventRecord `json:"Records"`
|
||||||
|
}
|
||||||
|
|
||||||
|
EventRecord struct {
|
||||||
|
EventVersion string `json:"eventVersion"`
|
||||||
|
EventSource string `json:"eventSource"` // frostfs:s3
|
||||||
|
AWSRegion string `json:"awsRegion,omitempty"` // empty
|
||||||
|
EventTime time.Time `json:"eventTime"`
|
||||||
|
EventName string `json:"eventName"`
|
||||||
|
UserIdentity UserIdentity `json:"userIdentity"`
|
||||||
|
RequestParameters RequestParameters `json:"requestParameters"`
|
||||||
|
ResponseElements map[string]string `json:"responseElements"`
|
||||||
|
S3 S3Entity `json:"s3"`
|
||||||
|
}
|
||||||
|
|
||||||
|
UserIdentity struct {
|
||||||
|
PrincipalID string `json:"principalId"`
|
||||||
|
}
|
||||||
|
|
||||||
|
RequestParameters struct {
|
||||||
|
SourceIPAddress string `json:"sourceIPAddress"`
|
||||||
|
}
|
||||||
|
|
||||||
|
S3Entity struct {
|
||||||
|
SchemaVersion string `json:"s3SchemaVersion"`
|
||||||
|
ConfigurationID string `json:"configurationId,omitempty"`
|
||||||
|
Bucket Bucket `json:"bucket"`
|
||||||
|
Object Object `json:"object"`
|
||||||
|
}
|
||||||
|
|
||||||
|
Bucket struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
OwnerIdentity UserIdentity `json:"ownerIdentity,omitempty"`
|
||||||
|
Arn string `json:"arn,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
Object struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Size uint64 `json:"size,omitempty"`
|
||||||
|
VersionID string `json:"versionId,omitempty"`
|
||||||
|
ETag string `json:"eTag,omitempty"`
|
||||||
|
Sequencer string `json:"sequencer,omitempty"`
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
func NewController(p *Options, l *zap.Logger) (*Controller, error) {
|
||||||
|
ncopts := []nats.Option{
|
||||||
|
nats.Timeout(p.Timeout),
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(p.TLSCertFilepath) != 0 && len(p.TLSAuthPrivateKeyFilePath) != 0 {
|
||||||
|
ncopts = append(ncopts, nats.ClientCert(p.TLSCertFilepath, p.TLSAuthPrivateKeyFilePath))
|
||||||
|
}
|
||||||
|
if len(p.RootCAFiles) != 0 {
|
||||||
|
ncopts = append(ncopts, nats.RootCAs(p.RootCAFiles...))
|
||||||
|
}
|
||||||
|
|
||||||
|
nc, err := nats.Connect(p.URL, ncopts...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("connect to nats: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
js, err := nc.JetStream()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("get jet stream: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Controller{
|
||||||
|
logger: l,
|
||||||
|
taskQueueConnection: nc,
|
||||||
|
jsClient: js,
|
||||||
|
handlers: make(map[string]Stream),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) Subscribe(_ context.Context, topic string, handler layer.MsgHandler) error {
|
||||||
|
ch := make(chan *nats.Msg, 1)
|
||||||
|
|
||||||
|
c.mu.RLock()
|
||||||
|
_, ok := c.handlers[topic]
|
||||||
|
c.mu.RUnlock()
|
||||||
|
if ok {
|
||||||
|
return fmt.Errorf("already subscribed to topic '%s'", topic)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := c.jsClient.AddStream(&nats.StreamConfig{Name: topic}); err != nil {
|
||||||
|
return fmt.Errorf("add stream: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := c.jsClient.ChanSubscribe(topic, ch); err != nil {
|
||||||
|
return fmt.Errorf("could not subscribe: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
c.mu.Lock()
|
||||||
|
c.handlers[topic] = Stream{
|
||||||
|
h: handler,
|
||||||
|
ch: ch,
|
||||||
|
}
|
||||||
|
c.mu.Unlock()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) Listen(ctx context.Context) {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
for _, stream := range c.handlers {
|
||||||
|
go func(stream Stream) {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case msg := <-stream.ch:
|
||||||
|
if err := stream.h.HandleMessage(ctx, msg); err != nil {
|
||||||
|
c.logger.Error(logs.CouldNotHandleMessage, zap.Error(err))
|
||||||
|
} else if err = msg.Ack(); err != nil {
|
||||||
|
c.logger.Error(logs.CouldNotACKMessage, zap.Error(err))
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}(stream)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) SendNotifications(topics map[string]string, p *handler.SendNotificationParams) error {
|
||||||
|
event := prepareEvent(p)
|
||||||
|
|
||||||
|
for id, topic := range topics {
|
||||||
|
event.Records[0].S3.ConfigurationID = id
|
||||||
|
msg, err := json.Marshal(event)
|
||||||
|
if err != nil {
|
||||||
|
c.logger.Error(logs.CouldntMarshalAnEvent, zap.String("subject", topic), zap.Error(err))
|
||||||
|
}
|
||||||
|
if err = c.publish(topic, msg); err != nil {
|
||||||
|
c.logger.Error(logs.CouldntSendAnEventToTopic, zap.String("subject", topic), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) SendTestNotification(topic, bucketName, requestID, HostID string, now time.Time) error {
|
||||||
|
event := &TestEvent{
|
||||||
|
Service: "FrostFS S3",
|
||||||
|
Event: "s3:TestEvent",
|
||||||
|
Time: now,
|
||||||
|
Bucket: bucketName,
|
||||||
|
RequestID: requestID,
|
||||||
|
HostID: HostID,
|
||||||
|
}
|
||||||
|
|
||||||
|
msg, err := json.Marshal(event)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("couldn't marshal test event: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.publish(topic, msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareEvent(p *handler.SendNotificationParams) *Event {
|
||||||
|
return &Event{
|
||||||
|
Records: []EventRecord{
|
||||||
|
{
|
||||||
|
EventVersion: EventVersion21,
|
||||||
|
EventSource: "frostfs:s3",
|
||||||
|
AWSRegion: "",
|
||||||
|
EventTime: p.Time,
|
||||||
|
EventName: p.Event,
|
||||||
|
UserIdentity: UserIdentity{
|
||||||
|
PrincipalID: p.User,
|
||||||
|
},
|
||||||
|
RequestParameters: RequestParameters{
|
||||||
|
SourceIPAddress: p.ReqInfo.RemoteHost,
|
||||||
|
},
|
||||||
|
ResponseElements: nil,
|
||||||
|
S3: S3Entity{
|
||||||
|
SchemaVersion: "1.0",
|
||||||
|
// ConfigurationID is skipped and will be placed later
|
||||||
|
Bucket: Bucket{
|
||||||
|
Name: p.BktInfo.Name,
|
||||||
|
OwnerIdentity: UserIdentity{PrincipalID: p.BktInfo.Owner.String()},
|
||||||
|
Arn: p.BktInfo.Name,
|
||||||
|
},
|
||||||
|
Object: Object{
|
||||||
|
Key: p.NotificationInfo.Name,
|
||||||
|
Size: p.NotificationInfo.Size,
|
||||||
|
VersionID: p.NotificationInfo.Version,
|
||||||
|
ETag: p.NotificationInfo.HashSum,
|
||||||
|
Sequencer: "",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Controller) publish(topic string, msg []byte) error {
|
||||||
|
if _, err := c.jsClient.Publish(topic, msg); err != nil {
|
||||||
|
return fmt.Errorf("couldn't send event: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -6,7 +6,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
|
||||||
|
@ -29,14 +29,20 @@ type FrostFS interface {
|
||||||
SystemDNS(context.Context) (string, error)
|
SystemDNS(context.Context) (string, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type Settings interface {
|
||||||
|
FormContainerZone(ns string) (zone string, isDefault bool)
|
||||||
|
}
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
FrostFS FrostFS
|
FrostFS FrostFS
|
||||||
RPCAddress string
|
RPCAddress string
|
||||||
|
Settings Settings
|
||||||
}
|
}
|
||||||
|
|
||||||
type BucketResolver struct {
|
type BucketResolver struct {
|
||||||
rpcAddress string
|
rpcAddress string
|
||||||
frostfs FrostFS
|
frostfs FrostFS
|
||||||
|
settings Settings
|
||||||
|
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
resolvers []*Resolver
|
resolvers []*Resolver
|
||||||
|
@ -44,15 +50,15 @@ type BucketResolver struct {
|
||||||
|
|
||||||
type Resolver struct {
|
type Resolver struct {
|
||||||
Name string
|
Name string
|
||||||
resolve func(context.Context, string, string) (cid.ID, error)
|
resolve func(context.Context, string) (cid.ID, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Resolver) SetResolveFunc(fn func(context.Context, string, string) (cid.ID, error)) {
|
func (r *Resolver) SetResolveFunc(fn func(context.Context, string) (cid.ID, error)) {
|
||||||
r.resolve = fn
|
r.resolve = fn
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Resolver) Resolve(ctx context.Context, zone, name string) (cid.ID, error) {
|
func (r *Resolver) Resolve(ctx context.Context, name string) (cid.ID, error) {
|
||||||
return r.resolve(ctx, zone, name)
|
return r.resolve(ctx, name)
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewBucketResolver(resolverNames []string, cfg *Config) (*BucketResolver, error) {
|
func NewBucketResolver(resolverNames []string, cfg *Config) (*BucketResolver, error) {
|
||||||
|
@ -81,12 +87,12 @@ func createResolvers(resolverNames []string, cfg *Config) ([]*Resolver, error) {
|
||||||
return resolvers, nil
|
return resolvers, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *BucketResolver) Resolve(ctx context.Context, zone, bktName string) (cnrID cid.ID, err error) {
|
func (r *BucketResolver) Resolve(ctx context.Context, bktName string) (cnrID cid.ID, err error) {
|
||||||
r.mu.RLock()
|
r.mu.RLock()
|
||||||
defer r.mu.RUnlock()
|
defer r.mu.RUnlock()
|
||||||
|
|
||||||
for _, resolver := range r.resolvers {
|
for _, resolver := range r.resolvers {
|
||||||
cnrID, resolverErr := resolver.Resolve(ctx, zone, bktName)
|
cnrID, resolverErr := resolver.Resolve(ctx, bktName)
|
||||||
if resolverErr != nil {
|
if resolverErr != nil {
|
||||||
resolverErr = fmt.Errorf("%s: %w", resolver.Name, resolverErr)
|
resolverErr = fmt.Errorf("%s: %w", resolver.Name, resolverErr)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
@ -117,6 +123,7 @@ func (r *BucketResolver) UpdateResolvers(resolverNames []string) error {
|
||||||
cfg := &Config{
|
cfg := &Config{
|
||||||
FrostFS: r.frostfs,
|
FrostFS: r.frostfs,
|
||||||
RPCAddress: r.rpcAddress,
|
RPCAddress: r.rpcAddress,
|
||||||
|
Settings: r.settings,
|
||||||
}
|
}
|
||||||
|
|
||||||
resolvers, err := createResolvers(resolverNames, cfg)
|
resolvers, err := createResolvers(resolverNames, cfg)
|
||||||
|
@ -145,25 +152,30 @@ func (r *BucketResolver) equals(resolverNames []string) bool {
|
||||||
func newResolver(name string, cfg *Config) (*Resolver, error) {
|
func newResolver(name string, cfg *Config) (*Resolver, error) {
|
||||||
switch name {
|
switch name {
|
||||||
case DNSResolver:
|
case DNSResolver:
|
||||||
return NewDNSResolver(cfg.FrostFS)
|
return NewDNSResolver(cfg.FrostFS, cfg.Settings)
|
||||||
case NNSResolver:
|
case NNSResolver:
|
||||||
return NewNNSResolver(cfg.RPCAddress)
|
return NewNNSResolver(cfg.RPCAddress, cfg.Settings)
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("unknown resolver: %s", name)
|
return nil, fmt.Errorf("unknown resolver: %s", name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewDNSResolver(frostFS FrostFS) (*Resolver, error) {
|
func NewDNSResolver(frostFS FrostFS, settings Settings) (*Resolver, error) {
|
||||||
if frostFS == nil {
|
if frostFS == nil {
|
||||||
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
|
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
|
||||||
}
|
}
|
||||||
|
if settings == nil {
|
||||||
|
return nil, fmt.Errorf("resolver settings must not be nil for DNS resolver")
|
||||||
|
}
|
||||||
|
|
||||||
var dns ns.DNS
|
var dns ns.DNS
|
||||||
|
|
||||||
resolveFunc := func(ctx context.Context, zone, name string) (cid.ID, error) {
|
resolveFunc := func(ctx context.Context, name string) (cid.ID, error) {
|
||||||
var err error
|
var err error
|
||||||
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
if zone == v2container.SysAttributeZoneDefault {
|
zone, isDefault := settings.FormContainerZone(reqInfo.Namespace)
|
||||||
|
if isDefault {
|
||||||
zone, err = frostFS.SystemDNS(ctx)
|
zone, err = frostFS.SystemDNS(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cid.ID{}, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err)
|
return cid.ID{}, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err)
|
||||||
|
@ -184,10 +196,13 @@ func NewDNSResolver(frostFS FrostFS) (*Resolver, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewNNSResolver(address string) (*Resolver, error) {
|
func NewNNSResolver(address string, settings Settings) (*Resolver, error) {
|
||||||
if address == "" {
|
if address == "" {
|
||||||
return nil, fmt.Errorf("rpc address must not be empty for NNS resolver")
|
return nil, fmt.Errorf("rpc address must not be empty for NNS resolver")
|
||||||
}
|
}
|
||||||
|
if settings == nil {
|
||||||
|
return nil, fmt.Errorf("resolver settings must not be nil for NNS resolver")
|
||||||
|
}
|
||||||
|
|
||||||
var nns ns.NNS
|
var nns ns.NNS
|
||||||
|
|
||||||
|
@ -195,9 +210,12 @@ func NewNNSResolver(address string) (*Resolver, error) {
|
||||||
return nil, fmt.Errorf("dial %s: %w", address, err)
|
return nil, fmt.Errorf("dial %s: %w", address, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
resolveFunc := func(_ context.Context, zone, name string) (cid.ID, error) {
|
resolveFunc := func(ctx context.Context, name string) (cid.ID, error) {
|
||||||
var d container.Domain
|
var d container.Domain
|
||||||
d.SetName(name)
|
d.SetName(name)
|
||||||
|
|
||||||
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
zone, _ := settings.FormContainerZone(reqInfo.Namespace)
|
||||||
d.SetZone(zone)
|
d.SetZone(zone)
|
||||||
|
|
||||||
cnrID, err := nns.ResolveContainerDomain(d)
|
cnrID, err := nns.ResolveContainerDomain(d)
|
||||||
|
|
|
@ -37,7 +37,6 @@ type (
|
||||||
PutObjectHandler(http.ResponseWriter, *http.Request)
|
PutObjectHandler(http.ResponseWriter, *http.Request)
|
||||||
DeleteObjectHandler(http.ResponseWriter, *http.Request)
|
DeleteObjectHandler(http.ResponseWriter, *http.Request)
|
||||||
GetBucketLocationHandler(http.ResponseWriter, *http.Request)
|
GetBucketLocationHandler(http.ResponseWriter, *http.Request)
|
||||||
GetBucketPolicyStatusHandler(http.ResponseWriter, *http.Request)
|
|
||||||
GetBucketPolicyHandler(http.ResponseWriter, *http.Request)
|
GetBucketPolicyHandler(http.ResponseWriter, *http.Request)
|
||||||
GetBucketLifecycleHandler(http.ResponseWriter, *http.Request)
|
GetBucketLifecycleHandler(http.ResponseWriter, *http.Request)
|
||||||
GetBucketEncryptionHandler(http.ResponseWriter, *http.Request)
|
GetBucketEncryptionHandler(http.ResponseWriter, *http.Request)
|
||||||
|
@ -87,7 +86,6 @@ type (
|
||||||
AbortMultipartUploadHandler(http.ResponseWriter, *http.Request)
|
AbortMultipartUploadHandler(http.ResponseWriter, *http.Request)
|
||||||
ListPartsHandler(w http.ResponseWriter, r *http.Request)
|
ListPartsHandler(w http.ResponseWriter, r *http.Request)
|
||||||
ListMultipartUploadsHandler(http.ResponseWriter, *http.Request)
|
ListMultipartUploadsHandler(http.ResponseWriter, *http.Request)
|
||||||
PatchObjectHandler(http.ResponseWriter, *http.Request)
|
|
||||||
|
|
||||||
ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error)
|
ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error)
|
||||||
ResolveCID(ctx context.Context, bucket string) (cid.ID, error)
|
ResolveCID(ctx context.Context, bucket string) (cid.ID, error)
|
||||||
|
@ -98,7 +96,7 @@ type Settings interface {
|
||||||
s3middleware.RequestSettings
|
s3middleware.RequestSettings
|
||||||
s3middleware.PolicySettings
|
s3middleware.PolicySettings
|
||||||
s3middleware.MetricsSettings
|
s3middleware.MetricsSettings
|
||||||
s3middleware.VHSSettings
|
s3middleware.HttpLogSettings
|
||||||
}
|
}
|
||||||
|
|
||||||
type FrostFSID interface {
|
type FrostFSID interface {
|
||||||
|
@ -115,19 +113,19 @@ type Config struct {
|
||||||
|
|
||||||
MiddlewareSettings Settings
|
MiddlewareSettings Settings
|
||||||
|
|
||||||
FrostfsID FrostFSID
|
// Domains optional. If empty no virtual hosted domains will be attached.
|
||||||
|
Domains []string
|
||||||
|
|
||||||
|
FrostfsID FrostFSID
|
||||||
FrostFSIDValidation bool
|
FrostFSIDValidation bool
|
||||||
|
|
||||||
PolicyChecker engine.ChainRouter
|
PolicyChecker engine.ChainRouter
|
||||||
|
|
||||||
XMLDecoder s3middleware.XMLDecoder
|
|
||||||
Tagging s3middleware.ResourceTagging
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewRouter(cfg Config) *chi.Mux {
|
func NewRouter(cfg Config) *chi.Mux {
|
||||||
api := chi.NewRouter()
|
api := chi.NewRouter()
|
||||||
api.Use(
|
api.Use(
|
||||||
|
s3middleware.LogHttp(cfg.Log, cfg.MiddlewareSettings),
|
||||||
s3middleware.Request(cfg.Log, cfg.MiddlewareSettings),
|
s3middleware.Request(cfg.Log, cfg.MiddlewareSettings),
|
||||||
middleware.ThrottleWithOpts(cfg.Throttle),
|
middleware.ThrottleWithOpts(cfg.Throttle),
|
||||||
middleware.Recoverer,
|
middleware.Recoverer,
|
||||||
|
@ -141,53 +139,32 @@ func NewRouter(cfg Config) *chi.Mux {
|
||||||
api.Use(s3middleware.FrostfsIDValidation(cfg.FrostfsID, cfg.Log))
|
api.Use(s3middleware.FrostfsIDValidation(cfg.FrostfsID, cfg.Log))
|
||||||
}
|
}
|
||||||
|
|
||||||
api.Use(s3middleware.PrepareAddressStyle(cfg.MiddlewareSettings, cfg.Log))
|
|
||||||
api.Use(s3middleware.PolicyCheck(s3middleware.PolicyConfig{
|
api.Use(s3middleware.PolicyCheck(s3middleware.PolicyConfig{
|
||||||
Storage: cfg.PolicyChecker,
|
Storage: cfg.PolicyChecker,
|
||||||
FrostfsID: cfg.FrostfsID,
|
FrostfsID: cfg.FrostfsID,
|
||||||
Settings: cfg.MiddlewareSettings,
|
Settings: cfg.MiddlewareSettings,
|
||||||
|
Domains: cfg.Domains,
|
||||||
Log: cfg.Log,
|
Log: cfg.Log,
|
||||||
BucketResolver: cfg.Handler.ResolveBucket,
|
BucketResolver: cfg.Handler.ResolveBucket,
|
||||||
Decoder: cfg.XMLDecoder,
|
|
||||||
Tagging: cfg.Tagging,
|
|
||||||
}))
|
}))
|
||||||
|
|
||||||
defaultRouter := chi.NewRouter()
|
defaultRouter := chi.NewRouter()
|
||||||
defaultRouter.Mount("/{bucket}", bucketRouter(cfg.Handler))
|
defaultRouter.Mount(fmt.Sprintf("/{%s}", s3middleware.BucketURLPrm), bucketRouter(cfg.Handler, cfg.Log))
|
||||||
defaultRouter.Get("/", named(s3middleware.ListBucketsOperation, cfg.Handler.ListBucketsHandler))
|
defaultRouter.Get("/", named("ListBuckets", cfg.Handler.ListBucketsHandler))
|
||||||
attachErrorHandler(defaultRouter)
|
attachErrorHandler(defaultRouter)
|
||||||
|
|
||||||
vhsRouter := bucketRouter(cfg.Handler)
|
hr := NewHostBucketRouter("bucket")
|
||||||
router := newGlobalRouter(defaultRouter, vhsRouter)
|
hr.Default(defaultRouter)
|
||||||
|
for _, domain := range cfg.Domains {
|
||||||
api.Mount("/", router)
|
hr.Map(domain, bucketRouter(cfg.Handler, cfg.Log))
|
||||||
|
}
|
||||||
|
api.Mount("/", hr)
|
||||||
|
|
||||||
attachErrorHandler(api)
|
attachErrorHandler(api)
|
||||||
|
|
||||||
return api
|
return api
|
||||||
}
|
}
|
||||||
|
|
||||||
type globalRouter struct {
|
|
||||||
pathStyleRouter chi.Router
|
|
||||||
vhsRouter chi.Router
|
|
||||||
}
|
|
||||||
|
|
||||||
func newGlobalRouter(pathStyleRouter, vhsRouter chi.Router) *globalRouter {
|
|
||||||
return &globalRouter{
|
|
||||||
pathStyleRouter: pathStyleRouter,
|
|
||||||
vhsRouter: vhsRouter,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *globalRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
||||||
router := g.pathStyleRouter
|
|
||||||
if reqInfo := s3middleware.GetReqInfo(r.Context()); reqInfo.RequestVHSEnabled {
|
|
||||||
router = g.vhsRouter
|
|
||||||
}
|
|
||||||
|
|
||||||
router.ServeHTTP(w, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
func named(name string, handlerFunc http.HandlerFunc) http.HandlerFunc {
|
func named(name string, handlerFunc http.HandlerFunc) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
reqInfo := s3middleware.GetReqInfo(r.Context())
|
reqInfo := s3middleware.GetReqInfo(r.Context())
|
||||||
|
@ -232,15 +209,16 @@ func attachErrorHandler(api *chi.Mux) {
|
||||||
api.MethodNotAllowed(named("MethodNotAllowed", errorHandler))
|
api.MethodNotAllowed(named("MethodNotAllowed", errorHandler))
|
||||||
}
|
}
|
||||||
|
|
||||||
func bucketRouter(h Handler) chi.Router {
|
func bucketRouter(h Handler, log *zap.Logger) chi.Router {
|
||||||
bktRouter := chi.NewRouter()
|
bktRouter := chi.NewRouter()
|
||||||
bktRouter.Use(
|
bktRouter.Use(
|
||||||
|
s3middleware.AddBucketName(log),
|
||||||
s3middleware.WrapHandler(h.AppendCORSHeaders),
|
s3middleware.WrapHandler(h.AppendCORSHeaders),
|
||||||
)
|
)
|
||||||
|
|
||||||
bktRouter.Mount("/", objectRouter(h))
|
bktRouter.Mount("/", objectRouter(h, log))
|
||||||
|
|
||||||
bktRouter.Options("/", named(s3middleware.OptionsBucketOperation, h.Preflight))
|
bktRouter.Options("/", h.Preflight)
|
||||||
|
|
||||||
bktRouter.Head("/", named(s3middleware.HeadBucketOperation, h.HeadBucketHandler))
|
bktRouter.Head("/", named(s3middleware.HeadBucketOperation, h.HeadBucketHandler))
|
||||||
|
|
||||||
|
@ -253,9 +231,6 @@ func bucketRouter(h Handler) chi.Router {
|
||||||
Add(NewFilter().
|
Add(NewFilter().
|
||||||
Queries(s3middleware.LocationQuery).
|
Queries(s3middleware.LocationQuery).
|
||||||
Handler(named(s3middleware.GetBucketLocationOperation, h.GetBucketLocationHandler))).
|
Handler(named(s3middleware.GetBucketLocationOperation, h.GetBucketLocationHandler))).
|
||||||
Add(NewFilter().
|
|
||||||
Queries(s3middleware.PolicyStatusQuery).
|
|
||||||
Handler(named(s3middleware.GetBucketPolicyStatusOperation, h.GetBucketPolicyStatusHandler))).
|
|
||||||
Add(NewFilter().
|
Add(NewFilter().
|
||||||
Queries(s3middleware.PolicyQuery).
|
Queries(s3middleware.PolicyQuery).
|
||||||
Handler(named(s3middleware.GetBucketPolicyOperation, h.GetBucketPolicyHandler))).
|
Handler(named(s3middleware.GetBucketPolicyOperation, h.GetBucketPolicyHandler))).
|
||||||
|
@ -310,7 +285,7 @@ func bucketRouter(h Handler) chi.Router {
|
||||||
Add(NewFilter().
|
Add(NewFilter().
|
||||||
Queries(s3middleware.VersionsQuery).
|
Queries(s3middleware.VersionsQuery).
|
||||||
Handler(named(s3middleware.ListBucketObjectVersionsOperation, h.ListBucketObjectVersionsHandler))).
|
Handler(named(s3middleware.ListBucketObjectVersionsOperation, h.ListBucketObjectVersionsHandler))).
|
||||||
DefaultHandler(listWrapper(h)))
|
DefaultHandler(named(s3middleware.ListObjectsV1Operation, h.ListObjectsV1Handler)))
|
||||||
})
|
})
|
||||||
|
|
||||||
// PUT method handlers
|
// PUT method handlers
|
||||||
|
@ -373,7 +348,7 @@ func bucketRouter(h Handler) chi.Router {
|
||||||
Handler(named(s3middleware.DeleteBucketPolicyOperation, h.DeleteBucketPolicyHandler))).
|
Handler(named(s3middleware.DeleteBucketPolicyOperation, h.DeleteBucketPolicyHandler))).
|
||||||
Add(NewFilter().
|
Add(NewFilter().
|
||||||
Queries(s3middleware.LifecycleQuery).
|
Queries(s3middleware.LifecycleQuery).
|
||||||
Handler(named(s3middleware.DeleteBucketLifecycleOperation, h.DeleteBucketLifecycleHandler))).
|
Handler(named(s3middleware.PutBucketLifecycleOperation, h.PutBucketLifecycleHandler))).
|
||||||
Add(NewFilter().
|
Add(NewFilter().
|
||||||
Queries(s3middleware.EncryptionQuery).
|
Queries(s3middleware.EncryptionQuery).
|
||||||
Handler(named(s3middleware.DeleteBucketEncryptionOperation, h.DeleteBucketEncryptionHandler))).
|
Handler(named(s3middleware.DeleteBucketEncryptionOperation, h.DeleteBucketEncryptionHandler))).
|
||||||
|
@ -385,27 +360,12 @@ func bucketRouter(h Handler) chi.Router {
|
||||||
return bktRouter
|
return bktRouter
|
||||||
}
|
}
|
||||||
|
|
||||||
func listWrapper(h Handler) http.HandlerFunc {
|
func objectRouter(h Handler, l *zap.Logger) chi.Router {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if reqInfo := s3middleware.GetReqInfo(r.Context()); reqInfo.BucketName == "" {
|
|
||||||
reqInfo.API = s3middleware.ListBucketsOperation
|
|
||||||
h.ListBucketsHandler(w, r)
|
|
||||||
} else {
|
|
||||||
reqInfo.API = s3middleware.ListObjectsV1Operation
|
|
||||||
h.ListObjectsV1Handler(w, r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func objectRouter(h Handler) chi.Router {
|
|
||||||
objRouter := chi.NewRouter()
|
objRouter := chi.NewRouter()
|
||||||
|
objRouter.Use(s3middleware.AddObjectName(l))
|
||||||
objRouter.Options("/*", named(s3middleware.OptionsObjectOperation, h.Preflight))
|
|
||||||
|
|
||||||
objRouter.Head("/*", named(s3middleware.HeadObjectOperation, h.HeadObjectHandler))
|
objRouter.Head("/*", named(s3middleware.HeadObjectOperation, h.HeadObjectHandler))
|
||||||
|
|
||||||
objRouter.Patch("/*", named(s3middleware.PatchObjectOperation, h.PatchObjectHandler))
|
|
||||||
|
|
||||||
// GET method handlers
|
// GET method handlers
|
||||||
objRouter.Group(func(r chi.Router) {
|
objRouter.Group(func(r chi.Router) {
|
||||||
r.Method(http.MethodGet, "/*", NewHandlerFilter().
|
r.Method(http.MethodGet, "/*", NewHandlerFilter().
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue