Compare commits

...

84 commits

Author SHA1 Message Date
a12fea8a5b Release v0.31.0
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2024-11-20 15:45:07 +03:00
9875307c9b [#556] Check bucket name not only during creation
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-11-20 08:13:27 +00:00
b1775f9478 [#553] authmate: Add retryer to create access box
After using AddChain to provide access to container we have to wait:
* tx with APE chain be accepted by blockchain
* cache in storage node be updated

it takes a while. So we add retry
 (the same as when we add bucket settings during bucket creation)

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-11-19 15:46:00 +03:00
4fa45bdac2 [#553] authmate: Don't use basic acl
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-11-19 15:45:54 +03:00
368c7d2acd [#549] Add tracing attributes
Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2024-11-18 11:55:31 +00:00
31076796ce [#550] Execute CI on push to master
Signed-off-by: Vitaliy Potyarkin <v.potyarkin@yadro.com>
2024-11-15 14:31:11 +03:00
eff0de43d5 [#538] Return headers with 304 Not Modified
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-11-13 13:49:09 +00:00
fb00dff83b [#540] Add md5 S3Tests compatability
Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>
2024-11-13 14:50:16 +03:00
d8f126b339 [#539] Fix listing v1 bookmark marker
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-11-12 12:58:09 +00:00
7ab902d8d2 [#536] Add rule ID generation
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-11-12 12:51:02 +00:00
0792fcf456 [#536] Fix error codes in lifecycle configuration check
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-11-12 12:51:02 +00:00
c46ffa8146 [#536] Add prefix to lifecycle rule
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-11-12 12:51:02 +00:00
3260308cc0 [#528] Check owner ID before deleting bucket
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-11-12 12:47:43 +00:00
d6e6a13576 [#542] Stop using obsolete .github directory
This commit is a part of multi-repo cleanup effort:
TrueCloudLab/frostfs-infra#136

Signed-off-by: Vitaliy Potyarkin <v.potyarkin@yadro.com>
2024-11-06 15:31:16 +03:00
17d40245de [#505] docs: Add example of uploading file using presigned URL
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-11-02 08:53:54 +00:00
979d85b046 [#505] authmate: Add flag for headers in generate-presigned-url cmd
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-11-02 08:53:54 +00:00
539dab8680 [#501] Add the trace id to the logs of middlware
Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2024-11-02 08:51:48 +00:00
76008d4ba1 [#501] Consider using request logger in logAndSendError
Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2024-11-02 08:51:48 +00:00
8bc19725ba [#521] Add documentation for multinet settings
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2024-10-29 15:55:27 +03:00
9e64304499 [#521] Use handler to register dial events
While frostfs-node uses dial handler to udpate metric
value, gateway starts with simple event logging.

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2024-10-29 15:55:27 +03:00
94504e9746 [#521] Use source dialer for gRPC connection to storage
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2024-10-29 15:55:27 +03:00
a8458dbc27 [#521] Add internal/net package with multinet dialer source
Code is taken from frostfs-node#1422
Author: Dmitrii Stepanov (dstepanov-yadro)

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2024-10-29 15:55:26 +03:00
424038de6c [#524] Update pool to treat maintenance mode differently
Contains these changes for pool component in SDK:
* frostfs-sdk-go#279 fix mm error counting during search operation
* frostfs-sdk-go#283 immediately mark mm node as unhealthy
* frostfs-sdk-go#278 do not reconnect to mm node

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2024-10-23 12:17:20 +00:00
3cf27d281d [#509] Support fallback address when getting box
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-10-23 15:01:31 +03:00
3c7cb82553 [#509] Init resolvers before first resolving
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-10-23 15:01:31 +03:00
57b7e83380 [#509] Save isCustom flag into accessbox
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-10-23 15:01:31 +03:00
6a90f4e624 [#509] Update docs
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-10-23 15:01:31 +03:00
cb3753f286 [#509] Add tests for custom credentials
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-10-23 15:01:31 +03:00
81209e308c [#509] Fix tests
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-10-23 15:01:31 +03:00
b78e55e101 [#509] Support custom AWS credentials
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-10-23 15:01:31 +03:00
25c24f5ce6 [#522] Add waiter to contract clients
Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
2024-10-23 09:22:19 +03:00
09c11262c6 [#516] Check Content-Md5 of lifecycle configuration
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-10-22 14:21:49 +00:00
f120715a37 [#516] Convert expiration date to epoch
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-10-22 14:21:49 +00:00
aaed083d82 [#520] Support the continuous use of interceptors
We can always add interceptors to the grpc
connection to the storage, since the actual
use will be controlled by the configuration
from the frostfs-observability library.

Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2024-10-21 11:49:22 +03:00
e35b582fe2 [#506] Deny bucket names with dot
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-10-08 12:50:22 +03:00
39fc7aa3ee [#404] Fix using encoding type in multipart upload listing
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-10-08 09:44:05 +00:00
da41f47826 [#404] Add check of encoding type in object listing
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-10-08 09:44:05 +00:00
9e5fb4be95 [#507] Return not implemented by default in bucket router
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-10-07 05:38:35 +00:00
346243b159 [#450] Fix getPutPayloadSize
If X-Amz-Decoded-Content-Length explicitly set use it even if value is 0

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-10-03 14:23:06 +03:00
03481274f0 [#467] authmate: Add sign command
Support singing arbitrary data using aws sigv4 algorithm

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-10-02 13:42:22 +00:00
c2adbd758a [#488] middleware/auth: Add frostfs-to-s3 error transformation
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
2024-10-02 13:18:25 +03:00
bc17ab5e47 [#488] middleware/policy: Add frostfs-to-s3 error transformation
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
2024-10-02 12:35:04 +03:00
9fadfbbc2f [#488] Renamed api/errors, layer/frostfs and layer/tree package names
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
2024-10-02 12:35:04 +03:00
827ea1a41e [#488] Move layer/frostfs.go to layer/frostfs/frostfs.go
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
2024-10-02 12:35:04 +03:00
968f10a72f [#488] Move layer/tree_service.go to layer/tree/tree_service.go
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
2024-10-02 12:35:04 +03:00
582e6ac642 [#503] Update SDK to fix error counting
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-09-27 10:36:48 +03:00
99f273f9af [#461] Configure logger sampling policy
Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>
2024-09-26 10:34:44 +03:00
cd96adef36 [#499] Fix of sighup traicing docs
Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2024-09-25 14:34:18 +03:00
738ce14f50 [#434] Remove container on failed bucket creation
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-09-25 07:15:24 +00:00
5358e39f71 [#496] Update frostfs-go-sdk for in-flight requests support
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
2024-09-23 08:57:40 +03:00
34c1426b9f [#484] Add root ca cert for telemetry configuration
Signed-off-by: Aleksey Savaitan <a.savaitan@yadro.com>
2024-09-19 11:07:13 +00:00
8ca73e2079 [#493] Fix of receiving VHS namespaces map
In the process of forming a map with namespaces
for which VHS is enabled, we resolve the alias
of the namespace. The problem is that to resolve,
we need default namespace names, which in turn do
not have time to decide by this time. Therefore,
now the check for the default name takes place
directly in the prepareVHSNamespaces function
based on previously read default names.

Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2024-09-17 16:57:05 +03:00
a87c636b4c [#493] Fix X-Frostfs-S3-VHS header processing
It is assumed that the X-Frostfs-S3-VHS header
will have the value enabled/disabled instead
of true/false.

Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2024-09-16 14:28:19 +03:00
26baf8a94e [#492] Add panic catchers to fuzzing tests
Signed-off-by: Roman Ognev <r.ognev@yadro.com>
2024-09-16 10:36:30 +00:00
f187141ae5 [#486] Fix PUT object with negative Content-Length
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-09-16 08:45:46 +00:00
3cffc782e9 [#486] Fix PUT encrypted object
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-09-16 08:45:46 +00:00
d0e4d55772 [#460] Add network info cache
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-09-13 09:56:24 +00:00
42e72889a5 [#450] Add test for chunk-encoded object size
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2024-09-13 12:00:24 +03:00
98815d5473 [#450] Fix aws-chunked header workflow
Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>
2024-09-13 11:59:07 +03:00
62615d7ab7 [#369] Request reproducer
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
2024-09-11 15:25:09 +03:00
575ab4d294 [#369] Enhanced http requests logging
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
2024-09-11 15:25:09 +03:00
d919e6cce2 [#482] Fix containers resolving
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-09-05 12:33:14 +03:00
056f168d77 [#448] multipart: Support removing duplicated parts
Previously after tree split we can have duplicated parts
(several objects and tree node referred to the same part number).
Some of them couldn't be deleted after abort or compete action.

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-09-03 13:20:38 +00:00
9bdfe2a016 [#479] Update APE to support s3:PatchObject action
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-09-03 11:57:59 +00:00
d6b506f6d9 [#466] Implement PATCH for multipart objects
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-09-03 11:57:59 +00:00
a2e0b92575 [#473] Add PATCH to extensions doc
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-09-03 11:57:59 +00:00
b08f476ea7 [#462] Implement PATCH for regular objects
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-09-03 11:57:59 +00:00
f4275d837a [#413] Add SECURITY.md
Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>
2024-09-03 11:45:05 +00:00
664f83b2b7 [#480] Add fuzzing tests
Signed-off-by: Roman Ognev <r.ognev@yadro.com>
2024-09-02 15:59:07 +03:00
136b5521fe [#475] Support graceful_close_on_switch_timeout param
This allows in-flight requests finish during rebalance

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-08-29 13:22:08 +00:00
a5f670d904 [#329] Reduce using mutex when update app settings
Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2024-08-29 12:03:26 +00:00
d76c4fe2a2 [#472] tree: Don't use sorted GetSubTree for nodes without FileName
Sorted GetSubTree doesn't return nodes without FileName attribute
if there are more than 1000 nodes in subtree

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-08-27 09:43:29 +00:00
0637133c61 [#470] Update Go version to 1.22
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
2024-08-23 12:18:42 +03:00
bf00fa6aa9 [#449] Add support headers for vhs and servername
Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2024-08-23 08:35:05 +00:00
ff690ce996 [#446] Add tests for address style middleware
Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2024-08-23 08:35:05 +00:00
534ae7f0f1 [#446] Add support virtual-hosted-style
Signed-off-by: Roman Loginov <r.loginov@yadro.com>
2024-08-23 08:35:05 +00:00
77673797f9 [#474] lint: Update golangci-lint to v1.60 and fix issues
Signed-off-by: Ekaterina Lebedeva <ekaterina.lebedeva@yadro.com>
2024-08-22 19:13:52 +03:00
9e1766ff74 [#459] Update APE
Update to policy engine version with lifecycle actions support

Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-08-21 16:42:34 +03:00
e73f11c251 [#452] tree: Fix logging
Don't log parsing tags error in listing

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-08-21 10:38:36 +03:00
5cb77018f8 [#42] Change valid time format for lifecycle configuration
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-08-21 10:38:36 +03:00
fa68a4ce40 [#412] Store creation epoch of delete markers
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-08-21 10:38:35 +03:00
0644067496 [#412] Store creation epoch in tree service
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-08-21 10:38:35 +03:00
481520705a [#42] Support expiration lifecycle
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-08-21 10:38:35 +03:00
28723f4a68 [#447] Add tree pool request duration metric
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
2024-08-21 06:36:55 +00:00
165 changed files with 11000 additions and 2429 deletions

View file

@ -1,13 +1,14 @@
FROM golang:1.21 AS builder
FROM golang:1.22 AS builder
ARG BUILD=now
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
ARG VERSION=dev
ARG GOFLAGS=""
WORKDIR /src
COPY . /src
RUN make
RUN make GOFLAGS=${GOFLAGS}
# Executable image
FROM alpine AS frostfs-s3-gw

View file

@ -1,3 +1,3 @@
.git
.cache
.github
.forgejo

View file

Before

Width:  |  Height:  |  Size: 5.5 KiB

After

Width:  |  Height:  |  Size: 5.5 KiB

View file

@ -1,4 +1,8 @@
on: [pull_request]
on:
pull_request:
push:
branches:
- master
jobs:
builds:
@ -6,7 +10,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
go_versions: [ '1.21', '1.22' ]
go_versions: [ '1.22', '1.23' ]
fail-fast: false
steps:
- uses: actions/checkout@v3

View file

@ -12,7 +12,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v3
with:
go-version: '1.22'
go-version: '1.23'
- name: Run commit format checker
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3

View file

@ -1,4 +1,8 @@
on: [pull_request]
on:
pull_request:
push:
branches:
- master
jobs:
lint:
@ -10,7 +14,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: '1.22'
go-version: '1.23'
cache: true
- name: Install linters
@ -24,7 +28,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
go_versions: [ '1.21', '1.22' ]
go_versions: [ '1.22', '1.23' ]
fail-fast: false
steps:
- uses: actions/checkout@v3

View file

@ -1,4 +1,8 @@
on: [pull_request]
on:
pull_request:
push:
branches:
- master
jobs:
vulncheck:
@ -12,7 +16,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v3
with:
go-version: '1.22'
go-version: '1.23'
- name: Install govulncheck
run: go install golang.org/x/vuln/cmd/govulncheck@latest

1
.github/CODEOWNERS vendored
View file

@ -1 +0,0 @@
* @alexvanin @dkirillov

View file

@ -12,7 +12,8 @@ run:
# output configuration options
output:
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
format: tab
formats:
- format: tab
# all available settings of specific linters
linters-settings:

View file

@ -4,6 +4,106 @@ This document outlines major changes between releases.
## [Unreleased]
## [0.31.0] - Rongbuk - 2024-11-20
### Fixed
- Docker warnings during image build (#421)
- `PartNumberMarker` in ListMultipart response (#451)
- PostObject handling (#456)
- Tag logging errors (#452)
- Removing of duplicated parts in tree service during split brain (#448)
- Container resolving (#482)
- FrostFS to S3 error transformation (#488)
- Default bucket routing (#507)
- encoding-type in ListBucketObjectVersions (#404)
- SIGHUP support for `tracing.enabled` config parameter (#520)
- `trace_id` parameter in logs (#501)
- Listing marker processing (#539)
- Content-MD5 header check (#540)
- Precondition check (#538)
- Bucket name check during all S3 operations (#556)
### Added
- Support for separate container for all CORS settings (#422)
- `X-Amz-Force-Delete-Bucket` header for forced bucket removal (#31)
- `Location` support in CompleteMultipart response (#451)
- Tree pool request duration metric (#447)
- Expiration lifecycle configuration support (#42, #412, #459, #460, #516, #536)
- Add support for virtual hosted style addressing (#446, #449, #493)
- Support `frostfs.graceful_close_on_switch_timeout` (#475)
- Vulnerability report document (#413)
- Support patch object method (#462, #473, #466, #479)
- Enhanced logging and request reproducer (#369)
- Root CA configuration for tracing (#484)
- Log sampling policy configuration (#461)
- `sign` command to `frostfs-s3-authmate` (#467)
- Support custom aws credentials (#509)
- Source IP binding configuration for FrostFS requests (#521)
- Tracing attributes (#549)
### Changed
- Split `FrostFS` interface into separate read methods (#427)
- golangci-lint v1.60 support (#474)
- Updated Go version to 1.22 (#470)
- Container removal after failed bucket creation (#434)
- Explicit check for `.` symbol in bucket name (#506)
- Transaction waiter in contract clients (#522)
- Avoid maintenance mode storage node during object operations (#524)
- Content-Type does not include in Presigned URL of s3-authmate (#505)
- Check owner ID before deleting bucket (#528)
- S3-Authmate now uses APE instead basic-ACL (#553)
### Removed
- Reduce using mutex when update app settings (#329)
## [0.30.8] - 2024-10-18
### Fixed
- Error handling for correct connection switch in SDK Pool (#517)
## [0.30.7] - 2024-10-03
### Fixed
- Correct aws-chunk encoding size handling (#511)
## [0.30.6] - 2024-09-17
### Fixed
- Object size of objects upload with aws-chunked encoding (#450)
- Object size of objects upload with negative Content-Length (#486)
## [0.30.5] - 2024-09-16
### Fixed
- Panic catchers for fuzzing tests (#492)
## [0.30.4] - 2024-09-03
### Added
- Fuzzing tests (#480)
## [0.30.3] - 2024-08-27
### Fixed
- Empty listing when multipart upload contains more than 1000 parts (#471)
## [0.30.2] - 2024-08-20
### Fixed
- Error counting in pool component before connection switch (#468)
### Added
- Log of endpoint address during tree pool errors (#468)
## [0.30.1] - 2024-07-25
### Fixed
- Redundant system node removal in tree service (#437)
### Added
- Log details on SDK Pool health status change (#439)
## [0.30.0] - Kangshung -2024-07-19
### Fixed
@ -233,4 +333,13 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs
[0.29.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.1...v0.29.2
[0.29.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.2...v0.29.3
[0.30.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.3...v0.30.0
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.0...master
[0.30.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.0...v0.30.1
[0.30.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.1...v0.30.2
[0.30.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.2...v0.30.3
[0.30.4]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.3...v0.30.4
[0.30.5]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.4...v0.30.5
[0.30.6]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.5...v0.30.6
[0.30.7]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.6...v0.30.7
[0.30.8]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.7...v0.30.8
[0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.8...v0.31.0
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.0...master

1
CODEOWNERS Normal file
View file

@ -0,0 +1 @@
.* @alexvanin @dkirillov

View file

@ -4,8 +4,8 @@
REPO ?= $(shell go list -m)
VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")
GO_VERSION ?= 1.22
LINT_VERSION ?= 1.56.1
TRUECLOUDLAB_LINT_VERSION ?= 0.0.5
LINT_VERSION ?= 1.60.1
TRUECLOUDLAB_LINT_VERSION ?= 0.0.6
BINDIR = bin
METRICS_DUMP_OUT ?= ./metrics-dump.json
@ -14,6 +14,8 @@ METRICS_DUMP_OUT ?= ./metrics-dump.json
CMDS = $(addprefix frostfs-, $(notdir $(wildcard cmd/*)))
BINS = $(addprefix $(BINDIR)/, $(CMDS))
GOFLAGS ?=
# Variables for docker
REPO_BASENAME = $(shell basename `go list -m`)
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
@ -23,6 +25,12 @@ OUTPUT_LINT_DIR ?= $(shell pwd)/bin
LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
TMP_DIR := .cache
# Variables for fuzzing
FUZZ_NGFUZZ_DIR ?= ""
FUZZ_TIMEOUT ?= 30
FUZZ_FUNCTIONS ?= "all"
FUZZ_AUX ?= ""
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
# .deb package versioning
@ -38,6 +46,7 @@ all: $(BINS)
$(BINS): $(BINDIR) dep
@echo "⇒ Build $@"
CGO_ENABLED=0 \
GOFLAGS=$(GOFLAGS) \
go build -v -trimpath \
-ldflags "-X $(REPO)/internal/version.Version=$(VERSION)" \
-o $@ ./cmd/$(subst frostfs-,,$(notdir $@))
@ -64,7 +73,7 @@ docker/%:
-w /src \
-u `stat -c "%u:%g" .` \
--env HOME=/src \
golang:$(GO_VERSION) make $*,\
golang:$(GO_VERSION) make GOFLAGS=$(GOFLAGS) $*,\
@echo "supported docker targets: all $(BINS) lint")
# Run tests
@ -76,6 +85,34 @@ cover:
@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
@go tool cover -html=coverage.txt -o coverage.html
# Run fuzzing
CLANG := $(shell which clang-17 2>/dev/null)
.PHONY: check-clang all
check-clang:
ifeq ($(CLANG),)
@echo "clang-17 is not installed. Please install it before proceeding - https://apt.llvm.org/llvm.sh "
@exit 1
endif
.PHONY: check-ngfuzz all
check-ngfuzz:
@if [ -z "$(FUZZ_NGFUZZ_DIR)" ]; then \
echo "Please set a variable FUZZ_NGFUZZ_DIR to specify path to the ngfuzz"; \
exit 1; \
fi
.PHONY: install-fuzzing-deps
install-fuzzing-deps: check-clang check-ngfuzz
.PHONY: fuzz
fuzz: install-fuzzing-deps
@START_PATH=$$(pwd); \
ROOT_PATH=$$(realpath --relative-to=$(FUZZ_NGFUZZ_DIR) $$START_PATH) ; \
cd $(FUZZ_NGFUZZ_DIR) && \
./ngfuzz -clean && \
./ngfuzz -fuzz $(FUZZ_FUNCTIONS) -rootdir $$ROOT_PATH -timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \
./ngfuzz -report
# Reformat code
format:
@echo "⇒ Processing gofmt check"
@ -87,6 +124,7 @@ image:
@docker build \
--build-arg REPO=$(REPO) \
--build-arg VERSION=$(VERSION) \
--build-arg GOFLAGS=$(GOFLAGS) \
--rm \
-f .docker/Dockerfile \
-t $(HUB_IMAGE):$(HUB_TAG) .

View file

@ -1,5 +1,5 @@
<p align="center">
<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
<img src="./.forgejo/logo.svg" width="500px" alt="FrostFS logo">
</p>
<p align="center">
<a href="https://frostfs.info">FrostFS</a> is a decentralized distributed object storage integrated with the <a href="https://neo.org">NEO Blockchain</a>.
@ -93,6 +93,24 @@ HTTP/1.1 200 OK
Also, you can configure domains using `.env` variables or `yaml` file.
## Fuzzing
To run fuzzing tests use the following command:
```shell
$ make fuzz
```
This command will install dependencies for the fuzzing process and run existing fuzzing tests.
You can also use the following arguments:
```
FUZZ_TIMEOUT - time to run each fuzzing test (default 30)
FUZZ_FUNCTIONS - fuzzing tests that will be started (default "all")
FUZZ_AUX - additional parameters for the fuzzer (for example, "-debug")
FUZZ_NGFUZZ_DIR - path to ngfuzz tool
````
## Documentation
- [Configuration](./docs/configuration.md)

26
SECURITY.md Normal file
View file

@ -0,0 +1,26 @@
# Security Policy
## How To Report a Vulnerability
If you think you have found a vulnerability in this repository, please report it to us through coordinated disclosure.
**Please do not report security vulnerabilities through public issues, discussions, or change requests.**
Instead, you can report it using one of the following ways:
* Contact the [TrueCloudLab Security Team](mailto:security@frostfs.info) via email
Please include as much of the information listed below as you can to help us better understand and resolve the issue:
* The type of issue (e.g., buffer overflow, or cross-site scripting)
* Affected version(s)
* Impact of the issue, including how an attacker might exploit the issue
* Step-by-step instructions to reproduce the issue
* The location of the affected source code (tag/branch/commit or direct URL)
* Full paths of source file(s) related to the manifestation of the issue
* Any special configuration required to reproduce the issue
* Any log files that are related to this issue (if possible)
* Proof-of-concept or exploit code (if possible)
This information will help us triage your report more quickly.

View file

@ -1 +1 @@
v0.30.0
v0.31.0

View file

@ -14,16 +14,17 @@ import (
"time"
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/aws/aws-sdk-go/aws/credentials"
)
// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
// AuthorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
var AuthorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
// postPolicyCredentialRegexp -- is regexp for credentials when uploading file using POST with policy.
var postPolicyCredentialRegexp = regexp.MustCompile(`(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request`)
@ -34,6 +35,11 @@ type (
postReg *RegexpSubmatcher
cli tokens.Credentials
allowedAccessKeyIDPrefixes []string // empty slice means all access key ids are allowed
settings CenterSettings
}
CenterSettings interface {
AccessBoxContainer() (cid.ID, bool)
}
//nolint:revive
@ -50,7 +56,6 @@ type (
)
const (
accessKeyPartsNum = 2
authHeaderPartsNum = 6
maxFormSizeMemory = 50 * 1048576 // 50 MB
@ -82,24 +87,20 @@ var ContentSHA256HeaderStandardValue = map[string]struct{}{
}
// New creates an instance of AuthCenter.
func New(creds tokens.Credentials, prefixes []string) *Center {
func New(creds tokens.Credentials, prefixes []string, settings CenterSettings) *Center {
return &Center{
cli: creds,
reg: NewRegexpMatcher(authorizationFieldRegexp),
reg: NewRegexpMatcher(AuthorizationFieldRegexp),
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
allowedAccessKeyIDPrefixes: prefixes,
settings: settings,
}
}
func (c *Center) parseAuthHeader(header string) (*AuthHeader, error) {
submatches := c.reg.GetSubmatches(header)
if len(submatches) != authHeaderPartsNum {
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrAuthorizationHeaderMalformed), header)
}
accessKey := strings.Split(submatches["access_key_id"], "0")
if len(accessKey) != accessKeyPartsNum {
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrInvalidAccessKeyID), accessKey)
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrAuthorizationHeaderMalformed), header)
}
signedFields := strings.Split(submatches["signed_header_fields"], ";")
@ -114,15 +115,6 @@ func (c *Center) parseAuthHeader(header string) (*AuthHeader, error) {
}, nil
}
func getAddress(accessKeyID string) (oid.Address, error) {
var addr oid.Address
if err := addr.DecodeString(strings.ReplaceAll(accessKeyID, "0", "/")); err != nil {
return addr, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrInvalidAccessKeyID), accessKeyID)
}
return addr, nil
}
func IsStandardContentSHA256(key string) bool {
_, ok := ContentSHA256HeaderStandardValue[key]
return ok
@ -181,14 +173,14 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
return nil, err
}
addr, err := getAddress(authHdr.AccessKeyID)
cnrID, err := c.getAccessBoxContainer(authHdr.AccessKeyID)
if err != nil {
return nil, err
}
box, attrs, err := c.cli.GetBox(r.Context(), addr)
box, attrs, err := c.cli.GetBox(r.Context(), cnrID, authHdr.AccessKeyID)
if err != nil {
return nil, fmt.Errorf("get box '%s': %w", addr, err)
return nil, fmt.Errorf("get box by access key '%s': %w", authHdr.AccessKeyID, err)
}
if err = checkFormatHashContentSHA256(r.Header.Get(AmzContentSHA256)); err != nil {
@ -216,15 +208,29 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) {
return result, nil
}
func (c *Center) getAccessBoxContainer(accessKeyID string) (cid.ID, error) {
var addr oid.Address
if err := addr.DecodeString(strings.ReplaceAll(accessKeyID, "0", "/")); err == nil {
return addr.Container(), nil
}
cnrID, ok := c.settings.AccessBoxContainer()
if ok {
return cnrID, nil
}
return cid.ID{}, fmt.Errorf("%w: unknown container for creds '%s'", apierr.GetAPIError(apierr.ErrInvalidAccessKeyID), accessKeyID)
}
func checkFormatHashContentSHA256(hash string) error {
if !IsStandardContentSHA256(hash) {
hashBinary, err := hex.DecodeString(hash)
if err != nil {
return fmt.Errorf("%w: decode hash: %s: %s", apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch),
return fmt.Errorf("%w: decode hash: %s: %s", apierr.GetAPIError(apierr.ErrContentSHA256Mismatch),
hash, err.Error())
}
if len(hashBinary) != sha256.Size && len(hash) != 0 {
return fmt.Errorf("%w: invalid hash size %d", apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch), len(hashBinary))
return fmt.Errorf("%w: invalid hash size %d", apierr.GetAPIError(apierr.ErrContentSHA256Mismatch), len(hashBinary))
}
}
@ -242,12 +248,12 @@ func (c Center) checkAccessKeyID(accessKeyID string) error {
}
}
return fmt.Errorf("%w: accesskeyID prefix isn't allowed", apiErrors.GetAPIError(apiErrors.ErrAccessDenied))
return fmt.Errorf("%w: accesskeyID prefix isn't allowed", apierr.GetAPIError(apierr.ErrAccessDenied))
}
func (c *Center) checkFormData(r *http.Request) (*middleware.Box, error) {
if err := r.ParseMultipartForm(maxFormSizeMemory); err != nil {
return nil, fmt.Errorf("%w: parse multipart form with max size %d", apiErrors.GetAPIError(apiErrors.ErrInvalidArgument), maxFormSizeMemory)
return nil, fmt.Errorf("%w: parse multipart form with max size %d", apierr.GetAPIError(apierr.ErrInvalidArgument), maxFormSizeMemory)
}
if err := prepareForm(r.MultipartForm); err != nil {
@ -262,7 +268,7 @@ func (c *Center) checkFormData(r *http.Request) (*middleware.Box, error) {
creds := MultipartFormValue(r, "x-amz-credential")
submatches := c.postReg.GetSubmatches(creds)
if len(submatches) != 4 {
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrAuthorizationHeaderMalformed), creds)
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrAuthorizationHeaderMalformed), creds)
}
signatureDateTime, err := time.Parse("20060102T150405Z", MultipartFormValue(r, "x-amz-date"))
@ -272,14 +278,14 @@ func (c *Center) checkFormData(r *http.Request) (*middleware.Box, error) {
accessKeyID := submatches["access_key_id"]
addr, err := getAddress(accessKeyID)
cnrID, err := c.getAccessBoxContainer(accessKeyID)
if err != nil {
return nil, err
}
box, attrs, err := c.cli.GetBox(r.Context(), addr)
box, attrs, err := c.cli.GetBox(r.Context(), cnrID, accessKeyID)
if err != nil {
return nil, fmt.Errorf("get box '%s': %w", addr, err)
return nil, fmt.Errorf("get box by accessKeyID '%s': %w", accessKeyID, err)
}
secret := box.Gate.SecretKey
@ -288,7 +294,7 @@ func (c *Center) checkFormData(r *http.Request) (*middleware.Box, error) {
signature := SignStr(secret, service, region, signatureDateTime, policy)
reqSignature := MultipartFormValue(r, "x-amz-signature")
if signature != reqSignature {
return nil, fmt.Errorf("%w: %s != %s", apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch),
return nil, fmt.Errorf("%w: %s != %s", apierr.GetAPIError(apierr.ErrSignatureDoesNotMatch),
reqSignature, signature)
}
@ -333,11 +339,11 @@ func (c *Center) checkSign(authHeader *AuthHeader, box *accessbox.Box, request *
if authHeader.IsPresigned {
now := time.Now()
if signatureDateTime.Add(authHeader.Expiration).Before(now) {
return fmt.Errorf("%w: expired: now %s, signature %s", apiErrors.GetAPIError(apiErrors.ErrExpiredPresignRequest),
return fmt.Errorf("%w: expired: now %s, signature %s", apierr.GetAPIError(apierr.ErrExpiredPresignRequest),
now.Format(time.RFC3339), signatureDateTime.Format(time.RFC3339))
}
if now.Before(signatureDateTime) {
return fmt.Errorf("%w: signature time from the future: now %s, signature %s", apiErrors.GetAPIError(apiErrors.ErrBadRequest),
return fmt.Errorf("%w: signature time from the future: now %s, signature %s", apierr.GetAPIError(apierr.ErrBadRequest),
now.Format(time.RFC3339), signatureDateTime.Format(time.RFC3339))
}
if _, err := signer.Presign(request, nil, authHeader.Service, authHeader.Region, authHeader.Expiration, signatureDateTime); err != nil {
@ -352,7 +358,7 @@ func (c *Center) checkSign(authHeader *AuthHeader, box *accessbox.Box, request *
}
if authHeader.SignatureV4 != signature {
return fmt.Errorf("%w: %s != %s: headers %v", apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch),
return fmt.Errorf("%w: %s != %s: headers %v", apierr.GetAPIError(apierr.ErrSignatureDoesNotMatch),
authHeader.SignatureV4, signature, authHeader.SignedFields)
}

View file

@ -0,0 +1,88 @@
//go:build gofuzz
// +build gofuzz
package auth
import (
"strings"
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/aws/aws-sdk-go/aws/credentials"
utils "github.com/trailofbits/go-fuzz-utils"
)
const (
fuzzSuccessExitCode = 0
fuzzFailExitCode = -1
)
func InitFuzzAuthenticate() {
}
func DoFuzzAuthenticate(input []byte) int {
// FUZZER INIT
if len(input) < 100 {
return fuzzFailExitCode
}
tp, err := utils.NewTypeProvider(input)
if err != nil {
return fuzzFailExitCode
}
var accessKeyAddr oid.Address
err = tp.Fill(accessKeyAddr)
if err != nil {
return fuzzFailExitCode
}
accessKeyID := strings.ReplaceAll(accessKeyAddr.String(), "/", "0")
secretKey, err := tp.GetString()
awsCreds := credentials.NewStaticCredentials(accessKeyID, secretKey, "")
reqData := RequestData{
Method: "GET",
Endpoint: "http://localhost:8084",
Bucket: "my-bucket",
Object: "@obj/name",
}
presignData := PresignData{
Service: "s3",
Region: "spb",
Lifetime: 10 * time.Minute,
SignTime: time.Now().UTC(),
}
req, err := PresignRequest(awsCreds, reqData, presignData)
if req == nil {
return fuzzFailExitCode
}
expBox := &accessbox.Box{
Gate: &accessbox.GateData{
SecretKey: secretKey,
},
}
mock := newTokensFrostfsMock()
mock.addBox(accessKeyAddr, expBox)
c := &Center{
cli: mock,
reg: NewRegexpMatcher(AuthorizationFieldRegexp),
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
}
_, _ = c.Authenticate(req)
return fuzzSuccessExitCode
}
func FuzzAuthenticate(f *testing.F) {
f.Fuzz(func(t *testing.T, data []byte) {
DoFuzzAuthenticate(data)
})
}

View file

@ -17,8 +17,9 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
@ -28,11 +29,23 @@ import (
"go.uber.org/zap/zaptest"
)
type centerSettingsMock struct {
accessBoxContainer *cid.ID
}
func (c *centerSettingsMock) AccessBoxContainer() (cid.ID, bool) {
if c.accessBoxContainer == nil {
return cid.ID{}, false
}
return *c.accessBoxContainer, true
}
func TestAuthHeaderParse(t *testing.T) {
defaultHeader := "AWS4-HMAC-SHA256 Credential=oid0cid/20210809/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2811ccb9e242f41426738fb1f"
center := &Center{
reg: NewRegexpMatcher(authorizationFieldRegexp),
reg: NewRegexpMatcher(AuthorizationFieldRegexp),
settings: &centerSettingsMock{},
}
for _, tc := range []struct {
@ -57,11 +70,6 @@ func TestAuthHeaderParse(t *testing.T) {
err: errors.GetAPIError(errors.ErrAuthorizationHeaderMalformed),
expected: nil,
},
{
header: strings.ReplaceAll(defaultHeader, "oid0cid", "oidcid"),
err: errors.GetAPIError(errors.ErrInvalidAccessKeyID),
expected: nil,
},
} {
authHeader, err := center.parseAuthHeader(tc.header)
require.ErrorIs(t, err, tc.err, tc.header)
@ -69,43 +77,6 @@ func TestAuthHeaderParse(t *testing.T) {
}
}
func TestAuthHeaderGetAddress(t *testing.T) {
defaulErr := errors.GetAPIError(errors.ErrInvalidAccessKeyID)
for _, tc := range []struct {
authHeader *AuthHeader
err error
}{
{
authHeader: &AuthHeader{
AccessKeyID: "vWqF8cMDRbJcvnPLALoQGnABPPhw8NyYMcGsfDPfZJM0HrgjonN8CgFvCZ3kh9BUXw4W2tJ5E7EAGhueSF122HB",
},
err: nil,
},
{
authHeader: &AuthHeader{
AccessKeyID: "vWqF8cMDRbJcvnPLALoQGnABPPhw8NyYMcGsfDPfZJMHrgjonN8CgFvCZ3kh9BUXw4W2tJ5E7EAGhueSF122HB",
},
err: defaulErr,
},
{
authHeader: &AuthHeader{
AccessKeyID: "oid0cid",
},
err: defaulErr,
},
{
authHeader: &AuthHeader{
AccessKeyID: "oidcid",
},
err: defaulErr,
},
} {
_, err := getAddress(tc.authHeader.AccessKeyID)
require.ErrorIs(t, err, tc.err, tc.authHeader.AccessKeyID)
}
}
func TestSignature(t *testing.T) {
secret := "66be461c3cd429941c55daf42fad2b8153e5a2016ba89c9494d97677cc9d3872"
strToSign := "eyAiZXhwaXJhdGlvbiI6ICIyMDE1LTEyLTMwVDEyOjAwOjAwLjAwMFoiLAogICJjb25kaXRpb25zIjogWwogICAgeyJidWNrZXQiOiAiYWNsIn0sCiAgICBbInN0YXJ0cy13aXRoIiwgIiRrZXkiLCAidXNlci91c2VyMS8iXSwKICAgIHsic3VjY2Vzc19hY3Rpb25fcmVkaXJlY3QiOiAiaHR0cDovL2xvY2FsaG9zdDo4MDg0L2FjbCJ9LAogICAgWyJzdGFydHMtd2l0aCIsICIkQ29udGVudC1UeXBlIiwgImltYWdlLyJdLAogICAgeyJ4LWFtei1tZXRhLXV1aWQiOiAiMTQzNjUxMjM2NTEyNzQifSwKICAgIFsic3RhcnRzLXdpdGgiLCAiJHgtYW16LW1ldGEtdGFnIiwgIiJdLAoKICAgIHsiWC1BbXotQ3JlZGVudGlhbCI6ICI4Vmk0MVBIbjVGMXNzY2J4OUhqMXdmMUU2aERUYURpNndxOGhxTU05NllKdTA1QzVDeUVkVlFoV1E2aVZGekFpTkxXaTlFc3BiUTE5ZDRuR3pTYnZVZm10TS8yMDE1MTIyOS91cy1lYXN0LTEvczMvYXdzNF9yZXF1ZXN0In0sCiAgICB7IngtYW16LWFsZ29yaXRobSI6ICJBV1M0LUhNQUMtU0hBMjU2In0sCiAgICB7IlgtQW16LURhdGUiOiAiMjAxNTEyMjlUMDAwMDAwWiIgfSwKICAgIHsieC1pZ25vcmUtdG1wIjogInNvbWV0aGluZyIgfQogIF0KfQ=="
@ -171,17 +142,17 @@ func TestCheckFormatContentSHA256(t *testing.T) {
}
type frostFSMock struct {
objects map[oid.Address]*object.Object
objects map[string]*object.Object
}
func newFrostFSMock() *frostFSMock {
return &frostFSMock{
objects: map[oid.Address]*object.Object{},
objects: map[string]*object.Object{},
}
}
func (f *frostFSMock) GetCredsObject(_ context.Context, address oid.Address) (*object.Object, error) {
obj, ok := f.objects[address]
func (f *frostFSMock) GetCredsObject(_ context.Context, prm tokens.PrmGetCredsObject) (*object.Object, error) {
obj, ok := f.objects[prm.AccessKeyID]
if !ok {
return nil, fmt.Errorf("not found")
}
@ -208,7 +179,7 @@ func TestAuthenticate(t *testing.T) {
GateKey: key.PublicKey(),
}}
accessBox, secret, err := accessbox.PackTokens(gateData, []byte("secret"))
accessBox, secret, err := accessbox.PackTokens(gateData, []byte("secret"), false)
require.NoError(t, err)
data, err := accessBox.Marshal()
require.NoError(t, err)
@ -219,10 +190,10 @@ func TestAuthenticate(t *testing.T) {
obj.SetContainerID(addr.Container())
obj.SetID(addr.Object())
frostfs := newFrostFSMock()
frostfs.objects[addr] = &obj
accessKeyID := getAccessKeyID(addr)
accessKeyID := addr.Container().String() + "0" + addr.Object().String()
frostfs := newFrostFSMock()
frostfs.objects[accessKeyID] = &obj
awsCreds := credentials.NewStaticCredentials(accessKeyID, secret.SecretKey, "")
defaultSigner := v4.NewSigner(awsCreds)
@ -413,13 +384,13 @@ func TestAuthenticate(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
creds := tokens.New(bigConfig)
cntr := New(creds, tc.prefixes)
cntr := New(creds, tc.prefixes, &centerSettingsMock{})
box, err := cntr.Authenticate(tc.request)
if tc.err {
require.Error(t, err)
if tc.errCode > 0 {
err = frostfsErrors.UnwrapErr(err)
err = frosterr.UnwrapErr(err)
require.Equal(t, errors.GetAPIError(tc.errCode), err)
}
} else {
@ -455,7 +426,7 @@ func TestHTTPPostAuthenticate(t *testing.T) {
GateKey: key.PublicKey(),
}}
accessBox, secret, err := accessbox.PackTokens(gateData, []byte("secret"))
accessBox, secret, err := accessbox.PackTokens(gateData, []byte("secret"), false)
require.NoError(t, err)
data, err := accessBox.Marshal()
require.NoError(t, err)
@ -466,10 +437,11 @@ func TestHTTPPostAuthenticate(t *testing.T) {
obj.SetContainerID(addr.Container())
obj.SetID(addr.Object())
frostfs := newFrostFSMock()
frostfs.objects[addr] = &obj
accessKeyID := getAccessKeyID(addr)
frostfs := newFrostFSMock()
frostfs.objects[accessKeyID] = &obj
accessKeyID := addr.Container().String() + "0" + addr.Object().String()
invalidAccessKeyID := oidtest.Address().String() + "0" + oidtest.Address().Object().String()
timeToSign := time.Now()
@ -590,13 +562,13 @@ func TestHTTPPostAuthenticate(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
creds := tokens.New(bigConfig)
cntr := New(creds, tc.prefixes)
cntr := New(creds, tc.prefixes, &centerSettingsMock{})
box, err := cntr.Authenticate(tc.request)
if tc.err {
require.Error(t, err)
if tc.errCode > 0 {
err = frostfsErrors.UnwrapErr(err)
err = frosterr.UnwrapErr(err)
require.Equal(t, errors.GetAPIError(tc.errCode), err)
}
} else {
@ -633,3 +605,7 @@ func getRequestWithMultipartForm(t *testing.T, policy, creds, date, sign, fieldN
return req
}
func getAccessKeyID(addr oid.Address) string {
return strings.ReplaceAll(addr.EncodeToString(), "/", "0")
}

View file

@ -23,6 +23,7 @@ type PresignData struct {
Region string
Lifetime time.Duration
SignTime time.Time
Headers map[string]string
}
// PresignRequest forms pre-signed request to access objects without aws credentials.
@ -34,7 +35,10 @@ func PresignRequest(creds *credentials.Credentials, reqData RequestData, presign
}
req.Header.Set(AmzDate, presignData.SignTime.Format("20060102T150405Z"))
req.Header.Set(ContentTypeHdr, "text/plain")
for k, v := range presignData.Headers {
req.Header.Set(k, v)
}
signer := v4.NewSigner(creds)
signer.DisableURIPathEscaping = true

View file

@ -29,11 +29,11 @@ func newTokensFrostfsMock() *credentialsMock {
}
func (m credentialsMock) addBox(addr oid.Address, box *accessbox.Box) {
m.boxes[addr.String()] = box
m.boxes[getAccessKeyID(addr)] = box
}
func (m credentialsMock) GetBox(_ context.Context, addr oid.Address) (*accessbox.Box, []object.Attribute, error) {
box, ok := m.boxes[addr.String()]
func (m credentialsMock) GetBox(_ context.Context, _ cid.ID, accessKeyID string) (*accessbox.Box, []object.Attribute, error) {
box, ok := m.boxes[accessKeyID]
if !ok {
return nil, nil, &apistatus.ObjectNotFound{}
}
@ -41,11 +41,11 @@ func (m credentialsMock) GetBox(_ context.Context, addr oid.Address) (*accessbox
return box, nil, nil
}
func (m credentialsMock) Put(context.Context, cid.ID, tokens.CredentialsParam) (oid.Address, error) {
func (m credentialsMock) Put(context.Context, tokens.CredentialsParam) (oid.Address, error) {
return oid.Address{}, nil
}
func (m credentialsMock) Update(context.Context, oid.Address, tokens.CredentialsParam) (oid.Address, error) {
func (m credentialsMock) Update(context.Context, tokens.CredentialsParam) (oid.Address, error) {
return oid.Address{}, nil
}
@ -84,9 +84,10 @@ func TestCheckSign(t *testing.T) {
mock.addBox(accessKeyAddr, expBox)
c := &Center{
cli: mock,
reg: NewRegexpMatcher(authorizationFieldRegexp),
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
cli: mock,
reg: NewRegexpMatcher(AuthorizationFieldRegexp),
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
settings: &centerSettingsMock{},
}
box, err := c.Authenticate(req)
require.NoError(t, err)

View file

@ -30,6 +30,7 @@ type (
Box *accessbox.Box
Attributes []object.Attribute
PutTime time.Time
Address *oid.Address
}
)
@ -57,8 +58,8 @@ func NewAccessBoxCache(config *Config) *AccessBoxCache {
}
// Get returns a cached accessbox.
func (o *AccessBoxCache) Get(address oid.Address) *AccessBoxCacheValue {
entry, err := o.cache.Get(address)
func (o *AccessBoxCache) Get(accessKeyID string) *AccessBoxCacheValue {
entry, err := o.cache.Get(accessKeyID)
if err != nil {
return nil
}
@ -74,16 +75,11 @@ func (o *AccessBoxCache) Get(address oid.Address) *AccessBoxCacheValue {
}
// Put stores an accessbox to cache.
func (o *AccessBoxCache) Put(address oid.Address, box *accessbox.Box, attrs []object.Attribute) error {
val := &AccessBoxCacheValue{
Box: box,
Attributes: attrs,
PutTime: time.Now(),
}
return o.cache.Set(address, val)
func (o *AccessBoxCache) Put(accessKeyID string, val *AccessBoxCacheValue) error {
return o.cache.Set(accessKeyID, val)
}
// Delete removes an accessbox from cache.
func (o *AccessBoxCache) Delete(address oid.Address) {
o.cache.Remove(address)
func (o *AccessBoxCache) Delete(accessKeyID string) {
o.cache.Remove(accessKeyID)
}

View file

@ -1,13 +1,14 @@
package cache
import (
"strings"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neo-go/pkg/util"
@ -22,18 +23,21 @@ func TestAccessBoxCacheType(t *testing.T) {
addr := oidtest.Address()
box := &accessbox.Box{}
var attrs []object.Attribute
val := &AccessBoxCacheValue{
Box: box,
}
err := cache.Put(addr, box, attrs)
accessKeyID := getAccessKeyID(addr)
err := cache.Put(accessKeyID, val)
require.NoError(t, err)
val := cache.Get(addr)
require.Equal(t, box, val.Box)
require.Equal(t, attrs, val.Attributes)
resVal := cache.Get(accessKeyID)
require.Equal(t, box, resVal.Box)
require.Equal(t, 0, observedLog.Len())
err = cache.cache.Set(addr, "tmp")
err = cache.cache.Set(accessKeyID, "tmp")
require.NoError(t, err)
assertInvalidCacheEntry(t, cache.Get(addr), observedLog)
assertInvalidCacheEntry(t, cache.Get(accessKeyID), observedLog)
}
func TestBucketsCacheType(t *testing.T) {
@ -230,3 +234,7 @@ func getObservedLogger() (*zap.Logger, *observer.ObservedLogs) {
loggerCore, observedLog := observer.New(zap.WarnLevel)
return zap.New(loggerCore), observedLog
}
func getAccessKeyID(addr oid.Address) string {
return strings.ReplaceAll(addr.EncodeToString(), "/", "0")
}

65
api/cache/network_info.go vendored Normal file
View file

@ -0,0 +1,65 @@
package cache
import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"github.com/bluele/gcache"
"go.uber.org/zap"
)
type (
// NetworkInfoCache provides cache for network info.
NetworkInfoCache struct {
cache gcache.Cache
logger *zap.Logger
}
// NetworkInfoCacheConfig stores expiration params for cache.
NetworkInfoCacheConfig struct {
Lifetime time.Duration
Logger *zap.Logger
}
)
const (
DefaultNetworkInfoCacheLifetime = 1 * time.Minute
networkInfoCacheSize = 1
networkInfoKey = "network_info"
)
// DefaultNetworkInfoConfig returns new default cache expiration values.
func DefaultNetworkInfoConfig(logger *zap.Logger) *NetworkInfoCacheConfig {
return &NetworkInfoCacheConfig{
Lifetime: DefaultNetworkInfoCacheLifetime,
Logger: logger,
}
}
// NewNetworkInfoCache creates an object of NetworkInfoCache.
func NewNetworkInfoCache(config *NetworkInfoCacheConfig) *NetworkInfoCache {
gc := gcache.New(networkInfoCacheSize).LRU().Expiration(config.Lifetime).Build()
return &NetworkInfoCache{cache: gc, logger: config.Logger}
}
func (c *NetworkInfoCache) Get() *netmap.NetworkInfo {
entry, err := c.cache.Get(networkInfoKey)
if err != nil {
return nil
}
result, ok := entry.(netmap.NetworkInfo)
if !ok {
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return nil
}
return &result
}
func (c *NetworkInfoCache) Put(info netmap.NetworkInfo) error {
return c.cache.Set(networkInfoKey, info)
}

20
api/cache/system.go vendored
View file

@ -88,6 +88,22 @@ func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration {
return result
}
func (o *SystemCache) GetLifecycleConfiguration(key string) *data.LifecycleConfiguration {
entry, err := o.cache.Get(key)
if err != nil {
return nil
}
result, ok := entry.(*data.LifecycleConfiguration)
if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
return nil
}
return result
}
func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
entry, err := o.cache.Get(key)
if err != nil {
@ -133,6 +149,10 @@ func (o *SystemCache) PutCORS(key string, obj *data.CORSConfiguration) error {
return o.cache.Set(key, obj)
}
func (o *SystemCache) PutLifecycleConfiguration(key string, obj *data.LifecycleConfiguration) error {
return o.cache.Set(key, obj)
}
func (o *SystemCache) PutSettings(key string, settings *data.BucketSettings) error {
return o.cache.Set(key, settings)
}

View file

@ -12,8 +12,9 @@ import (
)
const (
bktSettingsObject = ".s3-settings"
bktCORSConfigurationObject = ".s3-cors"
bktSettingsObject = ".s3-settings"
bktCORSConfigurationObject = ".s3-cors"
bktLifecycleConfigurationObject = ".s3-lifecycle"
VersioningUnversioned = "Unversioned"
VersioningEnabled = "Enabled"
@ -81,6 +82,15 @@ type (
VersionID string
NoErrorOnDeleteMarker bool
}
// CreatedObjectInfo stores created object info.
CreatedObjectInfo struct {
ID oid.ID
Size uint64
HashSum []byte
MD5Sum []byte
CreationEpoch uint64
}
)
// SettingsObjectName is a system name for a bucket settings file.
@ -91,6 +101,10 @@ func (b *BucketInfo) CORSObjectName() string {
return b.CID.EncodeToString() + bktCORSConfigurationObject
}
func (b *BucketInfo) LifecycleConfigurationObjectName() string {
return b.CID.EncodeToString() + bktLifecycleConfigurationObject
}
// VersionID returns object version from ObjectInfo.
func (o *ObjectInfo) VersionID() string { return o.ID.EncodeToString() }

56
api/data/lifecycle.go Normal file
View file

@ -0,0 +1,56 @@
package data
import "encoding/xml"
const (
LifecycleStatusEnabled = "Enabled"
LifecycleStatusDisabled = "Disabled"
)
type (
LifecycleConfiguration struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LifecycleConfiguration" json:"-"`
Rules []LifecycleRule `xml:"Rule"`
}
LifecycleRule struct {
Status string `xml:"Status,omitempty"`
AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
Expiration *LifecycleExpiration `xml:"Expiration,omitempty"`
Filter *LifecycleRuleFilter `xml:"Filter,omitempty"`
ID string `xml:"ID,omitempty"`
NonCurrentVersionExpiration *NonCurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
Prefix string `xml:"Prefix,omitempty"`
}
AbortIncompleteMultipartUpload struct {
DaysAfterInitiation *int `xml:"DaysAfterInitiation,omitempty"`
}
LifecycleExpiration struct {
Date string `xml:"Date,omitempty"`
Days *int `xml:"Days,omitempty"`
Epoch *uint64 `xml:"Epoch,omitempty"`
ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"`
}
LifecycleRuleFilter struct {
And *LifecycleRuleAndOperator `xml:"And,omitempty"`
ObjectSizeGreaterThan *uint64 `xml:"ObjectSizeGreaterThan,omitempty"`
ObjectSizeLessThan *uint64 `xml:"ObjectSizeLessThan,omitempty"`
Prefix string `xml:"Prefix,omitempty"`
Tag *Tag `xml:"Tag,omitempty"`
}
LifecycleRuleAndOperator struct {
ObjectSizeGreaterThan *uint64 `xml:"ObjectSizeGreaterThan,omitempty"`
ObjectSizeLessThan *uint64 `xml:"ObjectSizeLessThan,omitempty"`
Prefix string `xml:"Prefix,omitempty"`
Tags []Tag `xml:"Tag"`
}
NonCurrentVersionExpiration struct {
NewerNonCurrentVersions *int `xml:"NewerNoncurrentVersions,omitempty"`
NonCurrentDays *int `xml:"NoncurrentDays,omitempty"`
}
)

View file

@ -72,6 +72,7 @@ type BaseNodeVersion struct {
Created *time.Time
Owner *user.ID
IsDeleteMarker bool
CreationEpoch uint64
}
func (v *BaseNodeVersion) GetETag(md5Enabled bool) string {
@ -110,6 +111,7 @@ type MultipartInfo struct {
Meta map[string]string
CopiesNumbers []uint32
Finished bool
CreationEpoch uint64
}
// PartInfo is upload information about part.
@ -124,6 +126,14 @@ type PartInfo struct {
Created time.Time `json:"created"`
}
type PartInfoExtended struct {
PartInfo
// Timestamp is used to find the latest version of part info in case of tree split
// when there are multiple nodes for the same part.
Timestamp uint64
}
// ToHeaderString form short part representation to use in S3-Completed-Parts header.
func (p *PartInfo) ToHeaderString() string {
// ETag value contains SHA256 checksum which is used while getting object parts attributes.

View file

@ -1,10 +1,13 @@
package errors
import (
"errors"
"fmt"
"net/http"
frosterrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
)
type (
@ -187,6 +190,9 @@ const (
ErrInvalidRequestLargeCopy
ErrInvalidStorageClass
VersionIDMarkerWithoutKeyMarker
ErrInvalidRangeLength
ErrRangeOutOfBounds
ErrMissingContentRange
ErrMalformedJSON
ErrInsecureClientRequest
@ -1739,12 +1745,30 @@ var errorCodes = errorCodeMap{
Description: "Part number must be an integer between 1 and 10000, inclusive",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidRangeLength: {
ErrCode: ErrInvalidRangeLength,
Code: "InvalidRange",
Description: "Provided range length must be equal to content length",
HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
},
ErrRangeOutOfBounds: {
ErrCode: ErrRangeOutOfBounds,
Code: "InvalidRange",
Description: "Provided range is outside of object bounds",
HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
},
ErrMissingContentRange: {
ErrCode: ErrMissingContentRange,
Code: "MissingContentRange",
Description: "Content-Range header is mandatory for this type of request",
HTTPStatusCode: http.StatusBadRequest,
},
// Add your error structure here.
}
// IsS3Error checks if the provided error is a specific s3 error.
func IsS3Error(err error, code ErrorCode) bool {
err = frosterrors.UnwrapErr(err)
err = frosterr.UnwrapErr(err)
e, ok := err.(Error)
return ok && e.ErrCode == code
}
@ -1781,6 +1805,26 @@ func GetAPIErrorWithError(code ErrorCode, err error) Error {
return errorCodes.toAPIErrWithErr(code, err)
}
// TransformToS3Error converts FrostFS error to the corresponding S3 error type.
func TransformToS3Error(err error) error {
err = frosterr.UnwrapErr(err) // this wouldn't work with errors.Join
var s3err Error
if errors.As(err, &s3err) {
return err
}
if errors.Is(err, frostfs.ErrAccessDenied) ||
errors.Is(err, tree.ErrNodeAccessDenied) {
return GetAPIError(ErrAccessDenied)
}
if errors.Is(err, frostfs.ErrGatewayTimeout) {
return GetAPIError(ErrGatewayTimeout)
}
return GetAPIError(ErrInternalError)
}
// ObjectError -- error that is linked to a specific object.
type ObjectError struct {
Err error

View file

@ -2,7 +2,12 @@ package errors
import (
"errors"
"fmt"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
"github.com/stretchr/testify/require"
)
func BenchmarkErrCode(b *testing.B) {
@ -24,3 +29,56 @@ func BenchmarkErrorsIs(b *testing.B) {
}
}
}
func TestTransformS3Errors(t *testing.T) {
for _, tc := range []struct {
name string
err error
expected ErrorCode
}{
{
name: "simple std error to internal error",
err: errors.New("some error"),
expected: ErrInternalError,
},
{
name: "layer access denied error to s3 access denied error",
err: frostfs.ErrAccessDenied,
expected: ErrAccessDenied,
},
{
name: "wrapped layer access denied error to s3 access denied error",
err: fmt.Errorf("wrap: %w", frostfs.ErrAccessDenied),
expected: ErrAccessDenied,
},
{
name: "layer node access denied error to s3 access denied error",
err: tree.ErrNodeAccessDenied,
expected: ErrAccessDenied,
},
{
name: "layer gateway timeout error to s3 gateway timeout error",
err: frostfs.ErrGatewayTimeout,
expected: ErrGatewayTimeout,
},
{
name: "s3 error to s3 error",
err: GetAPIError(ErrInvalidPart),
expected: ErrInvalidPart,
},
{
name: "wrapped s3 error to s3 error",
err: fmt.Errorf("wrap: %w", GetAPIError(ErrInvalidPart)),
expected: ErrInvalidPart,
},
} {
t.Run(tc.name, func(t *testing.T) {
err := TransformToS3Error(tc.err)
s3err, ok := err.(Error)
require.True(t, ok, "error must be s3 error")
require.Equalf(t, tc.expected, s3err.ErrCode,
"expected: '%s', got: '%s'",
GetAPIError(tc.expected).Code, GetAPIError(s3err.ErrCode).Code)
})
}
}

View file

@ -52,18 +52,18 @@ func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't get bucket settings", reqInfo, err)
return
}
if err = middleware.EncodeToResponse(w, h.encodeBucketCannedACL(ctx, bktInfo, settings)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
return
}
}
@ -127,17 +127,18 @@ func getTokenIssuerKey(box *accessbox.Box) (*keys.PublicKey, error) {
}
func (h *handler) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't get bucket settings", reqInfo, err)
return
}
@ -149,30 +150,30 @@ func (h *handler) putBucketACLAPEHandler(w http.ResponseWriter, r *http.Request,
defer func() {
if errBody := r.Body.Close(); errBody != nil {
h.reqLogger(r.Context()).Warn(logs.CouldNotCloseRequestBody, zap.Error(errBody))
h.reqLogger(ctx).Warn(logs.CouldNotCloseRequestBody, zap.Error(errBody))
}
}()
written, err := io.Copy(io.Discard, r.Body)
if err != nil {
h.logAndSendError(w, "couldn't read request body", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't read request body", reqInfo, err)
return
}
if written != 0 || len(r.Header.Get(api.AmzACL)) == 0 {
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
h.logAndSendError(ctx, w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
return
}
cannedACL, err := parseCannedACL(r.Header)
if err != nil {
h.logAndSendError(w, "could not parse canned ACL", reqInfo, err)
h.logAndSendError(ctx, w, "could not parse canned ACL", reqInfo, err)
return
}
chainRules := bucketCannedACLToAPERules(cannedACL, reqInfo, bktInfo.CID)
if err = h.ape.SaveACLChains(bktInfo.CID.EncodeToString(), chainRules); err != nil {
h.logAndSendError(w, "failed to add morph rule chains", reqInfo, err)
h.logAndSendError(ctx, w, "failed to add morph rule chains", reqInfo, err)
return
}
@ -184,7 +185,7 @@ func (h *handler) putBucketACLAPEHandler(w http.ResponseWriter, r *http.Request,
}
if err = h.obj.PutBucketSettings(ctx, sp); err != nil {
h.logAndSendError(w, "couldn't save bucket settings", reqInfo, err,
h.logAndSendError(ctx, w, "couldn't save bucket settings", reqInfo, err,
zap.String("container_id", bktInfo.CID.EncodeToString()))
return
}
@ -198,18 +199,18 @@ func (h *handler) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't get bucket settings", reqInfo, err)
return
}
if err = middleware.EncodeToResponse(w, h.encodePrivateCannedACL(ctx, bktInfo, settings)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
return
}
}
@ -219,19 +220,20 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(ctx)
if _, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName); err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
h.logAndSendError(ctx, w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
}
func (h *handler) GetBucketPolicyStatusHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
@ -240,13 +242,13 @@ func (h *handler) GetBucketPolicyStatusHandler(w http.ResponseWriter, r *http.Re
if strings.Contains(err.Error(), "not found") {
err = fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchBucketPolicy), err.Error())
}
h.logAndSendError(w, "failed to get policy from storage", reqInfo, err)
h.logAndSendError(ctx, w, "failed to get policy from storage", reqInfo, err)
return
}
var bktPolicy engineiam.Policy
if err = json.Unmarshal(jsonPolicy, &bktPolicy); err != nil {
h.logAndSendError(w, "could not parse bucket policy", reqInfo, err)
h.logAndSendError(ctx, w, "could not parse bucket policy", reqInfo, err)
return
}
@ -263,17 +265,18 @@ func (h *handler) GetBucketPolicyStatusHandler(w http.ResponseWriter, r *http.Re
}
if err = middleware.EncodeToResponse(w, policyStatus); err != nil {
h.logAndSendError(w, "encode and write response", reqInfo, err)
h.logAndSendError(ctx, w, "encode and write response", reqInfo, err)
return
}
}
func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
@ -282,7 +285,7 @@ func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
if strings.Contains(err.Error(), "not found") {
err = fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchBucketPolicy), err.Error())
}
h.logAndSendError(w, "failed to get policy from storage", reqInfo, err)
h.logAndSendError(ctx, w, "failed to get policy from storage", reqInfo, err)
return
}
@ -290,22 +293,23 @@ func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
w.WriteHeader(http.StatusOK)
if _, err = w.Write(jsonPolicy); err != nil {
h.logAndSendError(w, "write json policy to client", reqInfo, err)
h.logAndSendError(ctx, w, "write json policy to client", reqInfo, err)
}
}
func (h *handler) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
chainIDs := []chain.ID{getBucketChainID(chain.S3, bktInfo), getBucketChainID(chain.Ingress, bktInfo)}
if err = h.ape.DeleteBucketPolicy(reqInfo.Namespace, bktInfo.CID, chainIDs); err != nil {
h.logAndSendError(w, "failed to delete policy from storage", reqInfo, err)
h.logAndSendError(ctx, w, "failed to delete policy from storage", reqInfo, err)
return
}
}
@ -324,40 +328,41 @@ func checkOwner(info *data.BucketInfo, owner string) error {
}
func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
jsonPolicy, err := io.ReadAll(r.Body)
if err != nil {
h.logAndSendError(w, "read body", reqInfo, err)
h.logAndSendError(ctx, w, "read body", reqInfo, err)
return
}
var bktPolicy engineiam.Policy
if err = json.Unmarshal(jsonPolicy, &bktPolicy); err != nil {
h.logAndSendError(w, "could not parse bucket policy", reqInfo, err)
h.logAndSendError(ctx, w, "could not parse bucket policy", reqInfo, err)
return
}
for _, stat := range bktPolicy.Statement {
if len(stat.NotResource) != 0 {
h.logAndSendError(w, "policy resource mismatched bucket", reqInfo, errors.GetAPIError(errors.ErrMalformedPolicy))
h.logAndSendError(ctx, w, "policy resource mismatched bucket", reqInfo, errors.GetAPIError(errors.ErrMalformedPolicy))
return
}
if len(stat.NotPrincipal) != 0 && stat.Effect == engineiam.AllowEffect {
h.logAndSendError(w, "invalid NotPrincipal", reqInfo, errors.GetAPIError(errors.ErrMalformedPolicyNotPrincipal))
h.logAndSendError(ctx, w, "invalid NotPrincipal", reqInfo, errors.GetAPIError(errors.ErrMalformedPolicyNotPrincipal))
return
}
for _, resource := range stat.Resource {
if reqInfo.BucketName != strings.Split(strings.TrimPrefix(resource, arnAwsPrefix), "/")[0] {
h.logAndSendError(w, "policy resource mismatched bucket", reqInfo, errors.GetAPIError(errors.ErrMalformedPolicy))
h.logAndSendError(ctx, w, "policy resource mismatched bucket", reqInfo, errors.GetAPIError(errors.ErrMalformedPolicy))
return
}
}
@ -365,7 +370,7 @@ func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
s3Chain, err := engineiam.ConvertToS3Chain(bktPolicy, h.frostfsid)
if err != nil {
h.logAndSendError(w, "could not convert s3 policy to chain policy", reqInfo, err)
h.logAndSendError(ctx, w, "could not convert s3 policy to chain policy", reqInfo, err)
return
}
s3Chain.ID = getBucketChainID(chain.S3, bktInfo)
@ -374,10 +379,10 @@ func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
if err == nil {
nativeChain.ID = getBucketChainID(chain.Ingress, bktInfo)
} else if !stderrors.Is(err, engineiam.ErrActionsNotApplicable) {
h.logAndSendError(w, "could not convert s3 policy to native chain policy", reqInfo, err)
h.logAndSendError(ctx, w, "could not convert s3 policy to native chain policy", reqInfo, err)
return
} else {
h.reqLogger(r.Context()).Warn(logs.PolicyCouldntBeConvertedToNativeRules)
h.reqLogger(ctx).Warn(logs.PolicyCouldntBeConvertedToNativeRules)
}
chainsToSave := []*chain.Chain{s3Chain}
@ -386,7 +391,7 @@ func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
}
if err = h.ape.PutBucketPolicy(reqInfo.Namespace, bktInfo.CID, jsonPolicy, chainsToSave); err != nil {
h.logAndSendError(w, "failed to update policy in contract", reqInfo, err)
h.logAndSendError(ctx, w, "failed to update policy in contract", reqInfo, err)
return
}
}

View file

@ -11,7 +11,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
@ -28,12 +28,12 @@ func TestPutObjectACLErrorAPE(t *testing.T) {
info := createBucket(hc, bktName)
putObjectWithHeadersAssertS3Error(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPublic}, s3errors.ErrAccessControlListNotSupported)
putObjectWithHeadersAssertS3Error(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPublic}, apierr.ErrAccessControlListNotSupported)
putObjectWithHeaders(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPrivate}) // only `private` canned acl is allowed, that is actually ignored
putObjectWithHeaders(hc, bktName, objName, nil)
aclBody := &AccessControlPolicy{}
putObjectACLAssertS3Error(hc, bktName, objName, info.Box, nil, aclBody, s3errors.ErrAccessControlListNotSupported)
putObjectACLAssertS3Error(hc, bktName, objName, info.Box, nil, aclBody, apierr.ErrAccessControlListNotSupported)
aclRes := getObjectACL(hc, bktName, objName)
checkPrivateACL(t, aclRes, info.Key.PublicKey())
@ -49,7 +49,7 @@ func TestCreateObjectACLErrorAPE(t *testing.T) {
copyObject(hc, bktName, objName, objNameCopy, CopyMeta{Headers: map[string]string{api.AmzACL: basicACLPublic}}, http.StatusBadRequest)
copyObject(hc, bktName, objName, objNameCopy, CopyMeta{Headers: map[string]string{api.AmzACL: basicACLPrivate}}, http.StatusOK)
createMultipartUploadAssertS3Error(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPublic}, s3errors.ErrAccessControlListNotSupported)
createMultipartUploadAssertS3Error(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPublic}, apierr.ErrAccessControlListNotSupported)
createMultipartUpload(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPrivate})
}
@ -60,7 +60,7 @@ func TestBucketACLAPE(t *testing.T) {
info := createBucket(hc, bktName)
aclBody := &AccessControlPolicy{}
putBucketACLAssertS3Error(hc, bktName, info.Box, nil, aclBody, s3errors.ErrAccessControlListNotSupported)
putBucketACLAssertS3Error(hc, bktName, info.Box, nil, aclBody, apierr.ErrAccessControlListNotSupported)
aclRes := getBucketACL(hc, bktName)
checkPrivateACL(t, aclRes, info.Key.PublicKey())
@ -113,7 +113,7 @@ func TestBucketPolicy(t *testing.T) {
createTestBucket(hc, bktName)
getBucketPolicy(hc, bktName, s3errors.ErrNoSuchBucketPolicy)
getBucketPolicy(hc, bktName, apierr.ErrNoSuchBucketPolicy)
newPolicy := engineiam.Policy{
Version: "2012-10-17",
@ -125,7 +125,7 @@ func TestBucketPolicy(t *testing.T) {
}},
}
putBucketPolicy(hc, bktName, newPolicy, s3errors.ErrMalformedPolicy)
putBucketPolicy(hc, bktName, newPolicy, apierr.ErrMalformedPolicy)
newPolicy.Statement[0].Resource[0] = arnAwsPrefix + bktName + "/*"
putBucketPolicy(hc, bktName, newPolicy)
@ -140,7 +140,7 @@ func TestBucketPolicyStatus(t *testing.T) {
createTestBucket(hc, bktName)
getBucketPolicy(hc, bktName, s3errors.ErrNoSuchBucketPolicy)
getBucketPolicy(hc, bktName, apierr.ErrNoSuchBucketPolicy)
newPolicy := engineiam.Policy{
Version: "2012-10-17",
@ -152,7 +152,7 @@ func TestBucketPolicyStatus(t *testing.T) {
}},
}
putBucketPolicy(hc, bktName, newPolicy, s3errors.ErrMalformedPolicyNotPrincipal)
putBucketPolicy(hc, bktName, newPolicy, apierr.ErrMalformedPolicyNotPrincipal)
newPolicy.Statement[0].NotPrincipal = nil
newPolicy.Statement[0].Principal = map[engineiam.PrincipalType][]string{engineiam.Wildcard: {}}
@ -191,6 +191,7 @@ func TestDeleteBucketWithPolicy(t *testing.T) {
require.Len(t, hc.h.ape.(*apeMock).policyMap, 1)
require.Len(t, hc.h.ape.(*apeMock).chainMap[engine.ContainerTarget(bi.CID.EncodeToString())], 4)
hc.owner = bi.Owner
deleteBucket(t, hc, bktName, http.StatusNoContent)
require.Empty(t, hc.h.ape.(*apeMock).policyMap)
@ -221,7 +222,7 @@ func TestPutBucketPolicy(t *testing.T) {
assertStatus(hc.t, w, http.StatusOK)
}
func getBucketPolicy(hc *handlerContext, bktName string, errCode ...s3errors.ErrorCode) engineiam.Policy {
func getBucketPolicy(hc *handlerContext, bktName string, errCode ...apierr.ErrorCode) engineiam.Policy {
w, r := prepareTestRequest(hc, bktName, "", nil)
hc.Handler().GetBucketPolicyHandler(w, r)
@ -231,13 +232,13 @@ func getBucketPolicy(hc *handlerContext, bktName string, errCode ...s3errors.Err
err := json.NewDecoder(w.Result().Body).Decode(&policy)
require.NoError(hc.t, err)
} else {
assertS3Error(hc.t, w, s3errors.GetAPIError(errCode[0]))
assertS3Error(hc.t, w, apierr.GetAPIError(errCode[0]))
}
return policy
}
func getBucketPolicyStatus(hc *handlerContext, bktName string, errCode ...s3errors.ErrorCode) PolicyStatus {
func getBucketPolicyStatus(hc *handlerContext, bktName string, errCode ...apierr.ErrorCode) PolicyStatus {
w, r := prepareTestRequest(hc, bktName, "", nil)
hc.Handler().GetBucketPolicyStatusHandler(w, r)
@ -247,13 +248,13 @@ func getBucketPolicyStatus(hc *handlerContext, bktName string, errCode ...s3erro
err := xml.NewDecoder(w.Result().Body).Decode(&policyStatus)
require.NoError(hc.t, err)
} else {
assertS3Error(hc.t, w, s3errors.GetAPIError(errCode[0]))
assertS3Error(hc.t, w, apierr.GetAPIError(errCode[0]))
}
return policyStatus
}
func putBucketPolicy(hc *handlerContext, bktName string, bktPolicy engineiam.Policy, errCode ...s3errors.ErrorCode) {
func putBucketPolicy(hc *handlerContext, bktName string, bktPolicy engineiam.Policy, errCode ...apierr.ErrorCode) {
body, err := json.Marshal(bktPolicy)
require.NoError(hc.t, err)
@ -263,7 +264,7 @@ func putBucketPolicy(hc *handlerContext, bktName string, bktPolicy engineiam.Pol
if len(errCode) == 0 {
assertStatus(hc.t, w, http.StatusOK)
} else {
assertS3Error(hc.t, w, s3errors.GetAPIError(errCode[0]))
assertS3Error(hc.t, w, apierr.GetAPIError(errCode[0]))
}
}
@ -312,9 +313,9 @@ func createBucket(hc *handlerContext, bktName string) *createBucketInfo {
}
}
func createBucketAssertS3Error(hc *handlerContext, bktName string, box *accessbox.Box, code s3errors.ErrorCode) {
func createBucketAssertS3Error(hc *handlerContext, bktName string, box *accessbox.Box, code apierr.ErrorCode) {
w := createBucketBase(hc, bktName, box)
assertS3Error(hc.t, w, s3errors.GetAPIError(code))
assertS3Error(hc.t, w, apierr.GetAPIError(code))
}
func createBucketBase(hc *handlerContext, bktName string, box *accessbox.Box) *httptest.ResponseRecorder {
@ -330,9 +331,9 @@ func putBucketACL(hc *handlerContext, bktName string, box *accessbox.Box, header
assertStatus(hc.t, w, http.StatusOK)
}
func putBucketACLAssertS3Error(hc *handlerContext, bktName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy, code s3errors.ErrorCode) {
func putBucketACLAssertS3Error(hc *handlerContext, bktName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy, code apierr.ErrorCode) {
w := putBucketACLBase(hc, bktName, box, header, body)
assertS3Error(hc.t, w, s3errors.GetAPIError(code))
assertS3Error(hc.t, w, apierr.GetAPIError(code))
}
func putBucketACLBase(hc *handlerContext, bktName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy) *httptest.ResponseRecorder {
@ -360,9 +361,9 @@ func getBucketACLBase(hc *handlerContext, bktName string) *httptest.ResponseReco
return w
}
func putObjectACLAssertS3Error(hc *handlerContext, bktName, objName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy, code s3errors.ErrorCode) {
func putObjectACLAssertS3Error(hc *handlerContext, bktName, objName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy, code apierr.ErrorCode) {
w := putObjectACLBase(hc, bktName, objName, box, header, body)
assertS3Error(hc.t, w, s3errors.GetAPIError(code))
assertS3Error(hc.t, w, apierr.GetAPIError(code))
}
func putObjectACLBase(hc *handlerContext, bktName, objName string, box *accessbox.Box, header map[string]string, body *AccessControlPolicy) *httptest.ResponseRecorder {
@ -396,9 +397,9 @@ func putObjectWithHeaders(hc *handlerContext, bktName, objName string, headers m
return w.Header()
}
func putObjectWithHeadersAssertS3Error(hc *handlerContext, bktName, objName string, headers map[string]string, code s3errors.ErrorCode) {
func putObjectWithHeadersAssertS3Error(hc *handlerContext, bktName, objName string, headers map[string]string, code apierr.ErrorCode) {
w := putObjectWithHeadersBase(hc, bktName, objName, headers, nil, nil)
assertS3Error(hc.t, w, s3errors.GetAPIError(code))
assertS3Error(hc.t, w, apierr.GetAPIError(code))
}
func putObjectWithHeadersBase(hc *handlerContext, bktName, objName string, headers map[string]string, box *accessbox.Box, data []byte) *httptest.ResponseRecorder {

View file

@ -41,7 +41,6 @@ type (
RetryMaxAttempts() int
RetryMaxBackoff() time.Duration
RetryStrategy() RetryStrategy
Domains() []string
}
FrostFSID interface {

View file

@ -70,17 +70,18 @@ var validAttributes = map[string]struct{}{
}
func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
params, err := parseGetObjectAttributeArgs(r)
if err != nil {
h.logAndSendError(w, "invalid request", reqInfo, err)
h.logAndSendError(ctx, w, "invalid request", reqInfo, err)
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
@ -90,44 +91,44 @@ func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Requ
VersionID: params.VersionID,
}
extendedInfo, err := h.obj.GetExtendedObjectInfo(r.Context(), p)
extendedInfo, err := h.obj.GetExtendedObjectInfo(ctx, p)
if err != nil {
h.logAndSendError(w, "could not fetch object info", reqInfo, err)
h.logAndSendError(ctx, w, "could not fetch object info", reqInfo, err)
return
}
info := extendedInfo.ObjectInfo
encryptionParams, err := formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
h.logAndSendError(ctx, w, "invalid sse headers", reqInfo, err)
return
}
if err = encryptionParams.MatchObjectEncryption(layer.FormEncryptionInfo(info.Headers)); err != nil {
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
h.logAndSendError(ctx, w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
return
}
if err = checkPreconditions(info, params.Conditional, h.cfg.MD5Enabled()); err != nil {
h.logAndSendError(w, "precondition failed", reqInfo, err)
h.logAndSendError(ctx, w, "precondition failed", reqInfo, err)
return
}
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
bktSettings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket settings", reqInfo, err)
return
}
response, err := encodeToObjectAttributesResponse(info, params, h.cfg.MD5Enabled())
if err != nil {
h.logAndSendError(w, "couldn't encode object info to response", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't encode object info to response", reqInfo, err)
return
}
writeAttributesHeaders(w.Header(), extendedInfo, bktSettings.Unversioned())
if err = middleware.EncodeToResponse(w, response); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}

View file

@ -65,7 +65,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
srcBucket, srcObject, err := path2BucketObject(src)
if err != nil {
h.logAndSendError(w, "invalid source copy", reqInfo, err)
h.logAndSendError(ctx, w, "invalid source copy", reqInfo, err)
return
}
@ -75,74 +75,74 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
}
if srcObjPrm.BktInfo, err = h.getBucketAndCheckOwner(r, srcBucket, api.AmzSourceExpectedBucketOwner); err != nil {
h.logAndSendError(w, "couldn't get source bucket", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't get source bucket", reqInfo, err)
return
}
dstBktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "couldn't get target bucket", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't get target bucket", reqInfo, err)
return
}
settings, err := h.obj.GetBucketSettings(ctx, dstBktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket settings", reqInfo, err)
return
}
if cannedACLStatus == aclStatusYes {
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
h.logAndSendError(ctx, w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
return
}
extendedSrcObjInfo, err := h.obj.GetExtendedObjectInfo(ctx, srcObjPrm)
if err != nil {
h.logAndSendError(w, "could not find object", reqInfo, err)
h.logAndSendError(ctx, w, "could not find object", reqInfo, err)
return
}
srcObjInfo := extendedSrcObjInfo.ObjectInfo
srcEncryptionParams, err := formCopySourceEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
h.logAndSendError(ctx, w, "invalid sse headers", reqInfo, err)
return
}
dstEncryptionParams, err := formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
h.logAndSendError(ctx, w, "invalid sse headers", reqInfo, err)
return
}
if err = srcEncryptionParams.MatchObjectEncryption(layer.FormEncryptionInfo(srcObjInfo.Headers)); err != nil {
if errors.IsS3Error(err, errors.ErrInvalidEncryptionParameters) || errors.IsS3Error(err, errors.ErrSSEEncryptedObject) ||
errors.IsS3Error(err, errors.ErrInvalidSSECustomerParameters) {
h.logAndSendError(w, "encryption doesn't match object", reqInfo, err, zap.Error(err))
h.logAndSendError(ctx, w, "encryption doesn't match object", reqInfo, err, zap.Error(err))
return
}
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
h.logAndSendError(ctx, w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
return
}
var dstSize uint64
srcSize, err := layer.GetObjectSize(srcObjInfo)
if err != nil {
h.logAndSendError(w, "failed to get source object size", reqInfo, err)
h.logAndSendError(ctx, w, "failed to get source object size", reqInfo, err)
return
} else if srcSize > layer.UploadMaxSize { // https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html
h.logAndSendError(w, "too bid object to copy with single copy operation, use multipart upload copy instead", reqInfo, errors.GetAPIError(errors.ErrInvalidRequestLargeCopy))
h.logAndSendError(ctx, w, "too bid object to copy with single copy operation, use multipart upload copy instead", reqInfo, errors.GetAPIError(errors.ErrInvalidRequestLargeCopy))
return
}
dstSize = srcSize
args, err := parseCopyObjectArgs(r.Header)
if err != nil {
h.logAndSendError(w, "could not parse request params", reqInfo, err)
h.logAndSendError(ctx, w, "could not parse request params", reqInfo, err)
return
}
if isCopyingToItselfForbidden(reqInfo, srcBucket, srcObject, settings, args) {
h.logAndSendError(w, "copying to itself without changing anything", reqInfo, errors.GetAPIError(errors.ErrInvalidCopyDest))
h.logAndSendError(ctx, w, "copying to itself without changing anything", reqInfo, errors.GetAPIError(errors.ErrInvalidCopyDest))
return
}
@ -153,7 +153,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
if args.TaggingDirective == replaceDirective {
tagSet, err = parseTaggingHeader(r.Header)
if err != nil {
h.logAndSendError(w, "could not parse tagging header", reqInfo, err)
h.logAndSendError(ctx, w, "could not parse tagging header", reqInfo, err)
return
}
} else {
@ -168,13 +168,13 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
_, tagSet, err = h.obj.GetObjectTagging(ctx, tagPrm)
if err != nil {
h.logAndSendError(w, "could not get object tagging", reqInfo, err)
h.logAndSendError(ctx, w, "could not get object tagging", reqInfo, err)
return
}
}
if err = checkPreconditions(srcObjInfo, args.Conditional, h.cfg.MD5Enabled()); err != nil {
h.logAndSendError(w, "precondition failed", reqInfo, errors.GetAPIError(errors.ErrPreconditionFailed))
h.logAndSendError(ctx, w, "precondition failed", reqInfo, errors.GetAPIError(errors.ErrPreconditionFailed))
return
}
@ -202,20 +202,20 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
params.CopiesNumbers, err = h.pickCopiesNumbers(metadata, reqInfo.Namespace, dstBktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err)
return
}
params.Lock, err = formObjectLock(ctx, dstBktInfo, settings.LockConfiguration, r.Header)
if err != nil {
h.logAndSendError(w, "could not form object lock", reqInfo, err)
h.logAndSendError(ctx, w, "could not form object lock", reqInfo, err)
return
}
additional := []zap.Field{zap.String("src_bucket_name", srcBucket), zap.String("src_object_name", srcObject)}
extendedDstObjInfo, err := h.obj.CopyObject(ctx, params)
if err != nil {
h.logAndSendError(w, "couldn't copy object", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "couldn't copy object", reqInfo, err, additional...)
return
}
dstObjInfo := extendedDstObjInfo.ObjectInfo
@ -224,7 +224,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
LastModified: dstObjInfo.Created.UTC().Format(time.RFC3339),
ETag: data.Quote(dstObjInfo.ETag(h.cfg.MD5Enabled())),
}); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err, additional...)
return
}
@ -239,7 +239,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
NodeVersion: extendedDstObjInfo.NodeVersion,
}
if err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
h.logAndSendError(ctx, w, "could not upload object tagging", reqInfo, err)
return
}
}

View file

@ -20,32 +20,34 @@ const (
)
func (h *handler) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
cors, err := h.obj.GetBucketCORS(r.Context(), bktInfo)
cors, err := h.obj.GetBucketCORS(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "could not get cors", reqInfo, err)
h.logAndSendError(ctx, w, "could not get cors", reqInfo, err)
return
}
if err = middleware.EncodeToResponse(w, cors); err != nil {
h.logAndSendError(w, "could not encode cors to response", reqInfo, err)
h.logAndSendError(ctx, w, "could not encode cors to response", reqInfo, err)
return
}
}
func (h *handler) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
@ -57,32 +59,33 @@ func (h *handler) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err)
return
}
if err = h.obj.PutBucketCORS(r.Context(), p); err != nil {
h.logAndSendError(w, "could not put cors configuration", reqInfo, err)
if err = h.obj.PutBucketCORS(ctx, p); err != nil {
h.logAndSendError(ctx, w, "could not put cors configuration", reqInfo, err)
return
}
if err = middleware.WriteSuccessResponseHeadersOnly(w); err != nil {
h.logAndSendError(w, "write response", reqInfo, err)
h.logAndSendError(ctx, w, "write response", reqInfo, err)
return
}
}
func (h *handler) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if err = h.obj.DeleteBucketCORS(r.Context(), bktInfo); err != nil {
h.logAndSendError(w, "could not delete cors", reqInfo, err)
if err = h.obj.DeleteBucketCORS(ctx, bktInfo); err != nil {
h.logAndSendError(ctx, w, "could not delete cors", reqInfo, err)
}
w.WriteHeader(http.StatusNoContent)
@ -102,7 +105,7 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
if reqInfo.BucketName == "" {
return
}
bktInfo, err := h.obj.GetBucketInfo(ctx, reqInfo.BucketName)
bktInfo, err := h.getBucketInfo(ctx, reqInfo.BucketName)
if err != nil {
h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err))
return
@ -149,21 +152,22 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
}
func (h *handler) Preflight(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName)
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketInfo(ctx, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
origin := r.Header.Get(api.Origin)
if origin == "" {
h.logAndSendError(w, "origin request header needed", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
h.logAndSendError(ctx, w, "origin request header needed", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
}
method := r.Header.Get(api.AccessControlRequestMethod)
if method == "" {
h.logAndSendError(w, "Access-Control-Request-Method request header needed", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
h.logAndSendError(ctx, w, "Access-Control-Request-Method request header needed", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
return
}
@ -173,9 +177,9 @@ func (h *handler) Preflight(w http.ResponseWriter, r *http.Request) {
headers = strings.Split(requestHeaders, ", ")
}
cors, err := h.obj.GetBucketCORS(r.Context(), bktInfo)
cors, err := h.obj.GetBucketCORS(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "could not get cors", reqInfo, err)
h.logAndSendError(ctx, w, "could not get cors", reqInfo, err)
return
}
@ -204,7 +208,7 @@ func (h *handler) Preflight(w http.ResponseWriter, r *http.Request) {
w.Header().Set(api.AccessControlAllowCredentials, "true")
}
if err = middleware.WriteSuccessResponseHeadersOnly(w); err != nil {
h.logAndSendError(w, "write response", reqInfo, err)
h.logAndSendError(ctx, w, "write response", reqInfo, err)
return
}
return
@ -213,7 +217,7 @@ func (h *handler) Preflight(w http.ResponseWriter, r *http.Request) {
}
}
}
h.logAndSendError(w, "Forbidden", reqInfo, errors.GetAPIError(errors.ErrAccessDenied))
h.logAndSendError(ctx, w, "Forbidden", reqInfo, errors.GetAPIError(errors.ErrAccessDenied))
}
func checkSubslice(slice []string, subSlice []string) bool {

View file

@ -71,28 +71,35 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
bktSettings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket settings", reqInfo, err)
return
}
networkInfo, err := h.obj.GetNetworkInfo(ctx)
if err != nil {
h.logAndSendError(ctx, w, "could not get network info", reqInfo, err)
return
}
p := &layer.DeleteObjectParams{
BktInfo: bktInfo,
Objects: versionedObject,
Settings: bktSettings,
BktInfo: bktInfo,
Objects: versionedObject,
Settings: bktSettings,
NetworkInfo: networkInfo,
}
deletedObjects := h.obj.DeleteObjects(ctx, p)
deletedObject := deletedObjects[0]
if deletedObject.Error != nil {
if isErrObjectLocked(deletedObject.Error) {
h.logAndSendError(w, "object is locked", reqInfo, errors.GetAPIError(errors.ErrAccessDenied))
h.logAndSendError(ctx, w, "object is locked", reqInfo, errors.GetAPIError(errors.ErrAccessDenied))
} else {
h.logAndSendError(w, "could not delete object", reqInfo, deletedObject.Error)
h.logAndSendError(ctx, w, "could not delete object", reqInfo, deletedObject.Error)
}
return
}
@ -127,26 +134,26 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
// Content-Md5 is required and should be set
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
if _, ok := r.Header[api.ContentMD5]; !ok {
h.logAndSendError(w, "missing Content-MD5", reqInfo, errors.GetAPIError(errors.ErrMissingContentMD5))
h.logAndSendError(ctx, w, "missing Content-MD5", reqInfo, errors.GetAPIError(errors.ErrMissingContentMD5))
return
}
// Content-Length is required and should be non-zero
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
if r.ContentLength <= 0 {
h.logAndSendError(w, "missing Content-Length", reqInfo, errors.GetAPIError(errors.ErrMissingContentLength))
h.logAndSendError(ctx, w, "missing Content-Length", reqInfo, errors.GetAPIError(errors.ErrMissingContentLength))
return
}
// Unmarshal list of keys to be deleted.
requested := &DeleteObjectsRequest{}
if err := h.cfg.NewXMLDecoder(r.Body).Decode(requested); err != nil {
h.logAndSendError(w, "couldn't decode body", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()))
h.logAndSendError(ctx, w, "couldn't decode body", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()))
return
}
if len(requested.Objects) == 0 || len(requested.Objects) > maxObjectsToDelete {
h.logAndSendError(w, "number of objects to delete must be greater than 0 and less or equal to 1000", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
h.logAndSendError(ctx, w, "number of objects to delete must be greater than 0 and less or equal to 1000", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
return
}
@ -171,21 +178,28 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
bktSettings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket settings", reqInfo, err)
return
}
networkInfo, err := h.obj.GetNetworkInfo(ctx)
if err != nil {
h.logAndSendError(ctx, w, "could not get network info", reqInfo, err)
return
}
p := &layer.DeleteObjectParams{
BktInfo: bktInfo,
Objects: toRemove,
Settings: bktSettings,
IsMultiple: true,
BktInfo: bktInfo,
Objects: toRemove,
Settings: bktSettings,
NetworkInfo: networkInfo,
IsMultiple: true,
}
deletedObjects := h.obj.DeleteObjects(ctx, p)
@ -217,22 +231,28 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
}
if err = middleware.EncodeToResponse(w, response); err != nil {
h.logAndSendError(w, "could not write response", reqInfo, err)
h.logAndSendError(ctx, w, "could not write response", reqInfo, err)
return
}
}
func (h *handler) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if err = checkOwner(bktInfo, reqInfo.User); err != nil {
h.logAndSendError(ctx, w, "request owner id does not match bucket owner id", reqInfo, err)
return
}
var sessionToken *session.Container
boxData, err := middleware.GetBoxData(r.Context())
boxData, err := middleware.GetBoxData(ctx)
if err == nil {
sessionToken = boxData.Gate.SessionTokenForDelete()
}
@ -245,12 +265,12 @@ func (h *handler) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
}
}
if err = h.obj.DeleteBucket(r.Context(), &layer.DeleteBucketParams{
if err = h.obj.DeleteBucket(ctx, &layer.DeleteBucketParams{
BktInfo: bktInfo,
SessionToken: sessionToken,
SkipCheck: skipObjCheck,
}); err != nil {
h.logAndSendError(w, "couldn't delete bucket", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't delete bucket", reqInfo, err)
return
}
@ -261,7 +281,7 @@ func (h *handler) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
getBucketCannedChainID(chain.Ingress, bktInfo.CID),
}
if err = h.ape.DeleteBucketPolicy(reqInfo.Namespace, bktInfo.CID, chainIDs); err != nil {
h.logAndSendError(w, "failed to delete policy from storage", reqInfo, err)
h.logAndSendError(ctx, w, "failed to delete policy from storage", reqInfo, err)
return
}

View file

@ -10,7 +10,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -37,6 +37,7 @@ func TestDeleteBucketOnAlreadyRemovedError(t *testing.T) {
deleteObjects(t, hc, bktName, [][2]string{{objName, emptyVersion}})
hc.owner = bktInfo.Owner
deleteBucket(t, hc, bktName, http.StatusNoContent)
}
@ -53,11 +54,12 @@ func TestDeleteBucket(t *testing.T) {
tc := prepareHandlerContext(t)
bktName, objName := "bucket-for-removal", "object-to-delete"
_, objInfo := createVersionedBucketAndObject(t, tc, bktName, objName)
bktInfo, objInfo := createVersionedBucketAndObject(t, tc, bktName, objName)
deleteMarkerVersion, isDeleteMarker := deleteObject(t, tc, bktName, objName, emptyVersion)
require.True(t, isDeleteMarker)
tc.owner = bktInfo.Owner
deleteBucket(t, tc, bktName, http.StatusConflict)
deleteObject(t, tc, bktName, objName, objInfo.VersionID())
deleteBucket(t, tc, bktName, http.StatusConflict)
@ -82,6 +84,7 @@ func TestDeleteBucketOnNotFoundError(t *testing.T) {
deleteObjects(t, hc, bktName, [][2]string{{objName, emptyVersion}})
hc.owner = bktInfo.Owner
deleteBucket(t, hc, bktName, http.StatusNoContent)
}
@ -99,6 +102,7 @@ func TestForceDeleteBucket(t *testing.T) {
addr.SetContainer(bktInfo.CID)
addr.SetObject(nodeVersion.OID)
hc.owner = bktInfo.Owner
deleteBucketForce(t, hc, bktName, http.StatusConflict, "false")
deleteBucketForce(t, hc, bktName, http.StatusNoContent, "true")
}
@ -131,7 +135,7 @@ func TestDeleteObjectsError(t *testing.T) {
addr.SetContainer(bktInfo.CID)
addr.SetObject(nodeVersion.OID)
expectedError := apiErrors.GetAPIError(apiErrors.ErrAccessDenied)
expectedError := apierr.GetAPIError(apierr.ErrAccessDenied)
hc.tp.SetObjectError(addr, expectedError)
w := deleteObjectsBase(hc, bktName, [][2]string{{objName, nodeVersion.OID.EncodeToString()}})
@ -457,6 +461,17 @@ func TestDeleteObjectCheckMarkerReturn(t *testing.T) {
require.Equal(t, deleteMarkerVersion, deleteMarkerVersion2)
}
func TestDeleteBucketByNotOwner(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-name"
bktInfo := createTestBucket(hc, bktName)
deleteBucket(t, hc, bktName, http.StatusForbidden)
hc.owner = bktInfo.Owner
deleteBucket(t, hc, bktName, http.StatusNoContent)
}
func createBucketAndObject(tc *handlerContext, bktName, objName string) (*data.BucketInfo, *data.ObjectInfo) {
bktInfo := createTestBucket(tc, bktName)
@ -553,9 +568,9 @@ func checkNotFound(t *testing.T, hc *handlerContext, bktName, objName, version s
assertStatus(t, w, http.StatusNotFound)
}
func headObjectAssertS3Error(hc *handlerContext, bktName, objName, version string, code apiErrors.ErrorCode) {
func headObjectAssertS3Error(hc *handlerContext, bktName, objName, version string, code apierr.ErrorCode) {
w := headObjectBase(hc, bktName, objName, version)
assertS3Error(hc.t, w, apiErrors.GetAPIError(code))
assertS3Error(hc.t, w, apierr.GetAPIError(code))
}
func checkFound(t *testing.T, hc *handlerContext, bktName, objName, version string) {
@ -563,6 +578,18 @@ func checkFound(t *testing.T, hc *handlerContext, bktName, objName, version stri
assertStatus(t, w, http.StatusOK)
}
func headObjectWithHeaders(hc *handlerContext, bktName, objName, version string, headers map[string]string) *httptest.ResponseRecorder {
query := make(url.Values)
query.Add(api.QueryVersionID, version)
w, r := prepareTestFullRequest(hc, bktName, objName, query, nil)
for k, v := range headers {
r.Header.Set(k, v)
}
hc.Handler().HeadObjectHandler(w, r)
return w
}
func headObjectBase(hc *handlerContext, bktName, objName, version string) *httptest.ResponseRecorder {
query := make(url.Values)
query.Add(api.QueryVersionID, version)

View file

@ -16,6 +16,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"github.com/stretchr/testify/require"
)
@ -37,7 +38,7 @@ func TestSimpleGetEncrypted(t *testing.T) {
objInfo, err := tc.Layer().GetObjectInfo(tc.Context(), &layer.HeadObjectParams{BktInfo: bktInfo, Object: objName})
require.NoError(t, err)
obj, err := tc.MockedPool().GetObject(tc.Context(), layer.PrmObjectGet{Container: bktInfo.CID, Object: objInfo.ID})
obj, err := tc.MockedPool().GetObject(tc.Context(), frostfs.PrmObjectGet{Container: bktInfo.CID, Object: objInfo.ID})
require.NoError(t, err)
encryptedContent, err := io.ReadAll(obj.Payload)
require.NoError(t, err)
@ -47,6 +48,25 @@ func TestSimpleGetEncrypted(t *testing.T) {
require.Equal(t, content, string(response))
}
func TestMD5HeaderBadOrEmpty(t *testing.T) {
tc := prepareHandlerContext(t)
bktName, objName := "bucket-for-sse-c", "object-to-encrypt"
createTestBucket(tc, bktName)
content := "content"
headers := map[string]string{
api.ContentMD5: "",
}
putEncryptedObjectWithHeadersErr(t, tc, bktName, objName, content, headers, errors.ErrInvalidDigest)
headers = map[string]string{
api.ContentMD5: "YWJjMTIzIT8kKiYoKSctPUB+",
}
putEncryptedObjectWithHeadersErr(t, tc, bktName, objName, content, headers, errors.ErrBadDigest)
}
func TestGetEncryptedRange(t *testing.T) {
tc := prepareHandlerContext(t)
@ -288,6 +308,21 @@ func completeMultipartUploadBase(hc *handlerContext, bktName, objName, uploadID
return w
}
func abortMultipartUpload(hc *handlerContext, bktName, objName, uploadID string) {
w := abortMultipartUploadBase(hc, bktName, objName, uploadID)
assertStatus(hc.t, w, http.StatusNoContent)
}
func abortMultipartUploadBase(hc *handlerContext, bktName, objName, uploadID string) *httptest.ResponseRecorder {
query := make(url.Values)
query.Set(uploadIDQuery, uploadID)
w, r := prepareTestFullRequest(hc, bktName, objName, query, nil)
hc.Handler().AbortMultipartUploadHandler(w, r)
return w
}
func uploadPartEncrypted(hc *handlerContext, bktName, objName, uploadID string, num, size int) (string, []byte) {
return uploadPartBase(hc, bktName, objName, true, uploadID, num, size)
}
@ -344,6 +379,15 @@ func putEncryptedObject(t *testing.T, tc *handlerContext, bktName, objName, cont
assertStatus(t, w, http.StatusOK)
}
func putEncryptedObjectWithHeadersErr(t *testing.T, tc *handlerContext, bktName, objName, content string, headers map[string]string, code errors.ErrorCode) {
body := bytes.NewReader([]byte(content))
w, r := prepareTestPayloadRequest(tc, bktName, objName, body)
setHeaders(r, headers)
tc.Handler().PutObjectHandler(w, r)
assertS3Error(t, w, errors.GetAPIError(code))
}
func getEncryptedObject(hc *handlerContext, bktName, objName string) ([]byte, http.Header) {
w, r := prepareTestRequest(hc, bktName, objName, nil)
setEncryptHeaders(r)
@ -355,6 +399,15 @@ func getObject(hc *handlerContext, bktName, objName string) ([]byte, http.Header
return getObjectBase(hc, w, r)
}
func getObjectWithHeaders(hc *handlerContext, bktName, objName string, headers map[string]string) *httptest.ResponseRecorder {
w, r := prepareTestRequest(hc, bktName, objName, nil)
for k, v := range headers {
r.Header.Set(k, v)
}
hc.Handler().GetObjectHandler(w, r)
return w
}
func getObjectBase(hc *handlerContext, w *httptest.ResponseRecorder, r *http.Request) ([]byte, http.Header) {
hc.Handler().GetObjectHandler(w, r)
assertStatus(hc.t, w, http.StatusOK)

View file

@ -78,6 +78,27 @@ func addSSECHeaders(responseHeader http.Header, requestHeader http.Header) {
responseHeader.Set(api.AmzServerSideEncryptionCustomerKeyMD5, requestHeader.Get(api.AmzServerSideEncryptionCustomerKeyMD5))
}
func writeNotModifiedHeaders(h http.Header, extendedInfo *data.ExtendedObjectInfo, tagSetLength int, isBucketUnversioned, md5Enabled bool) {
h.Set(api.ETag, data.Quote(extendedInfo.ObjectInfo.ETag(md5Enabled)))
h.Set(api.LastModified, extendedInfo.ObjectInfo.Created.UTC().Format(http.TimeFormat))
h.Set(api.AmzTaggingCount, strconv.Itoa(tagSetLength))
if !isBucketUnversioned {
h.Set(api.AmzVersionID, extendedInfo.Version())
}
if cacheControl := extendedInfo.ObjectInfo.Headers[api.CacheControl]; cacheControl != "" {
h.Set(api.CacheControl, cacheControl)
}
for key, val := range extendedInfo.ObjectInfo.Headers {
if layer.IsSystemHeader(key) {
continue
}
h[api.MetadataPrefix+key] = []string{val}
}
}
func writeHeaders(h http.Header, requestHeader http.Header, extendedInfo *data.ExtendedObjectInfo, tagSetLength int,
isBucketUnversioned, md5Enabled bool) {
info := extendedInfo.ObjectInfo
@ -129,18 +150,19 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
var (
params *layer.RangeParams
reqInfo = middleware.GetReqInfo(r.Context())
ctx = r.Context()
reqInfo = middleware.GetReqInfo(ctx)
)
conditional, err := parseConditionalHeaders(r.Header)
if err != nil {
h.logAndSendError(w, "could not parse request params", reqInfo, err)
h.logAndSendError(ctx, w, "could not parse request params", reqInfo, err)
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
@ -150,37 +172,16 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
extendedInfo, err := h.obj.GetExtendedObjectInfo(r.Context(), p)
extendedInfo, err := h.obj.GetExtendedObjectInfo(ctx, p)
if err != nil {
h.logAndSendError(w, "could not find object", reqInfo, err)
h.logAndSendError(ctx, w, "could not find object", reqInfo, err)
return
}
info := extendedInfo.ObjectInfo
if err = checkPreconditions(info, conditional, h.cfg.MD5Enabled()); err != nil {
h.logAndSendError(w, "precondition failed", reqInfo, err)
return
}
encryptionParams, err := formEncryptionParams(r)
bktSettings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
}
if err = encryptionParams.MatchObjectEncryption(layer.FormEncryptionInfo(info.Headers)); err != nil {
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
return
}
fullSize, err := layer.GetObjectSize(info)
if err != nil {
h.logAndSendError(w, "invalid size header", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
return
}
if params, err = fetchRangeHeader(r.Header, fullSize); err != nil {
h.logAndSendError(w, "could not parse range header", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket settings", reqInfo, err)
return
}
@ -190,24 +191,48 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
VersionID: info.VersionID(),
}
tagSet, lockInfo, err := h.obj.GetObjectTaggingAndLock(r.Context(), t, extendedInfo.NodeVersion)
tagSet, lockInfo, err := h.obj.GetObjectTaggingAndLock(ctx, t, extendedInfo.NodeVersion)
if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) {
h.logAndSendError(w, "could not get object meta data", reqInfo, err)
h.logAndSendError(ctx, w, "could not get object meta data", reqInfo, err)
return
}
if layer.IsAuthenticatedRequest(r.Context()) {
if err = checkPreconditions(info, conditional, h.cfg.MD5Enabled()); err != nil {
if errors.IsS3Error(err, errors.ErrNotModified) {
writeNotModifiedHeaders(w.Header(), extendedInfo, len(tagSet), bktSettings.Unversioned(), h.cfg.MD5Enabled())
}
h.logAndSendError(ctx, w, "precondition failed", reqInfo, err)
return
}
encryptionParams, err := formEncryptionParams(r)
if err != nil {
h.logAndSendError(ctx, w, "invalid sse headers", reqInfo, err)
return
}
if err = encryptionParams.MatchObjectEncryption(layer.FormEncryptionInfo(info.Headers)); err != nil {
h.logAndSendError(ctx, w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
return
}
fullSize, err := layer.GetObjectSize(info)
if err != nil {
h.logAndSendError(ctx, w, "invalid size header", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
return
}
if params, err = fetchRangeHeader(r.Header, fullSize); err != nil {
h.logAndSendError(ctx, w, "could not parse range header", reqInfo, err)
return
}
if layer.IsAuthenticatedRequest(ctx) {
overrideResponseHeaders(w.Header(), reqInfo.URL.Query())
}
if err = h.setLockingHeaders(bktInfo, lockInfo, w.Header()); err != nil {
h.logAndSendError(w, "could not get locking info", reqInfo, err)
return
}
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "could not get locking info", reqInfo, err)
return
}
@ -219,9 +244,9 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
Encryption: encryptionParams,
}
objPayload, err := h.obj.GetObject(r.Context(), getPayloadParams)
objPayload, err := h.obj.GetObject(ctx, getPayloadParams)
if err != nil {
h.logAndSendError(w, "could not get object payload", reqInfo, err)
h.logAndSendError(ctx, w, "could not get object payload", reqInfo, err)
return
}
@ -233,7 +258,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
}
if err = objPayload.StreamTo(w); err != nil {
h.logAndSendError(w, "could not stream object payload", reqInfo, err)
h.logAndSendError(ctx, w, "could not stream object payload", reqInfo, err)
return
}
}

View file

@ -2,7 +2,7 @@ package handler
import (
"bytes"
stderrors "errors"
"errors"
"fmt"
"io"
"net/http"
@ -13,7 +13,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
"github.com/stretchr/testify/require"
@ -89,7 +89,7 @@ func TestPreconditions(t *testing.T) {
name: "IfMatch false",
info: newInfo(etag, today),
args: &conditionalArgs{IfMatch: etag2},
expected: errors.GetAPIError(errors.ErrPreconditionFailed)},
expected: apierr.GetAPIError(apierr.ErrPreconditionFailed)},
{
name: "IfNoneMatch true",
info: newInfo(etag, today),
@ -99,7 +99,7 @@ func TestPreconditions(t *testing.T) {
name: "IfNoneMatch false",
info: newInfo(etag, today),
args: &conditionalArgs{IfNoneMatch: etag},
expected: errors.GetAPIError(errors.ErrNotModified)},
expected: apierr.GetAPIError(apierr.ErrNotModified)},
{
name: "IfModifiedSince true",
info: newInfo(etag, today),
@ -109,7 +109,7 @@ func TestPreconditions(t *testing.T) {
name: "IfModifiedSince false",
info: newInfo(etag, yesterday),
args: &conditionalArgs{IfModifiedSince: &today},
expected: errors.GetAPIError(errors.ErrNotModified)},
expected: apierr.GetAPIError(apierr.ErrNotModified)},
{
name: "IfUnmodifiedSince true",
info: newInfo(etag, yesterday),
@ -119,7 +119,7 @@ func TestPreconditions(t *testing.T) {
name: "IfUnmodifiedSince false",
info: newInfo(etag, today),
args: &conditionalArgs{IfUnmodifiedSince: &yesterday},
expected: errors.GetAPIError(errors.ErrPreconditionFailed)},
expected: apierr.GetAPIError(apierr.ErrPreconditionFailed)},
{
name: "IfMatch true, IfUnmodifiedSince false",
@ -131,19 +131,19 @@ func TestPreconditions(t *testing.T) {
name: "IfMatch false, IfUnmodifiedSince true",
info: newInfo(etag, yesterday),
args: &conditionalArgs{IfMatch: etag2, IfUnmodifiedSince: &today},
expected: errors.GetAPIError(errors.ErrPreconditionFailed),
expected: apierr.GetAPIError(apierr.ErrPreconditionFailed),
},
{
name: "IfNoneMatch false, IfModifiedSince true",
info: newInfo(etag, today),
args: &conditionalArgs{IfNoneMatch: etag, IfModifiedSince: &yesterday},
expected: errors.GetAPIError(errors.ErrNotModified),
expected: apierr.GetAPIError(apierr.ErrNotModified),
},
{
name: "IfNoneMatch true, IfModifiedSince false",
info: newInfo(etag, yesterday),
args: &conditionalArgs{IfNoneMatch: etag2, IfModifiedSince: &today},
expected: errors.GetAPIError(errors.ErrNotModified),
expected: apierr.GetAPIError(apierr.ErrNotModified),
},
} {
t.Run(tc.name, func(t *testing.T) {
@ -151,7 +151,7 @@ func TestPreconditions(t *testing.T) {
if tc.expected == nil {
require.NoError(t, actual)
} else {
require.True(t, stderrors.Is(actual, tc.expected), tc.expected, actual)
require.True(t, errors.Is(actual, tc.expected), tc.expected, actual)
}
})
}
@ -193,8 +193,8 @@ func TestGetObject(t *testing.T) {
hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectNotFound{})
getObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), errors.ErrNoSuchVersion)
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, errors.ErrNoSuchKey)
getObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), apierr.ErrNoSuchVersion)
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, apierr.ErrNoSuchKey)
}
func TestGetObjectEnabledMD5(t *testing.T) {
@ -210,6 +210,27 @@ func TestGetObjectEnabledMD5(t *testing.T) {
require.Equal(t, data.Quote(objInfo.MD5Sum), headers.Get(api.ETag))
}
func TestGetObjectNotModifiedHeaders(t *testing.T) {
hc := prepareHandlerContextWithMinCache(t)
bktName, objName, metadataHeader := "bucket", "obj", api.MetadataPrefix+"header"
createVersionedBucket(hc, bktName)
header := putObjectWithHeaders(hc, bktName, objName, map[string]string{api.CacheControl: "value", metadataHeader: "value"})
etag, versionID := header.Get(api.ETag), header.Get(api.AmzVersionID)
require.NotEmpty(t, etag)
require.NotEmpty(t, versionID)
putObjectTagging(t, hc, bktName, objName, map[string]string{"key": "value"})
w := getObjectWithHeaders(hc, bktName, objName, map[string]string{api.IfNoneMatch: etag})
require.Equal(t, http.StatusNotModified, w.Code)
require.Equal(t, "1", w.Header().Get(api.AmzTaggingCount))
require.Equal(t, etag, w.Header().Get(api.ETag))
require.NotEmpty(t, w.Header().Get(api.LastModified))
require.Equal(t, versionID, w.Header().Get(api.AmzVersionID))
require.Equal(t, "value", w.Header().Get(api.CacheControl))
require.Equal(t, []string{"value"}, w.Header()[metadataHeader])
}
func putObjectContent(hc *handlerContext, bktName, objName, content string) http.Header {
body := bytes.NewReader([]byte(content))
w, r := prepareTestPayloadRequest(hc, bktName, objName, body)
@ -228,9 +249,17 @@ func getObjectRange(t *testing.T, tc *handlerContext, bktName, objName string, s
return content
}
func getObjectAssertS3Error(hc *handlerContext, bktName, objName, version string, code errors.ErrorCode) {
func getObjectVersion(tc *handlerContext, bktName, objName, version string) []byte {
w := getObjectBaseResponse(tc, bktName, objName, version)
assertStatus(tc.t, w, http.StatusOK)
content, err := io.ReadAll(w.Result().Body)
require.NoError(tc.t, err)
return content
}
func getObjectAssertS3Error(hc *handlerContext, bktName, objName, version string, code apierr.ErrorCode) {
w := getObjectBaseResponse(hc, bktName, objName, version)
assertS3Error(hc.t, w, errors.GetAPIError(code))
assertS3Error(hc.t, w, apierr.GetAPIError(code))
}
func getObjectBaseResponse(hc *handlerContext, bktName, objName, version string) *httptest.ResponseRecorder {

File diff suppressed because it is too large Load diff

View file

@ -20,6 +20,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
@ -37,8 +38,12 @@ import (
)
type handlerContext struct {
*handlerContextBase
t *testing.T
}
type handlerContextBase struct {
owner user.ID
t *testing.T
h *handler
tp *layer.TestFrostFS
tree *tree.Tree
@ -50,19 +55,19 @@ type handlerContext struct {
cache *layer.Cache
}
func (hc *handlerContext) Handler() *handler {
func (hc *handlerContextBase) Handler() *handler {
return hc.h
}
func (hc *handlerContext) MockedPool() *layer.TestFrostFS {
func (hc *handlerContextBase) MockedPool() *layer.TestFrostFS {
return hc.tp
}
func (hc *handlerContext) Layer() *layer.Layer {
func (hc *handlerContextBase) Layer() *layer.Layer {
return hc.h.obj
}
func (hc *handlerContext) Context() context.Context {
func (hc *handlerContextBase) Context() context.Context {
return hc.context
}
@ -72,7 +77,6 @@ type configMock struct {
defaultCopiesNumbers []uint32
bypassContentEncodingInChunks bool
md5Enabled bool
domains []string
}
func (c *configMock) DefaultPlacementPolicy(_ string) netmap.PlacementPolicy {
@ -136,27 +140,35 @@ func (c *configMock) RetryStrategy() RetryStrategy {
return RetryStrategyConstant
}
func (c *configMock) Domains() []string {
return c.domains
}
func prepareHandlerContext(t *testing.T) *handlerContext {
return prepareHandlerContextBase(t, layer.DefaultCachesConfigs(zap.NewExample()))
hc, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(zap.NewExample()))
require.NoError(t, err)
return &handlerContext{
handlerContextBase: hc,
t: t,
}
}
func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext {
return prepareHandlerContextBase(t, getMinCacheConfig(zap.NewExample()))
hc, err := prepareHandlerContextBase(getMinCacheConfig(zap.NewExample()))
require.NoError(t, err)
return &handlerContext{
handlerContextBase: hc,
t: t,
}
}
func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *handlerContext {
func prepareHandlerContextBase(cacheCfg *layer.CachesConfig) (*handlerContextBase, error) {
key, err := keys.NewPrivateKey()
require.NoError(t, err)
if err != nil {
return nil, err
}
l := zap.NewExample()
log := zap.NewExample()
tp := layer.NewTestFrostFS(key)
testResolver := &resolver.Resolver{Name: "test_resolver"}
testResolver.SetResolveFunc(func(_ context.Context, name string) (cid.ID, error) {
testResolver.SetResolveFunc(func(_ context.Context, _, name string) (cid.ID, error) {
return tp.ContainerID(name)
})
@ -164,7 +176,9 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *hand
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
memCli, err := tree.NewTreeServiceClientMemory()
require.NoError(t, err)
if err != nil {
return nil, err
}
treeMock := tree.NewTree(memCli, zap.NewExample())
@ -181,32 +195,38 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *hand
var pp netmap.PlacementPolicy
err = pp.DecodeString("REP 1")
require.NoError(t, err)
if err != nil {
return nil, err
}
cfg := &configMock{
defaultPolicy: pp,
}
h := &handler{
log: l,
obj: layer.NewLayer(l, tp, layerCfg),
log: log,
obj: layer.NewLayer(log, tp, layerCfg),
cfg: cfg,
ape: newAPEMock(),
frostfsid: newFrostfsIDMock(),
}
return &handlerContext{
accessBox, err := newTestAccessBox(key)
if err != nil {
return nil, err
}
return &handlerContextBase{
owner: owner,
t: t,
h: h,
tp: tp,
tree: treeMock,
context: middleware.SetBox(context.Background(), &middleware.Box{AccessBox: newTestAccessBox(t, key)}),
context: middleware.SetBox(context.Background(), &middleware.Box{AccessBox: accessBox}),
config: cfg,
layerFeatures: features,
treeMock: memCli,
cache: layerCfg.Cache,
}
}, nil
}
func getMinCacheConfig(logger *zap.Logger) *layer.CachesConfig {
@ -224,12 +244,14 @@ func getMinCacheConfig(logger *zap.Logger) *layer.CachesConfig {
Buckets: minCacheCfg,
System: minCacheCfg,
AccessControl: minCacheCfg,
NetworkInfo: &cache.NetworkInfoCacheConfig{Lifetime: minCacheCfg.Lifetime},
}
}
type apeMock struct {
chainMap map[engine.Target][]*chain.Chain
policyMap map[string][]byte
err error
}
func newAPEMock() *apeMock {
@ -273,6 +295,10 @@ func (a *apeMock) DeletePolicy(namespace string, cnrID cid.ID) error {
}
func (a *apeMock) PutBucketPolicy(ns string, cnrID cid.ID, policy []byte, chain []*chain.Chain) error {
if a.err != nil {
return a.err
}
if err := a.PutPolicy(ns, cnrID, policy); err != nil {
return err
}
@ -287,6 +313,10 @@ func (a *apeMock) PutBucketPolicy(ns string, cnrID cid.ID, policy []byte, chain
}
func (a *apeMock) DeleteBucketPolicy(ns string, cnrID cid.ID, chainIDs []chain.ID) error {
if a.err != nil {
return a.err
}
if err := a.DeletePolicy(ns, cnrID); err != nil {
return err
}
@ -300,6 +330,10 @@ func (a *apeMock) DeleteBucketPolicy(ns string, cnrID cid.ID, chainIDs []chain.I
}
func (a *apeMock) GetBucketPolicy(ns string, cnrID cid.ID) ([]byte, error) {
if a.err != nil {
return nil, a.err
}
policy, ok := a.policyMap[ns+cnrID.EncodeToString()]
if !ok {
return nil, errors.New("not found")
@ -309,6 +343,10 @@ func (a *apeMock) GetBucketPolicy(ns string, cnrID cid.ID) ([]byte, error) {
}
func (a *apeMock) SaveACLChains(cid string, chains []*chain.Chain) error {
if a.err != nil {
return a.err
}
for i := range chains {
if err := a.AddChain(engine.ContainerTarget(cid), chains[i]); err != nil {
return err
@ -350,7 +388,7 @@ func createTestBucket(hc *handlerContext, bktName string) *data.BucketInfo {
}
func createTestBucketWithLock(hc *handlerContext, bktName string, conf *data.ObjectLockConfiguration) *data.BucketInfo {
res, err := hc.MockedPool().CreateContainer(hc.Context(), layer.PrmContainerCreate{
res, err := hc.MockedPool().CreateContainer(hc.Context(), frostfs.PrmContainerCreate{
Creator: hc.owner,
Name: bktName,
AdditionalAttributes: [][2]string{{layer.AttributeLockEnabled, "true"}},
@ -397,7 +435,7 @@ func createTestObject(hc *handlerContext, bktInfo *data.BucketInfo, objName stri
extObjInfo, err := hc.Layer().PutObject(hc.Context(), &layer.PutObjectParams{
BktInfo: bktInfo,
Object: objName,
Size: uint64(len(content)),
Size: ptr(uint64(len(content))),
Reader: bytes.NewReader(content),
Header: header,
Encryption: encryption,
@ -424,6 +462,7 @@ func prepareTestRequestWithQuery(hc *handlerContext, bktName, objName string, qu
r.URL.RawQuery = query.Encode()
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
reqInfo.User = hc.owner.String()
r = r.WithContext(middleware.SetReqInfo(hc.Context(), reqInfo))
return w, r

View file

@ -27,17 +27,18 @@ func getRangeToDetectContentType(maxSize uint64) *layer.RangeParams {
}
func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
conditional, err := parseConditionalHeaders(r.Header)
if err != nil {
h.logAndSendError(w, "could not parse request params", reqInfo, err)
h.logAndSendError(ctx, w, "could not parse request params", reqInfo, err)
return
}
@ -47,26 +48,27 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
extendedInfo, err := h.obj.GetExtendedObjectInfo(r.Context(), p)
extendedInfo, err := h.obj.GetExtendedObjectInfo(ctx, p)
if err != nil {
h.logAndSendError(w, "could not find object", reqInfo, err)
h.logAndSendError(ctx, w, "could not find object", reqInfo, err)
return
}
info := extendedInfo.ObjectInfo
encryptionParams, err := formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
h.logAndSendError(ctx, w, "invalid sse headers", reqInfo, err)
return
}
if err = encryptionParams.MatchObjectEncryption(layer.FormEncryptionInfo(info.Headers)); err != nil {
h.logAndSendError(w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
h.logAndSendError(ctx, w, "encryption doesn't match object", reqInfo, errors.GetAPIError(errors.ErrBadRequest), zap.Error(err))
return
}
if err = checkPreconditions(info, conditional, h.cfg.MD5Enabled()); err != nil {
h.logAndSendError(w, "precondition failed", reqInfo, err)
bktSettings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket settings", reqInfo, err)
return
}
@ -76,9 +78,17 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
VersionID: info.VersionID(),
}
tagSet, lockInfo, err := h.obj.GetObjectTaggingAndLock(r.Context(), t, extendedInfo.NodeVersion)
tagSet, lockInfo, err := h.obj.GetObjectTaggingAndLock(ctx, t, extendedInfo.NodeVersion)
if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchKey) {
h.logAndSendError(w, "could not get object meta data", reqInfo, err)
h.logAndSendError(ctx, w, "could not get object meta data", reqInfo, err)
return
}
if err = checkPreconditions(info, conditional, h.cfg.MD5Enabled()); err != nil {
if errors.IsS3Error(err, errors.ErrNotModified) {
writeNotModifiedHeaders(w.Header(), extendedInfo, len(tagSet), bktSettings.Unversioned(), h.cfg.MD5Enabled())
}
h.logAndSendError(ctx, w, "precondition failed", reqInfo, err)
return
}
@ -91,15 +101,15 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
BucketInfo: bktInfo,
}
objPayload, err := h.obj.GetObject(r.Context(), getParams)
objPayload, err := h.obj.GetObject(ctx, getParams)
if err != nil {
h.logAndSendError(w, "could not get object", reqInfo, err, zap.Stringer("oid", info.ID))
h.logAndSendError(ctx, w, "could not get object", reqInfo, err, zap.Stringer("oid", info.ID))
return
}
buffer, err := io.ReadAll(objPayload)
if err != nil {
h.logAndSendError(w, "could not partly read payload to detect content type", reqInfo, err, zap.Stringer("oid", info.ID))
h.logAndSendError(ctx, w, "could not partly read payload to detect content type", reqInfo, err, zap.Stringer("oid", info.ID))
return
}
@ -108,13 +118,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
}
if err = h.setLockingHeaders(bktInfo, lockInfo, w.Header()); err != nil {
h.logAndSendError(w, "could not get locking info", reqInfo, err)
return
}
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "could not get locking info", reqInfo, err)
return
}
@ -123,11 +127,12 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
}
func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
@ -141,7 +146,7 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
}
if err = middleware.WriteResponse(w, http.StatusOK, nil, middleware.MimeNone); err != nil {
h.logAndSendError(w, "write response", reqInfo, err)
h.logAndSendError(ctx, w, "write response", reqInfo, err)
return
}
}

View file

@ -6,7 +6,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
@ -95,8 +95,29 @@ func TestHeadObject(t *testing.T) {
hc.tp.SetObjectError(addr, &apistatus.ObjectNotFound{})
hc.tp.SetObjectError(objInfo.Address(), &apistatus.ObjectNotFound{})
headObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), s3errors.ErrNoSuchVersion)
headObjectAssertS3Error(hc, bktName, objName, emptyVersion, s3errors.ErrNoSuchKey)
headObjectAssertS3Error(hc, bktName, objName, objInfo.VersionID(), apierr.ErrNoSuchVersion)
headObjectAssertS3Error(hc, bktName, objName, emptyVersion, apierr.ErrNoSuchKey)
}
func TestHeadObjectNotModifiedHeaders(t *testing.T) {
hc := prepareHandlerContextWithMinCache(t)
bktName, objName, metadataHeader := "bucket", "obj", api.MetadataPrefix+"header"
createVersionedBucket(hc, bktName)
header := putObjectWithHeaders(hc, bktName, objName, map[string]string{api.CacheControl: "value", metadataHeader: "value"})
etag, versionID := header.Get(api.ETag), header.Get(api.AmzVersionID)
require.NotEmpty(t, etag)
require.NotEmpty(t, versionID)
putObjectTagging(t, hc, bktName, objName, map[string]string{"key": "value"})
w := headObjectWithHeaders(hc, bktName, objName, emptyVersion, map[string]string{api.IfNoneMatch: etag})
require.Equal(t, http.StatusNotModified, w.Code)
require.Equal(t, "1", w.Header().Get(api.AmzTaggingCount))
require.Equal(t, etag, w.Header().Get(api.ETag))
require.NotEmpty(t, w.Header().Get(api.LastModified))
require.Equal(t, versionID, w.Header().Get(api.AmzVersionID))
require.Equal(t, "value", w.Header().Get(api.CacheControl))
require.Equal(t, []string{"value"}, w.Header()[metadataHeader])
}
func TestIsAvailableToResolve(t *testing.T) {
@ -119,21 +140,25 @@ func TestIsAvailableToResolve(t *testing.T) {
}
}
func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box {
func newTestAccessBox(key *keys.PrivateKey) (*accessbox.Box, error) {
var err error
if key == nil {
key, err = keys.NewPrivateKey()
require.NoError(t, err)
if err != nil {
return nil, err
}
}
var btoken bearer.Token
btoken.SetImpersonate(true)
err = btoken.Sign(key.PrivateKey)
require.NoError(t, err)
if err != nil {
return nil, err
}
return &accessbox.Box{
Gate: &accessbox.GateData{
BearerToken: &btoken,
},
}
}, nil
}

View file

@ -7,15 +7,16 @@ import (
)
func (h *handler) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if err = middleware.EncodeToResponse(w, LocationResponse{Location: bktInfo.LocationConstraint}); err != nil {
h.logAndSendError(w, "couldn't encode bucket location response", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't encode bucket location response", reqInfo, err)
}
}

309
api/handler/lifecycle.go Normal file
View file

@ -0,0 +1,309 @@
package handler
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"fmt"
"io"
"net/http"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"github.com/google/uuid"
)
const (
maxRules = 1000
maxRuleIDLen = 255
maxNewerNoncurrentVersions = 100
)
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
cfg, err := h.obj.GetBucketLifecycleConfiguration(ctx, bktInfo)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket lifecycle configuration", reqInfo, err)
return
}
if err = middleware.EncodeToResponse(w, cfg); err != nil {
h.logAndSendError(ctx, w, "could not encode GetBucketLifecycle response", reqInfo, err)
return
}
}
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
var buf bytes.Buffer
tee := io.TeeReader(r.Body, &buf)
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
// Content-Md5 is required and should be set
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html
if _, ok := r.Header[api.ContentMD5]; !ok {
h.logAndSendError(ctx, w, "missing Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrMissingContentMD5))
return
}
headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5))
if err != nil {
h.logAndSendError(ctx, w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return
}
cfg := new(data.LifecycleConfiguration)
if err = h.cfg.NewXMLDecoder(tee).Decode(cfg); err != nil {
h.logAndSendError(ctx, w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
return
}
bodyMD5, err := getContentMD5(&buf)
if err != nil {
h.logAndSendError(ctx, w, "could not get content md5", reqInfo, err)
return
}
if !bytes.Equal(headerMD5, bodyMD5) {
h.logAndSendError(ctx, w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
networkInfo, err := h.obj.GetNetworkInfo(ctx)
if err != nil {
h.logAndSendError(ctx, w, "could not get network info", reqInfo, err)
return
}
if err = checkLifecycleConfiguration(ctx, cfg, &networkInfo); err != nil {
h.logAndSendError(ctx, w, "invalid lifecycle configuration", reqInfo, err)
return
}
params := &layer.PutBucketLifecycleParams{
BktInfo: bktInfo,
LifecycleCfg: cfg,
}
params.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err)
return
}
if err = h.obj.PutBucketLifecycleConfiguration(ctx, params); err != nil {
h.logAndSendError(ctx, w, "could not put bucket lifecycle configuration", reqInfo, err)
return
}
}
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if err = h.obj.DeleteBucketLifecycleConfiguration(ctx, bktInfo); err != nil {
h.logAndSendError(ctx, w, "could not delete bucket lifecycle configuration", reqInfo, err)
return
}
w.WriteHeader(http.StatusNoContent)
}
func checkLifecycleConfiguration(ctx context.Context, cfg *data.LifecycleConfiguration, ni *netmap.NetworkInfo) error {
now := layer.TimeNow(ctx)
if len(cfg.Rules) > maxRules {
return fmt.Errorf("%w: number of rules cannot be greater than %d", apierr.GetAPIError(apierr.ErrInvalidRequest), maxRules)
}
ids := make(map[string]struct{}, len(cfg.Rules))
for i, rule := range cfg.Rules {
if rule.ID == "" {
id, err := uuid.NewRandom()
if err != nil {
return fmt.Errorf("generate uuid: %w", err)
}
cfg.Rules[i].ID = id.String()
rule.ID = id.String()
}
if _, ok := ids[rule.ID]; ok {
return fmt.Errorf("%w: duplicate 'ID': %s", apierr.GetAPIError(apierr.ErrInvalidArgument), rule.ID)
}
ids[rule.ID] = struct{}{}
if len(rule.ID) > maxRuleIDLen {
return fmt.Errorf("%w: 'ID' value cannot be longer than %d characters", apierr.GetAPIError(apierr.ErrInvalidArgument), maxRuleIDLen)
}
if rule.Status != data.LifecycleStatusEnabled && rule.Status != data.LifecycleStatusDisabled {
return fmt.Errorf("%w: invalid lifecycle status: %s", apierr.GetAPIError(apierr.ErrMalformedXML), rule.Status)
}
if rule.AbortIncompleteMultipartUpload == nil && rule.Expiration == nil && rule.NonCurrentVersionExpiration == nil {
return fmt.Errorf("%w: at least one action needs to be specified in a rule", apierr.GetAPIError(apierr.ErrInvalidRequest))
}
if rule.AbortIncompleteMultipartUpload != nil {
if rule.AbortIncompleteMultipartUpload.DaysAfterInitiation != nil &&
*rule.AbortIncompleteMultipartUpload.DaysAfterInitiation <= 0 {
return fmt.Errorf("%w: days after initiation must be a positive integer", apierr.GetAPIError(apierr.ErrInvalidArgument))
}
if rule.Filter != nil && (rule.Filter.Tag != nil || (rule.Filter.And != nil && len(rule.Filter.And.Tags) > 0)) {
return fmt.Errorf("%w: abort incomplete multipart upload cannot be specified with tags", apierr.GetAPIError(apierr.ErrInvalidRequest))
}
}
if rule.Expiration != nil {
if rule.Expiration.ExpiredObjectDeleteMarker != nil {
if rule.Expiration.Days != nil || rule.Expiration.Date != "" {
return fmt.Errorf("%w: expired object delete marker cannot be specified with days or date", apierr.GetAPIError(apierr.ErrMalformedXML))
}
if rule.Filter != nil && (rule.Filter.Tag != nil || (rule.Filter.And != nil && len(rule.Filter.And.Tags) > 0)) {
return fmt.Errorf("%w: expired object delete marker cannot be specified with tags", apierr.GetAPIError(apierr.ErrInvalidRequest))
}
}
if rule.Expiration.Days != nil && *rule.Expiration.Days <= 0 {
return fmt.Errorf("%w: expiration days must be a positive integer", apierr.GetAPIError(apierr.ErrInvalidArgument))
}
if rule.Expiration.Date != "" {
parsedTime, err := time.Parse("2006-01-02T15:04:05Z", rule.Expiration.Date)
if err != nil {
return fmt.Errorf("%w: invalid value of expiration date: %s", apierr.GetAPIError(apierr.ErrInvalidArgument), rule.Expiration.Date)
}
epoch, err := util.TimeToEpoch(ni, now, parsedTime)
if err != nil {
return fmt.Errorf("convert time to epoch: %w", err)
}
cfg.Rules[i].Expiration.Epoch = &epoch
}
}
if rule.NonCurrentVersionExpiration != nil {
if rule.NonCurrentVersionExpiration.NewerNonCurrentVersions != nil && rule.NonCurrentVersionExpiration.NonCurrentDays == nil {
return fmt.Errorf("%w: newer noncurrent versions cannot be specified without noncurrent days", apierr.GetAPIError(apierr.ErrMalformedXML))
}
if rule.NonCurrentVersionExpiration.NewerNonCurrentVersions != nil &&
(*rule.NonCurrentVersionExpiration.NewerNonCurrentVersions > maxNewerNoncurrentVersions ||
*rule.NonCurrentVersionExpiration.NewerNonCurrentVersions <= 0) {
return fmt.Errorf("%w: newer noncurrent versions must be a positive integer up to %d", apierr.GetAPIError(apierr.ErrInvalidArgument),
maxNewerNoncurrentVersions)
}
if rule.NonCurrentVersionExpiration.NonCurrentDays != nil && *rule.NonCurrentVersionExpiration.NonCurrentDays <= 0 {
return fmt.Errorf("%w: noncurrent days must be a positive integer", apierr.GetAPIError(apierr.ErrInvalidArgument))
}
}
if err := checkLifecycleRuleFilter(rule.Filter); err != nil {
return err
}
if rule.Filter != nil && rule.Filter.Prefix != "" && rule.Prefix != "" {
return fmt.Errorf("%w: rule cannot have two prefixes", apierr.GetAPIError(apierr.ErrMalformedXML))
}
}
return nil
}
func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error {
if filter == nil {
return nil
}
var fields int
if filter.And != nil {
fields++
for _, tag := range filter.And.Tags {
err := checkTag(tag)
if err != nil {
return err
}
}
if filter.And.ObjectSizeLessThan != nil {
if *filter.And.ObjectSizeLessThan == 0 {
return fmt.Errorf("%w: the maximum object size must be more than 0", apierr.GetAPIError(apierr.ErrInvalidRequest))
}
if filter.And.ObjectSizeGreaterThan != nil && *filter.And.ObjectSizeLessThan <= *filter.And.ObjectSizeGreaterThan {
return fmt.Errorf("%w: the maximum object size must be larger than the minimum object size", apierr.GetAPIError(apierr.ErrInvalidRequest))
}
}
}
if filter.ObjectSizeGreaterThan != nil {
fields++
}
if filter.ObjectSizeLessThan != nil {
if *filter.ObjectSizeLessThan == 0 {
return fmt.Errorf("%w: the maximum object size must be more than 0", apierr.GetAPIError(apierr.ErrInvalidRequest))
}
fields++
}
if filter.Prefix != "" {
fields++
}
if filter.Tag != nil {
fields++
err := checkTag(*filter.Tag)
if err != nil {
return err
}
}
if fields > 1 {
return fmt.Errorf("%w: filter cannot have more than one field", apierr.GetAPIError(apierr.ErrMalformedXML))
}
return nil
}
func getContentMD5(reader io.Reader) ([]byte, error) {
hash := md5.New()
_, err := io.Copy(hash, reader)
if err != nil {
return nil, err
}
return hash.Sum(nil), nil
}

View file

@ -0,0 +1,563 @@
package handler
import (
"bytes"
"crypto/md5"
"crypto/rand"
"encoding/base64"
"encoding/xml"
"io"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/mr-tron/base58"
"github.com/stretchr/testify/require"
)
func TestPutBucketLifecycleConfiguration(t *testing.T) {
hc := prepareHandlerContextWithMinCache(t)
bktName := "bucket-lifecycle"
createBucket(hc, bktName)
for _, tc := range []struct {
name string
body *data.LifecycleConfiguration
errorCode apierr.ErrorCode
}{
{
name: "correct configuration",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
ID: "rule-1",
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
Date: time.Now().Format("2006-01-02T15:04:05.000Z"),
},
Filter: &data.LifecycleRuleFilter{
And: &data.LifecycleRuleAndOperator{
Prefix: "prefix/",
Tags: []data.Tag{{Key: "key", Value: "value"}, {Key: "tag", Value: ""}},
ObjectSizeGreaterThan: ptr(uint64(100)),
},
},
},
{
ID: "rule-2",
Status: data.LifecycleStatusEnabled,
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
DaysAfterInitiation: ptr(14),
},
Expiration: &data.LifecycleExpiration{
ExpiredObjectDeleteMarker: ptr(true),
},
Filter: &data.LifecycleRuleFilter{
ObjectSizeLessThan: ptr(uint64(100)),
},
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
NewerNonCurrentVersions: ptr(1),
NonCurrentDays: ptr(21),
},
},
},
},
},
{
name: "too many rules",
body: func() *data.LifecycleConfiguration {
lifecycle := new(data.LifecycleConfiguration)
for i := 0; i <= maxRules; i++ {
lifecycle.Rules = append(lifecycle.Rules, data.LifecycleRule{
ID: "Rule" + strconv.Itoa(i),
})
}
return lifecycle
}(),
errorCode: apierr.ErrInvalidRequest,
},
{
name: "duplicate rule ID",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
ID: "Rule",
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
},
{
ID: "Rule",
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
},
},
},
errorCode: apierr.ErrInvalidArgument,
},
{
name: "too long rule ID",
body: func() *data.LifecycleConfiguration {
id := make([]byte, maxRuleIDLen+1)
_, err := io.ReadFull(rand.Reader, id)
require.NoError(t, err)
return &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
ID: base58.Encode(id)[:maxRuleIDLen+1],
},
},
}
}(),
errorCode: apierr.ErrInvalidArgument,
},
{
name: "invalid status",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: "invalid",
},
},
},
errorCode: apierr.ErrMalformedXML,
},
{
name: "no actions",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Filter: &data.LifecycleRuleFilter{
Prefix: "prefix/",
},
},
},
},
errorCode: apierr.ErrInvalidRequest,
},
{
name: "invalid days after initiation",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
DaysAfterInitiation: ptr(0),
},
},
},
},
errorCode: apierr.ErrInvalidArgument,
},
{
name: "invalid expired object delete marker declaration",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
ExpiredObjectDeleteMarker: ptr(false),
},
},
},
},
errorCode: apierr.ErrMalformedXML,
},
{
name: "invalid expiration days",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(0),
},
},
},
},
errorCode: apierr.ErrInvalidArgument,
},
{
name: "invalid expiration date",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Date: "invalid",
},
},
},
},
errorCode: apierr.ErrInvalidArgument,
},
{
name: "newer noncurrent versions is too small",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
NonCurrentDays: ptr(1),
NewerNonCurrentVersions: ptr(0),
},
},
},
},
errorCode: apierr.ErrInvalidArgument,
},
{
name: "newer noncurrent versions is too large",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
NonCurrentDays: ptr(1),
NewerNonCurrentVersions: ptr(101),
},
},
},
},
errorCode: apierr.ErrInvalidArgument,
},
{
name: "invalid noncurrent days",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
NonCurrentDays: ptr(0),
},
},
},
},
errorCode: apierr.ErrInvalidArgument,
},
{
name: "more than one filter field",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
Filter: &data.LifecycleRuleFilter{
Prefix: "prefix/",
ObjectSizeGreaterThan: ptr(uint64(100)),
},
},
},
},
errorCode: apierr.ErrMalformedXML,
},
{
name: "invalid tag in filter",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
Filter: &data.LifecycleRuleFilter{
Tag: &data.Tag{},
},
},
},
},
errorCode: apierr.ErrInvalidTagKey,
},
{
name: "abort incomplete multipart upload with tag",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
DaysAfterInitiation: ptr(14),
},
Filter: &data.LifecycleRuleFilter{
Tag: &data.Tag{Key: "key", Value: "value"},
},
},
},
},
errorCode: apierr.ErrInvalidRequest,
},
{
name: "expired object delete marker with tag",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
ExpiredObjectDeleteMarker: ptr(true),
},
Filter: &data.LifecycleRuleFilter{
And: &data.LifecycleRuleAndOperator{
Tags: []data.Tag{{Key: "key", Value: "value"}},
},
},
},
},
},
errorCode: apierr.ErrInvalidRequest,
},
{
name: "invalid size range",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
Filter: &data.LifecycleRuleFilter{
And: &data.LifecycleRuleAndOperator{
ObjectSizeGreaterThan: ptr(uint64(100)),
ObjectSizeLessThan: ptr(uint64(100)),
},
},
},
},
},
errorCode: apierr.ErrInvalidRequest,
},
{
name: "two prefixes",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
Filter: &data.LifecycleRuleFilter{
Prefix: "prefix-1/",
},
Prefix: "prefix-2/",
},
},
},
errorCode: apierr.ErrMalformedXML,
},
{
name: "newer noncurrent versions without noncurrent days",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
NewerNonCurrentVersions: ptr(10),
},
},
},
},
errorCode: apierr.ErrMalformedXML,
},
{
name: "invalid maximum object size in filter",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
Filter: &data.LifecycleRuleFilter{
ObjectSizeLessThan: ptr(uint64(0)),
},
},
},
},
errorCode: apierr.ErrInvalidRequest,
},
{
name: "invalid maximum object size in filter and",
body: &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
Filter: &data.LifecycleRuleFilter{
And: &data.LifecycleRuleAndOperator{
Prefix: "prefix/",
ObjectSizeLessThan: ptr(uint64(0)),
},
},
},
},
},
errorCode: apierr.ErrInvalidRequest,
},
} {
t.Run(tc.name, func(t *testing.T) {
if tc.errorCode > 0 {
putBucketLifecycleConfigurationErr(hc, bktName, tc.body, apierr.GetAPIError(tc.errorCode))
return
}
putBucketLifecycleConfiguration(hc, bktName, tc.body)
cfg := getBucketLifecycleConfiguration(hc, bktName)
require.Equal(t, tc.body.Rules, cfg.Rules)
deleteBucketLifecycleConfiguration(hc, bktName)
getBucketLifecycleConfigurationErr(hc, bktName, apierr.GetAPIError(apierr.ErrNoSuchLifecycleConfiguration))
})
}
}
func TestPutBucketLifecycleIDGeneration(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-lifecycle-id"
createBucket(hc, bktName)
lifecycle := &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
},
{
Status: data.LifecycleStatusEnabled,
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
DaysAfterInitiation: ptr(14),
},
},
},
}
putBucketLifecycleConfiguration(hc, bktName, lifecycle)
cfg := getBucketLifecycleConfiguration(hc, bktName)
require.Len(t, cfg.Rules, 2)
require.NotEmpty(t, cfg.Rules[0].ID)
require.NotEmpty(t, cfg.Rules[1].ID)
}
func TestPutBucketLifecycleInvalidMD5(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-lifecycle-md5"
createBucket(hc, bktName)
lifecycle := &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
},
},
}
w, r := prepareTestRequest(hc, bktName, "", lifecycle)
hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMissingContentMD5))
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
r.Header.Set(api.ContentMD5, "")
hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
r.Header.Set(api.ContentMD5, "some-hash")
hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
}
func TestPutBucketLifecycleInvalidXML(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-lifecycle-invalid-xml"
createBucket(hc, bktName)
cfg := &data.CORSConfiguration{}
body, err := xml.Marshal(cfg)
require.NoError(t, err)
contentMD5, err := getContentMD5(bytes.NewReader(body))
require.NoError(t, err)
w, r := prepareTestRequest(hc, bktName, "", cfg)
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(contentMD5))
hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMalformedXML))
}
func putBucketLifecycleConfiguration(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) {
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
assertStatus(hc.t, w, http.StatusOK)
}
func putBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, err apierr.Error) {
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
assertS3Error(hc.t, w, err)
}
func putBucketLifecycleConfigurationBase(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) *httptest.ResponseRecorder {
w, r := prepareTestRequest(hc, bktName, "", cfg)
rawBody, err := xml.Marshal(cfg)
require.NoError(hc.t, err)
hash := md5.New()
hash.Write(rawBody)
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(hash.Sum(nil)))
hc.Handler().PutBucketLifecycleHandler(w, r)
return w
}
func getBucketLifecycleConfiguration(hc *handlerContext, bktName string) *data.LifecycleConfiguration {
w := getBucketLifecycleConfigurationBase(hc, bktName)
assertStatus(hc.t, w, http.StatusOK)
res := &data.LifecycleConfiguration{}
parseTestResponse(hc.t, w, res)
return res
}
func getBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, err apierr.Error) {
w := getBucketLifecycleConfigurationBase(hc, bktName)
assertS3Error(hc.t, w, err)
}
func getBucketLifecycleConfigurationBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder {
w, r := prepareTestRequest(hc, bktName, "", nil)
hc.Handler().GetBucketLifecycleHandler(w, r)
return w
}
func deleteBucketLifecycleConfiguration(hc *handlerContext, bktName string) {
w := deleteBucketLifecycleConfigurationBase(hc, bktName)
assertStatus(hc.t, w, http.StatusNoContent)
}
func deleteBucketLifecycleConfigurationBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder {
w, r := prepareTestRequest(hc, bktName, "", nil)
hc.Handler().DeleteBucketLifecycleHandler(w, r)
return w
}
func ptr[T any](t T) *T {
return &t
}

View file

@ -15,12 +15,13 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
var (
own user.ID
res *ListBucketsResponse
reqInfo = middleware.GetReqInfo(r.Context())
ctx = r.Context()
reqInfo = middleware.GetReqInfo(ctx)
)
list, err := h.obj.ListBuckets(r.Context())
list, err := h.obj.ListBuckets(ctx)
if err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
return
}
@ -43,6 +44,6 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
}
if err = middleware.EncodeToResponse(w, res); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}

View file

@ -9,7 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
)
@ -26,34 +26,35 @@ const (
)
func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if !bktInfo.ObjectLockEnabled {
h.logAndSendError(w, "couldn't put object locking configuration", reqInfo,
apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotAllowed))
h.logAndSendError(ctx, w, "couldn't put object locking configuration", reqInfo,
apierr.GetAPIError(apierr.ErrObjectLockConfigurationNotAllowed))
return
}
lockingConf := &data.ObjectLockConfiguration{}
if err = h.cfg.NewXMLDecoder(r.Body).Decode(lockingConf); err != nil {
h.logAndSendError(w, "couldn't parse locking configuration", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't parse locking configuration", reqInfo, err)
return
}
if err = checkLockConfiguration(lockingConf); err != nil {
h.logAndSendError(w, "invalid lock configuration", reqInfo, err)
h.logAndSendError(ctx, w, "invalid lock configuration", reqInfo, err)
return
}
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't get bucket settings", reqInfo, err)
return
}
@ -66,30 +67,31 @@ func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt
Settings: &newSettings,
}
if err = h.obj.PutBucketSettings(r.Context(), sp); err != nil {
h.logAndSendError(w, "couldn't put bucket settings", reqInfo, err)
if err = h.obj.PutBucketSettings(ctx, sp); err != nil {
h.logAndSendError(ctx, w, "couldn't put bucket settings", reqInfo, err)
return
}
}
func (h *handler) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if !bktInfo.ObjectLockEnabled {
h.logAndSendError(w, "object lock disabled", reqInfo,
apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound))
h.logAndSendError(ctx, w, "object lock disabled", reqInfo,
apierr.GetAPIError(apierr.ErrObjectLockConfigurationNotFound))
return
}
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't get bucket settings", reqInfo, err)
return
}
@ -101,33 +103,34 @@ func (h *handler) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt
}
if err = middleware.EncodeToResponse(w, settings.LockConfiguration); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}
func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if !bktInfo.ObjectLockEnabled {
h.logAndSendError(w, "object lock disabled", reqInfo,
apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound))
h.logAndSendError(ctx, w, "object lock disabled", reqInfo,
apierr.GetAPIError(apierr.ErrObjectLockConfigurationNotFound))
return
}
legalHold := &data.LegalHold{}
if err = h.cfg.NewXMLDecoder(r.Body).Decode(legalHold); err != nil {
h.logAndSendError(w, "couldn't parse legal hold configuration", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't parse legal hold configuration", reqInfo, err)
return
}
if legalHold.Status != legalHoldOn && legalHold.Status != legalHoldOff {
h.logAndSendError(w, "invalid legal hold status", reqInfo,
h.logAndSendError(ctx, w, "invalid legal hold status", reqInfo,
fmt.Errorf("invalid status %s", legalHold.Status))
return
}
@ -147,28 +150,29 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err)
return
}
if err = h.obj.PutLockInfo(r.Context(), p); err != nil {
h.logAndSendError(w, "couldn't head put legal hold", reqInfo, err)
if err = h.obj.PutLockInfo(ctx, p); err != nil {
h.logAndSendError(ctx, w, "couldn't head put legal hold", reqInfo, err)
return
}
}
func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if !bktInfo.ObjectLockEnabled {
h.logAndSendError(w, "object lock disabled", reqInfo,
apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound))
h.logAndSendError(ctx, w, "object lock disabled", reqInfo,
apierr.GetAPIError(apierr.ErrObjectLockConfigurationNotFound))
return
}
@ -178,9 +182,9 @@ func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
lockInfo, err := h.obj.GetLockInfo(r.Context(), p)
lockInfo, err := h.obj.GetLockInfo(ctx, p)
if err != nil {
h.logAndSendError(w, "couldn't head lock object", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't head lock object", reqInfo, err)
return
}
@ -190,33 +194,34 @@ func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
}
if err = middleware.EncodeToResponse(w, legalHold); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}
func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if !bktInfo.ObjectLockEnabled {
h.logAndSendError(w, "object lock disabled", reqInfo,
apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound))
h.logAndSendError(ctx, w, "object lock disabled", reqInfo,
apierr.GetAPIError(apierr.ErrObjectLockConfigurationNotFound))
return
}
retention := &data.Retention{}
if err = h.cfg.NewXMLDecoder(r.Body).Decode(retention); err != nil {
h.logAndSendError(w, "couldn't parse object retention", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't parse object retention", reqInfo, err)
return
}
lock, err := formObjectLockFromRetention(r.Context(), retention, r.Header)
lock, err := formObjectLockFromRetention(ctx, retention, r.Header)
if err != nil {
h.logAndSendError(w, "invalid retention configuration", reqInfo, err)
h.logAndSendError(ctx, w, "invalid retention configuration", reqInfo, err)
return
}
@ -231,28 +236,29 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err)
return
}
if err = h.obj.PutLockInfo(r.Context(), p); err != nil {
h.logAndSendError(w, "couldn't put legal hold", reqInfo, err)
if err = h.obj.PutLockInfo(ctx, p); err != nil {
h.logAndSendError(ctx, w, "couldn't put legal hold", reqInfo, err)
return
}
}
func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if !bktInfo.ObjectLockEnabled {
h.logAndSendError(w, "object lock disabled", reqInfo,
apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound))
h.logAndSendError(ctx, w, "object lock disabled", reqInfo,
apierr.GetAPIError(apierr.ErrObjectLockConfigurationNotFound))
return
}
@ -262,14 +268,14 @@ func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
lockInfo, err := h.obj.GetLockInfo(r.Context(), p)
lockInfo, err := h.obj.GetLockInfo(ctx, p)
if err != nil {
h.logAndSendError(w, "couldn't head lock object", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't head lock object", reqInfo, err)
return
}
if !lockInfo.IsRetentionSet() {
h.logAndSendError(w, "retention lock isn't set", reqInfo, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey))
h.logAndSendError(ctx, w, "retention lock isn't set", reqInfo, apierr.GetAPIError(apierr.ErrNoSuchKey))
return
}
@ -282,7 +288,7 @@ func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
}
if err = middleware.EncodeToResponse(w, retention); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}
@ -314,7 +320,7 @@ func checkLockConfiguration(conf *data.ObjectLockConfiguration) error {
func formObjectLock(ctx context.Context, bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConfiguration, header http.Header) (*data.ObjectLock, error) {
if !bktInfo.ObjectLockEnabled {
if existLockHeaders(header) {
return nil, apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound)
return nil, apierr.GetAPIError(apierr.ErrObjectLockConfigurationNotFound)
}
return nil, nil
}
@ -346,7 +352,7 @@ func formObjectLock(ctx context.Context, bktInfo *data.BucketInfo, defaultConfig
until := header.Get(api.AmzObjectLockRetainUntilDate)
if mode != "" && until == "" || mode == "" && until != "" {
return nil, apiErrors.GetAPIError(apiErrors.ErrObjectLockInvalidHeaders)
return nil, apierr.GetAPIError(apierr.ErrObjectLockInvalidHeaders)
}
if mode != "" {
@ -355,7 +361,7 @@ func formObjectLock(ctx context.Context, bktInfo *data.BucketInfo, defaultConfig
}
if mode != complianceMode && mode != governanceMode {
return nil, apiErrors.GetAPIError(apiErrors.ErrUnknownWORMModeDirective)
return nil, apierr.GetAPIError(apierr.ErrUnknownWORMModeDirective)
}
objectLock.Retention.IsCompliance = mode == complianceMode
@ -364,7 +370,7 @@ func formObjectLock(ctx context.Context, bktInfo *data.BucketInfo, defaultConfig
if until != "" {
retentionDate, err := time.Parse(time.RFC3339, until)
if err != nil {
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidRetentionDate)
return nil, apierr.GetAPIError(apierr.ErrInvalidRetentionDate)
}
if objectLock.Retention == nil {
objectLock.Retention = &data.RetentionLock{}
@ -382,7 +388,7 @@ func formObjectLock(ctx context.Context, bktInfo *data.BucketInfo, defaultConfig
}
if objectLock.Retention.Until.Before(layer.TimeNow(ctx)) {
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
return nil, apierr.GetAPIError(apierr.ErrPastObjectLockRetainDate)
}
}
@ -397,16 +403,16 @@ func existLockHeaders(header http.Header) bool {
func formObjectLockFromRetention(ctx context.Context, retention *data.Retention, header http.Header) (*data.ObjectLock, error) {
if retention.Mode != governanceMode && retention.Mode != complianceMode {
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
return nil, apierr.GetAPIError(apierr.ErrMalformedXML)
}
retentionDate, err := time.Parse(time.RFC3339, retention.RetainUntilDate)
if err != nil {
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
return nil, apierr.GetAPIError(apierr.ErrMalformedXML)
}
if retentionDate.Before(layer.TimeNow(ctx)) {
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
return nil, apierr.GetAPIError(apierr.ErrPastObjectLockRetainDate)
}
var bypass bool

View file

@ -12,7 +12,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"github.com/stretchr/testify/require"
@ -270,23 +270,24 @@ func TestPutBucketLockConfigurationHandler(t *testing.T) {
for _, tc := range []struct {
name string
bucket string
expectedError apiErrors.Error
expectedError apierr.Error
noError bool
configuration *data.ObjectLockConfiguration
}{
{
name: "bkt not found",
expectedError: apiErrors.GetAPIError(apiErrors.ErrNoSuchBucket),
bucket: "not-found-bucket",
expectedError: apierr.GetAPIError(apierr.ErrNoSuchBucket),
},
{
name: "bkt lock disabled",
bucket: bktLockDisabled,
expectedError: apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotAllowed),
expectedError: apierr.GetAPIError(apierr.ErrObjectLockConfigurationNotAllowed),
},
{
name: "invalid configuration",
bucket: bktLockEnabled,
expectedError: apiErrors.GetAPIError(apiErrors.ErrInternalError),
expectedError: apierr.GetAPIError(apierr.ErrInternalError),
configuration: &data.ObjectLockConfiguration{ObjectLockEnabled: "dummy"},
},
{
@ -359,18 +360,19 @@ func TestGetBucketLockConfigurationHandler(t *testing.T) {
for _, tc := range []struct {
name string
bucket string
expectedError apiErrors.Error
expectedError apierr.Error
noError bool
expectedConf *data.ObjectLockConfiguration
}{
{
name: "bkt not found",
expectedError: apiErrors.GetAPIError(apiErrors.ErrNoSuchBucket),
bucket: "not-found-bucket",
expectedError: apierr.GetAPIError(apierr.ErrNoSuchBucket),
},
{
name: "bkt lock disabled",
bucket: bktLockDisabled,
expectedError: apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound),
expectedError: apierr.GetAPIError(apierr.ErrObjectLockConfigurationNotFound),
},
{
name: "bkt lock enabled empty default",
@ -407,7 +409,7 @@ func TestGetBucketLockConfigurationHandler(t *testing.T) {
}
}
func assertS3Error(t *testing.T, w *httptest.ResponseRecorder, expectedError apiErrors.Error) {
func assertS3Error(t *testing.T, w *httptest.ResponseRecorder, expectedError apierr.Error) {
actualErrorResponse := &middleware.ErrorResponse{}
err := xml.NewDecoder(w.Result().Body).Decode(actualErrorResponse)
require.NoError(t, err)
@ -415,7 +417,7 @@ func assertS3Error(t *testing.T, w *httptest.ResponseRecorder, expectedError api
require.Equal(t, expectedError.HTTPStatusCode, w.Code)
require.Equal(t, expectedError.Code, actualErrorResponse.Code)
if expectedError.ErrCode != apiErrors.ErrInternalError {
if expectedError.ErrCode != apierr.ErrInternalError {
require.Equal(t, expectedError.Description, actualErrorResponse.Message)
}
}
@ -473,33 +475,33 @@ func TestObjectRetention(t *testing.T) {
objName := "obj-for-retention"
createTestObject(hc, bktInfo, objName, encryption.Params{})
getObjectRetention(hc, bktName, objName, nil, apiErrors.ErrNoSuchKey)
getObjectRetention(hc, bktName, objName, nil, apierr.ErrNoSuchKey)
retention := &data.Retention{Mode: governanceMode, RetainUntilDate: time.Now().Add(time.Minute).UTC().Format(time.RFC3339)}
putObjectRetention(hc, bktName, objName, retention, false, 0)
getObjectRetention(hc, bktName, objName, retention, 0)
retention = &data.Retention{Mode: governanceMode, RetainUntilDate: time.Now().UTC().Add(time.Minute).Format(time.RFC3339)}
putObjectRetention(hc, bktName, objName, retention, false, apiErrors.ErrInternalError)
putObjectRetention(hc, bktName, objName, retention, false, apierr.ErrInternalError)
retention = &data.Retention{Mode: complianceMode, RetainUntilDate: time.Now().Add(time.Minute).UTC().Format(time.RFC3339)}
putObjectRetention(hc, bktName, objName, retention, true, 0)
getObjectRetention(hc, bktName, objName, retention, 0)
putObjectRetention(hc, bktName, objName, retention, true, apiErrors.ErrInternalError)
putObjectRetention(hc, bktName, objName, retention, true, apierr.ErrInternalError)
}
func getObjectRetention(hc *handlerContext, bktName, objName string, retention *data.Retention, errCode apiErrors.ErrorCode) {
func getObjectRetention(hc *handlerContext, bktName, objName string, retention *data.Retention, errCode apierr.ErrorCode) {
w, r := prepareTestRequest(hc, bktName, objName, nil)
hc.Handler().GetObjectRetentionHandler(w, r)
if errCode == 0 {
assertRetention(hc.t, w, retention)
} else {
assertS3Error(hc.t, w, apiErrors.GetAPIError(errCode))
assertS3Error(hc.t, w, apierr.GetAPIError(errCode))
}
}
func putObjectRetention(hc *handlerContext, bktName, objName string, retention *data.Retention, byPass bool, errCode apiErrors.ErrorCode) {
func putObjectRetention(hc *handlerContext, bktName, objName string, retention *data.Retention, byPass bool, errCode apierr.ErrorCode) {
w, r := prepareTestRequest(hc, bktName, objName, retention)
if byPass {
r.Header.Set(api.AmzBypassGovernanceRetention, strconv.FormatBool(true))
@ -508,7 +510,7 @@ func putObjectRetention(hc *handlerContext, bktName, objName string, retention *
if errCode == 0 {
assertStatus(hc.t, w, http.StatusOK)
} else {
assertS3Error(hc.t, w, apiErrors.GetAPIError(errCode))
assertS3Error(hc.t, w, apierr.GetAPIError(errCode))
}
}
@ -572,37 +574,37 @@ func TestPutLockErrors(t *testing.T) {
createTestBucketWithLock(hc, bktName, nil)
headers := map[string]string{api.AmzObjectLockMode: complianceMode}
putObjectWithLockFailed(t, hc, bktName, objName, headers, apiErrors.ErrObjectLockInvalidHeaders)
putObjectWithLockFailed(t, hc, bktName, objName, headers, apierr.ErrObjectLockInvalidHeaders)
delete(headers, api.AmzObjectLockMode)
headers[api.AmzObjectLockRetainUntilDate] = time.Now().Add(time.Minute).Format(time.RFC3339)
putObjectWithLockFailed(t, hc, bktName, objName, headers, apiErrors.ErrObjectLockInvalidHeaders)
putObjectWithLockFailed(t, hc, bktName, objName, headers, apierr.ErrObjectLockInvalidHeaders)
headers[api.AmzObjectLockMode] = "dummy"
putObjectWithLockFailed(t, hc, bktName, objName, headers, apiErrors.ErrUnknownWORMModeDirective)
putObjectWithLockFailed(t, hc, bktName, objName, headers, apierr.ErrUnknownWORMModeDirective)
headers[api.AmzObjectLockMode] = complianceMode
headers[api.AmzObjectLockRetainUntilDate] = time.Now().Format(time.RFC3339)
putObjectWithLockFailed(t, hc, bktName, objName, headers, apiErrors.ErrPastObjectLockRetainDate)
putObjectWithLockFailed(t, hc, bktName, objName, headers, apierr.ErrPastObjectLockRetainDate)
headers[api.AmzObjectLockRetainUntilDate] = "dummy"
putObjectWithLockFailed(t, hc, bktName, objName, headers, apiErrors.ErrInvalidRetentionDate)
putObjectWithLockFailed(t, hc, bktName, objName, headers, apierr.ErrInvalidRetentionDate)
putObject(hc, bktName, objName)
retention := &data.Retention{Mode: governanceMode}
putObjectRetentionFailed(t, hc, bktName, objName, retention, apiErrors.ErrMalformedXML)
putObjectRetentionFailed(t, hc, bktName, objName, retention, apierr.ErrMalformedXML)
retention.Mode = "dummy"
retention.RetainUntilDate = time.Now().Add(time.Minute).UTC().Format(time.RFC3339)
putObjectRetentionFailed(t, hc, bktName, objName, retention, apiErrors.ErrMalformedXML)
putObjectRetentionFailed(t, hc, bktName, objName, retention, apierr.ErrMalformedXML)
retention.Mode = governanceMode
retention.RetainUntilDate = time.Now().UTC().Format(time.RFC3339)
putObjectRetentionFailed(t, hc, bktName, objName, retention, apiErrors.ErrPastObjectLockRetainDate)
putObjectRetentionFailed(t, hc, bktName, objName, retention, apierr.ErrPastObjectLockRetainDate)
}
func putObjectWithLockFailed(t *testing.T, hc *handlerContext, bktName, objName string, headers map[string]string, errCode apiErrors.ErrorCode) {
func putObjectWithLockFailed(t *testing.T, hc *handlerContext, bktName, objName string, headers map[string]string, errCode apierr.ErrorCode) {
w, r := prepareTestRequest(hc, bktName, objName, nil)
for key, val := range headers {
@ -610,13 +612,13 @@ func putObjectWithLockFailed(t *testing.T, hc *handlerContext, bktName, objName
}
hc.Handler().PutObjectHandler(w, r)
assertS3Error(t, w, apiErrors.GetAPIError(errCode))
assertS3Error(t, w, apierr.GetAPIError(errCode))
}
func putObjectRetentionFailed(t *testing.T, hc *handlerContext, bktName, objName string, retention *data.Retention, errCode apiErrors.ErrorCode) {
func putObjectRetentionFailed(t *testing.T, hc *handlerContext, bktName, objName string, retention *data.Retention, errCode apierr.ErrorCode) {
w, r := prepareTestRequest(hc, bktName, objName, retention)
hc.Handler().PutObjectRetentionHandler(w, r)
assertS3Error(t, w, apiErrors.GetAPIError(errCode))
assertS3Error(t, w, apierr.GetAPIError(errCode))
}
func assertRetentionApproximate(t *testing.T, w *httptest.ResponseRecorder, retention *data.Retention, delta float64) {

View file

@ -104,19 +104,20 @@ const (
)
func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
uploadID := uuid.New()
cannedACLStatus := aclHeadersStatus(r)
additional := []zap.Field{zap.String("uploadID", uploadID.String())}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if cannedACLStatus == aclStatusYes {
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
h.logAndSendError(ctx, w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
return
}
@ -132,14 +133,14 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
if len(r.Header.Get(api.AmzTagging)) > 0 {
p.Data.TagSet, err = parseTaggingHeader(r.Header)
if err != nil {
h.logAndSendError(w, "could not parse tagging", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "could not parse tagging", reqInfo, err, additional...)
return
}
}
p.Info.Encryption, err = formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "invalid sse headers", reqInfo, err, additional...)
return
}
@ -153,12 +154,12 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
p.CopiesNumbers, err = h.pickCopiesNumbers(p.Header, reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err, additional...)
return
}
if err = h.obj.CreateMultipartUpload(r.Context(), p); err != nil {
h.logAndSendError(w, "could create multipart upload", reqInfo, err, additional...)
if err = h.obj.CreateMultipartUpload(ctx, p); err != nil {
h.logAndSendError(ctx, w, "could create multipart upload", reqInfo, err, additional...)
return
}
@ -173,17 +174,18 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
}
if err = middleware.EncodeToResponse(w, resp); err != nil {
h.logAndSendError(w, "could not encode InitiateMultipartUploadResponse to response", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "could not encode InitiateMultipartUploadResponse to response", reqInfo, err, additional...)
return
}
}
func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
@ -196,20 +198,17 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
partNumber, err := strconv.Atoi(partNumStr)
if err != nil || partNumber < layer.UploadMinPartNumber || partNumber > layer.UploadMaxPartNumber {
h.logAndSendError(w, "invalid part number", reqInfo, errors.GetAPIError(errors.ErrInvalidPartNumber), additional...)
h.logAndSendError(ctx, w, "invalid part number", reqInfo, errors.GetAPIError(errors.ErrInvalidPartNumber), additional...)
return
}
body, err := h.getBodyReader(r)
if err != nil {
h.logAndSendError(w, "failed to get body reader", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "failed to get body reader", reqInfo, err, additional...)
return
}
var size uint64
if r.ContentLength > 0 {
size = uint64(r.ContentLength)
}
size := h.getPutPayloadSize(r)
p := &layer.UploadPartParams{
Info: &layer.UploadInfoParams{
@ -226,13 +225,13 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
p.Info.Encryption, err = formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "invalid sse headers", reqInfo, err, additional...)
return
}
hash, err := h.obj.UploadPart(r.Context(), p)
hash, err := h.obj.UploadPart(ctx, p)
if err != nil {
h.logAndSendError(w, "could not upload a part", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "could not upload a part", reqInfo, err, additional...)
return
}
@ -242,7 +241,7 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set(api.ETag, data.Quote(hash))
if err = middleware.WriteSuccessResponseHeadersOnly(w); err != nil {
h.logAndSendError(w, "write response", reqInfo, err)
h.logAndSendError(ctx, w, "write response", reqInfo, err)
return
}
}
@ -260,7 +259,7 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
partNumber, err := strconv.Atoi(partNumStr)
if err != nil || partNumber < layer.UploadMinPartNumber || partNumber > layer.UploadMaxPartNumber {
h.logAndSendError(w, "invalid part number", reqInfo, errors.GetAPIError(errors.ErrInvalidPartNumber), additional...)
h.logAndSendError(ctx, w, "invalid part number", reqInfo, errors.GetAPIError(errors.ErrInvalidPartNumber), additional...)
return
}
@ -271,26 +270,26 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
}
srcBucket, srcObject, err := path2BucketObject(src)
if err != nil {
h.logAndSendError(w, "invalid source copy", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "invalid source copy", reqInfo, err, additional...)
return
}
srcRange, err := parseRange(r.Header.Get(api.AmzCopySourceRange))
if err != nil {
h.logAndSendError(w, "could not parse copy range", reqInfo,
h.logAndSendError(ctx, w, "could not parse copy range", reqInfo,
errors.GetAPIError(errors.ErrInvalidCopyPartRange), additional...)
return
}
srcBktInfo, err := h.getBucketAndCheckOwner(r, srcBucket, api.AmzSourceExpectedBucketOwner)
if err != nil {
h.logAndSendError(w, "could not get source bucket info", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "could not get source bucket info", reqInfo, err, additional...)
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get target bucket info", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "could not get target bucket info", reqInfo, err, additional...)
return
}
@ -303,35 +302,35 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
srcInfo, err := h.obj.GetObjectInfo(ctx, headPrm)
if err != nil {
if errors.IsS3Error(err, errors.ErrNoSuchKey) && versionID != "" {
h.logAndSendError(w, "could not head source object version", reqInfo,
h.logAndSendError(ctx, w, "could not head source object version", reqInfo,
errors.GetAPIError(errors.ErrBadRequest), additional...)
return
}
h.logAndSendError(w, "could not head source object", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "could not head source object", reqInfo, err, additional...)
return
}
args, err := parseCopyObjectArgs(r.Header)
if err != nil {
h.logAndSendError(w, "could not parse copy object args", reqInfo,
h.logAndSendError(ctx, w, "could not parse copy object args", reqInfo,
errors.GetAPIError(errors.ErrInvalidCopyPartRange), additional...)
return
}
if err = checkPreconditions(srcInfo, args.Conditional, h.cfg.MD5Enabled()); err != nil {
h.logAndSendError(w, "precondition failed", reqInfo, errors.GetAPIError(errors.ErrPreconditionFailed),
h.logAndSendError(ctx, w, "precondition failed", reqInfo, errors.GetAPIError(errors.ErrPreconditionFailed),
additional...)
return
}
srcEncryptionParams, err := formCopySourceEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
h.logAndSendError(ctx, w, "invalid sse headers", reqInfo, err)
return
}
if err = srcEncryptionParams.MatchObjectEncryption(layer.FormEncryptionInfo(srcInfo.Headers)); err != nil {
h.logAndSendError(w, "encryption doesn't match object", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrBadRequest), err), additional...)
h.logAndSendError(ctx, w, "encryption doesn't match object", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrBadRequest), err), additional...)
return
}
@ -351,13 +350,13 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
p.Info.Encryption, err = formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "invalid sse headers", reqInfo, err, additional...)
return
}
info, err := h.obj.UploadPartCopy(ctx, p)
if err != nil {
h.logAndSendError(w, "could not upload part copy", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "could not upload part copy", reqInfo, err, additional...)
return
}
@ -371,22 +370,23 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
}
if err = middleware.EncodeToResponse(w, response); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err, additional...)
}
}
func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket settings", reqInfo, err)
return
}
@ -402,12 +402,12 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
reqBody := new(CompleteMultipartUpload)
if err = h.cfg.NewXMLDecoder(r.Body).Decode(reqBody); err != nil {
h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
h.logAndSendError(ctx, w, "could not read complete multipart upload xml", reqInfo,
fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()), additional...)
return
}
if len(reqBody.Parts) == 0 {
h.logAndSendError(w, "invalid xml with parts", reqInfo, errors.GetAPIError(errors.ErrMalformedXML), additional...)
h.logAndSendError(ctx, w, "invalid xml with parts", reqInfo, errors.GetAPIError(errors.ErrMalformedXML), additional...)
return
}
@ -421,7 +421,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
objInfo, err := h.completeMultipartUpload(r, c, bktInfo)
if err != nil {
h.logAndSendError(w, "complete multipart error", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "complete multipart error", reqInfo, err, additional...)
return
}
@ -429,7 +429,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
Bucket: objInfo.Bucket,
Key: objInfo.Name,
ETag: data.Quote(objInfo.ETag(h.cfg.MD5Enabled())),
Location: getObjectLocation(r, h.cfg.Domains(), reqInfo.BucketName, reqInfo.ObjectName),
Location: getObjectLocation(r, reqInfo.BucketName, reqInfo.ObjectName, reqInfo.RequestVHSEnabled),
}
if settings.VersioningEnabled() {
@ -437,7 +437,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
}
if err = middleware.EncodeToResponse(w, response); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err, additional...)
}
}
@ -450,7 +450,7 @@ func getURLScheme(r *http.Request) string {
}
// getObjectLocation gets the fully qualified URL of an object.
func getObjectLocation(r *http.Request, domains []string, bucket, object string) string {
func getObjectLocation(r *http.Request, bucket, object string, vhsEnabled bool) string {
proto := middleware.GetSourceScheme(r)
if proto == "" {
proto = getURLScheme(r)
@ -460,13 +460,12 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
Path: path.Join("/", bucket, object),
Scheme: proto,
}
// If domain is set then we need to use bucket DNS style.
for _, domain := range domains {
if strings.HasPrefix(r.Host, bucket+"."+domain) {
u.Path = path.Join("/", object)
break
}
// If vhs enabled then we need to use bucket DNS style.
if vhsEnabled {
u.Path = path.Join("/", object)
}
return u.String()
}
@ -497,11 +496,12 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
}
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
@ -514,7 +514,7 @@ func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Req
if maxUploadsStr != "" {
val, err := strconv.Atoi(maxUploadsStr)
if err != nil || val < 1 || val > 1000 {
h.logAndSendError(w, "invalid maxUploads", reqInfo, errors.GetAPIError(errors.ErrInvalidMaxUploads))
h.logAndSendError(ctx, w, "invalid maxUploads", reqInfo, errors.GetAPIError(errors.ErrInvalidMaxUploads))
return
}
maxUploads = val
@ -530,23 +530,29 @@ func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Req
UploadIDMarker: queryValues.Get(uploadIDMarkerQueryName),
}
list, err := h.obj.ListMultipartUploads(r.Context(), p)
if p.EncodingType != "" && strings.ToLower(p.EncodingType) != urlEncodingType {
h.logAndSendError(ctx, w, "invalid encoding type", reqInfo, errors.GetAPIError(errors.ErrInvalidEncodingMethod))
return
}
list, err := h.obj.ListMultipartUploads(ctx, p)
if err != nil {
h.logAndSendError(w, "could not list multipart uploads", reqInfo, err)
h.logAndSendError(ctx, w, "could not list multipart uploads", reqInfo, err)
return
}
if err = middleware.EncodeToResponse(w, encodeListMultipartUploadsToResponse(list, p)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}
func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
@ -555,14 +561,14 @@ func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
queryValues = reqInfo.URL.Query()
uploadID = queryValues.Get(uploadIDHeaderName)
additional = []zap.Field{zap.String("uploadID", uploadID), zap.String("Key", reqInfo.ObjectName)}
additional = []zap.Field{zap.String("uploadID", uploadID)}
maxParts = layer.MaxSizePartsList
)
if queryValues.Get("max-parts") != "" {
val, err := strconv.Atoi(queryValues.Get("max-parts"))
if err != nil || val < 0 {
h.logAndSendError(w, "invalid MaxParts", reqInfo, errors.GetAPIError(errors.ErrInvalidMaxParts), additional...)
h.logAndSendError(ctx, w, "invalid MaxParts", reqInfo, errors.GetAPIError(errors.ErrInvalidMaxParts), additional...)
return
}
if val < layer.MaxSizePartsList {
@ -572,7 +578,7 @@ func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
if queryValues.Get("part-number-marker") != "" {
if partNumberMarker, err = strconv.Atoi(queryValues.Get("part-number-marker")); err != nil || partNumberMarker < 0 {
h.logAndSendError(w, "invalid PartNumberMarker", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "invalid PartNumberMarker", reqInfo, err, additional...)
return
}
}
@ -589,32 +595,33 @@ func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
p.Info.Encryption, err = formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
h.logAndSendError(ctx, w, "invalid sse headers", reqInfo, err)
return
}
list, err := h.obj.ListParts(r.Context(), p)
list, err := h.obj.ListParts(ctx, p)
if err != nil {
h.logAndSendError(w, "could not list parts", reqInfo, err, additional...)
h.logAndSendError(ctx, w, "could not list parts", reqInfo, err, additional...)
return
}
if err = middleware.EncodeToResponse(w, encodeListPartsToResponse(list, p)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}
func (h *handler) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
uploadID := reqInfo.URL.Query().Get(uploadIDHeaderName)
additional := []zap.Field{zap.String("uploadID", uploadID), zap.String("Key", reqInfo.ObjectName)}
additional := []zap.Field{zap.String("uploadID", uploadID)}
p := &layer.UploadInfoParams{
UploadID: uploadID,
@ -624,12 +631,12 @@ func (h *handler) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Req
p.Encryption, err = formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
h.logAndSendError(ctx, w, "invalid sse headers", reqInfo, err)
return
}
if err = h.obj.AbortMultipartUpload(r.Context(), p); err != nil {
h.logAndSendError(w, "could not abort multipart upload", reqInfo, err, additional...)
if err = h.obj.AbortMultipartUpload(ctx, p); err != nil {
h.logAndSendError(ctx, w, "could not abort multipart upload", reqInfo, err, additional...)
return
}
@ -640,14 +647,14 @@ func encodeListMultipartUploadsToResponse(info *layer.ListMultipartUploadsInfo,
res := ListMultipartUploadsResponse{
Bucket: params.Bkt.Name,
CommonPrefixes: fillPrefixes(info.Prefixes, params.EncodingType),
Delimiter: params.Delimiter,
Delimiter: s3PathEncode(params.Delimiter, params.EncodingType),
EncodingType: params.EncodingType,
IsTruncated: info.IsTruncated,
KeyMarker: params.KeyMarker,
KeyMarker: s3PathEncode(params.KeyMarker, params.EncodingType),
MaxUploads: params.MaxUploads,
NextKeyMarker: info.NextKeyMarker,
NextKeyMarker: s3PathEncode(info.NextKeyMarker, params.EncodingType),
NextUploadIDMarker: info.NextUploadIDMarker,
Prefix: params.Prefix,
Prefix: s3PathEncode(params.Prefix, params.EncodingType),
UploadIDMarker: params.UploadIDMarker,
}
@ -659,7 +666,7 @@ func encodeListMultipartUploadsToResponse(info *layer.ListMultipartUploadsInfo,
ID: u.Owner.String(),
DisplayName: u.Owner.String(),
},
Key: u.Key,
Key: s3PathEncode(u.Key, params.EncodingType),
Owner: Owner{
ID: u.Owner.String(),
DisplayName: u.Owner.String(),

View file

@ -2,21 +2,27 @@ package handler
import (
"crypto/md5"
"crypto/rand"
"crypto/tls"
"encoding/base64"
"encoding/hex"
"encoding/xml"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3Errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
"github.com/stretchr/testify/require"
)
@ -35,7 +41,7 @@ func TestMultipartUploadInvalidPart(t *testing.T) {
etag1, _ := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 1, partSize)
etag2, _ := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 2, partSize)
w := completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1, etag2})
assertS3Error(hc.t, w, s3Errors.GetAPIError(s3Errors.ErrEntityTooSmall))
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrEntityTooSmall))
}
func TestDeleteMultipartAllParts(t *testing.T) {
@ -99,7 +105,7 @@ func TestMultipartReUploadPart(t *testing.T) {
require.Equal(t, etag2, list.Parts[1].ETag)
w := completeMultipartUploadBase(hc, bktName, objName, uploadInfo.UploadID, []string{etag1, etag2})
assertS3Error(hc.t, w, s3Errors.GetAPIError(s3Errors.ErrEntityTooSmall))
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrEntityTooSmall))
etag1, data1 := uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSizeFirst)
etag2, data2 := uploadPart(hc, bktName, objName, uploadInfo.UploadID, 2, partSizeLast)
@ -122,6 +128,108 @@ func TestMultipartReUploadPart(t *testing.T) {
equalDataSlices(t, append(data1, data2...), data)
}
func TestMultipartRemovePartsSplit(t *testing.T) {
bktName, objName := "bucket-to-upload-part", "object-multipart"
partSize := 8
t.Run("reupload part", func(t *testing.T) {
hc := prepareHandlerContext(t)
bktInfo := createTestBucket(hc, bktName)
uploadInfo := createMultipartUpload(hc, bktName, objName, map[string]string{})
uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSize)
multipartInfo, err := hc.tree.GetMultipartUpload(hc.Context(), bktInfo, uploadInfo.Key, uploadInfo.UploadID)
require.NoError(t, err)
objID := oidtest.ID()
_, err = hc.treeMock.AddNode(hc.Context(), bktInfo, "system", multipartInfo.ID, map[string]string{
"Number": "1",
"OID": objID.EncodeToString(),
"Owner": usertest.ID().EncodeToString(),
"ETag": "etag",
})
require.NoError(t, err)
hc.tp.AddObject(bktInfo.CID.EncodeToString()+"/"+objID.EncodeToString(), object.New())
require.Len(t, hc.tp.Objects(), 2)
list := listParts(hc, bktName, objName, uploadInfo.UploadID, "0", http.StatusOK)
require.Len(t, list.Parts, 1)
require.Equal(t, `"etag"`, list.Parts[0].ETag)
etag1, _ := uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSize)
list = listParts(hc, bktName, objName, uploadInfo.UploadID, "0", http.StatusOK)
require.Len(t, list.Parts, 1)
require.Equal(t, etag1, list.Parts[0].ETag)
require.Len(t, hc.tp.Objects(), 1)
})
t.Run("abort multipart", func(t *testing.T) {
hc := prepareHandlerContext(t)
bktInfo := createTestBucket(hc, bktName)
uploadInfo := createMultipartUpload(hc, bktName, objName, map[string]string{})
uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSize)
multipartInfo, err := hc.tree.GetMultipartUpload(hc.Context(), bktInfo, uploadInfo.Key, uploadInfo.UploadID)
require.NoError(t, err)
objID := oidtest.ID()
_, err = hc.treeMock.AddNode(hc.Context(), bktInfo, "system", multipartInfo.ID, map[string]string{
"Number": "1",
"OID": objID.EncodeToString(),
"Owner": usertest.ID().EncodeToString(),
"ETag": "etag",
})
require.NoError(t, err)
hc.tp.AddObject(bktInfo.CID.EncodeToString()+"/"+objID.EncodeToString(), object.New())
require.Len(t, hc.tp.Objects(), 2)
abortMultipartUpload(hc, bktName, objName, uploadInfo.UploadID)
require.Empty(t, hc.tp.Objects())
})
t.Run("complete multipart", func(t *testing.T) {
hc := prepareHandlerContext(t)
bktInfo := createTestBucket(hc, bktName)
uploadInfo := createMultipartUpload(hc, bktName, objName, map[string]string{})
etag1, _ := uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSize)
multipartInfo, err := hc.tree.GetMultipartUpload(hc.Context(), bktInfo, uploadInfo.Key, uploadInfo.UploadID)
require.NoError(t, err)
objID := oidtest.ID()
_, err = hc.treeMock.AddNode(hc.Context(), bktInfo, "system", multipartInfo.ID, map[string]string{
"Number": "1",
"OID": objID.EncodeToString(),
"Owner": usertest.ID().EncodeToString(),
"ETag": "etag",
})
require.NoError(t, err)
hc.tp.AddObject(bktInfo.CID.EncodeToString()+"/"+objID.EncodeToString(), object.New())
require.Len(t, hc.tp.Objects(), 2)
completeMultipartUpload(hc, bktName, objName, uploadInfo.UploadID, []string{etag1})
require.Falsef(t, containsOID(hc.tp.Objects(), objID), "frostfs contains '%s' object, but shouldn't", objID)
})
}
func containsOID(objects []*object.Object, objID oid.ID) bool {
for _, o := range objects {
oID, _ := o.ID()
if oID.Equals(objID) {
return true
}
}
return false
}
func TestListMultipartUploads(t *testing.T) {
hc := prepareHandlerContext(t)
@ -145,14 +253,14 @@ func TestListMultipartUploads(t *testing.T) {
})
t.Run("check max uploads", func(t *testing.T) {
listUploads := listMultipartUploadsBase(hc, bktName, "", "", "", "", 2)
listUploads := listMultipartUploads(hc, bktName, "", "", "", "", 2)
require.Len(t, listUploads.Uploads, 2)
require.Equal(t, uploadInfo1.UploadID, listUploads.Uploads[0].UploadID)
require.Equal(t, uploadInfo2.UploadID, listUploads.Uploads[1].UploadID)
})
t.Run("check prefix", func(t *testing.T) {
listUploads := listMultipartUploadsBase(hc, bktName, "/my", "", "", "", -1)
listUploads := listMultipartUploads(hc, bktName, "/my", "", "", "", -1)
require.Len(t, listUploads.Uploads, 2)
require.Equal(t, uploadInfo1.UploadID, listUploads.Uploads[0].UploadID)
require.Equal(t, uploadInfo2.UploadID, listUploads.Uploads[1].UploadID)
@ -160,7 +268,7 @@ func TestListMultipartUploads(t *testing.T) {
t.Run("check markers", func(t *testing.T) {
t.Run("check only key-marker", func(t *testing.T) {
listUploads := listMultipartUploadsBase(hc, bktName, "", "", "", objName2, -1)
listUploads := listMultipartUploads(hc, bktName, "", "", "", objName2, -1)
require.Len(t, listUploads.Uploads, 1)
// If upload-id-marker is not specified, only the keys lexicographically greater than the specified key-marker will be included in the list.
require.Equal(t, uploadInfo3.UploadID, listUploads.Uploads[0].UploadID)
@ -171,7 +279,7 @@ func TestListMultipartUploads(t *testing.T) {
if uploadIDMarker > uploadInfo2.UploadID {
uploadIDMarker = uploadInfo2.UploadID
}
listUploads := listMultipartUploadsBase(hc, bktName, "", "", uploadIDMarker, "", -1)
listUploads := listMultipartUploads(hc, bktName, "", "", uploadIDMarker, "", -1)
// If key-marker is not specified, the upload-id-marker parameter is ignored.
require.Len(t, listUploads.Uploads, 3)
})
@ -179,7 +287,7 @@ func TestListMultipartUploads(t *testing.T) {
t.Run("check key-marker along with upload-id-marker", func(t *testing.T) {
uploadIDMarker := "00000000-0000-0000-0000-000000000000"
listUploads := listMultipartUploadsBase(hc, bktName, "", "", uploadIDMarker, objName3, -1)
listUploads := listMultipartUploads(hc, bktName, "", "", uploadIDMarker, objName3, -1)
require.Len(t, listUploads.Uploads, 1)
// If upload-id-marker is specified, any multipart uploads for a key equal to the key-marker might also be included,
// provided those multipart uploads have upload IDs lexicographically greater than the specified upload-id-marker.
@ -414,7 +522,7 @@ func TestUploadPartCheckContentSHA256(t *testing.T) {
r.Header.Set(api.AmzContentSha256, tc.hash)
hc.Handler().UploadPartHandler(w, r)
if tc.error {
assertS3Error(t, w, s3Errors.GetAPIError(s3Errors.ErrContentSHA256Mismatch))
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch))
list := listParts(hc, bktName, objName, multipartUpload.UploadID, "0", http.StatusOK)
require.Len(t, list.Parts, 1)
@ -443,11 +551,11 @@ func TestUploadPartCheckContentSHA256(t *testing.T) {
func TestMultipartObjectLocation(t *testing.T) {
for _, tc := range []struct {
req *http.Request
bucket string
object string
domains []string
expected string
req *http.Request
bucket string
object string
vhsEnabled bool
expected string
}{
{
req: &http.Request{
@ -492,29 +600,96 @@ func TestMultipartObjectLocation(t *testing.T) {
req: &http.Request{
Host: "mybucket.s3dev.frostfs.devenv",
},
domains: []string{"s3dev.frostfs.devenv"},
bucket: "mybucket",
object: "test/1.txt",
expected: "http://mybucket.s3dev.frostfs.devenv/test/1.txt",
bucket: "mybucket",
object: "test/1.txt",
vhsEnabled: true,
expected: "http://mybucket.s3dev.frostfs.devenv/test/1.txt",
},
{
req: &http.Request{
Host: "mybucket.s3dev.frostfs.devenv",
Header: map[string][]string{"X-Forwarded-Scheme": {"https"}},
},
domains: []string{"s3dev.frostfs.devenv"},
bucket: "mybucket",
object: "test/1.txt",
expected: "https://mybucket.s3dev.frostfs.devenv/test/1.txt",
bucket: "mybucket",
object: "test/1.txt",
vhsEnabled: true,
expected: "https://mybucket.s3dev.frostfs.devenv/test/1.txt",
},
} {
t.Run("", func(t *testing.T) {
location := getObjectLocation(tc.req, tc.domains, tc.bucket, tc.object)
location := getObjectLocation(tc.req, tc.bucket, tc.object, tc.vhsEnabled)
require.Equal(t, tc.expected, location)
})
}
}
func TestUploadPartWithNegativeContentLength(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName := "bucket-to-upload-part", "object-multipart"
createTestBucket(hc, bktName)
partSize := 5 * 1024 * 1024
multipartUpload := createMultipartUpload(hc, bktName, objName, map[string]string{})
partBody := make([]byte, partSize)
_, err := rand.Read(partBody)
require.NoError(hc.t, err)
query := make(url.Values)
query.Set(uploadIDQuery, multipartUpload.UploadID)
query.Set(partNumberQuery, "1")
w, r := prepareTestRequestWithQuery(hc, bktName, objName, query, partBody)
r.ContentLength = -1
hc.Handler().UploadPartHandler(w, r)
assertStatus(hc.t, w, http.StatusOK)
completeMultipartUpload(hc, bktName, objName, multipartUpload.UploadID, []string{w.Header().Get(api.ETag)})
res, _ := getObject(hc, bktName, objName)
equalDataSlices(t, partBody, res)
resp := getObjectAttributes(hc, bktName, objName, objectParts)
require.Len(t, resp.ObjectParts.Parts, 1)
require.Equal(t, partSize, resp.ObjectParts.Parts[0].Size)
}
func TestListMultipartUploadsEncoding(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-to-list-uploads-encoding"
createTestBucket(hc, bktName)
listAllMultipartUploadsErr(hc, bktName, "invalid", apierr.GetAPIError(apierr.ErrInvalidEncodingMethod))
objects := []string{"foo()/bar", "foo()/bar/xyzzy", "asdf+b"}
for _, objName := range objects {
createMultipartUpload(hc, bktName, objName, nil)
}
listResponse := listMultipartUploadsURL(hc, bktName, "foo(", ")", "", "", -1)
require.Len(t, listResponse.CommonPrefixes, 1)
require.Equal(t, "foo%28%29", listResponse.CommonPrefixes[0].Prefix)
require.Equal(t, "foo%28", listResponse.Prefix)
require.Equal(t, "%29", listResponse.Delimiter)
require.Equal(t, "url", listResponse.EncodingType)
require.Equal(t, maxObjectList, listResponse.MaxUploads)
listResponse = listMultipartUploads(hc, bktName, "", "", "", "", 1)
require.Empty(t, listResponse.EncodingType)
listResponse = listMultipartUploadsURL(hc, bktName, "", "", "", listResponse.NextKeyMarker, 1)
require.Len(t, listResponse.CommonPrefixes, 0)
require.Len(t, listResponse.Uploads, 1)
require.Equal(t, "foo%28%29/bar", listResponse.Uploads[0].Key)
require.Equal(t, "asdf%2Bb", listResponse.KeyMarker)
require.Equal(t, "foo%28%29/bar", listResponse.NextKeyMarker)
require.Equal(t, "url", listResponse.EncodingType)
require.Equal(t, 1, listResponse.MaxUploads)
}
func uploadPartCopy(hc *handlerContext, bktName, objName, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
return uploadPartCopyBase(hc, bktName, objName, false, uploadID, num, srcObj, start, end)
}
@ -540,16 +715,42 @@ func uploadPartCopyBase(hc *handlerContext, bktName, objName string, encrypted b
return uploadPartCopyResponse
}
func listAllMultipartUploads(hc *handlerContext, bktName string) *ListMultipartUploadsResponse {
return listMultipartUploadsBase(hc, bktName, "", "", "", "", -1)
func listMultipartUploads(hc *handlerContext, bktName, prefix, delimiter, uploadIDMarker, keyMarker string, maxUploads int) *ListMultipartUploadsResponse {
w := listMultipartUploadsBase(hc, bktName, prefix, delimiter, uploadIDMarker, keyMarker, "", maxUploads)
assertStatus(hc.t, w, http.StatusOK)
res := &ListMultipartUploadsResponse{}
parseTestResponse(hc.t, w, res)
return res
}
func listMultipartUploadsBase(hc *handlerContext, bktName, prefix, delimiter, uploadIDMarker, keyMarker string, maxUploads int) *ListMultipartUploadsResponse {
func listMultipartUploadsURL(hc *handlerContext, bktName, prefix, delimiter, uploadIDMarker, keyMarker string, maxUploads int) *ListMultipartUploadsResponse {
w := listMultipartUploadsBase(hc, bktName, prefix, delimiter, uploadIDMarker, keyMarker, urlEncodingType, maxUploads)
assertStatus(hc.t, w, http.StatusOK)
res := &ListMultipartUploadsResponse{}
parseTestResponse(hc.t, w, res)
return res
}
func listAllMultipartUploads(hc *handlerContext, bktName string) *ListMultipartUploadsResponse {
w := listMultipartUploadsBase(hc, bktName, "", "", "", "", "", -1)
assertStatus(hc.t, w, http.StatusOK)
res := &ListMultipartUploadsResponse{}
parseTestResponse(hc.t, w, res)
return res
}
func listAllMultipartUploadsErr(hc *handlerContext, bktName, encoding string, err apierr.Error) {
w := listMultipartUploadsBase(hc, bktName, "", "", "", "", encoding, -1)
assertS3Error(hc.t, w, err)
}
func listMultipartUploadsBase(hc *handlerContext, bktName, prefix, delimiter, uploadIDMarker, keyMarker, encoding string, maxUploads int) *httptest.ResponseRecorder {
query := make(url.Values)
query.Set(prefixQueryName, prefix)
query.Set(delimiterQueryName, delimiter)
query.Set(uploadIDMarkerQueryName, uploadIDMarker)
query.Set(keyMarkerQueryName, keyMarker)
query.Set(encodingTypeQueryName, encoding)
if maxUploads != -1 {
query.Set(maxUploadsQueryName, strconv.Itoa(maxUploads))
}
@ -557,10 +758,7 @@ func listMultipartUploadsBase(hc *handlerContext, bktName, prefix, delimiter, up
w, r := prepareTestRequestWithQuery(hc, bktName, "", query, nil)
hc.Handler().ListMultipartUploadsHandler(w, r)
listPartsResponse := &ListMultipartUploadsResponse{}
readResponse(hc.t, w, http.StatusOK, listPartsResponse)
return listPartsResponse
return w
}
func listParts(hc *handlerContext, bktName, objName string, uploadID, partNumberMarker string, status int) *ListPartsResponse {

View file

@ -7,10 +7,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
)
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
}
func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
h.logAndSendError(r.Context(), w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
}

View file

@ -4,6 +4,7 @@ import (
"net/http"
"net/url"
"strconv"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
@ -16,26 +17,27 @@ import (
// ListObjectsV1Handler handles objects listing requests for API version 1.
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
params, err := parseListObjectsArgsV1(reqInfo)
if err != nil {
h.logAndSendError(w, "failed to parse arguments", reqInfo, err)
h.logAndSendError(ctx, w, "failed to parse arguments", reqInfo, err)
return
}
if params.BktInfo, err = h.getBucketAndCheckOwner(r, reqInfo.BucketName); err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
list, err := h.obj.ListObjectsV1(r.Context(), params)
list, err := h.obj.ListObjectsV1(ctx, params)
if err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
return
}
if err = middleware.EncodeToResponse(w, h.encodeV1(params, list)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}
@ -60,26 +62,27 @@ func (h *handler) encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjects
// ListObjectsV2Handler handles objects listing requests for API version 2.
func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
params, err := parseListObjectsArgsV2(reqInfo)
if err != nil {
h.logAndSendError(w, "failed to parse arguments", reqInfo, err)
h.logAndSendError(ctx, w, "failed to parse arguments", reqInfo, err)
return
}
if params.BktInfo, err = h.getBucketAndCheckOwner(r, reqInfo.BucketName); err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
list, err := h.obj.ListObjectsV2(r.Context(), params)
list, err := h.obj.ListObjectsV2(ctx, params)
if err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
return
}
if err = middleware.EncodeToResponse(w, h.encodeV2(params, list)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}
@ -153,6 +156,10 @@ func parseListObjectArgs(reqInfo *middleware.ReqInfo) (*layer.ListObjectsParamsC
res.Delimiter = queryValues.Get("delimiter")
res.Encode = queryValues.Get("encoding-type")
if res.Encode != "" && strings.ToLower(res.Encode) != urlEncodingType {
return nil, errors.GetAPIError(errors.ErrInvalidEncodingMethod)
}
if queryValues.Get("max-keys") == "" {
res.MaxKeys = maxObjectList
} else if res.MaxKeys, err = strconv.Atoi(queryValues.Get("max-keys")); err != nil || res.MaxKeys < 0 {
@ -214,27 +221,28 @@ func fillContents(src []*data.ExtendedNodeVersion, encode string, fetchOwner, md
}
func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
p, err := parseListObjectVersionsRequest(reqInfo)
if err != nil {
h.logAndSendError(w, "failed to parse request", reqInfo, err)
h.logAndSendError(ctx, w, "failed to parse request", reqInfo, err)
return
}
if p.BktInfo, err = h.getBucketAndCheckOwner(r, reqInfo.BucketName); err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
info, err := h.obj.ListObjectVersions(r.Context(), p)
info, err := h.obj.ListObjectVersions(ctx, p)
if err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
return
}
response := encodeListObjectVersionsToResponse(p, info, p.BktInfo.Name, h.cfg.MD5Enabled())
if err = middleware.EncodeToResponse(w, response); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}
@ -257,6 +265,10 @@ func parseListObjectVersionsRequest(reqInfo *middleware.ReqInfo) (*layer.ListObj
res.Encode = queryValues.Get("encoding-type")
res.VersionIDMarker = queryValues.Get("version-id-marker")
if res.Encode != "" && strings.ToLower(res.Encode) != urlEncodingType {
return nil, errors.GetAPIError(errors.ErrInvalidEncodingMethod)
}
if res.VersionIDMarker != "" && res.KeyMarker == "" {
return nil, errors.GetAPIError(errors.VersionIDMarkerWithoutKeyMarker)
}

View file

@ -4,19 +4,25 @@ import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"sort"
"strconv"
"strings"
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
"go.uber.org/zap/zaptest/observer"
)
func TestParseContinuationToken(t *testing.T) {
@ -93,8 +99,35 @@ func TestListObjectsWithOldTreeNodes(t *testing.T) {
checkListVersionsOldNodes(hc, listVers.Version, objInfos)
}
func TestListObjectsVersionsSkipLogTaggingNodesError(t *testing.T) {
loggerCore, observedLog := observer.New(zap.DebugLevel)
log := zap.New(loggerCore)
hcBase, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(log))
require.NoError(t, err)
hc := &handlerContext{
handlerContextBase: hcBase,
t: t,
}
bktName, objName := "bucket-versioning-enabled", "versions/object"
bktInfo := createTestBucket(hc, bktName)
createTestObject(hc, bktInfo, objName, encryption.Params{})
createTestObject(hc, bktInfo, objName, encryption.Params{})
putObjectTagging(hc.t, hc, bktName, objName, map[string]string{"tag1": "val1"})
listObjectsVersions(hc, bktName, "", "", "", "", -1)
filtered := observedLog.Filter(func(entry observer.LoggedEntry) bool {
return strings.Contains(entry.Message, logs.ParseTreeNode)
})
require.Empty(t, filtered)
}
func makeAllTreeObjectsOld(hc *handlerContext, bktInfo *data.BucketInfo) {
nodes, err := hc.treeMock.GetSubTree(hc.Context(), bktInfo, "version", []uint64{0}, 0)
nodes, err := hc.treeMock.GetSubTree(hc.Context(), bktInfo, "version", []uint64{0}, 0, true)
require.NoError(hc.t, err)
for _, node := range nodes {
@ -138,11 +171,17 @@ func checkListVersionsOldNodes(hc *handlerContext, list []ObjectVersionResponse,
}
func TestListObjectsContextCanceled(t *testing.T) {
layerCfg := layer.DefaultCachesConfigs(zaptest.NewLogger(t))
log := zaptest.NewLogger(t)
layerCfg := layer.DefaultCachesConfigs(log)
layerCfg.SessionList.Lifetime = time.Hour
layerCfg.SessionList.Size = 1
hc := prepareHandlerContextBase(t, layerCfg)
hcBase, err := prepareHandlerContextBase(layerCfg)
require.NoError(t, err)
hc := &handlerContext{
handlerContextBase: hcBase,
t: t,
}
bktName := "bucket-versioning-enabled"
bktInfo := createTestBucket(hc, bktName)
@ -642,6 +681,80 @@ func TestS3BucketListDelimiterNotSkipSpecial(t *testing.T) {
}
}
func TestS3BucketListMarkerUnreadable(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-for-listing"
bktInfo := createTestBucket(hc, bktName)
objects := []string{"bar", "baz", "foo", "quxx"}
for _, objName := range objects {
createTestObject(hc, bktInfo, objName, encryption.Params{})
}
list := listObjectsV1(hc, bktName, "", "", "\x0a", -1)
require.Equal(t, "\x0a", list.Marker)
require.False(t, list.IsTruncated)
require.Len(t, list.Contents, len(objects))
for i := 0; i < len(list.Contents); i++ {
require.Equal(t, objects[i], list.Contents[i].Key)
}
}
func TestS3BucketListMarkerNotInList(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-for-listing"
bktInfo := createTestBucket(hc, bktName)
objects := []string{"bar", "baz", "foo", "quxx"}
for _, objName := range objects {
createTestObject(hc, bktInfo, objName, encryption.Params{})
}
list := listObjectsV1(hc, bktName, "", "", "blah", -1)
require.Equal(t, "blah", list.Marker)
expected := []string{"foo", "quxx"}
require.Len(t, list.Contents, len(expected))
for i := 0; i < len(list.Contents); i++ {
require.Equal(t, expected[i], list.Contents[i].Key)
}
}
func TestListTruncatedCacheHit(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-for-listing"
bktInfo := createTestBucket(hc, bktName)
objects := []string{"bar", "baz", "foo", "quxx"}
for _, objName := range objects {
createTestObject(hc, bktInfo, objName, encryption.Params{})
}
list := listObjectsV1(hc, bktName, "", "", "", 2)
require.True(t, list.IsTruncated)
require.Len(t, list.Contents, 2)
for i := 0; i < len(list.Contents); i++ {
require.Equal(t, objects[i], list.Contents[i].Key)
}
cacheKey := cache.CreateListSessionCacheKey(bktInfo.CID, "", list.NextMarker)
list = listObjectsV1(hc, bktName, "", "", list.NextMarker, 2)
require.Nil(t, hc.cache.GetListSession(hc.owner, cacheKey))
require.False(t, list.IsTruncated)
require.Len(t, list.Contents, 2)
for i := 0; i < len(list.Contents); i++ {
require.Equal(t, objects[i+2], list.Contents[i].Key)
}
}
func TestMintVersioningListObjectVersionsVersionIDContinuation(t *testing.T) {
hc := prepareHandlerContext(t)
@ -718,6 +831,16 @@ func TestListObjectVersionsEncoding(t *testing.T) {
require.Equal(t, 3, listResponse.MaxKeys)
}
func TestListingsWithInvalidEncodingType(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-for-listing-invalid-encoding"
createTestBucket(hc, bktName)
listObjectsVersionsErr(hc, bktName, "invalid", apierr.GetAPIError(apierr.ErrInvalidEncodingMethod))
listObjectsV2Err(hc, bktName, "invalid", apierr.GetAPIError(apierr.ErrInvalidEncodingMethod))
listObjectsV1Err(hc, bktName, "invalid", apierr.GetAPIError(apierr.ErrInvalidEncodingMethod))
}
func checkVersionsNames(t *testing.T, versions *ListObjectsVersionsResponse, names []string) {
for i, v := range versions.Version {
require.Equal(t, names[i], v.Key)
@ -725,10 +848,19 @@ func checkVersionsNames(t *testing.T, versions *ListObjectsVersionsResponse, nam
}
func listObjectsV2(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
return listObjectsV2Ext(hc, bktName, prefix, delimiter, startAfter, continuationToken, "", maxKeys)
w := listObjectsV2Base(hc, bktName, prefix, delimiter, startAfter, continuationToken, "", maxKeys)
assertStatus(hc.t, w, http.StatusOK)
res := &ListObjectsV2Response{}
parseTestResponse(hc.t, w, res)
return res
}
func listObjectsV2Ext(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken, encodingType string, maxKeys int) *ListObjectsV2Response {
func listObjectsV2Err(hc *handlerContext, bktName, encoding string, err apierr.Error) {
w := listObjectsV2Base(hc, bktName, "", "", "", "", encoding, -1)
assertS3Error(hc.t, w, err)
}
func listObjectsV2Base(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken, encodingType string, maxKeys int) *httptest.ResponseRecorder {
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
query.Add("fetch-owner", "true")
if len(startAfter) != 0 {
@ -743,10 +875,7 @@ func listObjectsV2Ext(hc *handlerContext, bktName, prefix, delimiter, startAfter
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
hc.Handler().ListObjectsV2Handler(w, r)
assertStatus(hc.t, w, http.StatusOK)
res := &ListObjectsV2Response{}
parseTestResponse(hc.t, w, res)
return res
return w
}
func validateListV1(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int,
@ -806,28 +935,54 @@ func prepareCommonListObjectsQuery(prefix, delimiter string, maxKeys int) url.Va
}
func listObjectsV1(hc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int) *ListObjectsV1Response {
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
if len(marker) != 0 {
query.Add("marker", marker)
}
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
hc.Handler().ListObjectsV1Handler(w, r)
w := listObjectsV1Base(hc, bktName, prefix, delimiter, marker, "", maxKeys)
assertStatus(hc.t, w, http.StatusOK)
res := &ListObjectsV1Response{}
parseTestResponse(hc.t, w, res)
return res
}
func listObjectsV1Err(hc *handlerContext, bktName, encoding string, err apierr.Error) {
w := listObjectsV1Base(hc, bktName, "", "", "", encoding, -1)
assertS3Error(hc.t, w, err)
}
func listObjectsV1Base(hc *handlerContext, bktName, prefix, delimiter, marker, encoding string, maxKeys int) *httptest.ResponseRecorder {
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
if len(marker) != 0 {
query.Add("marker", marker)
}
if len(encoding) != 0 {
query.Add("encoding-type", encoding)
}
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
hc.Handler().ListObjectsV1Handler(w, r)
return w
}
func listObjectsVersions(hc *handlerContext, bktName, prefix, delimiter, keyMarker, versionIDMarker string, maxKeys int) *ListObjectsVersionsResponse {
return listObjectsVersionsBase(hc, bktName, prefix, delimiter, keyMarker, versionIDMarker, maxKeys, false)
w := listObjectsVersionsBase(hc, bktName, prefix, delimiter, keyMarker, versionIDMarker, "", maxKeys)
assertStatus(hc.t, w, http.StatusOK)
res := &ListObjectsVersionsResponse{}
parseTestResponse(hc.t, w, res)
return res
}
func listObjectsVersionsURL(hc *handlerContext, bktName, prefix, delimiter, keyMarker, versionIDMarker string, maxKeys int) *ListObjectsVersionsResponse {
return listObjectsVersionsBase(hc, bktName, prefix, delimiter, keyMarker, versionIDMarker, maxKeys, true)
w := listObjectsVersionsBase(hc, bktName, prefix, delimiter, keyMarker, versionIDMarker, urlEncodingType, maxKeys)
assertStatus(hc.t, w, http.StatusOK)
res := &ListObjectsVersionsResponse{}
parseTestResponse(hc.t, w, res)
return res
}
func listObjectsVersionsBase(hc *handlerContext, bktName, prefix, delimiter, keyMarker, versionIDMarker string, maxKeys int, encode bool) *ListObjectsVersionsResponse {
func listObjectsVersionsErr(hc *handlerContext, bktName, encoding string, err apierr.Error) {
w := listObjectsVersionsBase(hc, bktName, "", "", "", "", encoding, -1)
assertS3Error(hc.t, w, err)
}
func listObjectsVersionsBase(hc *handlerContext, bktName, prefix, delimiter, keyMarker, versionIDMarker, encoding string, maxKeys int) *httptest.ResponseRecorder {
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
if len(keyMarker) != 0 {
query.Add("key-marker", keyMarker)
@ -835,14 +990,11 @@ func listObjectsVersionsBase(hc *handlerContext, bktName, prefix, delimiter, key
if len(versionIDMarker) != 0 {
query.Add("version-id-marker", versionIDMarker)
}
if encode {
query.Add("encoding-type", "url")
if len(encoding) != 0 {
query.Add("encoding-type", encoding)
}
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
hc.Handler().ListBucketObjectVersionsHandler(w, r)
assertStatus(hc.t, w, http.StatusOK)
res := &ListObjectsVersionsResponse{}
parseTestResponse(hc.t, w, res)
return res
return w
}

195
api/handler/patch.go Normal file
View file

@ -0,0 +1,195 @@
package handler
import (
"fmt"
"net/http"
"strconv"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"go.uber.org/zap"
)
const maxPatchSize = 5 * 1024 * 1024 * 1024 // 5GB
func (h *handler) PatchObjectHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
if _, ok := r.Header[api.ContentRange]; !ok {
h.logAndSendError(ctx, w, "missing Content-Range", reqInfo, errors.GetAPIError(errors.ErrMissingContentRange))
return
}
if _, ok := r.Header[api.ContentLength]; !ok {
h.logAndSendError(ctx, w, "missing Content-Length", reqInfo, errors.GetAPIError(errors.ErrMissingContentLength))
return
}
conditional, err := parsePatchConditionalHeaders(r.Header)
if err != nil {
h.logAndSendError(ctx, w, "could not parse conditional headers", reqInfo, err)
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket settings", reqInfo, err)
return
}
srcObjPrm := &layer.HeadObjectParams{
Object: reqInfo.ObjectName,
BktInfo: bktInfo,
VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
}
extendedSrcObjInfo, err := h.obj.GetExtendedObjectInfo(ctx, srcObjPrm)
if err != nil {
h.logAndSendError(ctx, w, "could not find object", reqInfo, err)
return
}
srcObjInfo := extendedSrcObjInfo.ObjectInfo
if err = checkPreconditions(srcObjInfo, conditional, h.cfg.MD5Enabled()); err != nil {
h.logAndSendError(ctx, w, "precondition failed", reqInfo, err)
return
}
srcSize, err := layer.GetObjectSize(srcObjInfo)
if err != nil {
h.logAndSendError(ctx, w, "failed to get source object size", reqInfo, err)
return
}
byteRange, err := parsePatchByteRange(r.Header.Get(api.ContentRange), srcSize)
if err != nil {
h.logAndSendError(ctx, w, "could not parse byte range", reqInfo, errors.GetAPIError(errors.ErrInvalidRange), zap.Error(err))
return
}
if maxPatchSize < byteRange.End-byteRange.Start+1 {
h.logAndSendError(ctx, w, "byte range length is longer than allowed", reqInfo, errors.GetAPIError(errors.ErrInvalidRange), zap.Error(err))
return
}
if uint64(r.ContentLength) != (byteRange.End - byteRange.Start + 1) {
h.logAndSendError(ctx, w, "content-length must be equal to byte range length", reqInfo, errors.GetAPIError(errors.ErrInvalidRangeLength))
return
}
if byteRange.Start > srcSize {
h.logAndSendError(ctx, w, "start byte is greater than object size", reqInfo, errors.GetAPIError(errors.ErrRangeOutOfBounds))
return
}
params := &layer.PatchObjectParams{
Object: extendedSrcObjInfo,
BktInfo: bktInfo,
NewBytes: r.Body,
Range: byteRange,
VersioningEnabled: settings.VersioningEnabled(),
}
params.CopiesNumbers, err = h.pickCopiesNumbers(nil, reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err)
return
}
extendedObjInfo, err := h.obj.PatchObject(ctx, params)
if err != nil {
if isErrObjectLocked(err) {
h.logAndSendError(ctx, w, "object is locked", reqInfo, errors.GetAPIError(errors.ErrAccessDenied))
} else {
h.logAndSendError(ctx, w, "could not patch object", reqInfo, err)
}
return
}
if settings.VersioningEnabled() {
w.Header().Set(api.AmzVersionID, extendedObjInfo.ObjectInfo.VersionID())
}
w.Header().Set(api.ETag, data.Quote(extendedObjInfo.ObjectInfo.ETag(h.cfg.MD5Enabled())))
resp := PatchObjectResult{
Object: PatchObject{
LastModified: extendedObjInfo.ObjectInfo.Created.UTC().Format(time.RFC3339),
ETag: data.Quote(extendedObjInfo.ObjectInfo.ETag(h.cfg.MD5Enabled())),
},
}
if err = middleware.EncodeToResponse(w, resp); err != nil {
h.logAndSendError(ctx, w, "could not encode PatchObjectResult to response", reqInfo, err)
return
}
}
func parsePatchConditionalHeaders(headers http.Header) (*conditionalArgs, error) {
var err error
args := &conditionalArgs{
IfMatch: data.UnQuote(headers.Get(api.IfMatch)),
}
if args.IfUnmodifiedSince, err = parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err != nil {
return nil, err
}
return args, nil
}
func parsePatchByteRange(rangeStr string, objSize uint64) (*layer.RangeParams, error) {
const prefix = "bytes "
if rangeStr == "" {
return nil, fmt.Errorf("empty range")
}
if !strings.HasPrefix(rangeStr, prefix) {
return nil, fmt.Errorf("unknown unit in range header")
}
rangeStr, _, found := strings.Cut(strings.TrimPrefix(rangeStr, prefix), "/") // value after / is ignored
if !found {
return nil, fmt.Errorf("invalid range: %s", rangeStr)
}
startStr, endStr, found := strings.Cut(rangeStr, "-")
if !found {
return nil, fmt.Errorf("invalid range: %s", rangeStr)
}
start, err := strconv.ParseUint(startStr, 10, 64)
if err != nil {
return nil, fmt.Errorf("invalid start byte: %s", startStr)
}
end := objSize - 1
if len(endStr) > 0 {
end, err = strconv.ParseUint(endStr, 10, 64)
if err != nil {
return nil, fmt.Errorf("invalid end byte: %s", endStr)
}
}
if start > end {
return nil, fmt.Errorf("start byte is greater than end byte")
}
return &layer.RangeParams{
Start: start,
End: end,
}, nil
}

532
api/handler/patch_test.go Normal file
View file

@ -0,0 +1,532 @@
package handler
import (
"bytes"
"crypto/md5"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/stretchr/testify/require"
)
func TestPatch(t *testing.T) {
tc := prepareHandlerContext(t)
tc.config.md5Enabled = true
bktName, objName := "bucket-for-patch", "object-for-patch"
createTestBucket(tc, bktName)
content := []byte("old object content")
md5Hash := md5.New()
md5Hash.Write(content)
etag := data.Quote(hex.EncodeToString(md5Hash.Sum(nil)))
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
created := time.Now()
tc.Handler().PutObjectHandler(w, r)
require.Equal(t, etag, w.Header().Get(api.ETag))
patchPayload := []byte("new")
sha256Hash := sha256.New()
sha256Hash.Write(patchPayload)
sha256Hash.Write(content[len(patchPayload):])
hash := hex.EncodeToString(sha256Hash.Sum(nil))
for _, tt := range []struct {
name string
rng string
headers map[string]string
code apierr.ErrorCode
}{
{
name: "success",
rng: "bytes 0-2/*",
headers: map[string]string{
api.IfUnmodifiedSince: created.Format(http.TimeFormat),
api.IfMatch: etag,
},
},
{
name: "invalid range syntax",
rng: "bytes 0-2",
code: apierr.ErrInvalidRange,
},
{
name: "invalid range length",
rng: "bytes 0-5/*",
code: apierr.ErrInvalidRangeLength,
},
{
name: "invalid range start",
rng: "bytes 20-22/*",
code: apierr.ErrRangeOutOfBounds,
},
{
name: "range is too long",
rng: "bytes 0-5368709120/*",
code: apierr.ErrInvalidRange,
},
{
name: "If-Unmodified-Since precondition are not satisfied",
rng: "bytes 0-2/*",
headers: map[string]string{
api.IfUnmodifiedSince: created.Add(-24 * time.Hour).Format(http.TimeFormat),
},
code: apierr.ErrPreconditionFailed,
},
{
name: "If-Match precondition are not satisfied",
rng: "bytes 0-2/*",
headers: map[string]string{
api.IfMatch: "etag",
},
code: apierr.ErrPreconditionFailed,
},
} {
t.Run(tt.name, func(t *testing.T) {
if tt.code == 0 {
res := patchObject(t, tc, bktName, objName, tt.rng, patchPayload, tt.headers)
require.Equal(t, data.Quote(hash), res.Object.ETag)
} else {
patchObjectErr(tc, bktName, objName, tt.rng, patchPayload, tt.headers, tt.code)
}
})
}
}
func TestPatchMultipartObject(t *testing.T) {
tc := prepareHandlerContextWithMinCache(t)
tc.config.md5Enabled = true
bktName, objName, partSize := "bucket-for-multipart-patch", "object-for-multipart-patch", 5*1024*1024
createTestBucket(tc, bktName)
t.Run("patch beginning of the first part", func(t *testing.T) {
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
patchSize := partSize / 2
patchBody := make([]byte, patchSize)
_, err := rand.Read(patchBody)
require.NoError(t, err)
patchObject(t, tc, bktName, objName, "bytes 0-"+strconv.Itoa(patchSize-1)+"/*", patchBody, nil)
object, header := getObject(tc, bktName, objName)
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
require.NoError(t, err)
equalDataSlices(t, bytes.Join([][]byte{patchBody, data1[patchSize:], data2, data3}, []byte("")), object)
require.Equal(t, partSize*3, contentLen)
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
})
t.Run("patch middle of the first part", func(t *testing.T) {
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
patchSize := partSize / 2
patchBody := make([]byte, patchSize)
_, err := rand.Read(patchBody)
require.NoError(t, err)
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize/4)+"-"+strconv.Itoa(partSize*3/4-1)+"/*", patchBody, nil)
object, header := getObject(tc, bktName, objName)
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
require.NoError(t, err)
equalDataSlices(t, bytes.Join([][]byte{data1[:partSize/4], patchBody, data1[partSize*3/4:], data2, data3}, []byte("")), object)
require.Equal(t, partSize*3, contentLen)
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
})
t.Run("patch first and second parts", func(t *testing.T) {
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
patchSize := partSize / 2
patchBody := make([]byte, patchSize)
_, err := rand.Read(patchBody)
require.NoError(t, err)
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*3/4)+"-"+strconv.Itoa(partSize*5/4-1)+"/*", patchBody, nil)
object, header := getObject(tc, bktName, objName)
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
require.NoError(t, err)
equalDataSlices(t, bytes.Join([][]byte{data1[:partSize*3/4], patchBody, data2[partSize/4:], data3}, []byte("")), object)
require.Equal(t, partSize*3, contentLen)
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
})
t.Run("patch all parts", func(t *testing.T) {
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
etag2, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
patchSize := partSize * 2
patchBody := make([]byte, patchSize)
_, err := rand.Read(patchBody)
require.NoError(t, err)
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize/2-1)+"-"+strconv.Itoa(partSize/2+patchSize-2)+"/*", patchBody, nil)
object, header := getObject(tc, bktName, objName)
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
require.NoError(t, err)
equalDataSlices(t, bytes.Join([][]byte{data1[:partSize/2-1], patchBody, data3[partSize/2-1:]}, []byte("")), object)
require.Equal(t, partSize*3, contentLen)
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
})
t.Run("patch all parts and append bytes", func(t *testing.T) {
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
etag2, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
etag3, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
patchSize := partSize * 3
patchBody := make([]byte, patchSize)
_, err := rand.Read(patchBody)
require.NoError(t, err)
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize/2)+"-"+strconv.Itoa(partSize/2+patchSize-1)+"/*", patchBody, nil)
object, header := getObject(tc, bktName, objName)
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
require.NoError(t, err)
equalDataSlices(t, bytes.Join([][]byte{data1[:partSize/2], patchBody}, []byte("")), object)
require.Equal(t, partSize*7/2, contentLen)
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
})
t.Run("patch second part", func(t *testing.T) {
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
etag2, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
patchBody := make([]byte, partSize)
_, err := rand.Read(patchBody)
require.NoError(t, err)
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize)+"-"+strconv.Itoa(partSize*2-1)+"/*", patchBody, nil)
object, header := getObject(tc, bktName, objName)
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
require.NoError(t, err)
equalDataSlices(t, bytes.Join([][]byte{data1, patchBody, data3}, []byte("")), object)
require.Equal(t, partSize*3, contentLen)
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
})
t.Run("patch last part, equal size", func(t *testing.T) {
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
etag3, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
patchBody := make([]byte, partSize)
_, err := rand.Read(patchBody)
require.NoError(t, err)
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*2)+"-"+strconv.Itoa(partSize*3-1)+"/*", patchBody, nil)
object, header := getObject(tc, bktName, objName)
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
require.NoError(t, err)
equalDataSlices(t, bytes.Join([][]byte{data1, data2, patchBody}, []byte("")), object)
require.Equal(t, partSize*3, contentLen)
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
})
t.Run("patch last part, increase size", func(t *testing.T) {
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
etag3, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
patchBody := make([]byte, partSize+1)
_, err := rand.Read(patchBody)
require.NoError(t, err)
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*2)+"-"+strconv.Itoa(partSize*3)+"/*", patchBody, nil)
object, header := getObject(tc, bktName, objName)
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
require.NoError(t, err)
equalDataSlices(t, bytes.Join([][]byte{data1, data2, patchBody}, []byte("")), object)
require.Equal(t, partSize*3+1, contentLen)
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
})
t.Run("patch last part with offset and append bytes", func(t *testing.T) {
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
patchBody := make([]byte, partSize)
_, err := rand.Read(patchBody)
require.NoError(t, err)
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*2+3)+"-"+strconv.Itoa(partSize*3+2)+"/*", patchBody, nil)
object, header := getObject(tc, bktName, objName)
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
require.NoError(t, err)
equalDataSlices(t, bytes.Join([][]byte{data1, data2, data3[:3], patchBody}, []byte("")), object)
require.Equal(t, partSize*3+3, contentLen)
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
})
t.Run("append bytes", func(t *testing.T) {
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
etag1, data1 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, partSize)
etag2, data2 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 2, partSize)
etag3, data3 := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 3, partSize)
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag1, etag2, etag3})
patchBody := make([]byte, partSize)
_, err := rand.Read(patchBody)
require.NoError(t, err)
patchObject(t, tc, bktName, objName, "bytes "+strconv.Itoa(partSize*3)+"-"+strconv.Itoa(partSize*4-1)+"/*", patchBody, nil)
object, header := getObject(tc, bktName, objName)
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
require.NoError(t, err)
equalDataSlices(t, bytes.Join([][]byte{data1, data2, data3, patchBody}, []byte("")), object)
require.Equal(t, partSize*4, contentLen)
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-3"))
})
t.Run("patch empty multipart", func(t *testing.T) {
multipartInfo := createMultipartUpload(tc, bktName, objName, map[string]string{})
etag, _ := uploadPart(tc, bktName, objName, multipartInfo.UploadID, 1, 0)
completeMultipartUpload(tc, bktName, objName, multipartInfo.UploadID, []string{etag})
patchBody := make([]byte, partSize)
_, err := rand.Read(patchBody)
require.NoError(t, err)
patchObject(t, tc, bktName, objName, "bytes 0-"+strconv.Itoa(partSize-1)+"/*", patchBody, nil)
object, header := getObject(tc, bktName, objName)
contentLen, err := strconv.Atoi(header.Get(api.ContentLength))
require.NoError(t, err)
equalDataSlices(t, patchBody, object)
require.Equal(t, partSize, contentLen)
require.True(t, strings.HasSuffix(data.UnQuote(header.Get(api.ETag)), "-1"))
})
}
func TestPatchWithVersion(t *testing.T) {
hc := prepareHandlerContextWithMinCache(t)
bktName, objName := "bucket", "obj"
createVersionedBucket(hc, bktName)
objHeader := putObjectContent(hc, bktName, objName, "content")
putObjectContent(hc, bktName, objName, "some content")
patchObjectVersion(t, hc, bktName, objName, objHeader.Get(api.AmzVersionID), "bytes 7-14/*", []byte(" updated"))
res := listObjectsVersions(hc, bktName, "", "", "", "", 3)
require.False(t, res.IsTruncated)
require.Len(t, res.Version, 3)
for _, version := range res.Version {
content := getObjectVersion(hc, bktName, objName, version.VersionID)
if version.IsLatest {
require.Equal(t, []byte("content updated"), content)
continue
}
if version.VersionID == objHeader.Get(api.AmzVersionID) {
require.Equal(t, []byte("content"), content)
continue
}
require.Equal(t, []byte("some content"), content)
}
}
func TestPatchEncryptedObject(t *testing.T) {
tc := prepareHandlerContext(t)
bktName, objName := "bucket-for-patch-encrypted", "object-for-patch-encrypted"
createTestBucket(tc, bktName)
w, r := prepareTestPayloadRequest(tc, bktName, objName, strings.NewReader("object content"))
setEncryptHeaders(r)
tc.Handler().PutObjectHandler(w, r)
assertStatus(t, w, http.StatusOK)
patchObjectErr(tc, bktName, objName, "bytes 2-4/*", []byte("new"), nil, apierr.ErrInternalError)
}
func TestPatchMissingHeaders(t *testing.T) {
tc := prepareHandlerContext(t)
bktName, objName := "bucket-for-patch-missing-headers", "object-for-patch-missing-headers"
createTestBucket(tc, bktName)
w, r := prepareTestPayloadRequest(tc, bktName, objName, strings.NewReader("object content"))
setEncryptHeaders(r)
tc.Handler().PutObjectHandler(w, r)
assertStatus(t, w, http.StatusOK)
w = httptest.NewRecorder()
r = httptest.NewRequest(http.MethodPatch, defaultURL, strings.NewReader("new"))
tc.Handler().PatchObjectHandler(w, r)
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrMissingContentRange))
w = httptest.NewRecorder()
r = httptest.NewRequest(http.MethodPatch, defaultURL, strings.NewReader("new"))
r.Header.Set(api.ContentRange, "bytes 0-2/*")
tc.Handler().PatchObjectHandler(w, r)
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrMissingContentLength))
}
func TestPatchInvalidBucketName(t *testing.T) {
tc := prepareHandlerContext(t)
bktName, objName := "bucket", "object"
createTestBucket(tc, bktName)
patchObjectErr(tc, "bkt_name", objName, "bytes 2-4/*", []byte("new"), nil, apierr.ErrInvalidBucketName)
}
func TestParsePatchByteRange(t *testing.T) {
for _, tt := range []struct {
rng string
size uint64
expected *layer.RangeParams
err bool
}{
{
rng: "bytes 2-7/*",
expected: &layer.RangeParams{
Start: 2,
End: 7,
},
},
{
rng: "bytes 2-7/3",
expected: &layer.RangeParams{
Start: 2,
End: 7,
},
},
{
rng: "bytes 2-/*",
size: 9,
expected: &layer.RangeParams{
Start: 2,
End: 8,
},
},
{
rng: "bytes 2-/3",
size: 9,
expected: &layer.RangeParams{
Start: 2,
End: 8,
},
},
{
rng: "",
err: true,
},
{
rng: "2-7/*",
err: true,
},
{
rng: "bytes 7-2/*",
err: true,
},
{
rng: "bytes 2-7",
err: true,
},
{
rng: "bytes 2/*",
err: true,
},
{
rng: "bytes a-7/*",
err: true,
},
{
rng: "bytes 2-a/*",
err: true,
},
} {
t.Run(fmt.Sprintf("case: %s", tt.rng), func(t *testing.T) {
rng, err := parsePatchByteRange(tt.rng, tt.size)
if tt.err {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, tt.expected.Start, rng.Start)
require.Equal(t, tt.expected.End, rng.End)
}
})
}
}
func patchObject(t *testing.T, tc *handlerContext, bktName, objName, rng string, payload []byte, headers map[string]string) *PatchObjectResult {
w := patchObjectBase(tc, bktName, objName, "", rng, payload, headers)
assertStatus(t, w, http.StatusOK)
result := &PatchObjectResult{}
err := xml.NewDecoder(w.Result().Body).Decode(result)
require.NoError(t, err)
return result
}
func patchObjectVersion(t *testing.T, tc *handlerContext, bktName, objName, version, rng string, payload []byte) *PatchObjectResult {
w := patchObjectBase(tc, bktName, objName, version, rng, payload, nil)
assertStatus(t, w, http.StatusOK)
result := &PatchObjectResult{}
err := xml.NewDecoder(w.Result().Body).Decode(result)
require.NoError(t, err)
return result
}
func patchObjectErr(tc *handlerContext, bktName, objName, rng string, payload []byte, headers map[string]string, code apierr.ErrorCode) {
w := patchObjectBase(tc, bktName, objName, "", rng, payload, headers)
assertS3Error(tc.t, w, apierr.GetAPIError(code))
}
func patchObjectBase(tc *handlerContext, bktName, objName, version, rng string, payload []byte, headers map[string]string) *httptest.ResponseRecorder {
query := make(url.Values)
if len(version) > 0 {
query.Add(api.QueryVersionID, version)
}
w, r := prepareTestRequestWithQuery(tc, bktName, objName, query, payload)
r.Header.Set(api.ContentRange, rng)
r.Header.Set(api.ContentLength, strconv.Itoa(len(payload)))
for k, v := range headers {
r.Header.Set(k, v)
}
tc.Handler().PatchObjectHandler(w, r)
return w
}

View file

@ -2,11 +2,12 @@ package handler
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"encoding/json"
"encoding/xml"
stderrors "errors"
"errors"
"fmt"
"io"
"mime/multipart"
@ -20,7 +21,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
@ -91,11 +92,11 @@ func (p *postPolicy) CheckField(key string, value string) error {
}
cond := p.condition(key)
if cond == nil {
return errors.GetAPIError(errors.ErrPostPolicyConditionInvalidFormat)
return apierr.GetAPIError(apierr.ErrPostPolicyConditionInvalidFormat)
}
if !cond.match(value) {
return errors.GetAPIError(errors.ErrPostPolicyConditionInvalidFormat)
return apierr.GetAPIError(apierr.ErrPostPolicyConditionInvalidFormat)
}
return nil
@ -192,24 +193,24 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket objInfo", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket objInfo", reqInfo, err)
return
}
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket settings", reqInfo, err)
return
}
if cannedACLStatus == aclStatusYes {
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
h.logAndSendError(ctx, w, "acl not supported for this bucket", reqInfo, apierr.GetAPIError(apierr.ErrAccessControlListNotSupported))
return
}
tagSet, err := parseTaggingHeader(r.Header)
if err != nil {
h.logAndSendError(w, "could not parse tagging header", reqInfo, err)
h.logAndSendError(ctx, w, "could not parse tagging header", reqInfo, err)
return
}
@ -229,44 +230,44 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
encryptionParams, err := formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
h.logAndSendError(ctx, w, "invalid sse headers", reqInfo, err)
return
}
body, err := h.getBodyReader(r)
if err != nil {
h.logAndSendError(w, "failed to get body reader", reqInfo, err)
h.logAndSendError(ctx, w, "failed to get body reader", reqInfo, err)
return
}
if encodings := r.Header.Get(api.ContentEncoding); len(encodings) > 0 {
metadata[api.ContentEncoding] = encodings
}
var size uint64
if r.ContentLength > 0 {
size = uint64(r.ContentLength)
}
size := h.getPutPayloadSize(r)
params := &layer.PutObjectParams{
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
Reader: body,
Size: size,
Header: metadata,
Encryption: encryptionParams,
ContentMD5: r.Header.Get(api.ContentMD5),
ContentMD5: getMD5Header(r),
ContentSHA256Hash: r.Header.Get(api.AmzContentSha256),
}
if size > 0 {
params.Size = &size
}
params.CopiesNumbers, err = h.pickCopiesNumbers(metadata, reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err)
return
}
params.Lock, err = formObjectLock(ctx, bktInfo, settings.LockConfiguration, r.Header)
if err != nil {
h.logAndSendError(w, "could not form object lock", reqInfo, err)
h.logAndSendError(ctx, w, "could not form object lock", reqInfo, err)
return
}
@ -274,7 +275,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
if err != nil {
_, err2 := io.Copy(io.Discard, body)
err3 := body.Close()
h.logAndSendError(w, "could not upload object", reqInfo, err, zap.Errors("body close errors", []error{err2, err3}))
h.logAndSendError(ctx, w, "could not upload object", reqInfo, err, zap.Errors("body close errors", []error{err2, err3}))
return
}
objInfo := extendedObjInfo.ObjectInfo
@ -289,8 +290,8 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
TagSet: tagSet,
NodeVersion: extendedObjInfo.NodeVersion,
}
if err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
if err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
h.logAndSendError(ctx, w, "could not upload object tagging", reqInfo, err)
return
}
}
@ -305,7 +306,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set(api.ETag, data.Quote(objInfo.ETag(h.cfg.MD5Enabled())))
if err = middleware.WriteSuccessResponseHeadersOnly(w); err != nil {
h.logAndSendError(w, "write response", reqInfo, err)
h.logAndSendError(ctx, w, "write response", reqInfo, err)
return
}
}
@ -332,16 +333,16 @@ func (h *handler) getBodyReader(r *http.Request) (io.ReadCloser, error) {
if !chunkedEncoding && !h.cfg.BypassContentEncodingInChunks() {
return nil, fmt.Errorf("%w: request is not chunk encoded, encodings '%s'",
errors.GetAPIError(errors.ErrInvalidEncodingMethod), strings.Join(encodings, ","))
apierr.GetAPIError(apierr.ErrInvalidEncodingMethod), strings.Join(encodings, ","))
}
decodeContentSize := r.Header.Get(api.AmzDecodedContentLength)
if len(decodeContentSize) == 0 {
return nil, errors.GetAPIError(errors.ErrMissingContentLength)
return nil, apierr.GetAPIError(apierr.ErrMissingContentLength)
}
if _, err := strconv.Atoi(decodeContentSize); err != nil {
return nil, fmt.Errorf("%w: parse decoded content length: %s", errors.GetAPIError(errors.ErrMissingContentLength), err.Error())
return nil, fmt.Errorf("%w: parse decoded content length: %s", apierr.GetAPIError(apierr.ErrMissingContentLength), err.Error())
}
chunkReader, err := newSignV4ChunkedReader(r)
@ -377,43 +378,43 @@ func formEncryptionParamsBase(r *http.Request, isCopySource bool) (enc encryptio
}
if r.TLS == nil {
return enc, errors.GetAPIError(errors.ErrInsecureSSECustomerRequest)
return enc, apierr.GetAPIError(apierr.ErrInsecureSSECustomerRequest)
}
if len(sseCustomerKey) > 0 && len(sseCustomerAlgorithm) == 0 {
return enc, errors.GetAPIError(errors.ErrMissingSSECustomerAlgorithm)
return enc, apierr.GetAPIError(apierr.ErrMissingSSECustomerAlgorithm)
}
if len(sseCustomerAlgorithm) > 0 && len(sseCustomerKey) == 0 {
return enc, errors.GetAPIError(errors.ErrMissingSSECustomerKey)
return enc, apierr.GetAPIError(apierr.ErrMissingSSECustomerKey)
}
if sseCustomerAlgorithm != layer.AESEncryptionAlgorithm {
return enc, errors.GetAPIError(errors.ErrInvalidEncryptionAlgorithm)
return enc, apierr.GetAPIError(apierr.ErrInvalidEncryptionAlgorithm)
}
key, err := base64.StdEncoding.DecodeString(sseCustomerKey)
if err != nil {
if isCopySource {
return enc, errors.GetAPIError(errors.ErrInvalidSSECustomerParameters)
return enc, apierr.GetAPIError(apierr.ErrInvalidSSECustomerParameters)
}
return enc, errors.GetAPIError(errors.ErrInvalidSSECustomerKey)
return enc, apierr.GetAPIError(apierr.ErrInvalidSSECustomerKey)
}
if len(key) != layer.AESKeySize {
if isCopySource {
return enc, errors.GetAPIError(errors.ErrInvalidSSECustomerParameters)
return enc, apierr.GetAPIError(apierr.ErrInvalidSSECustomerParameters)
}
return enc, errors.GetAPIError(errors.ErrInvalidSSECustomerKey)
return enc, apierr.GetAPIError(apierr.ErrInvalidSSECustomerKey)
}
keyMD5, err := base64.StdEncoding.DecodeString(sseCustomerKeyMD5)
if err != nil {
return enc, errors.GetAPIError(errors.ErrSSECustomerKeyMD5Mismatch)
return enc, apierr.GetAPIError(apierr.ErrSSECustomerKeyMD5Mismatch)
}
md5Sum := md5.Sum(key)
if !bytes.Equal(md5Sum[:], keyMD5) {
return enc, errors.GetAPIError(errors.ErrSSECustomerKeyMD5Mismatch)
return enc, apierr.GetAPIError(apierr.ErrSSECustomerKeyMD5Mismatch)
}
params, err := encryption.NewParams(key)
@ -434,7 +435,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
policy, err := checkPostPolicy(r, reqInfo, metadata)
if err != nil {
h.logAndSendError(w, "failed check policy", reqInfo, err)
h.logAndSendError(ctx, w, "failed check policy", reqInfo, err)
return
}
@ -442,31 +443,31 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
buffer := bytes.NewBufferString(tagging)
tags := new(data.Tagging)
if err = h.cfg.NewXMLDecoder(buffer).Decode(tags); err != nil {
h.logAndSendError(w, "could not decode tag set", reqInfo,
fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()))
h.logAndSendError(ctx, w, "could not decode tag set", reqInfo,
fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
return
}
tagSet, err = h.readTagSet(tags)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
h.logAndSendError(ctx, w, "could not read tag set", reqInfo, err)
return
}
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket objInfo", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket objInfo", reqInfo, err)
return
}
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket settings", reqInfo, err)
return
}
if acl := auth.MultipartFormValue(r, "acl"); acl != "" && acl != basicACLPrivate {
h.logAndSendError(w, "acl not supported for this bucket", reqInfo, errors.GetAPIError(errors.ErrAccessControlListNotSupported))
h.logAndSendError(ctx, w, "acl not supported for this bucket", reqInfo, apierr.GetAPIError(apierr.ErrAccessControlListNotSupported))
return
}
@ -484,7 +485,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
if reqInfo.ObjectName == "" || strings.Contains(reqInfo.ObjectName, "${filename}") {
_, head, err := r.FormFile("file")
if err != nil {
h.logAndSendError(w, "could not parse file field", reqInfo, err)
h.logAndSendError(ctx, w, "could not parse file field", reqInfo, err)
return
}
filename = head.Filename
@ -493,7 +494,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
var head *multipart.FileHeader
contentReader, head, err = r.FormFile("file")
if err != nil {
h.logAndSendError(w, "could not parse file field", reqInfo, err)
h.logAndSendError(ctx, w, "could not parse file field", reqInfo, err)
return
}
size = uint64(head.Size)
@ -507,12 +508,12 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
}
if reqInfo.ObjectName == "" {
h.logAndSendError(w, "missing object name", reqInfo, errors.GetAPIError(errors.ErrInvalidArgument))
h.logAndSendError(ctx, w, "missing object name", reqInfo, apierr.GetAPIError(apierr.ErrInvalidArgument))
return
}
if !policy.CheckContentLength(size) {
h.logAndSendError(w, "invalid content-length", reqInfo, errors.GetAPIError(errors.ErrInvalidArgument))
h.logAndSendError(ctx, w, "invalid content-length", reqInfo, apierr.GetAPIError(apierr.ErrInvalidArgument))
return
}
@ -520,13 +521,13 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
Reader: contentReader,
Size: size,
Size: &size,
Header: metadata,
}
extendedObjInfo, err := h.obj.PutObject(ctx, params)
if err != nil {
h.logAndSendError(w, "could not upload object", reqInfo, err)
h.logAndSendError(ctx, w, "could not upload object", reqInfo, err)
return
}
objInfo := extendedObjInfo.ObjectInfo
@ -542,7 +543,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
}
if err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
h.logAndSendError(w, "could not upload object tagging", reqInfo, err)
h.logAndSendError(ctx, w, "could not upload object tagging", reqInfo, err)
return
}
}
@ -570,10 +571,10 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(status)
respData, err := middleware.EncodeResponse(resp)
if err != nil {
h.logAndSendError(w, "encode response", reqInfo, err)
h.logAndSendError(ctx, w, "encode response", reqInfo, err)
}
if _, err = w.Write(respData); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
return
}
@ -594,13 +595,13 @@ func checkPostPolicy(r *http.Request, reqInfo *middleware.ReqInfo, metadata map[
return nil, fmt.Errorf("could not unmarshal policy: %w", err)
}
if policy.Expiration.Before(time.Now()) {
return nil, fmt.Errorf("policy is expired: %w", errors.GetAPIError(errors.ErrInvalidArgument))
return nil, fmt.Errorf("policy is expired: %w", apierr.GetAPIError(apierr.ErrInvalidArgument))
}
policy.empty = false
}
if r.MultipartForm == nil {
return nil, stderrors.New("empty multipart form")
return nil, errors.New("empty multipart form")
}
for key, v := range r.MultipartForm.Value {
@ -631,7 +632,7 @@ func checkPostPolicy(r *http.Request, reqInfo *middleware.ReqInfo, metadata map[
for _, cond := range policy.Conditions {
if cond.Key == "bucket" {
if !cond.match(reqInfo.BucketName) {
return nil, errors.GetAPIError(errors.ErrPostPolicyConditionInvalidFormat)
return nil, apierr.GetAPIError(apierr.ErrPostPolicyConditionInvalidFormat)
}
}
}
@ -673,10 +674,10 @@ func parseTaggingHeader(header http.Header) (map[string]string, error) {
if tagging := header.Get(api.AmzTagging); len(tagging) > 0 {
queries, err := url.ParseQuery(tagging)
if err != nil {
return nil, errors.GetAPIError(errors.ErrInvalidArgument)
return nil, apierr.GetAPIError(apierr.ErrInvalidArgument)
}
if len(queries) > maxTags {
return nil, errors.GetAPIError(errors.ErrInvalidTagsSizeExceed)
return nil, apierr.GetAPIError(apierr.ErrInvalidTagsSizeExceed)
}
tagSet = make(map[string]string, len(queries))
for k, v := range queries {
@ -726,7 +727,7 @@ func (h *handler) parseCommonCreateBucketParams(reqInfo *middleware.ReqInfo, box
}
if p.SessionContainerCreation == nil {
return nil, nil, fmt.Errorf("%w: couldn't find session token for put", errors.GetAPIError(errors.ErrAccessDenied))
return nil, nil, fmt.Errorf("%w: couldn't find session token for put", apierr.GetAPIError(apierr.ErrAccessDenied))
}
if err := checkBucketName(reqInfo.BucketName); err != nil {
@ -758,32 +759,33 @@ func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Reque
boxData, err := middleware.GetBoxData(ctx)
if err != nil {
h.logAndSendError(w, "get access box from request", reqInfo, err)
h.logAndSendError(ctx, w, "get access box from request", reqInfo, err)
return
}
key, p, err := h.parseCommonCreateBucketParams(reqInfo, boxData, r)
if err != nil {
h.logAndSendError(w, "parse create bucket params", reqInfo, err)
h.logAndSendError(ctx, w, "parse create bucket params", reqInfo, err)
return
}
cannedACL, err := parseCannedACL(r.Header)
if err != nil {
h.logAndSendError(w, "could not parse canned ACL", reqInfo, err)
h.logAndSendError(ctx, w, "could not parse canned ACL", reqInfo, err)
return
}
bktInfo, err := h.obj.CreateBucket(ctx, p)
if err != nil {
h.logAndSendError(w, "could not create bucket", reqInfo, err)
h.logAndSendError(ctx, w, "could not create bucket", reqInfo, err)
return
}
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
chains := bucketCannedACLToAPERules(cannedACL, reqInfo, bktInfo.CID)
if err = h.ape.SaveACLChains(bktInfo.CID.EncodeToString(), chains); err != nil {
h.logAndSendError(w, "failed to add morph rule chain", reqInfo, err)
cleanErr := h.cleanupBucketCreation(ctx, reqInfo, bktInfo, boxData, chains)
h.logAndSendError(ctx, w, "failed to add morph rule chain", reqInfo, err, zap.NamedError("cleanup_error", cleanErr))
return
}
@ -804,17 +806,40 @@ func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Reque
return h.obj.PutBucketSettings(ctx, sp)
}, h.putBucketSettingsRetryer())
if err != nil {
h.logAndSendError(w, "couldn't save bucket settings", reqInfo, err,
zap.String("container_id", bktInfo.CID.EncodeToString()))
cleanErr := h.cleanupBucketCreation(ctx, reqInfo, bktInfo, boxData, chains)
h.logAndSendError(ctx, w, "couldn't save bucket settings", reqInfo, err,
zap.String("container_id", bktInfo.CID.EncodeToString()), zap.NamedError("cleanup_error", cleanErr))
return
}
if err = middleware.WriteSuccessResponseHeadersOnly(w); err != nil {
h.logAndSendError(w, "write response", reqInfo, err)
h.logAndSendError(ctx, w, "write response", reqInfo, err)
return
}
}
func (h *handler) cleanupBucketCreation(ctx context.Context, reqInfo *middleware.ReqInfo, bktInfo *data.BucketInfo, boxData *accessbox.Box, chains []*chain.Chain) error {
prm := &layer.DeleteBucketParams{
BktInfo: bktInfo,
SessionToken: boxData.Gate.SessionTokenForDelete(),
}
if err := h.obj.DeleteContainer(ctx, prm); err != nil {
return err
}
chainIDs := make([]chain.ID, len(chains))
for i, c := range chains {
chainIDs[i] = c.ID
}
if err := h.ape.DeleteBucketPolicy(reqInfo.Namespace, bktInfo.CID, chainIDs); err != nil {
return fmt.Errorf("delete bucket acl policy: %w", err)
}
return nil
}
func (h *handler) putBucketSettingsRetryer() aws.RetryerV2 {
return retry.NewStandard(func(options *retry.StandardOptions) {
options.MaxAttempts = h.cfg.RetryMaxAttempts()
@ -828,7 +853,7 @@ func (h *handler) putBucketSettingsRetryer() aws.RetryerV2 {
}
options.Retryables = []retry.IsErrorRetryable{retry.IsErrorRetryableFunc(func(err error) aws.Ternary {
if stderrors.Is(err, tree.ErrNodeAccessDenied) {
if errors.Is(err, tree.ErrNodeAccessDenied) {
return aws.TrueTernary
}
return aws.FalseTernary
@ -957,7 +982,7 @@ func (h handler) setPlacementPolicy(prm *layer.CreateBucketParams, namespace, lo
return nil
}
return errors.GetAPIError(errors.ErrInvalidLocationConstraint)
return apierr.GetAPIError(apierr.ErrInvalidLocationConstraint)
}
func isLockEnabled(log *zap.Logger, header http.Header) bool {
@ -976,28 +1001,22 @@ func isLockEnabled(log *zap.Logger, header http.Header) bool {
func checkBucketName(bucketName string) error {
if len(bucketName) < 3 || len(bucketName) > 63 {
return errors.GetAPIError(errors.ErrInvalidBucketName)
return apierr.GetAPIError(apierr.ErrInvalidBucketName)
}
if strings.HasPrefix(bucketName, "xn--") || strings.HasSuffix(bucketName, "-s3alias") {
return errors.GetAPIError(errors.ErrInvalidBucketName)
return apierr.GetAPIError(apierr.ErrInvalidBucketName)
}
if net.ParseIP(bucketName) != nil {
return errors.GetAPIError(errors.ErrInvalidBucketName)
return apierr.GetAPIError(apierr.ErrInvalidBucketName)
}
labels := strings.Split(bucketName, ".")
for _, label := range labels {
if len(label) == 0 {
return errors.GetAPIError(errors.ErrInvalidBucketName)
for i, r := range bucketName {
if r == '.' || (!isAlphaNum(r) && r != '-') {
return apierr.GetAPIError(apierr.ErrInvalidBucketName)
}
for i, r := range label {
if !isAlphaNum(r) && r != '-' {
return errors.GetAPIError(errors.ErrInvalidBucketName)
}
if (i == 0 || i == len(label)-1) && r == '-' {
return errors.GetAPIError(errors.ErrInvalidBucketName)
}
if (i == 0 || i == len(bucketName)-1) && r == '-' {
return apierr.GetAPIError(apierr.ErrInvalidBucketName)
}
}
@ -1015,7 +1034,17 @@ func (h *handler) parseLocationConstraint(r *http.Request) (*createBucketParams,
params := new(createBucketParams)
if err := h.cfg.NewXMLDecoder(r.Body).Decode(params); err != nil {
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error())
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error())
}
return params, nil
}
func getMD5Header(r *http.Request) *string {
var md5Hdr *string
if len(r.Header.Values(api.ContentMD5)) != 0 {
hdr := r.Header.Get(api.ContentMD5)
md5Hdr = &hdr
}
return md5Hdr
}

View file

@ -7,6 +7,7 @@ import (
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"io"
"mime/multipart"
"net/http"
@ -20,7 +21,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
@ -29,6 +30,11 @@ import (
"github.com/stretchr/testify/require"
)
const (
awsChunkedRequestExampleDecodedContentLength = 66560
awsChunkedRequestExampleContentLength = 66824
)
func TestCheckBucketName(t *testing.T) {
for _, tc := range []struct {
name string
@ -36,10 +42,10 @@ func TestCheckBucketName(t *testing.T) {
}{
{name: "bucket"},
{name: "2bucket"},
{name: "buc.ket"},
{name: "buc-ket"},
{name: "abc"},
{name: "63aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
{name: "buc.ket", err: true},
{name: "buc.-ket", err: true},
{name: "bucket.", err: true},
{name: ".bucket", err: true},
@ -199,7 +205,7 @@ func TestPostObject(t *testing.T) {
t.Run(tc.key+";"+tc.filename, func(t *testing.T) {
w := postObjectBase(hc, ns, bktName, tc.key, tc.filename, tc.content)
if tc.err {
assertS3Error(hc.t, w, s3errors.GetAPIError(s3errors.ErrInternalError))
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInternalError))
return
}
assertStatus(hc.t, w, http.StatusNoContent)
@ -245,6 +251,10 @@ func TestPutObjectWithNegativeContentLength(t *testing.T) {
tc.Handler().HeadObjectHandler(w, r)
assertStatus(t, w, http.StatusOK)
require.Equal(t, strconv.Itoa(len(content)), w.Header().Get(api.ContentLength))
result := listVersions(t, tc, bktName)
require.Len(t, result.Version, 1)
require.EqualValues(t, len(content), result.Version[0].Size)
}
func TestPutObjectWithStreamBodyError(t *testing.T) {
@ -258,7 +268,7 @@ func TestPutObjectWithStreamBodyError(t *testing.T) {
r.Header.Set(api.AmzContentSha256, api.StreamingContentSHA256)
r.Header.Set(api.ContentEncoding, api.AwsChunked)
tc.Handler().PutObjectHandler(w, r)
assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrMissingContentLength))
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrMissingContentLength))
checkNotFound(t, tc, bktName, objName, emptyVersion)
}
@ -274,7 +284,13 @@ func TestPutObjectWithInvalidContentMD5(t *testing.T) {
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString([]byte("invalid")))
tc.Handler().PutObjectHandler(w, r)
assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrInvalidDigest))
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrBadDigest))
content = []byte("content")
w, r = prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString([]byte("")))
tc.Handler().PutObjectHandler(w, r)
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
checkNotFound(t, tc, bktName, objName, emptyVersion)
}
@ -337,7 +353,7 @@ func TestPutObjectCheckContentSHA256(t *testing.T) {
hc.Handler().PutObjectHandler(w, r)
if tc.error {
assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch))
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch))
w, r := prepareTestRequest(hc, bktName, objName, nil)
hc.Handler().GetObjectHandler(w, r)
@ -361,12 +377,37 @@ func TestPutObjectWithStreamBodyAWSExample(t *testing.T) {
hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
data := getObjectRange(t, hc, bktName, objName, 0, 66824)
w, req = prepareTestRequest(hc, bktName, objName, nil)
hc.Handler().HeadObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
require.Equal(t, strconv.Itoa(awsChunkedRequestExampleDecodedContentLength), w.Header().Get(api.ContentLength))
data := getObjectRange(t, hc, bktName, objName, 0, awsChunkedRequestExampleDecodedContentLength)
for i := range chunk {
require.Equal(t, chunk[i], data[i])
}
}
func TestPutObjectWithStreamEmptyBodyAWSExample(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName := "dkirillov", "tmp"
createTestBucket(hc, bktName)
w, req := getEmptyChunkedRequest(hc.context, t, bktName, objName)
hc.Handler().PutObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
w, req = prepareTestRequest(hc, bktName, objName, nil)
hc.Handler().HeadObjectHandler(w, req)
assertStatus(t, w, http.StatusOK)
require.Equal(t, "0", w.Header().Get(api.ContentLength))
res := listObjectsV1(hc, bktName, "", "", "", -1)
require.Len(t, res.Contents, 1)
require.Empty(t, res.Contents[0].Size)
}
func TestPutChunkedTestContentEncoding(t *testing.T) {
hc := prepareHandlerContext(t)
@ -385,7 +426,7 @@ func TestPutChunkedTestContentEncoding(t *testing.T) {
w, req, _ = getChunkedRequest(hc.context, t, bktName, objName)
req.Header.Set(api.ContentEncoding, "gzip")
hc.Handler().PutObjectHandler(w, req)
assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrInvalidEncodingMethod))
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidEncodingMethod))
hc.config.bypassContentEncodingInChunks = true
w, req, _ = getChunkedRequest(hc.context, t, bktName, objName)
@ -397,6 +438,8 @@ func TestPutChunkedTestContentEncoding(t *testing.T) {
require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding))
}
// getChunkedRequest implements request example from
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
chunk := make([]byte, 65*1024)
for i := range chunk {
@ -424,9 +467,9 @@ func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName strin
req, err := http.NewRequest("PUT", "https://s3.amazonaws.com/"+bktName+"/"+objName, nil)
require.NoError(t, err)
req.Header.Set("content-encoding", "aws-chunked")
req.Header.Set("content-length", "66824")
req.Header.Set("content-length", strconv.Itoa(awsChunkedRequestExampleContentLength))
req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
req.Header.Set("x-amz-decoded-content-length", "66560")
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z")
@ -457,15 +500,69 @@ func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName strin
return w, req, chunk
}
func getEmptyChunkedRequest(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) {
AWSAccessKeyID := "48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh"
AWSSecretAccessKey := "09260955b4eb0279dc017ba20a1ddac909cbd226c86cbb2d868e55534c8e64b0"
reqBody := bytes.NewBufferString("0;chunk-signature=311a7142c8f3a07972c3aca65c36484b513a8fee48ab7178c7225388f2ae9894\r\n\r\n")
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, reqBody)
require.NoError(t, err)
req.Header.Set("Amz-Sdk-Invocation-Id", "8a8cd4be-aef8-8034-f08d-a6144ade41f9")
req.Header.Set("Amz-Sdk-Request", "attempt=1; max=2")
req.Header.Set(api.Authorization, "AWS4-HMAC-SHA256 Credential=48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh/20241003/us-east-1/s3/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-encoding;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length, Signature=4b530ab4af2381f214941af591266b209968264a2c94337fa1efc048c7dff352")
req.Header.Set(api.ContentEncoding, "aws-chunked")
req.Header.Set(api.ContentLength, "86")
req.Header.Set(api.ContentType, "text/plain; charset=UTF-8")
req.Header.Set(api.AmzDate, "20241003T100055Z")
req.Header.Set(api.AmzContentSha256, "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
req.Header.Set(api.AmzDecodedContentLength, "0")
signTime, err := time.Parse("20060102T150405Z", "20241003T100055Z")
require.NoError(t, err)
w := httptest.NewRecorder()
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "")
req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo))
req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{
ClientTime: signTime,
AuthHeaders: &middleware.AuthHeader{
AccessKeyID: AWSAccessKeyID,
SignatureV4: "4b530ab4af2381f214941af591266b209968264a2c94337fa1efc048c7dff352",
Region: "us-east-1",
},
AccessBox: &accessbox.Box{
Gate: &accessbox.GateData{
SecretKey: AWSSecretAccessKey,
},
},
}))
return w, req
}
func TestCreateBucket(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bkt-name"
info := createBucket(hc, bktName)
createBucketAssertS3Error(hc, bktName, info.Box, s3errors.ErrBucketAlreadyOwnedByYou)
createBucketAssertS3Error(hc, bktName, info.Box, apierr.ErrBucketAlreadyOwnedByYou)
box2, _ := createAccessBox(t)
createBucketAssertS3Error(hc, bktName, box2, s3errors.ErrBucketAlreadyExists)
createBucketAssertS3Error(hc, bktName, box2, apierr.ErrBucketAlreadyExists)
}
func TestCreateBucketWithoutPermissions(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bkt-name"
hc.h.ape.(*apeMock).err = errors.New("no permissions")
box, _ := createAccessBox(t)
createBucketAssertS3Error(hc, bktName, box, apierr.ErrInternalError)
_, err := hc.tp.ContainerID(bktName)
require.Errorf(t, err, "container exists after failed creation, but shouldn't")
}
func TestCreateNamespacedBucket(t *testing.T) {

View file

@ -195,6 +195,15 @@ type PostResponse struct {
ETag string `xml:"Etag"`
}
type PatchObjectResult struct {
Object PatchObject `xml:"Object"`
}
type PatchObject struct {
LastModified string `xml:"LastModified"`
ETag string `xml:"ETag"`
}
// MarshalXML -- StringMap marshals into XML.
func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
tokens := []xml.Token{start}

View file

@ -26,13 +26,13 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
tagSet, err := h.readTagSet(reqInfo.Tagging)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
h.logAndSendError(ctx, w, "could not read tag set", reqInfo, err)
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
@ -46,7 +46,7 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
}
if err = h.obj.PutObjectTagging(ctx, tagPrm); err != nil {
h.logAndSendError(w, "could not put object tagging", reqInfo, err)
h.logAndSendError(ctx, w, "could not put object tagging", reqInfo, err)
return
}
@ -54,17 +54,18 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
}
func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket settings", reqInfo, err)
return
}
@ -76,9 +77,9 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request
},
}
versionID, tagSet, err := h.obj.GetObjectTagging(r.Context(), tagPrm)
versionID, tagSet, err := h.obj.GetObjectTagging(ctx, tagPrm)
if err != nil {
h.logAndSendError(w, "could not get object tagging", reqInfo, err)
h.logAndSendError(ctx, w, "could not get object tagging", reqInfo, err)
return
}
@ -86,7 +87,7 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request
w.Header().Set(api.AmzVersionID, versionID)
}
if err = middleware.EncodeToResponse(w, encodeTagging(tagSet)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}
@ -96,7 +97,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
@ -107,7 +108,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
}
if err = h.obj.DeleteObjectTagging(ctx, p); err != nil {
h.logAndSendError(w, "could not delete object tagging", reqInfo, err)
h.logAndSendError(ctx, w, "could not delete object tagging", reqInfo, err)
return
}
@ -115,58 +116,61 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
}
func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
tagSet, err := h.readTagSet(reqInfo.Tagging)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
h.logAndSendError(ctx, w, "could not read tag set", reqInfo, err)
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if err = h.obj.PutBucketTagging(r.Context(), bktInfo, tagSet); err != nil {
h.logAndSendError(w, "could not put object tagging", reqInfo, err)
if err = h.obj.PutBucketTagging(ctx, bktInfo, tagSet); err != nil {
h.logAndSendError(ctx, w, "could not put object tagging", reqInfo, err)
return
}
}
func (h *handler) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
tagSet, err := h.obj.GetBucketTagging(r.Context(), bktInfo)
tagSet, err := h.obj.GetBucketTagging(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "could not get object tagging", reqInfo, err)
h.logAndSendError(ctx, w, "could not get object tagging", reqInfo, err)
return
}
if err = middleware.EncodeToResponse(w, encodeTagging(tagSet)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
return
}
}
func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if err = h.obj.DeleteBucketTagging(r.Context(), bktInfo); err != nil {
h.logAndSendError(w, "could not delete bucket tagging", reqInfo, err)
if err = h.obj.DeleteBucketTagging(ctx, bktInfo); err != nil {
h.logAndSendError(ctx, w, "could not delete bucket tagging", reqInfo, err)
return
}
w.WriteHeader(http.StatusNoContent)

View file

@ -6,7 +6,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"github.com/stretchr/testify/require"
)
@ -98,7 +98,7 @@ func TestPutObjectTaggingCheckUniqueness(t *testing.T) {
middleware.GetReqInfo(r.Context()).Tagging = tc.body
hc.Handler().PutObjectTaggingHandler(w, r)
if tc.error {
assertS3Error(t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidTagKeyUniqueness))
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidTagKeyUniqueness))
return
}
assertStatus(t, w, http.StatusOK)

View file

@ -8,61 +8,53 @@ import (
)
func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketAccelerateHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketLoggingHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketReplicationHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}

View file

@ -10,13 +10,11 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
frosterrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
)
@ -28,15 +26,14 @@ func (h *handler) reqLogger(ctx context.Context) *zap.Logger {
return h.log
}
func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo *middleware.ReqInfo, err error, additional ...zap.Field) {
func (h *handler) logAndSendError(ctx context.Context, w http.ResponseWriter, logText string, reqInfo *middleware.ReqInfo, err error, additional ...zap.Field) {
err = handleDeleteMarker(w, err)
if code, wrErr := middleware.WriteErrorResponse(w, reqInfo, transformToS3Error(err)); wrErr != nil {
if code, wrErr := middleware.WriteErrorResponse(w, reqInfo, apierr.TransformToS3Error(err)); wrErr != nil {
additional = append(additional, zap.NamedError("write_response_error", wrErr))
} else {
additional = append(additional, zap.Int("status", code))
}
fields := []zap.Field{
zap.String("request_id", reqInfo.RequestID),
zap.String("method", reqInfo.API),
zap.String("bucket", reqInfo.BucketName),
zap.String("object", reqInfo.ObjectName),
@ -44,10 +41,7 @@ func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo
zap.String("user", reqInfo.User),
zap.Error(err)}
fields = append(fields, additional...)
if traceID, err := trace.TraceIDFromHex(reqInfo.TraceID); err == nil && traceID.IsValid() {
fields = append(fields, zap.String("trace_id", reqInfo.TraceID))
}
h.log.Error(logs.RequestFailed, fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
h.reqLogger(ctx).Error(logs.RequestFailed, fields...)
}
func handleDeleteMarker(w http.ResponseWriter, err error) error {
@ -57,37 +51,26 @@ func handleDeleteMarker(w http.ResponseWriter, err error) error {
}
w.Header().Set(api.AmzDeleteMarker, "true")
return fmt.Errorf("%w: %s", s3errors.GetAPIError(target.ErrorCode), err)
}
func transformToS3Error(err error) error {
err = frosterrors.UnwrapErr(err) // this wouldn't work with errors.Join
if _, ok := err.(s3errors.Error); ok {
return err
}
if errors.Is(err, layer.ErrAccessDenied) ||
errors.Is(err, layer.ErrNodeAccessDenied) {
return s3errors.GetAPIError(s3errors.ErrAccessDenied)
}
if errors.Is(err, layer.ErrGatewayTimeout) {
return s3errors.GetAPIError(s3errors.ErrGatewayTimeout)
}
return s3errors.GetAPIError(s3errors.ErrInternalError)
return fmt.Errorf("%w: %s", apierr.GetAPIError(target.ErrorCode), err)
}
func (h *handler) ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error) {
return h.obj.GetBucketInfo(ctx, bucket)
return h.getBucketInfo(ctx, bucket)
}
func (h *handler) ResolveCID(ctx context.Context, bucket string) (cid.ID, error) {
return h.obj.ResolveCID(ctx, bucket)
}
func (h *handler) getBucketInfo(ctx context.Context, bucket string) (*data.BucketInfo, error) {
if err := checkBucketName(bucket); err != nil {
return nil, err
}
return h.obj.GetBucketInfo(ctx, bucket)
}
func (h *handler) getBucketAndCheckOwner(r *http.Request, bucket string, header ...string) (*data.BucketInfo, error) {
bktInfo, err := h.obj.GetBucketInfo(r.Context(), bucket)
bktInfo, err := h.getBucketInfo(r.Context(), bucket)
if err != nil {
return nil, err
}
@ -106,6 +89,19 @@ func (h *handler) getBucketAndCheckOwner(r *http.Request, bucket string, header
return bktInfo, checkOwner(bktInfo, expected)
}
func (h *handler) getPutPayloadSize(r *http.Request) uint64 {
decodeContentSize := r.Header.Get(api.AmzDecodedContentLength)
decodedSize, err := strconv.Atoi(decodeContentSize)
if err != nil {
if r.ContentLength >= 0 {
return uint64(r.ContentLength)
}
return 0
}
return uint64(decodedSize)
}
func parseRange(s string) (*layer.RangeParams, error) {
if s == "" {
return nil, nil
@ -114,26 +110,26 @@ func parseRange(s string) (*layer.RangeParams, error) {
prefix := "bytes="
if !strings.HasPrefix(s, prefix) {
return nil, s3errors.GetAPIError(s3errors.ErrInvalidRange)
return nil, apierr.GetAPIError(apierr.ErrInvalidRange)
}
s = strings.TrimPrefix(s, prefix)
valuesStr := strings.Split(s, "-")
if len(valuesStr) != 2 {
return nil, s3errors.GetAPIError(s3errors.ErrInvalidRange)
return nil, apierr.GetAPIError(apierr.ErrInvalidRange)
}
values := make([]uint64, 0, len(valuesStr))
for _, v := range valuesStr {
num, err := strconv.ParseUint(v, 10, 64)
if err != nil {
return nil, s3errors.GetAPIError(s3errors.ErrInvalidRange)
return nil, apierr.GetAPIError(apierr.ErrInvalidRange)
}
values = append(values, num)
}
if values[0] > values[1] {
return nil, s3errors.GetAPIError(s3errors.ErrInvalidRange)
return nil, apierr.GetAPIError(apierr.ErrInvalidRange)
}
return &layer.RangeParams{

View file

@ -1,64 +1 @@
package handler
import (
"errors"
"fmt"
"testing"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/stretchr/testify/require"
)
func TestTransformS3Errors(t *testing.T) {
for _, tc := range []struct {
name string
err error
expected s3errors.ErrorCode
}{
{
name: "simple std error to internal error",
err: errors.New("some error"),
expected: s3errors.ErrInternalError,
},
{
name: "layer access denied error to s3 access denied error",
err: layer.ErrAccessDenied,
expected: s3errors.ErrAccessDenied,
},
{
name: "wrapped layer access denied error to s3 access denied error",
err: fmt.Errorf("wrap: %w", layer.ErrAccessDenied),
expected: s3errors.ErrAccessDenied,
},
{
name: "layer node access denied error to s3 access denied error",
err: layer.ErrNodeAccessDenied,
expected: s3errors.ErrAccessDenied,
},
{
name: "layer gateway timeout error to s3 gateway timeout error",
err: layer.ErrGatewayTimeout,
expected: s3errors.ErrGatewayTimeout,
},
{
name: "s3 error to s3 error",
err: s3errors.GetAPIError(s3errors.ErrInvalidPart),
expected: s3errors.ErrInvalidPart,
},
{
name: "wrapped s3 error to s3 error",
err: fmt.Errorf("wrap: %w", s3errors.GetAPIError(s3errors.ErrInvalidPart)),
expected: s3errors.ErrInvalidPart,
},
} {
t.Run(tc.name, func(t *testing.T) {
err := transformToS3Error(tc.err)
s3err, ok := err.(s3errors.Error)
require.True(t, ok, "error must be s3 error")
require.Equalf(t, tc.expected, s3err.ErrCode,
"expected: '%s', got: '%s'",
s3errors.GetAPIError(tc.expected).Code, s3errors.GetAPIError(s3err.ErrCode).Code)
})
}
}

View file

@ -10,28 +10,29 @@ import (
)
func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
configuration := new(VersioningConfiguration)
if err := h.cfg.NewXMLDecoder(r.Body).Decode(configuration); err != nil {
h.logAndSendError(w, "couldn't decode versioning configuration", reqInfo, errors.GetAPIError(errors.ErrIllegalVersioningConfigurationException))
h.logAndSendError(ctx, w, "couldn't decode versioning configuration", reqInfo, errors.GetAPIError(errors.ErrIllegalVersioningConfigurationException))
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "couldn't get bucket settings", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't get bucket settings", reqInfo, err)
return
}
if configuration.Status != data.VersioningEnabled && configuration.Status != data.VersioningSuspended {
h.logAndSendError(w, "invalid versioning configuration", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
h.logAndSendError(ctx, w, "invalid versioning configuration", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
return
}
@ -45,33 +46,34 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ
}
if p.Settings.VersioningSuspended() && bktInfo.ObjectLockEnabled {
h.logAndSendError(w, "couldn't suspend bucket versioning", reqInfo, errors.GetAPIError(errors.ErrObjectLockConfigurationVersioningCannotBeChanged))
h.logAndSendError(ctx, w, "couldn't suspend bucket versioning", reqInfo, errors.GetAPIError(errors.ErrObjectLockConfigurationVersioningCannotBeChanged))
return
}
if err = h.obj.PutBucketSettings(r.Context(), p); err != nil {
h.logAndSendError(w, "couldn't put update versioning settings", reqInfo, err)
if err = h.obj.PutBucketSettings(ctx, p); err != nil {
h.logAndSendError(ctx, w, "couldn't put update versioning settings", reqInfo, err)
}
}
// GetBucketVersioningHandler implements bucket versioning getter handler.
func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
settings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(w, "couldn't get version settings", reqInfo, err)
h.logAndSendError(ctx, w, "couldn't get version settings", reqInfo, err)
return
}
if err = middleware.EncodeToResponse(w, formVersioningConfiguration(settings)); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
h.logAndSendError(ctx, w, "something went wrong", reqInfo, err)
}
}

View file

@ -5,6 +5,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"go.uber.org/zap"
@ -19,6 +20,7 @@ type Cache struct {
bucketCache *cache.BucketCache
systemCache *cache.SystemCache
accessCache *cache.AccessControlCache
networkInfoCache *cache.NetworkInfoCache
}
// CachesConfig contains params for caches.
@ -31,6 +33,7 @@ type CachesConfig struct {
Buckets *cache.Config
System *cache.Config
AccessControl *cache.Config
NetworkInfo *cache.NetworkInfoCacheConfig
}
// DefaultCachesConfigs returns filled configs.
@ -44,6 +47,7 @@ func DefaultCachesConfigs(logger *zap.Logger) *CachesConfig {
Buckets: cache.DefaultBucketConfig(logger),
System: cache.DefaultSystemConfig(logger),
AccessControl: cache.DefaultAccessControlConfig(logger),
NetworkInfo: cache.DefaultNetworkInfoConfig(logger),
}
}
@ -57,6 +61,7 @@ func NewCache(cfg *CachesConfig) *Cache {
bucketCache: cache.NewBucketCache(cfg.Buckets),
systemCache: cache.NewSystemCache(cfg.System),
accessCache: cache.NewAccessControlCache(cfg.AccessControl),
networkInfoCache: cache.NewNetworkInfoCache(cfg.NetworkInfo),
}
}
@ -257,3 +262,39 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf
func (c *Cache) DeleteCORS(bktInfo *data.BucketInfo) {
c.systemCache.Delete(bktInfo.CORSObjectName())
}
func (c *Cache) GetLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo) *data.LifecycleConfiguration {
key := bkt.LifecycleConfigurationObjectName()
if !c.accessCache.Get(owner, key) {
return nil
}
return c.systemCache.GetLifecycleConfiguration(key)
}
func (c *Cache) PutLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo, cfg *data.LifecycleConfiguration) {
key := bkt.LifecycleConfigurationObjectName()
if err := c.systemCache.PutLifecycleConfiguration(key, cfg); err != nil {
c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err))
}
if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
}
}
func (c *Cache) DeleteLifecycleConfiguration(bktInfo *data.BucketInfo) {
c.systemCache.Delete(bktInfo.LifecycleConfigurationObjectName())
}
func (c *Cache) GetNetworkInfo() *netmap.NetworkInfo {
return c.networkInfoCache.Get()
}
func (c *Cache) PutNetworkInfo(info netmap.NetworkInfo) {
if err := c.networkInfoCache.Put(info); err != nil {
c.logger.Warn(logs.CouldntCacheNetworkInfo, zap.Error(err))
}
}

View file

@ -6,7 +6,8 @@ import (
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
)
func (n *Layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) {
@ -29,8 +30,8 @@ func (n *Layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *data.Ob
tags, lockInfo, err = n.treeService.GetObjectTaggingAndLock(ctx, objVersion.BktInfo, nodeVersion)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, data.LockInfo{}, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, data.LockInfo{}, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error())
}
return nil, data.LockInfo{}, err
}

View file

@ -7,7 +7,8 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
@ -20,7 +21,7 @@ const (
AttributeLockEnabled = "LockEnabled"
)
func (n *Layer) containerInfo(ctx context.Context, prm PrmContainer) (*data.BucketInfo, error) {
func (n *Layer) containerInfo(ctx context.Context, prm frostfs.PrmContainer) (*data.BucketInfo, error) {
var (
err error
res *container.Container
@ -37,7 +38,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm PrmContainer) (*data.Buck
res, err = n.frostFS.Container(ctx, prm)
if err != nil {
if client.IsErrContainerNotFound(err) {
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchBucket), err.Error())
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchBucket), err.Error())
}
return nil, fmt.Errorf("get frostfs container: %w", err)
}
@ -64,7 +65,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm PrmContainer) (*data.Buck
}
}
zone, _ := n.features.FormContainerZone(reqInfo.Namespace)
zone := n.features.FormContainerZone(reqInfo.Namespace)
if zone != info.Zone {
return nil, fmt.Errorf("ns '%s' and zone '%s' are mismatched for container '%s'", zone, info.Zone, prm.ContainerID)
}
@ -77,7 +78,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm PrmContainer) (*data.Buck
func (n *Layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
stoken := n.SessionTokenForRead(ctx)
prm := PrmUserContainers{
prm := frostfs.PrmUserContainers{
UserID: n.BearerOwner(ctx),
SessionToken: stoken,
}
@ -90,7 +91,7 @@ func (n *Layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
list := make([]*data.BucketInfo, 0, len(res))
for i := range res {
getPrm := PrmContainer{
getPrm := frostfs.PrmContainer{
ContainerID: res[i],
SessionToken: stoken,
}
@ -111,7 +112,7 @@ func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
p.LocationConstraint = api.DefaultLocationConstraint // s3tests_boto3.functional.test_s3:test_bucket_get_location
}
zone, _ := n.features.FormContainerZone(p.Namespace)
zone := n.features.FormContainerZone(p.Namespace)
bktInfo := &data.BucketInfo{
Name: p.Name,
@ -132,7 +133,7 @@ func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
})
}
res, err := n.frostFS.CreateContainer(ctx, PrmContainerCreate{
res, err := n.frostFS.CreateContainer(ctx, frostfs.PrmContainerCreate{
Creator: bktInfo.Owner,
Policy: p.Policy,
Name: p.Name,
@ -140,7 +141,6 @@ func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
SessionToken: p.SessionContainerCreation,
CreationTime: bktInfo.Created,
AdditionalAttributes: attributes,
BasicACL: 0, // means APE
})
if err != nil {
return nil, fmt.Errorf("create container: %w", err)

View file

@ -3,12 +3,14 @@ package layer
import (
"bytes"
"context"
errorsStd "errors"
"errors"
"fmt"
"io"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -31,14 +33,14 @@ func (n *Layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
}
if cors.CORSRules == nil {
return errors.GetAPIError(errors.ErrMalformedXML)
return apierr.GetAPIError(apierr.ErrMalformedXML)
}
if err := checkCORS(cors); err != nil {
return err
}
prm := PrmObjectCreate{
prm := frostfs.PrmObjectCreate{
Payload: &buf,
Filepath: p.BktInfo.CORSObjectName(),
CreationTime: TimeNow(ctx),
@ -55,13 +57,13 @@ func (n *Layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
prm.Container = corsBkt.CID
_, objID, _, _, err := n.objectPutAndHash(ctx, prm, corsBkt)
createdObj, err := n.objectPutAndHash(ctx, prm, corsBkt)
if err != nil {
return fmt.Errorf("put cors object: %w", err)
}
objsToDelete, err := n.treeService.PutBucketCORS(ctx, p.BktInfo, newAddress(corsBkt.CID, objID))
objToDeleteNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
objsToDelete, err := n.treeService.PutBucketCORS(ctx, p.BktInfo, newAddress(corsBkt.CID, createdObj.ID))
objToDeleteNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !objToDeleteNotFound {
return err
}
@ -79,7 +81,7 @@ func (n *Layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
// deleteCORSObject removes object and logs in case of error.
func (n *Layer) deleteCORSObject(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) {
var prmAuth PrmAuth
var prmAuth frostfs.PrmAuth
corsBkt := bktInfo
if !addr.Container().Equals(bktInfo.CID) && !addr.Container().Equals(cid.ID{}) {
corsBkt = &data.BucketInfo{CID: addr.Container()}
@ -104,7 +106,7 @@ func (n *Layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*d
func (n *Layer) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) error {
objs, err := n.treeService.DeleteBucketCORS(ctx, bktInfo)
objNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
objNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !objNotFound {
return err
}
@ -124,12 +126,12 @@ func checkCORS(cors *data.CORSConfiguration) error {
for _, r := range cors.CORSRules {
for _, m := range r.AllowedMethods {
if _, ok := supportedMethods[m]; !ok {
return errors.GetAPIErrorWithError(errors.ErrCORSUnsupportedMethod, fmt.Errorf("unsupported method is %s", m))
return apierr.GetAPIErrorWithError(apierr.ErrCORSUnsupportedMethod, fmt.Errorf("unsupported method is %s", m))
}
}
for _, h := range r.ExposeHeaders {
if h == wildcard {
return errors.GetAPIError(errors.ErrCORSWildcardExposeHeaders)
return apierr.GetAPIError(apierr.ErrCORSWildcardExposeHeaders)
}
}
}

View file

@ -1,4 +1,4 @@
package layer
package frostfs
import (
"context"
@ -9,13 +9,13 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
)
// PrmContainerCreate groups parameters of FrostFS.CreateContainer operation.
@ -38,13 +38,19 @@ type PrmContainerCreate struct {
// Token of the container's creation session. Nil means session absence.
SessionToken *session.Container
// Basic ACL of the container.
BasicACL acl.Basic
// Attributes for optional parameters.
AdditionalAttributes [][2]string
}
// PrmAddContainerPolicyChain groups parameter of FrostFS.AddContainerPolicyChain operation.
type PrmAddContainerPolicyChain struct {
// ContainerID is a container identifier.
ContainerID cid.ID
// Chain is Access Policy Engine chain that contains rules which provide access to specific actions in container.
Chain chain.Chain
}
// PrmContainer groups parameters of FrostFS.Container operation.
type PrmContainer struct {
// Container identifier.
@ -166,6 +172,12 @@ type PrmObjectCreate struct {
BufferMaxSize uint64
}
// CreateObjectResult is a result parameter of FrostFS.CreateObject operation.
type CreateObjectResult struct {
ObjectID oid.ID
CreationEpoch uint64
}
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
type PrmObjectDelete struct {
// Authentication parameters.
@ -194,6 +206,27 @@ type PrmObjectSearch struct {
FilePrefix string
}
// PrmObjectPatch groups parameters of FrostFS.PatchObject operation.
type PrmObjectPatch struct {
// Authentication parameters.
PrmAuth
// Container of the patched object.
Container cid.ID
// Identifier of the patched object.
Object oid.ID
// Object patch payload encapsulated in io.Reader primitive.
Payload io.Reader
// Object range to patch.
Offset, Length uint64
// Size of original object payload.
ObjectSize uint64
}
var (
// ErrAccessDenied is returned from FrostFS in case of access violation.
ErrAccessDenied = errors.New("access denied")
@ -212,6 +245,10 @@ type FrostFS interface {
// prevented the container from being created.
CreateContainer(context.Context, PrmContainerCreate) (*ContainerCreateResult, error)
// AddContainerPolicyChain create new policy chain for container.
// Can be invoked only by container owner.
AddContainerPolicyChain(context.Context, PrmAddContainerPolicyChain) error
// Container reads a container from FrostFS by ID.
//
// It returns exactly one non-nil value. It returns any error encountered which
@ -261,15 +298,15 @@ type FrostFS interface {
// CreateObject creates and saves a parameterized object in the FrostFS container.
// It sets 'Timestamp' attribute to the current time.
// It returns the ID of the saved object.
// It returns the ID and creation epoch of the saved object.
//
// Creation time should be written into the object (UTC).
//
// It returns ErrAccessDenied on write access violation.
//
// It returns exactly one non-zero value. It returns any error encountered which
// prevented the container from being created.
CreateObject(context.Context, PrmObjectCreate) (oid.ID, error)
// It returns exactly one non-nil value. It returns any error encountered which
// prevented the object from being created.
CreateObject(context.Context, PrmObjectCreate) (*CreateObjectResult, error)
// DeleteObject marks the object to be removed from the FrostFS container by identifier.
// Successful return does not guarantee actual removal.
@ -288,6 +325,15 @@ type FrostFS interface {
// prevented the objects from being selected.
SearchObjects(context.Context, PrmObjectSearch) ([]oid.ID, error)
// PatchObject performs object patch in the FrostFS container.
// It returns the ID of the patched object.
//
// It returns ErrAccessDenied on selection access violation.
//
// It returns exactly one non-nil value. It returns any error encountered which
// prevented the objects from being patched.
PatchObject(context.Context, PrmObjectPatch) (oid.ID, error)
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided now and future time.
// Note:
// * future time must be after the now
@ -295,4 +341,7 @@ type FrostFS interface {
//
// It returns any error encountered which prevented computing epochs.
TimeToEpoch(ctx context.Context, now time.Time, future time.Time) (uint64, uint64, error)
// NetworkInfo returns parameters of FrostFS network.
NetworkInfo(context.Context) (netmap.NetworkInfo, error)
}

View file

@ -5,6 +5,7 @@ import (
"context"
"crypto/rand"
"crypto/sha256"
"errors"
"fmt"
"io"
"strings"
@ -12,16 +13,20 @@ import (
v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
)
@ -50,21 +55,22 @@ func (k *FeatureSettingsMock) SetMD5Enabled(md5Enabled bool) {
k.md5Enabled = md5Enabled
}
func (k *FeatureSettingsMock) FormContainerZone(ns string) (zone string, isDefault bool) {
func (k *FeatureSettingsMock) FormContainerZone(ns string) string {
if ns == "" {
return v2container.SysAttributeZoneDefault, true
return v2container.SysAttributeZoneDefault
}
return ns + ".ns", false
return ns + ".ns"
}
type TestFrostFS struct {
FrostFS
var _ frostfs.FrostFS = (*TestFrostFS)(nil)
type TestFrostFS struct {
objects map[string]*object.Object
objectErrors map[string]error
objectPutErrors map[string]error
containers map[string]*container.Container
chains map[string][]chain.Chain
currentEpoch uint64
key *keys.PrivateKey
}
@ -75,6 +81,7 @@ func NewTestFrostFS(key *keys.PrivateKey) *TestFrostFS {
objectErrors: make(map[string]error),
objectPutErrors: make(map[string]error),
containers: make(map[string]*container.Container),
chains: make(map[string][]chain.Chain),
key: key,
}
}
@ -137,12 +144,11 @@ func (t *TestFrostFS) SetContainer(cnrID cid.ID, cnr *container.Container) {
t.containers[cnrID.EncodeToString()] = cnr
}
func (t *TestFrostFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (*ContainerCreateResult, error) {
func (t *TestFrostFS) CreateContainer(_ context.Context, prm frostfs.PrmContainerCreate) (*frostfs.ContainerCreateResult, error) {
var cnr container.Container
cnr.Init()
cnr.SetOwner(prm.Creator)
cnr.SetPlacementPolicy(prm.Policy)
cnr.SetBasicACL(prm.BasicACL)
creationTime := prm.CreationTime
if creationTime.IsZero() {
@ -171,8 +177,9 @@ func (t *TestFrostFS) CreateContainer(_ context.Context, prm PrmContainerCreate)
var id cid.ID
id.SetSHA256(sha256.Sum256(b))
t.containers[id.EncodeToString()] = &cnr
t.chains[id.EncodeToString()] = []chain.Chain{}
return &ContainerCreateResult{ContainerID: id}, nil
return &frostfs.ContainerCreateResult{ContainerID: id}, nil
}
func (t *TestFrostFS) DeleteContainer(_ context.Context, cnrID cid.ID, _ *session.Container) error {
@ -181,7 +188,7 @@ func (t *TestFrostFS) DeleteContainer(_ context.Context, cnrID cid.ID, _ *sessio
return nil
}
func (t *TestFrostFS) Container(_ context.Context, prm PrmContainer) (*container.Container, error) {
func (t *TestFrostFS) Container(_ context.Context, prm frostfs.PrmContainer) (*container.Container, error) {
for k, v := range t.containers {
if k == prm.ContainerID.EncodeToString() {
return v, nil
@ -191,7 +198,7 @@ func (t *TestFrostFS) Container(_ context.Context, prm PrmContainer) (*container
return nil, fmt.Errorf("container not found %s", prm.ContainerID)
}
func (t *TestFrostFS) UserContainers(context.Context, PrmUserContainers) ([]cid.ID, error) {
func (t *TestFrostFS) UserContainers(context.Context, frostfs.PrmUserContainers) ([]cid.ID, error) {
var res []cid.ID
for k := range t.containers {
var idCnr cid.ID
@ -218,7 +225,7 @@ func (t *TestFrostFS) retrieveObject(ctx context.Context, cnrID cid.ID, objID oi
if obj, ok := t.objects[sAddr]; ok {
owner := getBearerOwner(ctx)
if !t.checkAccess(cnrID, owner) {
return nil, ErrAccessDenied
return nil, frostfs.ErrAccessDenied
}
return obj, nil
@ -227,23 +234,23 @@ func (t *TestFrostFS) retrieveObject(ctx context.Context, cnrID cid.ID, objID oi
return nil, fmt.Errorf("%w: %s", &apistatus.ObjectNotFound{}, addr)
}
func (t *TestFrostFS) HeadObject(ctx context.Context, prm PrmObjectHead) (*object.Object, error) {
func (t *TestFrostFS) HeadObject(ctx context.Context, prm frostfs.PrmObjectHead) (*object.Object, error) {
return t.retrieveObject(ctx, prm.Container, prm.Object)
}
func (t *TestFrostFS) GetObject(ctx context.Context, prm PrmObjectGet) (*Object, error) {
func (t *TestFrostFS) GetObject(ctx context.Context, prm frostfs.PrmObjectGet) (*frostfs.Object, error) {
obj, err := t.retrieveObject(ctx, prm.Container, prm.Object)
if err != nil {
return nil, err
}
return &Object{
return &frostfs.Object{
Header: *obj,
Payload: io.NopCloser(bytes.NewReader(obj.Payload())),
}, nil
}
func (t *TestFrostFS) RangeObject(ctx context.Context, prm PrmObjectRange) (io.ReadCloser, error) {
func (t *TestFrostFS) RangeObject(ctx context.Context, prm frostfs.PrmObjectRange) (io.ReadCloser, error) {
obj, err := t.retrieveObject(ctx, prm.Container, prm.Object)
if err != nil {
return nil, err
@ -255,10 +262,10 @@ func (t *TestFrostFS) RangeObject(ctx context.Context, prm PrmObjectRange) (io.R
return io.NopCloser(bytes.NewReader(payload)), nil
}
func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.ID, error) {
func (t *TestFrostFS) CreateObject(_ context.Context, prm frostfs.PrmObjectCreate) (*frostfs.CreateObjectResult, error) {
b := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, b); err != nil {
return oid.ID{}, err
return nil, err
}
var id oid.ID
id.SetSHA256(sha256.Sum256(b))
@ -266,7 +273,7 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.
attrs := make([]object.Attribute, 0)
if err := t.objectPutErrors[prm.Filepath]; err != nil {
return oid.ID{}, err
return nil, err
}
if prm.Filepath != "" {
@ -311,7 +318,7 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.
if prm.Payload != nil {
all, err := io.ReadAll(prm.Payload)
if err != nil {
return oid.ID{}, err
return nil, err
}
obj.SetPayload(all)
obj.SetPayloadSize(uint64(len(all)))
@ -325,10 +332,13 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.
addr := newAddress(cnrID, objID)
t.objects[addr.EncodeToString()] = obj
return objID, nil
return &frostfs.CreateObjectResult{
ObjectID: objID,
CreationEpoch: t.currentEpoch - 1,
}, nil
}
func (t *TestFrostFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
func (t *TestFrostFS) DeleteObject(ctx context.Context, prm frostfs.PrmObjectDelete) error {
var addr oid.Address
addr.SetContainer(prm.Container)
addr.SetObject(prm.Object)
@ -340,7 +350,7 @@ func (t *TestFrostFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) err
if _, ok := t.objects[addr.EncodeToString()]; ok {
owner := getBearerOwner(ctx)
if !t.checkAccess(prm.Container, owner) {
return ErrAccessDenied
return frostfs.ErrAccessDenied
}
delete(t.objects, addr.EncodeToString())
@ -367,7 +377,7 @@ func (t *TestFrostFS) AllObjects(cnrID cid.ID) []oid.ID {
return result
}
func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) ([]oid.ID, error) {
func (t *TestFrostFS) SearchObjects(_ context.Context, prm frostfs.PrmObjectSearch) ([]oid.ID, error) {
filters := object.NewSearchFilters()
filters.AddRootFilter()
@ -404,6 +414,62 @@ func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) ([]o
return res, nil
}
func (t *TestFrostFS) NetworkInfo(context.Context) (netmap.NetworkInfo, error) {
ni := netmap.NetworkInfo{}
ni.SetCurrentEpoch(t.currentEpoch)
ni.SetEpochDuration(60)
ni.SetMsPerBlock(1000)
return ni, nil
}
func (t *TestFrostFS) PatchObject(ctx context.Context, prm frostfs.PrmObjectPatch) (oid.ID, error) {
obj, err := t.retrieveObject(ctx, prm.Container, prm.Object)
if err != nil {
return oid.ID{}, err
}
newObj := *obj
patchBytes, err := io.ReadAll(prm.Payload)
if err != nil {
return oid.ID{}, err
}
var newPayload []byte
if prm.Offset > 0 {
newPayload = append(newPayload, obj.Payload()[:prm.Offset]...)
}
newPayload = append(newPayload, patchBytes...)
if prm.Offset+prm.Length < obj.PayloadSize() {
newPayload = append(newPayload, obj.Payload()[prm.Offset+prm.Length:]...)
}
newObj.SetPayload(newPayload)
newObj.SetPayloadSize(uint64(len(newPayload)))
var hash checksum.Checksum
checksum.Calculate(&hash, checksum.SHA256, newPayload)
newObj.SetPayloadChecksum(hash)
newID := oidtest.ID()
newObj.SetID(newID)
t.objects[newAddress(prm.Container, newID).EncodeToString()] = &newObj
return newID, nil
}
func (t *TestFrostFS) AddContainerPolicyChain(_ context.Context, prm frostfs.PrmAddContainerPolicyChain) error {
list, ok := t.chains[prm.ContainerID.EncodeToString()]
if !ok {
return errors.New("container not found")
}
t.chains[prm.ContainerID.EncodeToString()] = append(list, prm.Chain)
return nil
}
func (t *TestFrostFS) checkAccess(cnrID cid.ID, owner user.ID) bool {
cnr, ok := t.containers[cnrID.EncodeToString()]
if !ok {

View file

@ -6,7 +6,7 @@ import (
"crypto/rand"
"encoding/json"
"encoding/xml"
stderrors "errors"
"errors"
"fmt"
"io"
"net/url"
@ -17,9 +17,10 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
@ -35,39 +36,41 @@ import (
type (
BucketResolver interface {
Resolve(ctx context.Context, name string) (cid.ID, error)
Resolve(ctx context.Context, zone, name string) (cid.ID, error)
}
FeatureSettings interface {
ClientCut() bool
BufferMaxSizeForPut() uint64
MD5Enabled() bool
FormContainerZone(ns string) (zone string, isDefault bool)
FormContainerZone(ns string) string
}
Layer struct {
frostFS FrostFS
gateOwner user.ID
log *zap.Logger
anonKey AnonymousKey
resolver BucketResolver
cache *Cache
treeService TreeService
features FeatureSettings
gateKey *keys.PrivateKey
corsCnrInfo *data.BucketInfo
frostFS frostfs.FrostFS
gateOwner user.ID
log *zap.Logger
anonKey AnonymousKey
resolver BucketResolver
cache *Cache
treeService tree.Service
features FeatureSettings
gateKey *keys.PrivateKey
corsCnrInfo *data.BucketInfo
lifecycleCnrInfo *data.BucketInfo
}
Config struct {
GateOwner user.ID
ChainAddress string
Cache *Cache
AnonKey AnonymousKey
Resolver BucketResolver
TreeService TreeService
Features FeatureSettings
GateKey *keys.PrivateKey
CORSCnrInfo *data.BucketInfo
GateOwner user.ID
ChainAddress string
Cache *Cache
AnonKey AnonymousKey
Resolver BucketResolver
TreeService tree.Service
Features FeatureSettings
GateKey *keys.PrivateKey
CORSCnrInfo *data.BucketInfo
LifecycleCnrInfo *data.BucketInfo
}
// AnonymousKey contains data for anonymous requests.
@ -101,14 +104,14 @@ type (
PutObjectParams struct {
BktInfo *data.BucketInfo
Object string
Size uint64
Size *uint64
Reader io.Reader
Header map[string]string
Lock *data.ObjectLock
Encryption encryption.Params
CopiesNumbers []uint32
CompleteMD5Hash string
ContentMD5 string
ContentMD5 *string
ContentSHA256Hash string
}
@ -122,10 +125,11 @@ type (
}
DeleteObjectParams struct {
BktInfo *data.BucketInfo
Objects []*VersionedObject
Settings *data.BucketSettings
IsMultiple bool
BktInfo *data.BucketInfo
Objects []*VersionedObject
Settings *data.BucketSettings
NetworkInfo netmap.NetworkInfo
IsMultiple bool
}
// PutSettingsParams stores object copy request parameters.
@ -157,6 +161,7 @@ type (
DstEncryption encryption.Params
CopiesNumbers []uint32
}
// CreateBucketParams stores bucket create request parameters.
CreateBucketParams struct {
Name string
@ -231,18 +236,19 @@ func (p HeadObjectParams) Versioned() bool {
// NewLayer creates an instance of a Layer. It checks credentials
// and establishes gRPC connection with the node.
func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) *Layer {
func NewLayer(log *zap.Logger, frostFS frostfs.FrostFS, config *Config) *Layer {
return &Layer{
frostFS: frostFS,
log: log,
gateOwner: config.GateOwner,
anonKey: config.AnonKey,
resolver: config.Resolver,
cache: config.Cache,
treeService: config.TreeService,
features: config.Features,
gateKey: config.GateKey,
corsCnrInfo: config.CORSCnrInfo,
frostFS: frostFS,
log: log,
gateOwner: config.GateOwner,
anonKey: config.AnonKey,
resolver: config.Resolver,
cache: config.Cache,
treeService: config.TreeService,
features: config.Features,
gateKey: config.GateKey,
corsCnrInfo: config.CORSCnrInfo,
lifecycleCnrInfo: config.LifecycleCnrInfo,
}
}
@ -294,7 +300,7 @@ func (n *Layer) reqLogger(ctx context.Context) *zap.Logger {
return n.log
}
func (n *Layer) prepareAuthParameters(ctx context.Context, prm *PrmAuth, bktOwner user.ID) {
func (n *Layer) prepareAuthParameters(ctx context.Context, prm *frostfs.PrmAuth, bktOwner user.ID) {
if prm.BearerToken != nil || prm.PrivateKey != nil {
return
}
@ -317,21 +323,21 @@ func (n *Layer) GetBucketInfo(ctx context.Context, name string) (*data.BucketInf
}
reqInfo := middleware.GetReqInfo(ctx)
zone, _ := n.features.FormContainerZone(reqInfo.Namespace)
zone := n.features.FormContainerZone(reqInfo.Namespace)
if bktInfo := n.cache.GetBucket(zone, name); bktInfo != nil {
return bktInfo, nil
}
containerID, err := n.ResolveBucket(ctx, name)
containerID, err := n.ResolveBucket(ctx, zone, name)
if err != nil {
if strings.Contains(err.Error(), "not found") {
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchBucket), err.Error())
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchBucket), err.Error())
}
return nil, err
}
prm := PrmContainer{
prm := frostfs.PrmContainer{
ContainerID: containerID,
SessionToken: n.SessionTokenForRead(ctx),
}
@ -347,13 +353,13 @@ func (n *Layer) ResolveCID(ctx context.Context, name string) (cid.ID, error) {
}
reqInfo := middleware.GetReqInfo(ctx)
zone, _ := n.features.FormContainerZone(reqInfo.Namespace)
zone := n.features.FormContainerZone(reqInfo.Namespace)
if bktInfo := n.cache.GetBucket(zone, name); bktInfo != nil {
return bktInfo.CID, nil
}
return n.ResolveBucket(ctx, name)
return n.ResolveBucket(ctx, zone, name)
}
// ListBuckets returns all user containers. The name of the bucket is a container
@ -392,9 +398,9 @@ func (n *Layer) GetObject(ctx context.Context, p *GetObjectParams) (*ObjectPaylo
if err != nil {
if client.IsErrObjectNotFound(err) {
if p.Versioned {
err = fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchVersion), err.Error())
err = fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchVersion), err.Error())
} else {
err = fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchKey), err.Error())
err = fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error())
}
}
@ -523,7 +529,7 @@ func (n *Layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.Exte
return n.PutObject(ctx, &PutObjectParams{
BktInfo: p.DstBktInfo,
Object: p.DstObject,
Size: p.DstSize,
Size: &p.DstSize,
Reader: objPayload,
Header: p.Header,
Encryption: p.DstEncryption,
@ -542,7 +548,8 @@ func getRandomOID() (oid.ID, error) {
return objID, nil
}
func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings *data.BucketSettings, obj *VersionedObject) *VersionedObject {
func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings *data.BucketSettings, obj *VersionedObject,
networkInfo netmap.NetworkInfo) *VersionedObject {
if len(obj.VersionID) != 0 || settings.Unversioned() {
var nodeVersions []*data.NodeVersion
if nodeVersions, obj.Error = n.getNodeVersionsToDelete(ctx, bkt, obj); obj.Error != nil {
@ -624,6 +631,7 @@ func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
Created: &now,
Owner: &n.gateOwner,
IsDeleteMarker: true,
CreationEpoch: networkInfo.CurrentEpoch(),
},
IsUnversioned: settings.VersioningSuspended(),
}
@ -648,22 +656,22 @@ func (n *Layer) handleNotFoundError(bkt *data.BucketInfo, obj *VersionedObject)
}
func isNotFoundError(err error) bool {
return errors.IsS3Error(err, errors.ErrNoSuchKey) ||
errors.IsS3Error(err, errors.ErrNoSuchVersion)
return apierr.IsS3Error(err, apierr.ErrNoSuchKey) ||
apierr.IsS3Error(err, apierr.ErrNoSuchVersion)
}
func (n *Layer) getNodeVersionsToDelete(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject) ([]*data.NodeVersion, error) {
var versionsToDelete []*data.NodeVersion
versions, err := n.treeService.GetVersions(ctx, bkt, obj.Name)
if err != nil {
if stderrors.Is(err, ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error())
}
return nil, err
}
if len(versions) == 0 {
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", s3errors.GetAPIError(s3errors.ErrNoSuchVersion))
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", apierr.GetAPIError(apierr.ErrNoSuchVersion))
}
sort.Slice(versions, func(i, j int) bool {
@ -705,7 +713,7 @@ func (n *Layer) getNodeVersionsToDelete(ctx context.Context, bkt *data.BucketInf
}
if len(versionsToDelete) == 0 {
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", s3errors.GetAPIError(s3errors.ErrNoSuchVersion))
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", apierr.GetAPIError(apierr.ErrNoSuchVersion))
}
n.reqLogger(ctx).Debug(logs.GetTreeNodeToDelete, zap.Stringer("cid", bkt.CID), zap.Strings("oids", oids))
@ -766,7 +774,7 @@ func (n *Layer) removeCombinedObject(ctx context.Context, bkt *data.BucketInfo,
// DeleteObjects from the storage.
func (n *Layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*VersionedObject {
for i, obj := range p.Objects {
p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.Settings, obj)
p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.Settings, obj, p.NetworkInfo)
if p.IsMultiple && p.Objects[i].Error != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error))
}
@ -778,23 +786,23 @@ func (n *Layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*Ver
func (n *Layer) CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) {
bktInfo, err := n.GetBucketInfo(ctx, p.Name)
if err != nil {
if errors.IsS3Error(err, errors.ErrNoSuchBucket) {
if apierr.IsS3Error(err, apierr.ErrNoSuchBucket) {
return n.createContainer(ctx, p)
}
return nil, err
}
if p.SessionContainerCreation != nil && session.IssuedBy(*p.SessionContainerCreation, bktInfo.Owner) {
return nil, errors.GetAPIError(errors.ErrBucketAlreadyOwnedByYou)
return nil, apierr.GetAPIError(apierr.ErrBucketAlreadyOwnedByYou)
}
return nil, errors.GetAPIError(errors.ErrBucketAlreadyExists)
return nil, apierr.GetAPIError(apierr.ErrBucketAlreadyExists)
}
func (n *Layer) ResolveBucket(ctx context.Context, name string) (cid.ID, error) {
func (n *Layer) ResolveBucket(ctx context.Context, zone, name string) (cid.ID, error) {
var cnrID cid.ID
if err := cnrID.DecodeString(name); err != nil {
if cnrID, err = n.resolver.Resolve(ctx, name); err != nil {
if cnrID, err = n.resolver.Resolve(ctx, zone, name); err != nil {
return cid.ID{}, err
}
@ -815,7 +823,7 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
return err
}
if len(res) != 0 {
return errors.GetAPIError(errors.ErrBucketNotEmpty)
return apierr.GetAPIError(apierr.ErrBucketNotEmpty)
}
}
@ -826,6 +834,11 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
n.reqLogger(ctx).Error(logs.GetBucketCors, zap.Error(err))
}
lifecycleObj, treeErr := n.treeService.GetBucketLifecycleConfiguration(ctx, p.BktInfo)
if treeErr != nil {
n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr))
}
err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
if err != nil {
return fmt.Errorf("delete container: %w", err)
@ -835,5 +848,33 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
n.deleteCORSObject(ctx, p.BktInfo, corsObj)
}
if treeErr == nil && !lifecycleObj.Container().Equals(p.BktInfo.CID) {
n.deleteLifecycleObject(ctx, p.BktInfo, lifecycleObj)
}
return nil
}
func (n *Layer) DeleteContainer(ctx context.Context, p *DeleteBucketParams) error {
n.cache.DeleteBucket(p.BktInfo)
if err := n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken); err != nil {
return fmt.Errorf("delete container: %w", err)
}
return nil
}
func (n *Layer) GetNetworkInfo(ctx context.Context) (netmap.NetworkInfo, error) {
cachedInfo := n.cache.GetNetworkInfo()
if cachedInfo != nil {
return *cachedInfo, nil
}
networkInfo, err := n.frostFS.NetworkInfo(ctx)
if err != nil {
return netmap.NetworkInfo{}, fmt.Errorf("get network info: %w", err)
}
n.cache.PutNetworkInfo(networkInfo)
return networkInfo, nil
}

146
api/layer/lifecycle.go Normal file
View file

@ -0,0 +1,146 @@
package layer
import (
"bytes"
"context"
"encoding/xml"
"errors"
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"go.uber.org/zap"
)
type PutBucketLifecycleParams struct {
BktInfo *data.BucketInfo
LifecycleCfg *data.LifecycleConfiguration
CopiesNumbers []uint32
}
func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucketLifecycleParams) error {
cfgBytes, err := xml.Marshal(p.LifecycleCfg)
if err != nil {
return fmt.Errorf("marshal lifecycle configuration: %w", err)
}
prm := frostfs.PrmObjectCreate{
Payload: bytes.NewReader(cfgBytes),
Filepath: p.BktInfo.LifecycleConfigurationObjectName(),
CreationTime: TimeNow(ctx),
}
var lifecycleBkt *data.BucketInfo
if n.lifecycleCnrInfo == nil {
lifecycleBkt = p.BktInfo
prm.CopiesNumber = p.CopiesNumbers
} else {
lifecycleBkt = n.lifecycleCnrInfo
prm.PrmAuth.PrivateKey = &n.gateKey.PrivateKey
}
prm.Container = lifecycleBkt.CID
createdObj, err := n.objectPutAndHash(ctx, prm, lifecycleBkt)
if err != nil {
return fmt.Errorf("put lifecycle object: %w", err)
}
objsToDelete, err := n.treeService.PutBucketLifecycleConfiguration(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, createdObj.ID))
objsToDeleteNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !objsToDeleteNotFound {
return err
}
if !objsToDeleteNotFound {
for _, addr := range objsToDelete {
n.deleteLifecycleObject(ctx, p.BktInfo, addr)
}
}
n.cache.PutLifecycleConfiguration(n.BearerOwner(ctx), p.BktInfo, p.LifecycleCfg)
return nil
}
// deleteLifecycleObject removes object and logs in case of error.
func (n *Layer) deleteLifecycleObject(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) {
var prmAuth frostfs.PrmAuth
lifecycleBkt := bktInfo
if !addr.Container().Equals(bktInfo.CID) {
lifecycleBkt = &data.BucketInfo{CID: addr.Container()}
prmAuth.PrivateKey = &n.gateKey.PrivateKey
}
if err := n.objectDeleteWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth); err != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteLifecycleObject, zap.Error(err),
zap.String("cid", lifecycleBkt.CID.EncodeToString()),
zap.String("oid", addr.Object().EncodeToString()))
}
}
func (n *Layer) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (*data.LifecycleConfiguration, error) {
owner := n.BearerOwner(ctx)
if cfg := n.cache.GetLifecycleConfiguration(owner, bktInfo); cfg != nil {
return cfg, nil
}
addr, err := n.treeService.GetBucketLifecycleConfiguration(ctx, bktInfo)
objNotFound := errors.Is(err, tree.ErrNodeNotFound)
if err != nil && !objNotFound {
return nil, err
}
if objNotFound {
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchLifecycleConfiguration), err.Error())
}
var prmAuth frostfs.PrmAuth
lifecycleBkt := bktInfo
if !addr.Container().Equals(bktInfo.CID) {
lifecycleBkt = &data.BucketInfo{CID: addr.Container()}
prmAuth.PrivateKey = &n.gateKey.PrivateKey
}
obj, err := n.objectGetWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth)
if err != nil {
return nil, fmt.Errorf("get lifecycle object: %w", err)
}
lifecycleCfg := &data.LifecycleConfiguration{}
if err = xml.NewDecoder(obj.Payload).Decode(&lifecycleCfg); err != nil {
return nil, fmt.Errorf("unmarshal lifecycle configuration: %w", err)
}
n.cache.PutLifecycleConfiguration(owner, bktInfo, lifecycleCfg)
for i := range lifecycleCfg.Rules {
if lifecycleCfg.Rules[i].Expiration != nil {
lifecycleCfg.Rules[i].Expiration.Epoch = nil
}
}
return lifecycleCfg, nil
}
func (n *Layer) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) error {
objs, err := n.treeService.DeleteBucketLifecycleConfiguration(ctx, bktInfo)
objsNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !objsNotFound {
return err
}
if !objsNotFound {
for _, addr := range objs {
n.deleteLifecycleObject(ctx, bktInfo, addr)
}
}
n.cache.DeleteLifecycleConfiguration(bktInfo)
return nil
}

View file

@ -0,0 +1,61 @@
package layer
import (
"crypto/md5"
"encoding/xml"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"github.com/stretchr/testify/require"
)
func TestBucketLifecycle(t *testing.T) {
tc := prepareContext(t)
lifecycle := &data.LifecycleConfiguration{
XMLName: xml.Name{
Space: `http://s3.amazonaws.com/doc/2006-03-01/`,
Local: "LifecycleConfiguration",
},
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
},
},
}
lifecycleBytes, err := xml.Marshal(lifecycle)
require.NoError(t, err)
hash := md5.New()
hash.Write(lifecycleBytes)
_, err = tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
require.Equal(t, apierr.GetAPIError(apierr.ErrNoSuchLifecycleConfiguration), frosterr.UnwrapErr(err))
err = tc.layer.DeleteBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
require.NoError(t, err)
err = tc.layer.PutBucketLifecycleConfiguration(tc.ctx, &PutBucketLifecycleParams{
BktInfo: tc.bktInfo,
LifecycleCfg: lifecycle,
})
require.NoError(t, err)
cfg, err := tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
require.NoError(t, err)
require.Equal(t, *lifecycle, *cfg)
err = tc.layer.DeleteBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
require.NoError(t, err)
_, err = tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
require.Equal(t, apierr.GetAPIError(apierr.ErrNoSuchLifecycleConfiguration), frosterr.UnwrapErr(err))
}
func ptr[T any](t T) *T {
return &t
}

View file

@ -11,7 +11,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"github.com/panjf2000/ants/v2"
@ -585,7 +585,7 @@ func shouldSkip(node *data.ExtendedNodeVersion, p commonVersionsListingParams, e
return true
}
if p.Bookmark != "" {
if p.Bookmark != "" && p.Bookmark != p.Marker {
if _, ok := existed[continuationToken]; !ok {
if p.Bookmark != node.NodeVersion.OID.EncodeToString() {
return true
@ -691,10 +691,10 @@ func filterVersionsByMarker(objects []*data.ExtendedNodeVersion, p *ListObjectVe
return objects[j+1:], nil
}
}
return nil, s3errors.GetAPIError(s3errors.ErrInvalidVersion)
return nil, apierr.GetAPIError(apierr.ErrInvalidVersion)
} else if obj.NodeVersion.FilePath > p.KeyMarker {
if p.VersionIDMarker != "" {
return nil, s3errors.GetAPIError(s3errors.ErrInvalidVersion)
return nil, apierr.GetAPIError(apierr.ErrInvalidVersion)
}
return objects[i:], nil
}

View file

@ -17,8 +17,10 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
@ -150,6 +152,11 @@ func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
metaSize += len(p.Data.TagSet)
}
networkInfo, err := n.GetNetworkInfo(ctx)
if err != nil {
return err
}
info := &data.MultipartInfo{
Key: p.Info.Key,
UploadID: p.Info.UploadID,
@ -157,6 +164,7 @@ func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
Created: TimeNow(ctx),
Meta: make(map[string]string, metaSize),
CopiesNumbers: p.CopiesNumbers,
CreationEpoch: networkInfo.CurrentEpoch(),
}
for key, val := range p.Header {
@ -181,14 +189,14 @@ func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
func (n *Layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, error) {
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return "", fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchUpload), err.Error())
if errors.Is(err, tree.ErrNodeNotFound) {
return "", fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchUpload), err.Error())
}
return "", err
}
if p.Size > UploadMaxSize {
return "", fmt.Errorf("%w: %d/%d", s3errors.GetAPIError(s3errors.ErrEntityTooLarge), p.Size, UploadMaxSize)
return "", fmt.Errorf("%w: %d/%d", apierr.GetAPIError(apierr.ErrEntityTooLarge), p.Size, UploadMaxSize)
}
objInfo, err := n.uploadPart(ctx, multipartInfo, p)
@ -203,11 +211,11 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
encInfo := FormEncryptionInfo(multipartInfo.Meta)
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
return nil, s3errors.GetAPIError(s3errors.ErrInvalidEncryptionParameters)
return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters)
}
bktInfo := p.Info.Bkt
prm := PrmObjectCreate{
prm := frostfs.PrmObjectCreate{
Container: bktInfo.CID,
Attributes: make([][2]string, 2),
Payload: p.Reader,
@ -229,76 +237,78 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
prm.Attributes[0][0], prm.Attributes[0][1] = UploadIDAttributeName, p.Info.UploadID
prm.Attributes[1][0], prm.Attributes[1][1] = UploadPartNumberAttributeName, strconv.Itoa(p.PartNumber)
size, id, hash, md5Hash, err := n.objectPutAndHash(ctx, prm, bktInfo)
createdObj, err := n.objectPutAndHash(ctx, prm, bktInfo)
if err != nil {
return nil, err
}
if len(p.ContentMD5) > 0 {
hashBytes, err := base64.StdEncoding.DecodeString(p.ContentMD5)
if err != nil {
return nil, s3errors.GetAPIError(s3errors.ErrInvalidDigest)
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
}
if hex.EncodeToString(hashBytes) != hex.EncodeToString(md5Hash) {
prm := PrmObjectDelete{
Object: id,
if hex.EncodeToString(hashBytes) != hex.EncodeToString(createdObj.MD5Sum) {
prm := frostfs.PrmObjectDelete{
Object: createdObj.ID,
Container: bktInfo.CID,
}
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
err = n.frostFS.DeleteObject(ctx, prm)
if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
}
return nil, s3errors.GetAPIError(s3errors.ErrInvalidDigest)
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
}
}
if p.Info.Encryption.Enabled() {
size = decSize
createdObj.Size = decSize
}
if !p.Info.Encryption.Enabled() && len(p.ContentSHA256Hash) > 0 && !auth.IsStandardContentSHA256(p.ContentSHA256Hash) {
contentHashBytes, err := hex.DecodeString(p.ContentSHA256Hash)
if err != nil {
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
}
if !bytes.Equal(contentHashBytes, hash) {
err = n.objectDelete(ctx, bktInfo, id)
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
err = n.objectDelete(ctx, bktInfo, createdObj.ID)
if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
}
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
}
}
n.reqLogger(ctx).Debug(logs.UploadPart,
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
partInfo := &data.PartInfo{
Key: p.Info.Key,
UploadID: p.Info.UploadID,
Number: p.PartNumber,
OID: id,
Size: size,
ETag: hex.EncodeToString(hash),
OID: createdObj.ID,
Size: createdObj.Size,
ETag: hex.EncodeToString(createdObj.HashSum),
Created: prm.CreationTime,
MD5: hex.EncodeToString(md5Hash),
MD5: hex.EncodeToString(createdObj.MD5Sum),
}
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)
oldPartIDNotFound := errors.Is(err, ErrNoNodeToRemove)
oldPartIDs, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)
oldPartIDNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !oldPartIDNotFound {
return nil, err
}
if !oldPartIDNotFound {
if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteOldPartObject, zap.Error(err),
zap.String("cid", bktInfo.CID.EncodeToString()),
zap.String("oid", oldPartID.EncodeToString()))
for _, oldPartID := range oldPartIDs {
if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteOldPartObject, zap.Error(err),
zap.String("cid", bktInfo.CID.EncodeToString()),
zap.String("oid", oldPartID.EncodeToString()))
}
}
}
objInfo := &data.ObjectInfo{
ID: id,
ID: createdObj.ID,
CID: bktInfo.CID,
Owner: bktInfo.Owner,
@ -315,8 +325,8 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
func (n *Layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.ObjectInfo, error) {
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchUpload), err.Error())
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchUpload), err.Error())
}
return nil, err
}
@ -332,11 +342,11 @@ func (n *Layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
if p.Range != nil {
size = p.Range.End - p.Range.Start + 1
if p.Range.End > srcObjectSize {
return nil, fmt.Errorf("%w: %d-%d/%d", s3errors.GetAPIError(s3errors.ErrInvalidCopyPartRangeSource), p.Range.Start, p.Range.End, srcObjectSize)
return nil, fmt.Errorf("%w: %d-%d/%d", apierr.GetAPIError(apierr.ErrInvalidCopyPartRangeSource), p.Range.Start, p.Range.End, srcObjectSize)
}
}
if size > UploadMaxSize {
return nil, fmt.Errorf("%w: %d/%d", s3errors.GetAPIError(s3errors.ErrEntityTooLarge), size, UploadMaxSize)
return nil, fmt.Errorf("%w: %d/%d", apierr.GetAPIError(apierr.ErrEntityTooLarge), size, UploadMaxSize)
}
objPayload, err := n.GetObject(ctx, &GetObjectParams{
@ -363,7 +373,7 @@ func (n *Layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipartParams) (*UploadData, *data.ExtendedObjectInfo, error) {
for i := 1; i < len(p.Parts); i++ {
if p.Parts[i].PartNumber <= p.Parts[i-1].PartNumber {
return nil, nil, s3errors.GetAPIError(s3errors.ErrInvalidPartOrder)
return nil, nil, apierr.GetAPIError(apierr.ErrInvalidPartOrder)
}
}
@ -374,25 +384,24 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
encInfo := FormEncryptionInfo(multipartInfo.Meta)
if len(partsInfo) < len(p.Parts) {
return nil, nil, fmt.Errorf("%w: found %d parts, need %d", s3errors.GetAPIError(s3errors.ErrInvalidPart), len(partsInfo), len(p.Parts))
return nil, nil, fmt.Errorf("%w: found %d parts, need %d", apierr.GetAPIError(apierr.ErrInvalidPart), len(partsInfo), len(p.Parts))
}
var multipartObjetSize uint64
var encMultipartObjectSize uint64
parts := make([]*data.PartInfo, 0, len(p.Parts))
parts := make([]*data.PartInfoExtended, 0, len(p.Parts))
var completedPartsHeader strings.Builder
md5Hash := md5.New()
for i, part := range p.Parts {
partInfo := partsInfo[part.PartNumber]
if partInfo == nil || data.UnQuote(part.ETag) != partInfo.GetETag(n.features.MD5Enabled()) {
return nil, nil, fmt.Errorf("%w: unknown part %d or etag mismatched", s3errors.GetAPIError(s3errors.ErrInvalidPart), part.PartNumber)
partInfo := partsInfo.Extract(part.PartNumber, data.UnQuote(part.ETag), n.features.MD5Enabled())
if partInfo == nil {
return nil, nil, fmt.Errorf("%w: unknown part %d or etag mismatched", apierr.GetAPIError(apierr.ErrInvalidPart), part.PartNumber)
}
delete(partsInfo, part.PartNumber)
// for the last part we have no minimum size limit
if i != len(p.Parts)-1 && partInfo.Size < UploadMinSize {
return nil, nil, fmt.Errorf("%w: %d/%d", s3errors.GetAPIError(s3errors.ErrEntityTooSmall), partInfo.Size, UploadMinSize)
return nil, nil, fmt.Errorf("%w: %d/%d", apierr.GetAPIError(apierr.ErrEntityTooSmall), partInfo.Size, UploadMinSize)
}
parts = append(parts, partInfo)
multipartObjetSize += partInfo.Size // even if encryption is enabled size is actual (decrypted)
@ -453,7 +462,7 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
Object: p.Info.Key,
Reader: bytes.NewReader(partsData),
Header: initMetadata,
Size: multipartObjetSize,
Size: &multipartObjetSize,
Encryption: p.Info.Encryption,
CopiesNumbers: multipartInfo.CopiesNumbers,
CompleteMD5Hash: hex.EncodeToString(md5Hash.Sum(nil)) + "-" + strconv.Itoa(len(p.Parts)),
@ -464,19 +473,21 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
zap.String("uploadKey", p.Info.Key),
zap.Error(err))
return nil, nil, s3errors.GetAPIError(s3errors.ErrInternalError)
return nil, nil, apierr.GetAPIError(apierr.ErrInternalError)
}
var addr oid.Address
addr.SetContainer(p.Info.Bkt.CID)
for _, partInfo := range partsInfo {
if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil {
n.reqLogger(ctx).Warn(logs.CouldNotDeleteUploadPart,
zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID),
zap.Error(err))
for _, prts := range partsInfo {
for _, partInfo := range prts {
if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil {
n.reqLogger(ctx).Warn(logs.CouldNotDeleteUploadPart,
zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID),
zap.Error(err))
}
addr.SetObject(partInfo.OID)
n.cache.DeleteObject(addr)
}
addr.SetObject(partInfo.OID)
n.cache.DeleteObject(addr)
}
return uploadData, extObjInfo, n.treeService.DeleteMultipartUpload(ctx, p.Info.Bkt, multipartInfo)
@ -548,10 +559,12 @@ func (n *Layer) AbortMultipartUpload(ctx context.Context, p *UploadInfoParams) e
return err
}
for _, info := range parts {
if err = n.objectDelete(ctx, p.Bkt, info.OID); err != nil {
n.reqLogger(ctx).Warn(logs.CouldntDeletePart, zap.String("cid", p.Bkt.CID.EncodeToString()),
zap.String("oid", info.OID.EncodeToString()), zap.Int("part number", info.Number), zap.Error(err))
for _, infos := range parts {
for _, info := range infos {
if err = n.objectDelete(ctx, p.Bkt, info.OID); err != nil {
n.reqLogger(ctx).Warn(logs.CouldntDeletePart, zap.String("cid", p.Bkt.CID.EncodeToString()),
zap.String("oid", info.OID.EncodeToString()), zap.Int("part number", info.Number), zap.Error(err))
}
}
}
@ -568,14 +581,19 @@ func (n *Layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
encInfo := FormEncryptionInfo(multipartInfo.Meta)
if err = p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
return nil, s3errors.GetAPIError(s3errors.ErrInvalidEncryptionParameters)
return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters)
}
res.Owner = multipartInfo.Owner
parts := make([]*Part, 0, len(partsInfo))
for _, partInfo := range partsInfo {
for _, infos := range partsInfo {
sort.Slice(infos, func(i, j int) bool {
return infos[i].Timestamp < infos[j].Timestamp
})
partInfo := infos[len(infos)-1]
parts = append(parts, &Part{
ETag: data.Quote(partInfo.GetETag(n.features.MD5Enabled())),
LastModified: partInfo.Created.UTC().Format(time.RFC3339),
@ -612,11 +630,26 @@ func (n *Layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
return &res, nil
}
func (n *Layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.MultipartInfo, map[int]*data.PartInfo, error) {
type PartsInfo map[int][]*data.PartInfoExtended
func (p PartsInfo) Extract(part int, etag string, md5Enabled bool) *data.PartInfoExtended {
parts := p[part]
for i, info := range parts {
if info.GetETag(md5Enabled) == etag {
p[part] = append(parts[:i], parts[i+1:]...)
return info
}
}
return nil
}
func (n *Layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.MultipartInfo, PartsInfo, error) {
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Bkt, p.Key, p.UploadID)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchUpload), err.Error())
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchUpload), err.Error())
}
return nil, nil, err
}
@ -626,11 +659,11 @@ func (n *Layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
return nil, nil, err
}
res := make(map[int]*data.PartInfo, len(parts))
res := make(map[int][]*data.PartInfoExtended, len(parts))
partsNumbers := make([]int, len(parts))
oids := make([]string, len(parts))
for i, part := range parts {
res[part.Number] = part
res[part.Number] = append(res[part.Number], part)
partsNumbers[i] = part.Number
oids[i] = part.OID.EncodeToString()
}

View file

@ -12,6 +12,7 @@ import (
"fmt"
"io"
"mime"
"net/http"
"path/filepath"
"strconv"
"strings"
@ -19,8 +20,11 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/detector"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -47,7 +51,7 @@ type (
}
DeleteMarkerError struct {
ErrorCode apiErrors.ErrorCode
ErrorCode apierr.ErrorCode
}
)
@ -68,7 +72,7 @@ func newAddress(cnr cid.ID, obj oid.ID) oid.Address {
// objectHead returns all object's headers.
func (n *Layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) (*object.Object, error) {
prm := PrmObjectHead{
prm := frostfs.PrmObjectHead{
Container: bktInfo.CID,
Object: idObj,
}
@ -126,11 +130,11 @@ func (n *Layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Re
// initializes payload reader of the FrostFS object.
// Zero range corresponds to full payload (panics if only offset is set).
func (n *Layer) initFrostFSObjectPayloadReader(ctx context.Context, p getFrostFSParams) (io.Reader, error) {
var prmAuth PrmAuth
var prmAuth frostfs.PrmAuth
n.prepareAuthParameters(ctx, &prmAuth, p.bktInfo.Owner)
if p.off+p.ln != 0 {
prm := PrmObjectRange{
prm := frostfs.PrmObjectRange{
PrmAuth: prmAuth,
Container: p.bktInfo.CID,
Object: p.oid,
@ -140,7 +144,7 @@ func (n *Layer) initFrostFSObjectPayloadReader(ctx context.Context, p getFrostFS
return n.frostFS.RangeObject(ctx, prm)
}
prm := PrmObjectGet{
prm := frostfs.PrmObjectGet{
PrmAuth: prmAuth,
Container: p.bktInfo.CID,
Object: p.oid,
@ -155,17 +159,17 @@ func (n *Layer) initFrostFSObjectPayloadReader(ctx context.Context, p getFrostFS
}
// objectGet returns an object with payload in the object.
func (n *Layer) objectGet(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (*Object, error) {
return n.objectGetBase(ctx, bktInfo, objID, PrmAuth{})
func (n *Layer) objectGet(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (*frostfs.Object, error) {
return n.objectGetBase(ctx, bktInfo, objID, frostfs.PrmAuth{})
}
// objectGetWithAuth returns an object with payload in the object. Uses provided PrmAuth.
func (n *Layer) objectGetWithAuth(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, auth PrmAuth) (*Object, error) {
func (n *Layer) objectGetWithAuth(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, auth frostfs.PrmAuth) (*frostfs.Object, error) {
return n.objectGetBase(ctx, bktInfo, objID, auth)
}
func (n *Layer) objectGetBase(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, auth PrmAuth) (*Object, error) {
prm := PrmObjectGet{
func (n *Layer) objectGetBase(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, auth frostfs.PrmAuth) (*frostfs.Object, error) {
prm := frostfs.PrmObjectGet{
PrmAuth: auth,
Container: bktInfo.CID,
Object: objID,
@ -230,40 +234,46 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
r := p.Reader
if p.Encryption.Enabled() {
p.Header[AttributeDecryptedSize] = strconv.FormatUint(p.Size, 10)
var size uint64
if p.Size != nil {
size = *p.Size
}
p.Header[AttributeDecryptedSize] = strconv.FormatUint(size, 10)
if err = addEncryptionHeaders(p.Header, p.Encryption); err != nil {
return nil, fmt.Errorf("add encryption header: %w", err)
}
var encSize uint64
if r, encSize, err = encryptionReader(p.Reader, p.Size, p.Encryption.Key()); err != nil {
if r, encSize, err = encryptionReader(p.Reader, size, p.Encryption.Key()); err != nil {
return nil, fmt.Errorf("create encrypter: %w", err)
}
p.Size = encSize
p.Size = &encSize
}
if r != nil {
if len(p.Header[api.ContentType]) == 0 {
if contentType := MimeByFilePath(p.Object); len(contentType) == 0 {
d := newDetector(r)
d := detector.NewDetector(r, http.DetectContentType)
if contentType, err := d.Detect(); err == nil {
p.Header[api.ContentType] = contentType
}
r = d.MultiReader()
r = d.RestoredReader()
} else {
p.Header[api.ContentType] = contentType
}
}
}
prm := PrmObjectCreate{
prm := frostfs.PrmObjectCreate{
Container: p.BktInfo.CID,
PayloadSize: p.Size,
Filepath: p.Object,
Payload: r,
CreationTime: TimeNow(ctx),
CopiesNumber: p.CopiesNumbers,
}
if p.Size != nil {
prm.PayloadSize = *p.Size
}
prm.Attributes = make([][2]string, 0, len(p.Header))
@ -271,56 +281,67 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
prm.Attributes = append(prm.Attributes, [2]string{k, v})
}
size, id, hash, md5Hash, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
createdObj, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
if err != nil {
return nil, err
}
if len(p.ContentMD5) > 0 {
headerMd5Hash, err := base64.StdEncoding.DecodeString(p.ContentMD5)
if err != nil {
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)
if !p.Encryption.Enabled() && p.ContentMD5 != nil {
if len(*p.ContentMD5) == 0 {
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
}
if !bytes.Equal(headerMd5Hash, md5Hash) {
err = n.objectDelete(ctx, p.BktInfo, id)
headerMd5Hash, err := base64.StdEncoding.DecodeString(*p.ContentMD5)
if err != nil {
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
}
if !bytes.Equal(headerMd5Hash, createdObj.MD5Sum) {
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
}
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)
return nil, apierr.GetAPIError(apierr.ErrBadDigest)
}
}
if !p.Encryption.Enabled() && len(p.ContentSHA256Hash) > 0 && !auth.IsStandardContentSHA256(p.ContentSHA256Hash) {
contentHashBytes, err := hex.DecodeString(p.ContentSHA256Hash)
if err != nil {
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
}
if !bytes.Equal(contentHashBytes, hash) {
err = n.objectDelete(ctx, p.BktInfo, id)
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
}
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
}
}
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
now := TimeNow(ctx)
newVersion := &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{
OID: id,
ETag: hex.EncodeToString(hash),
FilePath: p.Object,
Size: p.Size,
Created: &now,
Owner: &n.gateOwner,
OID: createdObj.ID,
ETag: hex.EncodeToString(createdObj.HashSum),
FilePath: p.Object,
Created: &now,
Owner: &n.gateOwner,
CreationEpoch: createdObj.CreationEpoch,
},
IsUnversioned: !bktSettings.VersioningEnabled(),
IsCombined: p.Header[MultipartObjectSize] != "",
}
if len(p.CompleteMD5Hash) > 0 {
newVersion.MD5 = p.CompleteMD5Hash
} else {
newVersion.MD5 = hex.EncodeToString(md5Hash)
newVersion.MD5 = hex.EncodeToString(createdObj.MD5Sum)
}
if p.Size != nil {
newVersion.Size = *p.Size
} else {
newVersion.Size = createdObj.Size
}
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
@ -332,7 +353,7 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
ObjVersion: &data.ObjectVersion{
BktInfo: p.BktInfo,
ObjectName: p.Object,
VersionID: id.EncodeToString(),
VersionID: createdObj.ID.EncodeToString(),
},
NewLock: p.Lock,
CopiesNumbers: p.CopiesNumbers,
@ -347,13 +368,13 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
n.cache.CleanListCacheEntriesContainingObject(p.Object, p.BktInfo.CID)
objInfo := &data.ObjectInfo{
ID: id,
ID: createdObj.ID,
CID: p.BktInfo.CID,
Owner: n.gateOwner,
Bucket: p.BktInfo.Name,
Name: p.Object,
Size: size,
Size: createdObj.Size,
Created: prm.CreationTime,
Headers: p.Header,
ContentType: p.Header[api.ContentType],
@ -379,20 +400,20 @@ func (n *Layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.Bucke
node, err := n.treeService.GetLatestVersion(ctx, bkt, objectName)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrNoSuchKey), err.Error())
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error())
}
return nil, err
}
if node.IsDeleteMarker {
return nil, DeleteMarkerError{ErrorCode: apiErrors.ErrNoSuchKey}
return nil, DeleteMarkerError{ErrorCode: apierr.ErrNoSuchKey}
}
meta, err := n.objectHead(ctx, bkt, node.OID)
if err != nil {
if client.IsErrObjectNotFound(err) {
return nil, fmt.Errorf("%w: %s; %s", apiErrors.GetAPIError(apiErrors.ErrNoSuchKey), err.Error(), node.OID.EncodeToString())
return nil, fmt.Errorf("%w: %s; %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error(), node.OID.EncodeToString())
}
return nil, err
}
@ -415,8 +436,8 @@ func (n *Layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
if p.VersionID == data.UnversionedObjectVersionID {
foundVersion, err = n.treeService.GetUnversioned(ctx, bkt, p.Object)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion), err.Error())
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchVersion), err.Error())
}
return nil, err
}
@ -433,7 +454,7 @@ func (n *Layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
}
}
if foundVersion == nil {
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion))
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", apierr.GetAPIError(apierr.ErrNoSuchVersion))
}
}
@ -443,13 +464,13 @@ func (n *Layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
}
if foundVersion.IsDeleteMarker {
return nil, DeleteMarkerError{ErrorCode: apiErrors.ErrMethodNotAllowed}
return nil, DeleteMarkerError{ErrorCode: apierr.ErrMethodNotAllowed}
}
meta, err := n.objectHead(ctx, bkt, foundVersion.OID)
if err != nil {
if client.IsErrObjectNotFound(err) {
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion), err.Error())
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchVersion), err.Error())
}
return nil, err
}
@ -468,16 +489,16 @@ func (n *Layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
// objectDelete puts tombstone object into frostfs.
func (n *Layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) error {
return n.objectDeleteBase(ctx, bktInfo, idObj, PrmAuth{})
return n.objectDeleteBase(ctx, bktInfo, idObj, frostfs.PrmAuth{})
}
// objectDeleteWithAuth puts tombstone object into frostfs. Uses provided PrmAuth.
func (n *Layer) objectDeleteWithAuth(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID, auth PrmAuth) error {
func (n *Layer) objectDeleteWithAuth(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID, auth frostfs.PrmAuth) error {
return n.objectDeleteBase(ctx, bktInfo, idObj, auth)
}
func (n *Layer) objectDeleteBase(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID, auth PrmAuth) error {
prm := PrmObjectDelete{
func (n *Layer) objectDeleteBase(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID, auth frostfs.PrmAuth) error {
prm := frostfs.PrmObjectDelete{
PrmAuth: auth,
Container: bktInfo.CID,
Object: idObj,
@ -491,8 +512,7 @@ func (n *Layer) objectDeleteBase(ctx context.Context, bktInfo *data.BucketInfo,
}
// objectPutAndHash prepare auth parameters and invoke frostfs.CreateObject.
// Returns object ID and payload sha256 hash.
func (n *Layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (uint64, oid.ID, []byte, []byte, error) {
func (n *Layer) objectPutAndHash(ctx context.Context, prm frostfs.PrmObjectCreate, bktInfo *data.BucketInfo) (*data.CreatedObjectInfo, error) {
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
prm.ClientCut = n.features.ClientCut()
prm.BufferMaxSize = n.features.BufferMaxSizeForPut()
@ -505,15 +525,21 @@ func (n *Layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktIn
hash.Write(buf)
md5Hash.Write(buf)
})
id, err := n.frostFS.CreateObject(ctx, prm)
res, err := n.frostFS.CreateObject(ctx, prm)
if err != nil {
if _, errDiscard := io.Copy(io.Discard, prm.Payload); errDiscard != nil {
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard))
}
return 0, oid.ID{}, nil, nil, err
return nil, err
}
return size, id, hash.Sum(nil), md5Hash.Sum(nil), nil
return &data.CreatedObjectInfo{
ID: res.ObjectID,
Size: size,
HashSum: hash.Sum(nil),
MD5Sum: md5Hash.Sum(nil),
CreationEpoch: res.CreationEpoch,
}, nil
}
type logWrapper struct {

View file

@ -8,6 +8,7 @@ import (
"io"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"github.com/stretchr/testify/require"
)
@ -37,14 +38,14 @@ func TestGoroutinesDontLeakInPutAndHash(t *testing.T) {
require.NoError(t, err)
payload := bytes.NewReader(content)
prm := PrmObjectCreate{
prm := frostfs.PrmObjectCreate{
Filepath: tc.obj,
Payload: payload,
}
expErr := errors.New("some error")
tc.testFrostFS.SetObjectPutError(tc.obj, expErr)
_, _, _, _, err = tc.layer.objectPutAndHash(tc.ctx, prm, tc.bktInfo)
_, err = tc.layer.objectPutAndHash(tc.ctx, prm, tc.bktInfo)
require.ErrorIs(t, err, expErr)
require.Empty(t, payload.Len(), "body must be read out otherwise goroutines can leak in wrapReader")
}

265
api/layer/patch.go Normal file
View file

@ -0,0 +1,265 @@
package layer
import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"strconv"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
)
type PatchObjectParams struct {
Object *data.ExtendedObjectInfo
BktInfo *data.BucketInfo
NewBytes io.Reader
Range *RangeParams
VersioningEnabled bool
CopiesNumbers []uint32
}
func (n *Layer) PatchObject(ctx context.Context, p *PatchObjectParams) (*data.ExtendedObjectInfo, error) {
if p.Object.ObjectInfo.Headers[AttributeDecryptedSize] != "" {
return nil, fmt.Errorf("patch encrypted object")
}
if p.Object.ObjectInfo.Headers[MultipartObjectSize] != "" {
return n.patchMultipartObject(ctx, p)
}
prmPatch := frostfs.PrmObjectPatch{
Container: p.BktInfo.CID,
Object: p.Object.ObjectInfo.ID,
Payload: p.NewBytes,
Offset: p.Range.Start,
Length: p.Range.End - p.Range.Start + 1,
ObjectSize: p.Object.ObjectInfo.Size,
}
n.prepareAuthParameters(ctx, &prmPatch.PrmAuth, p.BktInfo.Owner)
createdObj, err := n.patchObject(ctx, prmPatch)
if err != nil {
return nil, fmt.Errorf("patch object: %w", err)
}
newVersion := &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{
OID: createdObj.ID,
ETag: hex.EncodeToString(createdObj.HashSum),
FilePath: p.Object.ObjectInfo.Name,
Size: createdObj.Size,
Created: &p.Object.ObjectInfo.Created,
Owner: &n.gateOwner,
CreationEpoch: p.Object.NodeVersion.CreationEpoch,
},
IsUnversioned: !p.VersioningEnabled,
IsCombined: p.Object.ObjectInfo.Headers[MultipartObjectSize] != "",
}
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
return nil, fmt.Errorf("couldn't add new version to tree service: %w", err)
}
p.Object.ObjectInfo.ID = createdObj.ID
p.Object.ObjectInfo.Size = createdObj.Size
p.Object.ObjectInfo.MD5Sum = ""
p.Object.ObjectInfo.HashSum = hex.EncodeToString(createdObj.HashSum)
p.Object.NodeVersion = newVersion
return p.Object, nil
}
func (n *Layer) patchObject(ctx context.Context, p frostfs.PrmObjectPatch) (*data.CreatedObjectInfo, error) {
objID, err := n.frostFS.PatchObject(ctx, p)
if err != nil {
return nil, fmt.Errorf("patch object: %w", err)
}
prmHead := frostfs.PrmObjectHead{
PrmAuth: p.PrmAuth,
Container: p.Container,
Object: objID,
}
obj, err := n.frostFS.HeadObject(ctx, prmHead)
if err != nil {
return nil, fmt.Errorf("head object: %w", err)
}
payloadChecksum, _ := obj.PayloadChecksum()
return &data.CreatedObjectInfo{
ID: objID,
Size: obj.PayloadSize(),
HashSum: payloadChecksum.Value(),
}, nil
}
func (n *Layer) patchMultipartObject(ctx context.Context, p *PatchObjectParams) (*data.ExtendedObjectInfo, error) {
combinedObj, err := n.objectGet(ctx, p.BktInfo, p.Object.ObjectInfo.ID)
if err != nil {
return nil, fmt.Errorf("get combined object '%s': %w", p.Object.ObjectInfo.ID.EncodeToString(), err)
}
var parts []*data.PartInfo
if err = json.NewDecoder(combinedObj.Payload).Decode(&parts); err != nil {
return nil, fmt.Errorf("unmarshal combined object parts: %w", err)
}
prmPatch := frostfs.PrmObjectPatch{
Container: p.BktInfo.CID,
}
n.prepareAuthParameters(ctx, &prmPatch.PrmAuth, p.BktInfo.Owner)
off, ln := p.Range.Start, p.Range.End-p.Range.Start+1
var multipartObjectSize uint64
for i, part := range parts {
if off > part.Size || (off == part.Size && i != len(parts)-1) || ln == 0 {
multipartObjectSize += part.Size
if ln != 0 {
off -= part.Size
}
continue
}
var createdObj *data.CreatedObjectInfo
createdObj, off, ln, err = n.patchPart(ctx, part, p, &prmPatch, off, ln, i == len(parts)-1)
if err != nil {
return nil, fmt.Errorf("patch part: %w", err)
}
parts[i].OID = createdObj.ID
parts[i].Size = createdObj.Size
parts[i].MD5 = ""
parts[i].ETag = hex.EncodeToString(createdObj.HashSum)
multipartObjectSize += createdObj.Size
}
return n.updateCombinedObject(ctx, parts, multipartObjectSize, p)
}
// Returns patched part info, updated offset and length.
func (n *Layer) patchPart(ctx context.Context, part *data.PartInfo, p *PatchObjectParams, prmPatch *frostfs.PrmObjectPatch, off, ln uint64, lastPart bool) (*data.CreatedObjectInfo, uint64, uint64, error) {
if off == 0 && ln >= part.Size {
curLen := part.Size
if lastPart {
curLen = ln
}
prm := frostfs.PrmObjectCreate{
Container: p.BktInfo.CID,
Payload: io.LimitReader(p.NewBytes, int64(curLen)),
CreationTime: part.Created,
CopiesNumber: p.CopiesNumbers,
}
createdObj, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
if err != nil {
return nil, 0, 0, fmt.Errorf("put new part object '%s': %w", part.OID.EncodeToString(), err)
}
ln -= curLen
return createdObj, off, ln, err
}
curLen := ln
if off+curLen > part.Size && !lastPart {
curLen = part.Size - off
}
prmPatch.Object = part.OID
prmPatch.ObjectSize = part.Size
prmPatch.Offset = off
prmPatch.Length = curLen
prmPatch.Payload = io.LimitReader(p.NewBytes, int64(prmPatch.Length))
createdObj, err := n.patchObject(ctx, *prmPatch)
if err != nil {
return nil, 0, 0, fmt.Errorf("patch part object '%s': %w", part.OID.EncodeToString(), err)
}
ln -= curLen
off = 0
return createdObj, off, ln, nil
}
func (n *Layer) updateCombinedObject(ctx context.Context, parts []*data.PartInfo, fullObjSize uint64, p *PatchObjectParams) (*data.ExtendedObjectInfo, error) {
newParts, err := json.Marshal(parts)
if err != nil {
return nil, fmt.Errorf("marshal parts for combined object: %w", err)
}
var headerParts strings.Builder
for i, part := range parts {
headerPart := part.ToHeaderString()
if i != len(parts)-1 {
headerPart += ","
}
headerParts.WriteString(headerPart)
}
prm := frostfs.PrmObjectCreate{
Container: p.BktInfo.CID,
PayloadSize: fullObjSize,
Filepath: p.Object.ObjectInfo.Name,
Payload: bytes.NewReader(newParts),
CreationTime: p.Object.ObjectInfo.Created,
CopiesNumber: p.CopiesNumbers,
}
prm.Attributes = make([][2]string, 0, len(p.Object.ObjectInfo.Headers)+1)
for k, v := range p.Object.ObjectInfo.Headers {
switch k {
case MultipartObjectSize:
prm.Attributes = append(prm.Attributes, [2]string{MultipartObjectSize, strconv.FormatUint(fullObjSize, 10)})
case UploadCompletedParts:
prm.Attributes = append(prm.Attributes, [2]string{UploadCompletedParts, headerParts.String()})
case api.ContentType:
default:
prm.Attributes = append(prm.Attributes, [2]string{k, v})
}
}
prm.Attributes = append(prm.Attributes, [2]string{api.ContentType, p.Object.ObjectInfo.ContentType})
createdObj, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
if err != nil {
return nil, fmt.Errorf("put new combined object: %w", err)
}
newVersion := &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{
OID: createdObj.ID,
ETag: hex.EncodeToString(createdObj.HashSum),
MD5: hex.EncodeToString(createdObj.MD5Sum) + "-" + strconv.Itoa(len(parts)),
FilePath: p.Object.ObjectInfo.Name,
Size: fullObjSize,
Created: &p.Object.ObjectInfo.Created,
Owner: &n.gateOwner,
CreationEpoch: p.Object.NodeVersion.CreationEpoch,
},
IsUnversioned: !p.VersioningEnabled,
IsCombined: p.Object.ObjectInfo.Headers[MultipartObjectSize] != "",
}
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
return nil, fmt.Errorf("couldn't add new version to tree service: %w", err)
}
p.Object.ObjectInfo.ID = createdObj.ID
p.Object.ObjectInfo.Size = createdObj.Size
p.Object.ObjectInfo.MD5Sum = hex.EncodeToString(createdObj.MD5Sum) + "-" + strconv.Itoa(len(parts))
p.Object.ObjectInfo.HashSum = hex.EncodeToString(createdObj.HashSum)
p.Object.ObjectInfo.Headers[MultipartObjectSize] = strconv.FormatUint(fullObjSize, 10)
p.Object.ObjectInfo.Headers[UploadCompletedParts] = headerParts.String()
p.Object.NodeVersion = newVersion
return p.Object, nil
}

View file

@ -3,7 +3,7 @@ package layer
import (
"context"
"encoding/xml"
errorsStd "errors"
"errors"
"fmt"
"math"
"strconv"
@ -11,7 +11,9 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
@ -40,7 +42,7 @@ func (n *Layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
}
lockInfo, err := n.treeService.GetLock(ctx, p.ObjVersion.BktInfo, versionNode.ID)
if err != nil && !errorsStd.Is(err, ErrNodeNotFound) {
if err != nil && !errors.Is(err, tree.ErrNodeNotFound) {
return err
}
@ -113,7 +115,7 @@ func (n *Layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion
}
func (n *Layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, lock *data.ObjectLock, copiesNumber []uint32) (oid.ID, error) {
prm := PrmObjectCreate{
prm := frostfs.PrmObjectCreate{
Container: bktInfo.CID,
Locks: []oid.ID{objID},
CreationTime: TimeNow(ctx),
@ -126,8 +128,12 @@ func (n *Layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
return oid.ID{}, err
}
_, id, _, _, err := n.objectPutAndHash(ctx, prm, bktInfo)
return id, err
createdObj, err := n.objectPutAndHash(ctx, prm, bktInfo)
if err != nil {
return oid.ID{}, err
}
return createdObj.ID, nil
}
func (n *Layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion) (*data.LockInfo, error) {
@ -142,7 +148,7 @@ func (n *Layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion)
}
lockInfo, err := n.treeService.GetLock(ctx, objVersion.BktInfo, versionNode.ID)
if err != nil && !errorsStd.Is(err, ErrNodeNotFound) {
if err != nil && !errors.Is(err, tree.ErrNodeNotFound) {
return nil, err
}
if lockInfo == nil {
@ -161,16 +167,16 @@ func (n *Layer) getCORS(ctx context.Context, bkt *data.BucketInfo) (*data.CORSCo
}
addr, err := n.treeService.GetBucketCORS(ctx, bkt)
objNotFound := errorsStd.Is(err, ErrNodeNotFound)
objNotFound := errors.Is(err, tree.ErrNodeNotFound)
if err != nil && !objNotFound {
return nil, err
}
if objNotFound {
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchCORSConfiguration), err.Error())
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchCORSConfiguration), err.Error())
}
var prmAuth PrmAuth
var prmAuth frostfs.PrmAuth
corsBkt := bkt
if !addr.Container().Equals(bkt.CID) && !addr.Container().Equals(cid.ID{}) {
corsBkt = &data.BucketInfo{CID: addr.Container()}
@ -205,7 +211,7 @@ func (n *Layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo)
settings, err := n.treeService.GetSettingsNode(ctx, bktInfo)
if err != nil {
if !errorsStd.Is(err, ErrNodeNotFound) {
if !errors.Is(err, tree.ErrNodeNotFound) {
return nil, err
}
settings = &data.BucketSettings{Versioning: data.VersioningUnversioned}

View file

@ -6,7 +6,8 @@ import (
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -39,8 +40,8 @@ func (n *Layer) GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingPa
tags, err := n.treeService.GetObjectTagging(ctx, p.ObjectVersion.BktInfo, nodeVersion)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return "", nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
if errors.Is(err, tree.ErrNodeNotFound) {
return "", nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error())
}
return "", nil, err
}
@ -62,8 +63,8 @@ func (n *Layer) PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingPa
err = n.treeService.PutObjectTagging(ctx, p.ObjectVersion.BktInfo, nodeVersion, p.TagSet)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
if errors.Is(err, tree.ErrNodeNotFound) {
return fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error())
}
return err
}
@ -81,8 +82,8 @@ func (n *Layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion)
err = n.treeService.DeleteObjectTagging(ctx, p.BktInfo, version)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
if errors.Is(err, tree.ErrNodeNotFound) {
return fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error())
}
return err
}
@ -102,7 +103,7 @@ func (n *Layer) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo)
}
tags, err := n.treeService.GetBucketTagging(ctx, bktInfo)
if err != nil && !errors.Is(err, ErrNodeNotFound) {
if err != nil && !errors.Is(err, tree.ErrNodeNotFound) {
return nil, err
}
@ -155,14 +156,14 @@ func (n *Layer) getNodeVersion(ctx context.Context, objVersion *data.ObjectVersi
}
}
if version == nil {
err = fmt.Errorf("%w: there isn't tree node with requested version id", s3errors.GetAPIError(s3errors.ErrNoSuchVersion))
err = fmt.Errorf("%w: there isn't tree node with requested version id", apierr.GetAPIError(apierr.ErrNoSuchVersion))
}
}
if err == nil && version.IsDeleteMarker && !objVersion.NoErrorOnDeleteMarker {
return nil, fmt.Errorf("%w: found version is delete marker", s3errors.GetAPIError(s3errors.ErrNoSuchKey))
} else if errors.Is(err, ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
return nil, fmt.Errorf("%w: found version is delete marker", apierr.GetAPIError(apierr.ErrNoSuchKey))
} else if errors.Is(err, tree.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchKey), err.Error())
}
if err == nil && version != nil && !version.IsDeleteMarker {

View file

@ -1,4 +1,4 @@
package layer
package tree
import (
"context"
@ -8,8 +8,8 @@ import (
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
// TreeService provide interface to interact with tree service using s3 data models.
type TreeService interface {
// Service provide interface to interact with tree service using s3 data models.
type Service interface {
// PutSettingsNode update or create new settings node in tree service.
PutSettingsNode(ctx context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error
@ -57,11 +57,15 @@ type TreeService interface {
GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error)
// AddPart puts a node to a system tree as a child of appropriate multipart upload
// and returns objectID of a previous part which must be deleted in FrostFS.
// and returns objectIDs of a previous part/s which must be deleted in FrostFS.
//
// If object id to remove is not found returns ErrNoNodeToRemove error.
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)
GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error)
// If object ids to remove is not found returns ErrNoNodeToRemove error.
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDsToDelete []oid.ID, err error)
GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfoExtended, error)
PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error)
GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error)
DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error)
// Compound methods for optimizations

View file

@ -6,8 +6,10 @@ import (
"io"
"sort"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
@ -33,7 +35,7 @@ type TreeServiceMock struct {
locks map[string]map[uint64]*data.LockInfo
tags map[string]map[uint64]map[string]string
multiparts map[string]map[string][]*data.MultipartInfo
parts map[string]map[int]*data.PartInfo
parts map[string]map[int]*data.PartInfoExtended
}
func (t *TreeServiceMock) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
@ -92,7 +94,7 @@ func NewTreeService() *TreeServiceMock {
locks: make(map[string]map[uint64]*data.LockInfo),
tags: make(map[string]map[uint64]map[string]string),
multiparts: make(map[string]map[string][]*data.MultipartInfo),
parts: make(map[string]map[int]*data.PartInfo),
parts: make(map[string]map[int]*data.PartInfoExtended),
}
}
@ -104,7 +106,7 @@ func (t *TreeServiceMock) PutSettingsNode(_ context.Context, bktInfo *data.Bucke
func (t *TreeServiceMock) GetSettingsNode(_ context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
settings, ok := t.settings[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
return nil, tree.ErrNodeNotFound
}
return settings, nil
@ -139,7 +141,7 @@ func (t *TreeServiceMock) PutBucketCORS(_ context.Context, bktInfo *data.BucketI
t.system[bktInfo.CID.EncodeToString()] = systemMap
return nil, ErrNoNodeToRemove
return nil, tree.ErrNoNodeToRemove
}
func (t *TreeServiceMock) DeleteBucketCORS(context.Context, *data.BucketInfo) ([]oid.Address, error) {
@ -149,12 +151,12 @@ func (t *TreeServiceMock) DeleteBucketCORS(context.Context, *data.BucketInfo) ([
func (t *TreeServiceMock) GetVersions(_ context.Context, bktInfo *data.BucketInfo, objectName string) ([]*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
return nil, tree.ErrNodeNotFound
}
versions, ok := cnrVersionsMap[objectName]
if !ok {
return nil, ErrNodeNotFound
return nil, tree.ErrNodeNotFound
}
return versions, nil
@ -163,12 +165,12 @@ func (t *TreeServiceMock) GetVersions(_ context.Context, bktInfo *data.BucketInf
func (t *TreeServiceMock) GetLatestVersion(_ context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
return nil, tree.ErrNodeNotFound
}
versions, ok := cnrVersionsMap[objectName]
if !ok {
return nil, ErrNodeNotFound
return nil, tree.ErrNodeNotFound
}
sort.Slice(versions, func(i, j int) bool {
@ -179,13 +181,13 @@ func (t *TreeServiceMock) GetLatestVersion(_ context.Context, bktInfo *data.Buck
return versions[len(versions)-1], nil
}
return nil, ErrNodeNotFound
return nil, tree.ErrNodeNotFound
}
func (t *TreeServiceMock) InitVersionsByPrefixStream(_ context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) (data.VersionsStream, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
return nil, tree.ErrNodeNotFound
}
var result []*data.NodeVersion
@ -217,12 +219,12 @@ func (t *TreeServiceMock) InitVersionsByPrefixStream(_ context.Context, bktInfo
func (t *TreeServiceMock) GetUnversioned(_ context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
return nil, tree.ErrNodeNotFound
}
versions, ok := cnrVersionsMap[objectName]
if !ok {
return nil, ErrNodeNotFound
return nil, tree.ErrNodeNotFound
}
for _, version := range versions {
@ -231,7 +233,7 @@ func (t *TreeServiceMock) GetUnversioned(_ context.Context, bktInfo *data.Bucket
}
}
return nil, ErrNodeNotFound
return nil, tree.ErrNodeNotFound
}
func (t *TreeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error) {
@ -277,7 +279,7 @@ func (t *TreeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo
func (t *TreeServiceMock) RemoveVersion(_ context.Context, bktInfo *data.BucketInfo, nodeID uint64) error {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return ErrNodeNotFound
return tree.ErrNodeNotFound
}
for key, versions := range cnrVersionsMap {
@ -289,7 +291,7 @@ func (t *TreeServiceMock) RemoveVersion(_ context.Context, bktInfo *data.BucketI
}
}
return ErrNodeNotFound
return tree.ErrNodeNotFound
}
func (t *TreeServiceMock) GetAllVersionsByPrefix(_ context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) {
@ -333,7 +335,7 @@ func (t *TreeServiceMock) GetMultipartUploadsByPrefix(context.Context, *data.Buc
func (t *TreeServiceMock) GetMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error) {
cnrMultipartsMap, ok := t.multiparts[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
return nil, tree.ErrNodeNotFound
}
multiparts := cnrMultipartsMap[objectName]
@ -343,31 +345,34 @@ func (t *TreeServiceMock) GetMultipartUpload(_ context.Context, bktInfo *data.Bu
}
}
return nil, ErrNodeNotFound
return nil, tree.ErrNodeNotFound
}
func (t *TreeServiceMock) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error) {
func (t *TreeServiceMock) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDsToDelete []oid.ID, err error) {
multipartInfo, err := t.GetMultipartUpload(ctx, bktInfo, info.Key, info.UploadID)
if err != nil {
return oid.ID{}, err
return nil, err
}
if multipartInfo.ID != multipartNodeID {
return oid.ID{}, fmt.Errorf("invalid multipart info id")
return nil, fmt.Errorf("invalid multipart info id")
}
partsMap, ok := t.parts[info.UploadID]
if !ok {
partsMap = make(map[int]*data.PartInfo)
partsMap = make(map[int]*data.PartInfoExtended)
}
partsMap[info.Number] = info
partsMap[info.Number] = &data.PartInfoExtended{
PartInfo: *info,
Timestamp: uint64(time.Now().UnixMicro()),
}
t.parts[info.UploadID] = partsMap
return oid.ID{}, nil
return nil, nil
}
func (t *TreeServiceMock) GetParts(_ context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error) {
func (t *TreeServiceMock) GetParts(_ context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfoExtended, error) {
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
var foundMultipart *data.MultipartInfo
@ -383,11 +388,11 @@ LOOP:
}
if foundMultipart == nil {
return nil, ErrNodeNotFound
return nil, tree.ErrNodeNotFound
}
partsMap := t.parts[foundMultipart.UploadID]
result := make([]*data.PartInfo, 0, len(partsMap))
result := make([]*data.PartInfoExtended, 0, len(partsMap))
for _, part := range partsMap {
result = append(result, part)
}
@ -395,6 +400,51 @@ LOOP:
return result, nil
}
func (t *TreeServiceMock) PutBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
if !ok {
systemMap = make(map[string]*data.BaseNodeVersion)
}
systemMap["lifecycle"] = &data.BaseNodeVersion{
OID: addr.Object(),
}
t.system[bktInfo.CID.EncodeToString()] = systemMap
return nil, tree.ErrNoNodeToRemove
}
func (t *TreeServiceMock) GetBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
if !ok {
return oid.Address{}, tree.ErrNodeNotFound
}
node, ok := systemMap["lifecycle"]
if !ok {
return oid.Address{}, tree.ErrNodeNotFound
}
return newAddress(bktInfo.CID, node.OID), nil
}
func (t *TreeServiceMock) DeleteBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
if !ok {
return nil, tree.ErrNoNodeToRemove
}
node, ok := systemMap["lifecycle"]
if !ok {
return nil, tree.ErrNoNodeToRemove
}
delete(systemMap, "lifecycle")
return []oid.Address{newAddress(bktInfo.CID, node.OID)}, nil
}
func (t *TreeServiceMock) DeleteMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
@ -412,7 +462,7 @@ LOOP:
}
if uploadID == "" {
return ErrNodeNotFound
return tree.ErrNodeNotFound
}
delete(t.parts, uploadID)

View file

@ -7,6 +7,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
@ -23,7 +24,7 @@ func (tc *testContext) putObject(content []byte) *data.ObjectInfo {
extObjInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{
BktInfo: tc.bktInfo,
Object: tc.obj,
Size: uint64(len(content)),
Size: ptr(uint64(len(content))),
Reader: bytes.NewReader(content),
Header: make(map[string]string),
})
@ -154,7 +155,7 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
tp := NewTestFrostFS(key)
bktName := "testbucket1"
res, err := tp.CreateContainer(ctx, PrmContainerCreate{
res, err := tp.CreateContainer(ctx, frostfs.PrmContainerCreate{
Name: bktName,
})
require.NoError(t, err)

View file

@ -0,0 +1,155 @@
package middleware
import (
"net/http"
"net/url"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"go.uber.org/zap"
)
const (
wildcardPlaceholder = "<wildcard>"
enabledVHS = "enabled"
disabledVHS = "disabled"
)
type VHSSettings interface {
Domains() []string
GlobalVHS() bool
VHSHeader() string
ServernameHeader() string
VHSNamespacesEnabled() map[string]bool
}
func PrepareAddressStyle(settings VHSSettings, log *zap.Logger) Func {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := GetReqInfo(ctx)
reqLogger := reqLogOrDefault(ctx, log)
statusVHS := r.Header.Get(settings.VHSHeader())
if isVHSAddress(statusVHS, settings.GlobalVHS(), settings.VHSNamespacesEnabled(), reqInfo.Namespace) {
prepareVHSAddress(reqInfo, r, settings)
} else {
preparePathStyleAddress(reqInfo, r, reqLogger)
}
h.ServeHTTP(w, r)
})
}
}
func isVHSAddress(statusVHS string, enabledFlag bool, vhsNamespaces map[string]bool, namespace string) bool {
switch statusVHS {
case enabledVHS:
return true
case disabledVHS:
return false
default:
result := enabledFlag
if v, ok := vhsNamespaces[namespace]; ok {
result = v
}
return result
}
}
func prepareVHSAddress(reqInfo *ReqInfo, r *http.Request, settings VHSSettings) {
reqInfo.RequestVHSEnabled = true
bktName, match := checkDomain(r.Host, getDomains(r, settings))
if match {
if bktName == "" {
reqInfo.RequestType = noneType
} else {
if objName := strings.TrimPrefix(r.URL.Path, "/"); objName != "" {
reqInfo.RequestType = objectType
reqInfo.ObjectName = objName
reqInfo.BucketName = bktName
} else {
reqInfo.RequestType = bucketType
reqInfo.BucketName = bktName
}
}
} else {
parts := strings.Split(r.Host, ".")
reqInfo.BucketName = parts[0]
if objName := strings.TrimPrefix(r.URL.Path, "/"); objName != "" {
reqInfo.RequestType = objectType
reqInfo.ObjectName = objName
} else {
reqInfo.RequestType = bucketType
}
}
}
func getDomains(r *http.Request, settings VHSSettings) []string {
if headerServername := r.Header.Get(settings.ServernameHeader()); headerServername != "" {
return []string{headerServername}
}
return settings.Domains()
}
func preparePathStyleAddress(reqInfo *ReqInfo, r *http.Request, reqLogger *zap.Logger) {
bktObj := strings.TrimPrefix(r.URL.Path, "/")
if bktObj == "" {
reqInfo.RequestType = noneType
} else if ind := strings.IndexByte(bktObj, '/'); ind != -1 && bktObj[ind+1:] != "" {
reqInfo.RequestType = objectType
reqInfo.BucketName = bktObj[:ind]
reqInfo.ObjectName = bktObj[ind+1:]
if r.URL.RawPath != "" {
// we have to do this because of
// https://github.com/go-chi/chi/issues/641
// https://github.com/go-chi/chi/issues/642
if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil {
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err))
} else {
reqInfo.ObjectName = obj
}
}
} else {
reqInfo.RequestType = bucketType
reqInfo.BucketName = strings.TrimSuffix(bktObj, "/")
}
}
func checkDomain(host string, domains []string) (bktName string, match bool) {
partsHost := strings.Split(host, ".")
for _, pattern := range domains {
partsPattern := strings.Split(pattern, ".")
bktName, match = compareMatch(partsHost, partsPattern)
if match {
break
}
}
return
}
func compareMatch(host, pattern []string) (bktName string, match bool) {
if len(host) < len(pattern) {
return "", false
}
i, j := len(host)-1, len(pattern)-1
for j >= 0 && (pattern[j] == wildcardPlaceholder || host[i] == pattern[j]) {
i--
j--
}
switch {
case i == -1:
return "", true
case i == 0 && (j != 0 || host[i] == pattern[j]):
return host[0], true
default:
return "", false
}
}

View file

@ -0,0 +1,443 @@
package middleware
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
)
const (
FrostfsVHSHeader = "X-Frostfs-S3-VHS"
FrostfsServernameHeader = "X-Frostfs-Servername"
)
type VHSSettingsMock struct {
domains []string
}
func (v *VHSSettingsMock) Domains() []string {
return v.domains
}
func (v *VHSSettingsMock) GlobalVHS() bool {
return false
}
func (v *VHSSettingsMock) VHSHeader() string {
return FrostfsVHSHeader
}
func (v *VHSSettingsMock) ServernameHeader() string {
return FrostfsServernameHeader
}
func (v *VHSSettingsMock) VHSNamespacesEnabled() map[string]bool {
return make(map[string]bool)
}
func TestIsVHSAddress(t *testing.T) {
for _, tc := range []struct {
name string
headerStatusVHS string
vhsEnabledFlag bool
vhsNamespaced map[string]bool
namespace string
expected bool
}{
{
name: "vhs disabled",
expected: false,
},
{
name: "vhs disabled for namespace",
vhsEnabledFlag: true,
vhsNamespaced: map[string]bool{
"kapusta": false,
},
namespace: "kapusta",
expected: false,
},
{
name: "vhs enabled (global vhs flag)",
vhsEnabledFlag: true,
expected: true,
},
{
name: "vhs enabled for namespace",
vhsNamespaced: map[string]bool{
"kapusta": true,
},
namespace: "kapusta",
expected: true,
},
{
name: "vhs enabled (header)",
headerStatusVHS: enabledVHS,
vhsEnabledFlag: false,
vhsNamespaced: map[string]bool{
"kapusta": false,
},
namespace: "kapusta",
expected: true,
},
{
name: "vhs disabled (header)",
headerStatusVHS: disabledVHS,
vhsEnabledFlag: true,
vhsNamespaced: map[string]bool{
"kapusta": true,
},
namespace: "kapusta",
expected: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
actual := isVHSAddress(tc.headerStatusVHS, tc.vhsEnabledFlag, tc.vhsNamespaced, tc.namespace)
require.Equal(t, tc.expected, actual)
})
}
}
func TestPreparePathStyleAddress(t *testing.T) {
bkt, obj := "test-bucket", "test-object"
for _, tc := range []struct {
name string
urlParams string
expectedReqType ReqType
expectedBktName string
expectedObjName string
}{
{
name: "bucket request",
urlParams: "/" + bkt,
expectedReqType: bucketType,
expectedBktName: bkt,
},
{
name: "bucket request with slash",
urlParams: "/" + bkt + "/",
expectedReqType: bucketType,
expectedBktName: bkt,
},
{
name: "object request",
urlParams: "/" + bkt + "/" + obj,
expectedReqType: objectType,
expectedBktName: bkt,
expectedObjName: obj,
},
{
name: "object request with slash",
urlParams: "/" + bkt + "/" + obj + "/",
expectedReqType: objectType,
expectedBktName: bkt,
expectedObjName: obj + "/",
},
{
name: "none type request",
urlParams: "/",
expectedReqType: noneType,
},
} {
t.Run(tc.name, func(t *testing.T) {
reqInfo := &ReqInfo{}
r := httptest.NewRequest(http.MethodGet, tc.urlParams, nil)
preparePathStyleAddress(reqInfo, r, reqLogOrDefault(r.Context(), zaptest.NewLogger(t)))
require.Equal(t, tc.expectedReqType, reqInfo.RequestType)
require.Equal(t, tc.expectedBktName, reqInfo.BucketName)
require.Equal(t, tc.expectedObjName, reqInfo.ObjectName)
})
}
}
func TestPrepareVHSAddress(t *testing.T) {
bkt, obj, domain := "test-bucket", "test-object", "domain.com"
for _, tc := range []struct {
name string
domains []string
host string
urlParams string
expectedReqType ReqType
expectedBktName string
expectedObjName string
}{
{
name: "bucket request, the domain matched",
domains: []string{domain},
host: bkt + "." + domain,
urlParams: "/",
expectedReqType: bucketType,
expectedBktName: bkt,
},
{
name: "object request, the domain matched",
domains: []string{domain},
host: bkt + "." + domain,
urlParams: "/" + obj,
expectedReqType: objectType,
expectedBktName: bkt,
expectedObjName: obj,
},
{
name: "object request with slash, the domain matched",
domains: []string{domain},
host: bkt + "." + domain,
urlParams: "/" + obj + "/",
expectedReqType: objectType,
expectedBktName: bkt,
expectedObjName: obj + "/",
},
{
name: "list-buckets request, the domain matched",
domains: []string{domain},
host: domain,
urlParams: "/",
expectedReqType: noneType,
},
{
name: "bucket request, the domain don't match",
host: bkt + "." + domain,
urlParams: "/",
expectedReqType: bucketType,
expectedBktName: bkt,
},
{
name: "object request, the domain don't match",
host: bkt + "." + domain,
urlParams: "/" + obj,
expectedReqType: objectType,
expectedBktName: bkt,
expectedObjName: obj,
},
{
name: "object request with slash, the domain don't match",
host: bkt + "." + domain,
urlParams: "/" + obj + "/",
expectedReqType: objectType,
expectedBktName: bkt,
expectedObjName: obj + "/",
},
{
name: "list-buckets request, the domain don't match (list-buckets isn't supported if the domains don't match)",
host: domain,
urlParams: "/",
expectedReqType: bucketType,
expectedBktName: strings.Split(domain, ".")[0],
},
} {
t.Run(tc.name, func(t *testing.T) {
reqInfo := &ReqInfo{}
vhsSettings := &VHSSettingsMock{domains: tc.domains}
r := httptest.NewRequest(http.MethodGet, tc.urlParams, nil)
r.Host = tc.host
prepareVHSAddress(reqInfo, r, vhsSettings)
require.Equal(t, tc.expectedReqType, reqInfo.RequestType)
require.Equal(t, tc.expectedBktName, reqInfo.BucketName)
require.Equal(t, tc.expectedObjName, reqInfo.ObjectName)
})
}
}
func TestCheckDomains(t *testing.T) {
for _, tc := range []struct {
name string
domains []string
requestURL string
expectedBktName string
expectedMatch bool
}{
{
name: "valid url with bktName and namespace (wildcard after protocol infix)",
domains: []string{"s3.<wildcard>.domain.com"},
requestURL: "bktA.s3.kapusta.domain.com",
expectedBktName: "bktA",
expectedMatch: true,
},
{
name: "valid url without bktName and namespace (wildcard after protocol infix)",
domains: []string{"s3.<wildcard>.domain.com"},
requestURL: "s3.kapusta.domain.com",
expectedBktName: "",
expectedMatch: true,
},
{
name: "invalid url with invalid bktName (wildcard after protocol infix)",
domains: []string{"s3.<wildcard>.domain.com"},
requestURL: "bktA.bktB.s3.kapusta.domain.com",
expectedMatch: false,
},
{
name: "invalid url without namespace (wildcard after protocol infix)",
domains: []string{"s3.<wildcard>.domain.com"},
requestURL: "bktA.s3.domain.com",
expectedMatch: false,
},
{
name: "invalid url with invalid infix (wildcard after protocol infix)",
domains: []string{"s3.<wildcard>.domain.com"},
requestURL: "bktA.s4.kapusta.domain.com",
expectedMatch: false,
},
{
name: "invalid url with invalid postfix (wildcard after protocol infix)",
domains: []string{"s3.<wildcard>.domain.com"},
requestURL: "bktA.s3.kapusta.dom.su",
expectedMatch: false,
},
{
name: "valid url with bktName and namespace (wildcard at the beginning of the domain)",
domains: []string{"<wildcard>.domain.com"},
requestURL: "bktA.kapusta.domain.com",
expectedBktName: "bktA",
expectedMatch: true,
},
{
name: "valid url without bktName and namespace (wildcard at the beginning of the domain)",
domains: []string{"<wildcard>.domain.com"},
requestURL: "kapusta.domain.com",
expectedBktName: "",
expectedMatch: true,
},
{
name: "invalid url with invalid bktName (wildcard at the beginning of the domain)",
domains: []string{"<wildcard>.domain.com"},
requestURL: "bktA.bktB.kapusta.domain.com",
expectedMatch: false,
},
{
name: "collision test - true, because we cannot clearly distinguish a namespace from a bucket (wildcard at the beginning of the domain)",
domains: []string{"<wildcard>.domain.com"},
requestURL: "bktA.domain.com",
expectedMatch: true,
},
{
name: "invalid url (fewer hosts)",
domains: []string{"<wildcard>.domain.com"},
requestURL: "domain.com",
expectedMatch: false,
},
{
name: "invalid url with invalid postfix (wildcard at the beginning of the domain)",
domains: []string{"<wildcard>.domain.com"},
requestURL: "bktA.kapusta.dom.su",
expectedMatch: false,
},
{
name: "valid url with bktName and without wildcard (root namaspace)",
domains: []string{"domain.com"},
requestURL: "bktA.domain.com",
expectedBktName: "bktA",
expectedMatch: true,
},
{
name: "valid url without bktName and without wildcard (root namaspace)",
domains: []string{"domain.com"},
requestURL: "domain.com",
expectedBktName: "",
expectedMatch: true,
},
{
name: "invalid url with bktName without wildcard (root namaspace)",
domains: []string{"domain.com"},
requestURL: "bktA.dom.su",
expectedMatch: false,
},
{
name: "invalid url without wildcard (root namaspace)",
domains: []string{"domain.com"},
requestURL: "dom.su",
expectedMatch: false,
},
{
name: "valid url, with a sorted list of domains",
domains: []string{"s3.<wildcard>.domain.com", "<wildcard>.domain.com", "domain.com"},
requestURL: "s3.kapusta.domain.com",
expectedBktName: "",
expectedMatch: true,
},
{
name: "valid url with bktName, multiple wildcards (wildcards at the beginning of the domain)",
domains: []string{"<wildcard>.<wildcard>.domain.com"},
requestURL: "bktA.s3.kapusta.domain.com",
expectedBktName: "bktA",
expectedMatch: true,
},
{
name: "valid url without bktName, multiple wildcards (wildcards at the beginning of the domain)",
domains: []string{"<wildcard>.<wildcard>.domain.com"},
requestURL: "s3.kapusta.domain.com",
expectedBktName: "",
expectedMatch: true,
},
{
name: "valid url with bktName, multiply wildcards",
domains: []string{"s3.<wildcard>.subdomain.<wildcard>.com"},
requestURL: "bktA.s3.kapusta.subdomain.domain.com",
expectedBktName: "bktA",
expectedMatch: true,
},
{
name: "valid url without bktName, multiply wildcards",
domains: []string{"s3.<wildcard>.subdomain.<wildcard>.com"},
requestURL: "s3.kapusta.subdomain.domain.com",
expectedBktName: "",
expectedMatch: true,
},
{
name: "invalid url without one wildcard",
domains: []string{"<wildcard>.<wildcard>.domain.com"},
requestURL: "kapusta.domain.com",
expectedMatch: false,
},
{
name: "invalid url, multiply wildcards",
domains: []string{"<wildcard>.<wildcard>.domain.com"},
requestURL: "s3.kapusta.dom.com",
expectedMatch: false,
},
{
name: "invalid url with invalid bktName, multiply wildcards",
domains: []string{"<wildcard>.<wildcard>.domain.com"},
requestURL: "bktA.bktB.s3.kapusta.domain.com",
expectedMatch: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
bktName, match := checkDomain(tc.requestURL, tc.domains)
require.Equal(t, tc.expectedBktName, bktName)
require.Equal(t, tc.expectedMatch, match)
})
}
}
func TestGetDomains(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "/", nil)
settings := &VHSSettingsMock{
domains: []string{
"s3.domain.com",
"s3.<wildcard>.domain.com",
"domain.com",
},
}
t.Run("the request does not contain the X-Frostfs-Servername header", func(t *testing.T) {
actualDomains := getDomains(req, settings)
require.Equal(t, settings.domains, actualDomains)
})
serverName := "domain.com"
req.Header.Set(settings.ServernameHeader(), serverName)
t.Run("the request contains the X-Frostfs-Servername header", func(t *testing.T) {
actualDomains := getDomains(req, settings)
require.Equal(t, []string{serverName}, actualDomains)
})
}

View file

@ -8,9 +8,8 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -57,9 +56,9 @@ func Auth(center Center, log *zap.Logger) Func {
reqLogOrDefault(ctx, log).Debug(logs.CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed, zap.Error(err))
} else {
reqLogOrDefault(ctx, log).Error(logs.FailedToPassAuthentication, zap.Error(err))
err = frostfsErrors.UnwrapErr(err)
if _, ok := err.(apiErrors.Error); !ok {
err = apiErrors.GetAPIError(apiErrors.ErrAccessDenied)
err = apierr.TransformToS3Error(err)
if err.(apierr.Error).ErrCode == apierr.ErrInternalError {
err = apierr.GetAPIError(apierr.ErrAccessDenied)
}
if _, wrErr := WriteErrorResponse(w, GetReqInfo(r.Context()), err); wrErr != nil {
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))

View file

@ -74,6 +74,7 @@ const (
AbortMultipartUploadOperation = "AbortMultipartUpload"
DeleteObjectTaggingOperation = "DeleteObjectTagging"
DeleteObjectOperation = "DeleteObject"
PatchObjectOperation = "PatchObject"
)
const (
@ -106,3 +107,9 @@ const (
PartNumberQuery = "partNumber"
LegalHoldQuery = "legal-hold"
)
const (
StdoutPath = "stdout"
StderrPath = "stderr"
SinkName = "lumberjack"
)

237
api/middleware/log_http.go Normal file
View file

@ -0,0 +1,237 @@
//go:build loghttp
package middleware
import (
"bytes"
"io"
"net/http"
"os"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/detector"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/xmlutils"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"gopkg.in/natefinch/lumberjack.v2"
)
type (
LogHTTPSettings interface {
LogHTTPConfig() LogHTTPConfig
}
LogHTTPConfig struct {
Enabled bool
MaxBody int64
MaxLogSize int
OutputPath string
UseGzip bool
log *httpLogger
}
httpLogger struct {
*zap.Logger
logRoller *lumberjack.Logger
}
// responseReadWriter helps read http response body.
responseReadWriter struct {
http.ResponseWriter
response *bytes.Buffer
statusCode int
}
)
const (
payloadLabel = "payload"
responseLabel = "response"
)
func (lc *LogHTTPConfig) InitHTTPLogger(log *zap.Logger) {
if err := lc.initHTTPLogger(); err != nil {
log.Error(logs.FailedToInitializeHTTPLogger, zap.Error(err))
}
}
// initHTTPLogger returns registers zap sink and returns new httpLogger.
func (lc *LogHTTPConfig) initHTTPLogger() (err error) {
lc.log = &httpLogger{
Logger: zap.NewNop(),
logRoller: &lumberjack.Logger{},
}
c := newLoggerConfig()
lc.log.Logger, err = c.Build()
if err != nil {
return err
}
lc.setLogOutput()
return nil
}
// newLoggerConfig creates new zap.Config with disabled base fields.
func newLoggerConfig() zap.Config {
c := zap.NewProductionConfig()
c.DisableCaller = true
c.DisableStacktrace = true
c.EncoderConfig = newEncoderConfig()
c.Sampling = nil
return c
}
func (lc *LogHTTPConfig) setLogOutput() {
var output zapcore.WriteSyncer
switch lc.OutputPath {
case "", StdoutPath:
output = zapcore.AddSync(os.Stdout)
case StderrPath:
output = zapcore.AddSync(os.Stderr)
default:
output = zapcore.AddSync(&lumberjack.Logger{
Filename: lc.OutputPath,
MaxSize: lc.MaxLogSize,
Compress: lc.UseGzip,
})
}
// create logger with new sync
lc.log.Logger = lc.log.Logger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core {
return zapcore.NewCore(zapcore.NewJSONEncoder(newEncoderConfig()), output, zapcore.InfoLevel)
}))
}
func newEncoderConfig() zapcore.EncoderConfig {
c := zap.NewProductionEncoderConfig()
c.MessageKey = zapcore.OmitKey
c.LevelKey = zapcore.OmitKey
c.TimeKey = zapcore.OmitKey
c.FunctionKey = zapcore.OmitKey
return c
}
func (ww *responseReadWriter) Write(data []byte) (int, error) {
ww.response.Write(data)
return ww.ResponseWriter.Write(data)
}
func (ww *responseReadWriter) WriteHeader(code int) {
ww.statusCode = code
ww.ResponseWriter.WriteHeader(code)
}
func (ww *responseReadWriter) Flush() {
if f, ok := ww.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
// LogHTTP logs http parameters from s3 request.
func LogHTTP(l *zap.Logger, settings LogHTTPSettings) Func {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
config := settings.LogHTTPConfig()
if !config.Enabled || config.log == nil {
h.ServeHTTP(w, r)
return
}
httplog := config.log.getHTTPLogger(r).
withFieldIfExist("query", r.URL.Query()).
withFieldIfExist("headers", r.Header)
payload := getBody(r.Body, l)
r.Body = io.NopCloser(bytes.NewReader(payload))
payloadReader := io.LimitReader(bytes.NewReader(payload), config.MaxBody)
httplog = httplog.withProcessedBody(payloadLabel, payloadReader, l)
wr := newResponseReadWriter(w)
h.ServeHTTP(wr, r)
respReader := io.LimitReader(wr.response, config.MaxBody)
httplog = httplog.withProcessedBody(responseLabel, respReader, l)
httplog = httplog.with(zap.Int("status", wr.statusCode))
httplog.Info(logs.LogHTTP)
})
}
}
// withFieldIfExist checks whether data is not empty and attach it to log output.
func (lg *httpLogger) withFieldIfExist(label string, data map[string][]string) *httpLogger {
if len(data) != 0 {
return lg.with(zap.Any(label, data))
}
return lg
}
func (lg *httpLogger) with(fields ...zap.Field) *httpLogger {
return &httpLogger{
Logger: lg.Logger.With(fields...),
logRoller: lg.logRoller,
}
}
func (lg *httpLogger) getHTTPLogger(r *http.Request) *httpLogger {
return lg.with(
zap.String("from", r.RemoteAddr),
zap.String("URI", r.RequestURI),
zap.String("method", r.Method),
zap.String("protocol", r.Proto),
)
}
func (lg *httpLogger) withProcessedBody(label string, bodyReader io.Reader, l *zap.Logger) *httpLogger {
resp, err := processBody(bodyReader)
if err != nil {
l.Error(logs.FailedToProcessHTTPBody,
zap.Error(err),
zap.String("body type", payloadLabel))
return lg
}
return lg.with(zap.ByteString(label, resp))
}
func newResponseReadWriter(w http.ResponseWriter) *responseReadWriter {
return &responseReadWriter{
ResponseWriter: w,
response: &bytes.Buffer{},
}
}
func getBody(httpBody io.ReadCloser, l *zap.Logger) []byte {
defer func(httpBody io.ReadCloser) {
if err := httpBody.Close(); err != nil {
l.Error(logs.FailedToCloseHTTPBody, zap.Error(err))
}
}(httpBody)
body, err := io.ReadAll(httpBody)
if err != nil {
l.Error(logs.FailedToReadHTTPBody,
zap.Error(err),
zap.String("body type", payloadLabel))
return nil
}
return body
}
// processBody reads body and base64 encode it if it's not XML.
func processBody(bodyReader io.Reader) ([]byte, error) {
resultBody := &bytes.Buffer{}
detect := detector.NewDetector(bodyReader, xmlutils.DetectXML)
dataType, err := detect.Detect()
if err != nil {
return nil, err
}
writer := xmlutils.ChooseWriter(dataType, resultBody)
if _, err = io.Copy(writer, detect.RestoredReader()); err != nil {
return nil, err
}
if err = writer.Close(); err != nil {
return nil, err
}
return resultBody.Bytes(), nil
}

View file

@ -0,0 +1,36 @@
//go:build !loghttp
package middleware
import (
"net/http"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"go.uber.org/zap"
)
type (
LogHTTPSettings interface {
LogHTTPConfig() LogHTTPConfig
}
LogHTTPConfig struct {
Enabled bool
MaxBody int64
MaxLogSize int
OutputPath string
UseGzip bool
}
)
func LogHTTP(l *zap.Logger, _ LogHTTPSettings) Func {
l.Warn(logs.LogHTTPDisabledInThisBuild)
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
h.ServeHTTP(w, r)
})
}
}
func (*LogHTTPConfig) InitHTTPLogger(*zap.Logger) {
// ignore
}

View file

@ -11,8 +11,7 @@ import (
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
@ -25,11 +24,15 @@ import (
)
const (
QueryVersionID = "versionId"
QueryPrefix = "prefix"
QueryDelimiter = "delimiter"
QueryMaxKeys = "max-keys"
amzTagging = "x-amz-tagging"
QueryVersionID = "versionId"
QueryPrefix = "prefix"
QueryDelimiter = "delimiter"
QueryMaxKeys = "max-keys"
QueryMarker = "marker"
QueryEncodingType = "encoding-type"
amzTagging = "x-amz-tagging"
unmatchedBucketOperation = "UnmatchedBucketOperation"
)
// In these operations we don't check resource tags because
@ -73,7 +76,6 @@ type PolicyConfig struct {
Storage engine.ChainRouter
FrostfsID FrostFSIDInformer
Settings PolicySettings
Domains []string
Log *zap.Logger
BucketResolver BucketResolveFunc
Decoder XMLDecoder
@ -86,7 +88,7 @@ func PolicyCheck(cfg PolicyConfig) Func {
ctx := r.Context()
if err := policyCheck(r, cfg); err != nil {
reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err))
err = frostfsErrors.UnwrapErr(err)
err = apierr.TransformToS3Error(err)
if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil {
reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
}
@ -99,21 +101,21 @@ func PolicyCheck(cfg PolicyConfig) Func {
}
func policyCheck(r *http.Request, cfg PolicyConfig) error {
reqType, bktName, objName := getBucketObject(r, cfg.Domains)
req, userKey, userGroups, err := getPolicyRequest(r, cfg, reqType, bktName, objName)
reqInfo := GetReqInfo(r.Context())
req, userKey, userGroups, err := getPolicyRequest(r, cfg, reqInfo.RequestType, reqInfo.BucketName, reqInfo.ObjectName)
if err != nil {
return err
}
var bktInfo *data.BucketInfo
if reqType != noneType && !strings.HasSuffix(req.Operation(), CreateBucketOperation) {
bktInfo, err = cfg.BucketResolver(r.Context(), bktName)
if reqInfo.RequestType != noneType && !strings.HasSuffix(req.Operation(), CreateBucketOperation) {
bktInfo, err = cfg.BucketResolver(r.Context(), reqInfo.BucketName)
if err != nil {
return err
}
}
reqInfo := GetReqInfo(r.Context())
target := engine.NewRequestTargetWithNamespace(reqInfo.Namespace)
if bktInfo != nil {
cnrTarget := engine.ContainerTarget(bktInfo.CID.EncodeToString())
@ -146,11 +148,11 @@ func policyCheck(r *http.Request, cfg PolicyConfig) error {
case st == chain.Allow:
return nil
case st != chain.NoRuleFound:
return apiErr.GetAPIErrorWithError(apiErr.ErrAccessDenied, fmt.Errorf("policy check: %s", st.String()))
return apierr.GetAPIErrorWithError(apierr.ErrAccessDenied, fmt.Errorf("policy check: %s", st.String()))
}
if cfg.Settings.PolicyDenyByDefault() {
return apiErr.GetAPIErrorWithError(apiErr.ErrAccessDenied, fmt.Errorf("policy check: %s", st.String()))
return apierr.GetAPIErrorWithError(apierr.ErrAccessDenied, fmt.Errorf("policy check: %s", st.String()))
}
return nil
@ -208,33 +210,6 @@ const (
objectType
)
func getBucketObject(r *http.Request, domains []string) (reqType ReqType, bktName string, objName string) {
for _, domain := range domains {
ind := strings.Index(r.Host, "."+domain)
if ind == -1 {
continue
}
bkt := r.Host[:ind]
if obj := strings.TrimPrefix(r.URL.Path, "/"); obj != "" {
return objectType, bkt, obj
}
return bucketType, bkt, ""
}
bktObj := strings.TrimPrefix(r.URL.Path, "/")
if bktObj == "" {
return noneType, "", ""
}
if ind := strings.IndexByte(bktObj, '/'); ind != -1 && bktObj[ind+1:] != "" {
return objectType, bktObj[:ind], bktObj[ind+1:]
}
return bucketType, strings.TrimSuffix(bktObj, "/"), ""
}
func determineOperation(r *http.Request, reqType ReqType) (operation string) {
switch reqType {
case objectType:
@ -297,8 +272,17 @@ func determineBucketOperation(r *http.Request) string {
return ListObjectsV2MOperation
case query.Get(ListTypeQuery) == "2":
return ListObjectsV2Operation
default:
case len(query) == 0 || func() bool {
for key := range query {
if key != QueryDelimiter && key != QueryMaxKeys && key != QueryPrefix && key != QueryMarker && key != QueryEncodingType {
return false
}
}
return true
}():
return ListObjectsV1Operation
default:
return unmatchedBucketOperation
}
case http.MethodPut:
switch {
@ -320,8 +304,10 @@ func determineBucketOperation(r *http.Request) string {
return PutBucketVersioningOperation
case query.Has(NotificationQuery):
return PutBucketNotificationOperation
default:
case len(query) == 0:
return CreateBucketOperation
default:
return unmatchedBucketOperation
}
case http.MethodPost:
switch {
@ -344,12 +330,14 @@ func determineBucketOperation(r *http.Request) string {
return DeleteBucketLifecycleOperation
case query.Has(EncryptionQuery):
return DeleteBucketEncryptionOperation
default:
case len(query) == 0:
return DeleteBucketOperation
default:
return unmatchedBucketOperation
}
}
return "UnmatchedBucketOperation"
return unmatchedBucketOperation
}
func determineObjectOperation(r *http.Request) string {
@ -357,6 +345,8 @@ func determineObjectOperation(r *http.Request) string {
switch r.Method {
case http.MethodOptions:
return OptionsObjectOperation
case http.MethodPatch:
return PatchObjectOperation
case http.MethodHead:
return HeadObjectOperation
case http.MethodGet:
@ -487,7 +477,7 @@ func determineRequestTags(r *http.Request, decoder XMLDecoder, op string) (map[s
if strings.HasSuffix(op, PutObjectTaggingOperation) || strings.HasSuffix(op, PutBucketTaggingOperation) {
tagging := new(data.Tagging)
if err := decoder.NewXMLDecoder(r.Body).Decode(tagging); err != nil {
return nil, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrMalformedXML), err.Error())
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error())
}
GetReqInfo(r.Context()).Tagging = tagging
@ -499,7 +489,7 @@ func determineRequestTags(r *http.Request, decoder XMLDecoder, op string) (map[s
if tagging := r.Header.Get(amzTagging); len(tagging) > 0 {
queries, err := url.ParseQuery(tagging)
if err != nil {
return nil, apiErr.GetAPIError(apiErr.ErrInvalidArgument)
return nil, apierr.GetAPIError(apierr.ErrInvalidArgument)
}
for key := range queries {
tags[fmt.Sprintf(s3.PropertyKeyFormatRequestTag, key)] = queries.Get(key)

View file

@ -8,79 +8,6 @@ import (
"github.com/stretchr/testify/require"
)
func TestReqTypeDetermination(t *testing.T) {
bkt, obj, domain := "test-bucket", "test-object", "domain"
for _, tc := range []struct {
name string
target string
host string
domains []string
expectedType ReqType
expectedBktName string
expectedObjName string
}{
{
name: "bucket request, path-style",
target: "/" + bkt,
expectedType: bucketType,
expectedBktName: bkt,
},
{
name: "bucket request with slash, path-style",
target: "/" + bkt + "/",
expectedType: bucketType,
expectedBktName: bkt,
},
{
name: "object request, path-style",
target: "/" + bkt + "/" + obj,
expectedType: objectType,
expectedBktName: bkt,
expectedObjName: obj,
},
{
name: "object request with slash, path-style",
target: "/" + bkt + "/" + obj + "/",
expectedType: objectType,
expectedBktName: bkt,
expectedObjName: obj + "/",
},
{
name: "none type request",
target: "/",
expectedType: noneType,
},
{
name: "bucket request, virtual-hosted style",
target: "/",
host: bkt + "." + domain,
domains: []string{"some-domain", domain},
expectedType: bucketType,
expectedBktName: bkt,
},
{
name: "object request, virtual-hosted style",
target: "/" + obj,
host: bkt + "." + domain,
domains: []string{"some-domain", domain},
expectedType: objectType,
expectedBktName: bkt,
expectedObjName: obj,
},
} {
t.Run(tc.name, func(t *testing.T) {
r := httptest.NewRequest(http.MethodPut, tc.target, nil)
r.Host = tc.host
reqType, bktName, objName := getBucketObject(r, tc.domains)
require.Equal(t, tc.expectedType, reqType)
require.Equal(t, tc.expectedBktName, bktName)
require.Equal(t, tc.expectedObjName, objName)
})
}
}
func TestDetermineBucketOperation(t *testing.T) {
const defaultValue = "value"
@ -225,6 +152,12 @@ func TestDetermineBucketOperation(t *testing.T) {
method: http.MethodGet,
expected: ListObjectsV1Operation,
},
{
name: "UnmatchedBucketOperation GET",
method: http.MethodGet,
queryParam: map[string]string{"query": ""},
expected: unmatchedBucketOperation,
},
{
name: "PutBucketCorsOperation",
method: http.MethodPut,
@ -284,6 +217,12 @@ func TestDetermineBucketOperation(t *testing.T) {
method: http.MethodPut,
expected: CreateBucketOperation,
},
{
name: "UnmatchedBucketOperation PUT",
method: http.MethodPut,
queryParam: map[string]string{"query": ""},
expected: unmatchedBucketOperation,
},
{
name: "DeleteMultipleObjectsOperation",
method: http.MethodPost,
@ -336,10 +275,16 @@ func TestDetermineBucketOperation(t *testing.T) {
method: http.MethodDelete,
expected: DeleteBucketOperation,
},
{
name: "UnmatchedBucketOperation DELETE",
method: http.MethodDelete,
queryParam: map[string]string{"query": ""},
expected: unmatchedBucketOperation,
},
{
name: "UnmatchedBucketOperation",
method: "invalid-method",
expected: "UnmatchedBucketOperation",
expected: unmatchedBucketOperation,
},
} {
t.Run(tc.name, func(t *testing.T) {

View file

@ -12,7 +12,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"go.uber.org/zap"
"google.golang.org/grpc/metadata"
@ -28,19 +27,21 @@ type (
// ReqInfo stores the request info.
ReqInfo struct {
sync.RWMutex
RemoteHost string // Client Host/IP
Host string // Node Host/IP
UserAgent string // User Agent
DeploymentID string // random generated s3-deployment-id
RequestID string // x-amz-request-id
API string // API name -- GetObject PutObject NewMultipartUpload etc.
BucketName string // Bucket name
ObjectName string // Object name
TraceID string // Trace ID
URL *url.URL // Request url
Namespace string
User string // User owner id
Tagging *data.Tagging
RemoteHost string // Client Host/IP
Host string // Node Host/IP
UserAgent string // User Agent
DeploymentID string // random generated s3-deployment-id
RequestID string // x-amz-request-id
API string // API name -- GetObject PutObject NewMultipartUpload etc.
BucketName string // Bucket name
ObjectName string // Object name
TraceID string // Trace ID
URL *url.URL // Request url
Namespace string
User string // User owner id
Tagging *data.Tagging
RequestVHSEnabled bool
RequestType ReqType
}
// ObjectRequest represents object request data.
@ -61,10 +62,6 @@ const (
const HdrAmzRequestID = "x-amz-request-id"
const (
BucketURLPrm = "bucket"
)
var deploymentID = uuid.Must(uuid.NewRandom())
var (
@ -190,65 +187,21 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
r = r.WithContext(treepool.SetRequestID(r.Context(), reqInfo.RequestID))
reqLogger := log.With(zap.String("request_id", reqInfo.RequestID))
r = r.WithContext(SetReqLogger(r.Context(), reqLogger))
fields := []zap.Field{zap.String("request_id", reqInfo.RequestID)}
ctx, span := StartHTTPServerSpan(r, "REQUEST S3")
if traceID := span.SpanContext().TraceID(); traceID.IsValid() {
fields = append(fields, zap.String("trace_id", traceID.String()))
}
lw := &traceResponseWriter{ResponseWriter: w, ctx: ctx, span: span}
reqLogger := log.With(fields...)
r = r.WithContext(SetReqLogger(ctx, reqLogger))
reqLogger.Info(logs.RequestStart, zap.String("host", r.Host),
zap.String("remote_host", reqInfo.RemoteHost), zap.String("namespace", reqInfo.Namespace))
// continue execution
h.ServeHTTP(w, r)
})
}
}
// AddBucketName adds bucket name to ReqInfo from context.
func AddBucketName(l *zap.Logger) Func {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := GetReqInfo(ctx)
reqInfo.BucketName = chi.URLParam(r, BucketURLPrm)
if reqInfo.BucketName != "" {
reqLogger := reqLogOrDefault(ctx, l)
r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("bucket", reqInfo.BucketName))))
}
h.ServeHTTP(w, r)
})
}
}
// AddObjectName adds objects name to ReqInfo from context.
func AddObjectName(l *zap.Logger) Func {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := GetReqInfo(ctx)
reqLogger := reqLogOrDefault(ctx, l)
rctx := chi.RouteContext(ctx)
// trim leading slash (always present)
reqInfo.ObjectName = rctx.RoutePath[1:]
if r.URL.RawPath != "" {
// we have to do this because of
// https://github.com/go-chi/chi/issues/641
// https://github.com/go-chi/chi/issues/642
if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil {
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err))
} else {
reqInfo.ObjectName = obj
}
}
if reqInfo.ObjectName != "" {
r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("object", reqInfo.ObjectName))))
}
h.ServeHTTP(w, r)
h.ServeHTTP(lw, r)
})
}
}

View file

@ -11,7 +11,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
)
@ -62,7 +61,7 @@ const (
hdrSSE = "X-Amz-Server-Side-Encryption"
// hdrSSECustomerKey is the HTTP header key referencing the
// SSE-C client-provided key..
// SSE-C client-provided key.
hdrSSECustomerKey = hdrSSE + "-Customer-Key"
// hdrSSECopyKey is the HTTP header key referencing the SSE-C
@ -74,7 +73,7 @@ var (
xmlHeader = []byte(xml.Header)
)
// Non exhaustive list of AWS S3 standard error responses -
// Non-exhaustive list of AWS S3 standard error responses -
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
var s3ErrorResponseMap = map[string]string{
"AccessDenied": "Access Denied.",
@ -332,10 +331,6 @@ func LogSuccessResponse(l *zap.Logger) Func {
fields = append(fields, zap.String("user", reqInfo.User))
}
if traceID, err := trace.TraceIDFromHex(reqInfo.TraceID); err == nil && traceID.IsValid() {
fields = append(fields, zap.String("trace_id", reqInfo.TraceID))
}
reqLogger.Info(logs.RequestEnd, fields...)
})
}

View file

@ -11,20 +11,6 @@ import (
"go.opentelemetry.io/otel/trace"
)
// Tracing adds tracing support for requests.
// Must be placed after prepareRequest middleware.
func Tracing() Func {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
appCtx, span := StartHTTPServerSpan(r, "REQUEST S3")
reqInfo := GetReqInfo(r.Context())
reqInfo.TraceID = span.SpanContext().TraceID().String()
lw := &traceResponseWriter{ResponseWriter: w, ctx: appCtx, span: span}
h.ServeHTTP(lw, r.WithContext(appCtx))
})
}
}
type traceResponseWriter struct {
sync.Once
http.ResponseWriter

View file

@ -6,7 +6,7 @@ import (
"fmt"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
@ -29,20 +29,14 @@ type FrostFS interface {
SystemDNS(context.Context) (string, error)
}
type Settings interface {
FormContainerZone(ns string) (zone string, isDefault bool)
}
type Config struct {
FrostFS FrostFS
RPCAddress string
Settings Settings
}
type BucketResolver struct {
rpcAddress string
frostfs FrostFS
settings Settings
mu sync.RWMutex
resolvers []*Resolver
@ -50,15 +44,15 @@ type BucketResolver struct {
type Resolver struct {
Name string
resolve func(context.Context, string) (cid.ID, error)
resolve func(context.Context, string, string) (cid.ID, error)
}
func (r *Resolver) SetResolveFunc(fn func(context.Context, string) (cid.ID, error)) {
func (r *Resolver) SetResolveFunc(fn func(context.Context, string, string) (cid.ID, error)) {
r.resolve = fn
}
func (r *Resolver) Resolve(ctx context.Context, name string) (cid.ID, error) {
return r.resolve(ctx, name)
func (r *Resolver) Resolve(ctx context.Context, zone, name string) (cid.ID, error) {
return r.resolve(ctx, zone, name)
}
func NewBucketResolver(resolverNames []string, cfg *Config) (*BucketResolver, error) {
@ -87,12 +81,12 @@ func createResolvers(resolverNames []string, cfg *Config) ([]*Resolver, error) {
return resolvers, nil
}
func (r *BucketResolver) Resolve(ctx context.Context, bktName string) (cnrID cid.ID, err error) {
func (r *BucketResolver) Resolve(ctx context.Context, zone, bktName string) (cnrID cid.ID, err error) {
r.mu.RLock()
defer r.mu.RUnlock()
for _, resolver := range r.resolvers {
cnrID, resolverErr := resolver.Resolve(ctx, bktName)
cnrID, resolverErr := resolver.Resolve(ctx, zone, bktName)
if resolverErr != nil {
resolverErr = fmt.Errorf("%s: %w", resolver.Name, resolverErr)
if err == nil {
@ -123,7 +117,6 @@ func (r *BucketResolver) UpdateResolvers(resolverNames []string) error {
cfg := &Config{
FrostFS: r.frostfs,
RPCAddress: r.rpcAddress,
Settings: r.settings,
}
resolvers, err := createResolvers(resolverNames, cfg)
@ -152,30 +145,25 @@ func (r *BucketResolver) equals(resolverNames []string) bool {
func newResolver(name string, cfg *Config) (*Resolver, error) {
switch name {
case DNSResolver:
return NewDNSResolver(cfg.FrostFS, cfg.Settings)
return NewDNSResolver(cfg.FrostFS)
case NNSResolver:
return NewNNSResolver(cfg.RPCAddress, cfg.Settings)
return NewNNSResolver(cfg.RPCAddress)
default:
return nil, fmt.Errorf("unknown resolver: %s", name)
}
}
func NewDNSResolver(frostFS FrostFS, settings Settings) (*Resolver, error) {
func NewDNSResolver(frostFS FrostFS) (*Resolver, error) {
if frostFS == nil {
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
}
if settings == nil {
return nil, fmt.Errorf("resolver settings must not be nil for DNS resolver")
}
var dns ns.DNS
resolveFunc := func(ctx context.Context, name string) (cid.ID, error) {
resolveFunc := func(ctx context.Context, zone, name string) (cid.ID, error) {
var err error
reqInfo := middleware.GetReqInfo(ctx)
zone, isDefault := settings.FormContainerZone(reqInfo.Namespace)
if isDefault {
if zone == v2container.SysAttributeZoneDefault {
zone, err = frostFS.SystemDNS(ctx)
if err != nil {
return cid.ID{}, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err)
@ -196,13 +184,10 @@ func NewDNSResolver(frostFS FrostFS, settings Settings) (*Resolver, error) {
}, nil
}
func NewNNSResolver(address string, settings Settings) (*Resolver, error) {
func NewNNSResolver(address string) (*Resolver, error) {
if address == "" {
return nil, fmt.Errorf("rpc address must not be empty for NNS resolver")
}
if settings == nil {
return nil, fmt.Errorf("resolver settings must not be nil for NNS resolver")
}
var nns ns.NNS
@ -210,12 +195,9 @@ func NewNNSResolver(address string, settings Settings) (*Resolver, error) {
return nil, fmt.Errorf("dial %s: %w", address, err)
}
resolveFunc := func(ctx context.Context, name string) (cid.ID, error) {
resolveFunc := func(_ context.Context, zone, name string) (cid.ID, error) {
var d container.Domain
d.SetName(name)
reqInfo := middleware.GetReqInfo(ctx)
zone, _ := settings.FormContainerZone(reqInfo.Namespace)
d.SetZone(zone)
cnrID, err := nns.ResolveContainerDomain(d)

Some files were not shown because too many files have changed in this diff Show more