Compare commits

...

75 commits

Author SHA1 Message Date
fb90c0f52c [#65] Enable pre-commit
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-03-24 07:28:04 +00:00
f2f90e260e [#66] Add Issue Template
Add bug report and feature request templates

Signed-off-by: Liza <e.chichindaeva@yadro.com>
2023-03-23 12:25:16 +03:00
a0937126cb [#64] Support new system attributes
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-22 11:46:58 +03:00
655889a1a2 [#60] Update docs
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-03-22 11:22:11 +03:00
ef556bd8ac [#60] Use session token to set eACL during Complete Multipart Upload
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-03-22 11:22:11 +03:00
5104683f68 [#60] Refactor start of periodic XML writer
Reduce code duplication for error handling

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-03-22 11:02:16 +03:00
8151753eeb [#60] Use periodic white space XML writer in Complete Multipart Upload
This mechanism is used by Amazon S3 to keep client's
connection alive while object is being constructed from
the upload parts.

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-03-20 13:34:22 +03:00
2282c32822 [#60] Add *NoHeader functions
Such functions should be used together with periodic white space
XML writer.

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-03-20 12:07:36 +03:00
43685e03d9 [#60] Implement flusher on all http.ResponseWriters
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-03-20 12:04:34 +03:00
cf18158da4 [#60] Implement periodic white space XML writer
Periodic white space XML writer sends XML header
and white spaces to the io.Writer.

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-03-20 12:04:34 +03:00
5c62010331 [#35] Update SDK to not count error on client aborting
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-10 11:08:21 +03:00
0af06c3bd9 [TrueCloudLab#40] Add param to configure xml decoder
This parameter enables parsing xml body without
xmlns="http://s3.amazonaws.com/doc/2006-03-01/" attribute
for CompleteMultipartUpload requests

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-09 15:44:13 +03:00
680c0dbe3d [#54] Update syncTree.sh due code relocation
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-09 14:59:32 +03:00
596381c382 [TrueCloudLab#32] Update CHANGELOG.md
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-09 11:19:01 +00:00
64e7356acc [TrueCloudLab#32] Add custom policy unmarshaler
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-09 11:19:01 +00:00
32bf915502 [TrueCloudLab#37] Limit number of objects to delete
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-09 11:16:22 +00:00
813aa2f173 Rename package name
Due to source code relocation from GitHub.

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-03-07 17:38:08 +03:00
6eb7966800 [TrueCloudLab#34] Fix resolve_bucket format in docs
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-02-20 19:23:11 +03:00
2dcb3c283d [TrueCloudLab#36] Update CHANGELOG.md
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-02-20 19:22:48 +03:00
740acadd37 [TrueCloudLab#36] Fix cors object payload
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-02-20 19:22:48 +03:00
3ab77c8990 [TrueCloudLab#25] Update changelog
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-02-17 13:47:48 +03:00
d00163aadc [TrueCloudLab#25] Update docs and config example
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-02-17 13:47:48 +03:00
aadefd98b6 [TrueCloudLab#25] Process allow and deny lists of zones in bucket head requests
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-02-17 13:47:48 +03:00
8ac630ee71 [TrueCloudLab#25] Add zone data to BucketInfo
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-02-17 13:47:48 +03:00
Aleksey Pastukhov
744b52322d [TrueCloudLab#28] Add generated deb builder files to gitignore, and fix typo
Signed-off-by: Aleksey Pastukhov <a.pastukhov@yadro.com>
2023-02-15 10:25:46 +03:00
8d9d1f9235 [TrueCloudLab#29] Update billing metric label name
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-02-13 16:40:43 +03:00
787d1a347a [TrueCloudLab#26] Update docs
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-02-10 10:55:38 +03:00
9f823bd65a [TrueCloudLab#26] Add billing metrics to separate registry
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-02-10 10:55:38 +03:00
9dcacc230e [TrueCloudLab#23] Return error on unknown LocationConstraint
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-02-08 15:04:12 +03:00
430f1e734f [TrueCloudLab#21] Update CHANGELOG.md
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-02-07 12:26:40 +03:00
1ce8b8a30d [TrueCloudLab#21] Support multiple configs
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-02-07 12:26:40 +03:00
b35f146cec [TrueCloudLab#20] Renew tokens beforehand
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-02-03 16:59:49 +03:00
5ee4bf80ae [#18] Return container name in head-bucket response
Signed-off-by: Artem Tataurov <a.tataurov@yadro.com>
2023-02-01 14:25:21 +03:00
f9f52ce8e0 [TrueCloudLab#5] Update CHANGELOG.md
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-25 11:02:23 +03:00
e278ab9362 [TrueCloudLab#5] Refactor middlewares
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-25 11:02:23 +03:00
c5570e661d [TrueCloudLab#5] Add traffic metrics per user
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-25 11:02:23 +03:00
fc5c09c084 [TrueCloudLab#5] Request metrics per user
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-25 11:02:23 +03:00
86e881694d [TrueCloudLab#16] Update CHANGELOG.md
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-25 10:39:44 +03:00
361d1d3881 [TrueCloudLab#16] Update go version to 1.18
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-25 10:39:44 +03:00
9ad7982807 [#14] Update neo-go and viper
Signed-off-by: Artem Tataurov <a.tataurov@yadro.com>
2023-01-25 10:23:16 +03:00
a0d5b18184 [TrueCloudLab#12] Update changelog
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-24 17:56:18 +03:00
533b12d8bb [TrueCloudLab#12] Require only one healthy server
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-24 17:56:18 +03:00
cafe079072 [TrueCloudLab#13] Update frostfs SDK
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-23 13:59:32 +03:00
9473335234 [#11] Update FrostFS SDK
Contains debug logs for switching
connections in pool.

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-01-18 12:22:44 +03:00
19d8f8fcfe [#7] Update SDK
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-10 11:24:43 +03:00
db603a9703 Update logo
Signed-off-by: Stanislav Bogatyrev <s.bogatyrev@yadro.com>
2023-01-09 11:12:11 +03:00
b2148cc36b Release v0.26.0
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
(cherry picked from commit 50d85dc7edabe6a753c346c388bf18bf9134cd90)
2022-12-28 17:06:32 +03:00
ff040f6785 [#568] Update docs
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2022-12-28 17:06:32 +03:00
8e14ccad9f [#754] Remove duplicating CID/OID logging calls
Signed-off-by: Artem Tataurov <a.tataurov@yadro.com>
(cherry picked from commit 8c178fc3c7265fecc897a2526419bb748847c668)
2022-12-28 17:06:32 +03:00
af7fbd6f32 [#754] Add CID and OID to logs
Signed-off-by: Artem Tataurov <a.tataurov@yadro.com>
2022-12-28 17:06:32 +03:00
Denis Kirillov
a68aca764b [#760] Update CHANGELOG.md
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
(cherry picked from commit 92a5e09207b424a668f5deb8993b6cb48c62a9db)
2022-12-28 17:06:32 +03:00
Denis Kirillov
f1696c4725 [#760] authmate: Add flag for pool timeouts
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-12-28 17:06:32 +03:00
000d9ed42a [#2] Update issue links
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2022-12-21 11:17:45 +03:00
388482e230 [#2] Rename internals
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2022-12-21 11:17:45 +03:00
c2567fcc73 [#2] Update syncTree.sh
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2022-12-20 18:24:30 +03:00
f744c672cb [#2] Update debpackage
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2022-12-20 18:24:30 +03:00
765fd28a49 [#2] Update build
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2022-12-20 18:24:30 +03:00
09c6e22b84 [#2] Update docs
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2022-12-20 18:24:30 +03:00
96dff367db [#1] Build S3 Gateway with FrostFS dependencies
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2022-12-15 12:43:52 +03:00
dd25331210 [#737] Debian packaging
Debian package includes:
 - user creation;
 - directories and permissions;
 - unit file for systemd

Signed-off-by: Dmitriy Zabolotskiy <d.zabolotskiy@yadro.com>
2022-11-29 17:58:48 +03:00
Denis Kirillov
d6424ebeac [#755] Remove deprecated linters
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-24 18:03:56 +03:00
Denis Kirillov
5265afe213 [#742] Update docs
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-24 17:48:26 +03:00
Denis Kirillov
dd4f66712c [#742] Add multiple listeners
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-24 17:48:26 +03:00
Denis Kirillov
556374e3b0 [#747] Update docs
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-18 13:14:25 +03:00
Denis Kirillov
d2587b21af [#747] Reload policies on SIGHUP
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-18 13:14:25 +03:00
Angira Kekteeva
f5fe9a9b4b [#741] Fix data race
Signed-off-by: Angira Kekteeva <kira@nspcc.ru>
2022-11-16 18:19:31 +03:00
Denis Kirillov
d1825f97fb [#750] Update SDK to support timeout for stream
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-15 18:10:28 +03:00
Denis Kirillov
8f1bbb7dc1 [#749] Update SDK
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-15 10:16:41 +03:00
Denis Kirillov
1c0f4d6df8 [#726] Update CHANGELOG.md
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-14 14:53:55 +03:00
Denis Kirillov
094eb12578 [#726] Use client time on regular requests
Use `X-Amz-Date` header as `now` when
* compute expiration epoch
* set Timestamp for object and container
* forming locks
* send notifications

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-14 14:53:55 +03:00
Denis Kirillov
d3702f86d1 [#568] Update docs
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-11 13:12:49 +03:00
Denis Kirillov
d47840f137 [#568] Add configuration for region to policy map
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-11 13:12:49 +03:00
Denis Kirillov
3212805955 [#740] Update docs
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-10 15:33:13 +03:00
Denis Kirillov
2886ac161c [#740] Fix forming policy by ast
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-10 15:33:03 +03:00
7b0492c468 [#738] Run CI on support branch PR
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2022-11-02 14:05:33 +03:00
154 changed files with 4480 additions and 1937 deletions

View file

@ -1,7 +1,7 @@
FROM golang:1.19 as builder
ARG BUILD=now
ARG REPO=github.com/nspcc-dev/neofs-s3-gw
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
ARG VERSION=dev
WORKDIR /src
@ -10,12 +10,12 @@ COPY . /src
RUN make
# Executable image
FROM alpine AS neofs-s3-gw
FROM alpine AS frostfs-s3-gw
RUN apk add --no-cache bash ca-certificates
WORKDIR /
COPY --from=builder /src/bin/neofs-s3-gw /bin/neofs-s3-gw
COPY --from=builder /src/bin/neofs-s3-authmate /bin/neofs-s3-authmate
COPY --from=builder /src/bin/frostfs-s3-gw /bin/frostfs-s3-gw
COPY --from=builder /src/bin/frostfs-s3-authmate /bin/frostfs-s3-authmate
ENTRYPOINT ["/bin/neofs-s3-gw"]
ENTRYPOINT ["/bin/frostfs-s3-gw"]

View file

@ -1,9 +1,9 @@
FROM alpine AS neofs-s3-gw
FROM alpine AS frostfs-s3-gw
RUN apk add --no-cache bash ca-certificates
WORKDIR /
COPY /bin/neofs-s3-gw /bin/neofs-s3-gw
COPY /bin/neofs-s3-authmate /bin/neofs-s3-authmate
COPY /bin/frostfs-s3-gw /bin/frostfs-s3-gw
COPY /bin/frostfs-s3-authmate /bin/frostfs-s3-authmate
ENTRYPOINT ["/bin/neofs-s3-gw"]
ENTRYPOINT ["/bin/frostfs-s3-gw"]

2
.github/CODEOWNERS vendored
View file

@ -1 +1 @@
* @alexvanin @masterSplinter01 @KirillovDenis
* @alexvanin @KirillovDenis

45
.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View file

@ -0,0 +1,45 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: community, triage, bug
assignees: ''
---
<!--- Provide a general summary of the issue in the Title above -->
## Expected Behavior
<!--- If you're describing a bug, tell us what should happen -->
<!--- If you're suggesting a change/improvement, tell us how it should work -->
## Current Behavior
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
## Possible Solution
<!--- Not obligatory -->
<!--- If no reason/fix/additions for the bug can be suggested, -->
<!--- uncomment the following phrase: -->
<!--- No fix can be suggested by a QA engineer. Further solutions shall be up to developers. -->
## Steps to Reproduce (for bugs)
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
<!--- reproduce this bug. -->
1.
## Context
<!--- How has this issue affected you? What are you trying to accomplish? -->
<!--- Providing context helps us come up with a solution that is most useful in the real world -->
## Regression
<!-- Is this issue a regression? (Yes / No) -->
<!-- If Yes, optionally please include version or commit id or PR# that caused this regression, if you have these details. -->
## Your Environment
<!--- Include as many relevant details about the environment you experienced the bug in -->
* Version used:
* Server setup and configuration:
* Operating System and version (`uname -a`):

1
.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View file

@ -0,0 +1 @@
blank_issues_enabled: false

View file

@ -0,0 +1,20 @@
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: community, triage
assignees: ''
---
## Is your feature request related to a problem? Please describe.
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
## Describe the solution you'd like
<!--- A clear and concise description of what you want to happen. -->
## Describe alternatives you've considered
<!--- A clear and concise description of any alternative solutions or features you've considered. -->
## Additional context
<!--- Add any other context or screenshots about the feature request here. -->

70
.github/logo.svg vendored Normal file
View file

@ -0,0 +1,70 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 25.0.1, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="Слой_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 184.2 51.8" style="enable-background:new 0 0 184.2 51.8;" xml:space="preserve">
<style type="text/css">
.st0{display:none;}
.st1{display:inline;}
.st2{fill:#01E397;}
.st3{display:inline;fill:#010032;}
.st4{display:inline;fill:#00E599;}
.st5{display:inline;fill:#00AF92;}
.st6{fill:#00C3E5;}
</style>
<g id="Layer_2">
<g id="Layer_1-2" class="st0">
<g class="st1">
<path class="st2" d="M146.6,18.3v7.2h10.9V29h-10.9v10.7h-4V14.8h18v3.5H146.6z"/>
<path class="st2" d="M180,15.7c1.7,0.9,3,2.2,4,3.8l-3,2.7c-0.6-1.3-1.5-2.4-2.6-3.3c-1.3-0.7-2.8-1-4.3-1
c-1.4-0.1-2.8,0.3-4,1.1c-0.9,0.5-1.5,1.5-1.4,2.6c0,1,0.5,1.9,1.4,2.4c1.5,0.8,3.2,1.3,4.9,1.5c1.9,0.3,3.7,0.8,5.4,1.6
c1.2,0.5,2.2,1.3,2.9,2.3c0.6,1,1,2.2,0.9,3.4c0,1.4-0.5,2.7-1.3,3.8c-0.9,1.2-2.1,2.1-3.5,2.6c-1.7,0.6-3.4,0.9-5.2,0.8
c-5,0-8.6-1.6-10.7-5l2.9-2.8c0.7,1.4,1.8,2.5,3.1,3.3c1.5,0.7,3.1,1.1,4.7,1c1.5,0.1,2.9-0.2,4.2-0.9c0.9-0.5,1.5-1.5,1.5-2.6
c0-0.9-0.5-1.8-1.3-2.2c-1.5-0.7-3.1-1.2-4.8-1.5c-1.9-0.3-3.7-0.8-5.5-1.5c-1.2-0.5-2.2-1.4-3-2.4c-0.6-1-1-2.2-0.9-3.4
c0-1.4,0.4-2.7,1.2-3.8c0.8-1.2,2-2.2,3.3-2.8c1.6-0.7,3.4-1.1,5.2-1C176.1,14.3,178.2,14.8,180,15.7z"/>
</g>
<path class="st3" d="M73.3,16.3c1.9,1.9,2.9,4.5,2.7,7.1v15.9h-4V24.8c0-2.6-0.5-4.5-1.6-5.7c-1.2-1.2-2.8-1.8-4.5-1.7
c-1.3,0-2.5,0.3-3.7,0.8c-1.2,0.7-2.2,1.7-2.9,2.9c-0.8,1.5-1.1,3.2-1.1,4.9v13.3h-4V15.1l3.6,1.5v1.7c0.8-1.5,2.1-2.6,3.6-3.3
c1.5-0.8,3.2-1.2,4.9-1.1C68.9,13.8,71.3,14.7,73.3,16.3z"/>
<path class="st3" d="M104.4,28.3H85.6c0.1,2.2,1,4.3,2.5,5.9c1.5,1.4,3.5,2.2,5.6,2.1c1.6,0.1,3.2-0.2,4.6-0.9
c1.1-0.6,2-1.6,2.5-2.8l3.3,1.8c-0.9,1.7-2.3,3.1-4,4c-2,1-4.2,1.5-6.4,1.4c-3.7,0-6.7-1.1-8.8-3.4s-3.2-5.5-3.2-9.6s1-7.2,3-9.5
s5-3.4,8.7-3.4c2.1-0.1,4.2,0.5,6.1,1.5c1.6,1,3,2.5,3.8,4.2c0.9,1.8,1.3,3.9,1.3,5.9C104.6,26.4,104.6,27.4,104.4,28.3z
M88.1,19.3c-1.4,1.5-2.2,3.4-2.4,5.5h15.1c-0.2-2-1-3.9-2.3-5.5c-1.4-1.3-3.2-2-5.1-1.9C91.5,17.3,89.6,18,88.1,19.3z"/>
<path class="st3" d="M131,17.3c2.2,2.3,3.2,5.5,3.2,9.5s-1,7.3-3.2,9.6s-5.1,3.4-8.8,3.4s-6.7-1.1-8.9-3.4s-3.2-5.5-3.2-9.6
s1.1-7.2,3.2-9.5s5.1-3.4,8.9-3.4S128.9,15,131,17.3z M116.2,19.9c-1.5,2-2.2,4.4-2.1,6.9c-0.2,2.5,0.6,5,2.1,7
c1.5,1.7,3.7,2.7,6,2.6c2.3,0.1,4.4-0.9,5.9-2.6c1.5-2,2.3-4.5,2.1-7c0.1-2.5-0.6-4.9-2.1-6.9c-1.5-1.7-3.6-2.7-5.9-2.6
C119.9,17.2,117.7,18.2,116.2,19.9z"/>
<polygon class="st4" points="0,9.1 0,43.7 22.5,51.8 22.5,16.9 46.8,7.9 24.8,0 "/>
<polygon class="st5" points="24.3,17.9 24.3,36.8 46.8,44.9 46.8,9.6 "/>
</g>
<g>
<g>
<path class="st6" d="M41.6,17.5H28.2v6.9h10.4v3.3H28.2v10.2h-3.9V14.2h17.2V17.5z"/>
<path class="st6" d="M45.8,37.9v-18h3.3l0.4,3.2c0.5-1.2,1.2-2.1,2.1-2.7c0.9-0.6,2.1-0.9,3.5-0.9c0.4,0,0.7,0,1.1,0.1
c0.4,0.1,0.7,0.2,0.9,0.3l-0.5,3.4c-0.3-0.1-0.6-0.2-0.9-0.2C55.4,23,54.9,23,54.4,23c-0.7,0-1.5,0.2-2.2,0.6
c-0.7,0.4-1.3,1-1.8,1.8s-0.7,1.8-0.7,3v9.5H45.8z"/>
<path class="st6" d="M68.6,19.6c1.8,0,3.3,0.4,4.6,1.1c1.3,0.7,2.4,1.8,3.1,3.2s1.1,3.1,1.1,5c0,1.9-0.4,3.6-1.1,5
c-0.8,1.4-1.8,2.5-3.1,3.2c-1.3,0.7-2.9,1.1-4.6,1.1s-3.3-0.4-4.6-1.1c-1.3-0.7-2.4-1.8-3.2-3.2c-0.8-1.4-1.2-3.1-1.2-5
c0-1.9,0.4-3.6,1.2-5s1.8-2.5,3.2-3.2C65.3,19.9,66.8,19.6,68.6,19.6z M68.6,22.6c-1.1,0-2,0.2-2.8,0.7c-0.8,0.5-1.3,1.2-1.7,2.1
s-0.6,2.1-0.6,3.5c0,1.3,0.2,2.5,0.6,3.4s1,1.7,1.7,2.2s1.7,0.7,2.8,0.7c1.1,0,2-0.2,2.7-0.7c0.7-0.5,1.3-1.2,1.7-2.2
s0.6-2.1,0.6-3.4c0-1.4-0.2-2.5-0.6-3.5s-1-1.6-1.7-2.1C70.6,22.8,69.6,22.6,68.6,22.6z"/>
<path class="st6" d="M89.2,38.3c-1.8,0-3.4-0.3-4.9-1c-1.5-0.7-2.7-1.7-3.5-3l2.7-2.3c0.5,1,1.3,1.8,2.3,2.4
c1,0.6,2.2,0.9,3.6,0.9c1.1,0,2-0.2,2.6-0.6c0.6-0.4,1-0.9,1-1.6c0-0.5-0.2-0.9-0.5-1.2s-0.9-0.6-1.7-0.8l-3.8-0.8
c-1.9-0.4-3.3-1-4.1-1.9c-0.8-0.9-1.2-1.9-1.2-3.3c0-1,0.3-1.9,0.9-2.7c0.6-0.8,1.4-1.5,2.5-2s2.5-0.8,4-0.8c1.8,0,3.3,0.3,4.6,1
c1.3,0.6,2.2,1.5,2.9,2.7l-2.7,2.2c-0.5-1-1.1-1.7-2-2.1c-0.9-0.5-1.8-0.7-2.8-0.7c-0.8,0-1.4,0.1-2,0.3c-0.6,0.2-1,0.5-1.3,0.8
c-0.3,0.3-0.4,0.7-0.4,1.2c0,0.5,0.2,0.9,0.5,1.3s1,0.6,1.9,0.8l4.1,0.9c1.7,0.3,2.9,0.9,3.7,1.7c0.7,0.8,1.1,1.8,1.1,2.9
c0,1.2-0.3,2.2-0.9,3c-0.6,0.9-1.5,1.6-2.6,2C92.1,38.1,90.7,38.3,89.2,38.3z"/>
<path class="st6" d="M112.8,19.9v3H99.3v-3H112.8z M106.6,14.6v17.9c0,0.9,0.2,1.5,0.7,1.9c0.5,0.4,1.1,0.6,1.9,0.6
c0.6,0,1.2-0.1,1.7-0.3c0.5-0.2,0.9-0.5,1.3-0.8l0.9,2.8c-0.6,0.5-1.2,0.9-2,1.1c-0.8,0.3-1.7,0.4-2.7,0.4c-1,0-2-0.2-2.8-0.5
s-1.5-0.9-2-1.6c-0.5-0.8-0.7-1.7-0.8-3V15.7L106.6,14.6z"/>
<path d="M137.9,17.5h-13.3v6.9h10.4v3.3h-10.4v10.2h-3.9V14.2h17.2V17.5z"/>
<path d="M150.9,13.8c2.1,0,4,0.4,5.5,1.2c1.6,0.8,2.9,2,4,3.5l-2.6,2.5c-0.9-1.4-1.9-2.4-3.1-3c-1.1-0.6-2.5-0.9-4-0.9
c-1.2,0-2.1,0.2-2.8,0.5c-0.7,0.3-1.3,0.7-1.6,1.2c-0.3,0.5-0.5,1.1-0.5,1.7c0,0.7,0.3,1.4,0.8,1.9c0.5,0.6,1.5,1,2.9,1.3
l4.8,1.1c2.3,0.5,3.9,1.3,4.9,2.3c1,1,1.4,2.3,1.4,3.9c0,1.5-0.4,2.7-1.2,3.8c-0.8,1.1-1.9,1.9-3.3,2.5s-3.1,0.9-5,0.9
c-1.7,0-3.2-0.2-4.5-0.6c-1.3-0.4-2.5-1-3.5-1.8c-1-0.7-1.8-1.6-2.5-2.6l2.7-2.7c0.5,0.8,1.1,1.6,1.9,2.2
c0.8,0.7,1.7,1.2,2.7,1.5c1,0.4,2.2,0.5,3.4,0.5c1.1,0,2.1-0.1,2.9-0.4c0.8-0.3,1.4-0.7,1.8-1.2c0.4-0.5,0.6-1.1,0.6-1.9
c0-0.7-0.2-1.3-0.7-1.8c-0.5-0.5-1.3-0.9-2.6-1.2l-5.2-1.2c-1.4-0.3-2.6-0.8-3.6-1.3c-0.9-0.6-1.6-1.3-2.1-2.1s-0.7-1.8-0.7-2.8
c0-1.3,0.4-2.6,1.1-3.7c0.7-1.1,1.8-2,3.2-2.6C147.3,14.1,148.9,13.8,150.9,13.8z"/>
</g>
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 5.5 KiB

View file

@ -4,6 +4,7 @@ on:
pull_request:
branches:
- master
- 'support/*'
types: [ opened, synchronize ]
paths-ignore:
- '**/*.md'

View file

@ -13,10 +13,10 @@ name: "CodeQL"
on:
push:
branches: [ master ]
branches: [ master, 'support/*' ]
pull_request:
# The branches below must be a subset of the branches above
branches: [ master ]
branches: [ master, 'support/*' ]
schedule:
- cron: '35 8 * * 1'

View file

@ -4,6 +4,7 @@ on:
pull_request:
branches:
- master
- 'support/*'
jobs:
commits_check_job:

View file

@ -4,6 +4,7 @@ on:
pull_request:
branches:
- master
- 'support/*'
types: [opened, synchronize]
paths-ignore:
- '**/*.md'
@ -67,7 +68,7 @@ jobs:
runs-on: ubuntu-20.04
strategy:
matrix:
go_versions: [ '1.17', '1.18.x', '1.19.x' ]
go_versions: [ '1.18.x', '1.19.x' ]
fail-fast: false
steps:
- uses: actions/checkout@v2

12
.gitignore vendored
View file

@ -3,7 +3,7 @@
.vscode
# Tree service
internal/neofs/services/tree/
internal/frostfs/services/tree/
# Vendoring
vendor
@ -19,3 +19,13 @@ bin/
# coverage
coverage.txt
coverage.html
# debhelpers
**/*debhelper*
# debian package build files
debian/files
debian/*.log
debian/*.substvars
debian/frostfs-s3-gw/

11
.gitlint Normal file
View file

@ -0,0 +1,11 @@
[general]
fail-without-commits=True
regex-style-search=True
contrib=CC1
[title-match-regex]
regex=^\[\#[0-9Xx]+\]\s
[ignore-by-title]
regex=^Release(.*)
ignore=title-match-regex

View file

@ -32,15 +32,12 @@ linters:
- revive
# some default golangci-lint linters
- deadcode
- errcheck
- gosimple
- ineffassign
- staticcheck
- structcheck
- typecheck
- unused
- varcheck
# extra linters
- exhaustive

45
.pre-commit-config.yaml Normal file
View file

@ -0,0 +1,45 @@
ci:
autofix_prs: false
repos:
- repo: https://github.com/jorisroovers/gitlint
rev: v0.19.1
hooks:
- id: gitlint
stages: [commit-msg]
- id: gitlint-ci
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: check-added-large-files
- id: check-case-conflict
- id: check-executables-have-shebangs
- id: check-shebang-scripts-are-executable
- id: check-merge-conflict
- id: check-json
- id: check-xml
- id: check-yaml
- id: trailing-whitespace
args: [--markdown-linebreak-ext=md]
- id: end-of-file-fixer
exclude: ".key$"
- repo: https://github.com/shellcheck-py/shellcheck-py
rev: v0.9.0.2
hooks:
- id: shellcheck
- repo: https://github.com/golangci/golangci-lint
rev: v1.51.2
hooks:
- id: golangci-lint
- repo: local
hooks:
- id: go-unit-tests
name: go unit tests
entry: make test
pass_filenames: false
types: [go]
language: system

View file

@ -4,6 +4,77 @@ This document outlines major changes between releases.
## [Unreleased]
### Fixed
- Get empty bucket CORS from frostfs (TrueCloudLab#36)
- Don't count pool error on client abort (#35)
### Added
- Return container name in `head-bucket` response (TrueCloudLab#18)
- Billing metrics (TrueCloudLab#5)
- Multiple configs support (TrueCloudLab#21)
- Bucket name resolving policy (TrueCloudLab#25)
- Support string `Action` and `Resource` fields in `bucketPolicy.Statement` (TrueCloudLab#32)
- Add new `kludge.use_default_xmlns_for_complete_multipart` config param (TrueCloudLab#40)
### Changed
- Update neo-go to v0.101.0 (#14)
- Update viper to v1.15.0 (#14)
- Using multiple servers require only one healthy (TrueCloudLab#12)
- Update go version to go1.18 (TrueCloudLab#16)
- Return error on invalid LocationConstraint (TrueCloudLab#23)
- Place billing metrics to separate url path (TrueCloudLab#26)
- Add generated deb builder files to .gitignore, and fix typo (TrueCloudLab#28)
- Limit number of objects to delete at one time (TrueCloudLab#37)
- CompleteMultipartUpload handler now sends whitespace characters to keep alive client's connection (#60)
- Support new system attributes (#64)
## [0.26.0] - 2022-12-28
### Added
- Use client time as `now` in some requests (#726)
- Reload policies on SIGHUP (#747)
- Authmate flags for pool timeouts (#760)
- Multiple server listeners (#742)
### Changed
- Placement policy configuration (#568)
- Improved debug logging of CID and OID values (#754)
### Removed
- Deprecated linters (#755)
### Updating from v0.25.1
New config parameters were added. And old one `defaul_policy` were changed.
```yaml
placement_policy:
default: "REP 3"
region_mapping: /path/to/container/policies.json
```
Make sure you update the config accordingly:
If you configure application using environment variables change:
* `S3_GW_DEFAULT_POLICY` -> `S3_GW_PLACEMENT_POLICY_DEFAULT_POLICY`
* `S3_GW_LISTEN_ADDRESS` -> `S3_GW_SERVER_0_ADDRESS`
* `S3_GW_TLS_CERT_FILE` -> `S3_GW_SERVER_0_TLS_CERT_FILE` (and set `S3_GW_SERVER_0_TLS_ENABLED=true`)
* `S3_GW_TLS_KEY_FILE` -> `S3_GW_SERVER_0_TLS_KEY_FILE` (and set `S3_GW_SERVER_0_TLS_ENABLED=true`)
If you configure application using `.yaml` file change:
* `defaul_policy` -> `placement_policy.default`
* `listen_address` -> `server.0.address`
* `tls.cert_file` -> `server.0.tls.cert_file` (and set `server.0.tls.enabled: true`)
* `tls.key_file` -> `server.0.tls.key_file` (and set `server.0.tls.enabled: true`)
## [0.25.1] - 2022-10-30
### Fixed
- Empty bucket policy (#740)
- Big object removal (#749)
- Checksum panic (#741)
### Added
- Debian packaging (#737)
- Timeout for individual operations in streaming RPC (#750)
## [0.25.0] - 2022-10-31
### Fixed

View file

@ -3,8 +3,8 @@
First, thank you for contributing! We love and encourage pull requests from
everyone. Please follow the guidelines:
- Check the open [issues](https://github.com/nspcc-dev/neofs-s3-gw/issues) and
[pull requests](https://github.com/nspcc-dev/neofs-s3-gw/pulls) for existing
- Check the open [issues](https://github.com/TrueCloudLab/frostfs-s3-gw/issues) and
[pull requests](https://github.com/TrueCloudLab/frostfs-s3-gw/pulls) for existing
discussions.
- Open an issue first, to discuss a new feature or enhancement.
@ -23,24 +23,24 @@ everyone. Please follow the guidelines:
## Development Workflow
Start by forking the `neofs-s3-gw` repository, make changes in a branch and then
Start by forking the `frostfs-s3-gw` repository, make changes in a branch and then
send a pull request. We encourage pull requests to discuss code changes. Here
are the steps in details:
### Set up your GitHub Repository
Fork [NeoFS S3 Gateway
upstream](https://github.com/nspcc-dev/neofs-s3-gw/fork) source repository
Fork [FrostFS S3 Gateway
upstream](https://github.com/TrueCloudLab/frostfs-s3-gw/fork) source repository
to your own personal repository. Copy the URL of your fork (you will need it for
the `git clone` command below).
```sh
$ git clone https://github.com/nspcc-dev/neofs-s3-gw
$ git clone https://github.com/TrueCloudLab/frostfs-s3-gw
```
### Set up git remote as ``upstream``
```sh
$ cd neofs-s3-gw
$ git remote add upstream https://github.com/nspcc-dev/neofs-s3-gw
$ cd frostfs-s3-gw
$ git remote add upstream https://github.com/TrueCloudLab/frostfs-s3-gw
$ git fetch upstream
$ git merge upstream/master
...
@ -107,7 +107,7 @@ contributors".
To sign your work, just add a line like this at the end of your commit message:
```
Signed-off-by: Samii Sakisaka <samii@nspcc.ru>
Signed-off-by: Samii Sakisaka <samii@frostfs.info>
```
This can be easily done with the `--signoff` option to `git commit`.

39
Makefile Normal file → Executable file
View file

@ -8,15 +8,22 @@ LINT_VERSION ?= 1.49.0
BINDIR = bin
# Binaries to build
CMDS = $(addprefix neofs-, $(notdir $(wildcard cmd/*)))
CMDS = $(addprefix frostfs-, $(notdir $(wildcard cmd/*)))
BINS = $(addprefix $(BINDIR)/, $(CMDS))
# Variables for docker
REPO_BASENAME = $(shell basename `go list -m`)
HUB_IMAGE ?= "nspccdev/$(REPO_BASENAME)"
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint version clean protoc
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
# .deb package versioning
OS_RELEASE = $(shell lsb_release -cs)
PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \
sed "s/-/~/")-${OS_RELEASE}
.PHONY: debpackage debclean
# Make all binaries
all: $(BINS)
@ -26,7 +33,7 @@ $(BINS): sync-tree $(BINDIR) dep
CGO_ENABLED=0 \
go build -v -trimpath \
-ldflags "-X $(REPO)/internal/version.Version=$(VERSION)" \
-o $@ ./cmd/$(subst neofs-,,$(notdir $@))
-o $@ ./cmd/$(subst frostfs-,,$(notdir $@))
$(BINDIR):
@echo "⇒ Ensure dir: $@"
@ -73,7 +80,7 @@ format:
# Build clean Docker image
image:
@echo "⇒ Build NeoFS S3 Gateway docker image "
@echo "⇒ Build FrostFS S3 Gateway docker image "
@docker build \
--build-arg REPO=$(REPO) \
--build-arg VERSION=$(VERSION) \
@ -88,7 +95,7 @@ image-push:
# Build dirty Docker image
dirty-image:
@echo "⇒ Build NeoFS S3 Gateway dirty docker image "
@echo "⇒ Build FrostFS S3 Gateway dirty docker image "
@docker build \
--build-arg REPO=$(REPO) \
--build-arg VERSION=$(VERSION) \
@ -108,6 +115,14 @@ docker/lint:
--env HOME=/src \
golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint'
# Activate pre-commit hooks
pre-commit:
pre-commit install -t pre-commit -t commit-msg
# Deactivate pre-commit hooks
unpre-commit:
pre-commit uninstall -t pre-commit -t commit-msg
# Show current version
version:
@echo $(VERSION)
@ -126,4 +141,16 @@ protoc:
done
rm -rf vendor
# Package for Debian
debpackage:
dch --package frostfs-s3-gw \
--controlmaint \
--newversion $(PKG_VERSION) \
--distribution $(OS_RELEASE) \
"Please see CHANGELOG.md for code changes for $(VERSION)"
dpkg-buildpackage --no-sign -b
debclean:
dh clean
include help.mk

View file

@ -1,13 +1,23 @@
# NeoFS S3 Gateway
<p align="center">
<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
</p>
<p align="center">
<a href="https://frostfs.info">FrostFS</a> is a decentralized distributed object storage integrated with the <a href="https://neo.org">NEO Blockchain</a>.
</p>
NeoFS S3 gateway provides API compatible with Amazon S3 cloud storage service.
---
[![Report](https://goreportcard.com/badge/git.frostfs.info/TrueCloudLab/frostfs-s3-gw)](https://goreportcard.com/report/git.frostfs.info/TrueCloudLab/frostfs-s3-gw)
# FrostFS S3 Gateway
FrostFS S3 gateway provides API compatible with Amazon S3 cloud storage service.
## Installation
```go get -u github.com/nspcc-dev/neofs-s3-gw```
```go get -u git.frostfs.info/TrueCloudLab/frostfs-s3-gw```
Or you can call `make` to build it from the cloned repository (the binary will
end up in `bin/neofs-s3-gw` with authmate helper in `bin/neofs-s3-authmate`).
end up in `bin/frostfs-s3-gw` with authmate helper in `bin/frostfs-s3-authmate`).
To build binaries in clean docker environment, call `make docker/all`.
Other notable make targets:
@ -22,36 +32,36 @@ version Show current version
```
Or you can also use a [Docker
image](https://hub.docker.com/r/nspccdev/neofs-s3-gw) provided for released
image](https://hub.docker.com/r/truecloudlab/frostfs-s3-gw) provided for released
(and occasionally unreleased) versions of gateway (`:latest` points to the
latest stable release).
## Execution
Minimalistic S3 gateway setup needs:
* NeoFS node(s) address (S3 gateway itself is not a NeoFS node)
* FrostFS node(s) address (S3 gateway itself is not a FrostFS node)
Passed via `-p` parameter or via `S3_GW_PEERS_<N>_ADDRESS` and
`S3_GW_PEERS_<N>_WEIGHT` environment variables (gateway supports multiple
NeoFS nodes with weighted load balancing).
* a wallet used to fetch key and communicate with NeoFS nodes
FrostFS nodes with weighted load balancing).
* a wallet used to fetch key and communicate with FrostFS nodes
Passed via `--wallet` parameter or `S3_GW_WALLET_PATH` environment variable.
These two commands are functionally equivalent, they run the gate with one
backend node, some keys and otherwise default settings:
```
$ neofs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json
$ frostfs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json
$ S3_GW_PEERS_0_ADDRESS=192.168.130.72:8080 \
S3_GW_WALLET=wallet.json \
neofs-s3-gw
frostfs-s3-gw
```
It's also possible to specify uri scheme (grpc or grpcs) when using `-p` or environment variables:
```
$ neofs-s3-gw -p grpc://192.168.130.72:8080 --wallet wallet.json
$ frostfs-s3-gw -p grpc://192.168.130.72:8080 --wallet wallet.json
$ S3_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 \
S3_GW_WALLET=wallet.json \
neofs-s3-gw
frostfs-s3-gw
```
## Domains
@ -60,7 +70,7 @@ By default, s3-gw enable only `path-style access`.
To be able to use both: `virtual-hosted-style` and `path-style` access you must configure `listen_domains`:
```shell
$ neofs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json --listen_domains your.first.domain --listen_domains your.second.domain
$ frostfs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json --listen_domains your.first.domain --listen_domains your.second.domain
```
So now you can use (e.g. `HeadBucket`. Make sure DNS is properly configured):
@ -84,8 +94,8 @@ Also, you can configure domains using `.env` variables or `yaml` file.
## Documentation
- [Configuration](./docs/configuration.md)
- [NeoFS S3 AuthMate](./docs/authmate.md)
- [NeoFS Tree service](./docs/tree_service.md)
- [FrostFS S3 AuthMate](./docs/authmate.md)
- [FrostFS Tree service](./docs/tree_service.md)
- [AWS CLI basic usage](./docs/aws_cli.md)
- [AWS S3 API compatibility](./docs/aws_s3_compat.md)
- [AWS S3 Compatibility test results](./docs/s3_test_results.md)

View file

@ -1 +1 @@
v0.25.0
v0.26.0

View file

@ -14,14 +14,14 @@ import (
"strings"
"time"
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
v4 "github.com/nspcc-dev/neofs-s3-gw/api/auth/signer/v4"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
@ -33,7 +33,13 @@ var postPolicyCredentialRegexp = regexp.MustCompile(`(?P<access_key_id>[^/]+)/(?
type (
// Center is a user authentication interface.
Center interface {
Authenticate(request *http.Request) (*accessbox.Box, error)
Authenticate(request *http.Request) (*Box, error)
}
// Box contains access box and additional info.
Box struct {
AccessBox *accessbox.Box
ClientTime time.Time
}
center struct {
@ -86,9 +92,9 @@ func (p prs) Seek(_ int64, _ int) (int64, error) {
var _ io.ReadSeeker = prs(0)
// New creates an instance of AuthCenter.
func New(neoFS tokens.NeoFS, key *keys.PrivateKey, prefixes []string, config *cache.Config) Center {
func New(frostFS tokens.FrostFS, key *keys.PrivateKey, prefixes []string, config *cache.Config) Center {
return &center{
cli: tokens.New(neoFS, key, config),
cli: tokens.New(frostFS, key, config),
reg: NewRegexpMatcher(authorizationFieldRegexp),
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
allowedAccessKeyIDPrefixes: prefixes,
@ -126,11 +132,12 @@ func (a *authHeader) getAddress() (oid.Address, error) {
return addr, nil
}
func (c *center) Authenticate(r *http.Request) (*accessbox.Box, error) {
func (c *center) Authenticate(r *http.Request) (*Box, error) {
var (
err error
authHdr *authHeader
signatureDateTimeStr string
needClientTime bool
)
queryValues := r.URL.Query()
@ -166,6 +173,7 @@ func (c *center) Authenticate(r *http.Request) (*accessbox.Box, error) {
return nil, err
}
signatureDateTimeStr = r.Header.Get(AmzDate)
needClientTime = true
}
signatureDateTime, err := time.Parse("20060102T150405Z", signatureDateTimeStr)
@ -192,7 +200,12 @@ func (c *center) Authenticate(r *http.Request) (*accessbox.Box, error) {
return nil, err
}
return box, nil
result := &Box{AccessBox: box}
if needClientTime {
result.ClientTime = signatureDateTime
}
return result, nil
}
func (c center) checkAccessKeyID(accessKeyID string) error {
@ -209,7 +222,7 @@ func (c center) checkAccessKeyID(accessKeyID string) error {
return apiErrors.GetAPIError(apiErrors.ErrAccessDenied)
}
func (c *center) checkFormData(r *http.Request) (*accessbox.Box, error) {
func (c *center) checkFormData(r *http.Request) (*Box, error) {
if err := r.ParseMultipartForm(maxFormSizeMemory); err != nil {
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidArgument)
}
@ -251,7 +264,7 @@ func (c *center) checkFormData(r *http.Request) (*accessbox.Box, error) {
return nil, apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch)
}
return box, nil
return &Box{AccessBox: box}, nil
}
func cloneRequest(r *http.Request, authHeader *authHeader) *http.Request {

View file

@ -5,7 +5,7 @@ import (
"testing"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/stretchr/testify/require"
)

View file

@ -4,8 +4,8 @@ import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-sdk-go/user"
"go.uber.org/zap"
)

View file

@ -4,9 +4,9 @@ import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)

View file

@ -4,8 +4,8 @@ import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"go.uber.org/zap"
)

View file

@ -3,10 +3,10 @@ package cache
import (
"testing"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
"go.uber.org/zap/zaptest/observer"

2
api/cache/names.go vendored
View file

@ -4,8 +4,8 @@ import (
"fmt"
"time"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/bluele/gcache"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)

View file

@ -4,9 +4,9 @@ import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)

View file

@ -4,9 +4,9 @@ import (
"testing"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)

View file

@ -6,9 +6,9 @@ import (
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"go.uber.org/zap"
)
@ -20,7 +20,7 @@ import (
After putting a record, it lives for a while (default value is 60 seconds).
When we receive a request from a user, we try to find the suitable and non-expired cache entry, go through the list
and get ObjectInfos from common object cache or with a request to NeoFS.
and get ObjectInfos from common object cache or with a request to FrostFS.
When we put an object into a container, we invalidate entries with prefixes that are prefixes of the object's name.
*/

View file

@ -4,9 +4,9 @@ import (
"testing"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)

2
api/cache/system.go vendored
View file

@ -4,8 +4,8 @@ import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"go.uber.org/zap"
)

View file

@ -4,9 +4,9 @@ import (
"encoding/xml"
"time"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
)
const (
@ -22,7 +22,8 @@ const (
type (
// BucketInfo stores basic bucket data.
BucketInfo struct {
Name string
Name string // container name from system attribute
Zone string // container zone from system attribute
CID cid.ID
Owner user.ID
Created time.Time

View file

@ -4,9 +4,9 @@ import (
"strconv"
"time"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
)
const (
@ -25,7 +25,7 @@ func (v NodeVersion) IsDeleteMarker() bool {
}
// DeleteMarkerInfo is used to save object info if node in the tree service is delete marker.
// We need this information because the "delete marker" object is no longer stored in NeoFS.
// We need this information because the "delete marker" object is no longer stored in FrostFS.
type DeleteMarkerInfo struct {
Created time.Time
Owner user.ID

View file

@ -90,6 +90,7 @@ const (
ErrMissingFields
ErrMissingCredTag
ErrCredMalformed
ErrInvalidLocationConstraint
ErrInvalidRegion
ErrInvalidServiceS3
ErrInvalidServiceSTS
@ -680,6 +681,12 @@ var errorCodes = errorCodeMap{
Description: "Error parsing the X-Amz-Credential parameter; the region is wrong;",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidLocationConstraint: {
ErrCode: ErrInvalidLocationConstraint,
Code: "InvalidLocationConstraint",
Description: "The specified location (Region) constraint is not valid.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidRegion: {
ErrCode: ErrInvalidRegion,
Code: "InvalidRegion",
@ -986,7 +993,7 @@ var errorCodes = errorCodeMap{
ErrNotSupported: {
ErrCode: ErrNotSupported,
Code: "BadRequest",
Description: "Not supported by NeoFS S3 Gateway",
Description: "Not supported by FrostFS S3 Gateway",
HTTPStatusCode: http.StatusNotImplemented,
},
ErrInvalidEncryptionMethod: {

View file

@ -14,16 +14,16 @@ import (
"strconv"
"strings"
v2acl "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
v2acl "github.com/nspcc-dev/neofs-api-go/v2/acl"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"go.uber.org/zap"
)
@ -158,6 +158,90 @@ func (s ServiceRecord) ToEACLRecord() *eacl.Record {
return serviceRecord
}
var (
errInvalidStatement = stderrors.New("invalid statement")
errInvalidPrincipal = stderrors.New("invalid principal")
)
func (s *statement) UnmarshalJSON(data []byte) error {
var statementMap map[string]interface{}
if err := json.Unmarshal(data, &statementMap); err != nil {
return err
}
sidField, ok := statementMap["Sid"]
if ok {
if s.Sid, ok = sidField.(string); !ok {
return errInvalidStatement
}
}
effectField, ok := statementMap["Effect"]
if ok {
if s.Effect, ok = effectField.(string); !ok {
return errInvalidStatement
}
}
principalField, ok := statementMap["Principal"]
if ok {
principalMap, ok := principalField.(map[string]interface{})
if !ok {
return errInvalidPrincipal
}
awsField, ok := principalMap["AWS"]
if ok {
if s.Principal.AWS, ok = awsField.(string); !ok {
return fmt.Errorf("%w: 'AWS' field must be string", errInvalidPrincipal)
}
}
canonicalUserField, ok := principalMap["CanonicalUser"]
if ok {
if s.Principal.CanonicalUser, ok = canonicalUserField.(string); !ok {
return errInvalidPrincipal
}
}
}
actionField, ok := statementMap["Action"]
if ok {
switch actionField := actionField.(type) {
case []interface{}:
s.Action = make([]string, len(actionField))
for i, action := range actionField {
if s.Action[i], ok = action.(string); !ok {
return errInvalidStatement
}
}
case string:
s.Action = []string{actionField}
default:
return errInvalidStatement
}
}
resourceField, ok := statementMap["Resource"]
if ok {
switch resourceField := resourceField.(type) {
case []interface{}:
s.Resource = make([]string, len(resourceField))
for i, action := range resourceField {
if s.Resource[i], ok = action.(string); !ok {
return errInvalidStatement
}
}
case string:
s.Resource = []string{resourceField}
default:
return errInvalidStatement
}
}
return nil
}
func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
@ -1001,6 +1085,10 @@ func policyToAst(bktPolicy *bucketPolicy) (*ast, error) {
trimmedResource := strings.TrimPrefix(resource, arnAwsPrefix)
r, ok := rr[trimmedResource]
if !ok {
if !strings.HasPrefix(trimmedResource, bktPolicy.Bucket) {
return nil, fmt.Errorf("resource '%s' must be in the same bucket '%s'", trimmedResource, bktPolicy.Bucket)
}
r = &astResource{
resourceInfo: resourceInfoFromName(trimmedResource, bktPolicy.Bucket),
}
@ -1046,9 +1134,6 @@ func astToPolicy(ast *ast) *bucketPolicy {
bktPolicy := &bucketPolicy{}
for _, resource := range ast.Resources {
if len(resource.Version) == 0 {
continue
}
allowed, denied := triageOperations(resource.Operations)
handleResourceOperations(bktPolicy, allowed, eacl.ActionAllow, resource.Name())
handleResourceOperations(bktPolicy, denied, eacl.ActionDeny, resource.Name())
@ -1080,7 +1165,7 @@ func handleResourceOperations(bktPolicy *bucketPolicy, list []*astOperation, eac
for action, ops := range actionToOpMap {
for _, op := range ops {
if !contains(userOps, op) {
break LOOP
continue LOOP
}
}
actions = append(actions, action)

View file

@ -1,25 +1,27 @@
package handler
import (
"bytes"
"context"
"crypto/ecdsa"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/stretchr/testify/require"
)
@ -1297,7 +1299,7 @@ func TestPutBucketACL(t *testing.T) {
tc := prepareHandlerContext(t)
bktName := "bucket-for-acl"
box := createAccessBox(t)
box, _ := createAccessBox(t)
bktInfo := createBucket(t, tc, bktName, box)
header := map[string]string{api.AmzACL: "public-read"}
@ -1308,6 +1310,149 @@ func TestPutBucketACL(t *testing.T) {
checkLastRecords(t, tc, bktInfo, eacl.ActionDeny)
}
func TestBucketPolicy(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-for-policy"
box, key := createAccessBox(t)
createBucket(t, hc, bktName, box)
bktPolicy := getBucketPolicy(hc, bktName)
for _, st := range bktPolicy.Statement {
if st.Effect == "Allow" {
require.Equal(t, hex.EncodeToString(key.PublicKey().Bytes()), st.Principal.CanonicalUser)
require.Equal(t, []string{arnAwsPrefix + bktName}, st.Resource)
} else {
require.Equal(t, allUsersWildcard, st.Principal.AWS)
require.Equal(t, "Deny", st.Effect)
require.Equal(t, []string{arnAwsPrefix + bktName}, st.Resource)
}
}
newPolicy := &bucketPolicy{
Statement: []statement{{
Effect: "Allow",
Principal: principal{AWS: allUsersWildcard},
Action: []string{s3GetObject},
Resource: []string{arnAwsPrefix + "dummy"},
}},
}
putBucketPolicy(hc, bktName, newPolicy, box, http.StatusInternalServerError)
newPolicy.Statement[0].Resource[0] = arnAwsPrefix + bktName
putBucketPolicy(hc, bktName, newPolicy, box, http.StatusOK)
bktPolicy = getBucketPolicy(hc, bktName)
for _, st := range bktPolicy.Statement {
if st.Effect == "Allow" && st.Principal.AWS == allUsersWildcard {
require.Equal(t, []string{arnAwsPrefix + bktName}, st.Resource)
require.ElementsMatch(t, []string{s3GetObject, s3ListBucket}, st.Action)
}
}
}
func TestBucketPolicyUnmarshal(t *testing.T) {
for _, tc := range []struct {
name string
policy string
}{
{
name: "action/resource array",
policy: `
{
"Version": "2012-10-17",
"Statement": [{
"Principal": {
"AWS": "arn:aws:iam::111122223333:role/JohnDoe"
},
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:GetObjectVersion"
],
"Resource": [
"arn:aws:s3:::DOC-EXAMPLE-BUCKET/*",
"arn:aws:s3:::DOC-EXAMPLE-BUCKET2/*"
]
}]
}
`,
},
{
name: "action/resource string",
policy: `
{
"Version": "2012-10-17",
"Statement": [{
"Principal": {
"AWS": "arn:aws:iam::111122223333:role/JohnDoe"
},
"Effect": "Deny",
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::DOC-EXAMPLE-BUCKET/*"
}]
}
`,
},
} {
t.Run(tc.name, func(t *testing.T) {
bktPolicy := &bucketPolicy{}
err := json.Unmarshal([]byte(tc.policy), bktPolicy)
require.NoError(t, err)
})
}
}
func TestPutBucketPolicy(t *testing.T) {
bktPolicy := `
{
"Version": "2012-10-17",
"Statement": [{
"Principal": {
"AWS": "*"
},
"Effect": "Deny",
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::bucket-for-policy/*"
}]
}
`
hc := prepareHandlerContext(t)
bktName := "bucket-for-policy"
box, _ := createAccessBox(t)
createBucket(t, hc, bktName, box)
w, r := prepareTestPayloadRequest(hc, bktName, "", bytes.NewReader([]byte(bktPolicy)))
ctx := context.WithValue(r.Context(), api.BoxData, box)
r = r.WithContext(ctx)
hc.Handler().PutBucketPolicyHandler(w, r)
assertStatus(hc.t, w, http.StatusOK)
}
func getBucketPolicy(hc *handlerContext, bktName string) *bucketPolicy {
w, r := prepareTestRequest(hc, bktName, "", nil)
hc.Handler().GetBucketPolicyHandler(w, r)
assertStatus(hc.t, w, http.StatusOK)
policy := &bucketPolicy{}
err := json.NewDecoder(w.Result().Body).Decode(policy)
require.NoError(hc.t, err)
return policy
}
func putBucketPolicy(hc *handlerContext, bktName string, bktPolicy *bucketPolicy, box *accessbox.Box, status int) {
body, err := json.Marshal(bktPolicy)
require.NoError(hc.t, err)
w, r := prepareTestPayloadRequest(hc, bktName, "", bytes.NewReader(body))
ctx := context.WithValue(r.Context(), api.BoxData, box)
r = r.WithContext(ctx)
hc.Handler().PutBucketPolicyHandler(w, r)
assertStatus(hc.t, w, status)
}
func checkLastRecords(t *testing.T, tc *handlerContext, bktInfo *data.BucketInfo, action eacl.Action) {
bktACL, err := tc.Layer().GetBucketACL(tc.Context(), bktInfo)
require.NoError(t, err)
@ -1325,7 +1470,7 @@ func checkLastRecords(t *testing.T, tc *handlerContext, bktInfo *data.BucketInfo
}
}
func createAccessBox(t *testing.T) *accessbox.Box {
func createAccessBox(t *testing.T) (*accessbox.Box, *keys.PrivateKey) {
key, err := keys.NewPrivateKey()
require.NoError(t, err)
@ -1345,7 +1490,7 @@ func createAccessBox(t *testing.T) *accessbox.Box {
},
}
return box
return box, key
}
func createBucket(t *testing.T, tc *handlerContext, bktName string, box *accessbox.Box) *data.BucketInfo {

View file

@ -1,11 +1,14 @@
package handler
import (
"encoding/xml"
"errors"
"io"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"go.uber.org/zap"
)
@ -19,21 +22,33 @@ type (
Notificator interface {
SendNotifications(topics map[string]string, p *SendNotificationParams) error
SendTestNotification(topic, bucketName, requestID, HostID string) error
SendTestNotification(topic, bucketName, requestID, HostID string, now time.Time) error
}
// Config contains data which handler needs to keep.
Config struct {
DefaultPolicy netmap.PlacementPolicy
DefaultMaxAge int
NotificatorEnabled bool
TLSEnabled bool
CopiesNumber uint32
Policy PlacementPolicy
XMLDecoder XMLDecoderProvider
DefaultMaxAge int
NotificatorEnabled bool
CopiesNumber uint32
ResolveZoneList []string
IsResolveListAllow bool // True if ResolveZoneList contains allowed zones
CompleteMultipartKeepalive time.Duration
}
PlacementPolicy interface {
Default() netmap.PlacementPolicy
Get(string) (netmap.PlacementPolicy, bool)
}
XMLDecoderProvider interface {
NewCompleteMultipartDecoder(io.Reader) *xml.Decoder
}
)
const (
// DefaultPolicy is a default policy of placing containers in NeoFS if it's not set at the request.
// DefaultPolicy is a default policy of placing containers in FrostFS if it's not set at the request.
DefaultPolicy = "REP 3"
// DefaultCopiesNumber is a default number of object copies that is enough to consider put successful if it's not set in config.
DefaultCopiesNumber uint32 = 0
@ -45,7 +60,7 @@ var _ api.Handler = (*handler)(nil)
func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg *Config) (api.Handler, error) {
switch {
case obj == nil:
return nil, errors.New("empty NeoFS Object Layer")
return nil, errors.New("empty FrostFS Object Layer")
case log == nil:
return nil, errors.New("empty logger")
}

View file

@ -6,10 +6,10 @@ import (
"strconv"
"strings"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"go.uber.org/zap"
)
@ -94,7 +94,7 @@ func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Requ
}
info := extendedInfo.ObjectInfo
encryptionParams, err := h.formEncryptionParams(r.Header)
encryptionParams, err := formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return

View file

@ -4,7 +4,7 @@ import (
"strings"
"testing"
"github.com/nspcc-dev/neofs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"github.com/stretchr/testify/require"
)

View file

@ -6,12 +6,12 @@ import (
"regexp"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"go.uber.org/zap"
)
@ -142,7 +142,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
}
}
encryptionParams, err := h.formEncryptionParams(r.Header)
encryptionParams, err := formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -184,7 +184,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
CopiesNuber: copiesNumber,
}
params.Lock, err = formObjectLock(dstBktInfo, settings.LockConfiguration, r.Header)
params.Lock, err = formObjectLock(r.Context(), dstBktInfo, settings.LockConfiguration, r.Header)
if err != nil {
h.logAndSendError(w, "could not form object lock", reqInfo, err)
return

View file

@ -6,7 +6,7 @@ import (
"net/url"
"testing"
"github.com/nspcc-dev/neofs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"github.com/stretchr/testify/require"
)

View file

@ -5,9 +5,9 @@ import (
"strconv"
"strings"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"go.uber.org/zap"
)

41
api/handler/cors_test.go Normal file
View file

@ -0,0 +1,41 @@
package handler
import (
"context"
"net/http"
"strings"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
)
func TestCORSOriginWildcard(t *testing.T) {
body := `
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<CORSRule>
<AllowedMethod>GET</AllowedMethod>
<AllowedOrigin>*</AllowedOrigin>
</CORSRule>
</CORSConfiguration>
`
hc := prepareHandlerContext(t)
bktName := "bucket-for-cors"
box, _ := createAccessBox(t)
w, r := prepareTestRequest(hc, bktName, "", nil)
ctx := context.WithValue(r.Context(), api.BoxData, box)
r = r.WithContext(ctx)
r.Header.Add(api.AmzACL, "public-read")
hc.Handler().CreateBucketHandler(w, r)
assertStatus(t, w, http.StatusOK)
w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
ctx = context.WithValue(r.Context(), api.BoxData, box)
r = r.WithContext(ctx)
hc.Handler().PutBucketCorsHandler(w, r)
assertStatus(t, w, http.StatusOK)
w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
hc.Handler().GetBucketCorsHandler(w, r)
assertStatus(t, w, http.StatusOK)
}

View file

@ -6,17 +6,20 @@ import (
"strconv"
"strings"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
// limitation of AWS https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
const maxObjectsToDelete = 1000
// DeleteObjectsRequest -- xml carrying the object key names which should be deleted.
type DeleteObjectsRequest struct {
// Element to enable quiet mode for the request
@ -176,6 +179,11 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
return
}
if len(requested.Objects) == 0 || len(requested.Objects) > maxObjectsToDelete {
h.logAndSendError(w, "number of objects to delete must be greater than 0 and less or equal to 1000", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
return
}
removed := make(map[string]*layer.VersionedObject)
toRemove := make([]*layer.VersionedObject, 0, len(requested.Objects))
for _, obj := range requested.Objects {

View file

@ -6,8 +6,8 @@ import (
"net/url"
"testing"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/stretchr/testify/require"
)
@ -41,7 +41,7 @@ func TestDeleteObject(t *testing.T) {
deleteObject(t, tc, bktName, objName, emptyVersion)
checkNotFound(t, tc, bktName, objName, emptyVersion)
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo))
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo))
}
func TestDeleteObjectFromSuspended(t *testing.T) {
@ -109,7 +109,7 @@ func TestDeleteObjectVersioned(t *testing.T) {
deleteObject(t, tc, bktName, objName, objInfo.VersionID())
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object exists but shouldn't")
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object exists but shouldn't")
}
func TestDeleteObjectUnversioned(t *testing.T) {
@ -126,7 +126,7 @@ func TestDeleteObjectUnversioned(t *testing.T) {
require.Len(t, versions.DeleteMarker, 0, "delete markers must be empty")
require.Len(t, versions.Version, 0, "versions must be empty")
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object exists but shouldn't")
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object exists but shouldn't")
}
func TestRemoveDeleteMarker(t *testing.T) {
@ -144,7 +144,7 @@ func TestRemoveDeleteMarker(t *testing.T) {
deleteObject(t, tc, bktName, objName, deleteMarkerVersion)
checkFound(t, tc, bktName, objName, emptyVersion)
require.True(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object doesn't exist but should")
require.True(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object doesn't exist but should")
}
func TestDeleteObjectCombined(t *testing.T) {
@ -161,7 +161,7 @@ func TestDeleteObjectCombined(t *testing.T) {
checkFound(t, tc, bktName, objName, objInfo.VersionID())
require.True(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object doesn't exist but should")
require.True(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object doesn't exist but should")
}
func TestDeleteObjectSuspended(t *testing.T) {
@ -181,7 +181,7 @@ func TestDeleteObjectSuspended(t *testing.T) {
deleteObject(t, tc, bktName, objName, emptyVersion)
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object exists but shouldn't")
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object exists but shouldn't")
}
func TestDeleteMarkers(t *testing.T) {
@ -200,7 +200,7 @@ func TestDeleteMarkers(t *testing.T) {
require.Len(t, versions.DeleteMarker, 3, "invalid delete markers length")
require.Len(t, versions.Version, 0, "versions must be empty")
require.Len(t, listOIDsFromMockedNeoFS(t, tc, bktName), 0, "shouldn't be any object in neofs")
require.Len(t, listOIDsFromMockedFrostFS(t, tc, bktName), 0, "shouldn't be any object in frostfs")
}
func TestDeleteObjectFromListCache(t *testing.T) {
@ -220,7 +220,7 @@ func TestDeleteObjectFromListCache(t *testing.T) {
versions = listObjectsV1(t, tc, bktName, "", "", "", -1)
require.Len(t, versions.Contents, 0)
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo))
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo))
}
func TestDeleteObjectCheckMarkerReturn(t *testing.T) {

View file

@ -3,6 +3,7 @@ package handler
import (
"bytes"
"crypto/rand"
"crypto/tls"
"fmt"
"io"
"net/http"
@ -11,8 +12,8 @@ import (
"strings"
"testing"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/stretchr/testify/require"
)
@ -284,6 +285,7 @@ func getEncryptedObjectRange(t *testing.T, tc *handlerContext, bktName, objName
}
func setEncryptHeaders(r *http.Request) {
r.TLS = &tls.ConnectionState{}
r.Header.Set(api.AmzServerSideEncryptionCustomerAlgorithm, layer.AESEncryptionAlgorithm)
r.Header.Set(api.AmzServerSideEncryptionCustomerKey, aes256Key)
r.Header.Set(api.AmzServerSideEncryptionCustomerKeyMD5, aes256KeyMD5)

View file

@ -8,10 +8,10 @@ import (
"strings"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"go.uber.org/zap"
)
@ -150,7 +150,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
encryptionParams, err := h.formEncryptionParams(r.Header)
encryptionParams, err := formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return

View file

@ -8,9 +8,9 @@ import (
"testing"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/stretchr/testify/require"
)

View file

@ -13,15 +13,16 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api/resolver"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
@ -30,7 +31,7 @@ type handlerContext struct {
owner user.ID
t *testing.T
h *handler
tp *layer.TestNeoFS
tp *layer.TestFrostFS
context context.Context
}
@ -38,7 +39,7 @@ func (hc *handlerContext) Handler() *handler {
return hc.h
}
func (hc *handlerContext) MockedPool() *layer.TestNeoFS {
func (hc *handlerContext) MockedPool() *layer.TestFrostFS {
return hc.tp
}
@ -50,12 +51,30 @@ func (hc *handlerContext) Context() context.Context {
return hc.context
}
type placementPolicyMock struct {
defaultPolicy netmap.PlacementPolicy
}
func (p *placementPolicyMock) Default() netmap.PlacementPolicy {
return p.defaultPolicy
}
func (p *placementPolicyMock) Get(string) (netmap.PlacementPolicy, bool) {
return netmap.PlacementPolicy{}, false
}
type xmlDecoderProviderMock struct{}
func (p *xmlDecoderProviderMock) NewCompleteMultipartDecoder(r io.Reader) *xml.Decoder {
return xml.NewDecoder(r)
}
func prepareHandlerContext(t *testing.T) *handlerContext {
key, err := keys.NewPrivateKey()
require.NoError(t, err)
l := zap.NewExample()
tp := layer.NewTestNeoFS()
tp := layer.NewTestFrostFS()
testResolver := &resolver.Resolver{Name: "test_resolver"}
testResolver.SetResolveFunc(func(_ context.Context, name string) (cid.ID, error) {
@ -72,11 +91,16 @@ func prepareHandlerContext(t *testing.T) *handlerContext {
TreeService: layer.NewTreeService(),
}
var pp netmap.PlacementPolicy
err = pp.DecodeString("REP 1")
require.NoError(t, err)
h := &handler{
log: l,
obj: layer.NewLayer(l, tp, layerCfg),
cfg: &Config{
TLSEnabled: true,
Policy: &placementPolicyMock{defaultPolicy: pp},
XMLDecoder: &xmlDecoderProviderMock{},
},
}
@ -191,7 +215,7 @@ func parseTestResponse(t *testing.T, response *httptest.ResponseRecorder, body i
require.NoError(t, err)
}
func existInMockedNeoFS(tc *handlerContext, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo) bool {
func existInMockedFrostFS(tc *handlerContext, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo) bool {
p := &layer.GetObjectParams{
BucketInfo: bktInfo,
ObjectInfo: objInfo,
@ -201,7 +225,7 @@ func existInMockedNeoFS(tc *handlerContext, bktInfo *data.BucketInfo, objInfo *d
return tc.Layer().GetObject(tc.Context(), p) == nil
}
func listOIDsFromMockedNeoFS(t *testing.T, tc *handlerContext, bktName string) []oid.ID {
func listOIDsFromMockedFrostFS(t *testing.T, tc *handlerContext, bktName string) []oid.ID {
bktInfo, err := tc.Layer().GetBucketInfo(tc.Context(), bktName)
require.NoError(t, err)

View file

@ -4,10 +4,10 @@ import (
"bytes"
"net/http"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"go.uber.org/zap"
)
@ -53,7 +53,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
}
info := extendedInfo.ObjectInfo
encryptionParams, err := h.formEncryptionParams(r.Header)
encryptionParams, err := formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -123,6 +123,13 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
}
w.Header().Set(api.ContainerID, bktInfo.CID.EncodeToString())
w.Header().Set(api.AmzBucketRegion, bktInfo.LocationConstraint)
if isAvailableToResolve(bktInfo.Zone, h.cfg.ResolveZoneList, h.cfg.IsResolveListAllow) {
w.Header().Set(api.ContainerName, bktInfo.Name)
w.Header().Set(api.ContainerZone, bktInfo.Zone)
}
api.WriteResponse(w, http.StatusOK, nil, api.MimeNone)
}
@ -156,3 +163,25 @@ func writeLockHeaders(h http.Header, legalHold *data.LegalHold, retention *data.
h.Set(api.AmzObjectLockMode, retention.Mode)
}
}
func isAvailableToResolve(zone string, list []string, isAllowList bool) bool {
// empty zone means container doesn't have proper system name,
// so we don't have to resolve it
if len(zone) == 0 {
return false
}
var zoneInList bool
for _, t := range list {
if t == zone {
zoneInList = true
break
}
}
// InList | IsAllowList | Result
// 0 0 1
// 0 1 0
// 1 0 0
// 1 1 1
return zoneInList == isAllowList
}

View file

@ -6,11 +6,11 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/stretchr/testify/require"
)
@ -86,6 +86,26 @@ func TestInvalidAccessThroughCache(t *testing.T) {
assertStatus(t, w, http.StatusForbidden)
}
func TestIsAvailableToResolve(t *testing.T) {
list := []string{"container", "s3"}
for i, testCase := range [...]struct {
isAllowList bool
list []string
zone string
expected bool
}{
{isAllowList: true, list: list, zone: "container", expected: true},
{isAllowList: true, list: list, zone: "sftp", expected: false},
{isAllowList: false, list: list, zone: "s3", expected: false},
{isAllowList: false, list: list, zone: "system", expected: true},
{isAllowList: true, list: list, zone: "", expected: false},
} {
result := isAvailableToResolve(testCase.zone, testCase.list, testCase.isAllowList)
require.Equal(t, testCase.expected, result, "case %d", i+1)
}
}
func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box {
var err error
if key == nil {

View file

@ -3,7 +3,7 @@ package handler
import (
"net/http"
"github.com/nspcc-dev/neofs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
)
func (h *handler) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {

View file

@ -4,8 +4,8 @@ import (
"net/http"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
)
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.

View file

@ -1,16 +1,17 @@
package handler
import (
"context"
"encoding/xml"
"fmt"
"net/http"
"strconv"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
)
const (
@ -208,7 +209,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
return
}
lock, err := formObjectLockFromRetention(retention, r.Header)
lock, err := formObjectLockFromRetention(r.Context(), retention, r.Header)
if err != nil {
h.logAndSendError(w, "invalid retention configuration", reqInfo, err)
return
@ -300,7 +301,7 @@ func checkLockConfiguration(conf *data.ObjectLockConfiguration) error {
return nil
}
func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConfiguration, header http.Header) (*data.ObjectLock, error) {
func formObjectLock(ctx context.Context, bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConfiguration, header http.Header) (*data.ObjectLock, error) {
if !bktInfo.ObjectLockEnabled {
if existLockHeaders(header) {
return nil, apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound)
@ -318,7 +319,7 @@ func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConf
retention := &data.RetentionLock{}
defaultRetention := defaultConfig.Rule.DefaultRetention
retention.IsCompliance = defaultRetention.Mode == complianceMode
now := time.Now()
now := layer.TimeNow(ctx)
if defaultRetention.Days != 0 {
retention.Until = now.Add(time.Duration(defaultRetention.Days) * dayDuration)
} else {
@ -370,7 +371,7 @@ func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConf
objectLock.Retention.ByPassedGovernance = bypass
}
if objectLock.Retention.Until.Before(time.Now()) {
if objectLock.Retention.Until.Before(layer.TimeNow(ctx)) {
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
}
}
@ -384,7 +385,7 @@ func existLockHeaders(header http.Header) bool {
header.Get(api.AmzObjectLockRetainUntilDate) != ""
}
func formObjectLockFromRetention(retention *data.Retention, header http.Header) (*data.ObjectLock, error) {
func formObjectLockFromRetention(ctx context.Context, retention *data.Retention, header http.Header) (*data.ObjectLock, error) {
if retention.Mode != governanceMode && retention.Mode != complianceMode {
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
}
@ -394,7 +395,7 @@ func formObjectLockFromRetention(retention *data.Retention, header http.Header)
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
}
if retentionDate.Before(time.Now()) {
if retentionDate.Before(layer.TimeNow(ctx)) {
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
}

View file

@ -10,15 +10,17 @@ import (
"testing"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/stretchr/testify/require"
)
const defaultURL = "http://localhost/"
func TestFormObjectLock(t *testing.T) {
ctx := context.Background()
for _, tc := range []struct {
name string
bktInfo *data.BucketInfo
@ -73,7 +75,7 @@ func TestFormObjectLock(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
actualObjLock, err := formObjectLock(tc.bktInfo, tc.config, tc.header)
actualObjLock, err := formObjectLock(ctx, tc.bktInfo, tc.config, tc.header)
if tc.expectedError {
require.Error(t, err)
return
@ -86,6 +88,8 @@ func TestFormObjectLock(t *testing.T) {
}
func TestFormObjectLockFromRetention(t *testing.T) {
ctx := context.Background()
for _, tc := range []struct {
name string
retention *data.Retention
@ -132,7 +136,7 @@ func TestFormObjectLockFromRetention(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
actualObjLock, err := formObjectLockFromRetention(tc.retention, tc.header)
actualObjLock, err := formObjectLockFromRetention(ctx, tc.retention, tc.header)
if tc.expectedError {
require.Error(t, err)
return

View file

@ -2,17 +2,18 @@ package handler
import (
"encoding/xml"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/google/uuid"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-sdk-go/session"
"go.uber.org/zap"
)
@ -137,7 +138,7 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
}
}
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
p.Info.Encryption, err = formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -226,7 +227,7 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
Reader: r.Body,
}
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
p.Info.Encryption, err = formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -331,7 +332,7 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
Range: srcRange,
}
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
p.Info.Encryption, err = formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -372,8 +373,6 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
}
var (
sessionTokenSetEACL *session.Container
uploadID = r.URL.Query().Get(uploadIDHeaderName)
uploadInfo = &layer.UploadInfoParams{
UploadID: uploadID,
@ -384,7 +383,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
)
reqBody := new(CompleteMultipartUpload)
if err = xml.NewDecoder(r.Body).Decode(reqBody); err != nil {
if err = h.cfg.XMLDecoder.NewCompleteMultipartDecoder(r.Body).Decode(reqBody); err != nil {
h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
errors.GetAPIError(errors.ErrMalformedXML), additional...)
return
@ -399,10 +398,51 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
Parts: reqBody.Parts,
}
// Next operations might take some time, so we want to keep client's
// connection alive. To do so, gateway sends periodic white spaces
// back to the client the same way as Amazon S3 service does.
stopPeriodicResponseWriter := periodicXMLWriter(w, h.cfg.CompleteMultipartKeepalive)
// Start complete multipart upload which may take some time to fetch object
// and re-upload it part by part.
objInfo, err := h.completeMultipartUpload(r, c, bktInfo, reqInfo)
// Stop periodic writer as complete multipart upload is finished
// successfully or not.
headerIsWritten := stopPeriodicResponseWriter()
responseWriter := api.EncodeToResponse
errLogger := h.logAndSendError
// Do not send XML and HTTP headers if periodic writer was invoked at this point.
if headerIsWritten {
responseWriter = api.EncodeToResponseNoHeader
errLogger = h.logAndSendErrorNoHeader
}
if err != nil {
errLogger(w, "complete multipart error", reqInfo, err, additional...)
return
}
response := CompleteMultipartUploadResponse{
Bucket: objInfo.Bucket,
ETag: objInfo.HashSum,
Key: objInfo.Name,
}
// Here we previously set api.AmzVersionID header for versioned bucket.
// It is not possible after #60, because of periodic white
// space XML writer to keep connection with the client.
if err = responseWriter(w, response); err != nil {
errLogger(w, "something went wrong", reqInfo, err)
}
}
func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMultipartParams, bktInfo *data.BucketInfo, reqInfo *api.ReqInfo) (*data.ObjectInfo, error) {
uploadData, extendedObjInfo, err := h.obj.CompleteMultipartUpload(r.Context(), c)
if err != nil {
h.logAndSendError(w, "could not complete multipart upload", reqInfo, err, additional...)
return
return nil, fmt.Errorf("could not complete multipart upload: %w", err)
}
objInfo := extendedObjInfo.ObjectInfo
@ -417,21 +457,22 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
NodeVersion: extendedObjInfo.NodeVersion,
}
if _, err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
h.logAndSendError(w, "could not put tagging file of completed multipart upload", reqInfo, err, additional...)
return
return nil, fmt.Errorf("could not put tagging file of completed multipart upload: %w", err)
}
}
if len(uploadData.ACLHeaders) != 0 {
sessionTokenSetEACL, err := getSessionTokenSetEACL(r.Context())
if err != nil {
return nil, fmt.Errorf("couldn't get eacl token: %w", err)
}
key, err := h.bearerTokenIssuerKey(r.Context())
if err != nil {
h.logAndSendError(w, "couldn't get gate key", reqInfo, err)
return
return nil, fmt.Errorf("couldn't get gate key: %w", err)
}
acl, err := parseACLHeaders(r.Header, key)
if err != nil {
h.logAndSendError(w, "could not parse acl", reqInfo, err)
return
return nil, fmt.Errorf("could not parse acl: %w", err)
}
resInfo := &resourceInfo{
@ -440,12 +481,10 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
}
astObject, err := aclToAst(acl, resInfo)
if err != nil {
h.logAndSendError(w, "could not translate acl of completed multipart upload to ast", reqInfo, err, additional...)
return
return nil, fmt.Errorf("could not translate acl of completed multipart upload to ast: %w", err)
}
if _, err = h.updateBucketACL(r, astObject, bktInfo, sessionTokenSetEACL); err != nil {
h.logAndSendError(w, "could not update bucket acl while completing multipart upload", reqInfo, err, additional...)
return
return nil, fmt.Errorf("could not update bucket acl while completing multipart upload: %w", err)
}
}
@ -459,24 +498,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
h.log.Error("couldn't send notification: %w", zap.Error(err))
}
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
}
response := CompleteMultipartUploadResponse{
Bucket: objInfo.Bucket,
ETag: objInfo.HashSum,
Key: objInfo.Name,
}
if bktSettings.VersioningEnabled() {
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
}
if err = api.EncodeToResponse(w, response); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
}
return objInfo, nil
}
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
@ -573,7 +595,7 @@ func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
PartNumberMarker: partNumberMarker,
}
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
p.Info.Encryption, err = formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -608,7 +630,7 @@ func (h *handler) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Req
Key: reqInfo.ObjectName,
}
p.Encryption, err = h.formEncryptionParams(r.Header)
p.Encryption, err = formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -681,3 +703,62 @@ func encodeListPartsToResponse(info *layer.ListPartsInfo, params *layer.ListPart
Parts: info.Parts,
}
}
// periodicXMLWriter creates go routine to write xml header and whitespaces
// over time to avoid connection drop from the client. To work properly,
// pass `http.ResponseWriter` with implemented `http.Flusher` interface.
// Returns stop function which returns boolean if writer has been used
// during goroutine execution. To disable writer, pass 0 duration value.
func periodicXMLWriter(w io.Writer, dur time.Duration) (stop func() bool) {
if dur == 0 { // 0 duration disables periodic writer
return func() bool { return false }
}
whitespaceChar := []byte(" ")
closer := make(chan struct{})
done := make(chan struct{})
headerWritten := false
go func() {
defer close(done)
tick := time.NewTicker(dur)
defer tick.Stop()
for {
select {
case <-tick.C:
if !headerWritten {
_, err := w.Write([]byte(xml.Header))
headerWritten = err == nil
}
_, err := w.Write(whitespaceChar)
if err != nil {
return // is there anything we can do better than ignore error?
}
if buffered, ok := w.(http.Flusher); ok {
buffered.Flush()
}
case <-closer:
return
}
}
}()
stop = func() bool {
close(closer)
<-done // wait for goroutine to stop
return headerWritten
}
return stop
}
// periodicWriterErrorSender returns handler function to send error. If header is
// alreay written by periodic XML writer, do not send HTTP and XML headers.
func (h *handler) periodicWriterErrorSender(headerWritten bool) func(http.ResponseWriter, string, *api.ReqInfo, error, ...zap.Field) {
if headerWritten {
return h.logAndSendErrorNoHeader
}
return h.logAndSendError
}

View file

@ -0,0 +1,48 @@
package handler
import (
"bytes"
"encoding/xml"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestPeriodicWriter(t *testing.T) {
const dur = 100 * time.Millisecond
const whitespaces = 8
expected := []byte(xml.Header)
for i := 0; i < whitespaces; i++ {
expected = append(expected, []byte(" ")...)
}
t.Run("writes data", func(t *testing.T) {
buf := bytes.NewBuffer(nil)
stop := periodicXMLWriter(buf, dur)
// N number of whitespaces + half durations to guarantee at least N writes in buffer
time.Sleep(whitespaces*dur + dur/2)
require.True(t, stop())
require.Equal(t, expected, buf.Bytes())
t.Run("no additional data after stop", func(t *testing.T) {
time.Sleep(2 * dur)
require.Equal(t, expected, buf.Bytes())
})
})
t.Run("does not write data", func(t *testing.T) {
buf := bytes.NewBuffer(nil)
stop := periodicXMLWriter(buf, dur)
time.Sleep(dur / 2)
require.False(t, stop())
require.Empty(t, buf.Bytes())
t.Run("disabled", func(t *testing.T) {
stop = periodicXMLWriter(buf, 0)
require.False(t, stop())
require.Empty(t, buf.Bytes())
})
})
}

View file

@ -3,8 +3,8 @@ package handler
import (
"net/http"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
)
func (h *handler) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {

View file

@ -6,13 +6,14 @@ import (
"fmt"
"net/http"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"github.com/google/uuid"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
)
type (
@ -22,6 +23,7 @@ type (
BktInfo *data.BucketInfo
ReqInfo *api.ReqInfo
User string
Time time.Time
}
NotificationConfiguration struct {
@ -107,7 +109,7 @@ func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Re
return
}
if _, err = h.checkBucketConfiguration(conf, reqInfo); err != nil {
if _, err = h.checkBucketConfiguration(r.Context(), conf, reqInfo); err != nil {
h.logAndSendError(w, "couldn't check bucket configuration", reqInfo, err)
return
}
@ -164,13 +166,15 @@ func (h *handler) sendNotifications(ctx context.Context, p *SendNotificationPara
p.User = bearer.ResolveIssuer(*box.Gate.BearerToken).EncodeToString()
}
p.Time = layer.TimeNow(ctx)
topics := filterSubjects(conf, p.Event, p.NotificationInfo.Name)
return h.notificator.SendNotifications(topics, p)
}
// checkBucketConfiguration checks notification configuration and generates an ID for configurations with empty ids.
func (h *handler) checkBucketConfiguration(conf *data.NotificationConfiguration, r *api.ReqInfo) (completed bool, err error) {
func (h *handler) checkBucketConfiguration(ctx context.Context, conf *data.NotificationConfiguration, r *api.ReqInfo) (completed bool, err error) {
if conf == nil {
return
}
@ -189,7 +193,7 @@ func (h *handler) checkBucketConfiguration(conf *data.NotificationConfiguration,
}
if h.cfg.NotificatorEnabled {
if err = h.notificator.SendTestNotification(q.QueueArn, r.BucketName, r.RequestID, r.Host); err != nil {
if err = h.notificator.SendTestNotification(q.QueueArn, r.BucketName, r.RequestID, r.Host, layer.TimeNow(ctx)); err != nil {
return
}
} else {

View file

@ -3,8 +3,8 @@ package handler
import (
"testing"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/stretchr/testify/require"
)

View file

@ -6,11 +6,11 @@ import (
"strconv"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
// ListObjectsV1Handler handles objects listing requests for API version 1.

View file

@ -6,7 +6,7 @@ import (
"strconv"
"testing"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/stretchr/testify/require"
)

View file

@ -16,15 +16,15 @@ import (
"strings"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"go.uber.org/zap"
)
@ -218,7 +218,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
encryption, err := h.formEncryptionParams(r.Header)
encryptionParams, err := formEncryptionParams(r)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -230,7 +230,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
Reader: r.Body,
Size: r.ContentLength,
Header: metadata,
Encryption: encryption,
Encryption: encryptionParams,
CopiesNumber: copiesNumber,
}
@ -240,7 +240,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
params.Lock, err = formObjectLock(bktInfo, settings.LockConfiguration, r.Header)
params.Lock, err = formObjectLock(r.Context(), bktInfo, settings.LockConfiguration, r.Header)
if err != nil {
h.logAndSendError(w, "could not form object lock", reqInfo, err)
return
@ -304,7 +304,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
if settings.VersioningEnabled() {
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
}
if encryption.Enabled() {
if encryptionParams.Enabled() {
addSSECHeaders(w.Header(), r.Header)
}
@ -313,7 +313,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
}
func getCopiesNumberOrDefault(metadata map[string]string, defaultCopiesNumber uint32) (uint32, error) {
copiesNumberStr, ok := metadata[layer.AttributeNeofsCopiesNumber]
copiesNumberStr, ok := metadata[layer.AttributeFrostfsCopiesNumber]
if !ok {
return defaultCopiesNumber, nil
}
@ -326,16 +326,16 @@ func getCopiesNumberOrDefault(metadata map[string]string, defaultCopiesNumber ui
return uint32(copiesNumber), nil
}
func (h handler) formEncryptionParams(header http.Header) (enc encryption.Params, err error) {
sseCustomerAlgorithm := header.Get(api.AmzServerSideEncryptionCustomerAlgorithm)
sseCustomerKey := header.Get(api.AmzServerSideEncryptionCustomerKey)
sseCustomerKeyMD5 := header.Get(api.AmzServerSideEncryptionCustomerKeyMD5)
func formEncryptionParams(r *http.Request) (enc encryption.Params, err error) {
sseCustomerAlgorithm := r.Header.Get(api.AmzServerSideEncryptionCustomerAlgorithm)
sseCustomerKey := r.Header.Get(api.AmzServerSideEncryptionCustomerKey)
sseCustomerKeyMD5 := r.Header.Get(api.AmzServerSideEncryptionCustomerKeyMD5)
if len(sseCustomerAlgorithm) == 0 && len(sseCustomerKey) == 0 && len(sseCustomerKeyMD5) == 0 {
return
}
if !h.cfg.TLSEnabled {
if r.TLS == nil {
return enc, errorsStd.New("encryption available only when TLS is enabled")
}
@ -667,12 +667,10 @@ func parseMetadata(r *http.Request) map[string]string {
}
func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
var (
reqInfo = api.GetReqInfo(r.Context())
p = layer.CreateBucketParams{
Name: reqInfo.BucketName,
}
)
reqInfo := api.GetReqInfo(r.Context())
p := &layer.CreateBucketParams{
Name: reqInfo.BucketName,
}
if err := checkBucketName(reqInfo.BucketName); err != nil {
h.logAndSendError(w, "invalid bucket name", reqInfo, err)
@ -722,29 +720,22 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
return
}
useDefaultPolicy := true
if createParams.LocationConstraint != "" {
for _, placementPolicy := range policies {
if placementPolicy.LocationConstraint == createParams.LocationConstraint {
p.Policy = placementPolicy.Policy
p.LocationConstraint = createParams.LocationConstraint
useDefaultPolicy = false
break
}
}
}
if useDefaultPolicy {
p.Policy = h.cfg.DefaultPolicy
if err = h.setPolicy(p, createParams.LocationConstraint, policies); err != nil {
h.logAndSendError(w, "couldn't set placement policy", reqInfo, err)
return
}
p.ObjectLockEnabled = isLockEnabled(r.Header)
bktInfo, err := h.obj.CreateBucket(r.Context(), &p)
bktInfo, err := h.obj.CreateBucket(r.Context(), p)
if err != nil {
h.logAndSendError(w, "could not create bucket", reqInfo, err)
return
}
h.log.Info("bucket is created", zap.String("reqId", reqInfo.RequestID),
zap.String("bucket", reqInfo.BucketName), zap.Stringer("container_id", bktInfo.CID))
if p.ObjectLockEnabled {
sp := &layer.PutSettingsParams{
BktInfo: bktInfo,
@ -757,11 +748,32 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
}
}
h.log.Info("bucket is created", zap.Stringer("container_id", bktInfo.CID))
api.WriteSuccessResponseHeadersOnly(w)
}
func (h handler) setPolicy(prm *layer.CreateBucketParams, locationConstraint string, userPolicies []*accessbox.ContainerPolicy) error {
prm.Policy = h.cfg.Policy.Default()
prm.LocationConstraint = locationConstraint
if locationConstraint == "" {
return nil
}
for _, placementPolicy := range userPolicies {
if placementPolicy.LocationConstraint == locationConstraint {
prm.Policy = placementPolicy.Policy
return nil
}
}
if policy, ok := h.cfg.Policy.Get(locationConstraint); ok {
prm.Policy = policy
return nil
}
return errors.GetAPIError(errors.ErrInvalidLocationConstraint)
}
func isLockEnabled(header http.Header) bool {
lockEnabledStr := header.Get(api.AmzBucketObjectLockEnabled)
lockEnabled, _ := strconv.ParseBool(lockEnabledStr)

View file

@ -8,8 +8,8 @@ import (
"testing"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/stretchr/testify/require"
)
@ -114,7 +114,7 @@ func TestPutObjectOverrideCopiesNumber(t *testing.T) {
bktInfo := createTestBucket(tc, bktName)
w, r := prepareTestRequest(tc, bktName, objName, nil)
r.Header.Set(api.MetadataPrefix+strings.ToUpper(layer.AttributeNeofsCopiesNumber), "1")
r.Header.Set(api.MetadataPrefix+strings.ToUpper(layer.AttributeFrostfsCopiesNumber), "1")
tc.Handler().PutObjectHandler(w, r)
p := &layer.HeadObjectParams{
@ -124,5 +124,5 @@ func TestPutObjectOverrideCopiesNumber(t *testing.T) {
objInfo, err := tc.Layer().GetObjectInfo(tc.Context(), p)
require.NoError(t, err)
require.Equal(t, "1", objInfo.Headers[layer.AttributeNeofsCopiesNumber])
require.Equal(t, "1", objInfo.Headers[layer.AttributeFrostfsCopiesNumber])
}

View file

@ -8,10 +8,10 @@ import (
"strings"
"unicode"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"go.uber.org/zap"
)

View file

@ -3,8 +3,8 @@ package handler
import (
"net/http"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
)
func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {

View file

@ -7,11 +7,11 @@ import (
"strconv"
"strings"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"go.uber.org/zap"
)
@ -29,6 +29,19 @@ func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo
h.log.Error("call method", fields...)
}
func (h *handler) logAndSendErrorNoHeader(w http.ResponseWriter, logText string, reqInfo *api.ReqInfo, err error, additional ...zap.Field) {
api.WriteErrorResponseNoHeader(w, reqInfo, transformToS3Error(err))
fields := []zap.Field{
zap.String("request_id", reqInfo.RequestID),
zap.String("method", reqInfo.API),
zap.String("bucket", reqInfo.BucketName),
zap.String("object", reqInfo.ObjectName),
zap.String("description", logText),
zap.Error(err)}
fields = append(fields, additional...)
h.log.Error("call method", fields...)
}
func transformToS3Error(err error) error {
if _, ok := err.(errors.Error); ok {
return err
@ -42,6 +55,10 @@ func transformToS3Error(err error) error {
return errors.GetAPIError(errors.ErrInternalError)
}
func (h *handler) ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error) {
return h.obj.GetBucketInfo(ctx, bucket)
}
func (h *handler) getBucketAndCheckOwner(r *http.Request, bucket string, header ...string) (*data.BucketInfo, error) {
bktInfo, err := h.obj.GetBucketInfo(r.Context(), bucket)
if err != nil {

View file

@ -4,10 +4,10 @@ import (
"encoding/xml"
"net/http"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
)
func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {

View file

@ -2,17 +2,17 @@ package api
// Standard S3 HTTP request/response constants.
const (
MetadataPrefix = "X-Amz-Meta-"
NeoFSSystemMetadataPrefix = "S3-"
AmzMetadataDirective = "X-Amz-Metadata-Directive"
AmzTaggingDirective = "X-Amz-Tagging-Directive"
AmzVersionID = "X-Amz-Version-Id"
AmzTaggingCount = "X-Amz-Tagging-Count"
AmzTagging = "X-Amz-Tagging"
AmzDeleteMarker = "X-Amz-Delete-Marker"
AmzCopySource = "X-Amz-Copy-Source"
AmzCopySourceRange = "X-Amz-Copy-Source-Range"
AmzDate = "X-Amz-Date"
MetadataPrefix = "X-Amz-Meta-"
FrostFSSystemMetadataPrefix = "S3-"
AmzMetadataDirective = "X-Amz-Metadata-Directive"
AmzTaggingDirective = "X-Amz-Tagging-Directive"
AmzVersionID = "X-Amz-Version-Id"
AmzTaggingCount = "X-Amz-Tagging-Count"
AmzTagging = "X-Amz-Tagging"
AmzDeleteMarker = "X-Amz-Delete-Marker"
AmzCopySource = "X-Amz-Copy-Source"
AmzCopySourceRange = "X-Amz-Copy-Source-Range"
AmzDate = "X-Amz-Date"
LastModified = "Last-Modified"
Date = "Date"
@ -62,7 +62,9 @@ const (
AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"
AmzServerSideEncryptionCustomerKeyMD5 = "x-amz-server-side-encryption-customer-key-MD5"
ContainerID = "X-Container-Id"
ContainerID = "X-Container-Id"
ContainerName = "X-Container-Name"
ContainerZone = "X-Container-Zone"
AccessControlAllowOrigin = "Access-Control-Allow-Origin"
AccessControlAllowMethods = "Access-Control-Allow-Methods"

View file

@ -1,11 +1,11 @@
package layer
import (
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"go.uber.org/zap"
)

View file

@ -4,8 +4,8 @@ import (
"context"
errorsStd "errors"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
)
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {

View file

@ -4,16 +4,16 @@ import (
"context"
"fmt"
"strconv"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-sdk-go/client"
"github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/session"
v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"go.uber.org/zap"
)
@ -42,14 +42,14 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
Name: idCnr.EncodeToString(),
}
)
res, err = n.neoFS.Container(ctx, idCnr)
res, err = n.frostFS.Container(ctx, idCnr)
if err != nil {
log.Error("could not fetch container", zap.Error(err))
if client.IsErrContainerNotFound(err) {
return nil, errors.GetAPIError(errors.ErrNoSuchBucket)
}
return nil, fmt.Errorf("get neofs container: %w", err)
return nil, fmt.Errorf("get frostfs container: %w", err)
}
cnr := *res
@ -57,6 +57,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
info.Owner = cnr.Owner()
if domain := container.ReadDomain(cnr); domain.Name() != "" {
info.Name = domain.Name()
info.Zone = domain.Zone()
}
info.Created = container.CreatedAt(cnr)
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
@ -84,7 +85,7 @@ func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
res []cid.ID
rid = api.GetRequestID(ctx)
)
res, err = n.neoFS.UserContainers(ctx, own)
res, err = n.frostFS.UserContainers(ctx, own)
if err != nil {
n.log.Error("could not list user containers",
zap.String("request_id", rid),
@ -115,8 +116,9 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
}
bktInfo := &data.BucketInfo{
Name: p.Name,
Zone: v2container.SysAttributeZoneDefault,
Owner: ownerID,
Created: time.Now(), // this can be a little incorrect since the real time is set later
Created: TimeNow(ctx),
LocationConstraint: p.LocationConstraint,
ObjectLockEnabled: p.ObjectLockEnabled,
}
@ -133,11 +135,12 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
})
}
idCnr, err := n.neoFS.CreateContainer(ctx, PrmContainerCreate{
idCnr, err := n.frostFS.CreateContainer(ctx, PrmContainerCreate{
Creator: bktInfo.Owner,
Policy: p.Policy,
Name: p.Name,
SessionToken: p.SessionContainerCreation,
CreationTime: bktInfo.Created,
AdditionalAttributes: attributes,
})
if err != nil {
@ -158,9 +161,9 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
func (n *layer) setContainerEACLTable(ctx context.Context, idCnr cid.ID, table *eacl.Table, sessionToken *session.Container) error {
table.SetCID(idCnr)
return n.neoFS.SetContainerEACL(ctx, *table, sessionToken)
return n.frostFS.SetContainerEACL(ctx, *table, sessionToken)
}
func (n *layer) GetContainerEACL(ctx context.Context, idCnr cid.ID) (*eacl.Table, error) {
return n.neoFS.ContainerEACL(ctx, idCnr)
return n.frostFS.ContainerEACL(ctx, idCnr)
}

View file

@ -8,8 +8,8 @@ import (
"fmt"
"io"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"go.uber.org/zap"
)
@ -39,8 +39,9 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
prm := PrmObjectCreate{
Container: p.BktInfo.CID,
Creator: p.BktInfo.Owner,
Payload: p.Reader,
Payload: &buf,
Filepath: p.BktInfo.CORSObjectName(),
CreationTime: TimeNow(ctx),
CopiesNumber: p.CopiesNumber,
}

View file

@ -7,21 +7,21 @@ import (
"io"
"time"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
"github.com/nspcc-dev/neofs-sdk-go/container"
"github.com/nspcc-dev/neofs-sdk-go/container/acl"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
)
// PrmContainerCreate groups parameters of NeoFS.CreateContainer operation.
// PrmContainerCreate groups parameters of FrostFS.CreateContainer operation.
type PrmContainerCreate struct {
// NeoFS identifier of the container creator.
// FrostFS identifier of the container creator.
Creator user.ID
// Container placement policy.
@ -30,6 +30,9 @@ type PrmContainerCreate struct {
// Name for the container.
Name string
// CreationTime value for Timestamp attribute
CreationTime time.Time
// Token of the container's creation session. Nil means session absence.
SessionToken *session.Container
@ -40,7 +43,7 @@ type PrmContainerCreate struct {
AdditionalAttributes [][2]string
}
// PrmAuth groups authentication parameters for the NeoFS operation.
// PrmAuth groups authentication parameters for the FrostFS operation.
type PrmAuth struct {
// Bearer token to be used for the operation. Overlaps PrivateKey. Optional.
BearerToken *bearer.Token
@ -49,7 +52,7 @@ type PrmAuth struct {
PrivateKey *ecdsa.PrivateKey
}
// PrmObjectRead groups parameters of NeoFS.ReadObject operation.
// PrmObjectRead groups parameters of FrostFS.ReadObject operation.
type PrmObjectRead struct {
// Authentication parameters.
PrmAuth
@ -70,7 +73,7 @@ type PrmObjectRead struct {
PayloadRange [2]uint64
}
// ObjectPart represents partially read NeoFS object.
// ObjectPart represents partially read FrostFS object.
type ObjectPart struct {
// Object header with optional in-memory payload part.
Head *object.Object
@ -80,7 +83,7 @@ type ObjectPart struct {
Payload io.ReadCloser
}
// PrmObjectCreate groups parameters of NeoFS.CreateObject operation.
// PrmObjectCreate groups parameters of FrostFS.CreateObject operation.
type PrmObjectCreate struct {
// Authentication parameters.
PrmAuth
@ -88,12 +91,15 @@ type PrmObjectCreate struct {
// Container to store the object.
Container cid.ID
// NeoFS identifier of the object creator.
// FrostFS identifier of the object creator.
Creator user.ID
// Key-value object attributes.
Attributes [][2]string
// Value for Timestamp attribute (optional).
CreationTime time.Time
// List of ids to lock (optional).
Locks []oid.ID
@ -110,7 +116,7 @@ type PrmObjectCreate struct {
CopiesNumber uint32
}
// PrmObjectDelete groups parameters of NeoFS.DeleteObject operation.
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
type PrmObjectDelete struct {
// Authentication parameters.
PrmAuth
@ -122,12 +128,12 @@ type PrmObjectDelete struct {
Object oid.ID
}
// ErrAccessDenied is returned from NeoFS in case of access violation.
// ErrAccessDenied is returned from FrostFS in case of access violation.
var ErrAccessDenied = errors.New("access denied")
// NeoFS represents virtual connection to NeoFS network.
type NeoFS interface {
// CreateContainer creates and saves parameterized container in NeoFS.
// FrostFS represents virtual connection to FrostFS network.
type FrostFS interface {
// CreateContainer creates and saves parameterized container in FrostFS.
// It sets 'Timestamp' attribute to the current time.
// It returns the ID of the saved container.
//
@ -137,7 +143,7 @@ type NeoFS interface {
// prevented the container from being created.
CreateContainer(context.Context, PrmContainerCreate) (cid.ID, error)
// Container reads a container from NeoFS by ID.
// Container reads a container from FrostFS by ID.
//
// It returns exactly one non-nil value. It returns any error encountered which
// prevented the container from being read.
@ -149,26 +155,26 @@ type NeoFS interface {
// prevented the containers from being listed.
UserContainers(context.Context, user.ID) ([]cid.ID, error)
// SetContainerEACL saves the eACL table of the container in NeoFS. The
// SetContainerEACL saves the eACL table of the container in FrostFS. The
// extended ACL is modified within session if session token is not nil.
//
// It returns any error encountered which prevented the eACL from being saved.
SetContainerEACL(context.Context, eacl.Table, *session.Container) error
// ContainerEACL reads the container eACL from NeoFS by the container ID.
// ContainerEACL reads the container eACL from FrostFS by the container ID.
//
// It returns exactly one non-nil value. It returns any error encountered which
// prevented the eACL from being read.
ContainerEACL(context.Context, cid.ID) (*eacl.Table, error)
// DeleteContainer marks the container to be removed from NeoFS by ID.
// DeleteContainer marks the container to be removed from FrostFS by ID.
// Request is sent within session if the session token is specified.
// Successful return does not guarantee actual removal.
//
// It returns any error encountered which prevented the removal request from being sent.
DeleteContainer(context.Context, cid.ID, *session.Container) error
// ReadObject reads a part of the object from the NeoFS container by identifier.
// ReadObject reads a part of the object from the FrostFS container by identifier.
// Exact part is returned according to the parameters:
// * with header only: empty payload (both in-mem and reader parts are nil);
// * with payload only: header is nil (zero range means full payload);
@ -184,7 +190,7 @@ type NeoFS interface {
// prevented the object header from being read.
ReadObject(context.Context, PrmObjectRead) (*ObjectPart, error)
// CreateObject creates and saves a parameterized object in the NeoFS container.
// CreateObject creates and saves a parameterized object in the FrostFS container.
// It sets 'Timestamp' attribute to the current time.
// It returns the ID of the saved object.
//
@ -196,7 +202,7 @@ type NeoFS interface {
// prevented the container from being created.
CreateObject(context.Context, PrmObjectCreate) (oid.ID, error)
// DeleteObject marks the object to be removed from the NeoFS container by identifier.
// DeleteObject marks the object to be removed from the FrostFS container by identifier.
// Successful return does not guarantee actual removal.
//
// It returns ErrAccessDenied on remove access violation.
@ -204,11 +210,11 @@ type NeoFS interface {
// It returns any error encountered which prevented the removal request from being sent.
DeleteObject(context.Context, PrmObjectDelete) error
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided time.
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided now and future time.
// Note:
// * time must be in the future
// * time will be ceil rounded to match epoch
// * future time must be after the now
// * future time will be ceil rounded to match epoch
//
// It returns any error encountered which prevented computing epochs.
TimeToEpoch(context.Context, time.Time) (uint64, uint64, error)
TimeToEpoch(ctx context.Context, now time.Time, future time.Time) (uint64, uint64, error)
}

View file

@ -11,20 +11,20 @@ import (
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nats-io/nats.go"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/user"
"go.uber.org/zap"
)
@ -45,7 +45,7 @@ type (
}
layer struct {
neoFS NeoFS
frostFS FrostFS
log *zap.Logger
anonKey AnonymousKey
resolver BucketResolver
@ -250,12 +250,12 @@ const (
AESEncryptionAlgorithm = "AES256"
AESKeySize = 32
AttributeEncryptionAlgorithm = api.NeoFSSystemMetadataPrefix + "Algorithm"
AttributeDecryptedSize = api.NeoFSSystemMetadataPrefix + "Decrypted-Size"
AttributeHMACSalt = api.NeoFSSystemMetadataPrefix + "HMAC-Salt"
AttributeHMACKey = api.NeoFSSystemMetadataPrefix + "HMAC-Key"
AttributeEncryptionAlgorithm = api.FrostFSSystemMetadataPrefix + "Algorithm"
AttributeDecryptedSize = api.FrostFSSystemMetadataPrefix + "Decrypted-Size"
AttributeHMACSalt = api.FrostFSSystemMetadataPrefix + "HMAC-Salt"
AttributeHMACKey = api.FrostFSSystemMetadataPrefix + "HMAC-Key"
AttributeNeofsCopiesNumber = "neofs-copies-number" // such formate to match X-Amz-Meta-Neofs-Copies-Number header
AttributeFrostfsCopiesNumber = "frostfs-copies-number" // such format to match X-Amz-Meta-Frostfs-Copies-Number header
)
func (t *VersionedObject) String() string {
@ -268,9 +268,9 @@ func (f MsgHandlerFunc) HandleMessage(ctx context.Context, msg *nats.Msg) error
// NewLayer creates an instance of a layer. It checks credentials
// and establishes gRPC connection with the node.
func NewLayer(log *zap.Logger, neoFS NeoFS, config *Config) Client {
func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) Client {
return &layer{
neoFS: neoFS,
frostFS: frostFS,
log: log,
anonKey: config.AnonKey,
resolver: config.Resolver,
@ -306,6 +306,15 @@ func IsAuthenticatedRequest(ctx context.Context) bool {
return ok
}
// TimeNow returns client time from request or time.Now().
func TimeNow(ctx context.Context) time.Time {
if now, ok := ctx.Value(api.ClientTime).(time.Time); ok {
return now
}
return time.Now()
}
// Owner returns owner id from BearerToken (context) or from client owner.
func (n *layer) Owner(ctx context.Context) user.ID {
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
@ -368,7 +377,7 @@ func (n *layer) PutBucketACL(ctx context.Context, param *PutBucketACLParams) err
}
// ListBuckets returns all user containers. The name of the bucket is a container
// id. Timestamp is omitted since it is not saved in neofs container.
// id. Timestamp is omitted since it is not saved in frostfs container.
func (n *layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) {
return n.containerList(ctx)
}
@ -473,11 +482,27 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.O
// GetExtendedObjectInfo returns meta information and corresponding info from the tree service about the object.
func (n *layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) {
var objInfo *data.ExtendedObjectInfo
var err error
if len(p.VersionID) == 0 {
return n.headLastVersionIfNotDeleted(ctx, p.BktInfo, p.Object)
objInfo, err = n.headLastVersionIfNotDeleted(ctx, p.BktInfo, p.Object)
} else {
objInfo, err = n.headVersion(ctx, p.BktInfo, p)
}
if err != nil {
return nil, err
}
return n.headVersion(ctx, p.BktInfo, p)
reqInfo := api.GetReqInfo(ctx)
n.log.Debug("get object",
zap.String("reqId", reqInfo.RequestID),
zap.String("bucket", p.BktInfo.Name),
zap.Stringer("cid", p.BktInfo.CID),
zap.String("object", objInfo.ObjectInfo.Name),
zap.Stringer("oid", objInfo.ObjectInfo.ID))
return objInfo, nil
}
// CopyObject from one bucket into another bucket.
@ -565,7 +590,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
FilePath: obj.Name,
},
DeleteMarker: &data.DeleteMarkerInfo{
Created: time.Now(),
Created: TimeNow(ctx),
Owner: n.Owner(ctx),
},
IsUnversioned: settings.VersioningSuspended(),
@ -636,7 +661,12 @@ func (n *layer) CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.
func (n *layer) ResolveBucket(ctx context.Context, name string) (cid.ID, error) {
var cnrID cid.ID
if err := cnrID.DecodeString(name); err != nil {
return n.resolver.Resolve(ctx, name)
if cnrID, err = n.resolver.Resolve(ctx, name); err != nil {
return cid.ID{}, err
}
reqInfo := api.GetReqInfo(ctx)
n.log.Info("resolve bucket", zap.String("reqId", reqInfo.RequestID), zap.String("bucket", name), zap.Stringer("cid", cnrID))
}
return cnrID, nil
@ -652,5 +682,5 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
}
n.cache.DeleteBucket(p.BktInfo.Name)
return n.neoFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
return n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
}

View file

@ -4,7 +4,8 @@ import (
"testing"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/stretchr/testify/require"
)
@ -43,10 +44,10 @@ func TestObjectLockAttributes(t *testing.T) {
expEpoch := false
for _, attr := range lockObj.Attributes() {
if attr.Key() == AttributeExpirationEpoch {
if attr.Key() == object.SysAttributeExpEpoch {
expEpoch = true
}
}
require.Truef(t, expEpoch, "system header __NEOFS__EXPIRATION_EPOCH presence")
require.Truef(t, expEpoch, "system header __SYSTEM__EXPIRATION_EPOCH presence")
}

View file

@ -11,12 +11,13 @@ import (
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/minio/sio"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
"go.uber.org/zap"
)
@ -143,7 +144,7 @@ func (n *layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
Key: p.Info.Key,
UploadID: p.Info.UploadID,
Owner: n.Owner(ctx),
Created: time.Now(),
Created: TimeNow(ctx),
Meta: make(map[string]string, metaSize),
CopiesNumber: p.CopiesNumber,
}
@ -205,6 +206,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
Creator: bktInfo.Owner,
Attributes: make([][2]string, 2),
Payload: p.Reader,
CreationTime: TimeNow(ctx),
CopiesNumber: multipartInfo.CopiesNumber,
}
@ -227,6 +229,13 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
return nil, err
}
reqInfo := api.GetReqInfo(ctx)
n.log.Debug("upload part",
zap.String("reqId", reqInfo.RequestID),
zap.String("bucket", bktInfo.Name), zap.Stringer("cid", bktInfo.CID),
zap.String("multipart upload", p.Info.UploadID),
zap.Int("part number", p.PartNumber), zap.String("object", p.Info.Key), zap.Stringer("oid", id))
partInfo := &data.PartInfo{
Key: p.Info.Key,
UploadID: p.Info.UploadID,
@ -234,7 +243,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
OID: id,
Size: decSize,
ETag: hex.EncodeToString(hash),
Created: time.Now(),
Created: prm.CreationTime,
}
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)
@ -310,7 +319,7 @@ func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
return n.uploadPart(ctx, multipartInfo, params)
}
// implements io.Reader of payloads of the object list stored in the NeoFS network.
// implements io.Reader of payloads of the object list stored in the FrostFS network.
type multiObjectReader struct {
ctx context.Context
@ -608,10 +617,24 @@ func (n *layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
}
res := make(map[int]*data.PartInfo, len(parts))
for _, part := range parts {
partsNumbers := make([]int, len(parts))
oids := make([]string, len(parts))
for i, part := range parts {
res[part.Number] = part
partsNumbers[i] = part.Number
oids[i] = part.OID.EncodeToString()
}
reqInfo := api.GetReqInfo(ctx)
n.log.Debug("part details",
zap.String("reqId", reqInfo.RequestID),
zap.String("bucket", p.Bkt.Name),
zap.Stringer("cid", p.Bkt.CID),
zap.String("object", p.Key),
zap.String("upload id", p.UploadID),
zap.Ints("part numbers", partsNumbers),
zap.Strings("oids", oids))
return multipartInfo, res, nil
}

View file

@ -10,22 +10,22 @@ import (
"io"
"time"
objectv2 "github.com/nspcc-dev/neofs-api-go/v2/object"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
"github.com/nspcc-dev/neofs-sdk-go/checksum"
"github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/user"
objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
)
type TestNeoFS struct {
NeoFS
type TestFrostFS struct {
FrostFS
objects map[string]*object.Object
containers map[string]*container.Container
@ -33,19 +33,19 @@ type TestNeoFS struct {
currentEpoch uint64
}
func NewTestNeoFS() *TestNeoFS {
return &TestNeoFS{
func NewTestFrostFS() *TestFrostFS {
return &TestFrostFS{
objects: make(map[string]*object.Object),
containers: make(map[string]*container.Container),
eaclTables: make(map[string]*eacl.Table),
}
}
func (t *TestNeoFS) CurrentEpoch() uint64 {
func (t *TestFrostFS) CurrentEpoch() uint64 {
return t.currentEpoch
}
func (t *TestNeoFS) Objects() []*object.Object {
func (t *TestFrostFS) Objects() []*object.Object {
res := make([]*object.Object, 0, len(t.objects))
for _, obj := range t.objects {
@ -55,11 +55,11 @@ func (t *TestNeoFS) Objects() []*object.Object {
return res
}
func (t *TestNeoFS) AddObject(key string, obj *object.Object) {
func (t *TestFrostFS) AddObject(key string, obj *object.Object) {
t.objects[key] = obj
}
func (t *TestNeoFS) ContainerID(name string) (cid.ID, error) {
func (t *TestFrostFS) ContainerID(name string) (cid.ID, error) {
for id, cnr := range t.containers {
if container.Name(*cnr) == name {
var cnrID cid.ID
@ -69,13 +69,18 @@ func (t *TestNeoFS) ContainerID(name string) (cid.ID, error) {
return cid.ID{}, fmt.Errorf("not found")
}
func (t *TestNeoFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (cid.ID, error) {
func (t *TestFrostFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (cid.ID, error) {
var cnr container.Container
cnr.Init()
cnr.SetOwner(prm.Creator)
cnr.SetPlacementPolicy(prm.Policy)
cnr.SetBasicACL(prm.BasicACL)
container.SetCreationTime(&cnr, time.Now())
creationTime := prm.CreationTime
if creationTime.IsZero() {
creationTime = time.Now()
}
container.SetCreationTime(&cnr, creationTime)
if prm.Name != "" {
var d container.Domain
@ -101,13 +106,13 @@ func (t *TestNeoFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (
return id, nil
}
func (t *TestNeoFS) DeleteContainer(_ context.Context, cnrID cid.ID, _ *session.Container) error {
func (t *TestFrostFS) DeleteContainer(_ context.Context, cnrID cid.ID, _ *session.Container) error {
delete(t.containers, cnrID.EncodeToString())
return nil
}
func (t *TestNeoFS) Container(_ context.Context, id cid.ID) (*container.Container, error) {
func (t *TestFrostFS) Container(_ context.Context, id cid.ID) (*container.Container, error) {
for k, v := range t.containers {
if k == id.EncodeToString() {
return v, nil
@ -117,7 +122,7 @@ func (t *TestNeoFS) Container(_ context.Context, id cid.ID) (*container.Containe
return nil, fmt.Errorf("container not found %s", id)
}
func (t *TestNeoFS) UserContainers(_ context.Context, _ user.ID) ([]cid.ID, error) {
func (t *TestFrostFS) UserContainers(_ context.Context, _ user.ID) ([]cid.ID, error) {
var res []cid.ID
for k := range t.containers {
var idCnr cid.ID
@ -130,7 +135,7 @@ func (t *TestNeoFS) UserContainers(_ context.Context, _ user.ID) ([]cid.ID, erro
return res, nil
}
func (t *TestNeoFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectPart, error) {
func (t *TestFrostFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectPart, error) {
var addr oid.Address
addr.SetContainer(prm.Container)
addr.SetObject(prm.Object)
@ -139,7 +144,7 @@ func (t *TestNeoFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectP
if obj, ok := t.objects[sAddr]; ok {
owner := getOwner(ctx)
if !obj.OwnerID().Equals(owner) {
if !obj.OwnerID().Equals(owner) && !t.isPublicRead(prm.Container) {
return nil, ErrAccessDenied
}
@ -159,7 +164,7 @@ func (t *TestNeoFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectP
return nil, fmt.Errorf("object not found %s", addr)
}
func (t *TestNeoFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oid.ID, error) {
func (t *TestFrostFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oid.ID, error) {
b := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, b); err != nil {
return oid.ID{}, err
@ -218,7 +223,7 @@ func (t *TestNeoFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oid.
return objID, nil
}
func (t *TestNeoFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
func (t *TestFrostFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
var addr oid.Address
addr.SetContainer(prm.Container)
addr.SetObject(prm.Object)
@ -235,11 +240,11 @@ func (t *TestNeoFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error
return nil
}
func (t *TestNeoFS) TimeToEpoch(_ context.Context, futureTime time.Time) (uint64, uint64, error) {
return t.currentEpoch, t.currentEpoch + uint64(futureTime.Second()), nil
func (t *TestFrostFS) TimeToEpoch(_ context.Context, now, futureTime time.Time) (uint64, uint64, error) {
return t.currentEpoch, t.currentEpoch + uint64(futureTime.Sub(now).Seconds()), nil
}
func (t *TestNeoFS) AllObjects(cnrID cid.ID) []oid.ID {
func (t *TestFrostFS) AllObjects(cnrID cid.ID) []oid.ID {
result := make([]oid.ID, 0, len(t.objects))
for _, val := range t.objects {
@ -253,7 +258,7 @@ func (t *TestNeoFS) AllObjects(cnrID cid.ID) []oid.ID {
return result
}
func (t *TestNeoFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *session.Container) error {
func (t *TestFrostFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *session.Container) error {
cnrID, ok := table.CID()
if !ok {
return errors.New("invalid cid")
@ -268,7 +273,7 @@ func (t *TestNeoFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *ses
return nil
}
func (t *TestNeoFS) ContainerEACL(_ context.Context, cnrID cid.ID) (*eacl.Table, error) {
func (t *TestFrostFS) ContainerEACL(_ context.Context, cnrID cid.ID) (*eacl.Table, error) {
table, ok := t.eaclTables[cnrID.EncodeToString()]
if !ok {
return nil, errors.New("not found")
@ -277,6 +282,25 @@ func (t *TestNeoFS) ContainerEACL(_ context.Context, cnrID cid.ID) (*eacl.Table,
return table, nil
}
func (t *TestFrostFS) isPublicRead(cnrID cid.ID) bool {
table, ok := t.eaclTables[cnrID.EncodeToString()]
if !ok {
return false
}
for _, rec := range table.Records() {
if rec.Operation() == eacl.OperationGet && len(rec.Filters()) == 0 {
for _, trgt := range rec.Targets() {
if trgt.Role() == eacl.RoleOthers {
return rec.Action() == eacl.ActionAllow
}
}
}
}
return false
}
func getOwner(ctx context.Context) user.ID {
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
return bearer.ResolveIssuer(*bd.Gate.BearerToken)

View file

@ -7,8 +7,8 @@ import (
errorsStd "errors"
"fmt"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"go.uber.org/zap"
)
@ -30,6 +30,7 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
Creator: p.BktInfo.Owner,
Payload: bytes.NewReader(confXML),
Filepath: p.BktInfo.NotificationConfigurationObjectName(),
CreationTime: TimeNow(ctx),
CopiesNumber: p.CopiesNumber,
}

View file

@ -13,17 +13,16 @@ import (
"strconv"
"strings"
"sync"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/minio/sio"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-sdk-go/client"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/panjf2000/ants/v2"
"go.uber.org/zap"
)
@ -91,7 +90,7 @@ func (n *layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
res, err := n.neoFS.ReadObject(ctx, prm)
res, err := n.frostFS.ReadObject(ctx, prm)
if err != nil {
return nil, err
}
@ -99,7 +98,7 @@ func (n *layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj
return res.Head, nil
}
// initializes payload reader of the NeoFS object.
// initializes payload reader of the FrostFS object.
// Zero range corresponds to full payload (panics if only offset is set).
func (n *layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Reader, error) {
prm := PrmObjectRead{
@ -111,7 +110,7 @@ func (n *layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Re
n.prepareAuthParameters(ctx, &prm.PrmAuth, p.bktInfo.Owner)
res, err := n.neoFS.ReadObject(ctx, prm)
res, err := n.frostFS.ReadObject(ctx, prm)
if err != nil {
return nil, err
}
@ -130,7 +129,7 @@ func (n *layer) objectGet(ctx context.Context, bktInfo *data.BucketInfo, objID o
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
res, err := n.neoFS.ReadObject(ctx, prm)
res, err := n.frostFS.ReadObject(ctx, prm)
if err != nil {
return nil, err
}
@ -183,7 +182,7 @@ func ParseCompletedPartHeader(hdr string) (*Part, error) {
}, nil
}
// PutObject stores object into NeoFS, took payload from io.Reader.
// PutObject stores object into FrostFS, took payload from io.Reader.
func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error) {
owner := n.Owner(ctx)
@ -234,6 +233,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
PayloadSize: uint64(p.Size),
Filepath: p.Object,
Payload: r,
CreationTime: TimeNow(ctx),
CopiesNumber: p.CopiesNumber,
}
@ -248,6 +248,12 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
return nil, err
}
reqInfo := api.GetReqInfo(ctx)
n.log.Debug("put object",
zap.String("reqId", reqInfo.RequestID),
zap.String("bucket", p.BktInfo.Name), zap.Stringer("cid", p.BktInfo.CID),
zap.String("object", p.Object), zap.Stringer("oid", id))
newVersion.OID = id
newVersion.ETag = hex.EncodeToString(hash)
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
@ -281,7 +287,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
Bucket: p.BktInfo.Name,
Name: p.Object,
Size: p.Size,
Created: time.Now(),
Created: prm.CreationTime,
Headers: p.Header,
ContentType: p.Header[api.ContentType],
HashSum: newVersion.ETag,
@ -383,7 +389,7 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
return extObjInfo, nil
}
// objectDelete puts tombstone object into neofs.
// objectDelete puts tombstone object into frostfs.
func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) error {
prm := PrmObjectDelete{
Container: bktInfo.CID,
@ -394,10 +400,10 @@ func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idOb
n.cache.DeleteObject(newAddress(bktInfo.CID, idObj))
return n.neoFS.DeleteObject(ctx, prm)
return n.frostFS.DeleteObject(ctx, prm)
}
// objectPutAndHash prepare auth parameters and invoke neofs.CreateObject.
// objectPutAndHash prepare auth parameters and invoke frostfs.CreateObject.
// Returns object ID and payload sha256 hash.
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (oid.ID, []byte, error) {
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
@ -405,8 +411,11 @@ func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktIn
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
hash.Write(buf)
})
id, err := n.neoFS.CreateObject(ctx, prm)
return id, hash.Sum(nil), err
id, err := n.frostFS.CreateObject(ctx, prm)
if err != nil {
return oid.ID{}, nil, err
}
return id, hash.Sum(nil), nil
}
// ListObjectsV1 returns objects in a bucket for requests of Version 1.
@ -574,10 +583,10 @@ func (n *layer) initWorkerPool(ctx context.Context, size int, p allObjectParams,
wg.Add(1)
err = pool.Submit(func() {
defer wg.Done()
oi := n.objectInfoFromObjectsCacheOrNeoFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter)
oi := n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter)
if oi == nil {
// try to get object again
if oi = n.objectInfoFromObjectsCacheOrNeoFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter); oi == nil {
if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter); oi == nil {
// form object info with data that the tree node contains
oi = getPartialObjectInfo(p.Bucket, node)
}
@ -643,14 +652,14 @@ func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo,
for _, nodeVersion := range nodeVersions {
oi := &data.ObjectInfo{}
if nodeVersion.IsDeleteMarker() { // delete marker does not match any object in NeoFS
if nodeVersion.IsDeleteMarker() { // delete marker does not match any object in FrostFS
oi.ID = nodeVersion.OID
oi.Name = nodeVersion.FilePath
oi.Owner = nodeVersion.DeleteMarker.Owner
oi.Created = nodeVersion.DeleteMarker.Created
oi.IsDeleteMarker = true
} else {
if oi = n.objectInfoFromObjectsCacheOrNeoFS(ctx, bkt, nodeVersion, prefix, delimiter); oi == nil {
if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, bkt, nodeVersion, prefix, delimiter); oi == nil {
continue
}
}
@ -674,7 +683,7 @@ func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo,
func IsSystemHeader(key string) bool {
_, ok := api.SystemMetadata[key]
return ok || strings.HasPrefix(key, api.NeoFSSystemMetadataPrefix)
return ok || strings.HasPrefix(key, api.FrostFSSystemMetadataPrefix)
}
func shouldSkip(node *data.NodeVersion, p allObjectParams, existed map[string]struct{}) bool {
@ -731,7 +740,7 @@ func triageExtendedObjects(allObjects []*data.ExtendedObjectInfo) (prefixes []st
return
}
func (n *layer) objectInfoFromObjectsCacheOrNeoFS(ctx context.Context, bktInfo *data.BucketInfo, node *data.NodeVersion, prefix, delimiter string) (oi *data.ObjectInfo) {
func (n *layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo *data.BucketInfo, node *data.NodeVersion, prefix, delimiter string) (oi *data.ObjectInfo) {
if oiDir := tryDirectory(bktInfo, node, prefix, delimiter); oiDir != nil {
return oiDir
}

View file

@ -9,14 +9,14 @@ import (
"strconv"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
const (
AttributeComplianceMode = ".s3-compliance-mode"
AttributeExpirationEpoch = "__NEOFS__EXPIRATION_EPOCH"
AttributeComplianceMode = ".s3-compliance-mode"
)
type PutLockInfoParams struct {
@ -32,7 +32,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
// sometimes node version can be provided from executing context
// if not, then receive node version from tree service
if versionNode == nil {
versionNode, err = n.getNodeVersionFromCacheOrNeofs(ctx, p.ObjVersion)
versionNode, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjVersion)
if err != nil {
return err
}
@ -100,7 +100,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
return nil
}
func (n *layer) getNodeVersionFromCacheOrNeofs(ctx context.Context, objVersion *ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
func (n *layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion *ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
// check cache if node version is stored inside extendedObjectVersion
nodeVersion = n.getNodeVersionFromCache(n.Owner(ctx), objVersion)
if nodeVersion == nil {
@ -116,6 +116,7 @@ func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
Container: bktInfo.CID,
Creator: bktInfo.Owner,
Locks: []oid.ID{objID},
CreationTime: TimeNow(ctx),
CopiesNumber: copiesNumber,
}
@ -227,7 +228,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
)
if lock.Retention != nil {
if _, expEpoch, err = n.neoFS.TimeToEpoch(ctx, lock.Retention.Until); err != nil {
if _, expEpoch, err = n.frostFS.TimeToEpoch(ctx, TimeNow(ctx), lock.Retention.Until); err != nil {
return nil, fmt.Errorf("fetch time to epoch: %w", err)
}
@ -237,7 +238,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
}
if lock.LegalHold != nil && lock.LegalHold.Enabled {
// todo: (@KirillovDenis) reconsider this when NeoFS will support Legal Hold https://github.com/nspcc-dev/neofs-contract/issues/247
// todo: (@KirillovDenis) reconsider this when FrostFS will support Legal Hold https://git.frostfs.info/TrueCloudLab/frostfs-contract/issues/2
// Currently lock object must have an expiration epoch.
// Besides we need to override retention expiration epoch since legal hold cannot be deleted yet.
expEpoch = math.MaxUint64
@ -245,7 +246,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
if expEpoch != 0 {
result = append(result, [2]string{
AttributeExpirationEpoch, strconv.FormatUint(expEpoch, 10),
object.SysAttributeExpEpoch, strconv.FormatUint(expEpoch, 10),
})
}

View file

@ -4,11 +4,13 @@ import (
"context"
errorsStd "errors"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"go.uber.org/zap"
)
type GetObjectTaggingParams struct {
@ -38,7 +40,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
nodeVersion := p.NodeVersion
if nodeVersion == nil {
nodeVersion, err = n.getNodeVersionFromCacheOrNeofs(ctx, p.ObjectVersion)
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
if err != nil {
return "", nil, err
}
@ -65,7 +67,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (nodeVersion *data.NodeVersion, err error) {
nodeVersion = p.NodeVersion
if nodeVersion == nil {
nodeVersion, err = n.getNodeVersionFromCacheOrNeofs(ctx, p.ObjectVersion)
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
if err != nil {
return nil, err
}
@ -175,6 +177,14 @@ func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (
return nil, errors.GetAPIError(errors.ErrNoSuchKey)
}
if err == nil && version != nil && !version.IsDeleteMarker() {
reqInfo := api.GetReqInfo(ctx)
n.log.Debug("target details",
zap.String("reqId", reqInfo.RequestID),
zap.String("bucket", objVersion.BktInfo.Name), zap.Stringer("cid", objVersion.BktInfo.CID),
zap.String("object", objVersion.ObjectName), zap.Stringer("oid", version.OID))
}
return version, err
}

View file

@ -6,8 +6,8 @@ import (
"sort"
"strings"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
type TreeServiceMock struct {
@ -109,11 +109,32 @@ func (t *TreeServiceMock) PutNotificationConfigurationNode(ctx context.Context,
}
func (t *TreeServiceMock) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
panic("implement me")
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
if !ok {
return oid.ID{}, nil
}
node, ok := systemMap["cors"]
if !ok {
return oid.ID{}, nil
}
return node.OID, nil
}
func (t *TreeServiceMock) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) {
panic("implement me")
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
if !ok {
systemMap = make(map[string]*data.BaseNodeVersion)
}
systemMap["cors"] = &data.BaseNodeVersion{
OID: objID,
}
t.system[bktInfo.CID.EncodeToString()] = systemMap
return oid.ID{}, ErrNoNodeToRemove
}
func (t *TreeServiceMock) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {

View file

@ -4,8 +4,8 @@ import (
"context"
"errors"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
// TreeService provide interface to interact with tree service using s3 data models.
@ -24,7 +24,7 @@ type TreeService interface {
GetNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
// PutNotificationConfigurationNode puts a node to a system tree
// and returns objectID of a previous notif config which must be deleted in NeoFS.
// and returns objectID of a previous notif config which must be deleted in FrostFS.
//
// If object id to remove is not found returns ErrNoNodeToRemove error.
PutNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
@ -34,12 +34,12 @@ type TreeService interface {
// If object id is not found returns ErrNodeNotFound error.
GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in NeoFS.
// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in FrostFS.
//
// If object id to remove is not found returns ErrNoNodeToRemove error.
PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in NeoFS.
// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in FrostFS.
//
// If object id to remove is not found returns ErrNoNodeToRemove error.
DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
@ -69,7 +69,7 @@ type TreeService interface {
GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error)
// AddPart puts a node to a system tree as a child of appropriate multipart upload
// and returns objectID of a previous part which must be deleted in NeoFS.
// and returns objectID of a previous part which must be deleted in FrostFS.
//
// If object id to remove is not found returns ErrNoNodeToRemove error.
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)

View file

@ -9,11 +9,11 @@ import (
"strings"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/object"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
)
type (

View file

@ -6,11 +6,11 @@ import (
"testing"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-sdk-go/checksum"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/stretchr/testify/require"
)

View file

@ -4,7 +4,7 @@ import (
"context"
"sort"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
)
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {

View file

@ -5,14 +5,14 @@ import (
"context"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
bearertest "github.com/nspcc-dev/neofs-sdk-go/bearer/test"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
@ -113,7 +113,7 @@ func (tc *testContext) checkListObjects(ids ...oid.ID) {
}
func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
for _, obj := range tc.testNeoFS.Objects() {
for _, obj := range tc.testFrostFS.Objects() {
id, _ := obj.ID()
if id.Equals(objID) {
return obj
@ -123,12 +123,12 @@ func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
}
type testContext struct {
t *testing.T
ctx context.Context
layer Client
bktInfo *data.BucketInfo
obj string
testNeoFS *TestNeoFS
t *testing.T
ctx context.Context
layer Client
bktInfo *data.BucketInfo
obj string
testFrostFS *TestFrostFS
}
func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
@ -146,7 +146,7 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
GateKey: key.PublicKey(),
},
})
tp := NewTestNeoFS()
tp := NewTestFrostFS()
bktName := "testbucket1"
bktID, err := tp.CreateContainer(ctx, PrmContainerCreate{
@ -176,9 +176,9 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
Owner: owner,
CID: bktID,
},
obj: "obj1",
t: t,
testNeoFS: tp,
obj: "obj1",
t: t,
testFrostFS: tp,
}
}

View file

@ -4,7 +4,7 @@ import (
"net/http"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
)
type (

305
api/metrics.go Normal file
View file

@ -0,0 +1,305 @@
package api
import (
"context"
"io"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"github.com/prometheus/client_golang/prometheus"
)
type RequestType int
const (
UNKNOWNRequest RequestType = iota
HEADRequest RequestType = iota
PUTRequest RequestType = iota
LISTRequest RequestType = iota
GETRequest RequestType = iota
DELETERequest RequestType = iota
)
func (t RequestType) String() string {
switch t {
case 1:
return "HEAD"
case 2:
return "PUT"
case 3:
return "LIST"
case 4:
return "GET"
case 5:
return "DELETE"
default:
return "Unknown"
}
}
func RequestTypeFromAPI(api string) RequestType {
switch api {
case "Options", "HeadObject", "HeadBucket":
return HEADRequest
case "CreateMultipartUpload", "UploadPartCopy", "UploadPart", "CompleteMultipartUpload",
"PutObjectACL", "PutObjectTagging", "CopyObject", "PutObjectRetention", "PutObjectLegalHold",
"PutObject", "PutBucketCors", "PutBucketACL", "PutBucketLifecycle", "PutBucketEncryption",
"PutBucketPolicy", "PutBucketObjectLockConfig", "PutBucketTagging", "PutBucketVersioning",
"PutBucketNotification", "CreateBucket", "PostObject":
return PUTRequest
case "ListObjectParts", "ListMultipartUploads", "ListObjectsV2M", "ListObjectsV2", "ListBucketVersions",
"ListObjectsV1", "ListBuckets":
return LISTRequest
case "GetObjectACL", "GetObjectTagging", "SelectObjectContent", "GetObjectRetention", "getobjectlegalhold",
"GetObjectAttributes", "GetObject", "GetBucketLocation", "GetBucketPolicy",
"GetBucketLifecycle", "GetBucketEncryption", "GetBucketCors", "GetBucketACL",
"GetBucketWebsite", "GetBucketAccelerate", "GetBucketRequestPayment", "GetBucketLogging",
"GetBucketReplication", "GetBucketTagging", "GetBucketObjectLockConfig",
"GetBucketVersioning", "GetBucketNotification", "ListenBucketNotification":
return GETRequest
case "AbortMultipartUpload", "DeleteObjectTagging", "DeleteObject", "DeleteBucketCors",
"DeleteBucketWebsite", "DeleteBucketTagging", "DeleteMultipleObjects", "DeleteBucketPolicy",
"DeleteBucketLifecycle", "DeleteBucketEncryption", "DeleteBucket":
return DELETERequest
default:
return UNKNOWNRequest
}
}
type (
// HTTPAPIStats holds statistics information about
// the API given in the requests.
HTTPAPIStats struct {
apiStats map[string]int
sync.RWMutex
}
UsersStat interface {
Update(user, bucket, cnrID string, reqType RequestType, in, out uint64)
}
// HTTPStats holds statistics information about
// HTTP requests made by all clients.
HTTPStats struct {
currentS3Requests HTTPAPIStats
totalS3Requests HTTPAPIStats
totalS3Errors HTTPAPIStats
totalInputBytes uint64
totalOutputBytes uint64
}
readCounter struct {
io.ReadCloser
countBytes uint64
}
writeCounter struct {
http.ResponseWriter
countBytes uint64
}
responseWrapper struct {
sync.Once
http.ResponseWriter
statusCode int
startTime time.Time
}
)
const systemPath = "/system"
var (
httpStatsMetric = new(HTTPStats)
httpRequestsDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "frostfs_s3_request_seconds",
Help: "Time taken by requests served by current FrostFS S3 Gate instance",
Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10},
},
[]string{"api"},
)
)
// Collects HTTP metrics for FrostFS S3 Gate in Prometheus specific format
// and sends to the given channel.
func collectHTTPMetrics(ch chan<- prometheus.Metric) {
for api, value := range httpStatsMetric.currentS3Requests.Load() {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("frostfs_s3", "requests", "current"),
"Total number of running s3 requests in current FrostFS S3 Gate instance",
[]string{"api"}, nil),
prometheus.CounterValue,
float64(value),
api,
)
}
for api, value := range httpStatsMetric.totalS3Requests.Load() {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("frostfs_s3", "requests", "total"),
"Total number of s3 requests in current FrostFS S3 Gate instance",
[]string{"api"}, nil),
prometheus.CounterValue,
float64(value),
api,
)
}
for api, value := range httpStatsMetric.totalS3Errors.Load() {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("frostfs_s3", "errors", "total"),
"Total number of s3 errors in current FrostFS S3 Gate instance",
[]string{"api"}, nil),
prometheus.CounterValue,
float64(value),
api,
)
}
}
// CIDResolveFunc is a func to resolve CID in Stats handler.
type CIDResolveFunc func(ctx context.Context, reqInfo *ReqInfo) (cnrID string)
// Stats is a handler that update metrics.
func Stats(f http.HandlerFunc, resolveCID CIDResolveFunc, usersStat UsersStat) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
reqInfo := GetReqInfo(r.Context())
httpStatsMetric.currentS3Requests.Inc(reqInfo.API)
defer httpStatsMetric.currentS3Requests.Dec(reqInfo.API)
in := &readCounter{ReadCloser: r.Body}
out := &writeCounter{ResponseWriter: w}
r.Body = in
statsWriter := &responseWrapper{
ResponseWriter: out,
startTime: time.Now(),
}
f(statsWriter, r)
// Time duration in secs since the call started.
// We don't need to do nanosecond precision here
// simply for the fact that it is not human-readable.
durationSecs := time.Since(statsWriter.startTime).Seconds()
user := resolveUser(r.Context())
cnrID := resolveCID(r.Context(), reqInfo)
usersStat.Update(user, reqInfo.BucketName, cnrID, RequestTypeFromAPI(reqInfo.API), in.countBytes, out.countBytes)
code := statsWriter.statusCode
// A successful request has a 2xx response code
successReq := code >= http.StatusOK && code < http.StatusMultipleChoices
if !strings.HasSuffix(r.URL.Path, systemPath) {
httpStatsMetric.totalS3Requests.Inc(reqInfo.API)
if !successReq && code != 0 {
httpStatsMetric.totalS3Errors.Inc(reqInfo.API)
}
}
if r.Method == http.MethodGet {
// Increment the prometheus http request response histogram with appropriate label
httpRequestsDuration.With(prometheus.Labels{"api": reqInfo.API}).Observe(durationSecs)
}
atomic.AddUint64(&httpStatsMetric.totalInputBytes, in.countBytes)
atomic.AddUint64(&httpStatsMetric.totalOutputBytes, out.countBytes)
}
}
func resolveUser(ctx context.Context) string {
user := "anon"
if bd, ok := ctx.Value(BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
user = bearer.ResolveIssuer(*bd.Gate.BearerToken).String()
}
return user
}
// Inc increments the api stats counter.
func (stats *HTTPAPIStats) Inc(api string) {
if stats == nil {
return
}
stats.Lock()
defer stats.Unlock()
if stats.apiStats == nil {
stats.apiStats = make(map[string]int)
}
stats.apiStats[api]++
}
// Dec increments the api stats counter.
func (stats *HTTPAPIStats) Dec(api string) {
if stats == nil {
return
}
stats.Lock()
defer stats.Unlock()
if val, ok := stats.apiStats[api]; ok && val > 0 {
stats.apiStats[api]--
}
}
// Load returns the recorded stats.
func (stats *HTTPAPIStats) Load() map[string]int {
stats.Lock()
defer stats.Unlock()
var apiStats = make(map[string]int, len(stats.apiStats))
for k, v := range stats.apiStats {
apiStats[k] = v
}
return apiStats
}
func (st *HTTPStats) getInputBytes() uint64 {
return atomic.LoadUint64(&st.totalInputBytes)
}
func (st *HTTPStats) getOutputBytes() uint64 {
return atomic.LoadUint64(&st.totalOutputBytes)
}
// WriteHeader -- writes http status code.
func (w *responseWrapper) WriteHeader(code int) {
w.Do(func() {
w.statusCode = code
w.ResponseWriter.WriteHeader(code)
})
}
// Flush -- calls the underlying Flush.
func (w *responseWrapper) Flush() {
if f, ok := w.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func (w *writeCounter) Flush() {
if f, ok := w.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func (w *writeCounter) Write(p []byte) (int, error) {
n, err := w.ResponseWriter.Write(p)
atomic.AddUint64(&w.countBytes, uint64(n))
return n, err
}
func (r *readCounter) Read(p []byte) (int, error) {
n, err := r.ReadCloser.Read(p)
atomic.AddUint64(&r.countBytes, uint64(n))
return n, err
}

View file

@ -1,229 +0,0 @@
package metrics
import (
"io"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/prometheus/client_golang/prometheus"
)
type (
// HTTPAPIStats holds statistics information about
// the API given in the requests.
HTTPAPIStats struct {
apiStats map[string]int
sync.RWMutex
}
// HTTPStats holds statistics information about
// HTTP requests made by all clients.
HTTPStats struct {
currentS3Requests HTTPAPIStats
totalS3Requests HTTPAPIStats
totalS3Errors HTTPAPIStats
totalInputBytes uint64
totalOutputBytes uint64
}
readCounter struct {
io.ReadCloser
countBytes uint64
}
writeCounter struct {
http.ResponseWriter
countBytes uint64
}
responseWrapper struct {
sync.Once
http.ResponseWriter
statusCode int
startTime time.Time
}
)
const systemPath = "/system"
var (
httpStatsMetric = new(HTTPStats)
httpRequestsDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "neofs_s3_request_seconds",
Help: "Time taken by requests served by current NeoFS S3 Gate instance",
Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10},
},
[]string{"api"},
)
)
// Collects HTTP metrics for NeoFS S3 Gate in Prometheus specific format
// and sends to the given channel.
func collectHTTPMetrics(ch chan<- prometheus.Metric) {
for api, value := range httpStatsMetric.currentS3Requests.Load() {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("neofs_s3", "requests", "current"),
"Total number of running s3 requests in current NeoFS S3 Gate instance",
[]string{"api"}, nil),
prometheus.CounterValue,
float64(value),
api,
)
}
for api, value := range httpStatsMetric.totalS3Requests.Load() {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("neofs_s3", "requests", "total"),
"Total number of s3 requests in current NeoFS S3 Gate instance",
[]string{"api"}, nil),
prometheus.CounterValue,
float64(value),
api,
)
}
for api, value := range httpStatsMetric.totalS3Errors.Load() {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("neofs_s3", "errors", "total"),
"Total number of s3 errors in current NeoFS S3 Gate instance",
[]string{"api"}, nil),
prometheus.CounterValue,
float64(value),
api,
)
}
}
// APIStats wraps http handler for api with basic statistics collection.
func APIStats(api string, f http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
httpStatsMetric.currentS3Requests.Inc(api)
defer httpStatsMetric.currentS3Requests.Dec(api)
in := &readCounter{ReadCloser: r.Body}
out := &writeCounter{ResponseWriter: w}
r.Body = in
statsWriter := &responseWrapper{
ResponseWriter: out,
startTime: time.Now(),
}
f.ServeHTTP(statsWriter, r)
// Time duration in secs since the call started.
// We don't need to do nanosecond precision here
// simply for the fact that it is not human readable.
durationSecs := time.Since(statsWriter.startTime).Seconds()
httpStatsMetric.updateStats(api, statsWriter, r, durationSecs)
atomic.AddUint64(&httpStatsMetric.totalInputBytes, in.countBytes)
atomic.AddUint64(&httpStatsMetric.totalOutputBytes, out.countBytes)
}
}
// Inc increments the api stats counter.
func (stats *HTTPAPIStats) Inc(api string) {
if stats == nil {
return
}
stats.Lock()
defer stats.Unlock()
if stats.apiStats == nil {
stats.apiStats = make(map[string]int)
}
stats.apiStats[api]++
}
// Dec increments the api stats counter.
func (stats *HTTPAPIStats) Dec(api string) {
if stats == nil {
return
}
stats.Lock()
defer stats.Unlock()
if val, ok := stats.apiStats[api]; ok && val > 0 {
stats.apiStats[api]--
}
}
// Load returns the recorded stats.
func (stats *HTTPAPIStats) Load() map[string]int {
stats.Lock()
defer stats.Unlock()
var apiStats = make(map[string]int, len(stats.apiStats))
for k, v := range stats.apiStats {
apiStats[k] = v
}
return apiStats
}
func (st *HTTPStats) getInputBytes() uint64 {
return atomic.LoadUint64(&st.totalInputBytes)
}
func (st *HTTPStats) getOutputBytes() uint64 {
return atomic.LoadUint64(&st.totalOutputBytes)
}
// Update statistics from http request and response data.
func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) {
var code int
if res, ok := w.(*responseWrapper); ok {
code = res.statusCode
}
// A successful request has a 2xx response code
successReq := code >= http.StatusOK && code < http.StatusMultipleChoices
if !strings.HasSuffix(r.URL.Path, systemPath) {
st.totalS3Requests.Inc(api)
if !successReq && code != 0 {
st.totalS3Errors.Inc(api)
}
}
if r.Method == http.MethodGet {
// Increment the prometheus http request response histogram with appropriate label
httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(durationSecs)
}
}
// WriteHeader -- writes http status code.
func (w *responseWrapper) WriteHeader(code int) {
w.Do(func() {
w.statusCode = code
w.ResponseWriter.WriteHeader(code)
})
}
// Flush -- calls the underlying Flush.
func (w *responseWrapper) Flush() {
if f, ok := w.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func (w *writeCounter) Write(p []byte) (int, error) {
n, err := w.ResponseWriter.Write(p)
atomic.AddUint64(&w.countBytes, uint64(n))
return n, err
}
func (r *readCounter) Read(p []byte) (int, error) {
n, err := r.ReadCloser.Read(p)
atomic.AddUint64(&r.countBytes, uint64(n))
return n, err
}

View file

@ -1,7 +1,7 @@
package metrics
package api
import (
"github.com/nspcc-dev/neofs-s3-gw/internal/version"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
"github.com/prometheus/client_golang/prometheus"
)
@ -12,9 +12,9 @@ type stats struct {
var (
versionInfo = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: "neofs_s3",
Namespace: "frostfs_s3",
Name: "version_info",
Help: "Version of current NeoFS S3 Gate instance",
Help: "Version of current FrostFS S3 Gate instance",
},
[]string{
// current version
@ -23,7 +23,7 @@ var (
)
statsMetrics = &stats{
desc: prometheus.NewDesc("neofs_s3_stats", "Statistics exposed by NeoFS S3 Gate instance", nil, nil),
desc: prometheus.NewDesc("frostfs_s3_stats", "Statistics exposed by FrostFS S3 Gate instance", nil, nil),
}
)
@ -37,8 +37,8 @@ func collectNetworkMetrics(ch chan<- prometheus.Metric) {
// Network Sent/Received Bytes (Outbound)
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("neofs_s3", "tx", "bytes_total"),
"Total number of bytes sent by current NeoFS S3 Gate instance",
prometheus.BuildFQName("frostfs_s3", "tx", "bytes_total"),
"Total number of bytes sent by current FrostFS S3 Gate instance",
nil, nil),
prometheus.CounterValue,
float64(httpStatsMetric.getInputBytes()),
@ -46,8 +46,8 @@ func collectNetworkMetrics(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("neofs_s3", "rx", "bytes_total"),
"Total number of bytes received by current NeoFS S3 Gate instance",
prometheus.BuildFQName("frostfs_s3", "rx", "bytes_total"),
"Total number of bytes received by current FrostFS S3 Gate instance",
nil, nil),
prometheus.CounterValue,
float64(httpStatsMetric.getOutputBytes()),

View file

@ -7,9 +7,9 @@ import (
"sync"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/nats-io/nats.go"
"github.com/nspcc-dev/neofs-s3-gw/api/handler"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"go.uber.org/zap"
)
@ -61,7 +61,7 @@ type (
EventRecord struct {
EventVersion string `json:"eventVersion"`
EventSource string `json:"eventSource"` // neofs:s3
EventSource string `json:"eventSource"` // frostfs:s3
AWSRegion string `json:"awsRegion,omitempty"` // empty
EventTime time.Time `json:"eventTime"`
EventName string `json:"eventName"`
@ -197,11 +197,11 @@ func (c *Controller) SendNotifications(topics map[string]string, p *handler.Send
return nil
}
func (c *Controller) SendTestNotification(topic, bucketName, requestID, HostID string) error {
func (c *Controller) SendTestNotification(topic, bucketName, requestID, HostID string, now time.Time) error {
event := &TestEvent{
Service: "NeoFS S3",
Service: "FrostFS S3",
Event: "s3:TestEvent",
Time: time.Now(),
Time: now,
Bucket: bucketName,
RequestID: requestID,
HostID: HostID,
@ -220,9 +220,9 @@ func prepareEvent(p *handler.SendNotificationParams) *Event {
Records: []EventRecord{
{
EventVersion: EventVersion21,
EventSource: "neofs:s3",
EventSource: "frostfs:s3",
AWSRegion: "",
EventTime: time.Now(),
EventTime: p.Time,
EventName: p.Event,
UserIdentity: UserIdentity{
PrincipalID: p.User,

View file

@ -45,7 +45,7 @@ type (
// Key used for Get/SetReqInfo.
type contextKeyType string
const ctxRequestInfo = contextKeyType("NeoFS-S3-GW")
const ctxRequestInfo = contextKeyType("FrostFS-S3-GW")
var (
// De-facto standard header keys.

View file

@ -6,8 +6,9 @@ import (
"fmt"
"sync"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/ns"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
)
const (
@ -18,9 +19,9 @@ const (
// ErrNoResolvers returns when trying to resolve container without any resolver.
var ErrNoResolvers = errors.New("no resolvers")
// NeoFS represents virtual connection to the NeoFS network.
type NeoFS interface {
// SystemDNS reads system DNS network parameters of the NeoFS.
// FrostFS represents virtual connection to the FrostFS network.
type FrostFS interface {
// SystemDNS reads system DNS network parameters of the FrostFS.
//
// It returns exactly on non-zero value. It returns any error encountered
// which prevented the parameter from being read.
@ -28,7 +29,7 @@ type NeoFS interface {
}
type Config struct {
NeoFS NeoFS
FrostFS FrostFS
RPCAddress string
}
@ -133,7 +134,7 @@ func (r *BucketResolver) equals(resolverNames []string) bool {
func newResolver(name string, cfg *Config) (*Resolver, error) {
switch name {
case DNSResolver:
return NewDNSResolver(cfg.NeoFS)
return NewDNSResolver(cfg.FrostFS)
case NNSResolver:
return NewNNSResolver(cfg.RPCAddress)
default:
@ -141,17 +142,17 @@ func newResolver(name string, cfg *Config) (*Resolver, error) {
}
}
func NewDNSResolver(neoFS NeoFS) (*Resolver, error) {
if neoFS == nil {
func NewDNSResolver(frostFS FrostFS) (*Resolver, error) {
if frostFS == nil {
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
}
var dns ns.DNS
resolveFunc := func(ctx context.Context, name string) (cid.ID, error) {
domain, err := neoFS.SystemDNS(ctx)
domain, err := frostFS.SystemDNS(ctx)
if err != nil {
return cid.ID{}, fmt.Errorf("read system DNS parameter of the NeoFS: %w", err)
return cid.ID{}, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err)
}
domain = name + "." + domain
@ -180,7 +181,10 @@ func NewNNSResolver(address string) (*Resolver, error) {
}
resolveFunc := func(_ context.Context, name string) (cid.ID, error) {
cnrID, err := nns.ResolveContainerName(name)
var d container.Domain
d.SetName(name)
cnrID, err := nns.ResolveContainerDomain(d)
if err != nil {
return cid.ID{}, fmt.Errorf("couldn't resolve container '%s': %w", name, err)
}

View file

@ -7,9 +7,9 @@ import (
"net/http"
"strconv"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
"github.com/google/uuid"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/internal/version"
)
type (
@ -117,7 +117,7 @@ func WriteErrorResponse(w http.ResponseWriter, reqInfo *ReqInfo, err error) int
code = e.HTTPStatusCode
switch e.Code {
case "SlowDown", "XNeoFSServerNotInitialized", "XNeoFSReadQuorum", "XNeoFSWriteQuorum":
case "SlowDown", "XFrostFSServerNotInitialized", "XFrostFSReadQuorum", "XFrostFSWriteQuorum":
// Set retry-after header to indicate user-agents to retry request after 120secs.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
w.Header().Set(hdrRetryAfter, "120")
@ -133,6 +133,13 @@ func WriteErrorResponse(w http.ResponseWriter, reqInfo *ReqInfo, err error) int
return code
}
// WriteErrorResponseNoHeader writes XML encoded error to the response body.
func WriteErrorResponseNoHeader(w http.ResponseWriter, reqInfo *ReqInfo, err error) {
errorResponse := getAPIErrorResponse(reqInfo, err)
encodedErrorResponse := EncodeResponse(errorResponse)
WriteResponseBody(w, encodedErrorResponse)
}
// If none of the http routes match respond with appropriate errors.
func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)
@ -172,6 +179,11 @@ func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType
return
}
WriteResponseBody(w, response)
}
// WriteResponseBody writes response into w.
func WriteResponseBody(w http.ResponseWriter, response []byte) {
_, _ = w.Write(response)
if flusher, ok := w.(http.Flusher); ok {
flusher.Flush()
@ -188,13 +200,30 @@ func EncodeResponse(response interface{}) []byte {
return bytesBuffer.Bytes()
}
// EncodeResponseNoHeader encodes response without setting xml.Header.
// Should be used with periodicXMLWriter which sends xml.Header to the client
// with whitespaces to keep connection alive.
func EncodeResponseNoHeader(response interface{}) []byte {
var bytesBuffer bytes.Buffer
_ = xml.NewEncoder(&bytesBuffer).Encode(response)
return bytesBuffer.Bytes()
}
// EncodeToResponse encodes the response into ResponseWriter.
func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
w.WriteHeader(http.StatusOK)
if _, err := w.Write(xmlHeader); err != nil {
return fmt.Errorf("write headers: %w", err)
} else if err = xml.NewEncoder(w).Encode(response); err != nil {
}
return EncodeToResponseNoHeader(w, response)
}
// EncodeToResponseNoHeader encodes the response into ResponseWriter without
// header status.
func EncodeToResponseNoHeader(w http.ResponseWriter, response interface{}) error {
if err := xml.NewEncoder(w).Encode(response); err != nil {
return fmt.Errorf("encode xml response: %w", err)
}

View file

@ -5,10 +5,10 @@ import (
"net/http"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/google/uuid"
"github.com/gorilla/mux"
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
"github.com/nspcc-dev/neofs-s3-gw/api/metrics"
"go.uber.org/zap"
"google.golang.org/grpc/metadata"
)
@ -82,6 +82,8 @@ type (
AbortMultipartUploadHandler(http.ResponseWriter, *http.Request)
ListPartsHandler(w http.ResponseWriter, r *http.Request)
ListMultipartUploadsHandler(http.ResponseWriter, *http.Request)
ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error)
}
// mimeType represents various MIME types used in API responses.
@ -106,7 +108,7 @@ const (
MimeXML mimeType = "application/xml"
)
var _ = logErrorResponse
var _ = logSuccessResponse
func (lrw *logResponseWriter) WriteHeader(code int) {
lrw.Do(func() {
@ -115,6 +117,12 @@ func (lrw *logResponseWriter) WriteHeader(code int) {
})
}
func (lrw *logResponseWriter) Flush() {
if f, ok := lrw.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func setRequestID(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// generate random UUIDv4
@ -145,7 +153,37 @@ func appendCORS(handler Handler) mux.MiddlewareFunc {
}
}
func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc {
// BucketResolveFunc is a func to resolve bucket info by name.
type BucketResolveFunc func(ctx context.Context, bucket string) (*data.BucketInfo, error)
// metricsMiddleware wraps http handler for api with basic statistics collection.
func metricsMiddleware(log *zap.Logger, resolveBucket BucketResolveFunc, usersStat UsersStat) mux.MiddlewareFunc {
return func(h http.Handler) http.Handler {
return Stats(h.ServeHTTP, resolveCID(log, resolveBucket), usersStat)
}
}
// resolveCID forms CIDResolveFunc using BucketResolveFunc.
func resolveCID(log *zap.Logger, resolveBucket BucketResolveFunc) CIDResolveFunc {
return func(ctx context.Context, reqInfo *ReqInfo) (cnrID string) {
if reqInfo.BucketName == "" || reqInfo.API == "CreateBucket" || reqInfo.API == "" {
return ""
}
bktInfo, err := resolveBucket(ctx, reqInfo.BucketName)
if err != nil {
log.Debug("failed to resolve CID",
zap.String("request_id", reqInfo.RequestID), zap.String("method", reqInfo.API),
zap.String("bucket", reqInfo.BucketName), zap.String("object", reqInfo.ObjectName),
zap.Error(err))
return ""
}
return bktInfo.CID.EncodeToString()
}
}
func logSuccessResponse(l *zap.Logger) mux.MiddlewareFunc {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
lw := &logResponseWriter{ResponseWriter: w}
@ -161,6 +199,7 @@ func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc {
l.Info("call method",
zap.Int("status", lw.statusCode),
zap.String("host", r.Host),
zap.String("request_id", GetRequestID(r.Context())),
zap.String("method", mux.CurrentRoute(r).GetName()),
zap.String("bucket", reqInfo.BucketName),
@ -182,21 +221,49 @@ func GetRequestID(v interface{}) string {
}
}
func setErrorAPI(apiName string, h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := SetReqInfo(r.Context(), &ReqInfo{API: apiName})
h.ServeHTTP(w, r.WithContext(ctx))
})
}
// attachErrorHandler set NotFoundHandler and MethodNotAllowedHandler for mux.Router.
func attachErrorHandler(api *mux.Router, log *zap.Logger, h Handler, center auth.Center, usersStat UsersStat) {
middlewares := []mux.MiddlewareFunc{
AuthMiddleware(log, center),
metricsMiddleware(log, h.ResolveBucket, usersStat),
}
var errorHandler http.Handler = http.HandlerFunc(errorResponseHandler)
for i := len(middlewares) - 1; i >= 0; i-- {
errorHandler = middlewares[i](errorHandler)
}
// If none of the routes match, add default error handler routes
api.NotFoundHandler = setErrorAPI("NotFound", errorHandler)
api.MethodNotAllowedHandler = setErrorAPI("MethodNotAllowed", errorHandler)
}
// Attach adds S3 API handlers from h to r for domains with m client limit using
// center authentication and log logger.
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) {
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger, usersStat UsersStat) {
api := r.PathPrefix(SlashSeparator).Subrouter()
api.Use(
// -- prepare request
setRequestID,
// Attach user authentication for all S3 routes.
AuthMiddleware(log, center),
metricsMiddleware(log, h.ResolveBucket, usersStat),
// -- logging error requests
logErrorResponse(log),
logSuccessResponse(log),
)
// Attach user authentication for all S3 routes.
AttachUserAuth(api, center, log)
attachErrorHandler(api, log, h, center, usersStat)
buckets := make([]*mux.Router, 0, len(domains)+1)
buckets = append(buckets, api.PathPrefix("/{bucket}").Subrouter())
@ -212,277 +279,327 @@ func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center aut
// -- append CORS headers to a response for
appendCORS(h),
)
bucket.Methods(http.MethodOptions).HandlerFunc(m.Handle(metrics.APIStats("preflight", h.Preflight))).Name("Options")
bucket.Methods(http.MethodOptions).HandlerFunc(
m.Handle(h.Preflight)).
Name("Options")
bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("headobject", h.HeadObjectHandler))).Name("HeadObject")
m.Handle(h.HeadObjectHandler)).
Name("HeadObject")
// CopyObjectPart
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(metrics.APIStats("uploadpartcopy", h.UploadPartCopy))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(
m.Handle(h.UploadPartCopy)).
Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
Name("UploadPartCopy")
// PutObjectPart
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("uploadpart", h.UploadPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
m.Handle(h.UploadPartHandler)).
Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
Name("UploadPart")
// ListParts
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("listobjectparts", h.ListPartsHandler))).Queries("uploadId", "{uploadId:.*}").
m.Handle(h.ListPartsHandler)).
Queries("uploadId", "{uploadId:.*}").
Name("ListObjectParts")
// CompleteMultipartUpload
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("completemutipartupload", h.CompleteMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}").
m.Handle(h.CompleteMultipartUploadHandler)).
Queries("uploadId", "{uploadId:.*}").
Name("CompleteMultipartUpload")
// CreateMultipartUpload
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("createmultipartupload", h.CreateMultipartUploadHandler))).Queries("uploads", "").
m.Handle(h.CreateMultipartUploadHandler)).
Queries("uploads", "").
Name("CreateMultipartUpload")
// AbortMultipartUpload
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("abortmultipartupload", h.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}").
m.Handle(h.AbortMultipartUploadHandler)).
Queries("uploadId", "{uploadId:.*}").
Name("AbortMultipartUpload")
// ListMultipartUploads
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("listmultipartuploads", h.ListMultipartUploadsHandler))).Queries("uploads", "").
m.Handle(h.ListMultipartUploadsHandler)).
Queries("uploads", "").
Name("ListMultipartUploads")
// GetObjectACL -- this is a dummy call.
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("getobjectacl", h.GetObjectACLHandler))).Queries("acl", "").
m.Handle(h.GetObjectACLHandler)).
Queries("acl", "").
Name("GetObjectACL")
// PutObjectACL -- this is a dummy call.
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("putobjectacl", h.PutObjectACLHandler))).Queries("acl", "").
m.Handle(h.PutObjectACLHandler)).
Queries("acl", "").
Name("PutObjectACL")
// GetObjectTagging
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("getobjecttagging", h.GetObjectTaggingHandler))).Queries("tagging", "").
m.Handle(h.GetObjectTaggingHandler)).
Queries("tagging", "").
Name("GetObjectTagging")
// PutObjectTagging
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("putobjecttagging", h.PutObjectTaggingHandler))).Queries("tagging", "").
m.Handle(h.PutObjectTaggingHandler)).
Queries("tagging", "").
Name("PutObjectTagging")
// DeleteObjectTagging
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("deleteobjecttagging", h.DeleteObjectTaggingHandler))).Queries("tagging", "").
m.Handle(h.DeleteObjectTaggingHandler)).
Queries("tagging", "").
Name("DeleteObjectTagging")
// SelectObjectContent
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("selectobjectcontent", h.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2").
m.Handle(h.SelectObjectContentHandler)).
Queries("select", "").Queries("select-type", "2").
Name("SelectObjectContent")
// GetObjectRetention
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("getobjectretention", h.GetObjectRetentionHandler))).Queries("retention", "").
m.Handle(h.GetObjectRetentionHandler)).
Queries("retention", "").
Name("GetObjectRetention")
// GetObjectLegalHold
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("getobjectlegalhold", h.GetObjectLegalHoldHandler))).Queries("legal-hold", "").
m.Handle(h.GetObjectLegalHoldHandler)).
Queries("legal-hold", "").
Name("GetObjectLegalHold")
// GetObjectAttributes
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("getobjectattributes", h.GetObjectAttributesHandler))).Queries("attributes", "").
m.Handle(h.GetObjectAttributesHandler)).
Queries("attributes", "").
Name("GetObjectAttributes")
// GetObject
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("getobject", h.GetObjectHandler))).
m.Handle(h.GetObjectHandler)).
Name("GetObject")
// CopyObject
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(metrics.APIStats("copyobject", h.CopyObjectHandler))).
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(
m.Handle(h.CopyObjectHandler)).
Name("CopyObject")
// PutObjectRetention
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("putobjectretention", h.PutObjectRetentionHandler))).Queries("retention", "").
m.Handle(h.PutObjectRetentionHandler)).
Queries("retention", "").
Name("PutObjectRetention")
// PutObjectLegalHold
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("putobjectlegalhold", h.PutObjectLegalHoldHandler))).Queries("legal-hold", "").
m.Handle(h.PutObjectLegalHoldHandler)).
Queries("legal-hold", "").
Name("PutObjectLegalHold")
// PutObject
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("putobject", h.PutObjectHandler))).
m.Handle(h.PutObjectHandler)).
Name("PutObject")
// DeleteObject
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
m.Handle(metrics.APIStats("deleteobject", h.DeleteObjectHandler))).
m.Handle(h.DeleteObjectHandler)).
Name("DeleteObject")
// Bucket operations
// GetBucketLocation
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketlocation", h.GetBucketLocationHandler))).Queries("location", "").
m.Handle(h.GetBucketLocationHandler)).
Queries("location", "").
Name("GetBucketLocation")
// GetBucketPolicy
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketpolicy", h.GetBucketPolicyHandler))).Queries("policy", "").
m.Handle(h.GetBucketPolicyHandler)).
Queries("policy", "").
Name("GetBucketPolicy")
// GetBucketLifecycle
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", "").
m.Handle(h.GetBucketLifecycleHandler)).
Queries("lifecycle", "").
Name("GetBucketLifecycle")
// GetBucketEncryption
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketencryption", h.GetBucketEncryptionHandler))).Queries("encryption", "").
m.Handle(h.GetBucketEncryptionHandler)).
Queries("encryption", "").
Name("GetBucketEncryption")
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketcors", h.GetBucketCorsHandler))).Queries("cors", "").
m.Handle(h.GetBucketCorsHandler)).
Queries("cors", "").
Name("GetBucketCors")
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(metrics.APIStats("putbucketcors", h.PutBucketCorsHandler))).Queries("cors", "").
m.Handle(h.PutBucketCorsHandler)).
Queries("cors", "").
Name("PutBucketCors")
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(metrics.APIStats("deletebucketcors", h.DeleteBucketCorsHandler))).Queries("cors", "").
m.Handle(h.DeleteBucketCorsHandler)).
Queries("cors", "").
Name("DeleteBucketCors")
// Dummy Bucket Calls
// GetBucketACL -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketacl", h.GetBucketACLHandler))).Queries("acl", "").
m.Handle(h.GetBucketACLHandler)).
Queries("acl", "").
Name("GetBucketACL")
// PutBucketACL -- this is a dummy call.
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(metrics.APIStats("putbucketacl", h.PutBucketACLHandler))).Queries("acl", "").
m.Handle(h.PutBucketACLHandler)).
Queries("acl", "").
Name("PutBucketACL")
// GetBucketWebsiteHandler -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketwebsite", h.GetBucketWebsiteHandler))).Queries("website", "").
m.Handle(h.GetBucketWebsiteHandler)).
Queries("website", "").
Name("GetBucketWebsite")
// GetBucketAccelerateHandler -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketaccelerate", h.GetBucketAccelerateHandler))).Queries("accelerate", "").
m.Handle(h.GetBucketAccelerateHandler)).
Queries("accelerate", "").
Name("GetBucketAccelerate")
// GetBucketRequestPaymentHandler -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketrequestpayment", h.GetBucketRequestPaymentHandler))).Queries("requestPayment", "").
m.Handle(h.GetBucketRequestPaymentHandler)).
Queries("requestPayment", "").
Name("GetBucketRequestPayment")
// GetBucketLoggingHandler -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketlogging", h.GetBucketLoggingHandler))).Queries("logging", "").
m.Handle(h.GetBucketLoggingHandler)).
Queries("logging", "").
Name("GetBucketLogging")
// GetBucketLifecycleHandler -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", "").
Name("GetBucketLifecycle")
// GetBucketReplicationHandler -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketreplication", h.GetBucketReplicationHandler))).Queries("replication", "").
m.Handle(h.GetBucketReplicationHandler)).
Queries("replication", "").
Name("GetBucketReplication")
// GetBucketTaggingHandler
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbuckettagging", h.GetBucketTaggingHandler))).Queries("tagging", "").
m.Handle(h.GetBucketTaggingHandler)).
Queries("tagging", "").
Name("GetBucketTagging")
// DeleteBucketWebsiteHandler
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(metrics.APIStats("deletebucketwebsite", h.DeleteBucketWebsiteHandler))).Queries("website", "").
m.Handle(h.DeleteBucketWebsiteHandler)).
Queries("website", "").
Name("DeleteBucketWebsite")
// DeleteBucketTaggingHandler
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(metrics.APIStats("deletebuckettagging", h.DeleteBucketTaggingHandler))).Queries("tagging", "").
m.Handle(h.DeleteBucketTaggingHandler)).
Queries("tagging", "").
Name("DeleteBucketTagging")
// GetBucketObjectLockConfig
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketobjectlockconfiguration", h.GetBucketObjectLockConfigHandler))).Queries("object-lock", "").
m.Handle(h.GetBucketObjectLockConfigHandler)).
Queries("object-lock", "").
Name("GetBucketObjectLockConfig")
// GetBucketVersioning
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketversioning", h.GetBucketVersioningHandler))).Queries("versioning", "").
m.Handle(h.GetBucketVersioningHandler)).
Queries("versioning", "").
Name("GetBucketVersioning")
// GetBucketNotification
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketnotification", h.GetBucketNotificationHandler))).Queries("notification", "").
m.Handle(h.GetBucketNotificationHandler)).
Queries("notification", "").
Name("GetBucketNotification")
// ListenBucketNotification
bucket.Methods(http.MethodGet).HandlerFunc(metrics.APIStats("listenbucketnotification", h.ListenBucketNotificationHandler)).Queries("events", "{events:.*}").
bucket.Methods(http.MethodGet).HandlerFunc(h.ListenBucketNotificationHandler).
Queries("events", "{events:.*}").
Name("ListenBucketNotification")
// ListObjectsV2M
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("listobjectsv2M", h.ListObjectsV2MHandler))).Queries("list-type", "2", "metadata", "true").
m.Handle(h.ListObjectsV2MHandler)).
Queries("list-type", "2", "metadata", "true").
Name("ListObjectsV2M")
// ListObjectsV2
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("listobjectsv2", h.ListObjectsV2Handler))).Queries("list-type", "2").
m.Handle(h.ListObjectsV2Handler)).
Queries("list-type", "2").
Name("ListObjectsV2")
// ListBucketVersions
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("listbucketversions", h.ListBucketObjectVersionsHandler))).Queries("versions", "").
m.Handle(h.ListBucketObjectVersionsHandler)).
Queries("versions", "").
Name("ListBucketVersions")
// ListObjectsV1 (Legacy)
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("listobjectsv1", h.ListObjectsV1Handler))).
m.Handle(h.ListObjectsV1Handler)).
Name("ListObjectsV1")
// PutBucketLifecycle
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(metrics.APIStats("putbucketlifecycle", h.PutBucketLifecycleHandler))).Queries("lifecycle", "").
m.Handle(h.PutBucketLifecycleHandler)).
Queries("lifecycle", "").
Name("PutBucketLifecycle")
// PutBucketEncryption
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(metrics.APIStats("putbucketencryption", h.PutBucketEncryptionHandler))).Queries("encryption", "").
m.Handle(h.PutBucketEncryptionHandler)).
Queries("encryption", "").
Name("PutBucketEncryption")
// PutBucketPolicy
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(metrics.APIStats("putbucketpolicy", h.PutBucketPolicyHandler))).Queries("policy", "").
m.Handle(h.PutBucketPolicyHandler)).
Queries("policy", "").
Name("PutBucketPolicy")
// PutBucketObjectLockConfig
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(metrics.APIStats("putbucketobjectlockconfig", h.PutBucketObjectLockConfigHandler))).Queries("object-lock", "").
m.Handle(h.PutBucketObjectLockConfigHandler)).
Queries("object-lock", "").
Name("PutBucketObjectLockConfig")
// PutBucketTaggingHandler
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(metrics.APIStats("putbuckettagging", h.PutBucketTaggingHandler))).Queries("tagging", "").
m.Handle(h.PutBucketTaggingHandler)).
Queries("tagging", "").
Name("PutBucketTagging")
// PutBucketVersioning
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(metrics.APIStats("putbucketversioning", h.PutBucketVersioningHandler))).Queries("versioning", "").
m.Handle(h.PutBucketVersioningHandler)).
Queries("versioning", "").
Name("PutBucketVersioning")
// PutBucketNotification
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(metrics.APIStats("putbucketnotification", h.PutBucketNotificationHandler))).Queries("notification", "").
m.Handle(h.PutBucketNotificationHandler)).
Queries("notification", "").
Name("PutBucketNotification")
// CreateBucket
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(metrics.APIStats("createbucket", h.CreateBucketHandler))).
m.Handle(h.CreateBucketHandler)).
Name("CreateBucket")
// HeadBucket
bucket.Methods(http.MethodHead).HandlerFunc(
m.Handle(metrics.APIStats("headbucket", h.HeadBucketHandler))).
m.Handle(h.HeadBucketHandler)).
Name("HeadBucket")
// PostPolicy
bucket.Methods(http.MethodPost).HeadersRegexp(hdrContentType, "multipart/form-data*").HandlerFunc(
m.Handle(metrics.APIStats("postobject", h.PostObject))).
m.Handle(h.PostObject)).
Name("PostObject")
// DeleteMultipleObjects
bucket.Methods(http.MethodPost).HandlerFunc(
m.Handle(metrics.APIStats("deletemultipleobjects", h.DeleteMultipleObjectsHandler))).Queries("delete", "").
m.Handle(h.DeleteMultipleObjectsHandler)).
Queries("delete", "").
Name("DeleteMultipleObjects")
// DeleteBucketPolicy
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(metrics.APIStats("deletebucketpolicy", h.DeleteBucketPolicyHandler))).Queries("policy", "").
m.Handle(h.DeleteBucketPolicyHandler)).
Queries("policy", "").
Name("DeleteBucketPolicy")
// DeleteBucketLifecycle
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(metrics.APIStats("deletebucketlifecycle", h.DeleteBucketLifecycleHandler))).Queries("lifecycle", "").
m.Handle(h.DeleteBucketLifecycleHandler)).
Queries("lifecycle", "").
Name("DeleteBucketLifecycle")
// DeleteBucketEncryption
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(metrics.APIStats("deletebucketencryption", h.DeleteBucketEncryptionHandler))).Queries("encryption", "").
m.Handle(h.DeleteBucketEncryptionHandler)).
Queries("encryption", "").
Name("DeleteBucketEncryption")
// DeleteBucket
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(metrics.APIStats("deletebucket", h.DeleteBucketHandler))).
m.Handle(h.DeleteBucketHandler)).
Name("DeleteBucket")
}
// Root operation
// ListBuckets
api.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
m.Handle(metrics.APIStats("listbuckets", h.ListBucketsHandler))).
m.Handle(h.ListBucketsHandler)).
Name("ListBuckets")
// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
// than failing with UnknownAPIRequest we simply handle it for now.
api.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
m.Handle(metrics.APIStats("listbuckets", h.ListBucketsHandler))).
m.Handle(h.ListBucketsHandler)).
Name("ListBuckets")
// If none of the routes match, add default error handler routes
api.NotFoundHandler = metrics.APIStats("notfound", errorResponseHandler)
api.MethodNotAllowedHandler = metrics.APIStats("methodnotallowed", errorResponseHandler)
}

View file

@ -4,9 +4,9 @@ import (
"context"
"net/http"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/gorilla/mux"
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"go.uber.org/zap"
)
@ -16,9 +16,12 @@ type KeyWrapper string
// BoxData is an ID used to store accessbox.Box in a context.
var BoxData = KeyWrapper("__context_box_key")
// AttachUserAuth adds user authentication via center to router using log for logging.
func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
router.Use(func(h http.Handler) http.Handler {
// ClientTime is an ID used to store client time.Time in a context.
var ClientTime = KeyWrapper("__context_client_time")
// AuthMiddleware adds user authentication via center to router using log for logging.
func AuthMiddleware(log *zap.Logger, center auth.Center) mux.MiddlewareFunc {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var ctx context.Context
box, err := center.Authenticate(r)
@ -35,10 +38,13 @@ func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
return
}
} else {
ctx = context.WithValue(r.Context(), BoxData, box)
ctx = context.WithValue(r.Context(), BoxData, box.AccessBox)
if !box.ClientTime.IsZero() {
ctx = context.WithValue(ctx, ClientTime, box.ClientTime)
}
}
h.ServeHTTP(w, r.WithContext(ctx))
})
})
}
}

View file

@ -11,25 +11,25 @@ import (
"os"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/google/uuid"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
neofsecdsa "github.com/nspcc-dev/neofs-sdk-go/crypto/ecdsa"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/user"
"go.uber.org/zap"
)
// PrmContainerCreate groups parameters of containers created by authmate.
type PrmContainerCreate struct {
// NeoFS identifier of the container creator.
// FrostFS identifier of the container creator.
Owner user.ID
// Container placement policy.
@ -39,26 +39,26 @@ type PrmContainerCreate struct {
FriendlyName string
}
// NetworkState represents NeoFS network state which is needed for authmate processing.
// NetworkState represents FrostFS network state which is needed for authmate processing.
type NetworkState struct {
// Current NeoFS time.
// Current FrostFS time.
Epoch uint64
// Duration of the Morph chain block in ms.
BlockDuration int64
// Duration of the NeoFS epoch in Morph chain blocks.
// Duration of the FrostFS epoch in Morph chain blocks.
EpochDuration uint64
}
// NeoFS represents virtual connection to NeoFS network.
type NeoFS interface {
// NeoFS interface required by credential tool.
tokens.NeoFS
// FrostFS represents virtual connection to FrostFS network.
type FrostFS interface {
// FrostFS interface required by credential tool.
tokens.FrostFS
// ContainerExists checks container presence in NeoFS by identifier.
// ContainerExists checks container presence in FrostFS by identifier.
// Returns nil if container exists.
ContainerExists(context.Context, cid.ID) error
// CreateContainer creates and saves parameterized container in NeoFS.
// CreateContainer creates and saves parameterized container in FrostFS.
// It sets 'Timestamp' attribute to the current time.
// It returns the ID of the saved container.
//
@ -78,25 +78,25 @@ type NeoFS interface {
TimeToEpoch(context.Context, time.Time) (uint64, uint64, error)
}
// Agent contains client communicating with NeoFS and logger.
// Agent contains client communicating with FrostFS and logger.
type Agent struct {
neoFS NeoFS
log *zap.Logger
frostFS FrostFS
log *zap.Logger
}
// New creates an object of type Agent that consists of Client and logger.
func New(log *zap.Logger, neoFS NeoFS) *Agent {
return &Agent{log: log, neoFS: neoFS}
func New(log *zap.Logger, frostFS FrostFS) *Agent {
return &Agent{log: log, frostFS: frostFS}
}
type (
// ContainerPolicies contains mapping of aws LocationConstraint to neofs PlacementPolicy.
// ContainerPolicies contains mapping of aws LocationConstraint to frostfs PlacementPolicy.
ContainerPolicies map[string]string
// IssueSecretOptions contains options for passing to Agent.IssueSecret method.
IssueSecretOptions struct {
Container ContainerOptions
NeoFSKey *keys.PrivateKey
FrostFSKey *keys.PrivateKey
GatesPublicKeys []*keys.PublicKey
EACLRules []byte
SessionTokenRules []byte
@ -120,7 +120,7 @@ type (
}
)
// lifetimeOptions holds NeoFS epochs, iat -- epoch which the token was issued at, exp -- epoch when the token expires.
// lifetimeOptions holds FrostFS epochs, iat -- epoch which the token was issued at, exp -- epoch when the token expires.
type lifetimeOptions struct {
Iat uint64
Exp uint64
@ -143,7 +143,7 @@ type (
func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwner user.ID) (cid.ID, error) {
if !opts.ID.Equals(cid.ID{}) {
return opts.ID, a.neoFS.ContainerExists(ctx, opts.ID)
return opts.ID, a.frostFS.ContainerExists(ctx, opts.ID)
}
var prm PrmContainerCreate
@ -156,9 +156,9 @@ func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwn
prm.Owner = idOwner
prm.FriendlyName = opts.FriendlyName
cnrID, err := a.neoFS.CreateContainer(ctx, prm)
cnrID, err := a.frostFS.CreateContainer(ctx, prm)
if err != nil {
return cid.ID{}, fmt.Errorf("create container in NeoFS: %w", err)
return cid.ID{}, fmt.Errorf("create container in FrostFS: %w", err)
}
return cnrID, nil
@ -200,7 +200,7 @@ func preparePolicy(policy ContainerPolicies) ([]*accessbox.AccessBox_ContainerPo
return result, nil
}
// IssueSecret creates an auth token, puts it in the NeoFS network and writes to io.Writer a new secret access key.
// IssueSecret creates an auth token, puts it in the FrostFS network and writes to io.Writer a new secret access key.
func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecretOptions) error {
var (
err error
@ -213,7 +213,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
return fmt.Errorf("prepare policies: %w", err)
}
lifetime.Iat, lifetime.Exp, err = a.neoFS.TimeToEpoch(ctx, time.Now().Add(options.Lifetime))
lifetime.Iat, lifetime.Exp, err = a.frostFS.TimeToEpoch(ctx, time.Now().Add(options.Lifetime))
if err != nil {
return fmt.Errorf("fetch time to epoch: %w", err)
}
@ -231,7 +231,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
box.ContainerPolicy = policies
var idOwner user.ID
user.IDFromKey(&idOwner, options.NeoFSKey.PrivateKey.PublicKey)
user.IDFromKey(&idOwner, options.FrostFSKey.PrivateKey.PublicKey)
a.log.Info("check container or create", zap.Stringer("cid", options.Container.ID),
zap.String("friendly_name", options.Container.FriendlyName),
@ -241,11 +241,11 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
return fmt.Errorf("check container: %w", err)
}
a.log.Info("store bearer token into NeoFS",
a.log.Info("store bearer token into FrostFS",
zap.Stringer("owner_tkn", idOwner))
addr, err := tokens.
New(a.neoFS, secrets.EphemeralKey, cache.DefaultAccessBoxConfig(a.log)).
New(a.frostFS, secrets.EphemeralKey, cache.DefaultAccessBoxConfig(a.log)).
Put(ctx, id, idOwner, box, lifetime.Exp, options.GatesPublicKeys...)
if err != nil {
return fmt.Errorf("failed to put bearer token: %w", err)
@ -260,7 +260,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
AccessKeyID: accessKeyID,
SecretAccessKey: secrets.AccessKey,
OwnerPrivateKey: hex.EncodeToString(secrets.EphemeralKey.Bytes()),
WalletPublicKey: hex.EncodeToString(options.NeoFSKey.PublicKey().Bytes()),
WalletPublicKey: hex.EncodeToString(options.FrostFSKey.PublicKey().Bytes()),
ContainerID: id.EncodeToString(),
}
@ -288,10 +288,10 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
return nil
}
// ObtainSecret receives an existing secret access key from NeoFS and
// ObtainSecret receives an existing secret access key from FrostFS and
// writes to io.Writer the secret access key.
func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSecretOptions) error {
bearerCreds := tokens.New(a.neoFS, options.GatePrivateKey, cache.DefaultAccessBoxConfig(a.log))
bearerCreds := tokens.New(a.frostFS, options.GatePrivateKey, cache.DefaultAccessBoxConfig(a.log))
var addr oid.Address
if err := addr.DecodeString(options.SecretAddress); err != nil {
@ -381,7 +381,7 @@ func buildSessionToken(key *keys.PrivateKey, lifetime lifetimeOptions, ctx sessi
tok.AppliedTo(ctx.containerID)
tok.SetID(uuid.New())
tok.SetAuthKey((*neofsecdsa.PublicKey)(gateKey))
tok.SetAuthKey((*frostfsecdsa.PublicKey)(gateKey))
tok.SetIat(lifetime.Iat)
tok.SetNbf(lifetime.Iat)
@ -413,7 +413,7 @@ func createTokens(options *IssueSecretOptions, lifetime lifetimeOptions) ([]*acc
if err != nil {
return nil, fmt.Errorf("failed to build eacl table: %w", err)
}
bearerTokens, err := buildBearerTokens(options.NeoFSKey, table, lifetime, options.GatesPublicKeys)
bearerTokens, err := buildBearerTokens(options.FrostFSKey, table, lifetime, options.GatesPublicKeys)
if err != nil {
return nil, fmt.Errorf("failed to build bearer tokens: %w", err)
}
@ -427,7 +427,7 @@ func createTokens(options *IssueSecretOptions, lifetime lifetimeOptions) ([]*acc
return nil, fmt.Errorf("failed to build context for session token: %w", err)
}
sessionTokens, err := buildSessionTokens(options.NeoFSKey, lifetime, sessionRules, options.GatesPublicKeys)
sessionTokens, err := buildSessionTokens(options.FrostFSKey, lifetime, sessionRules, options.GatesPublicKeys)
if err != nil {
return nil, fmt.Errorf("failed to biuild session token: %w", err)
}

Some files were not shown because too many files have changed in this diff Show more