forked from TrueCloudLab/frostfs-s3-gw
Compare commits
75 commits
support/v0
...
master
Author | SHA1 | Date | |
---|---|---|---|
fb90c0f52c | |||
f2f90e260e | |||
a0937126cb | |||
655889a1a2 | |||
ef556bd8ac | |||
5104683f68 | |||
8151753eeb | |||
2282c32822 | |||
43685e03d9 | |||
cf18158da4 | |||
5c62010331 | |||
0af06c3bd9 | |||
680c0dbe3d | |||
596381c382 | |||
64e7356acc | |||
32bf915502 | |||
813aa2f173 | |||
6eb7966800 | |||
2dcb3c283d | |||
740acadd37 | |||
3ab77c8990 | |||
d00163aadc | |||
aadefd98b6 | |||
8ac630ee71 | |||
|
744b52322d | ||
8d9d1f9235 | |||
787d1a347a | |||
9f823bd65a | |||
9dcacc230e | |||
430f1e734f | |||
1ce8b8a30d | |||
b35f146cec | |||
5ee4bf80ae | |||
f9f52ce8e0 | |||
e278ab9362 | |||
c5570e661d | |||
fc5c09c084 | |||
86e881694d | |||
361d1d3881 | |||
9ad7982807 | |||
a0d5b18184 | |||
533b12d8bb | |||
cafe079072 | |||
9473335234 | |||
19d8f8fcfe | |||
db603a9703 | |||
b2148cc36b | |||
ff040f6785 | |||
8e14ccad9f | |||
af7fbd6f32 | |||
|
a68aca764b | ||
|
f1696c4725 | ||
000d9ed42a | |||
388482e230 | |||
c2567fcc73 | |||
f744c672cb | |||
765fd28a49 | |||
09c6e22b84 | |||
96dff367db | |||
dd25331210 | |||
|
d6424ebeac | ||
|
5265afe213 | ||
|
dd4f66712c | ||
|
556374e3b0 | ||
|
d2587b21af | ||
|
f5fe9a9b4b | ||
|
d1825f97fb | ||
|
8f1bbb7dc1 | ||
|
1c0f4d6df8 | ||
|
094eb12578 | ||
|
d3702f86d1 | ||
|
d47840f137 | ||
|
3212805955 | ||
|
2886ac161c | ||
7b0492c468 |
151 changed files with 3996 additions and 1978 deletions
|
@ -1,7 +1,7 @@
|
||||||
FROM golang:1.19 as builder
|
FROM golang:1.19 as builder
|
||||||
|
|
||||||
ARG BUILD=now
|
ARG BUILD=now
|
||||||
ARG REPO=github.com/nspcc-dev/neofs-s3-gw
|
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
@ -10,12 +10,12 @@ COPY . /src
|
||||||
RUN make
|
RUN make
|
||||||
|
|
||||||
# Executable image
|
# Executable image
|
||||||
FROM alpine AS neofs-s3-gw
|
FROM alpine AS frostfs-s3-gw
|
||||||
RUN apk add --no-cache bash ca-certificates
|
RUN apk add --no-cache bash ca-certificates
|
||||||
|
|
||||||
WORKDIR /
|
WORKDIR /
|
||||||
|
|
||||||
COPY --from=builder /src/bin/neofs-s3-gw /bin/neofs-s3-gw
|
COPY --from=builder /src/bin/frostfs-s3-gw /bin/frostfs-s3-gw
|
||||||
COPY --from=builder /src/bin/neofs-s3-authmate /bin/neofs-s3-authmate
|
COPY --from=builder /src/bin/frostfs-s3-authmate /bin/frostfs-s3-authmate
|
||||||
|
|
||||||
ENTRYPOINT ["/bin/neofs-s3-gw"]
|
ENTRYPOINT ["/bin/frostfs-s3-gw"]
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
FROM alpine AS neofs-s3-gw
|
FROM alpine AS frostfs-s3-gw
|
||||||
RUN apk add --no-cache bash ca-certificates
|
RUN apk add --no-cache bash ca-certificates
|
||||||
|
|
||||||
WORKDIR /
|
WORKDIR /
|
||||||
|
|
||||||
COPY /bin/neofs-s3-gw /bin/neofs-s3-gw
|
COPY /bin/frostfs-s3-gw /bin/frostfs-s3-gw
|
||||||
COPY /bin/neofs-s3-authmate /bin/neofs-s3-authmate
|
COPY /bin/frostfs-s3-authmate /bin/frostfs-s3-authmate
|
||||||
|
|
||||||
ENTRYPOINT ["/bin/neofs-s3-gw"]
|
ENTRYPOINT ["/bin/frostfs-s3-gw"]
|
||||||
|
|
2
.github/CODEOWNERS
vendored
2
.github/CODEOWNERS
vendored
|
@ -1 +1 @@
|
||||||
* @alexvanin @masterSplinter01 @KirillovDenis
|
* @alexvanin @KirillovDenis
|
||||||
|
|
45
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
45
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
|
@ -0,0 +1,45 @@
|
||||||
|
---
|
||||||
|
name: Bug report
|
||||||
|
about: Create a report to help us improve
|
||||||
|
title: ''
|
||||||
|
labels: community, triage, bug
|
||||||
|
assignees: ''
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
<!--- Provide a general summary of the issue in the Title above -->
|
||||||
|
|
||||||
|
## Expected Behavior
|
||||||
|
<!--- If you're describing a bug, tell us what should happen -->
|
||||||
|
<!--- If you're suggesting a change/improvement, tell us how it should work -->
|
||||||
|
|
||||||
|
## Current Behavior
|
||||||
|
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
|
||||||
|
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
|
||||||
|
|
||||||
|
## Possible Solution
|
||||||
|
<!--- Not obligatory -->
|
||||||
|
<!--- If no reason/fix/additions for the bug can be suggested, -->
|
||||||
|
<!--- uncomment the following phrase: -->
|
||||||
|
|
||||||
|
<!--- No fix can be suggested by a QA engineer. Further solutions shall be up to developers. -->
|
||||||
|
|
||||||
|
## Steps to Reproduce (for bugs)
|
||||||
|
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
||||||
|
<!--- reproduce this bug. -->
|
||||||
|
|
||||||
|
1.
|
||||||
|
|
||||||
|
## Context
|
||||||
|
<!--- How has this issue affected you? What are you trying to accomplish? -->
|
||||||
|
<!--- Providing context helps us come up with a solution that is most useful in the real world -->
|
||||||
|
|
||||||
|
## Regression
|
||||||
|
<!-- Is this issue a regression? (Yes / No) -->
|
||||||
|
<!-- If Yes, optionally please include version or commit id or PR# that caused this regression, if you have these details. -->
|
||||||
|
|
||||||
|
## Your Environment
|
||||||
|
<!--- Include as many relevant details about the environment you experienced the bug in -->
|
||||||
|
* Version used:
|
||||||
|
* Server setup and configuration:
|
||||||
|
* Operating System and version (`uname -a`):
|
1
.github/ISSUE_TEMPLATE/config.yml
vendored
Normal file
1
.github/ISSUE_TEMPLATE/config.yml
vendored
Normal file
|
@ -0,0 +1 @@
|
||||||
|
blank_issues_enabled: false
|
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
Normal file
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
Normal file
|
@ -0,0 +1,20 @@
|
||||||
|
---
|
||||||
|
name: Feature request
|
||||||
|
about: Suggest an idea for this project
|
||||||
|
title: ''
|
||||||
|
labels: community, triage
|
||||||
|
assignees: ''
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Is your feature request related to a problem? Please describe.
|
||||||
|
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
|
||||||
|
|
||||||
|
## Describe the solution you'd like
|
||||||
|
<!--- A clear and concise description of what you want to happen. -->
|
||||||
|
|
||||||
|
## Describe alternatives you've considered
|
||||||
|
<!--- A clear and concise description of any alternative solutions or features you've considered. -->
|
||||||
|
|
||||||
|
## Additional context
|
||||||
|
<!--- Add any other context or screenshots about the feature request here. -->
|
70
.github/logo.svg
vendored
Normal file
70
.github/logo.svg
vendored
Normal file
|
@ -0,0 +1,70 @@
|
||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<!-- Generator: Adobe Illustrator 25.0.1, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||||
|
<svg version="1.1" id="Слой_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||||
|
viewBox="0 0 184.2 51.8" style="enable-background:new 0 0 184.2 51.8;" xml:space="preserve">
|
||||||
|
<style type="text/css">
|
||||||
|
.st0{display:none;}
|
||||||
|
.st1{display:inline;}
|
||||||
|
.st2{fill:#01E397;}
|
||||||
|
.st3{display:inline;fill:#010032;}
|
||||||
|
.st4{display:inline;fill:#00E599;}
|
||||||
|
.st5{display:inline;fill:#00AF92;}
|
||||||
|
.st6{fill:#00C3E5;}
|
||||||
|
</style>
|
||||||
|
<g id="Layer_2">
|
||||||
|
<g id="Layer_1-2" class="st0">
|
||||||
|
<g class="st1">
|
||||||
|
<path class="st2" d="M146.6,18.3v7.2h10.9V29h-10.9v10.7h-4V14.8h18v3.5H146.6z"/>
|
||||||
|
<path class="st2" d="M180,15.7c1.7,0.9,3,2.2,4,3.8l-3,2.7c-0.6-1.3-1.5-2.4-2.6-3.3c-1.3-0.7-2.8-1-4.3-1
|
||||||
|
c-1.4-0.1-2.8,0.3-4,1.1c-0.9,0.5-1.5,1.5-1.4,2.6c0,1,0.5,1.9,1.4,2.4c1.5,0.8,3.2,1.3,4.9,1.5c1.9,0.3,3.7,0.8,5.4,1.6
|
||||||
|
c1.2,0.5,2.2,1.3,2.9,2.3c0.6,1,1,2.2,0.9,3.4c0,1.4-0.5,2.7-1.3,3.8c-0.9,1.2-2.1,2.1-3.5,2.6c-1.7,0.6-3.4,0.9-5.2,0.8
|
||||||
|
c-5,0-8.6-1.6-10.7-5l2.9-2.8c0.7,1.4,1.8,2.5,3.1,3.3c1.5,0.7,3.1,1.1,4.7,1c1.5,0.1,2.9-0.2,4.2-0.9c0.9-0.5,1.5-1.5,1.5-2.6
|
||||||
|
c0-0.9-0.5-1.8-1.3-2.2c-1.5-0.7-3.1-1.2-4.8-1.5c-1.9-0.3-3.7-0.8-5.5-1.5c-1.2-0.5-2.2-1.4-3-2.4c-0.6-1-1-2.2-0.9-3.4
|
||||||
|
c0-1.4,0.4-2.7,1.2-3.8c0.8-1.2,2-2.2,3.3-2.8c1.6-0.7,3.4-1.1,5.2-1C176.1,14.3,178.2,14.8,180,15.7z"/>
|
||||||
|
</g>
|
||||||
|
<path class="st3" d="M73.3,16.3c1.9,1.9,2.9,4.5,2.7,7.1v15.9h-4V24.8c0-2.6-0.5-4.5-1.6-5.7c-1.2-1.2-2.8-1.8-4.5-1.7
|
||||||
|
c-1.3,0-2.5,0.3-3.7,0.8c-1.2,0.7-2.2,1.7-2.9,2.9c-0.8,1.5-1.1,3.2-1.1,4.9v13.3h-4V15.1l3.6,1.5v1.7c0.8-1.5,2.1-2.6,3.6-3.3
|
||||||
|
c1.5-0.8,3.2-1.2,4.9-1.1C68.9,13.8,71.3,14.7,73.3,16.3z"/>
|
||||||
|
<path class="st3" d="M104.4,28.3H85.6c0.1,2.2,1,4.3,2.5,5.9c1.5,1.4,3.5,2.2,5.6,2.1c1.6,0.1,3.2-0.2,4.6-0.9
|
||||||
|
c1.1-0.6,2-1.6,2.5-2.8l3.3,1.8c-0.9,1.7-2.3,3.1-4,4c-2,1-4.2,1.5-6.4,1.4c-3.7,0-6.7-1.1-8.8-3.4s-3.2-5.5-3.2-9.6s1-7.2,3-9.5
|
||||||
|
s5-3.4,8.7-3.4c2.1-0.1,4.2,0.5,6.1,1.5c1.6,1,3,2.5,3.8,4.2c0.9,1.8,1.3,3.9,1.3,5.9C104.6,26.4,104.6,27.4,104.4,28.3z
|
||||||
|
M88.1,19.3c-1.4,1.5-2.2,3.4-2.4,5.5h15.1c-0.2-2-1-3.9-2.3-5.5c-1.4-1.3-3.2-2-5.1-1.9C91.5,17.3,89.6,18,88.1,19.3z"/>
|
||||||
|
<path class="st3" d="M131,17.3c2.2,2.3,3.2,5.5,3.2,9.5s-1,7.3-3.2,9.6s-5.1,3.4-8.8,3.4s-6.7-1.1-8.9-3.4s-3.2-5.5-3.2-9.6
|
||||||
|
s1.1-7.2,3.2-9.5s5.1-3.4,8.9-3.4S128.9,15,131,17.3z M116.2,19.9c-1.5,2-2.2,4.4-2.1,6.9c-0.2,2.5,0.6,5,2.1,7
|
||||||
|
c1.5,1.7,3.7,2.7,6,2.6c2.3,0.1,4.4-0.9,5.9-2.6c1.5-2,2.3-4.5,2.1-7c0.1-2.5-0.6-4.9-2.1-6.9c-1.5-1.7-3.6-2.7-5.9-2.6
|
||||||
|
C119.9,17.2,117.7,18.2,116.2,19.9z"/>
|
||||||
|
<polygon class="st4" points="0,9.1 0,43.7 22.5,51.8 22.5,16.9 46.8,7.9 24.8,0 "/>
|
||||||
|
<polygon class="st5" points="24.3,17.9 24.3,36.8 46.8,44.9 46.8,9.6 "/>
|
||||||
|
</g>
|
||||||
|
<g>
|
||||||
|
<g>
|
||||||
|
<path class="st6" d="M41.6,17.5H28.2v6.9h10.4v3.3H28.2v10.2h-3.9V14.2h17.2V17.5z"/>
|
||||||
|
<path class="st6" d="M45.8,37.9v-18h3.3l0.4,3.2c0.5-1.2,1.2-2.1,2.1-2.7c0.9-0.6,2.1-0.9,3.5-0.9c0.4,0,0.7,0,1.1,0.1
|
||||||
|
c0.4,0.1,0.7,0.2,0.9,0.3l-0.5,3.4c-0.3-0.1-0.6-0.2-0.9-0.2C55.4,23,54.9,23,54.4,23c-0.7,0-1.5,0.2-2.2,0.6
|
||||||
|
c-0.7,0.4-1.3,1-1.8,1.8s-0.7,1.8-0.7,3v9.5H45.8z"/>
|
||||||
|
<path class="st6" d="M68.6,19.6c1.8,0,3.3,0.4,4.6,1.1c1.3,0.7,2.4,1.8,3.1,3.2s1.1,3.1,1.1,5c0,1.9-0.4,3.6-1.1,5
|
||||||
|
c-0.8,1.4-1.8,2.5-3.1,3.2c-1.3,0.7-2.9,1.1-4.6,1.1s-3.3-0.4-4.6-1.1c-1.3-0.7-2.4-1.8-3.2-3.2c-0.8-1.4-1.2-3.1-1.2-5
|
||||||
|
c0-1.9,0.4-3.6,1.2-5s1.8-2.5,3.2-3.2C65.3,19.9,66.8,19.6,68.6,19.6z M68.6,22.6c-1.1,0-2,0.2-2.8,0.7c-0.8,0.5-1.3,1.2-1.7,2.1
|
||||||
|
s-0.6,2.1-0.6,3.5c0,1.3,0.2,2.5,0.6,3.4s1,1.7,1.7,2.2s1.7,0.7,2.8,0.7c1.1,0,2-0.2,2.7-0.7c0.7-0.5,1.3-1.2,1.7-2.2
|
||||||
|
s0.6-2.1,0.6-3.4c0-1.4-0.2-2.5-0.6-3.5s-1-1.6-1.7-2.1C70.6,22.8,69.6,22.6,68.6,22.6z"/>
|
||||||
|
<path class="st6" d="M89.2,38.3c-1.8,0-3.4-0.3-4.9-1c-1.5-0.7-2.7-1.7-3.5-3l2.7-2.3c0.5,1,1.3,1.8,2.3,2.4
|
||||||
|
c1,0.6,2.2,0.9,3.6,0.9c1.1,0,2-0.2,2.6-0.6c0.6-0.4,1-0.9,1-1.6c0-0.5-0.2-0.9-0.5-1.2s-0.9-0.6-1.7-0.8l-3.8-0.8
|
||||||
|
c-1.9-0.4-3.3-1-4.1-1.9c-0.8-0.9-1.2-1.9-1.2-3.3c0-1,0.3-1.9,0.9-2.7c0.6-0.8,1.4-1.5,2.5-2s2.5-0.8,4-0.8c1.8,0,3.3,0.3,4.6,1
|
||||||
|
c1.3,0.6,2.2,1.5,2.9,2.7l-2.7,2.2c-0.5-1-1.1-1.7-2-2.1c-0.9-0.5-1.8-0.7-2.8-0.7c-0.8,0-1.4,0.1-2,0.3c-0.6,0.2-1,0.5-1.3,0.8
|
||||||
|
c-0.3,0.3-0.4,0.7-0.4,1.2c0,0.5,0.2,0.9,0.5,1.3s1,0.6,1.9,0.8l4.1,0.9c1.7,0.3,2.9,0.9,3.7,1.7c0.7,0.8,1.1,1.8,1.1,2.9
|
||||||
|
c0,1.2-0.3,2.2-0.9,3c-0.6,0.9-1.5,1.6-2.6,2C92.1,38.1,90.7,38.3,89.2,38.3z"/>
|
||||||
|
<path class="st6" d="M112.8,19.9v3H99.3v-3H112.8z M106.6,14.6v17.9c0,0.9,0.2,1.5,0.7,1.9c0.5,0.4,1.1,0.6,1.9,0.6
|
||||||
|
c0.6,0,1.2-0.1,1.7-0.3c0.5-0.2,0.9-0.5,1.3-0.8l0.9,2.8c-0.6,0.5-1.2,0.9-2,1.1c-0.8,0.3-1.7,0.4-2.7,0.4c-1,0-2-0.2-2.8-0.5
|
||||||
|
s-1.5-0.9-2-1.6c-0.5-0.8-0.7-1.7-0.8-3V15.7L106.6,14.6z"/>
|
||||||
|
<path d="M137.9,17.5h-13.3v6.9h10.4v3.3h-10.4v10.2h-3.9V14.2h17.2V17.5z"/>
|
||||||
|
<path d="M150.9,13.8c2.1,0,4,0.4,5.5,1.2c1.6,0.8,2.9,2,4,3.5l-2.6,2.5c-0.9-1.4-1.9-2.4-3.1-3c-1.1-0.6-2.5-0.9-4-0.9
|
||||||
|
c-1.2,0-2.1,0.2-2.8,0.5c-0.7,0.3-1.3,0.7-1.6,1.2c-0.3,0.5-0.5,1.1-0.5,1.7c0,0.7,0.3,1.4,0.8,1.9c0.5,0.6,1.5,1,2.9,1.3
|
||||||
|
l4.8,1.1c2.3,0.5,3.9,1.3,4.9,2.3c1,1,1.4,2.3,1.4,3.9c0,1.5-0.4,2.7-1.2,3.8c-0.8,1.1-1.9,1.9-3.3,2.5s-3.1,0.9-5,0.9
|
||||||
|
c-1.7,0-3.2-0.2-4.5-0.6c-1.3-0.4-2.5-1-3.5-1.8c-1-0.7-1.8-1.6-2.5-2.6l2.7-2.7c0.5,0.8,1.1,1.6,1.9,2.2
|
||||||
|
c0.8,0.7,1.7,1.2,2.7,1.5c1,0.4,2.2,0.5,3.4,0.5c1.1,0,2.1-0.1,2.9-0.4c0.8-0.3,1.4-0.7,1.8-1.2c0.4-0.5,0.6-1.1,0.6-1.9
|
||||||
|
c0-0.7-0.2-1.3-0.7-1.8c-0.5-0.5-1.3-0.9-2.6-1.2l-5.2-1.2c-1.4-0.3-2.6-0.8-3.6-1.3c-0.9-0.6-1.6-1.3-2.1-2.1s-0.7-1.8-0.7-2.8
|
||||||
|
c0-1.3,0.4-2.6,1.1-3.7c0.7-1.1,1.8-2,3.2-2.6C147.3,14.1,148.9,13.8,150.9,13.8z"/>
|
||||||
|
</g>
|
||||||
|
</g>
|
||||||
|
</g>
|
||||||
|
</svg>
|
After Width: | Height: | Size: 5.5 KiB |
2
.github/workflows/tests.yml
vendored
2
.github/workflows/tests.yml
vendored
|
@ -68,7 +68,7 @@ jobs:
|
||||||
runs-on: ubuntu-20.04
|
runs-on: ubuntu-20.04
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go_versions: [ '1.17', '1.18.x', '1.19.x' ]
|
go_versions: [ '1.18.x', '1.19.x' ]
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v2
|
||||||
|
|
11
.gitignore
vendored
11
.gitignore
vendored
|
@ -3,7 +3,7 @@
|
||||||
.vscode
|
.vscode
|
||||||
|
|
||||||
# Tree service
|
# Tree service
|
||||||
internal/neofs/services/tree/
|
internal/frostfs/services/tree/
|
||||||
|
|
||||||
# Vendoring
|
# Vendoring
|
||||||
vendor
|
vendor
|
||||||
|
@ -21,4 +21,11 @@ coverage.txt
|
||||||
coverage.html
|
coverage.html
|
||||||
|
|
||||||
# debhelpers
|
# debhelpers
|
||||||
**/.debhelper
|
**/*debhelper*
|
||||||
|
|
||||||
|
# debian package build files
|
||||||
|
debian/files
|
||||||
|
debian/*.log
|
||||||
|
debian/*.substvars
|
||||||
|
debian/frostfs-s3-gw/
|
||||||
|
|
||||||
|
|
11
.gitlint
Normal file
11
.gitlint
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
[general]
|
||||||
|
fail-without-commits=True
|
||||||
|
regex-style-search=True
|
||||||
|
contrib=CC1
|
||||||
|
|
||||||
|
[title-match-regex]
|
||||||
|
regex=^\[\#[0-9Xx]+\]\s
|
||||||
|
|
||||||
|
[ignore-by-title]
|
||||||
|
regex=^Release(.*)
|
||||||
|
ignore=title-match-regex
|
|
@ -32,15 +32,12 @@ linters:
|
||||||
- revive
|
- revive
|
||||||
|
|
||||||
# some default golangci-lint linters
|
# some default golangci-lint linters
|
||||||
- deadcode
|
|
||||||
- errcheck
|
- errcheck
|
||||||
- gosimple
|
- gosimple
|
||||||
- ineffassign
|
- ineffassign
|
||||||
- staticcheck
|
- staticcheck
|
||||||
- structcheck
|
|
||||||
- typecheck
|
- typecheck
|
||||||
- unused
|
- unused
|
||||||
- varcheck
|
|
||||||
|
|
||||||
# extra linters
|
# extra linters
|
||||||
- exhaustive
|
- exhaustive
|
||||||
|
|
45
.pre-commit-config.yaml
Normal file
45
.pre-commit-config.yaml
Normal file
|
@ -0,0 +1,45 @@
|
||||||
|
ci:
|
||||||
|
autofix_prs: false
|
||||||
|
|
||||||
|
repos:
|
||||||
|
- repo: https://github.com/jorisroovers/gitlint
|
||||||
|
rev: v0.19.1
|
||||||
|
hooks:
|
||||||
|
- id: gitlint
|
||||||
|
stages: [commit-msg]
|
||||||
|
- id: gitlint-ci
|
||||||
|
|
||||||
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
|
rev: v4.4.0
|
||||||
|
hooks:
|
||||||
|
- id: check-added-large-files
|
||||||
|
- id: check-case-conflict
|
||||||
|
- id: check-executables-have-shebangs
|
||||||
|
- id: check-shebang-scripts-are-executable
|
||||||
|
- id: check-merge-conflict
|
||||||
|
- id: check-json
|
||||||
|
- id: check-xml
|
||||||
|
- id: check-yaml
|
||||||
|
- id: trailing-whitespace
|
||||||
|
args: [--markdown-linebreak-ext=md]
|
||||||
|
- id: end-of-file-fixer
|
||||||
|
exclude: ".key$"
|
||||||
|
|
||||||
|
- repo: https://github.com/shellcheck-py/shellcheck-py
|
||||||
|
rev: v0.9.0.2
|
||||||
|
hooks:
|
||||||
|
- id: shellcheck
|
||||||
|
|
||||||
|
- repo: https://github.com/golangci/golangci-lint
|
||||||
|
rev: v1.51.2
|
||||||
|
hooks:
|
||||||
|
- id: golangci-lint
|
||||||
|
|
||||||
|
- repo: local
|
||||||
|
hooks:
|
||||||
|
- id: go-unit-tests
|
||||||
|
name: go unit tests
|
||||||
|
entry: make test
|
||||||
|
pass_filenames: false
|
||||||
|
types: [go]
|
||||||
|
language: system
|
65
CHANGELOG.md
65
CHANGELOG.md
|
@ -4,10 +4,70 @@ This document outlines major changes between releases.
|
||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Get empty bucket CORS from frostfs (TrueCloudLab#36)
|
||||||
|
- Don't count pool error on client abort (#35)
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Return container name in `head-bucket` response (TrueCloudLab#18)
|
||||||
|
- Billing metrics (TrueCloudLab#5)
|
||||||
|
- Multiple configs support (TrueCloudLab#21)
|
||||||
|
- Bucket name resolving policy (TrueCloudLab#25)
|
||||||
|
- Support string `Action` and `Resource` fields in `bucketPolicy.Statement` (TrueCloudLab#32)
|
||||||
|
- Add new `kludge.use_default_xmlns_for_complete_multipart` config param (TrueCloudLab#40)
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- Update neo-go to v0.101.0 (#14)
|
||||||
|
- Update viper to v1.15.0 (#14)
|
||||||
|
- Using multiple servers require only one healthy (TrueCloudLab#12)
|
||||||
|
- Update go version to go1.18 (TrueCloudLab#16)
|
||||||
|
- Return error on invalid LocationConstraint (TrueCloudLab#23)
|
||||||
|
- Place billing metrics to separate url path (TrueCloudLab#26)
|
||||||
|
- Add generated deb builder files to .gitignore, and fix typo (TrueCloudLab#28)
|
||||||
|
- Limit number of objects to delete at one time (TrueCloudLab#37)
|
||||||
|
- CompleteMultipartUpload handler now sends whitespace characters to keep alive client's connection (#60)
|
||||||
|
- Support new system attributes (#64)
|
||||||
|
|
||||||
|
## [0.26.0] - 2022-12-28
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Use client time as `now` in some requests (#726)
|
||||||
|
- Reload policies on SIGHUP (#747)
|
||||||
|
- Authmate flags for pool timeouts (#760)
|
||||||
|
- Multiple server listeners (#742)
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- Placement policy configuration (#568)
|
||||||
|
- Improved debug logging of CID and OID values (#754)
|
||||||
|
|
||||||
|
### Removed
|
||||||
|
- Deprecated linters (#755)
|
||||||
|
|
||||||
|
### Updating from v0.25.1
|
||||||
|
New config parameters were added. And old one `defaul_policy` were changed.
|
||||||
|
```yaml
|
||||||
|
placement_policy:
|
||||||
|
default: "REP 3"
|
||||||
|
region_mapping: /path/to/container/policies.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Make sure you update the config accordingly:
|
||||||
|
If you configure application using environment variables change:
|
||||||
|
* `S3_GW_DEFAULT_POLICY` -> `S3_GW_PLACEMENT_POLICY_DEFAULT_POLICY`
|
||||||
|
* `S3_GW_LISTEN_ADDRESS` -> `S3_GW_SERVER_0_ADDRESS`
|
||||||
|
* `S3_GW_TLS_CERT_FILE` -> `S3_GW_SERVER_0_TLS_CERT_FILE` (and set `S3_GW_SERVER_0_TLS_ENABLED=true`)
|
||||||
|
* `S3_GW_TLS_KEY_FILE` -> `S3_GW_SERVER_0_TLS_KEY_FILE` (and set `S3_GW_SERVER_0_TLS_ENABLED=true`)
|
||||||
|
|
||||||
|
If you configure application using `.yaml` file change:
|
||||||
|
* `defaul_policy` -> `placement_policy.default`
|
||||||
|
* `listen_address` -> `server.0.address`
|
||||||
|
* `tls.cert_file` -> `server.0.tls.cert_file` (and set `server.0.tls.enabled: true`)
|
||||||
|
* `tls.key_file` -> `server.0.tls.key_file` (and set `server.0.tls.enabled: true`)
|
||||||
|
|
||||||
## [0.25.1] - 2022-10-30
|
## [0.25.1] - 2022-10-30
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
- Empty bucket policy (#740)
|
- Empty bucket policy (#740)
|
||||||
- Big object removal (#749)
|
- Big object removal (#749)
|
||||||
- Checksum panic (#741)
|
- Checksum panic (#741)
|
||||||
|
|
||||||
|
@ -371,5 +431,4 @@ releases.
|
||||||
[0.23.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.22.0...v0.23.0
|
[0.23.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.22.0...v0.23.0
|
||||||
[0.24.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.23.0...v0.24.0
|
[0.24.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.23.0...v0.24.0
|
||||||
[0.25.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.24.0...v0.25.0
|
[0.25.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.24.0...v0.25.0
|
||||||
[0.25.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.25.0...v0.25.1
|
[Unreleased]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.25.0...master
|
||||||
[Unreleased]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.25.1...master
|
|
||||||
|
|
|
@ -3,8 +3,8 @@
|
||||||
First, thank you for contributing! We love and encourage pull requests from
|
First, thank you for contributing! We love and encourage pull requests from
|
||||||
everyone. Please follow the guidelines:
|
everyone. Please follow the guidelines:
|
||||||
|
|
||||||
- Check the open [issues](https://github.com/nspcc-dev/neofs-s3-gw/issues) and
|
- Check the open [issues](https://github.com/TrueCloudLab/frostfs-s3-gw/issues) and
|
||||||
[pull requests](https://github.com/nspcc-dev/neofs-s3-gw/pulls) for existing
|
[pull requests](https://github.com/TrueCloudLab/frostfs-s3-gw/pulls) for existing
|
||||||
discussions.
|
discussions.
|
||||||
|
|
||||||
- Open an issue first, to discuss a new feature or enhancement.
|
- Open an issue first, to discuss a new feature or enhancement.
|
||||||
|
@ -23,24 +23,24 @@ everyone. Please follow the guidelines:
|
||||||
|
|
||||||
## Development Workflow
|
## Development Workflow
|
||||||
|
|
||||||
Start by forking the `neofs-s3-gw` repository, make changes in a branch and then
|
Start by forking the `frostfs-s3-gw` repository, make changes in a branch and then
|
||||||
send a pull request. We encourage pull requests to discuss code changes. Here
|
send a pull request. We encourage pull requests to discuss code changes. Here
|
||||||
are the steps in details:
|
are the steps in details:
|
||||||
|
|
||||||
### Set up your GitHub Repository
|
### Set up your GitHub Repository
|
||||||
Fork [NeoFS S3 Gateway
|
Fork [FrostFS S3 Gateway
|
||||||
upstream](https://github.com/nspcc-dev/neofs-s3-gw/fork) source repository
|
upstream](https://github.com/TrueCloudLab/frostfs-s3-gw/fork) source repository
|
||||||
to your own personal repository. Copy the URL of your fork (you will need it for
|
to your own personal repository. Copy the URL of your fork (you will need it for
|
||||||
the `git clone` command below).
|
the `git clone` command below).
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
$ git clone https://github.com/nspcc-dev/neofs-s3-gw
|
$ git clone https://github.com/TrueCloudLab/frostfs-s3-gw
|
||||||
```
|
```
|
||||||
|
|
||||||
### Set up git remote as ``upstream``
|
### Set up git remote as ``upstream``
|
||||||
```sh
|
```sh
|
||||||
$ cd neofs-s3-gw
|
$ cd frostfs-s3-gw
|
||||||
$ git remote add upstream https://github.com/nspcc-dev/neofs-s3-gw
|
$ git remote add upstream https://github.com/TrueCloudLab/frostfs-s3-gw
|
||||||
$ git fetch upstream
|
$ git fetch upstream
|
||||||
$ git merge upstream/master
|
$ git merge upstream/master
|
||||||
...
|
...
|
||||||
|
@ -107,7 +107,7 @@ contributors".
|
||||||
To sign your work, just add a line like this at the end of your commit message:
|
To sign your work, just add a line like this at the end of your commit message:
|
||||||
|
|
||||||
```
|
```
|
||||||
Signed-off-by: Samii Sakisaka <samii@nspcc.ru>
|
Signed-off-by: Samii Sakisaka <samii@frostfs.info>
|
||||||
```
|
```
|
||||||
|
|
||||||
This can be easily done with the `--signoff` option to `git commit`.
|
This can be easily done with the `--signoff` option to `git commit`.
|
||||||
|
|
26
Makefile
Normal file → Executable file
26
Makefile
Normal file → Executable file
|
@ -8,22 +8,22 @@ LINT_VERSION ?= 1.49.0
|
||||||
BINDIR = bin
|
BINDIR = bin
|
||||||
|
|
||||||
# Binaries to build
|
# Binaries to build
|
||||||
CMDS = $(addprefix neofs-, $(notdir $(wildcard cmd/*)))
|
CMDS = $(addprefix frostfs-, $(notdir $(wildcard cmd/*)))
|
||||||
BINS = $(addprefix $(BINDIR)/, $(CMDS))
|
BINS = $(addprefix $(BINDIR)/, $(CMDS))
|
||||||
|
|
||||||
# Variables for docker
|
# Variables for docker
|
||||||
REPO_BASENAME = $(shell basename `go list -m`)
|
REPO_BASENAME = $(shell basename `go list -m`)
|
||||||
HUB_IMAGE ?= "nspccdev/$(REPO_BASENAME)"
|
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
|
||||||
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
||||||
|
|
||||||
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint version clean protoc
|
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
|
||||||
|
|
||||||
# .deb package versioning
|
# .deb package versioning
|
||||||
OS_RELEASE = $(shell lsb_release -cs)
|
OS_RELEASE = $(shell lsb_release -cs)
|
||||||
PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
|
PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
|
||||||
sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \
|
sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \
|
||||||
sed "s/-/~/")-${OS_RELEASE}
|
sed "s/-/~/")-${OS_RELEASE}
|
||||||
.PHONY: debpackage debclean
|
.PHONY: debpackage debclean
|
||||||
|
|
||||||
# Make all binaries
|
# Make all binaries
|
||||||
all: $(BINS)
|
all: $(BINS)
|
||||||
|
@ -33,7 +33,7 @@ $(BINS): sync-tree $(BINDIR) dep
|
||||||
CGO_ENABLED=0 \
|
CGO_ENABLED=0 \
|
||||||
go build -v -trimpath \
|
go build -v -trimpath \
|
||||||
-ldflags "-X $(REPO)/internal/version.Version=$(VERSION)" \
|
-ldflags "-X $(REPO)/internal/version.Version=$(VERSION)" \
|
||||||
-o $@ ./cmd/$(subst neofs-,,$(notdir $@))
|
-o $@ ./cmd/$(subst frostfs-,,$(notdir $@))
|
||||||
|
|
||||||
$(BINDIR):
|
$(BINDIR):
|
||||||
@echo "⇒ Ensure dir: $@"
|
@echo "⇒ Ensure dir: $@"
|
||||||
|
@ -80,7 +80,7 @@ format:
|
||||||
|
|
||||||
# Build clean Docker image
|
# Build clean Docker image
|
||||||
image:
|
image:
|
||||||
@echo "⇒ Build NeoFS S3 Gateway docker image "
|
@echo "⇒ Build FrostFS S3 Gateway docker image "
|
||||||
@docker build \
|
@docker build \
|
||||||
--build-arg REPO=$(REPO) \
|
--build-arg REPO=$(REPO) \
|
||||||
--build-arg VERSION=$(VERSION) \
|
--build-arg VERSION=$(VERSION) \
|
||||||
|
@ -95,7 +95,7 @@ image-push:
|
||||||
|
|
||||||
# Build dirty Docker image
|
# Build dirty Docker image
|
||||||
dirty-image:
|
dirty-image:
|
||||||
@echo "⇒ Build NeoFS S3 Gateway dirty docker image "
|
@echo "⇒ Build FrostFS S3 Gateway dirty docker image "
|
||||||
@docker build \
|
@docker build \
|
||||||
--build-arg REPO=$(REPO) \
|
--build-arg REPO=$(REPO) \
|
||||||
--build-arg VERSION=$(VERSION) \
|
--build-arg VERSION=$(VERSION) \
|
||||||
|
@ -115,6 +115,14 @@ docker/lint:
|
||||||
--env HOME=/src \
|
--env HOME=/src \
|
||||||
golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint'
|
golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint'
|
||||||
|
|
||||||
|
# Activate pre-commit hooks
|
||||||
|
pre-commit:
|
||||||
|
pre-commit install -t pre-commit -t commit-msg
|
||||||
|
|
||||||
|
# Deactivate pre-commit hooks
|
||||||
|
unpre-commit:
|
||||||
|
pre-commit uninstall -t pre-commit -t commit-msg
|
||||||
|
|
||||||
# Show current version
|
# Show current version
|
||||||
version:
|
version:
|
||||||
@echo $(VERSION)
|
@echo $(VERSION)
|
||||||
|
@ -135,7 +143,7 @@ protoc:
|
||||||
|
|
||||||
# Package for Debian
|
# Package for Debian
|
||||||
debpackage:
|
debpackage:
|
||||||
dch --package neofs-s3-gw \
|
dch --package frostfs-s3-gw \
|
||||||
--controlmaint \
|
--controlmaint \
|
||||||
--newversion $(PKG_VERSION) \
|
--newversion $(PKG_VERSION) \
|
||||||
--distribution $(OS_RELEASE) \
|
--distribution $(OS_RELEASE) \
|
||||||
|
@ -143,6 +151,6 @@ debpackage:
|
||||||
dpkg-buildpackage --no-sign -b
|
dpkg-buildpackage --no-sign -b
|
||||||
|
|
||||||
debclean:
|
debclean:
|
||||||
dh clean
|
dh clean
|
||||||
|
|
||||||
include help.mk
|
include help.mk
|
||||||
|
|
40
README.md
40
README.md
|
@ -1,13 +1,23 @@
|
||||||
# NeoFS S3 Gateway
|
<p align="center">
|
||||||
|
<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
|
||||||
|
</p>
|
||||||
|
<p align="center">
|
||||||
|
<a href="https://frostfs.info">FrostFS</a> is a decentralized distributed object storage integrated with the <a href="https://neo.org">NEO Blockchain</a>.
|
||||||
|
</p>
|
||||||
|
|
||||||
NeoFS S3 gateway provides API compatible with Amazon S3 cloud storage service.
|
---
|
||||||
|
[![Report](https://goreportcard.com/badge/git.frostfs.info/TrueCloudLab/frostfs-s3-gw)](https://goreportcard.com/report/git.frostfs.info/TrueCloudLab/frostfs-s3-gw)
|
||||||
|
|
||||||
|
# FrostFS S3 Gateway
|
||||||
|
|
||||||
|
FrostFS S3 gateway provides API compatible with Amazon S3 cloud storage service.
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
```go get -u github.com/nspcc-dev/neofs-s3-gw```
|
```go get -u git.frostfs.info/TrueCloudLab/frostfs-s3-gw```
|
||||||
|
|
||||||
Or you can call `make` to build it from the cloned repository (the binary will
|
Or you can call `make` to build it from the cloned repository (the binary will
|
||||||
end up in `bin/neofs-s3-gw` with authmate helper in `bin/neofs-s3-authmate`).
|
end up in `bin/frostfs-s3-gw` with authmate helper in `bin/frostfs-s3-authmate`).
|
||||||
To build binaries in clean docker environment, call `make docker/all`.
|
To build binaries in clean docker environment, call `make docker/all`.
|
||||||
|
|
||||||
Other notable make targets:
|
Other notable make targets:
|
||||||
|
@ -22,36 +32,36 @@ version Show current version
|
||||||
```
|
```
|
||||||
|
|
||||||
Or you can also use a [Docker
|
Or you can also use a [Docker
|
||||||
image](https://hub.docker.com/r/nspccdev/neofs-s3-gw) provided for released
|
image](https://hub.docker.com/r/truecloudlab/frostfs-s3-gw) provided for released
|
||||||
(and occasionally unreleased) versions of gateway (`:latest` points to the
|
(and occasionally unreleased) versions of gateway (`:latest` points to the
|
||||||
latest stable release).
|
latest stable release).
|
||||||
|
|
||||||
## Execution
|
## Execution
|
||||||
|
|
||||||
Minimalistic S3 gateway setup needs:
|
Minimalistic S3 gateway setup needs:
|
||||||
* NeoFS node(s) address (S3 gateway itself is not a NeoFS node)
|
* FrostFS node(s) address (S3 gateway itself is not a FrostFS node)
|
||||||
Passed via `-p` parameter or via `S3_GW_PEERS_<N>_ADDRESS` and
|
Passed via `-p` parameter or via `S3_GW_PEERS_<N>_ADDRESS` and
|
||||||
`S3_GW_PEERS_<N>_WEIGHT` environment variables (gateway supports multiple
|
`S3_GW_PEERS_<N>_WEIGHT` environment variables (gateway supports multiple
|
||||||
NeoFS nodes with weighted load balancing).
|
FrostFS nodes with weighted load balancing).
|
||||||
* a wallet used to fetch key and communicate with NeoFS nodes
|
* a wallet used to fetch key and communicate with FrostFS nodes
|
||||||
Passed via `--wallet` parameter or `S3_GW_WALLET_PATH` environment variable.
|
Passed via `--wallet` parameter or `S3_GW_WALLET_PATH` environment variable.
|
||||||
|
|
||||||
These two commands are functionally equivalent, they run the gate with one
|
These two commands are functionally equivalent, they run the gate with one
|
||||||
backend node, some keys and otherwise default settings:
|
backend node, some keys and otherwise default settings:
|
||||||
```
|
```
|
||||||
$ neofs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json
|
$ frostfs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json
|
||||||
|
|
||||||
$ S3_GW_PEERS_0_ADDRESS=192.168.130.72:8080 \
|
$ S3_GW_PEERS_0_ADDRESS=192.168.130.72:8080 \
|
||||||
S3_GW_WALLET=wallet.json \
|
S3_GW_WALLET=wallet.json \
|
||||||
neofs-s3-gw
|
frostfs-s3-gw
|
||||||
```
|
```
|
||||||
It's also possible to specify uri scheme (grpc or grpcs) when using `-p` or environment variables:
|
It's also possible to specify uri scheme (grpc or grpcs) when using `-p` or environment variables:
|
||||||
```
|
```
|
||||||
$ neofs-s3-gw -p grpc://192.168.130.72:8080 --wallet wallet.json
|
$ frostfs-s3-gw -p grpc://192.168.130.72:8080 --wallet wallet.json
|
||||||
|
|
||||||
$ S3_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 \
|
$ S3_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 \
|
||||||
S3_GW_WALLET=wallet.json \
|
S3_GW_WALLET=wallet.json \
|
||||||
neofs-s3-gw
|
frostfs-s3-gw
|
||||||
```
|
```
|
||||||
|
|
||||||
## Domains
|
## Domains
|
||||||
|
@ -60,7 +70,7 @@ By default, s3-gw enable only `path-style access`.
|
||||||
To be able to use both: `virtual-hosted-style` and `path-style` access you must configure `listen_domains`:
|
To be able to use both: `virtual-hosted-style` and `path-style` access you must configure `listen_domains`:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
$ neofs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json --listen_domains your.first.domain --listen_domains your.second.domain
|
$ frostfs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json --listen_domains your.first.domain --listen_domains your.second.domain
|
||||||
```
|
```
|
||||||
|
|
||||||
So now you can use (e.g. `HeadBucket`. Make sure DNS is properly configured):
|
So now you can use (e.g. `HeadBucket`. Make sure DNS is properly configured):
|
||||||
|
@ -84,8 +94,8 @@ Also, you can configure domains using `.env` variables or `yaml` file.
|
||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
- [Configuration](./docs/configuration.md)
|
- [Configuration](./docs/configuration.md)
|
||||||
- [NeoFS S3 AuthMate](./docs/authmate.md)
|
- [FrostFS S3 AuthMate](./docs/authmate.md)
|
||||||
- [NeoFS Tree service](./docs/tree_service.md)
|
- [FrostFS Tree service](./docs/tree_service.md)
|
||||||
- [AWS CLI basic usage](./docs/aws_cli.md)
|
- [AWS CLI basic usage](./docs/aws_cli.md)
|
||||||
- [AWS S3 API compatibility](./docs/aws_s3_compat.md)
|
- [AWS S3 API compatibility](./docs/aws_s3_compat.md)
|
||||||
- [AWS S3 Compatibility test results](./docs/s3_test_results.md)
|
- [AWS S3 Compatibility test results](./docs/s3_test_results.md)
|
||||||
|
|
2
VERSION
2
VERSION
|
@ -1 +1 @@
|
||||||
v0.25.1
|
v0.26.0
|
||||||
|
|
|
@ -14,14 +14,14 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
|
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
v4 "github.com/nspcc-dev/neofs-s3-gw/api/auth/signer/v4"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
|
||||||
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
|
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
|
// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
|
||||||
|
@ -33,7 +33,13 @@ var postPolicyCredentialRegexp = regexp.MustCompile(`(?P<access_key_id>[^/]+)/(?
|
||||||
type (
|
type (
|
||||||
// Center is a user authentication interface.
|
// Center is a user authentication interface.
|
||||||
Center interface {
|
Center interface {
|
||||||
Authenticate(request *http.Request) (*accessbox.Box, error)
|
Authenticate(request *http.Request) (*Box, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Box contains access box and additional info.
|
||||||
|
Box struct {
|
||||||
|
AccessBox *accessbox.Box
|
||||||
|
ClientTime time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
center struct {
|
center struct {
|
||||||
|
@ -86,9 +92,9 @@ func (p prs) Seek(_ int64, _ int) (int64, error) {
|
||||||
var _ io.ReadSeeker = prs(0)
|
var _ io.ReadSeeker = prs(0)
|
||||||
|
|
||||||
// New creates an instance of AuthCenter.
|
// New creates an instance of AuthCenter.
|
||||||
func New(neoFS tokens.NeoFS, key *keys.PrivateKey, prefixes []string, config *cache.Config) Center {
|
func New(frostFS tokens.FrostFS, key *keys.PrivateKey, prefixes []string, config *cache.Config) Center {
|
||||||
return ¢er{
|
return ¢er{
|
||||||
cli: tokens.New(neoFS, key, config),
|
cli: tokens.New(frostFS, key, config),
|
||||||
reg: NewRegexpMatcher(authorizationFieldRegexp),
|
reg: NewRegexpMatcher(authorizationFieldRegexp),
|
||||||
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
|
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
|
||||||
allowedAccessKeyIDPrefixes: prefixes,
|
allowedAccessKeyIDPrefixes: prefixes,
|
||||||
|
@ -126,11 +132,12 @@ func (a *authHeader) getAddress() (oid.Address, error) {
|
||||||
return addr, nil
|
return addr, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *center) Authenticate(r *http.Request) (*accessbox.Box, error) {
|
func (c *center) Authenticate(r *http.Request) (*Box, error) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
authHdr *authHeader
|
authHdr *authHeader
|
||||||
signatureDateTimeStr string
|
signatureDateTimeStr string
|
||||||
|
needClientTime bool
|
||||||
)
|
)
|
||||||
|
|
||||||
queryValues := r.URL.Query()
|
queryValues := r.URL.Query()
|
||||||
|
@ -166,6 +173,7 @@ func (c *center) Authenticate(r *http.Request) (*accessbox.Box, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
signatureDateTimeStr = r.Header.Get(AmzDate)
|
signatureDateTimeStr = r.Header.Get(AmzDate)
|
||||||
|
needClientTime = true
|
||||||
}
|
}
|
||||||
|
|
||||||
signatureDateTime, err := time.Parse("20060102T150405Z", signatureDateTimeStr)
|
signatureDateTime, err := time.Parse("20060102T150405Z", signatureDateTimeStr)
|
||||||
|
@ -192,7 +200,12 @@ func (c *center) Authenticate(r *http.Request) (*accessbox.Box, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return box, nil
|
result := &Box{AccessBox: box}
|
||||||
|
if needClientTime {
|
||||||
|
result.ClientTime = signatureDateTime
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c center) checkAccessKeyID(accessKeyID string) error {
|
func (c center) checkAccessKeyID(accessKeyID string) error {
|
||||||
|
@ -209,7 +222,7 @@ func (c center) checkAccessKeyID(accessKeyID string) error {
|
||||||
return apiErrors.GetAPIError(apiErrors.ErrAccessDenied)
|
return apiErrors.GetAPIError(apiErrors.ErrAccessDenied)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *center) checkFormData(r *http.Request) (*accessbox.Box, error) {
|
func (c *center) checkFormData(r *http.Request) (*Box, error) {
|
||||||
if err := r.ParseMultipartForm(maxFormSizeMemory); err != nil {
|
if err := r.ParseMultipartForm(maxFormSizeMemory); err != nil {
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidArgument)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidArgument)
|
||||||
}
|
}
|
||||||
|
@ -251,7 +264,7 @@ func (c *center) checkFormData(r *http.Request) (*accessbox.Box, error) {
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch)
|
||||||
}
|
}
|
||||||
|
|
||||||
return box, nil
|
return &Box{AccessBox: box}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneRequest(r *http.Request, authHeader *authHeader) *http.Request {
|
func cloneRequest(r *http.Request, authHeader *authHeader) *http.Request {
|
||||||
|
|
|
@ -5,7 +5,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
2
api/cache/access_control.go
vendored
2
api/cache/access_control.go
vendored
|
@ -4,8 +4,8 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
4
api/cache/accessbox.go
vendored
4
api/cache/accessbox.go
vendored
|
@ -4,9 +4,9 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
2
api/cache/buckets.go
vendored
2
api/cache/buckets.go
vendored
|
@ -4,8 +4,8 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
8
api/cache/cache_test.go
vendored
8
api/cache/cache_test.go
vendored
|
@ -3,10 +3,10 @@ package cache
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
|
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||||
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
|
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
"go.uber.org/zap/zaptest/observer"
|
"go.uber.org/zap/zaptest/observer"
|
||||||
|
|
2
api/cache/names.go
vendored
2
api/cache/names.go
vendored
|
@ -4,8 +4,8 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
4
api/cache/objects.go
vendored
4
api/cache/objects.go
vendored
|
@ -4,9 +4,9 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
6
api/cache/objects_test.go
vendored
6
api/cache/objects_test.go
vendored
|
@ -4,9 +4,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test"
|
objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
6
api/cache/objectslist.go
vendored
6
api/cache/objectslist.go
vendored
|
@ -6,9 +6,9 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -20,7 +20,7 @@ import (
|
||||||
After putting a record, it lives for a while (default value is 60 seconds).
|
After putting a record, it lives for a while (default value is 60 seconds).
|
||||||
|
|
||||||
When we receive a request from a user, we try to find the suitable and non-expired cache entry, go through the list
|
When we receive a request from a user, we try to find the suitable and non-expired cache entry, go through the list
|
||||||
and get ObjectInfos from common object cache or with a request to NeoFS.
|
and get ObjectInfos from common object cache or with a request to FrostFS.
|
||||||
|
|
||||||
When we put an object into a container, we invalidate entries with prefixes that are prefixes of the object's name.
|
When we put an object into a container, we invalidate entries with prefixes that are prefixes of the object's name.
|
||||||
*/
|
*/
|
||||||
|
|
6
api/cache/objectslist_test.go
vendored
6
api/cache/objectslist_test.go
vendored
|
@ -4,9 +4,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
|
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||||
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
|
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
2
api/cache/system.go
vendored
2
api/cache/system.go
vendored
|
@ -4,8 +4,8 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -4,9 +4,9 @@ import (
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -22,7 +22,8 @@ const (
|
||||||
type (
|
type (
|
||||||
// BucketInfo stores basic bucket data.
|
// BucketInfo stores basic bucket data.
|
||||||
BucketInfo struct {
|
BucketInfo struct {
|
||||||
Name string
|
Name string // container name from system attribute
|
||||||
|
Zone string // container zone from system attribute
|
||||||
CID cid.ID
|
CID cid.ID
|
||||||
Owner user.ID
|
Owner user.ID
|
||||||
Created time.Time
|
Created time.Time
|
||||||
|
|
|
@ -4,9 +4,9 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -25,7 +25,7 @@ func (v NodeVersion) IsDeleteMarker() bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteMarkerInfo is used to save object info if node in the tree service is delete marker.
|
// DeleteMarkerInfo is used to save object info if node in the tree service is delete marker.
|
||||||
// We need this information because the "delete marker" object is no longer stored in NeoFS.
|
// We need this information because the "delete marker" object is no longer stored in FrostFS.
|
||||||
type DeleteMarkerInfo struct {
|
type DeleteMarkerInfo struct {
|
||||||
Created time.Time
|
Created time.Time
|
||||||
Owner user.ID
|
Owner user.ID
|
||||||
|
|
|
@ -90,6 +90,7 @@ const (
|
||||||
ErrMissingFields
|
ErrMissingFields
|
||||||
ErrMissingCredTag
|
ErrMissingCredTag
|
||||||
ErrCredMalformed
|
ErrCredMalformed
|
||||||
|
ErrInvalidLocationConstraint
|
||||||
ErrInvalidRegion
|
ErrInvalidRegion
|
||||||
ErrInvalidServiceS3
|
ErrInvalidServiceS3
|
||||||
ErrInvalidServiceSTS
|
ErrInvalidServiceSTS
|
||||||
|
@ -680,6 +681,12 @@ var errorCodes = errorCodeMap{
|
||||||
Description: "Error parsing the X-Amz-Credential parameter; the region is wrong;",
|
Description: "Error parsing the X-Amz-Credential parameter; the region is wrong;",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrInvalidLocationConstraint: {
|
||||||
|
ErrCode: ErrInvalidLocationConstraint,
|
||||||
|
Code: "InvalidLocationConstraint",
|
||||||
|
Description: "The specified location (Region) constraint is not valid.",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrInvalidRegion: {
|
ErrInvalidRegion: {
|
||||||
ErrCode: ErrInvalidRegion,
|
ErrCode: ErrInvalidRegion,
|
||||||
Code: "InvalidRegion",
|
Code: "InvalidRegion",
|
||||||
|
@ -986,7 +993,7 @@ var errorCodes = errorCodeMap{
|
||||||
ErrNotSupported: {
|
ErrNotSupported: {
|
||||||
ErrCode: ErrNotSupported,
|
ErrCode: ErrNotSupported,
|
||||||
Code: "BadRequest",
|
Code: "BadRequest",
|
||||||
Description: "Not supported by NeoFS S3 Gateway",
|
Description: "Not supported by FrostFS S3 Gateway",
|
||||||
HTTPStatusCode: http.StatusNotImplemented,
|
HTTPStatusCode: http.StatusNotImplemented,
|
||||||
},
|
},
|
||||||
ErrInvalidEncryptionMethod: {
|
ErrInvalidEncryptionMethod: {
|
||||||
|
|
|
@ -14,16 +14,16 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
v2acl "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
v2acl "github.com/nspcc-dev/neofs-api-go/v2/acl"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -158,6 +158,90 @@ func (s ServiceRecord) ToEACLRecord() *eacl.Record {
|
||||||
return serviceRecord
|
return serviceRecord
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
errInvalidStatement = stderrors.New("invalid statement")
|
||||||
|
errInvalidPrincipal = stderrors.New("invalid principal")
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s *statement) UnmarshalJSON(data []byte) error {
|
||||||
|
var statementMap map[string]interface{}
|
||||||
|
if err := json.Unmarshal(data, &statementMap); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
sidField, ok := statementMap["Sid"]
|
||||||
|
if ok {
|
||||||
|
if s.Sid, ok = sidField.(string); !ok {
|
||||||
|
return errInvalidStatement
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
effectField, ok := statementMap["Effect"]
|
||||||
|
if ok {
|
||||||
|
if s.Effect, ok = effectField.(string); !ok {
|
||||||
|
return errInvalidStatement
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
principalField, ok := statementMap["Principal"]
|
||||||
|
if ok {
|
||||||
|
principalMap, ok := principalField.(map[string]interface{})
|
||||||
|
if !ok {
|
||||||
|
return errInvalidPrincipal
|
||||||
|
}
|
||||||
|
|
||||||
|
awsField, ok := principalMap["AWS"]
|
||||||
|
if ok {
|
||||||
|
if s.Principal.AWS, ok = awsField.(string); !ok {
|
||||||
|
return fmt.Errorf("%w: 'AWS' field must be string", errInvalidPrincipal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
canonicalUserField, ok := principalMap["CanonicalUser"]
|
||||||
|
if ok {
|
||||||
|
if s.Principal.CanonicalUser, ok = canonicalUserField.(string); !ok {
|
||||||
|
return errInvalidPrincipal
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
actionField, ok := statementMap["Action"]
|
||||||
|
if ok {
|
||||||
|
switch actionField := actionField.(type) {
|
||||||
|
case []interface{}:
|
||||||
|
s.Action = make([]string, len(actionField))
|
||||||
|
for i, action := range actionField {
|
||||||
|
if s.Action[i], ok = action.(string); !ok {
|
||||||
|
return errInvalidStatement
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case string:
|
||||||
|
s.Action = []string{actionField}
|
||||||
|
default:
|
||||||
|
return errInvalidStatement
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resourceField, ok := statementMap["Resource"]
|
||||||
|
if ok {
|
||||||
|
switch resourceField := resourceField.(type) {
|
||||||
|
case []interface{}:
|
||||||
|
s.Resource = make([]string, len(resourceField))
|
||||||
|
for i, action := range resourceField {
|
||||||
|
if s.Resource[i], ok = action.(string); !ok {
|
||||||
|
return errInvalidStatement
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case string:
|
||||||
|
s.Resource = []string{resourceField}
|
||||||
|
default:
|
||||||
|
return errInvalidStatement
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
reqInfo := api.GetReqInfo(r.Context())
|
reqInfo := api.GetReqInfo(r.Context())
|
||||||
|
|
||||||
|
|
|
@ -13,15 +13,15 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -1352,6 +1352,85 @@ func TestBucketPolicy(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBucketPolicyUnmarshal(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
policy string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "action/resource array",
|
||||||
|
policy: `
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Principal": {
|
||||||
|
"AWS": "arn:aws:iam::111122223333:role/JohnDoe"
|
||||||
|
},
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": [
|
||||||
|
"s3:GetObject",
|
||||||
|
"s3:GetObjectVersion"
|
||||||
|
],
|
||||||
|
"Resource": [
|
||||||
|
"arn:aws:s3:::DOC-EXAMPLE-BUCKET/*",
|
||||||
|
"arn:aws:s3:::DOC-EXAMPLE-BUCKET2/*"
|
||||||
|
]
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "action/resource string",
|
||||||
|
policy: `
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Principal": {
|
||||||
|
"AWS": "arn:aws:iam::111122223333:role/JohnDoe"
|
||||||
|
},
|
||||||
|
"Effect": "Deny",
|
||||||
|
"Action": "s3:GetObject",
|
||||||
|
"Resource": "arn:aws:s3:::DOC-EXAMPLE-BUCKET/*"
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
bktPolicy := &bucketPolicy{}
|
||||||
|
err := json.Unmarshal([]byte(tc.policy), bktPolicy)
|
||||||
|
require.NoError(t, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPutBucketPolicy(t *testing.T) {
|
||||||
|
bktPolicy := `
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Principal": {
|
||||||
|
"AWS": "*"
|
||||||
|
},
|
||||||
|
"Effect": "Deny",
|
||||||
|
"Action": "s3:GetObject",
|
||||||
|
"Resource": "arn:aws:s3:::bucket-for-policy/*"
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
`
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
bktName := "bucket-for-policy"
|
||||||
|
|
||||||
|
box, _ := createAccessBox(t)
|
||||||
|
createBucket(t, hc, bktName, box)
|
||||||
|
|
||||||
|
w, r := prepareTestPayloadRequest(hc, bktName, "", bytes.NewReader([]byte(bktPolicy)))
|
||||||
|
ctx := context.WithValue(r.Context(), api.BoxData, box)
|
||||||
|
r = r.WithContext(ctx)
|
||||||
|
hc.Handler().PutBucketPolicyHandler(w, r)
|
||||||
|
assertStatus(hc.t, w, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
func getBucketPolicy(hc *handlerContext, bktName string) *bucketPolicy {
|
func getBucketPolicy(hc *handlerContext, bktName string) *bucketPolicy {
|
||||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||||
hc.Handler().GetBucketPolicyHandler(w, r)
|
hc.Handler().GetBucketPolicyHandler(w, r)
|
||||||
|
|
|
@ -1,11 +1,14 @@
|
||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/xml"
|
||||||
"errors"
|
"errors"
|
||||||
|
"io"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -19,21 +22,33 @@ type (
|
||||||
|
|
||||||
Notificator interface {
|
Notificator interface {
|
||||||
SendNotifications(topics map[string]string, p *SendNotificationParams) error
|
SendNotifications(topics map[string]string, p *SendNotificationParams) error
|
||||||
SendTestNotification(topic, bucketName, requestID, HostID string) error
|
SendTestNotification(topic, bucketName, requestID, HostID string, now time.Time) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config contains data which handler needs to keep.
|
// Config contains data which handler needs to keep.
|
||||||
Config struct {
|
Config struct {
|
||||||
DefaultPolicy netmap.PlacementPolicy
|
Policy PlacementPolicy
|
||||||
DefaultMaxAge int
|
XMLDecoder XMLDecoderProvider
|
||||||
NotificatorEnabled bool
|
DefaultMaxAge int
|
||||||
TLSEnabled bool
|
NotificatorEnabled bool
|
||||||
CopiesNumber uint32
|
CopiesNumber uint32
|
||||||
|
ResolveZoneList []string
|
||||||
|
IsResolveListAllow bool // True if ResolveZoneList contains allowed zones
|
||||||
|
CompleteMultipartKeepalive time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
PlacementPolicy interface {
|
||||||
|
Default() netmap.PlacementPolicy
|
||||||
|
Get(string) (netmap.PlacementPolicy, bool)
|
||||||
|
}
|
||||||
|
|
||||||
|
XMLDecoderProvider interface {
|
||||||
|
NewCompleteMultipartDecoder(io.Reader) *xml.Decoder
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// DefaultPolicy is a default policy of placing containers in NeoFS if it's not set at the request.
|
// DefaultPolicy is a default policy of placing containers in FrostFS if it's not set at the request.
|
||||||
DefaultPolicy = "REP 3"
|
DefaultPolicy = "REP 3"
|
||||||
// DefaultCopiesNumber is a default number of object copies that is enough to consider put successful if it's not set in config.
|
// DefaultCopiesNumber is a default number of object copies that is enough to consider put successful if it's not set in config.
|
||||||
DefaultCopiesNumber uint32 = 0
|
DefaultCopiesNumber uint32 = 0
|
||||||
|
@ -45,7 +60,7 @@ var _ api.Handler = (*handler)(nil)
|
||||||
func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg *Config) (api.Handler, error) {
|
func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg *Config) (api.Handler, error) {
|
||||||
switch {
|
switch {
|
||||||
case obj == nil:
|
case obj == nil:
|
||||||
return nil, errors.New("empty NeoFS Object Layer")
|
return nil, errors.New("empty FrostFS Object Layer")
|
||||||
case log == nil:
|
case log == nil:
|
||||||
return nil, errors.New("empty logger")
|
return nil, errors.New("empty logger")
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,10 +6,10 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -94,7 +94,7 @@ func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Requ
|
||||||
}
|
}
|
||||||
info := extendedInfo.ObjectInfo
|
info := extendedInfo.ObjectInfo
|
||||||
|
|
||||||
encryptionParams, err := h.formEncryptionParams(r.Header)
|
encryptionParams, err := formEncryptionParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
|
|
@ -4,7 +4,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -6,12 +6,12 @@ import (
|
||||||
"regexp"
|
"regexp"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -142,7 +142,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
encryptionParams, err := h.formEncryptionParams(r.Header)
|
encryptionParams, err := formEncryptionParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -184,7 +184,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
CopiesNuber: copiesNumber,
|
CopiesNuber: copiesNumber,
|
||||||
}
|
}
|
||||||
|
|
||||||
params.Lock, err = formObjectLock(dstBktInfo, settings.LockConfiguration, r.Header)
|
params.Lock, err = formObjectLock(r.Context(), dstBktInfo, settings.LockConfiguration, r.Header)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not form object lock", reqInfo, err)
|
h.logAndSendError(w, "could not form object lock", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
|
|
@ -6,7 +6,7 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -5,9 +5,9 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
41
api/handler/cors_test.go
Normal file
41
api/handler/cors_test.go
Normal file
|
@ -0,0 +1,41 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCORSOriginWildcard(t *testing.T) {
|
||||||
|
body := `
|
||||||
|
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
<CORSRule>
|
||||||
|
<AllowedMethod>GET</AllowedMethod>
|
||||||
|
<AllowedOrigin>*</AllowedOrigin>
|
||||||
|
</CORSRule>
|
||||||
|
</CORSConfiguration>
|
||||||
|
`
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
bktName := "bucket-for-cors"
|
||||||
|
box, _ := createAccessBox(t)
|
||||||
|
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||||
|
ctx := context.WithValue(r.Context(), api.BoxData, box)
|
||||||
|
r = r.WithContext(ctx)
|
||||||
|
r.Header.Add(api.AmzACL, "public-read")
|
||||||
|
hc.Handler().CreateBucketHandler(w, r)
|
||||||
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
|
||||||
|
w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
|
||||||
|
ctx = context.WithValue(r.Context(), api.BoxData, box)
|
||||||
|
r = r.WithContext(ctx)
|
||||||
|
hc.Handler().PutBucketCorsHandler(w, r)
|
||||||
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
|
||||||
|
w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
|
||||||
|
hc.Handler().GetBucketCorsHandler(w, r)
|
||||||
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
}
|
|
@ -6,17 +6,20 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
|
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
"go.uber.org/zap/zapcore"
|
"go.uber.org/zap/zapcore"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// limitation of AWS https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
|
||||||
|
const maxObjectsToDelete = 1000
|
||||||
|
|
||||||
// DeleteObjectsRequest -- xml carrying the object key names which should be deleted.
|
// DeleteObjectsRequest -- xml carrying the object key names which should be deleted.
|
||||||
type DeleteObjectsRequest struct {
|
type DeleteObjectsRequest struct {
|
||||||
// Element to enable quiet mode for the request
|
// Element to enable quiet mode for the request
|
||||||
|
@ -176,6 +179,11 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(requested.Objects) == 0 || len(requested.Objects) > maxObjectsToDelete {
|
||||||
|
h.logAndSendError(w, "number of objects to delete must be greater than 0 and less or equal to 1000", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
removed := make(map[string]*layer.VersionedObject)
|
removed := make(map[string]*layer.VersionedObject)
|
||||||
toRemove := make([]*layer.VersionedObject, 0, len(requested.Objects))
|
toRemove := make([]*layer.VersionedObject, 0, len(requested.Objects))
|
||||||
for _, obj := range requested.Objects {
|
for _, obj := range requested.Objects {
|
||||||
|
|
|
@ -6,8 +6,8 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -41,7 +41,7 @@ func TestDeleteObject(t *testing.T) {
|
||||||
deleteObject(t, tc, bktName, objName, emptyVersion)
|
deleteObject(t, tc, bktName, objName, emptyVersion)
|
||||||
checkNotFound(t, tc, bktName, objName, emptyVersion)
|
checkNotFound(t, tc, bktName, objName, emptyVersion)
|
||||||
|
|
||||||
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo))
|
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeleteObjectFromSuspended(t *testing.T) {
|
func TestDeleteObjectFromSuspended(t *testing.T) {
|
||||||
|
@ -109,7 +109,7 @@ func TestDeleteObjectVersioned(t *testing.T) {
|
||||||
deleteObject(t, tc, bktName, objName, objInfo.VersionID())
|
deleteObject(t, tc, bktName, objName, objInfo.VersionID())
|
||||||
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
|
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
|
||||||
|
|
||||||
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object exists but shouldn't")
|
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object exists but shouldn't")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeleteObjectUnversioned(t *testing.T) {
|
func TestDeleteObjectUnversioned(t *testing.T) {
|
||||||
|
@ -126,7 +126,7 @@ func TestDeleteObjectUnversioned(t *testing.T) {
|
||||||
require.Len(t, versions.DeleteMarker, 0, "delete markers must be empty")
|
require.Len(t, versions.DeleteMarker, 0, "delete markers must be empty")
|
||||||
require.Len(t, versions.Version, 0, "versions must be empty")
|
require.Len(t, versions.Version, 0, "versions must be empty")
|
||||||
|
|
||||||
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object exists but shouldn't")
|
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object exists but shouldn't")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRemoveDeleteMarker(t *testing.T) {
|
func TestRemoveDeleteMarker(t *testing.T) {
|
||||||
|
@ -144,7 +144,7 @@ func TestRemoveDeleteMarker(t *testing.T) {
|
||||||
deleteObject(t, tc, bktName, objName, deleteMarkerVersion)
|
deleteObject(t, tc, bktName, objName, deleteMarkerVersion)
|
||||||
checkFound(t, tc, bktName, objName, emptyVersion)
|
checkFound(t, tc, bktName, objName, emptyVersion)
|
||||||
|
|
||||||
require.True(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object doesn't exist but should")
|
require.True(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object doesn't exist but should")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeleteObjectCombined(t *testing.T) {
|
func TestDeleteObjectCombined(t *testing.T) {
|
||||||
|
@ -161,7 +161,7 @@ func TestDeleteObjectCombined(t *testing.T) {
|
||||||
|
|
||||||
checkFound(t, tc, bktName, objName, objInfo.VersionID())
|
checkFound(t, tc, bktName, objName, objInfo.VersionID())
|
||||||
|
|
||||||
require.True(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object doesn't exist but should")
|
require.True(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object doesn't exist but should")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeleteObjectSuspended(t *testing.T) {
|
func TestDeleteObjectSuspended(t *testing.T) {
|
||||||
|
@ -181,7 +181,7 @@ func TestDeleteObjectSuspended(t *testing.T) {
|
||||||
deleteObject(t, tc, bktName, objName, emptyVersion)
|
deleteObject(t, tc, bktName, objName, emptyVersion)
|
||||||
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
|
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
|
||||||
|
|
||||||
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object exists but shouldn't")
|
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object exists but shouldn't")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeleteMarkers(t *testing.T) {
|
func TestDeleteMarkers(t *testing.T) {
|
||||||
|
@ -200,7 +200,7 @@ func TestDeleteMarkers(t *testing.T) {
|
||||||
require.Len(t, versions.DeleteMarker, 3, "invalid delete markers length")
|
require.Len(t, versions.DeleteMarker, 3, "invalid delete markers length")
|
||||||
require.Len(t, versions.Version, 0, "versions must be empty")
|
require.Len(t, versions.Version, 0, "versions must be empty")
|
||||||
|
|
||||||
require.Len(t, listOIDsFromMockedNeoFS(t, tc, bktName), 0, "shouldn't be any object in neofs")
|
require.Len(t, listOIDsFromMockedFrostFS(t, tc, bktName), 0, "shouldn't be any object in frostfs")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeleteObjectFromListCache(t *testing.T) {
|
func TestDeleteObjectFromListCache(t *testing.T) {
|
||||||
|
@ -220,7 +220,7 @@ func TestDeleteObjectFromListCache(t *testing.T) {
|
||||||
versions = listObjectsV1(t, tc, bktName, "", "", "", -1)
|
versions = listObjectsV1(t, tc, bktName, "", "", "", -1)
|
||||||
require.Len(t, versions.Contents, 0)
|
require.Len(t, versions.Contents, 0)
|
||||||
|
|
||||||
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo))
|
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeleteObjectCheckMarkerReturn(t *testing.T) {
|
func TestDeleteObjectCheckMarkerReturn(t *testing.T) {
|
||||||
|
|
|
@ -3,6 +3,7 @@ package handler
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
|
"crypto/tls"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
@ -11,8 +12,8 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -284,6 +285,7 @@ func getEncryptedObjectRange(t *testing.T, tc *handlerContext, bktName, objName
|
||||||
}
|
}
|
||||||
|
|
||||||
func setEncryptHeaders(r *http.Request) {
|
func setEncryptHeaders(r *http.Request) {
|
||||||
|
r.TLS = &tls.ConnectionState{}
|
||||||
r.Header.Set(api.AmzServerSideEncryptionCustomerAlgorithm, layer.AESEncryptionAlgorithm)
|
r.Header.Set(api.AmzServerSideEncryptionCustomerAlgorithm, layer.AESEncryptionAlgorithm)
|
||||||
r.Header.Set(api.AmzServerSideEncryptionCustomerKey, aes256Key)
|
r.Header.Set(api.AmzServerSideEncryptionCustomerKey, aes256Key)
|
||||||
r.Header.Set(api.AmzServerSideEncryptionCustomerKeyMD5, aes256KeyMD5)
|
r.Header.Set(api.AmzServerSideEncryptionCustomerKeyMD5, aes256KeyMD5)
|
||||||
|
|
|
@ -8,10 +8,10 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -150,7 +150,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
encryptionParams, err := h.formEncryptionParams(r.Header)
|
encryptionParams, err := formEncryptionParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
|
|
@ -8,9 +8,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -13,15 +13,16 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
||||||
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/resolver"
|
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -30,7 +31,7 @@ type handlerContext struct {
|
||||||
owner user.ID
|
owner user.ID
|
||||||
t *testing.T
|
t *testing.T
|
||||||
h *handler
|
h *handler
|
||||||
tp *layer.TestNeoFS
|
tp *layer.TestFrostFS
|
||||||
context context.Context
|
context context.Context
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -38,7 +39,7 @@ func (hc *handlerContext) Handler() *handler {
|
||||||
return hc.h
|
return hc.h
|
||||||
}
|
}
|
||||||
|
|
||||||
func (hc *handlerContext) MockedPool() *layer.TestNeoFS {
|
func (hc *handlerContext) MockedPool() *layer.TestFrostFS {
|
||||||
return hc.tp
|
return hc.tp
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -50,12 +51,30 @@ func (hc *handlerContext) Context() context.Context {
|
||||||
return hc.context
|
return hc.context
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type placementPolicyMock struct {
|
||||||
|
defaultPolicy netmap.PlacementPolicy
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *placementPolicyMock) Default() netmap.PlacementPolicy {
|
||||||
|
return p.defaultPolicy
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *placementPolicyMock) Get(string) (netmap.PlacementPolicy, bool) {
|
||||||
|
return netmap.PlacementPolicy{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
type xmlDecoderProviderMock struct{}
|
||||||
|
|
||||||
|
func (p *xmlDecoderProviderMock) NewCompleteMultipartDecoder(r io.Reader) *xml.Decoder {
|
||||||
|
return xml.NewDecoder(r)
|
||||||
|
}
|
||||||
|
|
||||||
func prepareHandlerContext(t *testing.T) *handlerContext {
|
func prepareHandlerContext(t *testing.T) *handlerContext {
|
||||||
key, err := keys.NewPrivateKey()
|
key, err := keys.NewPrivateKey()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
l := zap.NewExample()
|
l := zap.NewExample()
|
||||||
tp := layer.NewTestNeoFS()
|
tp := layer.NewTestFrostFS()
|
||||||
|
|
||||||
testResolver := &resolver.Resolver{Name: "test_resolver"}
|
testResolver := &resolver.Resolver{Name: "test_resolver"}
|
||||||
testResolver.SetResolveFunc(func(_ context.Context, name string) (cid.ID, error) {
|
testResolver.SetResolveFunc(func(_ context.Context, name string) (cid.ID, error) {
|
||||||
|
@ -72,11 +91,16 @@ func prepareHandlerContext(t *testing.T) *handlerContext {
|
||||||
TreeService: layer.NewTreeService(),
|
TreeService: layer.NewTreeService(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var pp netmap.PlacementPolicy
|
||||||
|
err = pp.DecodeString("REP 1")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
h := &handler{
|
h := &handler{
|
||||||
log: l,
|
log: l,
|
||||||
obj: layer.NewLayer(l, tp, layerCfg),
|
obj: layer.NewLayer(l, tp, layerCfg),
|
||||||
cfg: &Config{
|
cfg: &Config{
|
||||||
TLSEnabled: true,
|
Policy: &placementPolicyMock{defaultPolicy: pp},
|
||||||
|
XMLDecoder: &xmlDecoderProviderMock{},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -191,7 +215,7 @@ func parseTestResponse(t *testing.T, response *httptest.ResponseRecorder, body i
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func existInMockedNeoFS(tc *handlerContext, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo) bool {
|
func existInMockedFrostFS(tc *handlerContext, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo) bool {
|
||||||
p := &layer.GetObjectParams{
|
p := &layer.GetObjectParams{
|
||||||
BucketInfo: bktInfo,
|
BucketInfo: bktInfo,
|
||||||
ObjectInfo: objInfo,
|
ObjectInfo: objInfo,
|
||||||
|
@ -201,7 +225,7 @@ func existInMockedNeoFS(tc *handlerContext, bktInfo *data.BucketInfo, objInfo *d
|
||||||
return tc.Layer().GetObject(tc.Context(), p) == nil
|
return tc.Layer().GetObject(tc.Context(), p) == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func listOIDsFromMockedNeoFS(t *testing.T, tc *handlerContext, bktName string) []oid.ID {
|
func listOIDsFromMockedFrostFS(t *testing.T, tc *handlerContext, bktName string) []oid.ID {
|
||||||
bktInfo, err := tc.Layer().GetBucketInfo(tc.Context(), bktName)
|
bktInfo, err := tc.Layer().GetBucketInfo(tc.Context(), bktName)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
|
|
@ -4,10 +4,10 @@ import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -53,7 +53,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
info := extendedInfo.ObjectInfo
|
info := extendedInfo.ObjectInfo
|
||||||
|
|
||||||
encryptionParams, err := h.formEncryptionParams(r.Header)
|
encryptionParams, err := formEncryptionParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -123,6 +123,13 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Header().Set(api.ContainerID, bktInfo.CID.EncodeToString())
|
w.Header().Set(api.ContainerID, bktInfo.CID.EncodeToString())
|
||||||
|
w.Header().Set(api.AmzBucketRegion, bktInfo.LocationConstraint)
|
||||||
|
|
||||||
|
if isAvailableToResolve(bktInfo.Zone, h.cfg.ResolveZoneList, h.cfg.IsResolveListAllow) {
|
||||||
|
w.Header().Set(api.ContainerName, bktInfo.Name)
|
||||||
|
w.Header().Set(api.ContainerZone, bktInfo.Zone)
|
||||||
|
}
|
||||||
|
|
||||||
api.WriteResponse(w, http.StatusOK, nil, api.MimeNone)
|
api.WriteResponse(w, http.StatusOK, nil, api.MimeNone)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -156,3 +163,25 @@ func writeLockHeaders(h http.Header, legalHold *data.LegalHold, retention *data.
|
||||||
h.Set(api.AmzObjectLockMode, retention.Mode)
|
h.Set(api.AmzObjectLockMode, retention.Mode)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isAvailableToResolve(zone string, list []string, isAllowList bool) bool {
|
||||||
|
// empty zone means container doesn't have proper system name,
|
||||||
|
// so we don't have to resolve it
|
||||||
|
if len(zone) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
var zoneInList bool
|
||||||
|
for _, t := range list {
|
||||||
|
if t == zone {
|
||||||
|
zoneInList = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// InList | IsAllowList | Result
|
||||||
|
// 0 0 1
|
||||||
|
// 0 1 0
|
||||||
|
// 1 0 0
|
||||||
|
// 1 1 1
|
||||||
|
return zoneInList == isAllowList
|
||||||
|
}
|
||||||
|
|
|
@ -6,11 +6,11 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -86,6 +86,26 @@ func TestInvalidAccessThroughCache(t *testing.T) {
|
||||||
assertStatus(t, w, http.StatusForbidden)
|
assertStatus(t, w, http.StatusForbidden)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIsAvailableToResolve(t *testing.T) {
|
||||||
|
list := []string{"container", "s3"}
|
||||||
|
|
||||||
|
for i, testCase := range [...]struct {
|
||||||
|
isAllowList bool
|
||||||
|
list []string
|
||||||
|
zone string
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
{isAllowList: true, list: list, zone: "container", expected: true},
|
||||||
|
{isAllowList: true, list: list, zone: "sftp", expected: false},
|
||||||
|
{isAllowList: false, list: list, zone: "s3", expected: false},
|
||||||
|
{isAllowList: false, list: list, zone: "system", expected: true},
|
||||||
|
{isAllowList: true, list: list, zone: "", expected: false},
|
||||||
|
} {
|
||||||
|
result := isAvailableToResolve(testCase.zone, testCase.list, testCase.isAllowList)
|
||||||
|
require.Equal(t, testCase.expected, result, "case %d", i+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box {
|
func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box {
|
||||||
var err error
|
var err error
|
||||||
if key == nil {
|
if key == nil {
|
||||||
|
|
|
@ -3,7 +3,7 @@ package handler
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
|
@ -4,8 +4,8 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
)
|
)
|
||||||
|
|
||||||
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||||
|
|
|
@ -1,16 +1,17 @@
|
||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -208,7 +209,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
lock, err := formObjectLockFromRetention(retention, r.Header)
|
lock, err := formObjectLockFromRetention(r.Context(), retention, r.Header)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid retention configuration", reqInfo, err)
|
h.logAndSendError(w, "invalid retention configuration", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -300,7 +301,7 @@ func checkLockConfiguration(conf *data.ObjectLockConfiguration) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConfiguration, header http.Header) (*data.ObjectLock, error) {
|
func formObjectLock(ctx context.Context, bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConfiguration, header http.Header) (*data.ObjectLock, error) {
|
||||||
if !bktInfo.ObjectLockEnabled {
|
if !bktInfo.ObjectLockEnabled {
|
||||||
if existLockHeaders(header) {
|
if existLockHeaders(header) {
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound)
|
||||||
|
@ -318,7 +319,7 @@ func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConf
|
||||||
retention := &data.RetentionLock{}
|
retention := &data.RetentionLock{}
|
||||||
defaultRetention := defaultConfig.Rule.DefaultRetention
|
defaultRetention := defaultConfig.Rule.DefaultRetention
|
||||||
retention.IsCompliance = defaultRetention.Mode == complianceMode
|
retention.IsCompliance = defaultRetention.Mode == complianceMode
|
||||||
now := time.Now()
|
now := layer.TimeNow(ctx)
|
||||||
if defaultRetention.Days != 0 {
|
if defaultRetention.Days != 0 {
|
||||||
retention.Until = now.Add(time.Duration(defaultRetention.Days) * dayDuration)
|
retention.Until = now.Add(time.Duration(defaultRetention.Days) * dayDuration)
|
||||||
} else {
|
} else {
|
||||||
|
@ -370,7 +371,7 @@ func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConf
|
||||||
objectLock.Retention.ByPassedGovernance = bypass
|
objectLock.Retention.ByPassedGovernance = bypass
|
||||||
}
|
}
|
||||||
|
|
||||||
if objectLock.Retention.Until.Before(time.Now()) {
|
if objectLock.Retention.Until.Before(layer.TimeNow(ctx)) {
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -384,7 +385,7 @@ func existLockHeaders(header http.Header) bool {
|
||||||
header.Get(api.AmzObjectLockRetainUntilDate) != ""
|
header.Get(api.AmzObjectLockRetainUntilDate) != ""
|
||||||
}
|
}
|
||||||
|
|
||||||
func formObjectLockFromRetention(retention *data.Retention, header http.Header) (*data.ObjectLock, error) {
|
func formObjectLockFromRetention(ctx context.Context, retention *data.Retention, header http.Header) (*data.ObjectLock, error) {
|
||||||
if retention.Mode != governanceMode && retention.Mode != complianceMode {
|
if retention.Mode != governanceMode && retention.Mode != complianceMode {
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
|
||||||
}
|
}
|
||||||
|
@ -394,7 +395,7 @@ func formObjectLockFromRetention(retention *data.Retention, header http.Header)
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retentionDate.Before(time.Now()) {
|
if retentionDate.Before(layer.TimeNow(ctx)) {
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -10,15 +10,17 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
const defaultURL = "http://localhost/"
|
const defaultURL = "http://localhost/"
|
||||||
|
|
||||||
func TestFormObjectLock(t *testing.T) {
|
func TestFormObjectLock(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
bktInfo *data.BucketInfo
|
bktInfo *data.BucketInfo
|
||||||
|
@ -73,7 +75,7 @@ func TestFormObjectLock(t *testing.T) {
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
actualObjLock, err := formObjectLock(tc.bktInfo, tc.config, tc.header)
|
actualObjLock, err := formObjectLock(ctx, tc.bktInfo, tc.config, tc.header)
|
||||||
if tc.expectedError {
|
if tc.expectedError {
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
return
|
return
|
||||||
|
@ -86,6 +88,8 @@ func TestFormObjectLock(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFormObjectLockFromRetention(t *testing.T) {
|
func TestFormObjectLockFromRetention(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
retention *data.Retention
|
retention *data.Retention
|
||||||
|
@ -132,7 +136,7 @@ func TestFormObjectLockFromRetention(t *testing.T) {
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
actualObjLock, err := formObjectLockFromRetention(tc.retention, tc.header)
|
actualObjLock, err := formObjectLockFromRetention(ctx, tc.retention, tc.header)
|
||||||
if tc.expectedError {
|
if tc.expectedError {
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
return
|
return
|
||||||
|
|
|
@ -2,17 +2,18 @@ package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -137,7 +138,7 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
|
p.Info.Encryption, err = formEncryptionParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -226,7 +227,7 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
Reader: r.Body,
|
Reader: r.Body,
|
||||||
}
|
}
|
||||||
|
|
||||||
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
|
p.Info.Encryption, err = formEncryptionParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -331,7 +332,7 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
||||||
Range: srcRange,
|
Range: srcRange,
|
||||||
}
|
}
|
||||||
|
|
||||||
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
|
p.Info.Encryption, err = formEncryptionParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -372,8 +373,6 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
sessionTokenSetEACL *session.Container
|
|
||||||
|
|
||||||
uploadID = r.URL.Query().Get(uploadIDHeaderName)
|
uploadID = r.URL.Query().Get(uploadIDHeaderName)
|
||||||
uploadInfo = &layer.UploadInfoParams{
|
uploadInfo = &layer.UploadInfoParams{
|
||||||
UploadID: uploadID,
|
UploadID: uploadID,
|
||||||
|
@ -384,7 +383,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
)
|
)
|
||||||
|
|
||||||
reqBody := new(CompleteMultipartUpload)
|
reqBody := new(CompleteMultipartUpload)
|
||||||
if err = xml.NewDecoder(r.Body).Decode(reqBody); err != nil {
|
if err = h.cfg.XMLDecoder.NewCompleteMultipartDecoder(r.Body).Decode(reqBody); err != nil {
|
||||||
h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
|
h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
|
||||||
errors.GetAPIError(errors.ErrMalformedXML), additional...)
|
errors.GetAPIError(errors.ErrMalformedXML), additional...)
|
||||||
return
|
return
|
||||||
|
@ -399,10 +398,51 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
Parts: reqBody.Parts,
|
Parts: reqBody.Parts,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Next operations might take some time, so we want to keep client's
|
||||||
|
// connection alive. To do so, gateway sends periodic white spaces
|
||||||
|
// back to the client the same way as Amazon S3 service does.
|
||||||
|
stopPeriodicResponseWriter := periodicXMLWriter(w, h.cfg.CompleteMultipartKeepalive)
|
||||||
|
|
||||||
|
// Start complete multipart upload which may take some time to fetch object
|
||||||
|
// and re-upload it part by part.
|
||||||
|
objInfo, err := h.completeMultipartUpload(r, c, bktInfo, reqInfo)
|
||||||
|
|
||||||
|
// Stop periodic writer as complete multipart upload is finished
|
||||||
|
// successfully or not.
|
||||||
|
headerIsWritten := stopPeriodicResponseWriter()
|
||||||
|
|
||||||
|
responseWriter := api.EncodeToResponse
|
||||||
|
errLogger := h.logAndSendError
|
||||||
|
// Do not send XML and HTTP headers if periodic writer was invoked at this point.
|
||||||
|
if headerIsWritten {
|
||||||
|
responseWriter = api.EncodeToResponseNoHeader
|
||||||
|
errLogger = h.logAndSendErrorNoHeader
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
errLogger(w, "complete multipart error", reqInfo, err, additional...)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
response := CompleteMultipartUploadResponse{
|
||||||
|
Bucket: objInfo.Bucket,
|
||||||
|
ETag: objInfo.HashSum,
|
||||||
|
Key: objInfo.Name,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Here we previously set api.AmzVersionID header for versioned bucket.
|
||||||
|
// It is not possible after #60, because of periodic white
|
||||||
|
// space XML writer to keep connection with the client.
|
||||||
|
|
||||||
|
if err = responseWriter(w, response); err != nil {
|
||||||
|
errLogger(w, "something went wrong", reqInfo, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMultipartParams, bktInfo *data.BucketInfo, reqInfo *api.ReqInfo) (*data.ObjectInfo, error) {
|
||||||
uploadData, extendedObjInfo, err := h.obj.CompleteMultipartUpload(r.Context(), c)
|
uploadData, extendedObjInfo, err := h.obj.CompleteMultipartUpload(r.Context(), c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not complete multipart upload", reqInfo, err, additional...)
|
return nil, fmt.Errorf("could not complete multipart upload: %w", err)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
objInfo := extendedObjInfo.ObjectInfo
|
objInfo := extendedObjInfo.ObjectInfo
|
||||||
|
|
||||||
|
@ -417,21 +457,22 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
NodeVersion: extendedObjInfo.NodeVersion,
|
NodeVersion: extendedObjInfo.NodeVersion,
|
||||||
}
|
}
|
||||||
if _, err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
|
if _, err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
|
||||||
h.logAndSendError(w, "could not put tagging file of completed multipart upload", reqInfo, err, additional...)
|
return nil, fmt.Errorf("could not put tagging file of completed multipart upload: %w", err)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(uploadData.ACLHeaders) != 0 {
|
if len(uploadData.ACLHeaders) != 0 {
|
||||||
|
sessionTokenSetEACL, err := getSessionTokenSetEACL(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("couldn't get eacl token: %w", err)
|
||||||
|
}
|
||||||
key, err := h.bearerTokenIssuerKey(r.Context())
|
key, err := h.bearerTokenIssuerKey(r.Context())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "couldn't get gate key", reqInfo, err)
|
return nil, fmt.Errorf("couldn't get gate key: %w", err)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
acl, err := parseACLHeaders(r.Header, key)
|
acl, err := parseACLHeaders(r.Header, key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not parse acl", reqInfo, err)
|
return nil, fmt.Errorf("could not parse acl: %w", err)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
resInfo := &resourceInfo{
|
resInfo := &resourceInfo{
|
||||||
|
@ -440,12 +481,10 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
}
|
}
|
||||||
astObject, err := aclToAst(acl, resInfo)
|
astObject, err := aclToAst(acl, resInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not translate acl of completed multipart upload to ast", reqInfo, err, additional...)
|
return nil, fmt.Errorf("could not translate acl of completed multipart upload to ast: %w", err)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
if _, err = h.updateBucketACL(r, astObject, bktInfo, sessionTokenSetEACL); err != nil {
|
if _, err = h.updateBucketACL(r, astObject, bktInfo, sessionTokenSetEACL); err != nil {
|
||||||
h.logAndSendError(w, "could not update bucket acl while completing multipart upload", reqInfo, err, additional...)
|
return nil, fmt.Errorf("could not update bucket acl while completing multipart upload: %w", err)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -459,24 +498,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
return objInfo, nil
|
||||||
if err != nil {
|
|
||||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response := CompleteMultipartUploadResponse{
|
|
||||||
Bucket: objInfo.Bucket,
|
|
||||||
ETag: objInfo.HashSum,
|
|
||||||
Key: objInfo.Name,
|
|
||||||
}
|
|
||||||
|
|
||||||
if bktSettings.VersioningEnabled() {
|
|
||||||
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = api.EncodeToResponse(w, response); err != nil {
|
|
||||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
@ -573,7 +595,7 @@ func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
PartNumberMarker: partNumberMarker,
|
PartNumberMarker: partNumberMarker,
|
||||||
}
|
}
|
||||||
|
|
||||||
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
|
p.Info.Encryption, err = formEncryptionParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -608,7 +630,7 @@ func (h *handler) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Req
|
||||||
Key: reqInfo.ObjectName,
|
Key: reqInfo.ObjectName,
|
||||||
}
|
}
|
||||||
|
|
||||||
p.Encryption, err = h.formEncryptionParams(r.Header)
|
p.Encryption, err = formEncryptionParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -681,3 +703,62 @@ func encodeListPartsToResponse(info *layer.ListPartsInfo, params *layer.ListPart
|
||||||
Parts: info.Parts,
|
Parts: info.Parts,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// periodicXMLWriter creates go routine to write xml header and whitespaces
|
||||||
|
// over time to avoid connection drop from the client. To work properly,
|
||||||
|
// pass `http.ResponseWriter` with implemented `http.Flusher` interface.
|
||||||
|
// Returns stop function which returns boolean if writer has been used
|
||||||
|
// during goroutine execution. To disable writer, pass 0 duration value.
|
||||||
|
func periodicXMLWriter(w io.Writer, dur time.Duration) (stop func() bool) {
|
||||||
|
if dur == 0 { // 0 duration disables periodic writer
|
||||||
|
return func() bool { return false }
|
||||||
|
}
|
||||||
|
|
||||||
|
whitespaceChar := []byte(" ")
|
||||||
|
closer := make(chan struct{})
|
||||||
|
done := make(chan struct{})
|
||||||
|
headerWritten := false
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
|
||||||
|
tick := time.NewTicker(dur)
|
||||||
|
defer tick.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-tick.C:
|
||||||
|
if !headerWritten {
|
||||||
|
_, err := w.Write([]byte(xml.Header))
|
||||||
|
headerWritten = err == nil
|
||||||
|
}
|
||||||
|
_, err := w.Write(whitespaceChar)
|
||||||
|
if err != nil {
|
||||||
|
return // is there anything we can do better than ignore error?
|
||||||
|
}
|
||||||
|
if buffered, ok := w.(http.Flusher); ok {
|
||||||
|
buffered.Flush()
|
||||||
|
}
|
||||||
|
case <-closer:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
stop = func() bool {
|
||||||
|
close(closer)
|
||||||
|
<-done // wait for goroutine to stop
|
||||||
|
return headerWritten
|
||||||
|
}
|
||||||
|
|
||||||
|
return stop
|
||||||
|
}
|
||||||
|
|
||||||
|
// periodicWriterErrorSender returns handler function to send error. If header is
|
||||||
|
// alreay written by periodic XML writer, do not send HTTP and XML headers.
|
||||||
|
func (h *handler) periodicWriterErrorSender(headerWritten bool) func(http.ResponseWriter, string, *api.ReqInfo, error, ...zap.Field) {
|
||||||
|
if headerWritten {
|
||||||
|
return h.logAndSendErrorNoHeader
|
||||||
|
}
|
||||||
|
return h.logAndSendError
|
||||||
|
}
|
||||||
|
|
48
api/handler/multipart_upload_test.go
Normal file
48
api/handler/multipart_upload_test.go
Normal file
|
@ -0,0 +1,48 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/xml"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPeriodicWriter(t *testing.T) {
|
||||||
|
const dur = 100 * time.Millisecond
|
||||||
|
const whitespaces = 8
|
||||||
|
expected := []byte(xml.Header)
|
||||||
|
for i := 0; i < whitespaces; i++ {
|
||||||
|
expected = append(expected, []byte(" ")...)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("writes data", func(t *testing.T) {
|
||||||
|
buf := bytes.NewBuffer(nil)
|
||||||
|
stop := periodicXMLWriter(buf, dur)
|
||||||
|
|
||||||
|
// N number of whitespaces + half durations to guarantee at least N writes in buffer
|
||||||
|
time.Sleep(whitespaces*dur + dur/2)
|
||||||
|
require.True(t, stop())
|
||||||
|
require.Equal(t, expected, buf.Bytes())
|
||||||
|
|
||||||
|
t.Run("no additional data after stop", func(t *testing.T) {
|
||||||
|
time.Sleep(2 * dur)
|
||||||
|
require.Equal(t, expected, buf.Bytes())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("does not write data", func(t *testing.T) {
|
||||||
|
buf := bytes.NewBuffer(nil)
|
||||||
|
stop := periodicXMLWriter(buf, dur)
|
||||||
|
time.Sleep(dur / 2)
|
||||||
|
require.False(t, stop())
|
||||||
|
require.Empty(t, buf.Bytes())
|
||||||
|
|
||||||
|
t.Run("disabled", func(t *testing.T) {
|
||||||
|
stop = periodicXMLWriter(buf, 0)
|
||||||
|
require.False(t, stop())
|
||||||
|
require.Empty(t, buf.Bytes())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
|
@ -3,8 +3,8 @@ package handler
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
|
@ -6,13 +6,14 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
@ -22,6 +23,7 @@ type (
|
||||||
BktInfo *data.BucketInfo
|
BktInfo *data.BucketInfo
|
||||||
ReqInfo *api.ReqInfo
|
ReqInfo *api.ReqInfo
|
||||||
User string
|
User string
|
||||||
|
Time time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
NotificationConfiguration struct {
|
NotificationConfiguration struct {
|
||||||
|
@ -107,7 +109,7 @@ func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Re
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err = h.checkBucketConfiguration(conf, reqInfo); err != nil {
|
if _, err = h.checkBucketConfiguration(r.Context(), conf, reqInfo); err != nil {
|
||||||
h.logAndSendError(w, "couldn't check bucket configuration", reqInfo, err)
|
h.logAndSendError(w, "couldn't check bucket configuration", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -164,13 +166,15 @@ func (h *handler) sendNotifications(ctx context.Context, p *SendNotificationPara
|
||||||
p.User = bearer.ResolveIssuer(*box.Gate.BearerToken).EncodeToString()
|
p.User = bearer.ResolveIssuer(*box.Gate.BearerToken).EncodeToString()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
p.Time = layer.TimeNow(ctx)
|
||||||
|
|
||||||
topics := filterSubjects(conf, p.Event, p.NotificationInfo.Name)
|
topics := filterSubjects(conf, p.Event, p.NotificationInfo.Name)
|
||||||
|
|
||||||
return h.notificator.SendNotifications(topics, p)
|
return h.notificator.SendNotifications(topics, p)
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkBucketConfiguration checks notification configuration and generates an ID for configurations with empty ids.
|
// checkBucketConfiguration checks notification configuration and generates an ID for configurations with empty ids.
|
||||||
func (h *handler) checkBucketConfiguration(conf *data.NotificationConfiguration, r *api.ReqInfo) (completed bool, err error) {
|
func (h *handler) checkBucketConfiguration(ctx context.Context, conf *data.NotificationConfiguration, r *api.ReqInfo) (completed bool, err error) {
|
||||||
if conf == nil {
|
if conf == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -189,7 +193,7 @@ func (h *handler) checkBucketConfiguration(conf *data.NotificationConfiguration,
|
||||||
}
|
}
|
||||||
|
|
||||||
if h.cfg.NotificatorEnabled {
|
if h.cfg.NotificatorEnabled {
|
||||||
if err = h.notificator.SendTestNotification(q.QueueArn, r.BucketName, r.RequestID, r.Host); err != nil {
|
if err = h.notificator.SendTestNotification(q.QueueArn, r.BucketName, r.RequestID, r.Host, layer.TimeNow(ctx)); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -3,8 +3,8 @@ package handler
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -6,11 +6,11 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ListObjectsV1Handler handles objects listing requests for API version 1.
|
// ListObjectsV1Handler handles objects listing requests for API version 1.
|
||||||
|
|
|
@ -6,7 +6,7 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -16,15 +16,15 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -218,7 +218,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
encryption, err := h.formEncryptionParams(r.Header)
|
encryptionParams, err := formEncryptionParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -230,7 +230,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
Reader: r.Body,
|
Reader: r.Body,
|
||||||
Size: r.ContentLength,
|
Size: r.ContentLength,
|
||||||
Header: metadata,
|
Header: metadata,
|
||||||
Encryption: encryption,
|
Encryption: encryptionParams,
|
||||||
CopiesNumber: copiesNumber,
|
CopiesNumber: copiesNumber,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -240,7 +240,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
params.Lock, err = formObjectLock(bktInfo, settings.LockConfiguration, r.Header)
|
params.Lock, err = formObjectLock(r.Context(), bktInfo, settings.LockConfiguration, r.Header)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not form object lock", reqInfo, err)
|
h.logAndSendError(w, "could not form object lock", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -304,7 +304,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
if settings.VersioningEnabled() {
|
if settings.VersioningEnabled() {
|
||||||
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
||||||
}
|
}
|
||||||
if encryption.Enabled() {
|
if encryptionParams.Enabled() {
|
||||||
addSSECHeaders(w.Header(), r.Header)
|
addSSECHeaders(w.Header(), r.Header)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -313,7 +313,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func getCopiesNumberOrDefault(metadata map[string]string, defaultCopiesNumber uint32) (uint32, error) {
|
func getCopiesNumberOrDefault(metadata map[string]string, defaultCopiesNumber uint32) (uint32, error) {
|
||||||
copiesNumberStr, ok := metadata[layer.AttributeNeofsCopiesNumber]
|
copiesNumberStr, ok := metadata[layer.AttributeFrostfsCopiesNumber]
|
||||||
if !ok {
|
if !ok {
|
||||||
return defaultCopiesNumber, nil
|
return defaultCopiesNumber, nil
|
||||||
}
|
}
|
||||||
|
@ -326,16 +326,16 @@ func getCopiesNumberOrDefault(metadata map[string]string, defaultCopiesNumber ui
|
||||||
return uint32(copiesNumber), nil
|
return uint32(copiesNumber), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h handler) formEncryptionParams(header http.Header) (enc encryption.Params, err error) {
|
func formEncryptionParams(r *http.Request) (enc encryption.Params, err error) {
|
||||||
sseCustomerAlgorithm := header.Get(api.AmzServerSideEncryptionCustomerAlgorithm)
|
sseCustomerAlgorithm := r.Header.Get(api.AmzServerSideEncryptionCustomerAlgorithm)
|
||||||
sseCustomerKey := header.Get(api.AmzServerSideEncryptionCustomerKey)
|
sseCustomerKey := r.Header.Get(api.AmzServerSideEncryptionCustomerKey)
|
||||||
sseCustomerKeyMD5 := header.Get(api.AmzServerSideEncryptionCustomerKeyMD5)
|
sseCustomerKeyMD5 := r.Header.Get(api.AmzServerSideEncryptionCustomerKeyMD5)
|
||||||
|
|
||||||
if len(sseCustomerAlgorithm) == 0 && len(sseCustomerKey) == 0 && len(sseCustomerKeyMD5) == 0 {
|
if len(sseCustomerAlgorithm) == 0 && len(sseCustomerKey) == 0 && len(sseCustomerKeyMD5) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if !h.cfg.TLSEnabled {
|
if r.TLS == nil {
|
||||||
return enc, errorsStd.New("encryption available only when TLS is enabled")
|
return enc, errorsStd.New("encryption available only when TLS is enabled")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -667,12 +667,10 @@ func parseMetadata(r *http.Request) map[string]string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
var (
|
reqInfo := api.GetReqInfo(r.Context())
|
||||||
reqInfo = api.GetReqInfo(r.Context())
|
p := &layer.CreateBucketParams{
|
||||||
p = layer.CreateBucketParams{
|
Name: reqInfo.BucketName,
|
||||||
Name: reqInfo.BucketName,
|
}
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
if err := checkBucketName(reqInfo.BucketName); err != nil {
|
if err := checkBucketName(reqInfo.BucketName); err != nil {
|
||||||
h.logAndSendError(w, "invalid bucket name", reqInfo, err)
|
h.logAndSendError(w, "invalid bucket name", reqInfo, err)
|
||||||
|
@ -722,29 +720,22 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
useDefaultPolicy := true
|
if err = h.setPolicy(p, createParams.LocationConstraint, policies); err != nil {
|
||||||
if createParams.LocationConstraint != "" {
|
h.logAndSendError(w, "couldn't set placement policy", reqInfo, err)
|
||||||
for _, placementPolicy := range policies {
|
return
|
||||||
if placementPolicy.LocationConstraint == createParams.LocationConstraint {
|
|
||||||
p.Policy = placementPolicy.Policy
|
|
||||||
p.LocationConstraint = createParams.LocationConstraint
|
|
||||||
useDefaultPolicy = false
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if useDefaultPolicy {
|
|
||||||
p.Policy = h.cfg.DefaultPolicy
|
|
||||||
}
|
}
|
||||||
|
|
||||||
p.ObjectLockEnabled = isLockEnabled(r.Header)
|
p.ObjectLockEnabled = isLockEnabled(r.Header)
|
||||||
|
|
||||||
bktInfo, err := h.obj.CreateBucket(r.Context(), &p)
|
bktInfo, err := h.obj.CreateBucket(r.Context(), p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "could not create bucket", reqInfo, err)
|
h.logAndSendError(w, "could not create bucket", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
h.log.Info("bucket is created", zap.String("reqId", reqInfo.RequestID),
|
||||||
|
zap.String("bucket", reqInfo.BucketName), zap.Stringer("container_id", bktInfo.CID))
|
||||||
|
|
||||||
if p.ObjectLockEnabled {
|
if p.ObjectLockEnabled {
|
||||||
sp := &layer.PutSettingsParams{
|
sp := &layer.PutSettingsParams{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
|
@ -757,11 +748,32 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
h.log.Info("bucket is created", zap.Stringer("container_id", bktInfo.CID))
|
|
||||||
|
|
||||||
api.WriteSuccessResponseHeadersOnly(w)
|
api.WriteSuccessResponseHeadersOnly(w)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h handler) setPolicy(prm *layer.CreateBucketParams, locationConstraint string, userPolicies []*accessbox.ContainerPolicy) error {
|
||||||
|
prm.Policy = h.cfg.Policy.Default()
|
||||||
|
prm.LocationConstraint = locationConstraint
|
||||||
|
|
||||||
|
if locationConstraint == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, placementPolicy := range userPolicies {
|
||||||
|
if placementPolicy.LocationConstraint == locationConstraint {
|
||||||
|
prm.Policy = placementPolicy.Policy
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if policy, ok := h.cfg.Policy.Get(locationConstraint); ok {
|
||||||
|
prm.Policy = policy
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return errors.GetAPIError(errors.ErrInvalidLocationConstraint)
|
||||||
|
}
|
||||||
|
|
||||||
func isLockEnabled(header http.Header) bool {
|
func isLockEnabled(header http.Header) bool {
|
||||||
lockEnabledStr := header.Get(api.AmzBucketObjectLockEnabled)
|
lockEnabledStr := header.Get(api.AmzBucketObjectLockEnabled)
|
||||||
lockEnabled, _ := strconv.ParseBool(lockEnabledStr)
|
lockEnabled, _ := strconv.ParseBool(lockEnabledStr)
|
||||||
|
|
|
@ -8,8 +8,8 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -114,7 +114,7 @@ func TestPutObjectOverrideCopiesNumber(t *testing.T) {
|
||||||
bktInfo := createTestBucket(tc, bktName)
|
bktInfo := createTestBucket(tc, bktName)
|
||||||
|
|
||||||
w, r := prepareTestRequest(tc, bktName, objName, nil)
|
w, r := prepareTestRequest(tc, bktName, objName, nil)
|
||||||
r.Header.Set(api.MetadataPrefix+strings.ToUpper(layer.AttributeNeofsCopiesNumber), "1")
|
r.Header.Set(api.MetadataPrefix+strings.ToUpper(layer.AttributeFrostfsCopiesNumber), "1")
|
||||||
tc.Handler().PutObjectHandler(w, r)
|
tc.Handler().PutObjectHandler(w, r)
|
||||||
|
|
||||||
p := &layer.HeadObjectParams{
|
p := &layer.HeadObjectParams{
|
||||||
|
@ -124,5 +124,5 @@ func TestPutObjectOverrideCopiesNumber(t *testing.T) {
|
||||||
|
|
||||||
objInfo, err := tc.Layer().GetObjectInfo(tc.Context(), p)
|
objInfo, err := tc.Layer().GetObjectInfo(tc.Context(), p)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, "1", objInfo.Headers[layer.AttributeNeofsCopiesNumber])
|
require.Equal(t, "1", objInfo.Headers[layer.AttributeFrostfsCopiesNumber])
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,10 +8,10 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"unicode"
|
"unicode"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -3,8 +3,8 @@ package handler
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
|
@ -7,11 +7,11 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -29,6 +29,19 @@ func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo
|
||||||
h.log.Error("call method", fields...)
|
h.log.Error("call method", fields...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *handler) logAndSendErrorNoHeader(w http.ResponseWriter, logText string, reqInfo *api.ReqInfo, err error, additional ...zap.Field) {
|
||||||
|
api.WriteErrorResponseNoHeader(w, reqInfo, transformToS3Error(err))
|
||||||
|
fields := []zap.Field{
|
||||||
|
zap.String("request_id", reqInfo.RequestID),
|
||||||
|
zap.String("method", reqInfo.API),
|
||||||
|
zap.String("bucket", reqInfo.BucketName),
|
||||||
|
zap.String("object", reqInfo.ObjectName),
|
||||||
|
zap.String("description", logText),
|
||||||
|
zap.Error(err)}
|
||||||
|
fields = append(fields, additional...)
|
||||||
|
h.log.Error("call method", fields...)
|
||||||
|
}
|
||||||
|
|
||||||
func transformToS3Error(err error) error {
|
func transformToS3Error(err error) error {
|
||||||
if _, ok := err.(errors.Error); ok {
|
if _, ok := err.(errors.Error); ok {
|
||||||
return err
|
return err
|
||||||
|
@ -42,6 +55,10 @@ func transformToS3Error(err error) error {
|
||||||
return errors.GetAPIError(errors.ErrInternalError)
|
return errors.GetAPIError(errors.ErrInternalError)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *handler) ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error) {
|
||||||
|
return h.obj.GetBucketInfo(ctx, bucket)
|
||||||
|
}
|
||||||
|
|
||||||
func (h *handler) getBucketAndCheckOwner(r *http.Request, bucket string, header ...string) (*data.BucketInfo, error) {
|
func (h *handler) getBucketAndCheckOwner(r *http.Request, bucket string, header ...string) (*data.BucketInfo, error) {
|
||||||
bktInfo, err := h.obj.GetBucketInfo(r.Context(), bucket)
|
bktInfo, err := h.obj.GetBucketInfo(r.Context(), bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -4,10 +4,10 @@ import (
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
|
@ -2,17 +2,17 @@ package api
|
||||||
|
|
||||||
// Standard S3 HTTP request/response constants.
|
// Standard S3 HTTP request/response constants.
|
||||||
const (
|
const (
|
||||||
MetadataPrefix = "X-Amz-Meta-"
|
MetadataPrefix = "X-Amz-Meta-"
|
||||||
NeoFSSystemMetadataPrefix = "S3-"
|
FrostFSSystemMetadataPrefix = "S3-"
|
||||||
AmzMetadataDirective = "X-Amz-Metadata-Directive"
|
AmzMetadataDirective = "X-Amz-Metadata-Directive"
|
||||||
AmzTaggingDirective = "X-Amz-Tagging-Directive"
|
AmzTaggingDirective = "X-Amz-Tagging-Directive"
|
||||||
AmzVersionID = "X-Amz-Version-Id"
|
AmzVersionID = "X-Amz-Version-Id"
|
||||||
AmzTaggingCount = "X-Amz-Tagging-Count"
|
AmzTaggingCount = "X-Amz-Tagging-Count"
|
||||||
AmzTagging = "X-Amz-Tagging"
|
AmzTagging = "X-Amz-Tagging"
|
||||||
AmzDeleteMarker = "X-Amz-Delete-Marker"
|
AmzDeleteMarker = "X-Amz-Delete-Marker"
|
||||||
AmzCopySource = "X-Amz-Copy-Source"
|
AmzCopySource = "X-Amz-Copy-Source"
|
||||||
AmzCopySourceRange = "X-Amz-Copy-Source-Range"
|
AmzCopySourceRange = "X-Amz-Copy-Source-Range"
|
||||||
AmzDate = "X-Amz-Date"
|
AmzDate = "X-Amz-Date"
|
||||||
|
|
||||||
LastModified = "Last-Modified"
|
LastModified = "Last-Modified"
|
||||||
Date = "Date"
|
Date = "Date"
|
||||||
|
@ -62,7 +62,9 @@ const (
|
||||||
AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"
|
AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"
|
||||||
AmzServerSideEncryptionCustomerKeyMD5 = "x-amz-server-side-encryption-customer-key-MD5"
|
AmzServerSideEncryptionCustomerKeyMD5 = "x-amz-server-side-encryption-customer-key-MD5"
|
||||||
|
|
||||||
ContainerID = "X-Container-Id"
|
ContainerID = "X-Container-Id"
|
||||||
|
ContainerName = "X-Container-Name"
|
||||||
|
ContainerZone = "X-Container-Zone"
|
||||||
|
|
||||||
AccessControlAllowOrigin = "Access-Control-Allow-Origin"
|
AccessControlAllowOrigin = "Access-Control-Allow-Origin"
|
||||||
AccessControlAllowMethods = "Access-Control-Allow-Methods"
|
AccessControlAllowMethods = "Access-Control-Allow-Methods"
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
package layer
|
package layer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -4,8 +4,8 @@ import (
|
||||||
"context"
|
"context"
|
||||||
errorsStd "errors"
|
errorsStd "errors"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
|
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
|
||||||
|
|
|
@ -4,16 +4,16 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/client"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -42,14 +42,14 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
|
||||||
Name: idCnr.EncodeToString(),
|
Name: idCnr.EncodeToString(),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
res, err = n.neoFS.Container(ctx, idCnr)
|
res, err = n.frostFS.Container(ctx, idCnr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("could not fetch container", zap.Error(err))
|
log.Error("could not fetch container", zap.Error(err))
|
||||||
|
|
||||||
if client.IsErrContainerNotFound(err) {
|
if client.IsErrContainerNotFound(err) {
|
||||||
return nil, errors.GetAPIError(errors.ErrNoSuchBucket)
|
return nil, errors.GetAPIError(errors.ErrNoSuchBucket)
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("get neofs container: %w", err)
|
return nil, fmt.Errorf("get frostfs container: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cnr := *res
|
cnr := *res
|
||||||
|
@ -57,6 +57,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
|
||||||
info.Owner = cnr.Owner()
|
info.Owner = cnr.Owner()
|
||||||
if domain := container.ReadDomain(cnr); domain.Name() != "" {
|
if domain := container.ReadDomain(cnr); domain.Name() != "" {
|
||||||
info.Name = domain.Name()
|
info.Name = domain.Name()
|
||||||
|
info.Zone = domain.Zone()
|
||||||
}
|
}
|
||||||
info.Created = container.CreatedAt(cnr)
|
info.Created = container.CreatedAt(cnr)
|
||||||
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
|
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
|
||||||
|
@ -84,7 +85,7 @@ func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||||
res []cid.ID
|
res []cid.ID
|
||||||
rid = api.GetRequestID(ctx)
|
rid = api.GetRequestID(ctx)
|
||||||
)
|
)
|
||||||
res, err = n.neoFS.UserContainers(ctx, own)
|
res, err = n.frostFS.UserContainers(ctx, own)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.log.Error("could not list user containers",
|
n.log.Error("could not list user containers",
|
||||||
zap.String("request_id", rid),
|
zap.String("request_id", rid),
|
||||||
|
@ -115,8 +116,9 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
||||||
}
|
}
|
||||||
bktInfo := &data.BucketInfo{
|
bktInfo := &data.BucketInfo{
|
||||||
Name: p.Name,
|
Name: p.Name,
|
||||||
|
Zone: v2container.SysAttributeZoneDefault,
|
||||||
Owner: ownerID,
|
Owner: ownerID,
|
||||||
Created: time.Now(), // this can be a little incorrect since the real time is set later
|
Created: TimeNow(ctx),
|
||||||
LocationConstraint: p.LocationConstraint,
|
LocationConstraint: p.LocationConstraint,
|
||||||
ObjectLockEnabled: p.ObjectLockEnabled,
|
ObjectLockEnabled: p.ObjectLockEnabled,
|
||||||
}
|
}
|
||||||
|
@ -133,11 +135,12 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
idCnr, err := n.neoFS.CreateContainer(ctx, PrmContainerCreate{
|
idCnr, err := n.frostFS.CreateContainer(ctx, PrmContainerCreate{
|
||||||
Creator: bktInfo.Owner,
|
Creator: bktInfo.Owner,
|
||||||
Policy: p.Policy,
|
Policy: p.Policy,
|
||||||
Name: p.Name,
|
Name: p.Name,
|
||||||
SessionToken: p.SessionContainerCreation,
|
SessionToken: p.SessionContainerCreation,
|
||||||
|
CreationTime: bktInfo.Created,
|
||||||
AdditionalAttributes: attributes,
|
AdditionalAttributes: attributes,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -158,9 +161,9 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
||||||
func (n *layer) setContainerEACLTable(ctx context.Context, idCnr cid.ID, table *eacl.Table, sessionToken *session.Container) error {
|
func (n *layer) setContainerEACLTable(ctx context.Context, idCnr cid.ID, table *eacl.Table, sessionToken *session.Container) error {
|
||||||
table.SetCID(idCnr)
|
table.SetCID(idCnr)
|
||||||
|
|
||||||
return n.neoFS.SetContainerEACL(ctx, *table, sessionToken)
|
return n.frostFS.SetContainerEACL(ctx, *table, sessionToken)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) GetContainerEACL(ctx context.Context, idCnr cid.ID) (*eacl.Table, error) {
|
func (n *layer) GetContainerEACL(ctx context.Context, idCnr cid.ID) (*eacl.Table, error) {
|
||||||
return n.neoFS.ContainerEACL(ctx, idCnr)
|
return n.frostFS.ContainerEACL(ctx, idCnr)
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,8 +8,8 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -39,8 +39,9 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
||||||
prm := PrmObjectCreate{
|
prm := PrmObjectCreate{
|
||||||
Container: p.BktInfo.CID,
|
Container: p.BktInfo.CID,
|
||||||
Creator: p.BktInfo.Owner,
|
Creator: p.BktInfo.Owner,
|
||||||
Payload: p.Reader,
|
Payload: &buf,
|
||||||
Filepath: p.BktInfo.CORSObjectName(),
|
Filepath: p.BktInfo.CORSObjectName(),
|
||||||
|
CreationTime: TimeNow(ctx),
|
||||||
CopiesNumber: p.CopiesNumber,
|
CopiesNumber: p.CopiesNumber,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -7,21 +7,21 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/container/acl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
)
|
)
|
||||||
|
|
||||||
// PrmContainerCreate groups parameters of NeoFS.CreateContainer operation.
|
// PrmContainerCreate groups parameters of FrostFS.CreateContainer operation.
|
||||||
type PrmContainerCreate struct {
|
type PrmContainerCreate struct {
|
||||||
// NeoFS identifier of the container creator.
|
// FrostFS identifier of the container creator.
|
||||||
Creator user.ID
|
Creator user.ID
|
||||||
|
|
||||||
// Container placement policy.
|
// Container placement policy.
|
||||||
|
@ -30,6 +30,9 @@ type PrmContainerCreate struct {
|
||||||
// Name for the container.
|
// Name for the container.
|
||||||
Name string
|
Name string
|
||||||
|
|
||||||
|
// CreationTime value for Timestamp attribute
|
||||||
|
CreationTime time.Time
|
||||||
|
|
||||||
// Token of the container's creation session. Nil means session absence.
|
// Token of the container's creation session. Nil means session absence.
|
||||||
SessionToken *session.Container
|
SessionToken *session.Container
|
||||||
|
|
||||||
|
@ -40,7 +43,7 @@ type PrmContainerCreate struct {
|
||||||
AdditionalAttributes [][2]string
|
AdditionalAttributes [][2]string
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrmAuth groups authentication parameters for the NeoFS operation.
|
// PrmAuth groups authentication parameters for the FrostFS operation.
|
||||||
type PrmAuth struct {
|
type PrmAuth struct {
|
||||||
// Bearer token to be used for the operation. Overlaps PrivateKey. Optional.
|
// Bearer token to be used for the operation. Overlaps PrivateKey. Optional.
|
||||||
BearerToken *bearer.Token
|
BearerToken *bearer.Token
|
||||||
|
@ -49,7 +52,7 @@ type PrmAuth struct {
|
||||||
PrivateKey *ecdsa.PrivateKey
|
PrivateKey *ecdsa.PrivateKey
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrmObjectRead groups parameters of NeoFS.ReadObject operation.
|
// PrmObjectRead groups parameters of FrostFS.ReadObject operation.
|
||||||
type PrmObjectRead struct {
|
type PrmObjectRead struct {
|
||||||
// Authentication parameters.
|
// Authentication parameters.
|
||||||
PrmAuth
|
PrmAuth
|
||||||
|
@ -70,7 +73,7 @@ type PrmObjectRead struct {
|
||||||
PayloadRange [2]uint64
|
PayloadRange [2]uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
// ObjectPart represents partially read NeoFS object.
|
// ObjectPart represents partially read FrostFS object.
|
||||||
type ObjectPart struct {
|
type ObjectPart struct {
|
||||||
// Object header with optional in-memory payload part.
|
// Object header with optional in-memory payload part.
|
||||||
Head *object.Object
|
Head *object.Object
|
||||||
|
@ -80,7 +83,7 @@ type ObjectPart struct {
|
||||||
Payload io.ReadCloser
|
Payload io.ReadCloser
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrmObjectCreate groups parameters of NeoFS.CreateObject operation.
|
// PrmObjectCreate groups parameters of FrostFS.CreateObject operation.
|
||||||
type PrmObjectCreate struct {
|
type PrmObjectCreate struct {
|
||||||
// Authentication parameters.
|
// Authentication parameters.
|
||||||
PrmAuth
|
PrmAuth
|
||||||
|
@ -88,12 +91,15 @@ type PrmObjectCreate struct {
|
||||||
// Container to store the object.
|
// Container to store the object.
|
||||||
Container cid.ID
|
Container cid.ID
|
||||||
|
|
||||||
// NeoFS identifier of the object creator.
|
// FrostFS identifier of the object creator.
|
||||||
Creator user.ID
|
Creator user.ID
|
||||||
|
|
||||||
// Key-value object attributes.
|
// Key-value object attributes.
|
||||||
Attributes [][2]string
|
Attributes [][2]string
|
||||||
|
|
||||||
|
// Value for Timestamp attribute (optional).
|
||||||
|
CreationTime time.Time
|
||||||
|
|
||||||
// List of ids to lock (optional).
|
// List of ids to lock (optional).
|
||||||
Locks []oid.ID
|
Locks []oid.ID
|
||||||
|
|
||||||
|
@ -110,7 +116,7 @@ type PrmObjectCreate struct {
|
||||||
CopiesNumber uint32
|
CopiesNumber uint32
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrmObjectDelete groups parameters of NeoFS.DeleteObject operation.
|
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
|
||||||
type PrmObjectDelete struct {
|
type PrmObjectDelete struct {
|
||||||
// Authentication parameters.
|
// Authentication parameters.
|
||||||
PrmAuth
|
PrmAuth
|
||||||
|
@ -122,12 +128,12 @@ type PrmObjectDelete struct {
|
||||||
Object oid.ID
|
Object oid.ID
|
||||||
}
|
}
|
||||||
|
|
||||||
// ErrAccessDenied is returned from NeoFS in case of access violation.
|
// ErrAccessDenied is returned from FrostFS in case of access violation.
|
||||||
var ErrAccessDenied = errors.New("access denied")
|
var ErrAccessDenied = errors.New("access denied")
|
||||||
|
|
||||||
// NeoFS represents virtual connection to NeoFS network.
|
// FrostFS represents virtual connection to FrostFS network.
|
||||||
type NeoFS interface {
|
type FrostFS interface {
|
||||||
// CreateContainer creates and saves parameterized container in NeoFS.
|
// CreateContainer creates and saves parameterized container in FrostFS.
|
||||||
// It sets 'Timestamp' attribute to the current time.
|
// It sets 'Timestamp' attribute to the current time.
|
||||||
// It returns the ID of the saved container.
|
// It returns the ID of the saved container.
|
||||||
//
|
//
|
||||||
|
@ -137,7 +143,7 @@ type NeoFS interface {
|
||||||
// prevented the container from being created.
|
// prevented the container from being created.
|
||||||
CreateContainer(context.Context, PrmContainerCreate) (cid.ID, error)
|
CreateContainer(context.Context, PrmContainerCreate) (cid.ID, error)
|
||||||
|
|
||||||
// Container reads a container from NeoFS by ID.
|
// Container reads a container from FrostFS by ID.
|
||||||
//
|
//
|
||||||
// It returns exactly one non-nil value. It returns any error encountered which
|
// It returns exactly one non-nil value. It returns any error encountered which
|
||||||
// prevented the container from being read.
|
// prevented the container from being read.
|
||||||
|
@ -149,26 +155,26 @@ type NeoFS interface {
|
||||||
// prevented the containers from being listed.
|
// prevented the containers from being listed.
|
||||||
UserContainers(context.Context, user.ID) ([]cid.ID, error)
|
UserContainers(context.Context, user.ID) ([]cid.ID, error)
|
||||||
|
|
||||||
// SetContainerEACL saves the eACL table of the container in NeoFS. The
|
// SetContainerEACL saves the eACL table of the container in FrostFS. The
|
||||||
// extended ACL is modified within session if session token is not nil.
|
// extended ACL is modified within session if session token is not nil.
|
||||||
//
|
//
|
||||||
// It returns any error encountered which prevented the eACL from being saved.
|
// It returns any error encountered which prevented the eACL from being saved.
|
||||||
SetContainerEACL(context.Context, eacl.Table, *session.Container) error
|
SetContainerEACL(context.Context, eacl.Table, *session.Container) error
|
||||||
|
|
||||||
// ContainerEACL reads the container eACL from NeoFS by the container ID.
|
// ContainerEACL reads the container eACL from FrostFS by the container ID.
|
||||||
//
|
//
|
||||||
// It returns exactly one non-nil value. It returns any error encountered which
|
// It returns exactly one non-nil value. It returns any error encountered which
|
||||||
// prevented the eACL from being read.
|
// prevented the eACL from being read.
|
||||||
ContainerEACL(context.Context, cid.ID) (*eacl.Table, error)
|
ContainerEACL(context.Context, cid.ID) (*eacl.Table, error)
|
||||||
|
|
||||||
// DeleteContainer marks the container to be removed from NeoFS by ID.
|
// DeleteContainer marks the container to be removed from FrostFS by ID.
|
||||||
// Request is sent within session if the session token is specified.
|
// Request is sent within session if the session token is specified.
|
||||||
// Successful return does not guarantee actual removal.
|
// Successful return does not guarantee actual removal.
|
||||||
//
|
//
|
||||||
// It returns any error encountered which prevented the removal request from being sent.
|
// It returns any error encountered which prevented the removal request from being sent.
|
||||||
DeleteContainer(context.Context, cid.ID, *session.Container) error
|
DeleteContainer(context.Context, cid.ID, *session.Container) error
|
||||||
|
|
||||||
// ReadObject reads a part of the object from the NeoFS container by identifier.
|
// ReadObject reads a part of the object from the FrostFS container by identifier.
|
||||||
// Exact part is returned according to the parameters:
|
// Exact part is returned according to the parameters:
|
||||||
// * with header only: empty payload (both in-mem and reader parts are nil);
|
// * with header only: empty payload (both in-mem and reader parts are nil);
|
||||||
// * with payload only: header is nil (zero range means full payload);
|
// * with payload only: header is nil (zero range means full payload);
|
||||||
|
@ -184,7 +190,7 @@ type NeoFS interface {
|
||||||
// prevented the object header from being read.
|
// prevented the object header from being read.
|
||||||
ReadObject(context.Context, PrmObjectRead) (*ObjectPart, error)
|
ReadObject(context.Context, PrmObjectRead) (*ObjectPart, error)
|
||||||
|
|
||||||
// CreateObject creates and saves a parameterized object in the NeoFS container.
|
// CreateObject creates and saves a parameterized object in the FrostFS container.
|
||||||
// It sets 'Timestamp' attribute to the current time.
|
// It sets 'Timestamp' attribute to the current time.
|
||||||
// It returns the ID of the saved object.
|
// It returns the ID of the saved object.
|
||||||
//
|
//
|
||||||
|
@ -196,7 +202,7 @@ type NeoFS interface {
|
||||||
// prevented the container from being created.
|
// prevented the container from being created.
|
||||||
CreateObject(context.Context, PrmObjectCreate) (oid.ID, error)
|
CreateObject(context.Context, PrmObjectCreate) (oid.ID, error)
|
||||||
|
|
||||||
// DeleteObject marks the object to be removed from the NeoFS container by identifier.
|
// DeleteObject marks the object to be removed from the FrostFS container by identifier.
|
||||||
// Successful return does not guarantee actual removal.
|
// Successful return does not guarantee actual removal.
|
||||||
//
|
//
|
||||||
// It returns ErrAccessDenied on remove access violation.
|
// It returns ErrAccessDenied on remove access violation.
|
||||||
|
@ -204,11 +210,11 @@ type NeoFS interface {
|
||||||
// It returns any error encountered which prevented the removal request from being sent.
|
// It returns any error encountered which prevented the removal request from being sent.
|
||||||
DeleteObject(context.Context, PrmObjectDelete) error
|
DeleteObject(context.Context, PrmObjectDelete) error
|
||||||
|
|
||||||
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided time.
|
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided now and future time.
|
||||||
// Note:
|
// Note:
|
||||||
// * time must be in the future
|
// * future time must be after the now
|
||||||
// * time will be ceil rounded to match epoch
|
// * future time will be ceil rounded to match epoch
|
||||||
//
|
//
|
||||||
// It returns any error encountered which prevented computing epochs.
|
// It returns any error encountered which prevented computing epochs.
|
||||||
TimeToEpoch(context.Context, time.Time) (uint64, uint64, error)
|
TimeToEpoch(ctx context.Context, now time.Time, future time.Time) (uint64, uint64, error)
|
||||||
}
|
}
|
|
@ -11,20 +11,20 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -45,7 +45,7 @@ type (
|
||||||
}
|
}
|
||||||
|
|
||||||
layer struct {
|
layer struct {
|
||||||
neoFS NeoFS
|
frostFS FrostFS
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
anonKey AnonymousKey
|
anonKey AnonymousKey
|
||||||
resolver BucketResolver
|
resolver BucketResolver
|
||||||
|
@ -250,12 +250,12 @@ const (
|
||||||
|
|
||||||
AESEncryptionAlgorithm = "AES256"
|
AESEncryptionAlgorithm = "AES256"
|
||||||
AESKeySize = 32
|
AESKeySize = 32
|
||||||
AttributeEncryptionAlgorithm = api.NeoFSSystemMetadataPrefix + "Algorithm"
|
AttributeEncryptionAlgorithm = api.FrostFSSystemMetadataPrefix + "Algorithm"
|
||||||
AttributeDecryptedSize = api.NeoFSSystemMetadataPrefix + "Decrypted-Size"
|
AttributeDecryptedSize = api.FrostFSSystemMetadataPrefix + "Decrypted-Size"
|
||||||
AttributeHMACSalt = api.NeoFSSystemMetadataPrefix + "HMAC-Salt"
|
AttributeHMACSalt = api.FrostFSSystemMetadataPrefix + "HMAC-Salt"
|
||||||
AttributeHMACKey = api.NeoFSSystemMetadataPrefix + "HMAC-Key"
|
AttributeHMACKey = api.FrostFSSystemMetadataPrefix + "HMAC-Key"
|
||||||
|
|
||||||
AttributeNeofsCopiesNumber = "neofs-copies-number" // such formate to match X-Amz-Meta-Neofs-Copies-Number header
|
AttributeFrostfsCopiesNumber = "frostfs-copies-number" // such format to match X-Amz-Meta-Frostfs-Copies-Number header
|
||||||
)
|
)
|
||||||
|
|
||||||
func (t *VersionedObject) String() string {
|
func (t *VersionedObject) String() string {
|
||||||
|
@ -268,9 +268,9 @@ func (f MsgHandlerFunc) HandleMessage(ctx context.Context, msg *nats.Msg) error
|
||||||
|
|
||||||
// NewLayer creates an instance of a layer. It checks credentials
|
// NewLayer creates an instance of a layer. It checks credentials
|
||||||
// and establishes gRPC connection with the node.
|
// and establishes gRPC connection with the node.
|
||||||
func NewLayer(log *zap.Logger, neoFS NeoFS, config *Config) Client {
|
func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) Client {
|
||||||
return &layer{
|
return &layer{
|
||||||
neoFS: neoFS,
|
frostFS: frostFS,
|
||||||
log: log,
|
log: log,
|
||||||
anonKey: config.AnonKey,
|
anonKey: config.AnonKey,
|
||||||
resolver: config.Resolver,
|
resolver: config.Resolver,
|
||||||
|
@ -306,6 +306,15 @@ func IsAuthenticatedRequest(ctx context.Context) bool {
|
||||||
return ok
|
return ok
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TimeNow returns client time from request or time.Now().
|
||||||
|
func TimeNow(ctx context.Context) time.Time {
|
||||||
|
if now, ok := ctx.Value(api.ClientTime).(time.Time); ok {
|
||||||
|
return now
|
||||||
|
}
|
||||||
|
|
||||||
|
return time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
// Owner returns owner id from BearerToken (context) or from client owner.
|
// Owner returns owner id from BearerToken (context) or from client owner.
|
||||||
func (n *layer) Owner(ctx context.Context) user.ID {
|
func (n *layer) Owner(ctx context.Context) user.ID {
|
||||||
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
|
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
|
||||||
|
@ -368,7 +377,7 @@ func (n *layer) PutBucketACL(ctx context.Context, param *PutBucketACLParams) err
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListBuckets returns all user containers. The name of the bucket is a container
|
// ListBuckets returns all user containers. The name of the bucket is a container
|
||||||
// id. Timestamp is omitted since it is not saved in neofs container.
|
// id. Timestamp is omitted since it is not saved in frostfs container.
|
||||||
func (n *layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) {
|
func (n *layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||||
return n.containerList(ctx)
|
return n.containerList(ctx)
|
||||||
}
|
}
|
||||||
|
@ -473,11 +482,27 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.O
|
||||||
|
|
||||||
// GetExtendedObjectInfo returns meta information and corresponding info from the tree service about the object.
|
// GetExtendedObjectInfo returns meta information and corresponding info from the tree service about the object.
|
||||||
func (n *layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) {
|
func (n *layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) {
|
||||||
|
var objInfo *data.ExtendedObjectInfo
|
||||||
|
var err error
|
||||||
|
|
||||||
if len(p.VersionID) == 0 {
|
if len(p.VersionID) == 0 {
|
||||||
return n.headLastVersionIfNotDeleted(ctx, p.BktInfo, p.Object)
|
objInfo, err = n.headLastVersionIfNotDeleted(ctx, p.BktInfo, p.Object)
|
||||||
|
} else {
|
||||||
|
objInfo, err = n.headVersion(ctx, p.BktInfo, p)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return n.headVersion(ctx, p.BktInfo, p)
|
reqInfo := api.GetReqInfo(ctx)
|
||||||
|
n.log.Debug("get object",
|
||||||
|
zap.String("reqId", reqInfo.RequestID),
|
||||||
|
zap.String("bucket", p.BktInfo.Name),
|
||||||
|
zap.Stringer("cid", p.BktInfo.CID),
|
||||||
|
zap.String("object", objInfo.ObjectInfo.Name),
|
||||||
|
zap.Stringer("oid", objInfo.ObjectInfo.ID))
|
||||||
|
|
||||||
|
return objInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CopyObject from one bucket into another bucket.
|
// CopyObject from one bucket into another bucket.
|
||||||
|
@ -565,7 +590,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
||||||
FilePath: obj.Name,
|
FilePath: obj.Name,
|
||||||
},
|
},
|
||||||
DeleteMarker: &data.DeleteMarkerInfo{
|
DeleteMarker: &data.DeleteMarkerInfo{
|
||||||
Created: time.Now(),
|
Created: TimeNow(ctx),
|
||||||
Owner: n.Owner(ctx),
|
Owner: n.Owner(ctx),
|
||||||
},
|
},
|
||||||
IsUnversioned: settings.VersioningSuspended(),
|
IsUnversioned: settings.VersioningSuspended(),
|
||||||
|
@ -636,7 +661,12 @@ func (n *layer) CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.
|
||||||
func (n *layer) ResolveBucket(ctx context.Context, name string) (cid.ID, error) {
|
func (n *layer) ResolveBucket(ctx context.Context, name string) (cid.ID, error) {
|
||||||
var cnrID cid.ID
|
var cnrID cid.ID
|
||||||
if err := cnrID.DecodeString(name); err != nil {
|
if err := cnrID.DecodeString(name); err != nil {
|
||||||
return n.resolver.Resolve(ctx, name)
|
if cnrID, err = n.resolver.Resolve(ctx, name); err != nil {
|
||||||
|
return cid.ID{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
reqInfo := api.GetReqInfo(ctx)
|
||||||
|
n.log.Info("resolve bucket", zap.String("reqId", reqInfo.RequestID), zap.String("bucket", name), zap.Stringer("cid", cnrID))
|
||||||
}
|
}
|
||||||
|
|
||||||
return cnrID, nil
|
return cnrID, nil
|
||||||
|
@ -652,5 +682,5 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
n.cache.DeleteBucket(p.BktInfo.Name)
|
n.cache.DeleteBucket(p.BktInfo.Name)
|
||||||
return n.neoFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
return n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,8 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -43,10 +44,10 @@ func TestObjectLockAttributes(t *testing.T) {
|
||||||
|
|
||||||
expEpoch := false
|
expEpoch := false
|
||||||
for _, attr := range lockObj.Attributes() {
|
for _, attr := range lockObj.Attributes() {
|
||||||
if attr.Key() == AttributeExpirationEpoch {
|
if attr.Key() == object.SysAttributeExpEpoch {
|
||||||
expEpoch = true
|
expEpoch = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
require.Truef(t, expEpoch, "system header __NEOFS__EXPIRATION_EPOCH presence")
|
require.Truef(t, expEpoch, "system header __SYSTEM__EXPIRATION_EPOCH presence")
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,12 +11,13 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/minio/sio"
|
"github.com/minio/sio"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -143,7 +144,7 @@ func (n *layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
|
||||||
Key: p.Info.Key,
|
Key: p.Info.Key,
|
||||||
UploadID: p.Info.UploadID,
|
UploadID: p.Info.UploadID,
|
||||||
Owner: n.Owner(ctx),
|
Owner: n.Owner(ctx),
|
||||||
Created: time.Now(),
|
Created: TimeNow(ctx),
|
||||||
Meta: make(map[string]string, metaSize),
|
Meta: make(map[string]string, metaSize),
|
||||||
CopiesNumber: p.CopiesNumber,
|
CopiesNumber: p.CopiesNumber,
|
||||||
}
|
}
|
||||||
|
@ -205,6 +206,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
Creator: bktInfo.Owner,
|
Creator: bktInfo.Owner,
|
||||||
Attributes: make([][2]string, 2),
|
Attributes: make([][2]string, 2),
|
||||||
Payload: p.Reader,
|
Payload: p.Reader,
|
||||||
|
CreationTime: TimeNow(ctx),
|
||||||
CopiesNumber: multipartInfo.CopiesNumber,
|
CopiesNumber: multipartInfo.CopiesNumber,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -227,6 +229,13 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
reqInfo := api.GetReqInfo(ctx)
|
||||||
|
n.log.Debug("upload part",
|
||||||
|
zap.String("reqId", reqInfo.RequestID),
|
||||||
|
zap.String("bucket", bktInfo.Name), zap.Stringer("cid", bktInfo.CID),
|
||||||
|
zap.String("multipart upload", p.Info.UploadID),
|
||||||
|
zap.Int("part number", p.PartNumber), zap.String("object", p.Info.Key), zap.Stringer("oid", id))
|
||||||
|
|
||||||
partInfo := &data.PartInfo{
|
partInfo := &data.PartInfo{
|
||||||
Key: p.Info.Key,
|
Key: p.Info.Key,
|
||||||
UploadID: p.Info.UploadID,
|
UploadID: p.Info.UploadID,
|
||||||
|
@ -234,7 +243,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
OID: id,
|
OID: id,
|
||||||
Size: decSize,
|
Size: decSize,
|
||||||
ETag: hex.EncodeToString(hash),
|
ETag: hex.EncodeToString(hash),
|
||||||
Created: time.Now(),
|
Created: prm.CreationTime,
|
||||||
}
|
}
|
||||||
|
|
||||||
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)
|
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)
|
||||||
|
@ -310,7 +319,7 @@ func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
|
||||||
return n.uploadPart(ctx, multipartInfo, params)
|
return n.uploadPart(ctx, multipartInfo, params)
|
||||||
}
|
}
|
||||||
|
|
||||||
// implements io.Reader of payloads of the object list stored in the NeoFS network.
|
// implements io.Reader of payloads of the object list stored in the FrostFS network.
|
||||||
type multiObjectReader struct {
|
type multiObjectReader struct {
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
|
|
||||||
|
@ -608,10 +617,24 @@ func (n *layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
|
||||||
}
|
}
|
||||||
|
|
||||||
res := make(map[int]*data.PartInfo, len(parts))
|
res := make(map[int]*data.PartInfo, len(parts))
|
||||||
for _, part := range parts {
|
partsNumbers := make([]int, len(parts))
|
||||||
|
oids := make([]string, len(parts))
|
||||||
|
for i, part := range parts {
|
||||||
res[part.Number] = part
|
res[part.Number] = part
|
||||||
|
partsNumbers[i] = part.Number
|
||||||
|
oids[i] = part.OID.EncodeToString()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
reqInfo := api.GetReqInfo(ctx)
|
||||||
|
n.log.Debug("part details",
|
||||||
|
zap.String("reqId", reqInfo.RequestID),
|
||||||
|
zap.String("bucket", p.Bkt.Name),
|
||||||
|
zap.Stringer("cid", p.Bkt.CID),
|
||||||
|
zap.String("object", p.Key),
|
||||||
|
zap.String("upload id", p.UploadID),
|
||||||
|
zap.Ints("part numbers", partsNumbers),
|
||||||
|
zap.Strings("oids", oids))
|
||||||
|
|
||||||
return multipartInfo, res, nil
|
return multipartInfo, res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -10,22 +10,22 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
objectv2 "github.com/nspcc-dev/neofs-api-go/v2/object"
|
objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/checksum"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
)
|
)
|
||||||
|
|
||||||
type TestNeoFS struct {
|
type TestFrostFS struct {
|
||||||
NeoFS
|
FrostFS
|
||||||
|
|
||||||
objects map[string]*object.Object
|
objects map[string]*object.Object
|
||||||
containers map[string]*container.Container
|
containers map[string]*container.Container
|
||||||
|
@ -33,19 +33,19 @@ type TestNeoFS struct {
|
||||||
currentEpoch uint64
|
currentEpoch uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewTestNeoFS() *TestNeoFS {
|
func NewTestFrostFS() *TestFrostFS {
|
||||||
return &TestNeoFS{
|
return &TestFrostFS{
|
||||||
objects: make(map[string]*object.Object),
|
objects: make(map[string]*object.Object),
|
||||||
containers: make(map[string]*container.Container),
|
containers: make(map[string]*container.Container),
|
||||||
eaclTables: make(map[string]*eacl.Table),
|
eaclTables: make(map[string]*eacl.Table),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) CurrentEpoch() uint64 {
|
func (t *TestFrostFS) CurrentEpoch() uint64 {
|
||||||
return t.currentEpoch
|
return t.currentEpoch
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) Objects() []*object.Object {
|
func (t *TestFrostFS) Objects() []*object.Object {
|
||||||
res := make([]*object.Object, 0, len(t.objects))
|
res := make([]*object.Object, 0, len(t.objects))
|
||||||
|
|
||||||
for _, obj := range t.objects {
|
for _, obj := range t.objects {
|
||||||
|
@ -55,11 +55,11 @@ func (t *TestNeoFS) Objects() []*object.Object {
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) AddObject(key string, obj *object.Object) {
|
func (t *TestFrostFS) AddObject(key string, obj *object.Object) {
|
||||||
t.objects[key] = obj
|
t.objects[key] = obj
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) ContainerID(name string) (cid.ID, error) {
|
func (t *TestFrostFS) ContainerID(name string) (cid.ID, error) {
|
||||||
for id, cnr := range t.containers {
|
for id, cnr := range t.containers {
|
||||||
if container.Name(*cnr) == name {
|
if container.Name(*cnr) == name {
|
||||||
var cnrID cid.ID
|
var cnrID cid.ID
|
||||||
|
@ -69,13 +69,18 @@ func (t *TestNeoFS) ContainerID(name string) (cid.ID, error) {
|
||||||
return cid.ID{}, fmt.Errorf("not found")
|
return cid.ID{}, fmt.Errorf("not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (cid.ID, error) {
|
func (t *TestFrostFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (cid.ID, error) {
|
||||||
var cnr container.Container
|
var cnr container.Container
|
||||||
cnr.Init()
|
cnr.Init()
|
||||||
cnr.SetOwner(prm.Creator)
|
cnr.SetOwner(prm.Creator)
|
||||||
cnr.SetPlacementPolicy(prm.Policy)
|
cnr.SetPlacementPolicy(prm.Policy)
|
||||||
cnr.SetBasicACL(prm.BasicACL)
|
cnr.SetBasicACL(prm.BasicACL)
|
||||||
container.SetCreationTime(&cnr, time.Now())
|
|
||||||
|
creationTime := prm.CreationTime
|
||||||
|
if creationTime.IsZero() {
|
||||||
|
creationTime = time.Now()
|
||||||
|
}
|
||||||
|
container.SetCreationTime(&cnr, creationTime)
|
||||||
|
|
||||||
if prm.Name != "" {
|
if prm.Name != "" {
|
||||||
var d container.Domain
|
var d container.Domain
|
||||||
|
@ -101,13 +106,13 @@ func (t *TestNeoFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (
|
||||||
return id, nil
|
return id, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) DeleteContainer(_ context.Context, cnrID cid.ID, _ *session.Container) error {
|
func (t *TestFrostFS) DeleteContainer(_ context.Context, cnrID cid.ID, _ *session.Container) error {
|
||||||
delete(t.containers, cnrID.EncodeToString())
|
delete(t.containers, cnrID.EncodeToString())
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) Container(_ context.Context, id cid.ID) (*container.Container, error) {
|
func (t *TestFrostFS) Container(_ context.Context, id cid.ID) (*container.Container, error) {
|
||||||
for k, v := range t.containers {
|
for k, v := range t.containers {
|
||||||
if k == id.EncodeToString() {
|
if k == id.EncodeToString() {
|
||||||
return v, nil
|
return v, nil
|
||||||
|
@ -117,7 +122,7 @@ func (t *TestNeoFS) Container(_ context.Context, id cid.ID) (*container.Containe
|
||||||
return nil, fmt.Errorf("container not found %s", id)
|
return nil, fmt.Errorf("container not found %s", id)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) UserContainers(_ context.Context, _ user.ID) ([]cid.ID, error) {
|
func (t *TestFrostFS) UserContainers(_ context.Context, _ user.ID) ([]cid.ID, error) {
|
||||||
var res []cid.ID
|
var res []cid.ID
|
||||||
for k := range t.containers {
|
for k := range t.containers {
|
||||||
var idCnr cid.ID
|
var idCnr cid.ID
|
||||||
|
@ -130,7 +135,7 @@ func (t *TestNeoFS) UserContainers(_ context.Context, _ user.ID) ([]cid.ID, erro
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectPart, error) {
|
func (t *TestFrostFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectPart, error) {
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(prm.Container)
|
addr.SetContainer(prm.Container)
|
||||||
addr.SetObject(prm.Object)
|
addr.SetObject(prm.Object)
|
||||||
|
@ -139,7 +144,7 @@ func (t *TestNeoFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectP
|
||||||
|
|
||||||
if obj, ok := t.objects[sAddr]; ok {
|
if obj, ok := t.objects[sAddr]; ok {
|
||||||
owner := getOwner(ctx)
|
owner := getOwner(ctx)
|
||||||
if !obj.OwnerID().Equals(owner) {
|
if !obj.OwnerID().Equals(owner) && !t.isPublicRead(prm.Container) {
|
||||||
return nil, ErrAccessDenied
|
return nil, ErrAccessDenied
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -159,7 +164,7 @@ func (t *TestNeoFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectP
|
||||||
return nil, fmt.Errorf("object not found %s", addr)
|
return nil, fmt.Errorf("object not found %s", addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oid.ID, error) {
|
func (t *TestFrostFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oid.ID, error) {
|
||||||
b := make([]byte, 32)
|
b := make([]byte, 32)
|
||||||
if _, err := io.ReadFull(rand.Reader, b); err != nil {
|
if _, err := io.ReadFull(rand.Reader, b); err != nil {
|
||||||
return oid.ID{}, err
|
return oid.ID{}, err
|
||||||
|
@ -218,7 +223,7 @@ func (t *TestNeoFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oid.
|
||||||
return objID, nil
|
return objID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
|
func (t *TestFrostFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(prm.Container)
|
addr.SetContainer(prm.Container)
|
||||||
addr.SetObject(prm.Object)
|
addr.SetObject(prm.Object)
|
||||||
|
@ -235,11 +240,11 @@ func (t *TestNeoFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) TimeToEpoch(_ context.Context, futureTime time.Time) (uint64, uint64, error) {
|
func (t *TestFrostFS) TimeToEpoch(_ context.Context, now, futureTime time.Time) (uint64, uint64, error) {
|
||||||
return t.currentEpoch, t.currentEpoch + uint64(futureTime.Second()), nil
|
return t.currentEpoch, t.currentEpoch + uint64(futureTime.Sub(now).Seconds()), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) AllObjects(cnrID cid.ID) []oid.ID {
|
func (t *TestFrostFS) AllObjects(cnrID cid.ID) []oid.ID {
|
||||||
result := make([]oid.ID, 0, len(t.objects))
|
result := make([]oid.ID, 0, len(t.objects))
|
||||||
|
|
||||||
for _, val := range t.objects {
|
for _, val := range t.objects {
|
||||||
|
@ -253,7 +258,7 @@ func (t *TestNeoFS) AllObjects(cnrID cid.ID) []oid.ID {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *session.Container) error {
|
func (t *TestFrostFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *session.Container) error {
|
||||||
cnrID, ok := table.CID()
|
cnrID, ok := table.CID()
|
||||||
if !ok {
|
if !ok {
|
||||||
return errors.New("invalid cid")
|
return errors.New("invalid cid")
|
||||||
|
@ -268,7 +273,7 @@ func (t *TestNeoFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *ses
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestNeoFS) ContainerEACL(_ context.Context, cnrID cid.ID) (*eacl.Table, error) {
|
func (t *TestFrostFS) ContainerEACL(_ context.Context, cnrID cid.ID) (*eacl.Table, error) {
|
||||||
table, ok := t.eaclTables[cnrID.EncodeToString()]
|
table, ok := t.eaclTables[cnrID.EncodeToString()]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, errors.New("not found")
|
return nil, errors.New("not found")
|
||||||
|
@ -277,6 +282,25 @@ func (t *TestNeoFS) ContainerEACL(_ context.Context, cnrID cid.ID) (*eacl.Table,
|
||||||
return table, nil
|
return table, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (t *TestFrostFS) isPublicRead(cnrID cid.ID) bool {
|
||||||
|
table, ok := t.eaclTables[cnrID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, rec := range table.Records() {
|
||||||
|
if rec.Operation() == eacl.OperationGet && len(rec.Filters()) == 0 {
|
||||||
|
for _, trgt := range rec.Targets() {
|
||||||
|
if trgt.Role() == eacl.RoleOthers {
|
||||||
|
return rec.Action() == eacl.ActionAllow
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func getOwner(ctx context.Context) user.ID {
|
func getOwner(ctx context.Context) user.ID {
|
||||||
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
|
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
|
||||||
return bearer.ResolveIssuer(*bd.Gate.BearerToken)
|
return bearer.ResolveIssuer(*bd.Gate.BearerToken)
|
||||||
|
|
|
@ -7,8 +7,8 @@ import (
|
||||||
errorsStd "errors"
|
errorsStd "errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -30,6 +30,7 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
|
||||||
Creator: p.BktInfo.Owner,
|
Creator: p.BktInfo.Owner,
|
||||||
Payload: bytes.NewReader(confXML),
|
Payload: bytes.NewReader(confXML),
|
||||||
Filepath: p.BktInfo.NotificationConfigurationObjectName(),
|
Filepath: p.BktInfo.NotificationConfigurationObjectName(),
|
||||||
|
CreationTime: TimeNow(ctx),
|
||||||
CopiesNumber: p.CopiesNumber,
|
CopiesNumber: p.CopiesNumber,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -13,17 +13,16 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/minio/sio"
|
"github.com/minio/sio"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/client"
|
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
||||||
"github.com/panjf2000/ants/v2"
|
"github.com/panjf2000/ants/v2"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -91,7 +90,7 @@ func (n *layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj
|
||||||
|
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||||
|
|
||||||
res, err := n.neoFS.ReadObject(ctx, prm)
|
res, err := n.frostFS.ReadObject(ctx, prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -99,7 +98,7 @@ func (n *layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj
|
||||||
return res.Head, nil
|
return res.Head, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// initializes payload reader of the NeoFS object.
|
// initializes payload reader of the FrostFS object.
|
||||||
// Zero range corresponds to full payload (panics if only offset is set).
|
// Zero range corresponds to full payload (panics if only offset is set).
|
||||||
func (n *layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Reader, error) {
|
func (n *layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Reader, error) {
|
||||||
prm := PrmObjectRead{
|
prm := PrmObjectRead{
|
||||||
|
@ -111,7 +110,7 @@ func (n *layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Re
|
||||||
|
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, p.bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, p.bktInfo.Owner)
|
||||||
|
|
||||||
res, err := n.neoFS.ReadObject(ctx, prm)
|
res, err := n.frostFS.ReadObject(ctx, prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -130,7 +129,7 @@ func (n *layer) objectGet(ctx context.Context, bktInfo *data.BucketInfo, objID o
|
||||||
|
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||||
|
|
||||||
res, err := n.neoFS.ReadObject(ctx, prm)
|
res, err := n.frostFS.ReadObject(ctx, prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -183,7 +182,7 @@ func ParseCompletedPartHeader(hdr string) (*Part, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PutObject stores object into NeoFS, took payload from io.Reader.
|
// PutObject stores object into FrostFS, took payload from io.Reader.
|
||||||
func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error) {
|
func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error) {
|
||||||
owner := n.Owner(ctx)
|
owner := n.Owner(ctx)
|
||||||
|
|
||||||
|
@ -234,6 +233,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
PayloadSize: uint64(p.Size),
|
PayloadSize: uint64(p.Size),
|
||||||
Filepath: p.Object,
|
Filepath: p.Object,
|
||||||
Payload: r,
|
Payload: r,
|
||||||
|
CreationTime: TimeNow(ctx),
|
||||||
CopiesNumber: p.CopiesNumber,
|
CopiesNumber: p.CopiesNumber,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -248,6 +248,12 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
reqInfo := api.GetReqInfo(ctx)
|
||||||
|
n.log.Debug("put object",
|
||||||
|
zap.String("reqId", reqInfo.RequestID),
|
||||||
|
zap.String("bucket", p.BktInfo.Name), zap.Stringer("cid", p.BktInfo.CID),
|
||||||
|
zap.String("object", p.Object), zap.Stringer("oid", id))
|
||||||
|
|
||||||
newVersion.OID = id
|
newVersion.OID = id
|
||||||
newVersion.ETag = hex.EncodeToString(hash)
|
newVersion.ETag = hex.EncodeToString(hash)
|
||||||
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
|
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
|
||||||
|
@ -281,7 +287,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
Bucket: p.BktInfo.Name,
|
Bucket: p.BktInfo.Name,
|
||||||
Name: p.Object,
|
Name: p.Object,
|
||||||
Size: p.Size,
|
Size: p.Size,
|
||||||
Created: time.Now(),
|
Created: prm.CreationTime,
|
||||||
Headers: p.Header,
|
Headers: p.Header,
|
||||||
ContentType: p.Header[api.ContentType],
|
ContentType: p.Header[api.ContentType],
|
||||||
HashSum: newVersion.ETag,
|
HashSum: newVersion.ETag,
|
||||||
|
@ -383,7 +389,7 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
|
||||||
return extObjInfo, nil
|
return extObjInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectDelete puts tombstone object into neofs.
|
// objectDelete puts tombstone object into frostfs.
|
||||||
func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) error {
|
func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) error {
|
||||||
prm := PrmObjectDelete{
|
prm := PrmObjectDelete{
|
||||||
Container: bktInfo.CID,
|
Container: bktInfo.CID,
|
||||||
|
@ -394,10 +400,10 @@ func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idOb
|
||||||
|
|
||||||
n.cache.DeleteObject(newAddress(bktInfo.CID, idObj))
|
n.cache.DeleteObject(newAddress(bktInfo.CID, idObj))
|
||||||
|
|
||||||
return n.neoFS.DeleteObject(ctx, prm)
|
return n.frostFS.DeleteObject(ctx, prm)
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectPutAndHash prepare auth parameters and invoke neofs.CreateObject.
|
// objectPutAndHash prepare auth parameters and invoke frostfs.CreateObject.
|
||||||
// Returns object ID and payload sha256 hash.
|
// Returns object ID and payload sha256 hash.
|
||||||
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (oid.ID, []byte, error) {
|
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (oid.ID, []byte, error) {
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||||
|
@ -405,7 +411,7 @@ func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktIn
|
||||||
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
|
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
|
||||||
hash.Write(buf)
|
hash.Write(buf)
|
||||||
})
|
})
|
||||||
id, err := n.neoFS.CreateObject(ctx, prm)
|
id, err := n.frostFS.CreateObject(ctx, prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return oid.ID{}, nil, err
|
return oid.ID{}, nil, err
|
||||||
}
|
}
|
||||||
|
@ -577,10 +583,10 @@ func (n *layer) initWorkerPool(ctx context.Context, size int, p allObjectParams,
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
err = pool.Submit(func() {
|
err = pool.Submit(func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
oi := n.objectInfoFromObjectsCacheOrNeoFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter)
|
oi := n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter)
|
||||||
if oi == nil {
|
if oi == nil {
|
||||||
// try to get object again
|
// try to get object again
|
||||||
if oi = n.objectInfoFromObjectsCacheOrNeoFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter); oi == nil {
|
if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter); oi == nil {
|
||||||
// form object info with data that the tree node contains
|
// form object info with data that the tree node contains
|
||||||
oi = getPartialObjectInfo(p.Bucket, node)
|
oi = getPartialObjectInfo(p.Bucket, node)
|
||||||
}
|
}
|
||||||
|
@ -646,14 +652,14 @@ func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo,
|
||||||
for _, nodeVersion := range nodeVersions {
|
for _, nodeVersion := range nodeVersions {
|
||||||
oi := &data.ObjectInfo{}
|
oi := &data.ObjectInfo{}
|
||||||
|
|
||||||
if nodeVersion.IsDeleteMarker() { // delete marker does not match any object in NeoFS
|
if nodeVersion.IsDeleteMarker() { // delete marker does not match any object in FrostFS
|
||||||
oi.ID = nodeVersion.OID
|
oi.ID = nodeVersion.OID
|
||||||
oi.Name = nodeVersion.FilePath
|
oi.Name = nodeVersion.FilePath
|
||||||
oi.Owner = nodeVersion.DeleteMarker.Owner
|
oi.Owner = nodeVersion.DeleteMarker.Owner
|
||||||
oi.Created = nodeVersion.DeleteMarker.Created
|
oi.Created = nodeVersion.DeleteMarker.Created
|
||||||
oi.IsDeleteMarker = true
|
oi.IsDeleteMarker = true
|
||||||
} else {
|
} else {
|
||||||
if oi = n.objectInfoFromObjectsCacheOrNeoFS(ctx, bkt, nodeVersion, prefix, delimiter); oi == nil {
|
if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, bkt, nodeVersion, prefix, delimiter); oi == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -677,7 +683,7 @@ func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo,
|
||||||
|
|
||||||
func IsSystemHeader(key string) bool {
|
func IsSystemHeader(key string) bool {
|
||||||
_, ok := api.SystemMetadata[key]
|
_, ok := api.SystemMetadata[key]
|
||||||
return ok || strings.HasPrefix(key, api.NeoFSSystemMetadataPrefix)
|
return ok || strings.HasPrefix(key, api.FrostFSSystemMetadataPrefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
func shouldSkip(node *data.NodeVersion, p allObjectParams, existed map[string]struct{}) bool {
|
func shouldSkip(node *data.NodeVersion, p allObjectParams, existed map[string]struct{}) bool {
|
||||||
|
@ -734,7 +740,7 @@ func triageExtendedObjects(allObjects []*data.ExtendedObjectInfo) (prefixes []st
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) objectInfoFromObjectsCacheOrNeoFS(ctx context.Context, bktInfo *data.BucketInfo, node *data.NodeVersion, prefix, delimiter string) (oi *data.ObjectInfo) {
|
func (n *layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo *data.BucketInfo, node *data.NodeVersion, prefix, delimiter string) (oi *data.ObjectInfo) {
|
||||||
if oiDir := tryDirectory(bktInfo, node, prefix, delimiter); oiDir != nil {
|
if oiDir := tryDirectory(bktInfo, node, prefix, delimiter); oiDir != nil {
|
||||||
return oiDir
|
return oiDir
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,14 +9,14 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
AttributeComplianceMode = ".s3-compliance-mode"
|
AttributeComplianceMode = ".s3-compliance-mode"
|
||||||
AttributeExpirationEpoch = "__NEOFS__EXPIRATION_EPOCH"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type PutLockInfoParams struct {
|
type PutLockInfoParams struct {
|
||||||
|
@ -32,7 +32,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
|
||||||
// sometimes node version can be provided from executing context
|
// sometimes node version can be provided from executing context
|
||||||
// if not, then receive node version from tree service
|
// if not, then receive node version from tree service
|
||||||
if versionNode == nil {
|
if versionNode == nil {
|
||||||
versionNode, err = n.getNodeVersionFromCacheOrNeofs(ctx, p.ObjVersion)
|
versionNode, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjVersion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -100,7 +100,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) getNodeVersionFromCacheOrNeofs(ctx context.Context, objVersion *ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
|
func (n *layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion *ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
|
||||||
// check cache if node version is stored inside extendedObjectVersion
|
// check cache if node version is stored inside extendedObjectVersion
|
||||||
nodeVersion = n.getNodeVersionFromCache(n.Owner(ctx), objVersion)
|
nodeVersion = n.getNodeVersionFromCache(n.Owner(ctx), objVersion)
|
||||||
if nodeVersion == nil {
|
if nodeVersion == nil {
|
||||||
|
@ -116,6 +116,7 @@ func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
|
||||||
Container: bktInfo.CID,
|
Container: bktInfo.CID,
|
||||||
Creator: bktInfo.Owner,
|
Creator: bktInfo.Owner,
|
||||||
Locks: []oid.ID{objID},
|
Locks: []oid.ID{objID},
|
||||||
|
CreationTime: TimeNow(ctx),
|
||||||
CopiesNumber: copiesNumber,
|
CopiesNumber: copiesNumber,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -227,7 +228,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
|
||||||
)
|
)
|
||||||
|
|
||||||
if lock.Retention != nil {
|
if lock.Retention != nil {
|
||||||
if _, expEpoch, err = n.neoFS.TimeToEpoch(ctx, lock.Retention.Until); err != nil {
|
if _, expEpoch, err = n.frostFS.TimeToEpoch(ctx, TimeNow(ctx), lock.Retention.Until); err != nil {
|
||||||
return nil, fmt.Errorf("fetch time to epoch: %w", err)
|
return nil, fmt.Errorf("fetch time to epoch: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -237,7 +238,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
|
||||||
}
|
}
|
||||||
|
|
||||||
if lock.LegalHold != nil && lock.LegalHold.Enabled {
|
if lock.LegalHold != nil && lock.LegalHold.Enabled {
|
||||||
// todo: (@KirillovDenis) reconsider this when NeoFS will support Legal Hold https://github.com/nspcc-dev/neofs-contract/issues/247
|
// todo: (@KirillovDenis) reconsider this when FrostFS will support Legal Hold https://git.frostfs.info/TrueCloudLab/frostfs-contract/issues/2
|
||||||
// Currently lock object must have an expiration epoch.
|
// Currently lock object must have an expiration epoch.
|
||||||
// Besides we need to override retention expiration epoch since legal hold cannot be deleted yet.
|
// Besides we need to override retention expiration epoch since legal hold cannot be deleted yet.
|
||||||
expEpoch = math.MaxUint64
|
expEpoch = math.MaxUint64
|
||||||
|
@ -245,7 +246,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
|
||||||
|
|
||||||
if expEpoch != 0 {
|
if expEpoch != 0 {
|
||||||
result = append(result, [2]string{
|
result = append(result, [2]string{
|
||||||
AttributeExpirationEpoch, strconv.FormatUint(expEpoch, 10),
|
object.SysAttributeExpEpoch, strconv.FormatUint(expEpoch, 10),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -4,11 +4,13 @@ import (
|
||||||
"context"
|
"context"
|
||||||
errorsStd "errors"
|
errorsStd "errors"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
type GetObjectTaggingParams struct {
|
type GetObjectTaggingParams struct {
|
||||||
|
@ -38,7 +40,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
|
||||||
|
|
||||||
nodeVersion := p.NodeVersion
|
nodeVersion := p.NodeVersion
|
||||||
if nodeVersion == nil {
|
if nodeVersion == nil {
|
||||||
nodeVersion, err = n.getNodeVersionFromCacheOrNeofs(ctx, p.ObjectVersion)
|
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", nil, err
|
return "", nil, err
|
||||||
}
|
}
|
||||||
|
@ -65,7 +67,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
|
||||||
func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (nodeVersion *data.NodeVersion, err error) {
|
func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (nodeVersion *data.NodeVersion, err error) {
|
||||||
nodeVersion = p.NodeVersion
|
nodeVersion = p.NodeVersion
|
||||||
if nodeVersion == nil {
|
if nodeVersion == nil {
|
||||||
nodeVersion, err = n.getNodeVersionFromCacheOrNeofs(ctx, p.ObjectVersion)
|
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -175,6 +177,14 @@ func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (
|
||||||
return nil, errors.GetAPIError(errors.ErrNoSuchKey)
|
return nil, errors.GetAPIError(errors.ErrNoSuchKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err == nil && version != nil && !version.IsDeleteMarker() {
|
||||||
|
reqInfo := api.GetReqInfo(ctx)
|
||||||
|
n.log.Debug("target details",
|
||||||
|
zap.String("reqId", reqInfo.RequestID),
|
||||||
|
zap.String("bucket", objVersion.BktInfo.Name), zap.Stringer("cid", objVersion.BktInfo.CID),
|
||||||
|
zap.String("object", objVersion.ObjectName), zap.Stringer("oid", version.OID))
|
||||||
|
}
|
||||||
|
|
||||||
return version, err
|
return version, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,8 +6,8 @@ import (
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
)
|
)
|
||||||
|
|
||||||
type TreeServiceMock struct {
|
type TreeServiceMock struct {
|
||||||
|
@ -109,11 +109,32 @@ func (t *TreeServiceMock) PutNotificationConfigurationNode(ctx context.Context,
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
|
func (t *TreeServiceMock) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
|
||||||
panic("implement me")
|
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
return oid.ID{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
node, ok := systemMap["cors"]
|
||||||
|
if !ok {
|
||||||
|
return oid.ID{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return node.OID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) {
|
func (t *TreeServiceMock) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) {
|
||||||
panic("implement me")
|
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
systemMap = make(map[string]*data.BaseNodeVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
systemMap["cors"] = &data.BaseNodeVersion{
|
||||||
|
OID: objID,
|
||||||
|
}
|
||||||
|
|
||||||
|
t.system[bktInfo.CID.EncodeToString()] = systemMap
|
||||||
|
|
||||||
|
return oid.ID{}, ErrNoNodeToRemove
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
|
func (t *TreeServiceMock) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
|
||||||
|
|
|
@ -4,8 +4,8 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TreeService provide interface to interact with tree service using s3 data models.
|
// TreeService provide interface to interact with tree service using s3 data models.
|
||||||
|
@ -24,7 +24,7 @@ type TreeService interface {
|
||||||
GetNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
GetNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
||||||
|
|
||||||
// PutNotificationConfigurationNode puts a node to a system tree
|
// PutNotificationConfigurationNode puts a node to a system tree
|
||||||
// and returns objectID of a previous notif config which must be deleted in NeoFS.
|
// and returns objectID of a previous notif config which must be deleted in FrostFS.
|
||||||
//
|
//
|
||||||
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||||
PutNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
|
PutNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
|
||||||
|
@ -34,12 +34,12 @@ type TreeService interface {
|
||||||
// If object id is not found returns ErrNodeNotFound error.
|
// If object id is not found returns ErrNodeNotFound error.
|
||||||
GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
||||||
|
|
||||||
// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in NeoFS.
|
// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in FrostFS.
|
||||||
//
|
//
|
||||||
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||||
PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
|
PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
|
||||||
|
|
||||||
// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in NeoFS.
|
// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in FrostFS.
|
||||||
//
|
//
|
||||||
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||||
DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
||||||
|
@ -69,7 +69,7 @@ type TreeService interface {
|
||||||
GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error)
|
GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error)
|
||||||
|
|
||||||
// AddPart puts a node to a system tree as a child of appropriate multipart upload
|
// AddPart puts a node to a system tree as a child of appropriate multipart upload
|
||||||
// and returns objectID of a previous part which must be deleted in NeoFS.
|
// and returns objectID of a previous part which must be deleted in FrostFS.
|
||||||
//
|
//
|
||||||
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||||
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)
|
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)
|
||||||
|
|
|
@ -9,11 +9,11 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
|
|
@ -6,11 +6,11 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/checksum"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
||||||
|
|
|
@ -5,14 +5,14 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
|
bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
|
||||||
bearertest "github.com/nspcc-dev/neofs-sdk-go/bearer/test"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -113,7 +113,7 @@ func (tc *testContext) checkListObjects(ids ...oid.ID) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
|
func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
|
||||||
for _, obj := range tc.testNeoFS.Objects() {
|
for _, obj := range tc.testFrostFS.Objects() {
|
||||||
id, _ := obj.ID()
|
id, _ := obj.ID()
|
||||||
if id.Equals(objID) {
|
if id.Equals(objID) {
|
||||||
return obj
|
return obj
|
||||||
|
@ -123,12 +123,12 @@ func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
|
||||||
}
|
}
|
||||||
|
|
||||||
type testContext struct {
|
type testContext struct {
|
||||||
t *testing.T
|
t *testing.T
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
layer Client
|
layer Client
|
||||||
bktInfo *data.BucketInfo
|
bktInfo *data.BucketInfo
|
||||||
obj string
|
obj string
|
||||||
testNeoFS *TestNeoFS
|
testFrostFS *TestFrostFS
|
||||||
}
|
}
|
||||||
|
|
||||||
func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
||||||
|
@ -146,7 +146,7 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
||||||
GateKey: key.PublicKey(),
|
GateKey: key.PublicKey(),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
tp := NewTestNeoFS()
|
tp := NewTestFrostFS()
|
||||||
|
|
||||||
bktName := "testbucket1"
|
bktName := "testbucket1"
|
||||||
bktID, err := tp.CreateContainer(ctx, PrmContainerCreate{
|
bktID, err := tp.CreateContainer(ctx, PrmContainerCreate{
|
||||||
|
@ -176,9 +176,9 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
||||||
Owner: owner,
|
Owner: owner,
|
||||||
CID: bktID,
|
CID: bktID,
|
||||||
},
|
},
|
||||||
obj: "obj1",
|
obj: "obj1",
|
||||||
t: t,
|
t: t,
|
||||||
testNeoFS: tp,
|
testFrostFS: tp,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,7 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
|
305
api/metrics.go
Normal file
305
api/metrics.go
Normal file
|
@ -0,0 +1,305 @@
|
||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
)
|
||||||
|
|
||||||
|
type RequestType int
|
||||||
|
|
||||||
|
const (
|
||||||
|
UNKNOWNRequest RequestType = iota
|
||||||
|
HEADRequest RequestType = iota
|
||||||
|
PUTRequest RequestType = iota
|
||||||
|
LISTRequest RequestType = iota
|
||||||
|
GETRequest RequestType = iota
|
||||||
|
DELETERequest RequestType = iota
|
||||||
|
)
|
||||||
|
|
||||||
|
func (t RequestType) String() string {
|
||||||
|
switch t {
|
||||||
|
case 1:
|
||||||
|
return "HEAD"
|
||||||
|
case 2:
|
||||||
|
return "PUT"
|
||||||
|
case 3:
|
||||||
|
return "LIST"
|
||||||
|
case 4:
|
||||||
|
return "GET"
|
||||||
|
case 5:
|
||||||
|
return "DELETE"
|
||||||
|
default:
|
||||||
|
return "Unknown"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func RequestTypeFromAPI(api string) RequestType {
|
||||||
|
switch api {
|
||||||
|
case "Options", "HeadObject", "HeadBucket":
|
||||||
|
return HEADRequest
|
||||||
|
case "CreateMultipartUpload", "UploadPartCopy", "UploadPart", "CompleteMultipartUpload",
|
||||||
|
"PutObjectACL", "PutObjectTagging", "CopyObject", "PutObjectRetention", "PutObjectLegalHold",
|
||||||
|
"PutObject", "PutBucketCors", "PutBucketACL", "PutBucketLifecycle", "PutBucketEncryption",
|
||||||
|
"PutBucketPolicy", "PutBucketObjectLockConfig", "PutBucketTagging", "PutBucketVersioning",
|
||||||
|
"PutBucketNotification", "CreateBucket", "PostObject":
|
||||||
|
return PUTRequest
|
||||||
|
case "ListObjectParts", "ListMultipartUploads", "ListObjectsV2M", "ListObjectsV2", "ListBucketVersions",
|
||||||
|
"ListObjectsV1", "ListBuckets":
|
||||||
|
return LISTRequest
|
||||||
|
case "GetObjectACL", "GetObjectTagging", "SelectObjectContent", "GetObjectRetention", "getobjectlegalhold",
|
||||||
|
"GetObjectAttributes", "GetObject", "GetBucketLocation", "GetBucketPolicy",
|
||||||
|
"GetBucketLifecycle", "GetBucketEncryption", "GetBucketCors", "GetBucketACL",
|
||||||
|
"GetBucketWebsite", "GetBucketAccelerate", "GetBucketRequestPayment", "GetBucketLogging",
|
||||||
|
"GetBucketReplication", "GetBucketTagging", "GetBucketObjectLockConfig",
|
||||||
|
"GetBucketVersioning", "GetBucketNotification", "ListenBucketNotification":
|
||||||
|
return GETRequest
|
||||||
|
case "AbortMultipartUpload", "DeleteObjectTagging", "DeleteObject", "DeleteBucketCors",
|
||||||
|
"DeleteBucketWebsite", "DeleteBucketTagging", "DeleteMultipleObjects", "DeleteBucketPolicy",
|
||||||
|
"DeleteBucketLifecycle", "DeleteBucketEncryption", "DeleteBucket":
|
||||||
|
return DELETERequest
|
||||||
|
default:
|
||||||
|
return UNKNOWNRequest
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type (
|
||||||
|
// HTTPAPIStats holds statistics information about
|
||||||
|
// the API given in the requests.
|
||||||
|
HTTPAPIStats struct {
|
||||||
|
apiStats map[string]int
|
||||||
|
sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
UsersStat interface {
|
||||||
|
Update(user, bucket, cnrID string, reqType RequestType, in, out uint64)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HTTPStats holds statistics information about
|
||||||
|
// HTTP requests made by all clients.
|
||||||
|
HTTPStats struct {
|
||||||
|
currentS3Requests HTTPAPIStats
|
||||||
|
totalS3Requests HTTPAPIStats
|
||||||
|
totalS3Errors HTTPAPIStats
|
||||||
|
|
||||||
|
totalInputBytes uint64
|
||||||
|
totalOutputBytes uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
readCounter struct {
|
||||||
|
io.ReadCloser
|
||||||
|
countBytes uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
writeCounter struct {
|
||||||
|
http.ResponseWriter
|
||||||
|
countBytes uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
responseWrapper struct {
|
||||||
|
sync.Once
|
||||||
|
http.ResponseWriter
|
||||||
|
|
||||||
|
statusCode int
|
||||||
|
startTime time.Time
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
const systemPath = "/system"
|
||||||
|
|
||||||
|
var (
|
||||||
|
httpStatsMetric = new(HTTPStats)
|
||||||
|
httpRequestsDuration = prometheus.NewHistogramVec(
|
||||||
|
prometheus.HistogramOpts{
|
||||||
|
Name: "frostfs_s3_request_seconds",
|
||||||
|
Help: "Time taken by requests served by current FrostFS S3 Gate instance",
|
||||||
|
Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10},
|
||||||
|
},
|
||||||
|
[]string{"api"},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
// Collects HTTP metrics for FrostFS S3 Gate in Prometheus specific format
|
||||||
|
// and sends to the given channel.
|
||||||
|
func collectHTTPMetrics(ch chan<- prometheus.Metric) {
|
||||||
|
for api, value := range httpStatsMetric.currentS3Requests.Load() {
|
||||||
|
ch <- prometheus.MustNewConstMetric(
|
||||||
|
prometheus.NewDesc(
|
||||||
|
prometheus.BuildFQName("frostfs_s3", "requests", "current"),
|
||||||
|
"Total number of running s3 requests in current FrostFS S3 Gate instance",
|
||||||
|
[]string{"api"}, nil),
|
||||||
|
prometheus.CounterValue,
|
||||||
|
float64(value),
|
||||||
|
api,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
for api, value := range httpStatsMetric.totalS3Requests.Load() {
|
||||||
|
ch <- prometheus.MustNewConstMetric(
|
||||||
|
prometheus.NewDesc(
|
||||||
|
prometheus.BuildFQName("frostfs_s3", "requests", "total"),
|
||||||
|
"Total number of s3 requests in current FrostFS S3 Gate instance",
|
||||||
|
[]string{"api"}, nil),
|
||||||
|
prometheus.CounterValue,
|
||||||
|
float64(value),
|
||||||
|
api,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
for api, value := range httpStatsMetric.totalS3Errors.Load() {
|
||||||
|
ch <- prometheus.MustNewConstMetric(
|
||||||
|
prometheus.NewDesc(
|
||||||
|
prometheus.BuildFQName("frostfs_s3", "errors", "total"),
|
||||||
|
"Total number of s3 errors in current FrostFS S3 Gate instance",
|
||||||
|
[]string{"api"}, nil),
|
||||||
|
prometheus.CounterValue,
|
||||||
|
float64(value),
|
||||||
|
api,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CIDResolveFunc is a func to resolve CID in Stats handler.
|
||||||
|
type CIDResolveFunc func(ctx context.Context, reqInfo *ReqInfo) (cnrID string)
|
||||||
|
|
||||||
|
// Stats is a handler that update metrics.
|
||||||
|
func Stats(f http.HandlerFunc, resolveCID CIDResolveFunc, usersStat UsersStat) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
reqInfo := GetReqInfo(r.Context())
|
||||||
|
|
||||||
|
httpStatsMetric.currentS3Requests.Inc(reqInfo.API)
|
||||||
|
defer httpStatsMetric.currentS3Requests.Dec(reqInfo.API)
|
||||||
|
|
||||||
|
in := &readCounter{ReadCloser: r.Body}
|
||||||
|
out := &writeCounter{ResponseWriter: w}
|
||||||
|
|
||||||
|
r.Body = in
|
||||||
|
|
||||||
|
statsWriter := &responseWrapper{
|
||||||
|
ResponseWriter: out,
|
||||||
|
startTime: time.Now(),
|
||||||
|
}
|
||||||
|
|
||||||
|
f(statsWriter, r)
|
||||||
|
|
||||||
|
// Time duration in secs since the call started.
|
||||||
|
// We don't need to do nanosecond precision here
|
||||||
|
// simply for the fact that it is not human-readable.
|
||||||
|
durationSecs := time.Since(statsWriter.startTime).Seconds()
|
||||||
|
|
||||||
|
user := resolveUser(r.Context())
|
||||||
|
cnrID := resolveCID(r.Context(), reqInfo)
|
||||||
|
usersStat.Update(user, reqInfo.BucketName, cnrID, RequestTypeFromAPI(reqInfo.API), in.countBytes, out.countBytes)
|
||||||
|
|
||||||
|
code := statsWriter.statusCode
|
||||||
|
// A successful request has a 2xx response code
|
||||||
|
successReq := code >= http.StatusOK && code < http.StatusMultipleChoices
|
||||||
|
if !strings.HasSuffix(r.URL.Path, systemPath) {
|
||||||
|
httpStatsMetric.totalS3Requests.Inc(reqInfo.API)
|
||||||
|
if !successReq && code != 0 {
|
||||||
|
httpStatsMetric.totalS3Errors.Inc(reqInfo.API)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.Method == http.MethodGet {
|
||||||
|
// Increment the prometheus http request response histogram with appropriate label
|
||||||
|
httpRequestsDuration.With(prometheus.Labels{"api": reqInfo.API}).Observe(durationSecs)
|
||||||
|
}
|
||||||
|
|
||||||
|
atomic.AddUint64(&httpStatsMetric.totalInputBytes, in.countBytes)
|
||||||
|
atomic.AddUint64(&httpStatsMetric.totalOutputBytes, out.countBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveUser(ctx context.Context) string {
|
||||||
|
user := "anon"
|
||||||
|
if bd, ok := ctx.Value(BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
|
||||||
|
user = bearer.ResolveIssuer(*bd.Gate.BearerToken).String()
|
||||||
|
}
|
||||||
|
return user
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inc increments the api stats counter.
|
||||||
|
func (stats *HTTPAPIStats) Inc(api string) {
|
||||||
|
if stats == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
stats.Lock()
|
||||||
|
defer stats.Unlock()
|
||||||
|
if stats.apiStats == nil {
|
||||||
|
stats.apiStats = make(map[string]int)
|
||||||
|
}
|
||||||
|
stats.apiStats[api]++
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dec increments the api stats counter.
|
||||||
|
func (stats *HTTPAPIStats) Dec(api string) {
|
||||||
|
if stats == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
stats.Lock()
|
||||||
|
defer stats.Unlock()
|
||||||
|
if val, ok := stats.apiStats[api]; ok && val > 0 {
|
||||||
|
stats.apiStats[api]--
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load returns the recorded stats.
|
||||||
|
func (stats *HTTPAPIStats) Load() map[string]int {
|
||||||
|
stats.Lock()
|
||||||
|
defer stats.Unlock()
|
||||||
|
var apiStats = make(map[string]int, len(stats.apiStats))
|
||||||
|
for k, v := range stats.apiStats {
|
||||||
|
apiStats[k] = v
|
||||||
|
}
|
||||||
|
return apiStats
|
||||||
|
}
|
||||||
|
|
||||||
|
func (st *HTTPStats) getInputBytes() uint64 {
|
||||||
|
return atomic.LoadUint64(&st.totalInputBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (st *HTTPStats) getOutputBytes() uint64 {
|
||||||
|
return atomic.LoadUint64(&st.totalOutputBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteHeader -- writes http status code.
|
||||||
|
func (w *responseWrapper) WriteHeader(code int) {
|
||||||
|
w.Do(func() {
|
||||||
|
w.statusCode = code
|
||||||
|
w.ResponseWriter.WriteHeader(code)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flush -- calls the underlying Flush.
|
||||||
|
func (w *responseWrapper) Flush() {
|
||||||
|
if f, ok := w.ResponseWriter.(http.Flusher); ok {
|
||||||
|
f.Flush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *writeCounter) Flush() {
|
||||||
|
if f, ok := w.ResponseWriter.(http.Flusher); ok {
|
||||||
|
f.Flush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *writeCounter) Write(p []byte) (int, error) {
|
||||||
|
n, err := w.ResponseWriter.Write(p)
|
||||||
|
atomic.AddUint64(&w.countBytes, uint64(n))
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *readCounter) Read(p []byte) (int, error) {
|
||||||
|
n, err := r.ReadCloser.Read(p)
|
||||||
|
atomic.AddUint64(&r.countBytes, uint64(n))
|
||||||
|
return n, err
|
||||||
|
}
|
|
@ -1,229 +0,0 @@
|
||||||
package metrics
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
)
|
|
||||||
|
|
||||||
type (
|
|
||||||
// HTTPAPIStats holds statistics information about
|
|
||||||
// the API given in the requests.
|
|
||||||
HTTPAPIStats struct {
|
|
||||||
apiStats map[string]int
|
|
||||||
sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
// HTTPStats holds statistics information about
|
|
||||||
// HTTP requests made by all clients.
|
|
||||||
HTTPStats struct {
|
|
||||||
currentS3Requests HTTPAPIStats
|
|
||||||
totalS3Requests HTTPAPIStats
|
|
||||||
totalS3Errors HTTPAPIStats
|
|
||||||
|
|
||||||
totalInputBytes uint64
|
|
||||||
totalOutputBytes uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
readCounter struct {
|
|
||||||
io.ReadCloser
|
|
||||||
countBytes uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
writeCounter struct {
|
|
||||||
http.ResponseWriter
|
|
||||||
countBytes uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
responseWrapper struct {
|
|
||||||
sync.Once
|
|
||||||
http.ResponseWriter
|
|
||||||
|
|
||||||
statusCode int
|
|
||||||
startTime time.Time
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
const systemPath = "/system"
|
|
||||||
|
|
||||||
var (
|
|
||||||
httpStatsMetric = new(HTTPStats)
|
|
||||||
httpRequestsDuration = prometheus.NewHistogramVec(
|
|
||||||
prometheus.HistogramOpts{
|
|
||||||
Name: "neofs_s3_request_seconds",
|
|
||||||
Help: "Time taken by requests served by current NeoFS S3 Gate instance",
|
|
||||||
Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10},
|
|
||||||
},
|
|
||||||
[]string{"api"},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
// Collects HTTP metrics for NeoFS S3 Gate in Prometheus specific format
|
|
||||||
// and sends to the given channel.
|
|
||||||
func collectHTTPMetrics(ch chan<- prometheus.Metric) {
|
|
||||||
for api, value := range httpStatsMetric.currentS3Requests.Load() {
|
|
||||||
ch <- prometheus.MustNewConstMetric(
|
|
||||||
prometheus.NewDesc(
|
|
||||||
prometheus.BuildFQName("neofs_s3", "requests", "current"),
|
|
||||||
"Total number of running s3 requests in current NeoFS S3 Gate instance",
|
|
||||||
[]string{"api"}, nil),
|
|
||||||
prometheus.CounterValue,
|
|
||||||
float64(value),
|
|
||||||
api,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
for api, value := range httpStatsMetric.totalS3Requests.Load() {
|
|
||||||
ch <- prometheus.MustNewConstMetric(
|
|
||||||
prometheus.NewDesc(
|
|
||||||
prometheus.BuildFQName("neofs_s3", "requests", "total"),
|
|
||||||
"Total number of s3 requests in current NeoFS S3 Gate instance",
|
|
||||||
[]string{"api"}, nil),
|
|
||||||
prometheus.CounterValue,
|
|
||||||
float64(value),
|
|
||||||
api,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
for api, value := range httpStatsMetric.totalS3Errors.Load() {
|
|
||||||
ch <- prometheus.MustNewConstMetric(
|
|
||||||
prometheus.NewDesc(
|
|
||||||
prometheus.BuildFQName("neofs_s3", "errors", "total"),
|
|
||||||
"Total number of s3 errors in current NeoFS S3 Gate instance",
|
|
||||||
[]string{"api"}, nil),
|
|
||||||
prometheus.CounterValue,
|
|
||||||
float64(value),
|
|
||||||
api,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// APIStats wraps http handler for api with basic statistics collection.
|
|
||||||
func APIStats(api string, f http.HandlerFunc) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
httpStatsMetric.currentS3Requests.Inc(api)
|
|
||||||
defer httpStatsMetric.currentS3Requests.Dec(api)
|
|
||||||
|
|
||||||
in := &readCounter{ReadCloser: r.Body}
|
|
||||||
out := &writeCounter{ResponseWriter: w}
|
|
||||||
|
|
||||||
r.Body = in
|
|
||||||
|
|
||||||
statsWriter := &responseWrapper{
|
|
||||||
ResponseWriter: out,
|
|
||||||
startTime: time.Now(),
|
|
||||||
}
|
|
||||||
|
|
||||||
f.ServeHTTP(statsWriter, r)
|
|
||||||
|
|
||||||
// Time duration in secs since the call started.
|
|
||||||
// We don't need to do nanosecond precision here
|
|
||||||
// simply for the fact that it is not human readable.
|
|
||||||
durationSecs := time.Since(statsWriter.startTime).Seconds()
|
|
||||||
|
|
||||||
httpStatsMetric.updateStats(api, statsWriter, r, durationSecs)
|
|
||||||
|
|
||||||
atomic.AddUint64(&httpStatsMetric.totalInputBytes, in.countBytes)
|
|
||||||
atomic.AddUint64(&httpStatsMetric.totalOutputBytes, out.countBytes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Inc increments the api stats counter.
|
|
||||||
func (stats *HTTPAPIStats) Inc(api string) {
|
|
||||||
if stats == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
stats.Lock()
|
|
||||||
defer stats.Unlock()
|
|
||||||
if stats.apiStats == nil {
|
|
||||||
stats.apiStats = make(map[string]int)
|
|
||||||
}
|
|
||||||
stats.apiStats[api]++
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dec increments the api stats counter.
|
|
||||||
func (stats *HTTPAPIStats) Dec(api string) {
|
|
||||||
if stats == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
stats.Lock()
|
|
||||||
defer stats.Unlock()
|
|
||||||
if val, ok := stats.apiStats[api]; ok && val > 0 {
|
|
||||||
stats.apiStats[api]--
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load returns the recorded stats.
|
|
||||||
func (stats *HTTPAPIStats) Load() map[string]int {
|
|
||||||
stats.Lock()
|
|
||||||
defer stats.Unlock()
|
|
||||||
var apiStats = make(map[string]int, len(stats.apiStats))
|
|
||||||
for k, v := range stats.apiStats {
|
|
||||||
apiStats[k] = v
|
|
||||||
}
|
|
||||||
return apiStats
|
|
||||||
}
|
|
||||||
|
|
||||||
func (st *HTTPStats) getInputBytes() uint64 {
|
|
||||||
return atomic.LoadUint64(&st.totalInputBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (st *HTTPStats) getOutputBytes() uint64 {
|
|
||||||
return atomic.LoadUint64(&st.totalOutputBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update statistics from http request and response data.
|
|
||||||
func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) {
|
|
||||||
var code int
|
|
||||||
|
|
||||||
if res, ok := w.(*responseWrapper); ok {
|
|
||||||
code = res.statusCode
|
|
||||||
}
|
|
||||||
|
|
||||||
// A successful request has a 2xx response code
|
|
||||||
successReq := code >= http.StatusOK && code < http.StatusMultipleChoices
|
|
||||||
|
|
||||||
if !strings.HasSuffix(r.URL.Path, systemPath) {
|
|
||||||
st.totalS3Requests.Inc(api)
|
|
||||||
if !successReq && code != 0 {
|
|
||||||
st.totalS3Errors.Inc(api)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Method == http.MethodGet {
|
|
||||||
// Increment the prometheus http request response histogram with appropriate label
|
|
||||||
httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(durationSecs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteHeader -- writes http status code.
|
|
||||||
func (w *responseWrapper) WriteHeader(code int) {
|
|
||||||
w.Do(func() {
|
|
||||||
w.statusCode = code
|
|
||||||
w.ResponseWriter.WriteHeader(code)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Flush -- calls the underlying Flush.
|
|
||||||
func (w *responseWrapper) Flush() {
|
|
||||||
if f, ok := w.ResponseWriter.(http.Flusher); ok {
|
|
||||||
f.Flush()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *writeCounter) Write(p []byte) (int, error) {
|
|
||||||
n, err := w.ResponseWriter.Write(p)
|
|
||||||
atomic.AddUint64(&w.countBytes, uint64(n))
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *readCounter) Read(p []byte) (int, error) {
|
|
||||||
n, err := r.ReadCloser.Read(p)
|
|
||||||
atomic.AddUint64(&r.countBytes, uint64(n))
|
|
||||||
return n, err
|
|
||||||
}
|
|
|
@ -1,7 +1,7 @@
|
||||||
package metrics
|
package api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/internal/version"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -12,9 +12,9 @@ type stats struct {
|
||||||
var (
|
var (
|
||||||
versionInfo = prometheus.NewGaugeVec(
|
versionInfo = prometheus.NewGaugeVec(
|
||||||
prometheus.GaugeOpts{
|
prometheus.GaugeOpts{
|
||||||
Namespace: "neofs_s3",
|
Namespace: "frostfs_s3",
|
||||||
Name: "version_info",
|
Name: "version_info",
|
||||||
Help: "Version of current NeoFS S3 Gate instance",
|
Help: "Version of current FrostFS S3 Gate instance",
|
||||||
},
|
},
|
||||||
[]string{
|
[]string{
|
||||||
// current version
|
// current version
|
||||||
|
@ -23,7 +23,7 @@ var (
|
||||||
)
|
)
|
||||||
|
|
||||||
statsMetrics = &stats{
|
statsMetrics = &stats{
|
||||||
desc: prometheus.NewDesc("neofs_s3_stats", "Statistics exposed by NeoFS S3 Gate instance", nil, nil),
|
desc: prometheus.NewDesc("frostfs_s3_stats", "Statistics exposed by FrostFS S3 Gate instance", nil, nil),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -37,8 +37,8 @@ func collectNetworkMetrics(ch chan<- prometheus.Metric) {
|
||||||
// Network Sent/Received Bytes (Outbound)
|
// Network Sent/Received Bytes (Outbound)
|
||||||
ch <- prometheus.MustNewConstMetric(
|
ch <- prometheus.MustNewConstMetric(
|
||||||
prometheus.NewDesc(
|
prometheus.NewDesc(
|
||||||
prometheus.BuildFQName("neofs_s3", "tx", "bytes_total"),
|
prometheus.BuildFQName("frostfs_s3", "tx", "bytes_total"),
|
||||||
"Total number of bytes sent by current NeoFS S3 Gate instance",
|
"Total number of bytes sent by current FrostFS S3 Gate instance",
|
||||||
nil, nil),
|
nil, nil),
|
||||||
prometheus.CounterValue,
|
prometheus.CounterValue,
|
||||||
float64(httpStatsMetric.getInputBytes()),
|
float64(httpStatsMetric.getInputBytes()),
|
||||||
|
@ -46,8 +46,8 @@ func collectNetworkMetrics(ch chan<- prometheus.Metric) {
|
||||||
|
|
||||||
ch <- prometheus.MustNewConstMetric(
|
ch <- prometheus.MustNewConstMetric(
|
||||||
prometheus.NewDesc(
|
prometheus.NewDesc(
|
||||||
prometheus.BuildFQName("neofs_s3", "rx", "bytes_total"),
|
prometheus.BuildFQName("frostfs_s3", "rx", "bytes_total"),
|
||||||
"Total number of bytes received by current NeoFS S3 Gate instance",
|
"Total number of bytes received by current FrostFS S3 Gate instance",
|
||||||
nil, nil),
|
nil, nil),
|
||||||
prometheus.CounterValue,
|
prometheus.CounterValue,
|
||||||
float64(httpStatsMetric.getOutputBytes()),
|
float64(httpStatsMetric.getOutputBytes()),
|
|
@ -7,9 +7,9 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/handler"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -61,7 +61,7 @@ type (
|
||||||
|
|
||||||
EventRecord struct {
|
EventRecord struct {
|
||||||
EventVersion string `json:"eventVersion"`
|
EventVersion string `json:"eventVersion"`
|
||||||
EventSource string `json:"eventSource"` // neofs:s3
|
EventSource string `json:"eventSource"` // frostfs:s3
|
||||||
AWSRegion string `json:"awsRegion,omitempty"` // empty
|
AWSRegion string `json:"awsRegion,omitempty"` // empty
|
||||||
EventTime time.Time `json:"eventTime"`
|
EventTime time.Time `json:"eventTime"`
|
||||||
EventName string `json:"eventName"`
|
EventName string `json:"eventName"`
|
||||||
|
@ -197,11 +197,11 @@ func (c *Controller) SendNotifications(topics map[string]string, p *handler.Send
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Controller) SendTestNotification(topic, bucketName, requestID, HostID string) error {
|
func (c *Controller) SendTestNotification(topic, bucketName, requestID, HostID string, now time.Time) error {
|
||||||
event := &TestEvent{
|
event := &TestEvent{
|
||||||
Service: "NeoFS S3",
|
Service: "FrostFS S3",
|
||||||
Event: "s3:TestEvent",
|
Event: "s3:TestEvent",
|
||||||
Time: time.Now(),
|
Time: now,
|
||||||
Bucket: bucketName,
|
Bucket: bucketName,
|
||||||
RequestID: requestID,
|
RequestID: requestID,
|
||||||
HostID: HostID,
|
HostID: HostID,
|
||||||
|
@ -220,9 +220,9 @@ func prepareEvent(p *handler.SendNotificationParams) *Event {
|
||||||
Records: []EventRecord{
|
Records: []EventRecord{
|
||||||
{
|
{
|
||||||
EventVersion: EventVersion21,
|
EventVersion: EventVersion21,
|
||||||
EventSource: "neofs:s3",
|
EventSource: "frostfs:s3",
|
||||||
AWSRegion: "",
|
AWSRegion: "",
|
||||||
EventTime: time.Now(),
|
EventTime: p.Time,
|
||||||
EventName: p.Event,
|
EventName: p.Event,
|
||||||
UserIdentity: UserIdentity{
|
UserIdentity: UserIdentity{
|
||||||
PrincipalID: p.User,
|
PrincipalID: p.User,
|
||||||
|
|
|
@ -45,7 +45,7 @@ type (
|
||||||
// Key used for Get/SetReqInfo.
|
// Key used for Get/SetReqInfo.
|
||||||
type contextKeyType string
|
type contextKeyType string
|
||||||
|
|
||||||
const ctxRequestInfo = contextKeyType("NeoFS-S3-GW")
|
const ctxRequestInfo = contextKeyType("FrostFS-S3-GW")
|
||||||
|
|
||||||
var (
|
var (
|
||||||
// De-facto standard header keys.
|
// De-facto standard header keys.
|
||||||
|
|
|
@ -6,9 +6,9 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/ns"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -19,9 +19,9 @@ const (
|
||||||
// ErrNoResolvers returns when trying to resolve container without any resolver.
|
// ErrNoResolvers returns when trying to resolve container without any resolver.
|
||||||
var ErrNoResolvers = errors.New("no resolvers")
|
var ErrNoResolvers = errors.New("no resolvers")
|
||||||
|
|
||||||
// NeoFS represents virtual connection to the NeoFS network.
|
// FrostFS represents virtual connection to the FrostFS network.
|
||||||
type NeoFS interface {
|
type FrostFS interface {
|
||||||
// SystemDNS reads system DNS network parameters of the NeoFS.
|
// SystemDNS reads system DNS network parameters of the FrostFS.
|
||||||
//
|
//
|
||||||
// It returns exactly on non-zero value. It returns any error encountered
|
// It returns exactly on non-zero value. It returns any error encountered
|
||||||
// which prevented the parameter from being read.
|
// which prevented the parameter from being read.
|
||||||
|
@ -29,7 +29,7 @@ type NeoFS interface {
|
||||||
}
|
}
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
NeoFS NeoFS
|
FrostFS FrostFS
|
||||||
RPCAddress string
|
RPCAddress string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -134,7 +134,7 @@ func (r *BucketResolver) equals(resolverNames []string) bool {
|
||||||
func newResolver(name string, cfg *Config) (*Resolver, error) {
|
func newResolver(name string, cfg *Config) (*Resolver, error) {
|
||||||
switch name {
|
switch name {
|
||||||
case DNSResolver:
|
case DNSResolver:
|
||||||
return NewDNSResolver(cfg.NeoFS)
|
return NewDNSResolver(cfg.FrostFS)
|
||||||
case NNSResolver:
|
case NNSResolver:
|
||||||
return NewNNSResolver(cfg.RPCAddress)
|
return NewNNSResolver(cfg.RPCAddress)
|
||||||
default:
|
default:
|
||||||
|
@ -142,17 +142,17 @@ func newResolver(name string, cfg *Config) (*Resolver, error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewDNSResolver(neoFS NeoFS) (*Resolver, error) {
|
func NewDNSResolver(frostFS FrostFS) (*Resolver, error) {
|
||||||
if neoFS == nil {
|
if frostFS == nil {
|
||||||
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
|
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
|
||||||
}
|
}
|
||||||
|
|
||||||
var dns ns.DNS
|
var dns ns.DNS
|
||||||
|
|
||||||
resolveFunc := func(ctx context.Context, name string) (cid.ID, error) {
|
resolveFunc := func(ctx context.Context, name string) (cid.ID, error) {
|
||||||
domain, err := neoFS.SystemDNS(ctx)
|
domain, err := frostFS.SystemDNS(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cid.ID{}, fmt.Errorf("read system DNS parameter of the NeoFS: %w", err)
|
return cid.ID{}, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
domain = name + "." + domain
|
domain = name + "." + domain
|
||||||
|
|
|
@ -7,9 +7,9 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/internal/version"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
@ -117,7 +117,7 @@ func WriteErrorResponse(w http.ResponseWriter, reqInfo *ReqInfo, err error) int
|
||||||
code = e.HTTPStatusCode
|
code = e.HTTPStatusCode
|
||||||
|
|
||||||
switch e.Code {
|
switch e.Code {
|
||||||
case "SlowDown", "XNeoFSServerNotInitialized", "XNeoFSReadQuorum", "XNeoFSWriteQuorum":
|
case "SlowDown", "XFrostFSServerNotInitialized", "XFrostFSReadQuorum", "XFrostFSWriteQuorum":
|
||||||
// Set retry-after header to indicate user-agents to retry request after 120secs.
|
// Set retry-after header to indicate user-agents to retry request after 120secs.
|
||||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
||||||
w.Header().Set(hdrRetryAfter, "120")
|
w.Header().Set(hdrRetryAfter, "120")
|
||||||
|
@ -133,6 +133,13 @@ func WriteErrorResponse(w http.ResponseWriter, reqInfo *ReqInfo, err error) int
|
||||||
return code
|
return code
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WriteErrorResponseNoHeader writes XML encoded error to the response body.
|
||||||
|
func WriteErrorResponseNoHeader(w http.ResponseWriter, reqInfo *ReqInfo, err error) {
|
||||||
|
errorResponse := getAPIErrorResponse(reqInfo, err)
|
||||||
|
encodedErrorResponse := EncodeResponse(errorResponse)
|
||||||
|
WriteResponseBody(w, encodedErrorResponse)
|
||||||
|
}
|
||||||
|
|
||||||
// If none of the http routes match respond with appropriate errors.
|
// If none of the http routes match respond with appropriate errors.
|
||||||
func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)
|
desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)
|
||||||
|
@ -172,6 +179,11 @@ func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
WriteResponseBody(w, response)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteResponseBody writes response into w.
|
||||||
|
func WriteResponseBody(w http.ResponseWriter, response []byte) {
|
||||||
_, _ = w.Write(response)
|
_, _ = w.Write(response)
|
||||||
if flusher, ok := w.(http.Flusher); ok {
|
if flusher, ok := w.(http.Flusher); ok {
|
||||||
flusher.Flush()
|
flusher.Flush()
|
||||||
|
@ -188,13 +200,30 @@ func EncodeResponse(response interface{}) []byte {
|
||||||
return bytesBuffer.Bytes()
|
return bytesBuffer.Bytes()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EncodeResponseNoHeader encodes response without setting xml.Header.
|
||||||
|
// Should be used with periodicXMLWriter which sends xml.Header to the client
|
||||||
|
// with whitespaces to keep connection alive.
|
||||||
|
func EncodeResponseNoHeader(response interface{}) []byte {
|
||||||
|
var bytesBuffer bytes.Buffer
|
||||||
|
_ = xml.NewEncoder(&bytesBuffer).Encode(response)
|
||||||
|
return bytesBuffer.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
// EncodeToResponse encodes the response into ResponseWriter.
|
// EncodeToResponse encodes the response into ResponseWriter.
|
||||||
func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
|
func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
|
|
||||||
if _, err := w.Write(xmlHeader); err != nil {
|
if _, err := w.Write(xmlHeader); err != nil {
|
||||||
return fmt.Errorf("write headers: %w", err)
|
return fmt.Errorf("write headers: %w", err)
|
||||||
} else if err = xml.NewEncoder(w).Encode(response); err != nil {
|
}
|
||||||
|
|
||||||
|
return EncodeToResponseNoHeader(w, response)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeToResponseNoHeader encodes the response into ResponseWriter without
|
||||||
|
// header status.
|
||||||
|
func EncodeToResponseNoHeader(w http.ResponseWriter, response interface{}) error {
|
||||||
|
if err := xml.NewEncoder(w).Encode(response); err != nil {
|
||||||
return fmt.Errorf("encode xml response: %w", err)
|
return fmt.Errorf("encode xml response: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
281
api/router.go
281
api/router.go
|
@ -5,10 +5,10 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/metrics"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
"google.golang.org/grpc/metadata"
|
"google.golang.org/grpc/metadata"
|
||||||
)
|
)
|
||||||
|
@ -82,6 +82,8 @@ type (
|
||||||
AbortMultipartUploadHandler(http.ResponseWriter, *http.Request)
|
AbortMultipartUploadHandler(http.ResponseWriter, *http.Request)
|
||||||
ListPartsHandler(w http.ResponseWriter, r *http.Request)
|
ListPartsHandler(w http.ResponseWriter, r *http.Request)
|
||||||
ListMultipartUploadsHandler(http.ResponseWriter, *http.Request)
|
ListMultipartUploadsHandler(http.ResponseWriter, *http.Request)
|
||||||
|
|
||||||
|
ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// mimeType represents various MIME types used in API responses.
|
// mimeType represents various MIME types used in API responses.
|
||||||
|
@ -106,7 +108,7 @@ const (
|
||||||
MimeXML mimeType = "application/xml"
|
MimeXML mimeType = "application/xml"
|
||||||
)
|
)
|
||||||
|
|
||||||
var _ = logErrorResponse
|
var _ = logSuccessResponse
|
||||||
|
|
||||||
func (lrw *logResponseWriter) WriteHeader(code int) {
|
func (lrw *logResponseWriter) WriteHeader(code int) {
|
||||||
lrw.Do(func() {
|
lrw.Do(func() {
|
||||||
|
@ -115,6 +117,12 @@ func (lrw *logResponseWriter) WriteHeader(code int) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (lrw *logResponseWriter) Flush() {
|
||||||
|
if f, ok := lrw.ResponseWriter.(http.Flusher); ok {
|
||||||
|
f.Flush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func setRequestID(h http.Handler) http.Handler {
|
func setRequestID(h http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
// generate random UUIDv4
|
// generate random UUIDv4
|
||||||
|
@ -145,7 +153,37 @@ func appendCORS(handler Handler) mux.MiddlewareFunc {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc {
|
// BucketResolveFunc is a func to resolve bucket info by name.
|
||||||
|
type BucketResolveFunc func(ctx context.Context, bucket string) (*data.BucketInfo, error)
|
||||||
|
|
||||||
|
// metricsMiddleware wraps http handler for api with basic statistics collection.
|
||||||
|
func metricsMiddleware(log *zap.Logger, resolveBucket BucketResolveFunc, usersStat UsersStat) mux.MiddlewareFunc {
|
||||||
|
return func(h http.Handler) http.Handler {
|
||||||
|
return Stats(h.ServeHTTP, resolveCID(log, resolveBucket), usersStat)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveCID forms CIDResolveFunc using BucketResolveFunc.
|
||||||
|
func resolveCID(log *zap.Logger, resolveBucket BucketResolveFunc) CIDResolveFunc {
|
||||||
|
return func(ctx context.Context, reqInfo *ReqInfo) (cnrID string) {
|
||||||
|
if reqInfo.BucketName == "" || reqInfo.API == "CreateBucket" || reqInfo.API == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
bktInfo, err := resolveBucket(ctx, reqInfo.BucketName)
|
||||||
|
if err != nil {
|
||||||
|
log.Debug("failed to resolve CID",
|
||||||
|
zap.String("request_id", reqInfo.RequestID), zap.String("method", reqInfo.API),
|
||||||
|
zap.String("bucket", reqInfo.BucketName), zap.String("object", reqInfo.ObjectName),
|
||||||
|
zap.Error(err))
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
return bktInfo.CID.EncodeToString()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func logSuccessResponse(l *zap.Logger) mux.MiddlewareFunc {
|
||||||
return func(h http.Handler) http.Handler {
|
return func(h http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
lw := &logResponseWriter{ResponseWriter: w}
|
lw := &logResponseWriter{ResponseWriter: w}
|
||||||
|
@ -161,6 +199,7 @@ func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc {
|
||||||
|
|
||||||
l.Info("call method",
|
l.Info("call method",
|
||||||
zap.Int("status", lw.statusCode),
|
zap.Int("status", lw.statusCode),
|
||||||
|
zap.String("host", r.Host),
|
||||||
zap.String("request_id", GetRequestID(r.Context())),
|
zap.String("request_id", GetRequestID(r.Context())),
|
||||||
zap.String("method", mux.CurrentRoute(r).GetName()),
|
zap.String("method", mux.CurrentRoute(r).GetName()),
|
||||||
zap.String("bucket", reqInfo.BucketName),
|
zap.String("bucket", reqInfo.BucketName),
|
||||||
|
@ -182,21 +221,49 @@ func GetRequestID(v interface{}) string {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func setErrorAPI(apiName string, h http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := SetReqInfo(r.Context(), &ReqInfo{API: apiName})
|
||||||
|
h.ServeHTTP(w, r.WithContext(ctx))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// attachErrorHandler set NotFoundHandler and MethodNotAllowedHandler for mux.Router.
|
||||||
|
func attachErrorHandler(api *mux.Router, log *zap.Logger, h Handler, center auth.Center, usersStat UsersStat) {
|
||||||
|
middlewares := []mux.MiddlewareFunc{
|
||||||
|
AuthMiddleware(log, center),
|
||||||
|
metricsMiddleware(log, h.ResolveBucket, usersStat),
|
||||||
|
}
|
||||||
|
|
||||||
|
var errorHandler http.Handler = http.HandlerFunc(errorResponseHandler)
|
||||||
|
for i := len(middlewares) - 1; i >= 0; i-- {
|
||||||
|
errorHandler = middlewares[i](errorHandler)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If none of the routes match, add default error handler routes
|
||||||
|
api.NotFoundHandler = setErrorAPI("NotFound", errorHandler)
|
||||||
|
api.MethodNotAllowedHandler = setErrorAPI("MethodNotAllowed", errorHandler)
|
||||||
|
}
|
||||||
|
|
||||||
// Attach adds S3 API handlers from h to r for domains with m client limit using
|
// Attach adds S3 API handlers from h to r for domains with m client limit using
|
||||||
// center authentication and log logger.
|
// center authentication and log logger.
|
||||||
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) {
|
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger, usersStat UsersStat) {
|
||||||
api := r.PathPrefix(SlashSeparator).Subrouter()
|
api := r.PathPrefix(SlashSeparator).Subrouter()
|
||||||
|
|
||||||
api.Use(
|
api.Use(
|
||||||
// -- prepare request
|
// -- prepare request
|
||||||
setRequestID,
|
setRequestID,
|
||||||
|
|
||||||
|
// Attach user authentication for all S3 routes.
|
||||||
|
AuthMiddleware(log, center),
|
||||||
|
|
||||||
|
metricsMiddleware(log, h.ResolveBucket, usersStat),
|
||||||
|
|
||||||
// -- logging error requests
|
// -- logging error requests
|
||||||
logErrorResponse(log),
|
logSuccessResponse(log),
|
||||||
)
|
)
|
||||||
|
|
||||||
// Attach user authentication for all S3 routes.
|
attachErrorHandler(api, log, h, center, usersStat)
|
||||||
AttachUserAuth(api, center, log)
|
|
||||||
|
|
||||||
buckets := make([]*mux.Router, 0, len(domains)+1)
|
buckets := make([]*mux.Router, 0, len(domains)+1)
|
||||||
buckets = append(buckets, api.PathPrefix("/{bucket}").Subrouter())
|
buckets = append(buckets, api.PathPrefix("/{bucket}").Subrouter())
|
||||||
|
@ -212,277 +279,327 @@ func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center aut
|
||||||
// -- append CORS headers to a response for
|
// -- append CORS headers to a response for
|
||||||
appendCORS(h),
|
appendCORS(h),
|
||||||
)
|
)
|
||||||
bucket.Methods(http.MethodOptions).HandlerFunc(m.Handle(metrics.APIStats("preflight", h.Preflight))).Name("Options")
|
bucket.Methods(http.MethodOptions).HandlerFunc(
|
||||||
|
m.Handle(h.Preflight)).
|
||||||
|
Name("Options")
|
||||||
bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("headobject", h.HeadObjectHandler))).Name("HeadObject")
|
m.Handle(h.HeadObjectHandler)).
|
||||||
|
Name("HeadObject")
|
||||||
// CopyObjectPart
|
// CopyObjectPart
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(metrics.APIStats("uploadpartcopy", h.UploadPartCopy))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(
|
||||||
|
m.Handle(h.UploadPartCopy)).
|
||||||
|
Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
|
||||||
Name("UploadPartCopy")
|
Name("UploadPartCopy")
|
||||||
// PutObjectPart
|
// PutObjectPart
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("uploadpart", h.UploadPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
|
m.Handle(h.UploadPartHandler)).
|
||||||
|
Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
|
||||||
Name("UploadPart")
|
Name("UploadPart")
|
||||||
// ListParts
|
// ListParts
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listobjectparts", h.ListPartsHandler))).Queries("uploadId", "{uploadId:.*}").
|
m.Handle(h.ListPartsHandler)).
|
||||||
|
Queries("uploadId", "{uploadId:.*}").
|
||||||
Name("ListObjectParts")
|
Name("ListObjectParts")
|
||||||
// CompleteMultipartUpload
|
// CompleteMultipartUpload
|
||||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("completemutipartupload", h.CompleteMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}").
|
m.Handle(h.CompleteMultipartUploadHandler)).
|
||||||
|
Queries("uploadId", "{uploadId:.*}").
|
||||||
Name("CompleteMultipartUpload")
|
Name("CompleteMultipartUpload")
|
||||||
// CreateMultipartUpload
|
// CreateMultipartUpload
|
||||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("createmultipartupload", h.CreateMultipartUploadHandler))).Queries("uploads", "").
|
m.Handle(h.CreateMultipartUploadHandler)).
|
||||||
|
Queries("uploads", "").
|
||||||
Name("CreateMultipartUpload")
|
Name("CreateMultipartUpload")
|
||||||
// AbortMultipartUpload
|
// AbortMultipartUpload
|
||||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("abortmultipartupload", h.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}").
|
m.Handle(h.AbortMultipartUploadHandler)).
|
||||||
|
Queries("uploadId", "{uploadId:.*}").
|
||||||
Name("AbortMultipartUpload")
|
Name("AbortMultipartUpload")
|
||||||
// ListMultipartUploads
|
// ListMultipartUploads
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listmultipartuploads", h.ListMultipartUploadsHandler))).Queries("uploads", "").
|
m.Handle(h.ListMultipartUploadsHandler)).
|
||||||
|
Queries("uploads", "").
|
||||||
Name("ListMultipartUploads")
|
Name("ListMultipartUploads")
|
||||||
// GetObjectACL -- this is a dummy call.
|
// GetObjectACL -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getobjectacl", h.GetObjectACLHandler))).Queries("acl", "").
|
m.Handle(h.GetObjectACLHandler)).
|
||||||
|
Queries("acl", "").
|
||||||
Name("GetObjectACL")
|
Name("GetObjectACL")
|
||||||
// PutObjectACL -- this is a dummy call.
|
// PutObjectACL -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putobjectacl", h.PutObjectACLHandler))).Queries("acl", "").
|
m.Handle(h.PutObjectACLHandler)).
|
||||||
|
Queries("acl", "").
|
||||||
Name("PutObjectACL")
|
Name("PutObjectACL")
|
||||||
// GetObjectTagging
|
// GetObjectTagging
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getobjecttagging", h.GetObjectTaggingHandler))).Queries("tagging", "").
|
m.Handle(h.GetObjectTaggingHandler)).
|
||||||
|
Queries("tagging", "").
|
||||||
Name("GetObjectTagging")
|
Name("GetObjectTagging")
|
||||||
// PutObjectTagging
|
// PutObjectTagging
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putobjecttagging", h.PutObjectTaggingHandler))).Queries("tagging", "").
|
m.Handle(h.PutObjectTaggingHandler)).
|
||||||
|
Queries("tagging", "").
|
||||||
Name("PutObjectTagging")
|
Name("PutObjectTagging")
|
||||||
// DeleteObjectTagging
|
// DeleteObjectTagging
|
||||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deleteobjecttagging", h.DeleteObjectTaggingHandler))).Queries("tagging", "").
|
m.Handle(h.DeleteObjectTaggingHandler)).
|
||||||
|
Queries("tagging", "").
|
||||||
Name("DeleteObjectTagging")
|
Name("DeleteObjectTagging")
|
||||||
// SelectObjectContent
|
// SelectObjectContent
|
||||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("selectobjectcontent", h.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2").
|
m.Handle(h.SelectObjectContentHandler)).
|
||||||
|
Queries("select", "").Queries("select-type", "2").
|
||||||
Name("SelectObjectContent")
|
Name("SelectObjectContent")
|
||||||
// GetObjectRetention
|
// GetObjectRetention
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getobjectretention", h.GetObjectRetentionHandler))).Queries("retention", "").
|
m.Handle(h.GetObjectRetentionHandler)).
|
||||||
|
Queries("retention", "").
|
||||||
Name("GetObjectRetention")
|
Name("GetObjectRetention")
|
||||||
// GetObjectLegalHold
|
// GetObjectLegalHold
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getobjectlegalhold", h.GetObjectLegalHoldHandler))).Queries("legal-hold", "").
|
m.Handle(h.GetObjectLegalHoldHandler)).
|
||||||
|
Queries("legal-hold", "").
|
||||||
Name("GetObjectLegalHold")
|
Name("GetObjectLegalHold")
|
||||||
// GetObjectAttributes
|
// GetObjectAttributes
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getobjectattributes", h.GetObjectAttributesHandler))).Queries("attributes", "").
|
m.Handle(h.GetObjectAttributesHandler)).
|
||||||
|
Queries("attributes", "").
|
||||||
Name("GetObjectAttributes")
|
Name("GetObjectAttributes")
|
||||||
// GetObject
|
// GetObject
|
||||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getobject", h.GetObjectHandler))).
|
m.Handle(h.GetObjectHandler)).
|
||||||
Name("GetObject")
|
Name("GetObject")
|
||||||
// CopyObject
|
// CopyObject
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(metrics.APIStats("copyobject", h.CopyObjectHandler))).
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(
|
||||||
|
m.Handle(h.CopyObjectHandler)).
|
||||||
Name("CopyObject")
|
Name("CopyObject")
|
||||||
// PutObjectRetention
|
// PutObjectRetention
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putobjectretention", h.PutObjectRetentionHandler))).Queries("retention", "").
|
m.Handle(h.PutObjectRetentionHandler)).
|
||||||
|
Queries("retention", "").
|
||||||
Name("PutObjectRetention")
|
Name("PutObjectRetention")
|
||||||
// PutObjectLegalHold
|
// PutObjectLegalHold
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putobjectlegalhold", h.PutObjectLegalHoldHandler))).Queries("legal-hold", "").
|
m.Handle(h.PutObjectLegalHoldHandler)).
|
||||||
|
Queries("legal-hold", "").
|
||||||
Name("PutObjectLegalHold")
|
Name("PutObjectLegalHold")
|
||||||
|
|
||||||
// PutObject
|
// PutObject
|
||||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putobject", h.PutObjectHandler))).
|
m.Handle(h.PutObjectHandler)).
|
||||||
Name("PutObject")
|
Name("PutObject")
|
||||||
// DeleteObject
|
// DeleteObject
|
||||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deleteobject", h.DeleteObjectHandler))).
|
m.Handle(h.DeleteObjectHandler)).
|
||||||
Name("DeleteObject")
|
Name("DeleteObject")
|
||||||
|
|
||||||
// Bucket operations
|
// Bucket operations
|
||||||
// GetBucketLocation
|
// GetBucketLocation
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketlocation", h.GetBucketLocationHandler))).Queries("location", "").
|
m.Handle(h.GetBucketLocationHandler)).
|
||||||
|
Queries("location", "").
|
||||||
Name("GetBucketLocation")
|
Name("GetBucketLocation")
|
||||||
// GetBucketPolicy
|
// GetBucketPolicy
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketpolicy", h.GetBucketPolicyHandler))).Queries("policy", "").
|
m.Handle(h.GetBucketPolicyHandler)).
|
||||||
|
Queries("policy", "").
|
||||||
Name("GetBucketPolicy")
|
Name("GetBucketPolicy")
|
||||||
// GetBucketLifecycle
|
// GetBucketLifecycle
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", "").
|
m.Handle(h.GetBucketLifecycleHandler)).
|
||||||
|
Queries("lifecycle", "").
|
||||||
Name("GetBucketLifecycle")
|
Name("GetBucketLifecycle")
|
||||||
// GetBucketEncryption
|
// GetBucketEncryption
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketencryption", h.GetBucketEncryptionHandler))).Queries("encryption", "").
|
m.Handle(h.GetBucketEncryptionHandler)).
|
||||||
|
Queries("encryption", "").
|
||||||
Name("GetBucketEncryption")
|
Name("GetBucketEncryption")
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketcors", h.GetBucketCorsHandler))).Queries("cors", "").
|
m.Handle(h.GetBucketCorsHandler)).
|
||||||
|
Queries("cors", "").
|
||||||
Name("GetBucketCors")
|
Name("GetBucketCors")
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketcors", h.PutBucketCorsHandler))).Queries("cors", "").
|
m.Handle(h.PutBucketCorsHandler)).
|
||||||
|
Queries("cors", "").
|
||||||
Name("PutBucketCors")
|
Name("PutBucketCors")
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebucketcors", h.DeleteBucketCorsHandler))).Queries("cors", "").
|
m.Handle(h.DeleteBucketCorsHandler)).
|
||||||
|
Queries("cors", "").
|
||||||
Name("DeleteBucketCors")
|
Name("DeleteBucketCors")
|
||||||
// Dummy Bucket Calls
|
// Dummy Bucket Calls
|
||||||
// GetBucketACL -- this is a dummy call.
|
// GetBucketACL -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketacl", h.GetBucketACLHandler))).Queries("acl", "").
|
m.Handle(h.GetBucketACLHandler)).
|
||||||
|
Queries("acl", "").
|
||||||
Name("GetBucketACL")
|
Name("GetBucketACL")
|
||||||
// PutBucketACL -- this is a dummy call.
|
// PutBucketACL -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketacl", h.PutBucketACLHandler))).Queries("acl", "").
|
m.Handle(h.PutBucketACLHandler)).
|
||||||
|
Queries("acl", "").
|
||||||
Name("PutBucketACL")
|
Name("PutBucketACL")
|
||||||
// GetBucketWebsiteHandler -- this is a dummy call.
|
// GetBucketWebsiteHandler -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketwebsite", h.GetBucketWebsiteHandler))).Queries("website", "").
|
m.Handle(h.GetBucketWebsiteHandler)).
|
||||||
|
Queries("website", "").
|
||||||
Name("GetBucketWebsite")
|
Name("GetBucketWebsite")
|
||||||
// GetBucketAccelerateHandler -- this is a dummy call.
|
// GetBucketAccelerateHandler -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketaccelerate", h.GetBucketAccelerateHandler))).Queries("accelerate", "").
|
m.Handle(h.GetBucketAccelerateHandler)).
|
||||||
|
Queries("accelerate", "").
|
||||||
Name("GetBucketAccelerate")
|
Name("GetBucketAccelerate")
|
||||||
// GetBucketRequestPaymentHandler -- this is a dummy call.
|
// GetBucketRequestPaymentHandler -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketrequestpayment", h.GetBucketRequestPaymentHandler))).Queries("requestPayment", "").
|
m.Handle(h.GetBucketRequestPaymentHandler)).
|
||||||
|
Queries("requestPayment", "").
|
||||||
Name("GetBucketRequestPayment")
|
Name("GetBucketRequestPayment")
|
||||||
// GetBucketLoggingHandler -- this is a dummy call.
|
// GetBucketLoggingHandler -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketlogging", h.GetBucketLoggingHandler))).Queries("logging", "").
|
m.Handle(h.GetBucketLoggingHandler)).
|
||||||
|
Queries("logging", "").
|
||||||
Name("GetBucketLogging")
|
Name("GetBucketLogging")
|
||||||
// GetBucketLifecycleHandler -- this is a dummy call.
|
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
|
||||||
m.Handle(metrics.APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", "").
|
|
||||||
Name("GetBucketLifecycle")
|
|
||||||
// GetBucketReplicationHandler -- this is a dummy call.
|
// GetBucketReplicationHandler -- this is a dummy call.
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketreplication", h.GetBucketReplicationHandler))).Queries("replication", "").
|
m.Handle(h.GetBucketReplicationHandler)).
|
||||||
|
Queries("replication", "").
|
||||||
Name("GetBucketReplication")
|
Name("GetBucketReplication")
|
||||||
// GetBucketTaggingHandler
|
// GetBucketTaggingHandler
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbuckettagging", h.GetBucketTaggingHandler))).Queries("tagging", "").
|
m.Handle(h.GetBucketTaggingHandler)).
|
||||||
|
Queries("tagging", "").
|
||||||
Name("GetBucketTagging")
|
Name("GetBucketTagging")
|
||||||
// DeleteBucketWebsiteHandler
|
// DeleteBucketWebsiteHandler
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebucketwebsite", h.DeleteBucketWebsiteHandler))).Queries("website", "").
|
m.Handle(h.DeleteBucketWebsiteHandler)).
|
||||||
|
Queries("website", "").
|
||||||
Name("DeleteBucketWebsite")
|
Name("DeleteBucketWebsite")
|
||||||
// DeleteBucketTaggingHandler
|
// DeleteBucketTaggingHandler
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebuckettagging", h.DeleteBucketTaggingHandler))).Queries("tagging", "").
|
m.Handle(h.DeleteBucketTaggingHandler)).
|
||||||
|
Queries("tagging", "").
|
||||||
Name("DeleteBucketTagging")
|
Name("DeleteBucketTagging")
|
||||||
|
|
||||||
// GetBucketObjectLockConfig
|
// GetBucketObjectLockConfig
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketobjectlockconfiguration", h.GetBucketObjectLockConfigHandler))).Queries("object-lock", "").
|
m.Handle(h.GetBucketObjectLockConfigHandler)).
|
||||||
|
Queries("object-lock", "").
|
||||||
Name("GetBucketObjectLockConfig")
|
Name("GetBucketObjectLockConfig")
|
||||||
// GetBucketVersioning
|
// GetBucketVersioning
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketversioning", h.GetBucketVersioningHandler))).Queries("versioning", "").
|
m.Handle(h.GetBucketVersioningHandler)).
|
||||||
|
Queries("versioning", "").
|
||||||
Name("GetBucketVersioning")
|
Name("GetBucketVersioning")
|
||||||
// GetBucketNotification
|
// GetBucketNotification
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("getbucketnotification", h.GetBucketNotificationHandler))).Queries("notification", "").
|
m.Handle(h.GetBucketNotificationHandler)).
|
||||||
|
Queries("notification", "").
|
||||||
Name("GetBucketNotification")
|
Name("GetBucketNotification")
|
||||||
// ListenBucketNotification
|
// ListenBucketNotification
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(metrics.APIStats("listenbucketnotification", h.ListenBucketNotificationHandler)).Queries("events", "{events:.*}").
|
bucket.Methods(http.MethodGet).HandlerFunc(h.ListenBucketNotificationHandler).
|
||||||
|
Queries("events", "{events:.*}").
|
||||||
Name("ListenBucketNotification")
|
Name("ListenBucketNotification")
|
||||||
// ListObjectsV2M
|
// ListObjectsV2M
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listobjectsv2M", h.ListObjectsV2MHandler))).Queries("list-type", "2", "metadata", "true").
|
m.Handle(h.ListObjectsV2MHandler)).
|
||||||
|
Queries("list-type", "2", "metadata", "true").
|
||||||
Name("ListObjectsV2M")
|
Name("ListObjectsV2M")
|
||||||
// ListObjectsV2
|
// ListObjectsV2
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listobjectsv2", h.ListObjectsV2Handler))).Queries("list-type", "2").
|
m.Handle(h.ListObjectsV2Handler)).
|
||||||
|
Queries("list-type", "2").
|
||||||
Name("ListObjectsV2")
|
Name("ListObjectsV2")
|
||||||
// ListBucketVersions
|
// ListBucketVersions
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listbucketversions", h.ListBucketObjectVersionsHandler))).Queries("versions", "").
|
m.Handle(h.ListBucketObjectVersionsHandler)).
|
||||||
|
Queries("versions", "").
|
||||||
Name("ListBucketVersions")
|
Name("ListBucketVersions")
|
||||||
// ListObjectsV1 (Legacy)
|
// ListObjectsV1 (Legacy)
|
||||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listobjectsv1", h.ListObjectsV1Handler))).
|
m.Handle(h.ListObjectsV1Handler)).
|
||||||
Name("ListObjectsV1")
|
Name("ListObjectsV1")
|
||||||
// PutBucketLifecycle
|
// PutBucketLifecycle
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketlifecycle", h.PutBucketLifecycleHandler))).Queries("lifecycle", "").
|
m.Handle(h.PutBucketLifecycleHandler)).
|
||||||
|
Queries("lifecycle", "").
|
||||||
Name("PutBucketLifecycle")
|
Name("PutBucketLifecycle")
|
||||||
// PutBucketEncryption
|
// PutBucketEncryption
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketencryption", h.PutBucketEncryptionHandler))).Queries("encryption", "").
|
m.Handle(h.PutBucketEncryptionHandler)).
|
||||||
|
Queries("encryption", "").
|
||||||
Name("PutBucketEncryption")
|
Name("PutBucketEncryption")
|
||||||
|
|
||||||
// PutBucketPolicy
|
// PutBucketPolicy
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketpolicy", h.PutBucketPolicyHandler))).Queries("policy", "").
|
m.Handle(h.PutBucketPolicyHandler)).
|
||||||
|
Queries("policy", "").
|
||||||
Name("PutBucketPolicy")
|
Name("PutBucketPolicy")
|
||||||
|
|
||||||
// PutBucketObjectLockConfig
|
// PutBucketObjectLockConfig
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketobjectlockconfig", h.PutBucketObjectLockConfigHandler))).Queries("object-lock", "").
|
m.Handle(h.PutBucketObjectLockConfigHandler)).
|
||||||
|
Queries("object-lock", "").
|
||||||
Name("PutBucketObjectLockConfig")
|
Name("PutBucketObjectLockConfig")
|
||||||
// PutBucketTaggingHandler
|
// PutBucketTaggingHandler
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbuckettagging", h.PutBucketTaggingHandler))).Queries("tagging", "").
|
m.Handle(h.PutBucketTaggingHandler)).
|
||||||
|
Queries("tagging", "").
|
||||||
Name("PutBucketTagging")
|
Name("PutBucketTagging")
|
||||||
// PutBucketVersioning
|
// PutBucketVersioning
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketversioning", h.PutBucketVersioningHandler))).Queries("versioning", "").
|
m.Handle(h.PutBucketVersioningHandler)).
|
||||||
|
Queries("versioning", "").
|
||||||
Name("PutBucketVersioning")
|
Name("PutBucketVersioning")
|
||||||
// PutBucketNotification
|
// PutBucketNotification
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("putbucketnotification", h.PutBucketNotificationHandler))).Queries("notification", "").
|
m.Handle(h.PutBucketNotificationHandler)).
|
||||||
|
Queries("notification", "").
|
||||||
Name("PutBucketNotification")
|
Name("PutBucketNotification")
|
||||||
// CreateBucket
|
// CreateBucket
|
||||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("createbucket", h.CreateBucketHandler))).
|
m.Handle(h.CreateBucketHandler)).
|
||||||
Name("CreateBucket")
|
Name("CreateBucket")
|
||||||
// HeadBucket
|
// HeadBucket
|
||||||
bucket.Methods(http.MethodHead).HandlerFunc(
|
bucket.Methods(http.MethodHead).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("headbucket", h.HeadBucketHandler))).
|
m.Handle(h.HeadBucketHandler)).
|
||||||
Name("HeadBucket")
|
Name("HeadBucket")
|
||||||
// PostPolicy
|
// PostPolicy
|
||||||
bucket.Methods(http.MethodPost).HeadersRegexp(hdrContentType, "multipart/form-data*").HandlerFunc(
|
bucket.Methods(http.MethodPost).HeadersRegexp(hdrContentType, "multipart/form-data*").HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("postobject", h.PostObject))).
|
m.Handle(h.PostObject)).
|
||||||
Name("PostObject")
|
Name("PostObject")
|
||||||
// DeleteMultipleObjects
|
// DeleteMultipleObjects
|
||||||
bucket.Methods(http.MethodPost).HandlerFunc(
|
bucket.Methods(http.MethodPost).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletemultipleobjects", h.DeleteMultipleObjectsHandler))).Queries("delete", "").
|
m.Handle(h.DeleteMultipleObjectsHandler)).
|
||||||
|
Queries("delete", "").
|
||||||
Name("DeleteMultipleObjects")
|
Name("DeleteMultipleObjects")
|
||||||
// DeleteBucketPolicy
|
// DeleteBucketPolicy
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebucketpolicy", h.DeleteBucketPolicyHandler))).Queries("policy", "").
|
m.Handle(h.DeleteBucketPolicyHandler)).
|
||||||
|
Queries("policy", "").
|
||||||
Name("DeleteBucketPolicy")
|
Name("DeleteBucketPolicy")
|
||||||
// DeleteBucketLifecycle
|
// DeleteBucketLifecycle
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebucketlifecycle", h.DeleteBucketLifecycleHandler))).Queries("lifecycle", "").
|
m.Handle(h.DeleteBucketLifecycleHandler)).
|
||||||
|
Queries("lifecycle", "").
|
||||||
Name("DeleteBucketLifecycle")
|
Name("DeleteBucketLifecycle")
|
||||||
// DeleteBucketEncryption
|
// DeleteBucketEncryption
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebucketencryption", h.DeleteBucketEncryptionHandler))).Queries("encryption", "").
|
m.Handle(h.DeleteBucketEncryptionHandler)).
|
||||||
|
Queries("encryption", "").
|
||||||
Name("DeleteBucketEncryption")
|
Name("DeleteBucketEncryption")
|
||||||
// DeleteBucket
|
// DeleteBucket
|
||||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("deletebucket", h.DeleteBucketHandler))).
|
m.Handle(h.DeleteBucketHandler)).
|
||||||
Name("DeleteBucket")
|
Name("DeleteBucket")
|
||||||
}
|
}
|
||||||
// Root operation
|
// Root operation
|
||||||
|
|
||||||
// ListBuckets
|
// ListBuckets
|
||||||
api.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
|
api.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listbuckets", h.ListBucketsHandler))).
|
m.Handle(h.ListBucketsHandler)).
|
||||||
Name("ListBuckets")
|
Name("ListBuckets")
|
||||||
|
|
||||||
// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
|
// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
|
||||||
// than failing with UnknownAPIRequest we simply handle it for now.
|
// than failing with UnknownAPIRequest we simply handle it for now.
|
||||||
api.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
|
api.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
|
||||||
m.Handle(metrics.APIStats("listbuckets", h.ListBucketsHandler))).
|
m.Handle(h.ListBucketsHandler)).
|
||||||
Name("ListBuckets")
|
Name("ListBuckets")
|
||||||
|
|
||||||
// If none of the routes match, add default error handler routes
|
|
||||||
api.NotFoundHandler = metrics.APIStats("notfound", errorResponseHandler)
|
|
||||||
api.MethodNotAllowedHandler = metrics.APIStats("methodnotallowed", errorResponseHandler)
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,9 +4,9 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -16,9 +16,12 @@ type KeyWrapper string
|
||||||
// BoxData is an ID used to store accessbox.Box in a context.
|
// BoxData is an ID used to store accessbox.Box in a context.
|
||||||
var BoxData = KeyWrapper("__context_box_key")
|
var BoxData = KeyWrapper("__context_box_key")
|
||||||
|
|
||||||
// AttachUserAuth adds user authentication via center to router using log for logging.
|
// ClientTime is an ID used to store client time.Time in a context.
|
||||||
func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
|
var ClientTime = KeyWrapper("__context_client_time")
|
||||||
router.Use(func(h http.Handler) http.Handler {
|
|
||||||
|
// AuthMiddleware adds user authentication via center to router using log for logging.
|
||||||
|
func AuthMiddleware(log *zap.Logger, center auth.Center) mux.MiddlewareFunc {
|
||||||
|
return func(h http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
var ctx context.Context
|
var ctx context.Context
|
||||||
box, err := center.Authenticate(r)
|
box, err := center.Authenticate(r)
|
||||||
|
@ -35,10 +38,13 @@ func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
ctx = context.WithValue(r.Context(), BoxData, box)
|
ctx = context.WithValue(r.Context(), BoxData, box.AccessBox)
|
||||||
|
if !box.ClientTime.IsZero() {
|
||||||
|
ctx = context.WithValue(ctx, ClientTime, box.ClientTime)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
h.ServeHTTP(w, r.WithContext(ctx))
|
h.ServeHTTP(w, r.WithContext(ctx))
|
||||||
})
|
})
|
||||||
})
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,25 +11,25 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
|
||||||
neofsecdsa "github.com/nspcc-dev/neofs-sdk-go/crypto/ecdsa"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
|
||||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
// PrmContainerCreate groups parameters of containers created by authmate.
|
// PrmContainerCreate groups parameters of containers created by authmate.
|
||||||
type PrmContainerCreate struct {
|
type PrmContainerCreate struct {
|
||||||
// NeoFS identifier of the container creator.
|
// FrostFS identifier of the container creator.
|
||||||
Owner user.ID
|
Owner user.ID
|
||||||
|
|
||||||
// Container placement policy.
|
// Container placement policy.
|
||||||
|
@ -39,26 +39,26 @@ type PrmContainerCreate struct {
|
||||||
FriendlyName string
|
FriendlyName string
|
||||||
}
|
}
|
||||||
|
|
||||||
// NetworkState represents NeoFS network state which is needed for authmate processing.
|
// NetworkState represents FrostFS network state which is needed for authmate processing.
|
||||||
type NetworkState struct {
|
type NetworkState struct {
|
||||||
// Current NeoFS time.
|
// Current FrostFS time.
|
||||||
Epoch uint64
|
Epoch uint64
|
||||||
// Duration of the Morph chain block in ms.
|
// Duration of the Morph chain block in ms.
|
||||||
BlockDuration int64
|
BlockDuration int64
|
||||||
// Duration of the NeoFS epoch in Morph chain blocks.
|
// Duration of the FrostFS epoch in Morph chain blocks.
|
||||||
EpochDuration uint64
|
EpochDuration uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
// NeoFS represents virtual connection to NeoFS network.
|
// FrostFS represents virtual connection to FrostFS network.
|
||||||
type NeoFS interface {
|
type FrostFS interface {
|
||||||
// NeoFS interface required by credential tool.
|
// FrostFS interface required by credential tool.
|
||||||
tokens.NeoFS
|
tokens.FrostFS
|
||||||
|
|
||||||
// ContainerExists checks container presence in NeoFS by identifier.
|
// ContainerExists checks container presence in FrostFS by identifier.
|
||||||
// Returns nil if container exists.
|
// Returns nil if container exists.
|
||||||
ContainerExists(context.Context, cid.ID) error
|
ContainerExists(context.Context, cid.ID) error
|
||||||
|
|
||||||
// CreateContainer creates and saves parameterized container in NeoFS.
|
// CreateContainer creates and saves parameterized container in FrostFS.
|
||||||
// It sets 'Timestamp' attribute to the current time.
|
// It sets 'Timestamp' attribute to the current time.
|
||||||
// It returns the ID of the saved container.
|
// It returns the ID of the saved container.
|
||||||
//
|
//
|
||||||
|
@ -78,25 +78,25 @@ type NeoFS interface {
|
||||||
TimeToEpoch(context.Context, time.Time) (uint64, uint64, error)
|
TimeToEpoch(context.Context, time.Time) (uint64, uint64, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Agent contains client communicating with NeoFS and logger.
|
// Agent contains client communicating with FrostFS and logger.
|
||||||
type Agent struct {
|
type Agent struct {
|
||||||
neoFS NeoFS
|
frostFS FrostFS
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates an object of type Agent that consists of Client and logger.
|
// New creates an object of type Agent that consists of Client and logger.
|
||||||
func New(log *zap.Logger, neoFS NeoFS) *Agent {
|
func New(log *zap.Logger, frostFS FrostFS) *Agent {
|
||||||
return &Agent{log: log, neoFS: neoFS}
|
return &Agent{log: log, frostFS: frostFS}
|
||||||
}
|
}
|
||||||
|
|
||||||
type (
|
type (
|
||||||
// ContainerPolicies contains mapping of aws LocationConstraint to neofs PlacementPolicy.
|
// ContainerPolicies contains mapping of aws LocationConstraint to frostfs PlacementPolicy.
|
||||||
ContainerPolicies map[string]string
|
ContainerPolicies map[string]string
|
||||||
|
|
||||||
// IssueSecretOptions contains options for passing to Agent.IssueSecret method.
|
// IssueSecretOptions contains options for passing to Agent.IssueSecret method.
|
||||||
IssueSecretOptions struct {
|
IssueSecretOptions struct {
|
||||||
Container ContainerOptions
|
Container ContainerOptions
|
||||||
NeoFSKey *keys.PrivateKey
|
FrostFSKey *keys.PrivateKey
|
||||||
GatesPublicKeys []*keys.PublicKey
|
GatesPublicKeys []*keys.PublicKey
|
||||||
EACLRules []byte
|
EACLRules []byte
|
||||||
SessionTokenRules []byte
|
SessionTokenRules []byte
|
||||||
|
@ -120,7 +120,7 @@ type (
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
// lifetimeOptions holds NeoFS epochs, iat -- epoch which the token was issued at, exp -- epoch when the token expires.
|
// lifetimeOptions holds FrostFS epochs, iat -- epoch which the token was issued at, exp -- epoch when the token expires.
|
||||||
type lifetimeOptions struct {
|
type lifetimeOptions struct {
|
||||||
Iat uint64
|
Iat uint64
|
||||||
Exp uint64
|
Exp uint64
|
||||||
|
@ -143,7 +143,7 @@ type (
|
||||||
|
|
||||||
func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwner user.ID) (cid.ID, error) {
|
func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwner user.ID) (cid.ID, error) {
|
||||||
if !opts.ID.Equals(cid.ID{}) {
|
if !opts.ID.Equals(cid.ID{}) {
|
||||||
return opts.ID, a.neoFS.ContainerExists(ctx, opts.ID)
|
return opts.ID, a.frostFS.ContainerExists(ctx, opts.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
var prm PrmContainerCreate
|
var prm PrmContainerCreate
|
||||||
|
@ -156,9 +156,9 @@ func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwn
|
||||||
prm.Owner = idOwner
|
prm.Owner = idOwner
|
||||||
prm.FriendlyName = opts.FriendlyName
|
prm.FriendlyName = opts.FriendlyName
|
||||||
|
|
||||||
cnrID, err := a.neoFS.CreateContainer(ctx, prm)
|
cnrID, err := a.frostFS.CreateContainer(ctx, prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cid.ID{}, fmt.Errorf("create container in NeoFS: %w", err)
|
return cid.ID{}, fmt.Errorf("create container in FrostFS: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return cnrID, nil
|
return cnrID, nil
|
||||||
|
@ -200,7 +200,7 @@ func preparePolicy(policy ContainerPolicies) ([]*accessbox.AccessBox_ContainerPo
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// IssueSecret creates an auth token, puts it in the NeoFS network and writes to io.Writer a new secret access key.
|
// IssueSecret creates an auth token, puts it in the FrostFS network and writes to io.Writer a new secret access key.
|
||||||
func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecretOptions) error {
|
func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecretOptions) error {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
|
@ -213,7 +213,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
||||||
return fmt.Errorf("prepare policies: %w", err)
|
return fmt.Errorf("prepare policies: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
lifetime.Iat, lifetime.Exp, err = a.neoFS.TimeToEpoch(ctx, time.Now().Add(options.Lifetime))
|
lifetime.Iat, lifetime.Exp, err = a.frostFS.TimeToEpoch(ctx, time.Now().Add(options.Lifetime))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("fetch time to epoch: %w", err)
|
return fmt.Errorf("fetch time to epoch: %w", err)
|
||||||
}
|
}
|
||||||
|
@ -231,7 +231,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
||||||
box.ContainerPolicy = policies
|
box.ContainerPolicy = policies
|
||||||
|
|
||||||
var idOwner user.ID
|
var idOwner user.ID
|
||||||
user.IDFromKey(&idOwner, options.NeoFSKey.PrivateKey.PublicKey)
|
user.IDFromKey(&idOwner, options.FrostFSKey.PrivateKey.PublicKey)
|
||||||
|
|
||||||
a.log.Info("check container or create", zap.Stringer("cid", options.Container.ID),
|
a.log.Info("check container or create", zap.Stringer("cid", options.Container.ID),
|
||||||
zap.String("friendly_name", options.Container.FriendlyName),
|
zap.String("friendly_name", options.Container.FriendlyName),
|
||||||
|
@ -241,11 +241,11 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
||||||
return fmt.Errorf("check container: %w", err)
|
return fmt.Errorf("check container: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
a.log.Info("store bearer token into NeoFS",
|
a.log.Info("store bearer token into FrostFS",
|
||||||
zap.Stringer("owner_tkn", idOwner))
|
zap.Stringer("owner_tkn", idOwner))
|
||||||
|
|
||||||
addr, err := tokens.
|
addr, err := tokens.
|
||||||
New(a.neoFS, secrets.EphemeralKey, cache.DefaultAccessBoxConfig(a.log)).
|
New(a.frostFS, secrets.EphemeralKey, cache.DefaultAccessBoxConfig(a.log)).
|
||||||
Put(ctx, id, idOwner, box, lifetime.Exp, options.GatesPublicKeys...)
|
Put(ctx, id, idOwner, box, lifetime.Exp, options.GatesPublicKeys...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to put bearer token: %w", err)
|
return fmt.Errorf("failed to put bearer token: %w", err)
|
||||||
|
@ -260,7 +260,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
||||||
AccessKeyID: accessKeyID,
|
AccessKeyID: accessKeyID,
|
||||||
SecretAccessKey: secrets.AccessKey,
|
SecretAccessKey: secrets.AccessKey,
|
||||||
OwnerPrivateKey: hex.EncodeToString(secrets.EphemeralKey.Bytes()),
|
OwnerPrivateKey: hex.EncodeToString(secrets.EphemeralKey.Bytes()),
|
||||||
WalletPublicKey: hex.EncodeToString(options.NeoFSKey.PublicKey().Bytes()),
|
WalletPublicKey: hex.EncodeToString(options.FrostFSKey.PublicKey().Bytes()),
|
||||||
ContainerID: id.EncodeToString(),
|
ContainerID: id.EncodeToString(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -288,10 +288,10 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ObtainSecret receives an existing secret access key from NeoFS and
|
// ObtainSecret receives an existing secret access key from FrostFS and
|
||||||
// writes to io.Writer the secret access key.
|
// writes to io.Writer the secret access key.
|
||||||
func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSecretOptions) error {
|
func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSecretOptions) error {
|
||||||
bearerCreds := tokens.New(a.neoFS, options.GatePrivateKey, cache.DefaultAccessBoxConfig(a.log))
|
bearerCreds := tokens.New(a.frostFS, options.GatePrivateKey, cache.DefaultAccessBoxConfig(a.log))
|
||||||
|
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
if err := addr.DecodeString(options.SecretAddress); err != nil {
|
if err := addr.DecodeString(options.SecretAddress); err != nil {
|
||||||
|
@ -381,7 +381,7 @@ func buildSessionToken(key *keys.PrivateKey, lifetime lifetimeOptions, ctx sessi
|
||||||
tok.AppliedTo(ctx.containerID)
|
tok.AppliedTo(ctx.containerID)
|
||||||
|
|
||||||
tok.SetID(uuid.New())
|
tok.SetID(uuid.New())
|
||||||
tok.SetAuthKey((*neofsecdsa.PublicKey)(gateKey))
|
tok.SetAuthKey((*frostfsecdsa.PublicKey)(gateKey))
|
||||||
|
|
||||||
tok.SetIat(lifetime.Iat)
|
tok.SetIat(lifetime.Iat)
|
||||||
tok.SetNbf(lifetime.Iat)
|
tok.SetNbf(lifetime.Iat)
|
||||||
|
@ -413,7 +413,7 @@ func createTokens(options *IssueSecretOptions, lifetime lifetimeOptions) ([]*acc
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to build eacl table: %w", err)
|
return nil, fmt.Errorf("failed to build eacl table: %w", err)
|
||||||
}
|
}
|
||||||
bearerTokens, err := buildBearerTokens(options.NeoFSKey, table, lifetime, options.GatesPublicKeys)
|
bearerTokens, err := buildBearerTokens(options.FrostFSKey, table, lifetime, options.GatesPublicKeys)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to build bearer tokens: %w", err)
|
return nil, fmt.Errorf("failed to build bearer tokens: %w", err)
|
||||||
}
|
}
|
||||||
|
@ -427,7 +427,7 @@ func createTokens(options *IssueSecretOptions, lifetime lifetimeOptions) ([]*acc
|
||||||
return nil, fmt.Errorf("failed to build context for session token: %w", err)
|
return nil, fmt.Errorf("failed to build context for session token: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
sessionTokens, err := buildSessionTokens(options.NeoFSKey, lifetime, sessionRules, options.GatesPublicKeys)
|
sessionTokens, err := buildSessionTokens(options.FrostFSKey, lifetime, sessionRules, options.GatesPublicKeys)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to biuild session token: %w", err)
|
return nil, fmt.Errorf("failed to biuild session token: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,9 +4,9 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
apisession "github.com/nspcc-dev/neofs-api-go/v2/session"
|
apisession "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session"
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
|
|
@ -3,7 +3,7 @@ package authmate
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -13,18 +13,18 @@ import (
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet"
|
||||||
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
"github.com/aws/aws-sdk-go/aws"
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
"github.com/aws/aws-sdk-go/aws/session"
|
"github.com/aws/aws-sdk-go/aws/session"
|
||||||
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
|
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/authmate"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/internal/neofs"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/internal/version"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/internal/wallet"
|
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
"github.com/urfave/cli/v2"
|
"github.com/urfave/cli/v2"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -32,13 +32,25 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
poolConnectTimeout = 5 * time.Second
|
poolDialTimeout = 5 * time.Second
|
||||||
poolRequestTimeout = 5 * time.Second
|
poolHealthcheckTimeout = 5 * time.Second
|
||||||
|
poolRebalanceInterval = 30 * time.Second
|
||||||
|
poolStreamTimeout = 10 * time.Second
|
||||||
|
|
||||||
// a month.
|
// a month.
|
||||||
defaultLifetime = 30 * 24 * time.Hour
|
defaultLifetime = 30 * 24 * time.Hour
|
||||||
defaultPresignedLifetime = 12 * time.Hour
|
defaultPresignedLifetime = 12 * time.Hour
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type PoolConfig struct {
|
||||||
|
Key *ecdsa.PrivateKey
|
||||||
|
Address string
|
||||||
|
DialTimeout time.Duration
|
||||||
|
HealthcheckTimeout time.Duration
|
||||||
|
StreamTimeout time.Duration
|
||||||
|
RebalanceInterval time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
walletPathFlag string
|
walletPathFlag string
|
||||||
accountAddressFlag string
|
accountAddressFlag string
|
||||||
|
@ -65,6 +77,12 @@ var (
|
||||||
containerPolicies string
|
containerPolicies string
|
||||||
awcCliCredFile string
|
awcCliCredFile string
|
||||||
timeoutFlag time.Duration
|
timeoutFlag time.Duration
|
||||||
|
|
||||||
|
// pool timeouts flag.
|
||||||
|
poolDialTimeoutFlag time.Duration
|
||||||
|
poolHealthcheckTimeoutFlag time.Duration
|
||||||
|
poolRebalanceIntervalFlag time.Duration
|
||||||
|
poolStreamTimeoutFlag time.Duration
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -110,8 +128,8 @@ func prepare() (context.Context, *zap.Logger) {
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
app := &cli.App{
|
app := &cli.App{
|
||||||
Name: "NeoFS S3 Authmate",
|
Name: "FrostFS S3 Authmate",
|
||||||
Usage: "Helps manage delegated access via gates to data stored in NeoFS network",
|
Usage: "Helps manage delegated access via gates to data stored in FrostFS network",
|
||||||
Version: version.Version,
|
Version: version.Version,
|
||||||
Flags: appFlags(),
|
Flags: appFlags(),
|
||||||
Commands: appCommands(),
|
Commands: appCommands(),
|
||||||
|
@ -164,7 +182,7 @@ func appCommands() []*cli.Command {
|
||||||
func issueSecret() *cli.Command {
|
func issueSecret() *cli.Command {
|
||||||
return &cli.Command{
|
return &cli.Command{
|
||||||
Name: "issue-secret",
|
Name: "issue-secret",
|
||||||
Usage: "Issue a secret in NeoFS network",
|
Usage: "Issue a secret in FrostFS network",
|
||||||
Flags: []cli.Flag{
|
Flags: []cli.Flag{
|
||||||
&cli.StringFlag{
|
&cli.StringFlag{
|
||||||
Name: "wallet",
|
Name: "wallet",
|
||||||
|
@ -183,7 +201,7 @@ func issueSecret() *cli.Command {
|
||||||
&cli.StringFlag{
|
&cli.StringFlag{
|
||||||
Name: "peer",
|
Name: "peer",
|
||||||
Value: "",
|
Value: "",
|
||||||
Usage: "address of a neofs peer to connect to",
|
Usage: "address of a frostfs peer to connect to",
|
||||||
Required: true,
|
Required: true,
|
||||||
Destination: &peerAddressFlag,
|
Destination: &peerAddressFlag,
|
||||||
},
|
},
|
||||||
|
@ -235,7 +253,7 @@ It will be ceil rounded to the nearest amount of epoch.`,
|
||||||
},
|
},
|
||||||
&cli.StringFlag{
|
&cli.StringFlag{
|
||||||
Name: "container-policy",
|
Name: "container-policy",
|
||||||
Usage: "mapping AWS storage class to NeoFS storage policy as plain json string or path to json file",
|
Usage: "mapping AWS storage class to FrostFS storage policy as plain json string or path to json file",
|
||||||
Required: false,
|
Required: false,
|
||||||
Destination: &containerPolicies,
|
Destination: &containerPolicies,
|
||||||
},
|
},
|
||||||
|
@ -245,6 +263,34 @@ It will be ceil rounded to the nearest amount of epoch.`,
|
||||||
Required: false,
|
Required: false,
|
||||||
Destination: &awcCliCredFile,
|
Destination: &awcCliCredFile,
|
||||||
},
|
},
|
||||||
|
&cli.DurationFlag{
|
||||||
|
Name: "pool-dial-timeout",
|
||||||
|
Usage: `Timeout for connection to the node in pool to be established`,
|
||||||
|
Required: false,
|
||||||
|
Destination: &poolDialTimeoutFlag,
|
||||||
|
Value: poolDialTimeout,
|
||||||
|
},
|
||||||
|
&cli.DurationFlag{
|
||||||
|
Name: "pool-healthcheck-timeout",
|
||||||
|
Usage: `Timeout for request to node to decide if it is alive`,
|
||||||
|
Required: false,
|
||||||
|
Destination: &poolHealthcheckTimeoutFlag,
|
||||||
|
Value: poolHealthcheckTimeout,
|
||||||
|
},
|
||||||
|
&cli.DurationFlag{
|
||||||
|
Name: "pool-rebalance-interval",
|
||||||
|
Usage: `Interval for updating nodes health status`,
|
||||||
|
Required: false,
|
||||||
|
Destination: &poolRebalanceIntervalFlag,
|
||||||
|
Value: poolRebalanceInterval,
|
||||||
|
},
|
||||||
|
&cli.DurationFlag{
|
||||||
|
Name: "pool-stream-timeout",
|
||||||
|
Usage: `Timeout for individual operation in streaming RPC`,
|
||||||
|
Required: false,
|
||||||
|
Destination: &poolStreamTimeoutFlag,
|
||||||
|
Value: poolStreamTimeout,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
Action: func(c *cli.Context) error {
|
Action: func(c *cli.Context) error {
|
||||||
ctx, log := prepare()
|
ctx, log := prepare()
|
||||||
|
@ -252,18 +298,27 @@ It will be ceil rounded to the nearest amount of epoch.`,
|
||||||
password := wallet.GetPassword(viper.GetViper(), envWalletPassphrase)
|
password := wallet.GetPassword(viper.GetViper(), envWalletPassphrase)
|
||||||
key, err := wallet.GetKeyFromPath(walletPathFlag, accountAddressFlag, password)
|
key, err := wallet.GetKeyFromPath(walletPathFlag, accountAddressFlag, password)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cli.Exit(fmt.Sprintf("failed to load neofs private key: %s", err), 1)
|
return cli.Exit(fmt.Sprintf("failed to load frostfs private key: %s", err), 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
neoFS, err := createNeoFS(ctx, log, &key.PrivateKey, peerAddressFlag)
|
poolCfg := PoolConfig{
|
||||||
if err != nil {
|
Key: &key.PrivateKey,
|
||||||
return cli.Exit(fmt.Sprintf("failed to create NeoFS component: %s", err), 2)
|
Address: peerAddressFlag,
|
||||||
|
DialTimeout: poolDialTimeoutFlag,
|
||||||
|
HealthcheckTimeout: poolHealthcheckTimeoutFlag,
|
||||||
|
StreamTimeout: poolStreamTimeoutFlag,
|
||||||
|
RebalanceInterval: poolRebalanceIntervalFlag,
|
||||||
}
|
}
|
||||||
|
|
||||||
agent := authmate.New(log, neoFS)
|
frostFS, err := createFrostFS(ctx, log, poolCfg)
|
||||||
|
if err != nil {
|
||||||
|
return cli.Exit(fmt.Sprintf("failed to create FrostFS component: %s", err), 2)
|
||||||
|
}
|
||||||
|
|
||||||
|
agent := authmate.New(log, frostFS)
|
||||||
|
|
||||||
var containerID cid.ID
|
var containerID cid.ID
|
||||||
if len(containerIDFlag) > 0 {
|
if len(containerIDFlag) > 0 {
|
||||||
|
@ -306,7 +361,7 @@ It will be ceil rounded to the nearest amount of epoch.`,
|
||||||
FriendlyName: containerFriendlyName,
|
FriendlyName: containerFriendlyName,
|
||||||
PlacementPolicy: containerPlacementPolicy,
|
PlacementPolicy: containerPlacementPolicy,
|
||||||
},
|
},
|
||||||
NeoFSKey: key,
|
FrostFSKey: key,
|
||||||
GatesPublicKeys: gatesPublicKeys,
|
GatesPublicKeys: gatesPublicKeys,
|
||||||
EACLRules: bearerRules,
|
EACLRules: bearerRules,
|
||||||
SessionTokenRules: sessionRules,
|
SessionTokenRules: sessionRules,
|
||||||
|
@ -335,7 +390,7 @@ func generatePresignedURL() *cli.Command {
|
||||||
You provide profile to load using --profile flag or explicitly provide credentials and region using
|
You provide profile to load using --profile flag or explicitly provide credentials and region using
|
||||||
--aws-access-key-id, --aws-secret-access-key, --region.
|
--aws-access-key-id, --aws-secret-access-key, --region.
|
||||||
Note to override credentials you must provide both access key and secret key.`,
|
Note to override credentials you must provide both access key and secret key.`,
|
||||||
Usage: "generate-presigned-url --endpoint http://s3.neofs.devenv:8080 --bucket bucket-name --object object-name --method get --profile aws-profile",
|
Usage: "generate-presigned-url --endpoint http://s3.frostfs.devenv:8080 --bucket bucket-name --object object-name --method get --profile aws-profile",
|
||||||
Flags: []cli.Flag{
|
Flags: []cli.Flag{
|
||||||
&cli.DurationFlag{
|
&cli.DurationFlag{
|
||||||
Name: "lifetime",
|
Name: "lifetime",
|
||||||
|
@ -499,7 +554,7 @@ func getSessionRules(r string) ([]byte, bool, error) {
|
||||||
func obtainSecret() *cli.Command {
|
func obtainSecret() *cli.Command {
|
||||||
command := &cli.Command{
|
command := &cli.Command{
|
||||||
Name: "obtain-secret",
|
Name: "obtain-secret",
|
||||||
Usage: "Obtain a secret from NeoFS network",
|
Usage: "Obtain a secret from FrostFS network",
|
||||||
Flags: []cli.Flag{
|
Flags: []cli.Flag{
|
||||||
&cli.StringFlag{
|
&cli.StringFlag{
|
||||||
Name: "wallet",
|
Name: "wallet",
|
||||||
|
@ -518,7 +573,7 @@ func obtainSecret() *cli.Command {
|
||||||
&cli.StringFlag{
|
&cli.StringFlag{
|
||||||
Name: "peer",
|
Name: "peer",
|
||||||
Value: "",
|
Value: "",
|
||||||
Usage: "address of neofs peer to connect to",
|
Usage: "address of frostfs peer to connect to",
|
||||||
Required: true,
|
Required: true,
|
||||||
Destination: &peerAddressFlag,
|
Destination: &peerAddressFlag,
|
||||||
},
|
},
|
||||||
|
@ -542,6 +597,34 @@ func obtainSecret() *cli.Command {
|
||||||
Required: true,
|
Required: true,
|
||||||
Destination: &accessKeyIDFlag,
|
Destination: &accessKeyIDFlag,
|
||||||
},
|
},
|
||||||
|
&cli.DurationFlag{
|
||||||
|
Name: "pool-dial-timeout",
|
||||||
|
Usage: `Timeout for connection to the node in pool to be established`,
|
||||||
|
Required: false,
|
||||||
|
Destination: &poolDialTimeoutFlag,
|
||||||
|
Value: poolDialTimeout,
|
||||||
|
},
|
||||||
|
&cli.DurationFlag{
|
||||||
|
Name: "pool-healthcheck-timeout",
|
||||||
|
Usage: `Timeout for request to node to decide if it is alive`,
|
||||||
|
Required: false,
|
||||||
|
Destination: &poolHealthcheckTimeoutFlag,
|
||||||
|
Value: poolHealthcheckTimeout,
|
||||||
|
},
|
||||||
|
&cli.DurationFlag{
|
||||||
|
Name: "pool-rebalance-interval",
|
||||||
|
Usage: `Interval for updating nodes health status`,
|
||||||
|
Required: false,
|
||||||
|
Destination: &poolRebalanceIntervalFlag,
|
||||||
|
Value: poolRebalanceInterval,
|
||||||
|
},
|
||||||
|
&cli.DurationFlag{
|
||||||
|
Name: "pool-stream-timeout",
|
||||||
|
Usage: `Timeout for individual operation in streaming RPC`,
|
||||||
|
Required: false,
|
||||||
|
Destination: &poolStreamTimeoutFlag,
|
||||||
|
Value: poolStreamTimeout,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
Action: func(c *cli.Context) error {
|
Action: func(c *cli.Context) error {
|
||||||
ctx, log := prepare()
|
ctx, log := prepare()
|
||||||
|
@ -549,18 +632,27 @@ func obtainSecret() *cli.Command {
|
||||||
password := wallet.GetPassword(viper.GetViper(), envWalletPassphrase)
|
password := wallet.GetPassword(viper.GetViper(), envWalletPassphrase)
|
||||||
key, err := wallet.GetKeyFromPath(walletPathFlag, accountAddressFlag, password)
|
key, err := wallet.GetKeyFromPath(walletPathFlag, accountAddressFlag, password)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cli.Exit(fmt.Sprintf("failed to load neofs private key: %s", err), 1)
|
return cli.Exit(fmt.Sprintf("failed to load frostfs private key: %s", err), 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
neoFS, err := createNeoFS(ctx, log, &key.PrivateKey, peerAddressFlag)
|
poolCfg := PoolConfig{
|
||||||
if err != nil {
|
Key: &key.PrivateKey,
|
||||||
return cli.Exit(fmt.Sprintf("failed to create NeoFS component: %s", err), 2)
|
Address: peerAddressFlag,
|
||||||
|
DialTimeout: poolDialTimeoutFlag,
|
||||||
|
HealthcheckTimeout: poolHealthcheckTimeoutFlag,
|
||||||
|
StreamTimeout: poolStreamTimeoutFlag,
|
||||||
|
RebalanceInterval: poolRebalanceIntervalFlag,
|
||||||
}
|
}
|
||||||
|
|
||||||
agent := authmate.New(log, neoFS)
|
frostFS, err := createFrostFS(ctx, log, poolCfg)
|
||||||
|
if err != nil {
|
||||||
|
return cli.Exit(fmt.Sprintf("failed to create FrostFS component: %s", err), 2)
|
||||||
|
}
|
||||||
|
|
||||||
|
agent := authmate.New(log, frostFS)
|
||||||
|
|
||||||
var _ = agent
|
var _ = agent
|
||||||
|
|
||||||
|
@ -591,14 +683,16 @@ func obtainSecret() *cli.Command {
|
||||||
return command
|
return command
|
||||||
}
|
}
|
||||||
|
|
||||||
func createNeoFS(ctx context.Context, log *zap.Logger, key *ecdsa.PrivateKey, peerAddress string) (authmate.NeoFS, error) {
|
func createFrostFS(ctx context.Context, log *zap.Logger, cfg PoolConfig) (authmate.FrostFS, error) {
|
||||||
log.Debug("prepare connection pool")
|
log.Debug("prepare connection pool")
|
||||||
|
|
||||||
var prm pool.InitParameters
|
var prm pool.InitParameters
|
||||||
prm.SetKey(key)
|
prm.SetKey(cfg.Key)
|
||||||
prm.SetNodeDialTimeout(poolConnectTimeout)
|
prm.SetNodeDialTimeout(cfg.DialTimeout)
|
||||||
prm.SetHealthcheckTimeout(poolRequestTimeout)
|
prm.SetHealthcheckTimeout(cfg.HealthcheckTimeout)
|
||||||
prm.AddNode(pool.NewNodeParam(1, peerAddress, 1))
|
prm.SetNodeStreamTimeout(cfg.StreamTimeout)
|
||||||
|
prm.SetClientRebalanceInterval(cfg.RebalanceInterval)
|
||||||
|
prm.AddNode(pool.NewNodeParam(1, cfg.Address, 1))
|
||||||
|
|
||||||
p, err := pool.NewPool(prm)
|
p, err := pool.NewPool(prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -609,5 +703,5 @@ func createNeoFS(ctx context.Context, log *zap.Logger, key *ecdsa.PrivateKey, pe
|
||||||
return nil, fmt.Errorf("dial pool: %w", err)
|
return nil, fmt.Errorf("dial pool: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return neofs.NewAuthmateNeoFS(p), nil
|
return frostfs.NewAuthmateFrostFS(p), nil
|
||||||
}
|
}
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue