forked from TrueCloudLab/frostfs-s3-gw
Compare commits
75 commits
support/v0
...
master
Author | SHA1 | Date | |
---|---|---|---|
fb90c0f52c | |||
f2f90e260e | |||
a0937126cb | |||
655889a1a2 | |||
ef556bd8ac | |||
5104683f68 | |||
8151753eeb | |||
2282c32822 | |||
43685e03d9 | |||
cf18158da4 | |||
5c62010331 | |||
0af06c3bd9 | |||
680c0dbe3d | |||
596381c382 | |||
64e7356acc | |||
32bf915502 | |||
813aa2f173 | |||
6eb7966800 | |||
2dcb3c283d | |||
740acadd37 | |||
3ab77c8990 | |||
d00163aadc | |||
aadefd98b6 | |||
8ac630ee71 | |||
|
744b52322d | ||
8d9d1f9235 | |||
787d1a347a | |||
9f823bd65a | |||
9dcacc230e | |||
430f1e734f | |||
1ce8b8a30d | |||
b35f146cec | |||
5ee4bf80ae | |||
f9f52ce8e0 | |||
e278ab9362 | |||
c5570e661d | |||
fc5c09c084 | |||
86e881694d | |||
361d1d3881 | |||
9ad7982807 | |||
a0d5b18184 | |||
533b12d8bb | |||
cafe079072 | |||
9473335234 | |||
19d8f8fcfe | |||
db603a9703 | |||
b2148cc36b | |||
ff040f6785 | |||
8e14ccad9f | |||
af7fbd6f32 | |||
|
a68aca764b | ||
|
f1696c4725 | ||
000d9ed42a | |||
388482e230 | |||
c2567fcc73 | |||
f744c672cb | |||
765fd28a49 | |||
09c6e22b84 | |||
96dff367db | |||
dd25331210 | |||
|
d6424ebeac | ||
|
5265afe213 | ||
|
dd4f66712c | ||
|
556374e3b0 | ||
|
d2587b21af | ||
|
f5fe9a9b4b | ||
|
d1825f97fb | ||
|
8f1bbb7dc1 | ||
|
1c0f4d6df8 | ||
|
094eb12578 | ||
|
d3702f86d1 | ||
|
d47840f137 | ||
|
3212805955 | ||
|
2886ac161c | ||
7b0492c468 |
154 changed files with 4480 additions and 1937 deletions
|
@ -1,7 +1,7 @@
|
|||
FROM golang:1.19 as builder
|
||||
|
||||
ARG BUILD=now
|
||||
ARG REPO=github.com/nspcc-dev/neofs-s3-gw
|
||||
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
|
||||
ARG VERSION=dev
|
||||
|
||||
WORKDIR /src
|
||||
|
@ -10,12 +10,12 @@ COPY . /src
|
|||
RUN make
|
||||
|
||||
# Executable image
|
||||
FROM alpine AS neofs-s3-gw
|
||||
FROM alpine AS frostfs-s3-gw
|
||||
RUN apk add --no-cache bash ca-certificates
|
||||
|
||||
WORKDIR /
|
||||
|
||||
COPY --from=builder /src/bin/neofs-s3-gw /bin/neofs-s3-gw
|
||||
COPY --from=builder /src/bin/neofs-s3-authmate /bin/neofs-s3-authmate
|
||||
COPY --from=builder /src/bin/frostfs-s3-gw /bin/frostfs-s3-gw
|
||||
COPY --from=builder /src/bin/frostfs-s3-authmate /bin/frostfs-s3-authmate
|
||||
|
||||
ENTRYPOINT ["/bin/neofs-s3-gw"]
|
||||
ENTRYPOINT ["/bin/frostfs-s3-gw"]
|
||||
|
|
|
@ -1,9 +1,9 @@
|
|||
FROM alpine AS neofs-s3-gw
|
||||
FROM alpine AS frostfs-s3-gw
|
||||
RUN apk add --no-cache bash ca-certificates
|
||||
|
||||
WORKDIR /
|
||||
|
||||
COPY /bin/neofs-s3-gw /bin/neofs-s3-gw
|
||||
COPY /bin/neofs-s3-authmate /bin/neofs-s3-authmate
|
||||
COPY /bin/frostfs-s3-gw /bin/frostfs-s3-gw
|
||||
COPY /bin/frostfs-s3-authmate /bin/frostfs-s3-authmate
|
||||
|
||||
ENTRYPOINT ["/bin/neofs-s3-gw"]
|
||||
ENTRYPOINT ["/bin/frostfs-s3-gw"]
|
||||
|
|
2
.github/CODEOWNERS
vendored
2
.github/CODEOWNERS
vendored
|
@ -1 +1 @@
|
|||
* @alexvanin @masterSplinter01 @KirillovDenis
|
||||
* @alexvanin @KirillovDenis
|
||||
|
|
45
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
45
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
|
@ -0,0 +1,45 @@
|
|||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: community, triage, bug
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
<!--- Provide a general summary of the issue in the Title above -->
|
||||
|
||||
## Expected Behavior
|
||||
<!--- If you're describing a bug, tell us what should happen -->
|
||||
<!--- If you're suggesting a change/improvement, tell us how it should work -->
|
||||
|
||||
## Current Behavior
|
||||
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
|
||||
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
|
||||
|
||||
## Possible Solution
|
||||
<!--- Not obligatory -->
|
||||
<!--- If no reason/fix/additions for the bug can be suggested, -->
|
||||
<!--- uncomment the following phrase: -->
|
||||
|
||||
<!--- No fix can be suggested by a QA engineer. Further solutions shall be up to developers. -->
|
||||
|
||||
## Steps to Reproduce (for bugs)
|
||||
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
||||
<!--- reproduce this bug. -->
|
||||
|
||||
1.
|
||||
|
||||
## Context
|
||||
<!--- How has this issue affected you? What are you trying to accomplish? -->
|
||||
<!--- Providing context helps us come up with a solution that is most useful in the real world -->
|
||||
|
||||
## Regression
|
||||
<!-- Is this issue a regression? (Yes / No) -->
|
||||
<!-- If Yes, optionally please include version or commit id or PR# that caused this regression, if you have these details. -->
|
||||
|
||||
## Your Environment
|
||||
<!--- Include as many relevant details about the environment you experienced the bug in -->
|
||||
* Version used:
|
||||
* Server setup and configuration:
|
||||
* Operating System and version (`uname -a`):
|
1
.github/ISSUE_TEMPLATE/config.yml
vendored
Normal file
1
.github/ISSUE_TEMPLATE/config.yml
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
blank_issues_enabled: false
|
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
Normal file
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
Normal file
|
@ -0,0 +1,20 @@
|
|||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: community, triage
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## Is your feature request related to a problem? Please describe.
|
||||
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
|
||||
|
||||
## Describe the solution you'd like
|
||||
<!--- A clear and concise description of what you want to happen. -->
|
||||
|
||||
## Describe alternatives you've considered
|
||||
<!--- A clear and concise description of any alternative solutions or features you've considered. -->
|
||||
|
||||
## Additional context
|
||||
<!--- Add any other context or screenshots about the feature request here. -->
|
70
.github/logo.svg
vendored
Normal file
70
.github/logo.svg
vendored
Normal file
|
@ -0,0 +1,70 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Generator: Adobe Illustrator 25.0.1, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||
<svg version="1.1" id="Слой_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||
viewBox="0 0 184.2 51.8" style="enable-background:new 0 0 184.2 51.8;" xml:space="preserve">
|
||||
<style type="text/css">
|
||||
.st0{display:none;}
|
||||
.st1{display:inline;}
|
||||
.st2{fill:#01E397;}
|
||||
.st3{display:inline;fill:#010032;}
|
||||
.st4{display:inline;fill:#00E599;}
|
||||
.st5{display:inline;fill:#00AF92;}
|
||||
.st6{fill:#00C3E5;}
|
||||
</style>
|
||||
<g id="Layer_2">
|
||||
<g id="Layer_1-2" class="st0">
|
||||
<g class="st1">
|
||||
<path class="st2" d="M146.6,18.3v7.2h10.9V29h-10.9v10.7h-4V14.8h18v3.5H146.6z"/>
|
||||
<path class="st2" d="M180,15.7c1.7,0.9,3,2.2,4,3.8l-3,2.7c-0.6-1.3-1.5-2.4-2.6-3.3c-1.3-0.7-2.8-1-4.3-1
|
||||
c-1.4-0.1-2.8,0.3-4,1.1c-0.9,0.5-1.5,1.5-1.4,2.6c0,1,0.5,1.9,1.4,2.4c1.5,0.8,3.2,1.3,4.9,1.5c1.9,0.3,3.7,0.8,5.4,1.6
|
||||
c1.2,0.5,2.2,1.3,2.9,2.3c0.6,1,1,2.2,0.9,3.4c0,1.4-0.5,2.7-1.3,3.8c-0.9,1.2-2.1,2.1-3.5,2.6c-1.7,0.6-3.4,0.9-5.2,0.8
|
||||
c-5,0-8.6-1.6-10.7-5l2.9-2.8c0.7,1.4,1.8,2.5,3.1,3.3c1.5,0.7,3.1,1.1,4.7,1c1.5,0.1,2.9-0.2,4.2-0.9c0.9-0.5,1.5-1.5,1.5-2.6
|
||||
c0-0.9-0.5-1.8-1.3-2.2c-1.5-0.7-3.1-1.2-4.8-1.5c-1.9-0.3-3.7-0.8-5.5-1.5c-1.2-0.5-2.2-1.4-3-2.4c-0.6-1-1-2.2-0.9-3.4
|
||||
c0-1.4,0.4-2.7,1.2-3.8c0.8-1.2,2-2.2,3.3-2.8c1.6-0.7,3.4-1.1,5.2-1C176.1,14.3,178.2,14.8,180,15.7z"/>
|
||||
</g>
|
||||
<path class="st3" d="M73.3,16.3c1.9,1.9,2.9,4.5,2.7,7.1v15.9h-4V24.8c0-2.6-0.5-4.5-1.6-5.7c-1.2-1.2-2.8-1.8-4.5-1.7
|
||||
c-1.3,0-2.5,0.3-3.7,0.8c-1.2,0.7-2.2,1.7-2.9,2.9c-0.8,1.5-1.1,3.2-1.1,4.9v13.3h-4V15.1l3.6,1.5v1.7c0.8-1.5,2.1-2.6,3.6-3.3
|
||||
c1.5-0.8,3.2-1.2,4.9-1.1C68.9,13.8,71.3,14.7,73.3,16.3z"/>
|
||||
<path class="st3" d="M104.4,28.3H85.6c0.1,2.2,1,4.3,2.5,5.9c1.5,1.4,3.5,2.2,5.6,2.1c1.6,0.1,3.2-0.2,4.6-0.9
|
||||
c1.1-0.6,2-1.6,2.5-2.8l3.3,1.8c-0.9,1.7-2.3,3.1-4,4c-2,1-4.2,1.5-6.4,1.4c-3.7,0-6.7-1.1-8.8-3.4s-3.2-5.5-3.2-9.6s1-7.2,3-9.5
|
||||
s5-3.4,8.7-3.4c2.1-0.1,4.2,0.5,6.1,1.5c1.6,1,3,2.5,3.8,4.2c0.9,1.8,1.3,3.9,1.3,5.9C104.6,26.4,104.6,27.4,104.4,28.3z
|
||||
M88.1,19.3c-1.4,1.5-2.2,3.4-2.4,5.5h15.1c-0.2-2-1-3.9-2.3-5.5c-1.4-1.3-3.2-2-5.1-1.9C91.5,17.3,89.6,18,88.1,19.3z"/>
|
||||
<path class="st3" d="M131,17.3c2.2,2.3,3.2,5.5,3.2,9.5s-1,7.3-3.2,9.6s-5.1,3.4-8.8,3.4s-6.7-1.1-8.9-3.4s-3.2-5.5-3.2-9.6
|
||||
s1.1-7.2,3.2-9.5s5.1-3.4,8.9-3.4S128.9,15,131,17.3z M116.2,19.9c-1.5,2-2.2,4.4-2.1,6.9c-0.2,2.5,0.6,5,2.1,7
|
||||
c1.5,1.7,3.7,2.7,6,2.6c2.3,0.1,4.4-0.9,5.9-2.6c1.5-2,2.3-4.5,2.1-7c0.1-2.5-0.6-4.9-2.1-6.9c-1.5-1.7-3.6-2.7-5.9-2.6
|
||||
C119.9,17.2,117.7,18.2,116.2,19.9z"/>
|
||||
<polygon class="st4" points="0,9.1 0,43.7 22.5,51.8 22.5,16.9 46.8,7.9 24.8,0 "/>
|
||||
<polygon class="st5" points="24.3,17.9 24.3,36.8 46.8,44.9 46.8,9.6 "/>
|
||||
</g>
|
||||
<g>
|
||||
<g>
|
||||
<path class="st6" d="M41.6,17.5H28.2v6.9h10.4v3.3H28.2v10.2h-3.9V14.2h17.2V17.5z"/>
|
||||
<path class="st6" d="M45.8,37.9v-18h3.3l0.4,3.2c0.5-1.2,1.2-2.1,2.1-2.7c0.9-0.6,2.1-0.9,3.5-0.9c0.4,0,0.7,0,1.1,0.1
|
||||
c0.4,0.1,0.7,0.2,0.9,0.3l-0.5,3.4c-0.3-0.1-0.6-0.2-0.9-0.2C55.4,23,54.9,23,54.4,23c-0.7,0-1.5,0.2-2.2,0.6
|
||||
c-0.7,0.4-1.3,1-1.8,1.8s-0.7,1.8-0.7,3v9.5H45.8z"/>
|
||||
<path class="st6" d="M68.6,19.6c1.8,0,3.3,0.4,4.6,1.1c1.3,0.7,2.4,1.8,3.1,3.2s1.1,3.1,1.1,5c0,1.9-0.4,3.6-1.1,5
|
||||
c-0.8,1.4-1.8,2.5-3.1,3.2c-1.3,0.7-2.9,1.1-4.6,1.1s-3.3-0.4-4.6-1.1c-1.3-0.7-2.4-1.8-3.2-3.2c-0.8-1.4-1.2-3.1-1.2-5
|
||||
c0-1.9,0.4-3.6,1.2-5s1.8-2.5,3.2-3.2C65.3,19.9,66.8,19.6,68.6,19.6z M68.6,22.6c-1.1,0-2,0.2-2.8,0.7c-0.8,0.5-1.3,1.2-1.7,2.1
|
||||
s-0.6,2.1-0.6,3.5c0,1.3,0.2,2.5,0.6,3.4s1,1.7,1.7,2.2s1.7,0.7,2.8,0.7c1.1,0,2-0.2,2.7-0.7c0.7-0.5,1.3-1.2,1.7-2.2
|
||||
s0.6-2.1,0.6-3.4c0-1.4-0.2-2.5-0.6-3.5s-1-1.6-1.7-2.1C70.6,22.8,69.6,22.6,68.6,22.6z"/>
|
||||
<path class="st6" d="M89.2,38.3c-1.8,0-3.4-0.3-4.9-1c-1.5-0.7-2.7-1.7-3.5-3l2.7-2.3c0.5,1,1.3,1.8,2.3,2.4
|
||||
c1,0.6,2.2,0.9,3.6,0.9c1.1,0,2-0.2,2.6-0.6c0.6-0.4,1-0.9,1-1.6c0-0.5-0.2-0.9-0.5-1.2s-0.9-0.6-1.7-0.8l-3.8-0.8
|
||||
c-1.9-0.4-3.3-1-4.1-1.9c-0.8-0.9-1.2-1.9-1.2-3.3c0-1,0.3-1.9,0.9-2.7c0.6-0.8,1.4-1.5,2.5-2s2.5-0.8,4-0.8c1.8,0,3.3,0.3,4.6,1
|
||||
c1.3,0.6,2.2,1.5,2.9,2.7l-2.7,2.2c-0.5-1-1.1-1.7-2-2.1c-0.9-0.5-1.8-0.7-2.8-0.7c-0.8,0-1.4,0.1-2,0.3c-0.6,0.2-1,0.5-1.3,0.8
|
||||
c-0.3,0.3-0.4,0.7-0.4,1.2c0,0.5,0.2,0.9,0.5,1.3s1,0.6,1.9,0.8l4.1,0.9c1.7,0.3,2.9,0.9,3.7,1.7c0.7,0.8,1.1,1.8,1.1,2.9
|
||||
c0,1.2-0.3,2.2-0.9,3c-0.6,0.9-1.5,1.6-2.6,2C92.1,38.1,90.7,38.3,89.2,38.3z"/>
|
||||
<path class="st6" d="M112.8,19.9v3H99.3v-3H112.8z M106.6,14.6v17.9c0,0.9,0.2,1.5,0.7,1.9c0.5,0.4,1.1,0.6,1.9,0.6
|
||||
c0.6,0,1.2-0.1,1.7-0.3c0.5-0.2,0.9-0.5,1.3-0.8l0.9,2.8c-0.6,0.5-1.2,0.9-2,1.1c-0.8,0.3-1.7,0.4-2.7,0.4c-1,0-2-0.2-2.8-0.5
|
||||
s-1.5-0.9-2-1.6c-0.5-0.8-0.7-1.7-0.8-3V15.7L106.6,14.6z"/>
|
||||
<path d="M137.9,17.5h-13.3v6.9h10.4v3.3h-10.4v10.2h-3.9V14.2h17.2V17.5z"/>
|
||||
<path d="M150.9,13.8c2.1,0,4,0.4,5.5,1.2c1.6,0.8,2.9,2,4,3.5l-2.6,2.5c-0.9-1.4-1.9-2.4-3.1-3c-1.1-0.6-2.5-0.9-4-0.9
|
||||
c-1.2,0-2.1,0.2-2.8,0.5c-0.7,0.3-1.3,0.7-1.6,1.2c-0.3,0.5-0.5,1.1-0.5,1.7c0,0.7,0.3,1.4,0.8,1.9c0.5,0.6,1.5,1,2.9,1.3
|
||||
l4.8,1.1c2.3,0.5,3.9,1.3,4.9,2.3c1,1,1.4,2.3,1.4,3.9c0,1.5-0.4,2.7-1.2,3.8c-0.8,1.1-1.9,1.9-3.3,2.5s-3.1,0.9-5,0.9
|
||||
c-1.7,0-3.2-0.2-4.5-0.6c-1.3-0.4-2.5-1-3.5-1.8c-1-0.7-1.8-1.6-2.5-2.6l2.7-2.7c0.5,0.8,1.1,1.6,1.9,2.2
|
||||
c0.8,0.7,1.7,1.2,2.7,1.5c1,0.4,2.2,0.5,3.4,0.5c1.1,0,2.1-0.1,2.9-0.4c0.8-0.3,1.4-0.7,1.8-1.2c0.4-0.5,0.6-1.1,0.6-1.9
|
||||
c0-0.7-0.2-1.3-0.7-1.8c-0.5-0.5-1.3-0.9-2.6-1.2l-5.2-1.2c-1.4-0.3-2.6-0.8-3.6-1.3c-0.9-0.6-1.6-1.3-2.1-2.1s-0.7-1.8-0.7-2.8
|
||||
c0-1.3,0.4-2.6,1.1-3.7c0.7-1.1,1.8-2,3.2-2.6C147.3,14.1,148.9,13.8,150.9,13.8z"/>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
After Width: | Height: | Size: 5.5 KiB |
1
.github/workflows/builds.yml
vendored
1
.github/workflows/builds.yml
vendored
|
@ -4,6 +4,7 @@ on:
|
|||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- 'support/*'
|
||||
types: [ opened, synchronize ]
|
||||
paths-ignore:
|
||||
- '**/*.md'
|
||||
|
|
4
.github/workflows/codeql-analysis.yml
vendored
4
.github/workflows/codeql-analysis.yml
vendored
|
@ -13,10 +13,10 @@ name: "CodeQL"
|
|||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
branches: [ master, 'support/*' ]
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: [ master ]
|
||||
branches: [ master, 'support/*' ]
|
||||
schedule:
|
||||
- cron: '35 8 * * 1'
|
||||
|
||||
|
|
1
.github/workflows/dco.yml
vendored
1
.github/workflows/dco.yml
vendored
|
@ -4,6 +4,7 @@ on:
|
|||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- 'support/*'
|
||||
|
||||
jobs:
|
||||
commits_check_job:
|
||||
|
|
3
.github/workflows/tests.yml
vendored
3
.github/workflows/tests.yml
vendored
|
@ -4,6 +4,7 @@ on:
|
|||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- 'support/*'
|
||||
types: [opened, synchronize]
|
||||
paths-ignore:
|
||||
- '**/*.md'
|
||||
|
@ -67,7 +68,7 @@ jobs:
|
|||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
go_versions: [ '1.17', '1.18.x', '1.19.x' ]
|
||||
go_versions: [ '1.18.x', '1.19.x' ]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
12
.gitignore
vendored
12
.gitignore
vendored
|
@ -3,7 +3,7 @@
|
|||
.vscode
|
||||
|
||||
# Tree service
|
||||
internal/neofs/services/tree/
|
||||
internal/frostfs/services/tree/
|
||||
|
||||
# Vendoring
|
||||
vendor
|
||||
|
@ -19,3 +19,13 @@ bin/
|
|||
# coverage
|
||||
coverage.txt
|
||||
coverage.html
|
||||
|
||||
# debhelpers
|
||||
**/*debhelper*
|
||||
|
||||
# debian package build files
|
||||
debian/files
|
||||
debian/*.log
|
||||
debian/*.substvars
|
||||
debian/frostfs-s3-gw/
|
||||
|
||||
|
|
11
.gitlint
Normal file
11
.gitlint
Normal file
|
@ -0,0 +1,11 @@
|
|||
[general]
|
||||
fail-without-commits=True
|
||||
regex-style-search=True
|
||||
contrib=CC1
|
||||
|
||||
[title-match-regex]
|
||||
regex=^\[\#[0-9Xx]+\]\s
|
||||
|
||||
[ignore-by-title]
|
||||
regex=^Release(.*)
|
||||
ignore=title-match-regex
|
|
@ -32,15 +32,12 @@ linters:
|
|||
- revive
|
||||
|
||||
# some default golangci-lint linters
|
||||
- deadcode
|
||||
- errcheck
|
||||
- gosimple
|
||||
- ineffassign
|
||||
- staticcheck
|
||||
- structcheck
|
||||
- typecheck
|
||||
- unused
|
||||
- varcheck
|
||||
|
||||
# extra linters
|
||||
- exhaustive
|
||||
|
|
45
.pre-commit-config.yaml
Normal file
45
.pre-commit-config.yaml
Normal file
|
@ -0,0 +1,45 @@
|
|||
ci:
|
||||
autofix_prs: false
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/jorisroovers/gitlint
|
||||
rev: v0.19.1
|
||||
hooks:
|
||||
- id: gitlint
|
||||
stages: [commit-msg]
|
||||
- id: gitlint-ci
|
||||
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.4.0
|
||||
hooks:
|
||||
- id: check-added-large-files
|
||||
- id: check-case-conflict
|
||||
- id: check-executables-have-shebangs
|
||||
- id: check-shebang-scripts-are-executable
|
||||
- id: check-merge-conflict
|
||||
- id: check-json
|
||||
- id: check-xml
|
||||
- id: check-yaml
|
||||
- id: trailing-whitespace
|
||||
args: [--markdown-linebreak-ext=md]
|
||||
- id: end-of-file-fixer
|
||||
exclude: ".key$"
|
||||
|
||||
- repo: https://github.com/shellcheck-py/shellcheck-py
|
||||
rev: v0.9.0.2
|
||||
hooks:
|
||||
- id: shellcheck
|
||||
|
||||
- repo: https://github.com/golangci/golangci-lint
|
||||
rev: v1.51.2
|
||||
hooks:
|
||||
- id: golangci-lint
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: go-unit-tests
|
||||
name: go unit tests
|
||||
entry: make test
|
||||
pass_filenames: false
|
||||
types: [go]
|
||||
language: system
|
71
CHANGELOG.md
71
CHANGELOG.md
|
@ -4,6 +4,77 @@ This document outlines major changes between releases.
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
- Get empty bucket CORS from frostfs (TrueCloudLab#36)
|
||||
- Don't count pool error on client abort (#35)
|
||||
|
||||
### Added
|
||||
- Return container name in `head-bucket` response (TrueCloudLab#18)
|
||||
- Billing metrics (TrueCloudLab#5)
|
||||
- Multiple configs support (TrueCloudLab#21)
|
||||
- Bucket name resolving policy (TrueCloudLab#25)
|
||||
- Support string `Action` and `Resource` fields in `bucketPolicy.Statement` (TrueCloudLab#32)
|
||||
- Add new `kludge.use_default_xmlns_for_complete_multipart` config param (TrueCloudLab#40)
|
||||
|
||||
### Changed
|
||||
- Update neo-go to v0.101.0 (#14)
|
||||
- Update viper to v1.15.0 (#14)
|
||||
- Using multiple servers require only one healthy (TrueCloudLab#12)
|
||||
- Update go version to go1.18 (TrueCloudLab#16)
|
||||
- Return error on invalid LocationConstraint (TrueCloudLab#23)
|
||||
- Place billing metrics to separate url path (TrueCloudLab#26)
|
||||
- Add generated deb builder files to .gitignore, and fix typo (TrueCloudLab#28)
|
||||
- Limit number of objects to delete at one time (TrueCloudLab#37)
|
||||
- CompleteMultipartUpload handler now sends whitespace characters to keep alive client's connection (#60)
|
||||
- Support new system attributes (#64)
|
||||
|
||||
## [0.26.0] - 2022-12-28
|
||||
|
||||
### Added
|
||||
- Use client time as `now` in some requests (#726)
|
||||
- Reload policies on SIGHUP (#747)
|
||||
- Authmate flags for pool timeouts (#760)
|
||||
- Multiple server listeners (#742)
|
||||
|
||||
### Changed
|
||||
- Placement policy configuration (#568)
|
||||
- Improved debug logging of CID and OID values (#754)
|
||||
|
||||
### Removed
|
||||
- Deprecated linters (#755)
|
||||
|
||||
### Updating from v0.25.1
|
||||
New config parameters were added. And old one `defaul_policy` were changed.
|
||||
```yaml
|
||||
placement_policy:
|
||||
default: "REP 3"
|
||||
region_mapping: /path/to/container/policies.json
|
||||
```
|
||||
|
||||
Make sure you update the config accordingly:
|
||||
If you configure application using environment variables change:
|
||||
* `S3_GW_DEFAULT_POLICY` -> `S3_GW_PLACEMENT_POLICY_DEFAULT_POLICY`
|
||||
* `S3_GW_LISTEN_ADDRESS` -> `S3_GW_SERVER_0_ADDRESS`
|
||||
* `S3_GW_TLS_CERT_FILE` -> `S3_GW_SERVER_0_TLS_CERT_FILE` (and set `S3_GW_SERVER_0_TLS_ENABLED=true`)
|
||||
* `S3_GW_TLS_KEY_FILE` -> `S3_GW_SERVER_0_TLS_KEY_FILE` (and set `S3_GW_SERVER_0_TLS_ENABLED=true`)
|
||||
|
||||
If you configure application using `.yaml` file change:
|
||||
* `defaul_policy` -> `placement_policy.default`
|
||||
* `listen_address` -> `server.0.address`
|
||||
* `tls.cert_file` -> `server.0.tls.cert_file` (and set `server.0.tls.enabled: true`)
|
||||
* `tls.key_file` -> `server.0.tls.key_file` (and set `server.0.tls.enabled: true`)
|
||||
|
||||
## [0.25.1] - 2022-10-30
|
||||
|
||||
### Fixed
|
||||
- Empty bucket policy (#740)
|
||||
- Big object removal (#749)
|
||||
- Checksum panic (#741)
|
||||
|
||||
### Added
|
||||
- Debian packaging (#737)
|
||||
- Timeout for individual operations in streaming RPC (#750)
|
||||
|
||||
## [0.25.0] - 2022-10-31
|
||||
|
||||
### Fixed
|
||||
|
|
|
@ -3,8 +3,8 @@
|
|||
First, thank you for contributing! We love and encourage pull requests from
|
||||
everyone. Please follow the guidelines:
|
||||
|
||||
- Check the open [issues](https://github.com/nspcc-dev/neofs-s3-gw/issues) and
|
||||
[pull requests](https://github.com/nspcc-dev/neofs-s3-gw/pulls) for existing
|
||||
- Check the open [issues](https://github.com/TrueCloudLab/frostfs-s3-gw/issues) and
|
||||
[pull requests](https://github.com/TrueCloudLab/frostfs-s3-gw/pulls) for existing
|
||||
discussions.
|
||||
|
||||
- Open an issue first, to discuss a new feature or enhancement.
|
||||
|
@ -23,24 +23,24 @@ everyone. Please follow the guidelines:
|
|||
|
||||
## Development Workflow
|
||||
|
||||
Start by forking the `neofs-s3-gw` repository, make changes in a branch and then
|
||||
Start by forking the `frostfs-s3-gw` repository, make changes in a branch and then
|
||||
send a pull request. We encourage pull requests to discuss code changes. Here
|
||||
are the steps in details:
|
||||
|
||||
### Set up your GitHub Repository
|
||||
Fork [NeoFS S3 Gateway
|
||||
upstream](https://github.com/nspcc-dev/neofs-s3-gw/fork) source repository
|
||||
Fork [FrostFS S3 Gateway
|
||||
upstream](https://github.com/TrueCloudLab/frostfs-s3-gw/fork) source repository
|
||||
to your own personal repository. Copy the URL of your fork (you will need it for
|
||||
the `git clone` command below).
|
||||
|
||||
```sh
|
||||
$ git clone https://github.com/nspcc-dev/neofs-s3-gw
|
||||
$ git clone https://github.com/TrueCloudLab/frostfs-s3-gw
|
||||
```
|
||||
|
||||
### Set up git remote as ``upstream``
|
||||
```sh
|
||||
$ cd neofs-s3-gw
|
||||
$ git remote add upstream https://github.com/nspcc-dev/neofs-s3-gw
|
||||
$ cd frostfs-s3-gw
|
||||
$ git remote add upstream https://github.com/TrueCloudLab/frostfs-s3-gw
|
||||
$ git fetch upstream
|
||||
$ git merge upstream/master
|
||||
...
|
||||
|
@ -107,7 +107,7 @@ contributors".
|
|||
To sign your work, just add a line like this at the end of your commit message:
|
||||
|
||||
```
|
||||
Signed-off-by: Samii Sakisaka <samii@nspcc.ru>
|
||||
Signed-off-by: Samii Sakisaka <samii@frostfs.info>
|
||||
```
|
||||
|
||||
This can be easily done with the `--signoff` option to `git commit`.
|
||||
|
|
39
Makefile
Normal file → Executable file
39
Makefile
Normal file → Executable file
|
@ -8,15 +8,22 @@ LINT_VERSION ?= 1.49.0
|
|||
BINDIR = bin
|
||||
|
||||
# Binaries to build
|
||||
CMDS = $(addprefix neofs-, $(notdir $(wildcard cmd/*)))
|
||||
CMDS = $(addprefix frostfs-, $(notdir $(wildcard cmd/*)))
|
||||
BINS = $(addprefix $(BINDIR)/, $(CMDS))
|
||||
|
||||
# Variables for docker
|
||||
REPO_BASENAME = $(shell basename `go list -m`)
|
||||
HUB_IMAGE ?= "nspccdev/$(REPO_BASENAME)"
|
||||
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
|
||||
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
||||
|
||||
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint version clean protoc
|
||||
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
|
||||
|
||||
# .deb package versioning
|
||||
OS_RELEASE = $(shell lsb_release -cs)
|
||||
PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
|
||||
sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \
|
||||
sed "s/-/~/")-${OS_RELEASE}
|
||||
.PHONY: debpackage debclean
|
||||
|
||||
# Make all binaries
|
||||
all: $(BINS)
|
||||
|
@ -26,7 +33,7 @@ $(BINS): sync-tree $(BINDIR) dep
|
|||
CGO_ENABLED=0 \
|
||||
go build -v -trimpath \
|
||||
-ldflags "-X $(REPO)/internal/version.Version=$(VERSION)" \
|
||||
-o $@ ./cmd/$(subst neofs-,,$(notdir $@))
|
||||
-o $@ ./cmd/$(subst frostfs-,,$(notdir $@))
|
||||
|
||||
$(BINDIR):
|
||||
@echo "⇒ Ensure dir: $@"
|
||||
|
@ -73,7 +80,7 @@ format:
|
|||
|
||||
# Build clean Docker image
|
||||
image:
|
||||
@echo "⇒ Build NeoFS S3 Gateway docker image "
|
||||
@echo "⇒ Build FrostFS S3 Gateway docker image "
|
||||
@docker build \
|
||||
--build-arg REPO=$(REPO) \
|
||||
--build-arg VERSION=$(VERSION) \
|
||||
|
@ -88,7 +95,7 @@ image-push:
|
|||
|
||||
# Build dirty Docker image
|
||||
dirty-image:
|
||||
@echo "⇒ Build NeoFS S3 Gateway dirty docker image "
|
||||
@echo "⇒ Build FrostFS S3 Gateway dirty docker image "
|
||||
@docker build \
|
||||
--build-arg REPO=$(REPO) \
|
||||
--build-arg VERSION=$(VERSION) \
|
||||
|
@ -108,6 +115,14 @@ docker/lint:
|
|||
--env HOME=/src \
|
||||
golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint'
|
||||
|
||||
# Activate pre-commit hooks
|
||||
pre-commit:
|
||||
pre-commit install -t pre-commit -t commit-msg
|
||||
|
||||
# Deactivate pre-commit hooks
|
||||
unpre-commit:
|
||||
pre-commit uninstall -t pre-commit -t commit-msg
|
||||
|
||||
# Show current version
|
||||
version:
|
||||
@echo $(VERSION)
|
||||
|
@ -126,4 +141,16 @@ protoc:
|
|||
done
|
||||
rm -rf vendor
|
||||
|
||||
# Package for Debian
|
||||
debpackage:
|
||||
dch --package frostfs-s3-gw \
|
||||
--controlmaint \
|
||||
--newversion $(PKG_VERSION) \
|
||||
--distribution $(OS_RELEASE) \
|
||||
"Please see CHANGELOG.md for code changes for $(VERSION)"
|
||||
dpkg-buildpackage --no-sign -b
|
||||
|
||||
debclean:
|
||||
dh clean
|
||||
|
||||
include help.mk
|
||||
|
|
40
README.md
40
README.md
|
@ -1,13 +1,23 @@
|
|||
# NeoFS S3 Gateway
|
||||
<p align="center">
|
||||
<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="https://frostfs.info">FrostFS</a> is a decentralized distributed object storage integrated with the <a href="https://neo.org">NEO Blockchain</a>.
|
||||
</p>
|
||||
|
||||
NeoFS S3 gateway provides API compatible with Amazon S3 cloud storage service.
|
||||
---
|
||||
[![Report](https://goreportcard.com/badge/git.frostfs.info/TrueCloudLab/frostfs-s3-gw)](https://goreportcard.com/report/git.frostfs.info/TrueCloudLab/frostfs-s3-gw)
|
||||
|
||||
# FrostFS S3 Gateway
|
||||
|
||||
FrostFS S3 gateway provides API compatible with Amazon S3 cloud storage service.
|
||||
|
||||
## Installation
|
||||
|
||||
```go get -u github.com/nspcc-dev/neofs-s3-gw```
|
||||
```go get -u git.frostfs.info/TrueCloudLab/frostfs-s3-gw```
|
||||
|
||||
Or you can call `make` to build it from the cloned repository (the binary will
|
||||
end up in `bin/neofs-s3-gw` with authmate helper in `bin/neofs-s3-authmate`).
|
||||
end up in `bin/frostfs-s3-gw` with authmate helper in `bin/frostfs-s3-authmate`).
|
||||
To build binaries in clean docker environment, call `make docker/all`.
|
||||
|
||||
Other notable make targets:
|
||||
|
@ -22,36 +32,36 @@ version Show current version
|
|||
```
|
||||
|
||||
Or you can also use a [Docker
|
||||
image](https://hub.docker.com/r/nspccdev/neofs-s3-gw) provided for released
|
||||
image](https://hub.docker.com/r/truecloudlab/frostfs-s3-gw) provided for released
|
||||
(and occasionally unreleased) versions of gateway (`:latest` points to the
|
||||
latest stable release).
|
||||
|
||||
## Execution
|
||||
|
||||
Minimalistic S3 gateway setup needs:
|
||||
* NeoFS node(s) address (S3 gateway itself is not a NeoFS node)
|
||||
* FrostFS node(s) address (S3 gateway itself is not a FrostFS node)
|
||||
Passed via `-p` parameter or via `S3_GW_PEERS_<N>_ADDRESS` and
|
||||
`S3_GW_PEERS_<N>_WEIGHT` environment variables (gateway supports multiple
|
||||
NeoFS nodes with weighted load balancing).
|
||||
* a wallet used to fetch key and communicate with NeoFS nodes
|
||||
FrostFS nodes with weighted load balancing).
|
||||
* a wallet used to fetch key and communicate with FrostFS nodes
|
||||
Passed via `--wallet` parameter or `S3_GW_WALLET_PATH` environment variable.
|
||||
|
||||
These two commands are functionally equivalent, they run the gate with one
|
||||
backend node, some keys and otherwise default settings:
|
||||
```
|
||||
$ neofs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json
|
||||
$ frostfs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json
|
||||
|
||||
$ S3_GW_PEERS_0_ADDRESS=192.168.130.72:8080 \
|
||||
S3_GW_WALLET=wallet.json \
|
||||
neofs-s3-gw
|
||||
frostfs-s3-gw
|
||||
```
|
||||
It's also possible to specify uri scheme (grpc or grpcs) when using `-p` or environment variables:
|
||||
```
|
||||
$ neofs-s3-gw -p grpc://192.168.130.72:8080 --wallet wallet.json
|
||||
$ frostfs-s3-gw -p grpc://192.168.130.72:8080 --wallet wallet.json
|
||||
|
||||
$ S3_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 \
|
||||
S3_GW_WALLET=wallet.json \
|
||||
neofs-s3-gw
|
||||
frostfs-s3-gw
|
||||
```
|
||||
|
||||
## Domains
|
||||
|
@ -60,7 +70,7 @@ By default, s3-gw enable only `path-style access`.
|
|||
To be able to use both: `virtual-hosted-style` and `path-style` access you must configure `listen_domains`:
|
||||
|
||||
```shell
|
||||
$ neofs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json --listen_domains your.first.domain --listen_domains your.second.domain
|
||||
$ frostfs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json --listen_domains your.first.domain --listen_domains your.second.domain
|
||||
```
|
||||
|
||||
So now you can use (e.g. `HeadBucket`. Make sure DNS is properly configured):
|
||||
|
@ -84,8 +94,8 @@ Also, you can configure domains using `.env` variables or `yaml` file.
|
|||
## Documentation
|
||||
|
||||
- [Configuration](./docs/configuration.md)
|
||||
- [NeoFS S3 AuthMate](./docs/authmate.md)
|
||||
- [NeoFS Tree service](./docs/tree_service.md)
|
||||
- [FrostFS S3 AuthMate](./docs/authmate.md)
|
||||
- [FrostFS Tree service](./docs/tree_service.md)
|
||||
- [AWS CLI basic usage](./docs/aws_cli.md)
|
||||
- [AWS S3 API compatibility](./docs/aws_s3_compat.md)
|
||||
- [AWS S3 Compatibility test results](./docs/s3_test_results.md)
|
||||
|
|
2
VERSION
2
VERSION
|
@ -1 +1 @@
|
|||
v0.25.0
|
||||
v0.26.0
|
||||
|
|
|
@ -14,14 +14,14 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
v4 "github.com/nspcc-dev/neofs-s3-gw/api/auth/signer/v4"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
|
||||
|
@ -33,7 +33,13 @@ var postPolicyCredentialRegexp = regexp.MustCompile(`(?P<access_key_id>[^/]+)/(?
|
|||
type (
|
||||
// Center is a user authentication interface.
|
||||
Center interface {
|
||||
Authenticate(request *http.Request) (*accessbox.Box, error)
|
||||
Authenticate(request *http.Request) (*Box, error)
|
||||
}
|
||||
|
||||
// Box contains access box and additional info.
|
||||
Box struct {
|
||||
AccessBox *accessbox.Box
|
||||
ClientTime time.Time
|
||||
}
|
||||
|
||||
center struct {
|
||||
|
@ -86,9 +92,9 @@ func (p prs) Seek(_ int64, _ int) (int64, error) {
|
|||
var _ io.ReadSeeker = prs(0)
|
||||
|
||||
// New creates an instance of AuthCenter.
|
||||
func New(neoFS tokens.NeoFS, key *keys.PrivateKey, prefixes []string, config *cache.Config) Center {
|
||||
func New(frostFS tokens.FrostFS, key *keys.PrivateKey, prefixes []string, config *cache.Config) Center {
|
||||
return ¢er{
|
||||
cli: tokens.New(neoFS, key, config),
|
||||
cli: tokens.New(frostFS, key, config),
|
||||
reg: NewRegexpMatcher(authorizationFieldRegexp),
|
||||
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
|
||||
allowedAccessKeyIDPrefixes: prefixes,
|
||||
|
@ -126,11 +132,12 @@ func (a *authHeader) getAddress() (oid.Address, error) {
|
|||
return addr, nil
|
||||
}
|
||||
|
||||
func (c *center) Authenticate(r *http.Request) (*accessbox.Box, error) {
|
||||
func (c *center) Authenticate(r *http.Request) (*Box, error) {
|
||||
var (
|
||||
err error
|
||||
authHdr *authHeader
|
||||
signatureDateTimeStr string
|
||||
needClientTime bool
|
||||
)
|
||||
|
||||
queryValues := r.URL.Query()
|
||||
|
@ -166,6 +173,7 @@ func (c *center) Authenticate(r *http.Request) (*accessbox.Box, error) {
|
|||
return nil, err
|
||||
}
|
||||
signatureDateTimeStr = r.Header.Get(AmzDate)
|
||||
needClientTime = true
|
||||
}
|
||||
|
||||
signatureDateTime, err := time.Parse("20060102T150405Z", signatureDateTimeStr)
|
||||
|
@ -192,7 +200,12 @@ func (c *center) Authenticate(r *http.Request) (*accessbox.Box, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
return box, nil
|
||||
result := &Box{AccessBox: box}
|
||||
if needClientTime {
|
||||
result.ClientTime = signatureDateTime
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c center) checkAccessKeyID(accessKeyID string) error {
|
||||
|
@ -209,7 +222,7 @@ func (c center) checkAccessKeyID(accessKeyID string) error {
|
|||
return apiErrors.GetAPIError(apiErrors.ErrAccessDenied)
|
||||
}
|
||||
|
||||
func (c *center) checkFormData(r *http.Request) (*accessbox.Box, error) {
|
||||
func (c *center) checkFormData(r *http.Request) (*Box, error) {
|
||||
if err := r.ParseMultipartForm(maxFormSizeMemory); err != nil {
|
||||
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidArgument)
|
||||
}
|
||||
|
@ -251,7 +264,7 @@ func (c *center) checkFormData(r *http.Request) (*accessbox.Box, error) {
|
|||
return nil, apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch)
|
||||
}
|
||||
|
||||
return box, nil
|
||||
return &Box{AccessBox: box}, nil
|
||||
}
|
||||
|
||||
func cloneRequest(r *http.Request, authHeader *authHeader) *http.Request {
|
||||
|
|
|
@ -5,7 +5,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
|
2
api/cache/access_control.go
vendored
2
api/cache/access_control.go
vendored
|
@ -4,8 +4,8 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/bluele/gcache"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
|
4
api/cache/accessbox.go
vendored
4
api/cache/accessbox.go
vendored
|
@ -4,9 +4,9 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/bluele/gcache"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
|
2
api/cache/buckets.go
vendored
2
api/cache/buckets.go
vendored
|
@ -4,8 +4,8 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/bluele/gcache"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
|
8
api/cache/cache_test.go
vendored
8
api/cache/cache_test.go
vendored
|
@ -3,10 +3,10 @@ package cache
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
|
||||
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zaptest/observer"
|
||||
|
|
2
api/cache/names.go
vendored
2
api/cache/names.go
vendored
|
@ -4,8 +4,8 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/bluele/gcache"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
|
4
api/cache/objects.go
vendored
4
api/cache/objects.go
vendored
|
@ -4,9 +4,9 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/bluele/gcache"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
|
6
api/cache/objects_test.go
vendored
6
api/cache/objects_test.go
vendored
|
@ -4,9 +4,9 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
|
6
api/cache/objectslist.go
vendored
6
api/cache/objectslist.go
vendored
|
@ -6,9 +6,9 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"github.com/bluele/gcache"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -20,7 +20,7 @@ import (
|
|||
After putting a record, it lives for a while (default value is 60 seconds).
|
||||
|
||||
When we receive a request from a user, we try to find the suitable and non-expired cache entry, go through the list
|
||||
and get ObjectInfos from common object cache or with a request to NeoFS.
|
||||
and get ObjectInfos from common object cache or with a request to FrostFS.
|
||||
|
||||
When we put an object into a container, we invalidate entries with prefixes that are prefixes of the object's name.
|
||||
*/
|
||||
|
|
6
api/cache/objectslist_test.go
vendored
6
api/cache/objectslist_test.go
vendored
|
@ -4,9 +4,9 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
|
||||
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
|
2
api/cache/system.go
vendored
2
api/cache/system.go
vendored
|
@ -4,8 +4,8 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/bluele/gcache"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
|
|
@ -4,9 +4,9 @@ import (
|
|||
"encoding/xml"
|
||||
"time"
|
||||
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -22,7 +22,8 @@ const (
|
|||
type (
|
||||
// BucketInfo stores basic bucket data.
|
||||
BucketInfo struct {
|
||||
Name string
|
||||
Name string // container name from system attribute
|
||||
Zone string // container zone from system attribute
|
||||
CID cid.ID
|
||||
Owner user.ID
|
||||
Created time.Time
|
||||
|
|
|
@ -4,9 +4,9 @@ import (
|
|||
"strconv"
|
||||
"time"
|
||||
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -25,7 +25,7 @@ func (v NodeVersion) IsDeleteMarker() bool {
|
|||
}
|
||||
|
||||
// DeleteMarkerInfo is used to save object info if node in the tree service is delete marker.
|
||||
// We need this information because the "delete marker" object is no longer stored in NeoFS.
|
||||
// We need this information because the "delete marker" object is no longer stored in FrostFS.
|
||||
type DeleteMarkerInfo struct {
|
||||
Created time.Time
|
||||
Owner user.ID
|
||||
|
|
|
@ -90,6 +90,7 @@ const (
|
|||
ErrMissingFields
|
||||
ErrMissingCredTag
|
||||
ErrCredMalformed
|
||||
ErrInvalidLocationConstraint
|
||||
ErrInvalidRegion
|
||||
ErrInvalidServiceS3
|
||||
ErrInvalidServiceSTS
|
||||
|
@ -680,6 +681,12 @@ var errorCodes = errorCodeMap{
|
|||
Description: "Error parsing the X-Amz-Credential parameter; the region is wrong;",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidLocationConstraint: {
|
||||
ErrCode: ErrInvalidLocationConstraint,
|
||||
Code: "InvalidLocationConstraint",
|
||||
Description: "The specified location (Region) constraint is not valid.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidRegion: {
|
||||
ErrCode: ErrInvalidRegion,
|
||||
Code: "InvalidRegion",
|
||||
|
@ -986,7 +993,7 @@ var errorCodes = errorCodeMap{
|
|||
ErrNotSupported: {
|
||||
ErrCode: ErrNotSupported,
|
||||
Code: "BadRequest",
|
||||
Description: "Not supported by NeoFS S3 Gateway",
|
||||
Description: "Not supported by FrostFS S3 Gateway",
|
||||
HTTPStatusCode: http.StatusNotImplemented,
|
||||
},
|
||||
ErrInvalidEncryptionMethod: {
|
||||
|
|
|
@ -14,16 +14,16 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
|
||||
v2acl "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
v2acl "github.com/nspcc-dev/neofs-api-go/v2/acl"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -158,6 +158,90 @@ func (s ServiceRecord) ToEACLRecord() *eacl.Record {
|
|||
return serviceRecord
|
||||
}
|
||||
|
||||
var (
|
||||
errInvalidStatement = stderrors.New("invalid statement")
|
||||
errInvalidPrincipal = stderrors.New("invalid principal")
|
||||
)
|
||||
|
||||
func (s *statement) UnmarshalJSON(data []byte) error {
|
||||
var statementMap map[string]interface{}
|
||||
if err := json.Unmarshal(data, &statementMap); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sidField, ok := statementMap["Sid"]
|
||||
if ok {
|
||||
if s.Sid, ok = sidField.(string); !ok {
|
||||
return errInvalidStatement
|
||||
}
|
||||
}
|
||||
|
||||
effectField, ok := statementMap["Effect"]
|
||||
if ok {
|
||||
if s.Effect, ok = effectField.(string); !ok {
|
||||
return errInvalidStatement
|
||||
}
|
||||
}
|
||||
|
||||
principalField, ok := statementMap["Principal"]
|
||||
if ok {
|
||||
principalMap, ok := principalField.(map[string]interface{})
|
||||
if !ok {
|
||||
return errInvalidPrincipal
|
||||
}
|
||||
|
||||
awsField, ok := principalMap["AWS"]
|
||||
if ok {
|
||||
if s.Principal.AWS, ok = awsField.(string); !ok {
|
||||
return fmt.Errorf("%w: 'AWS' field must be string", errInvalidPrincipal)
|
||||
}
|
||||
}
|
||||
|
||||
canonicalUserField, ok := principalMap["CanonicalUser"]
|
||||
if ok {
|
||||
if s.Principal.CanonicalUser, ok = canonicalUserField.(string); !ok {
|
||||
return errInvalidPrincipal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
actionField, ok := statementMap["Action"]
|
||||
if ok {
|
||||
switch actionField := actionField.(type) {
|
||||
case []interface{}:
|
||||
s.Action = make([]string, len(actionField))
|
||||
for i, action := range actionField {
|
||||
if s.Action[i], ok = action.(string); !ok {
|
||||
return errInvalidStatement
|
||||
}
|
||||
}
|
||||
case string:
|
||||
s.Action = []string{actionField}
|
||||
default:
|
||||
return errInvalidStatement
|
||||
}
|
||||
}
|
||||
|
||||
resourceField, ok := statementMap["Resource"]
|
||||
if ok {
|
||||
switch resourceField := resourceField.(type) {
|
||||
case []interface{}:
|
||||
s.Resource = make([]string, len(resourceField))
|
||||
for i, action := range resourceField {
|
||||
if s.Resource[i], ok = action.(string); !ok {
|
||||
return errInvalidStatement
|
||||
}
|
||||
}
|
||||
case string:
|
||||
s.Resource = []string{resourceField}
|
||||
default:
|
||||
return errInvalidStatement
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
|
||||
|
@ -1001,6 +1085,10 @@ func policyToAst(bktPolicy *bucketPolicy) (*ast, error) {
|
|||
trimmedResource := strings.TrimPrefix(resource, arnAwsPrefix)
|
||||
r, ok := rr[trimmedResource]
|
||||
if !ok {
|
||||
if !strings.HasPrefix(trimmedResource, bktPolicy.Bucket) {
|
||||
return nil, fmt.Errorf("resource '%s' must be in the same bucket '%s'", trimmedResource, bktPolicy.Bucket)
|
||||
}
|
||||
|
||||
r = &astResource{
|
||||
resourceInfo: resourceInfoFromName(trimmedResource, bktPolicy.Bucket),
|
||||
}
|
||||
|
@ -1046,9 +1134,6 @@ func astToPolicy(ast *ast) *bucketPolicy {
|
|||
bktPolicy := &bucketPolicy{}
|
||||
|
||||
for _, resource := range ast.Resources {
|
||||
if len(resource.Version) == 0 {
|
||||
continue
|
||||
}
|
||||
allowed, denied := triageOperations(resource.Operations)
|
||||
handleResourceOperations(bktPolicy, allowed, eacl.ActionAllow, resource.Name())
|
||||
handleResourceOperations(bktPolicy, denied, eacl.ActionDeny, resource.Name())
|
||||
|
@ -1080,7 +1165,7 @@ func handleResourceOperations(bktPolicy *bucketPolicy, list []*astOperation, eac
|
|||
for action, ops := range actionToOpMap {
|
||||
for _, op := range ops {
|
||||
if !contains(userOps, op) {
|
||||
break LOOP
|
||||
continue LOOP
|
||||
}
|
||||
}
|
||||
actions = append(actions, action)
|
||||
|
|
|
@ -1,25 +1,27 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -1297,7 +1299,7 @@ func TestPutBucketACL(t *testing.T) {
|
|||
tc := prepareHandlerContext(t)
|
||||
bktName := "bucket-for-acl"
|
||||
|
||||
box := createAccessBox(t)
|
||||
box, _ := createAccessBox(t)
|
||||
bktInfo := createBucket(t, tc, bktName, box)
|
||||
|
||||
header := map[string]string{api.AmzACL: "public-read"}
|
||||
|
@ -1308,6 +1310,149 @@ func TestPutBucketACL(t *testing.T) {
|
|||
checkLastRecords(t, tc, bktInfo, eacl.ActionDeny)
|
||||
}
|
||||
|
||||
func TestBucketPolicy(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName := "bucket-for-policy"
|
||||
|
||||
box, key := createAccessBox(t)
|
||||
createBucket(t, hc, bktName, box)
|
||||
|
||||
bktPolicy := getBucketPolicy(hc, bktName)
|
||||
for _, st := range bktPolicy.Statement {
|
||||
if st.Effect == "Allow" {
|
||||
require.Equal(t, hex.EncodeToString(key.PublicKey().Bytes()), st.Principal.CanonicalUser)
|
||||
require.Equal(t, []string{arnAwsPrefix + bktName}, st.Resource)
|
||||
} else {
|
||||
require.Equal(t, allUsersWildcard, st.Principal.AWS)
|
||||
require.Equal(t, "Deny", st.Effect)
|
||||
require.Equal(t, []string{arnAwsPrefix + bktName}, st.Resource)
|
||||
}
|
||||
}
|
||||
|
||||
newPolicy := &bucketPolicy{
|
||||
Statement: []statement{{
|
||||
Effect: "Allow",
|
||||
Principal: principal{AWS: allUsersWildcard},
|
||||
Action: []string{s3GetObject},
|
||||
Resource: []string{arnAwsPrefix + "dummy"},
|
||||
}},
|
||||
}
|
||||
|
||||
putBucketPolicy(hc, bktName, newPolicy, box, http.StatusInternalServerError)
|
||||
|
||||
newPolicy.Statement[0].Resource[0] = arnAwsPrefix + bktName
|
||||
putBucketPolicy(hc, bktName, newPolicy, box, http.StatusOK)
|
||||
|
||||
bktPolicy = getBucketPolicy(hc, bktName)
|
||||
for _, st := range bktPolicy.Statement {
|
||||
if st.Effect == "Allow" && st.Principal.AWS == allUsersWildcard {
|
||||
require.Equal(t, []string{arnAwsPrefix + bktName}, st.Resource)
|
||||
require.ElementsMatch(t, []string{s3GetObject, s3ListBucket}, st.Action)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketPolicyUnmarshal(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
policy string
|
||||
}{
|
||||
{
|
||||
name: "action/resource array",
|
||||
policy: `
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Principal": {
|
||||
"AWS": "arn:aws:iam::111122223333:role/JohnDoe"
|
||||
},
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:GetObjectVersion"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::DOC-EXAMPLE-BUCKET/*",
|
||||
"arn:aws:s3:::DOC-EXAMPLE-BUCKET2/*"
|
||||
]
|
||||
}]
|
||||
}
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "action/resource string",
|
||||
policy: `
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Principal": {
|
||||
"AWS": "arn:aws:iam::111122223333:role/JohnDoe"
|
||||
},
|
||||
"Effect": "Deny",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::DOC-EXAMPLE-BUCKET/*"
|
||||
}]
|
||||
}
|
||||
`,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
bktPolicy := &bucketPolicy{}
|
||||
err := json.Unmarshal([]byte(tc.policy), bktPolicy)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutBucketPolicy(t *testing.T) {
|
||||
bktPolicy := `
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Principal": {
|
||||
"AWS": "*"
|
||||
},
|
||||
"Effect": "Deny",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::bucket-for-policy/*"
|
||||
}]
|
||||
}
|
||||
`
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName := "bucket-for-policy"
|
||||
|
||||
box, _ := createAccessBox(t)
|
||||
createBucket(t, hc, bktName, box)
|
||||
|
||||
w, r := prepareTestPayloadRequest(hc, bktName, "", bytes.NewReader([]byte(bktPolicy)))
|
||||
ctx := context.WithValue(r.Context(), api.BoxData, box)
|
||||
r = r.WithContext(ctx)
|
||||
hc.Handler().PutBucketPolicyHandler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func getBucketPolicy(hc *handlerContext, bktName string) *bucketPolicy {
|
||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||
hc.Handler().GetBucketPolicyHandler(w, r)
|
||||
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
policy := &bucketPolicy{}
|
||||
err := json.NewDecoder(w.Result().Body).Decode(policy)
|
||||
require.NoError(hc.t, err)
|
||||
return policy
|
||||
}
|
||||
|
||||
func putBucketPolicy(hc *handlerContext, bktName string, bktPolicy *bucketPolicy, box *accessbox.Box, status int) {
|
||||
body, err := json.Marshal(bktPolicy)
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
w, r := prepareTestPayloadRequest(hc, bktName, "", bytes.NewReader(body))
|
||||
ctx := context.WithValue(r.Context(), api.BoxData, box)
|
||||
r = r.WithContext(ctx)
|
||||
hc.Handler().PutBucketPolicyHandler(w, r)
|
||||
assertStatus(hc.t, w, status)
|
||||
}
|
||||
|
||||
func checkLastRecords(t *testing.T, tc *handlerContext, bktInfo *data.BucketInfo, action eacl.Action) {
|
||||
bktACL, err := tc.Layer().GetBucketACL(tc.Context(), bktInfo)
|
||||
require.NoError(t, err)
|
||||
|
@ -1325,7 +1470,7 @@ func checkLastRecords(t *testing.T, tc *handlerContext, bktInfo *data.BucketInfo
|
|||
}
|
||||
}
|
||||
|
||||
func createAccessBox(t *testing.T) *accessbox.Box {
|
||||
func createAccessBox(t *testing.T) (*accessbox.Box, *keys.PrivateKey) {
|
||||
key, err := keys.NewPrivateKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
|
@ -1345,7 +1490,7 @@ func createAccessBox(t *testing.T) *accessbox.Box {
|
|||
},
|
||||
}
|
||||
|
||||
return box
|
||||
return box, key
|
||||
}
|
||||
|
||||
func createBucket(t *testing.T, tc *handlerContext, bktName string, box *accessbox.Box) *data.BucketInfo {
|
||||
|
|
|
@ -1,11 +1,14 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -19,21 +22,33 @@ type (
|
|||
|
||||
Notificator interface {
|
||||
SendNotifications(topics map[string]string, p *SendNotificationParams) error
|
||||
SendTestNotification(topic, bucketName, requestID, HostID string) error
|
||||
SendTestNotification(topic, bucketName, requestID, HostID string, now time.Time) error
|
||||
}
|
||||
|
||||
// Config contains data which handler needs to keep.
|
||||
Config struct {
|
||||
DefaultPolicy netmap.PlacementPolicy
|
||||
DefaultMaxAge int
|
||||
NotificatorEnabled bool
|
||||
TLSEnabled bool
|
||||
CopiesNumber uint32
|
||||
Policy PlacementPolicy
|
||||
XMLDecoder XMLDecoderProvider
|
||||
DefaultMaxAge int
|
||||
NotificatorEnabled bool
|
||||
CopiesNumber uint32
|
||||
ResolveZoneList []string
|
||||
IsResolveListAllow bool // True if ResolveZoneList contains allowed zones
|
||||
CompleteMultipartKeepalive time.Duration
|
||||
}
|
||||
|
||||
PlacementPolicy interface {
|
||||
Default() netmap.PlacementPolicy
|
||||
Get(string) (netmap.PlacementPolicy, bool)
|
||||
}
|
||||
|
||||
XMLDecoderProvider interface {
|
||||
NewCompleteMultipartDecoder(io.Reader) *xml.Decoder
|
||||
}
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultPolicy is a default policy of placing containers in NeoFS if it's not set at the request.
|
||||
// DefaultPolicy is a default policy of placing containers in FrostFS if it's not set at the request.
|
||||
DefaultPolicy = "REP 3"
|
||||
// DefaultCopiesNumber is a default number of object copies that is enough to consider put successful if it's not set in config.
|
||||
DefaultCopiesNumber uint32 = 0
|
||||
|
@ -45,7 +60,7 @@ var _ api.Handler = (*handler)(nil)
|
|||
func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg *Config) (api.Handler, error) {
|
||||
switch {
|
||||
case obj == nil:
|
||||
return nil, errors.New("empty NeoFS Object Layer")
|
||||
return nil, errors.New("empty FrostFS Object Layer")
|
||||
case log == nil:
|
||||
return nil, errors.New("empty logger")
|
||||
}
|
||||
|
|
|
@ -6,10 +6,10 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -94,7 +94,7 @@ func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Requ
|
|||
}
|
||||
info := extendedInfo.ObjectInfo
|
||||
|
||||
encryptionParams, err := h.formEncryptionParams(r.Header)
|
||||
encryptionParams, err := formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
|
|
|
@ -4,7 +4,7 @@ import (
|
|||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
|
|
@ -6,12 +6,12 @@ import (
|
|||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -142,7 +142,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
}
|
||||
|
||||
encryptionParams, err := h.formEncryptionParams(r.Header)
|
||||
encryptionParams, err := formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
|
@ -184,7 +184,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
CopiesNuber: copiesNumber,
|
||||
}
|
||||
|
||||
params.Lock, err = formObjectLock(dstBktInfo, settings.LockConfiguration, r.Header)
|
||||
params.Lock, err = formObjectLock(r.Context(), dstBktInfo, settings.LockConfiguration, r.Header)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not form object lock", reqInfo, err)
|
||||
return
|
||||
|
|
|
@ -6,7 +6,7 @@ import (
|
|||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
|
|
@ -5,9 +5,9 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
|
41
api/handler/cors_test.go
Normal file
41
api/handler/cors_test.go
Normal file
|
@ -0,0 +1,41 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
)
|
||||
|
||||
func TestCORSOriginWildcard(t *testing.T) {
|
||||
body := `
|
||||
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<CORSRule>
|
||||
<AllowedMethod>GET</AllowedMethod>
|
||||
<AllowedOrigin>*</AllowedOrigin>
|
||||
</CORSRule>
|
||||
</CORSConfiguration>
|
||||
`
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-cors"
|
||||
box, _ := createAccessBox(t)
|
||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||
ctx := context.WithValue(r.Context(), api.BoxData, box)
|
||||
r = r.WithContext(ctx)
|
||||
r.Header.Add(api.AmzACL, "public-read")
|
||||
hc.Handler().CreateBucketHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
|
||||
ctx = context.WithValue(r.Context(), api.BoxData, box)
|
||||
r = r.WithContext(ctx)
|
||||
hc.Handler().PutBucketCorsHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
|
||||
hc.Handler().GetBucketCorsHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
}
|
|
@ -6,17 +6,20 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
)
|
||||
|
||||
// limitation of AWS https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
|
||||
const maxObjectsToDelete = 1000
|
||||
|
||||
// DeleteObjectsRequest -- xml carrying the object key names which should be deleted.
|
||||
type DeleteObjectsRequest struct {
|
||||
// Element to enable quiet mode for the request
|
||||
|
@ -176,6 +179,11 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
|||
return
|
||||
}
|
||||
|
||||
if len(requested.Objects) == 0 || len(requested.Objects) > maxObjectsToDelete {
|
||||
h.logAndSendError(w, "number of objects to delete must be greater than 0 and less or equal to 1000", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
|
||||
return
|
||||
}
|
||||
|
||||
removed := make(map[string]*layer.VersionedObject)
|
||||
toRemove := make([]*layer.VersionedObject, 0, len(requested.Objects))
|
||||
for _, obj := range requested.Objects {
|
||||
|
|
|
@ -6,8 +6,8 @@ import (
|
|||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -41,7 +41,7 @@ func TestDeleteObject(t *testing.T) {
|
|||
deleteObject(t, tc, bktName, objName, emptyVersion)
|
||||
checkNotFound(t, tc, bktName, objName, emptyVersion)
|
||||
|
||||
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo))
|
||||
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo))
|
||||
}
|
||||
|
||||
func TestDeleteObjectFromSuspended(t *testing.T) {
|
||||
|
@ -109,7 +109,7 @@ func TestDeleteObjectVersioned(t *testing.T) {
|
|||
deleteObject(t, tc, bktName, objName, objInfo.VersionID())
|
||||
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
|
||||
|
||||
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object exists but shouldn't")
|
||||
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object exists but shouldn't")
|
||||
}
|
||||
|
||||
func TestDeleteObjectUnversioned(t *testing.T) {
|
||||
|
@ -126,7 +126,7 @@ func TestDeleteObjectUnversioned(t *testing.T) {
|
|||
require.Len(t, versions.DeleteMarker, 0, "delete markers must be empty")
|
||||
require.Len(t, versions.Version, 0, "versions must be empty")
|
||||
|
||||
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object exists but shouldn't")
|
||||
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object exists but shouldn't")
|
||||
}
|
||||
|
||||
func TestRemoveDeleteMarker(t *testing.T) {
|
||||
|
@ -144,7 +144,7 @@ func TestRemoveDeleteMarker(t *testing.T) {
|
|||
deleteObject(t, tc, bktName, objName, deleteMarkerVersion)
|
||||
checkFound(t, tc, bktName, objName, emptyVersion)
|
||||
|
||||
require.True(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object doesn't exist but should")
|
||||
require.True(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object doesn't exist but should")
|
||||
}
|
||||
|
||||
func TestDeleteObjectCombined(t *testing.T) {
|
||||
|
@ -161,7 +161,7 @@ func TestDeleteObjectCombined(t *testing.T) {
|
|||
|
||||
checkFound(t, tc, bktName, objName, objInfo.VersionID())
|
||||
|
||||
require.True(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object doesn't exist but should")
|
||||
require.True(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object doesn't exist but should")
|
||||
}
|
||||
|
||||
func TestDeleteObjectSuspended(t *testing.T) {
|
||||
|
@ -181,7 +181,7 @@ func TestDeleteObjectSuspended(t *testing.T) {
|
|||
deleteObject(t, tc, bktName, objName, emptyVersion)
|
||||
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
|
||||
|
||||
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object exists but shouldn't")
|
||||
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object exists but shouldn't")
|
||||
}
|
||||
|
||||
func TestDeleteMarkers(t *testing.T) {
|
||||
|
@ -200,7 +200,7 @@ func TestDeleteMarkers(t *testing.T) {
|
|||
require.Len(t, versions.DeleteMarker, 3, "invalid delete markers length")
|
||||
require.Len(t, versions.Version, 0, "versions must be empty")
|
||||
|
||||
require.Len(t, listOIDsFromMockedNeoFS(t, tc, bktName), 0, "shouldn't be any object in neofs")
|
||||
require.Len(t, listOIDsFromMockedFrostFS(t, tc, bktName), 0, "shouldn't be any object in frostfs")
|
||||
}
|
||||
|
||||
func TestDeleteObjectFromListCache(t *testing.T) {
|
||||
|
@ -220,7 +220,7 @@ func TestDeleteObjectFromListCache(t *testing.T) {
|
|||
versions = listObjectsV1(t, tc, bktName, "", "", "", -1)
|
||||
require.Len(t, versions.Contents, 0)
|
||||
|
||||
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo))
|
||||
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo))
|
||||
}
|
||||
|
||||
func TestDeleteObjectCheckMarkerReturn(t *testing.T) {
|
||||
|
|
|
@ -3,6 +3,7 @@ package handler
|
|||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
@ -11,8 +12,8 @@ import (
|
|||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -284,6 +285,7 @@ func getEncryptedObjectRange(t *testing.T, tc *handlerContext, bktName, objName
|
|||
}
|
||||
|
||||
func setEncryptHeaders(r *http.Request) {
|
||||
r.TLS = &tls.ConnectionState{}
|
||||
r.Header.Set(api.AmzServerSideEncryptionCustomerAlgorithm, layer.AESEncryptionAlgorithm)
|
||||
r.Header.Set(api.AmzServerSideEncryptionCustomerKey, aes256Key)
|
||||
r.Header.Set(api.AmzServerSideEncryptionCustomerKeyMD5, aes256KeyMD5)
|
||||
|
|
|
@ -8,10 +8,10 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -150,7 +150,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
encryptionParams, err := h.formEncryptionParams(r.Header)
|
||||
encryptionParams, err := formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
|
|
|
@ -8,9 +8,9 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
|
|
@ -13,15 +13,16 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/resolver"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -30,7 +31,7 @@ type handlerContext struct {
|
|||
owner user.ID
|
||||
t *testing.T
|
||||
h *handler
|
||||
tp *layer.TestNeoFS
|
||||
tp *layer.TestFrostFS
|
||||
context context.Context
|
||||
}
|
||||
|
||||
|
@ -38,7 +39,7 @@ func (hc *handlerContext) Handler() *handler {
|
|||
return hc.h
|
||||
}
|
||||
|
||||
func (hc *handlerContext) MockedPool() *layer.TestNeoFS {
|
||||
func (hc *handlerContext) MockedPool() *layer.TestFrostFS {
|
||||
return hc.tp
|
||||
}
|
||||
|
||||
|
@ -50,12 +51,30 @@ func (hc *handlerContext) Context() context.Context {
|
|||
return hc.context
|
||||
}
|
||||
|
||||
type placementPolicyMock struct {
|
||||
defaultPolicy netmap.PlacementPolicy
|
||||
}
|
||||
|
||||
func (p *placementPolicyMock) Default() netmap.PlacementPolicy {
|
||||
return p.defaultPolicy
|
||||
}
|
||||
|
||||
func (p *placementPolicyMock) Get(string) (netmap.PlacementPolicy, bool) {
|
||||
return netmap.PlacementPolicy{}, false
|
||||
}
|
||||
|
||||
type xmlDecoderProviderMock struct{}
|
||||
|
||||
func (p *xmlDecoderProviderMock) NewCompleteMultipartDecoder(r io.Reader) *xml.Decoder {
|
||||
return xml.NewDecoder(r)
|
||||
}
|
||||
|
||||
func prepareHandlerContext(t *testing.T) *handlerContext {
|
||||
key, err := keys.NewPrivateKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
l := zap.NewExample()
|
||||
tp := layer.NewTestNeoFS()
|
||||
tp := layer.NewTestFrostFS()
|
||||
|
||||
testResolver := &resolver.Resolver{Name: "test_resolver"}
|
||||
testResolver.SetResolveFunc(func(_ context.Context, name string) (cid.ID, error) {
|
||||
|
@ -72,11 +91,16 @@ func prepareHandlerContext(t *testing.T) *handlerContext {
|
|||
TreeService: layer.NewTreeService(),
|
||||
}
|
||||
|
||||
var pp netmap.PlacementPolicy
|
||||
err = pp.DecodeString("REP 1")
|
||||
require.NoError(t, err)
|
||||
|
||||
h := &handler{
|
||||
log: l,
|
||||
obj: layer.NewLayer(l, tp, layerCfg),
|
||||
cfg: &Config{
|
||||
TLSEnabled: true,
|
||||
Policy: &placementPolicyMock{defaultPolicy: pp},
|
||||
XMLDecoder: &xmlDecoderProviderMock{},
|
||||
},
|
||||
}
|
||||
|
||||
|
@ -191,7 +215,7 @@ func parseTestResponse(t *testing.T, response *httptest.ResponseRecorder, body i
|
|||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func existInMockedNeoFS(tc *handlerContext, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo) bool {
|
||||
func existInMockedFrostFS(tc *handlerContext, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo) bool {
|
||||
p := &layer.GetObjectParams{
|
||||
BucketInfo: bktInfo,
|
||||
ObjectInfo: objInfo,
|
||||
|
@ -201,7 +225,7 @@ func existInMockedNeoFS(tc *handlerContext, bktInfo *data.BucketInfo, objInfo *d
|
|||
return tc.Layer().GetObject(tc.Context(), p) == nil
|
||||
}
|
||||
|
||||
func listOIDsFromMockedNeoFS(t *testing.T, tc *handlerContext, bktName string) []oid.ID {
|
||||
func listOIDsFromMockedFrostFS(t *testing.T, tc *handlerContext, bktName string) []oid.ID {
|
||||
bktInfo, err := tc.Layer().GetBucketInfo(tc.Context(), bktName)
|
||||
require.NoError(t, err)
|
||||
|
||||
|
|
|
@ -4,10 +4,10 @@ import (
|
|||
"bytes"
|
||||
"net/http"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -53,7 +53,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
info := extendedInfo.ObjectInfo
|
||||
|
||||
encryptionParams, err := h.formEncryptionParams(r.Header)
|
||||
encryptionParams, err := formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
|
@ -123,6 +123,13 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
w.Header().Set(api.ContainerID, bktInfo.CID.EncodeToString())
|
||||
w.Header().Set(api.AmzBucketRegion, bktInfo.LocationConstraint)
|
||||
|
||||
if isAvailableToResolve(bktInfo.Zone, h.cfg.ResolveZoneList, h.cfg.IsResolveListAllow) {
|
||||
w.Header().Set(api.ContainerName, bktInfo.Name)
|
||||
w.Header().Set(api.ContainerZone, bktInfo.Zone)
|
||||
}
|
||||
|
||||
api.WriteResponse(w, http.StatusOK, nil, api.MimeNone)
|
||||
}
|
||||
|
||||
|
@ -156,3 +163,25 @@ func writeLockHeaders(h http.Header, legalHold *data.LegalHold, retention *data.
|
|||
h.Set(api.AmzObjectLockMode, retention.Mode)
|
||||
}
|
||||
}
|
||||
|
||||
func isAvailableToResolve(zone string, list []string, isAllowList bool) bool {
|
||||
// empty zone means container doesn't have proper system name,
|
||||
// so we don't have to resolve it
|
||||
if len(zone) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
var zoneInList bool
|
||||
for _, t := range list {
|
||||
if t == zone {
|
||||
zoneInList = true
|
||||
break
|
||||
}
|
||||
}
|
||||
// InList | IsAllowList | Result
|
||||
// 0 0 1
|
||||
// 0 1 0
|
||||
// 1 0 0
|
||||
// 1 1 1
|
||||
return zoneInList == isAllowList
|
||||
}
|
||||
|
|
|
@ -6,11 +6,11 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -86,6 +86,26 @@ func TestInvalidAccessThroughCache(t *testing.T) {
|
|||
assertStatus(t, w, http.StatusForbidden)
|
||||
}
|
||||
|
||||
func TestIsAvailableToResolve(t *testing.T) {
|
||||
list := []string{"container", "s3"}
|
||||
|
||||
for i, testCase := range [...]struct {
|
||||
isAllowList bool
|
||||
list []string
|
||||
zone string
|
||||
expected bool
|
||||
}{
|
||||
{isAllowList: true, list: list, zone: "container", expected: true},
|
||||
{isAllowList: true, list: list, zone: "sftp", expected: false},
|
||||
{isAllowList: false, list: list, zone: "s3", expected: false},
|
||||
{isAllowList: false, list: list, zone: "system", expected: true},
|
||||
{isAllowList: true, list: list, zone: "", expected: false},
|
||||
} {
|
||||
result := isAvailableToResolve(testCase.zone, testCase.list, testCase.isAllowList)
|
||||
require.Equal(t, testCase.expected, result, "case %d", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box {
|
||||
var err error
|
||||
if key == nil {
|
||||
|
|
|
@ -3,7 +3,7 @@ package handler
|
|||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
)
|
||||
|
||||
func (h *handler) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
|
@ -4,8 +4,8 @@ import (
|
|||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
)
|
||||
|
||||
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||
|
|
|
@ -1,16 +1,17 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -208,7 +209,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
|
|||
return
|
||||
}
|
||||
|
||||
lock, err := formObjectLockFromRetention(retention, r.Header)
|
||||
lock, err := formObjectLockFromRetention(r.Context(), retention, r.Header)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid retention configuration", reqInfo, err)
|
||||
return
|
||||
|
@ -300,7 +301,7 @@ func checkLockConfiguration(conf *data.ObjectLockConfiguration) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConfiguration, header http.Header) (*data.ObjectLock, error) {
|
||||
func formObjectLock(ctx context.Context, bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConfiguration, header http.Header) (*data.ObjectLock, error) {
|
||||
if !bktInfo.ObjectLockEnabled {
|
||||
if existLockHeaders(header) {
|
||||
return nil, apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound)
|
||||
|
@ -318,7 +319,7 @@ func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConf
|
|||
retention := &data.RetentionLock{}
|
||||
defaultRetention := defaultConfig.Rule.DefaultRetention
|
||||
retention.IsCompliance = defaultRetention.Mode == complianceMode
|
||||
now := time.Now()
|
||||
now := layer.TimeNow(ctx)
|
||||
if defaultRetention.Days != 0 {
|
||||
retention.Until = now.Add(time.Duration(defaultRetention.Days) * dayDuration)
|
||||
} else {
|
||||
|
@ -370,7 +371,7 @@ func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConf
|
|||
objectLock.Retention.ByPassedGovernance = bypass
|
||||
}
|
||||
|
||||
if objectLock.Retention.Until.Before(time.Now()) {
|
||||
if objectLock.Retention.Until.Before(layer.TimeNow(ctx)) {
|
||||
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
|
||||
}
|
||||
}
|
||||
|
@ -384,7 +385,7 @@ func existLockHeaders(header http.Header) bool {
|
|||
header.Get(api.AmzObjectLockRetainUntilDate) != ""
|
||||
}
|
||||
|
||||
func formObjectLockFromRetention(retention *data.Retention, header http.Header) (*data.ObjectLock, error) {
|
||||
func formObjectLockFromRetention(ctx context.Context, retention *data.Retention, header http.Header) (*data.ObjectLock, error) {
|
||||
if retention.Mode != governanceMode && retention.Mode != complianceMode {
|
||||
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
|
||||
}
|
||||
|
@ -394,7 +395,7 @@ func formObjectLockFromRetention(retention *data.Retention, header http.Header)
|
|||
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
|
||||
}
|
||||
|
||||
if retentionDate.Before(time.Now()) {
|
||||
if retentionDate.Before(layer.TimeNow(ctx)) {
|
||||
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
|
||||
}
|
||||
|
||||
|
|
|
@ -10,15 +10,17 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const defaultURL = "http://localhost/"
|
||||
|
||||
func TestFormObjectLock(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
bktInfo *data.BucketInfo
|
||||
|
@ -73,7 +75,7 @@ func TestFormObjectLock(t *testing.T) {
|
|||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
actualObjLock, err := formObjectLock(tc.bktInfo, tc.config, tc.header)
|
||||
actualObjLock, err := formObjectLock(ctx, tc.bktInfo, tc.config, tc.header)
|
||||
if tc.expectedError {
|
||||
require.Error(t, err)
|
||||
return
|
||||
|
@ -86,6 +88,8 @@ func TestFormObjectLock(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestFormObjectLockFromRetention(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
retention *data.Retention
|
||||
|
@ -132,7 +136,7 @@ func TestFormObjectLockFromRetention(t *testing.T) {
|
|||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
actualObjLock, err := formObjectLockFromRetention(tc.retention, tc.header)
|
||||
actualObjLock, err := formObjectLockFromRetention(ctx, tc.retention, tc.header)
|
||||
if tc.expectedError {
|
||||
require.Error(t, err)
|
||||
return
|
||||
|
|
|
@ -2,17 +2,18 @@ package handler
|
|||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/google/uuid"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -137,7 +138,7 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
|
|||
}
|
||||
}
|
||||
|
||||
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
|
||||
p.Info.Encryption, err = formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
|
@ -226,7 +227,7 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
|
|||
Reader: r.Body,
|
||||
}
|
||||
|
||||
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
|
||||
p.Info.Encryption, err = formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
|
@ -331,7 +332,7 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
|||
Range: srcRange,
|
||||
}
|
||||
|
||||
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
|
||||
p.Info.Encryption, err = formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
|
@ -372,8 +373,6 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
|||
}
|
||||
|
||||
var (
|
||||
sessionTokenSetEACL *session.Container
|
||||
|
||||
uploadID = r.URL.Query().Get(uploadIDHeaderName)
|
||||
uploadInfo = &layer.UploadInfoParams{
|
||||
UploadID: uploadID,
|
||||
|
@ -384,7 +383,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
|||
)
|
||||
|
||||
reqBody := new(CompleteMultipartUpload)
|
||||
if err = xml.NewDecoder(r.Body).Decode(reqBody); err != nil {
|
||||
if err = h.cfg.XMLDecoder.NewCompleteMultipartDecoder(r.Body).Decode(reqBody); err != nil {
|
||||
h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
|
||||
errors.GetAPIError(errors.ErrMalformedXML), additional...)
|
||||
return
|
||||
|
@ -399,10 +398,51 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
|||
Parts: reqBody.Parts,
|
||||
}
|
||||
|
||||
// Next operations might take some time, so we want to keep client's
|
||||
// connection alive. To do so, gateway sends periodic white spaces
|
||||
// back to the client the same way as Amazon S3 service does.
|
||||
stopPeriodicResponseWriter := periodicXMLWriter(w, h.cfg.CompleteMultipartKeepalive)
|
||||
|
||||
// Start complete multipart upload which may take some time to fetch object
|
||||
// and re-upload it part by part.
|
||||
objInfo, err := h.completeMultipartUpload(r, c, bktInfo, reqInfo)
|
||||
|
||||
// Stop periodic writer as complete multipart upload is finished
|
||||
// successfully or not.
|
||||
headerIsWritten := stopPeriodicResponseWriter()
|
||||
|
||||
responseWriter := api.EncodeToResponse
|
||||
errLogger := h.logAndSendError
|
||||
// Do not send XML and HTTP headers if periodic writer was invoked at this point.
|
||||
if headerIsWritten {
|
||||
responseWriter = api.EncodeToResponseNoHeader
|
||||
errLogger = h.logAndSendErrorNoHeader
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
errLogger(w, "complete multipart error", reqInfo, err, additional...)
|
||||
return
|
||||
}
|
||||
|
||||
response := CompleteMultipartUploadResponse{
|
||||
Bucket: objInfo.Bucket,
|
||||
ETag: objInfo.HashSum,
|
||||
Key: objInfo.Name,
|
||||
}
|
||||
|
||||
// Here we previously set api.AmzVersionID header for versioned bucket.
|
||||
// It is not possible after #60, because of periodic white
|
||||
// space XML writer to keep connection with the client.
|
||||
|
||||
if err = responseWriter(w, response); err != nil {
|
||||
errLogger(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMultipartParams, bktInfo *data.BucketInfo, reqInfo *api.ReqInfo) (*data.ObjectInfo, error) {
|
||||
uploadData, extendedObjInfo, err := h.obj.CompleteMultipartUpload(r.Context(), c)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not complete multipart upload", reqInfo, err, additional...)
|
||||
return
|
||||
return nil, fmt.Errorf("could not complete multipart upload: %w", err)
|
||||
}
|
||||
objInfo := extendedObjInfo.ObjectInfo
|
||||
|
||||
|
@ -417,21 +457,22 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
|||
NodeVersion: extendedObjInfo.NodeVersion,
|
||||
}
|
||||
if _, err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
|
||||
h.logAndSendError(w, "could not put tagging file of completed multipart upload", reqInfo, err, additional...)
|
||||
return
|
||||
return nil, fmt.Errorf("could not put tagging file of completed multipart upload: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(uploadData.ACLHeaders) != 0 {
|
||||
sessionTokenSetEACL, err := getSessionTokenSetEACL(r.Context())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get eacl token: %w", err)
|
||||
}
|
||||
key, err := h.bearerTokenIssuerKey(r.Context())
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "couldn't get gate key", reqInfo, err)
|
||||
return
|
||||
return nil, fmt.Errorf("couldn't get gate key: %w", err)
|
||||
}
|
||||
acl, err := parseACLHeaders(r.Header, key)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not parse acl", reqInfo, err)
|
||||
return
|
||||
return nil, fmt.Errorf("could not parse acl: %w", err)
|
||||
}
|
||||
|
||||
resInfo := &resourceInfo{
|
||||
|
@ -440,12 +481,10 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
|||
}
|
||||
astObject, err := aclToAst(acl, resInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not translate acl of completed multipart upload to ast", reqInfo, err, additional...)
|
||||
return
|
||||
return nil, fmt.Errorf("could not translate acl of completed multipart upload to ast: %w", err)
|
||||
}
|
||||
if _, err = h.updateBucketACL(r, astObject, bktInfo, sessionTokenSetEACL); err != nil {
|
||||
h.logAndSendError(w, "could not update bucket acl while completing multipart upload", reqInfo, err, additional...)
|
||||
return
|
||||
return nil, fmt.Errorf("could not update bucket acl while completing multipart upload: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -459,24 +498,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
|||
h.log.Error("couldn't send notification: %w", zap.Error(err))
|
||||
}
|
||||
|
||||
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
||||
}
|
||||
|
||||
response := CompleteMultipartUploadResponse{
|
||||
Bucket: objInfo.Bucket,
|
||||
ETag: objInfo.HashSum,
|
||||
Key: objInfo.Name,
|
||||
}
|
||||
|
||||
if bktSettings.VersioningEnabled() {
|
||||
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
||||
}
|
||||
|
||||
if err = api.EncodeToResponse(w, response); err != nil {
|
||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||
}
|
||||
return objInfo, nil
|
||||
}
|
||||
|
||||
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
@ -573,7 +595,7 @@ func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
|
|||
PartNumberMarker: partNumberMarker,
|
||||
}
|
||||
|
||||
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
|
||||
p.Info.Encryption, err = formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
|
@ -608,7 +630,7 @@ func (h *handler) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Req
|
|||
Key: reqInfo.ObjectName,
|
||||
}
|
||||
|
||||
p.Encryption, err = h.formEncryptionParams(r.Header)
|
||||
p.Encryption, err = formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
|
@ -681,3 +703,62 @@ func encodeListPartsToResponse(info *layer.ListPartsInfo, params *layer.ListPart
|
|||
Parts: info.Parts,
|
||||
}
|
||||
}
|
||||
|
||||
// periodicXMLWriter creates go routine to write xml header and whitespaces
|
||||
// over time to avoid connection drop from the client. To work properly,
|
||||
// pass `http.ResponseWriter` with implemented `http.Flusher` interface.
|
||||
// Returns stop function which returns boolean if writer has been used
|
||||
// during goroutine execution. To disable writer, pass 0 duration value.
|
||||
func periodicXMLWriter(w io.Writer, dur time.Duration) (stop func() bool) {
|
||||
if dur == 0 { // 0 duration disables periodic writer
|
||||
return func() bool { return false }
|
||||
}
|
||||
|
||||
whitespaceChar := []byte(" ")
|
||||
closer := make(chan struct{})
|
||||
done := make(chan struct{})
|
||||
headerWritten := false
|
||||
|
||||
go func() {
|
||||
defer close(done)
|
||||
|
||||
tick := time.NewTicker(dur)
|
||||
defer tick.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-tick.C:
|
||||
if !headerWritten {
|
||||
_, err := w.Write([]byte(xml.Header))
|
||||
headerWritten = err == nil
|
||||
}
|
||||
_, err := w.Write(whitespaceChar)
|
||||
if err != nil {
|
||||
return // is there anything we can do better than ignore error?
|
||||
}
|
||||
if buffered, ok := w.(http.Flusher); ok {
|
||||
buffered.Flush()
|
||||
}
|
||||
case <-closer:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
stop = func() bool {
|
||||
close(closer)
|
||||
<-done // wait for goroutine to stop
|
||||
return headerWritten
|
||||
}
|
||||
|
||||
return stop
|
||||
}
|
||||
|
||||
// periodicWriterErrorSender returns handler function to send error. If header is
|
||||
// alreay written by periodic XML writer, do not send HTTP and XML headers.
|
||||
func (h *handler) periodicWriterErrorSender(headerWritten bool) func(http.ResponseWriter, string, *api.ReqInfo, error, ...zap.Field) {
|
||||
if headerWritten {
|
||||
return h.logAndSendErrorNoHeader
|
||||
}
|
||||
return h.logAndSendError
|
||||
}
|
||||
|
|
48
api/handler/multipart_upload_test.go
Normal file
48
api/handler/multipart_upload_test.go
Normal file
|
@ -0,0 +1,48 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPeriodicWriter(t *testing.T) {
|
||||
const dur = 100 * time.Millisecond
|
||||
const whitespaces = 8
|
||||
expected := []byte(xml.Header)
|
||||
for i := 0; i < whitespaces; i++ {
|
||||
expected = append(expected, []byte(" ")...)
|
||||
}
|
||||
|
||||
t.Run("writes data", func(t *testing.T) {
|
||||
buf := bytes.NewBuffer(nil)
|
||||
stop := periodicXMLWriter(buf, dur)
|
||||
|
||||
// N number of whitespaces + half durations to guarantee at least N writes in buffer
|
||||
time.Sleep(whitespaces*dur + dur/2)
|
||||
require.True(t, stop())
|
||||
require.Equal(t, expected, buf.Bytes())
|
||||
|
||||
t.Run("no additional data after stop", func(t *testing.T) {
|
||||
time.Sleep(2 * dur)
|
||||
require.Equal(t, expected, buf.Bytes())
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("does not write data", func(t *testing.T) {
|
||||
buf := bytes.NewBuffer(nil)
|
||||
stop := periodicXMLWriter(buf, dur)
|
||||
time.Sleep(dur / 2)
|
||||
require.False(t, stop())
|
||||
require.Empty(t, buf.Bytes())
|
||||
|
||||
t.Run("disabled", func(t *testing.T) {
|
||||
stop = periodicXMLWriter(buf, 0)
|
||||
require.False(t, stop())
|
||||
require.Empty(t, buf.Bytes())
|
||||
})
|
||||
})
|
||||
}
|
|
@ -3,8 +3,8 @@ package handler
|
|||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
)
|
||||
|
||||
func (h *handler) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
|
@ -6,13 +6,14 @@ import (
|
|||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"github.com/google/uuid"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
||||
)
|
||||
|
||||
type (
|
||||
|
@ -22,6 +23,7 @@ type (
|
|||
BktInfo *data.BucketInfo
|
||||
ReqInfo *api.ReqInfo
|
||||
User string
|
||||
Time time.Time
|
||||
}
|
||||
|
||||
NotificationConfiguration struct {
|
||||
|
@ -107,7 +109,7 @@ func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Re
|
|||
return
|
||||
}
|
||||
|
||||
if _, err = h.checkBucketConfiguration(conf, reqInfo); err != nil {
|
||||
if _, err = h.checkBucketConfiguration(r.Context(), conf, reqInfo); err != nil {
|
||||
h.logAndSendError(w, "couldn't check bucket configuration", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
@ -164,13 +166,15 @@ func (h *handler) sendNotifications(ctx context.Context, p *SendNotificationPara
|
|||
p.User = bearer.ResolveIssuer(*box.Gate.BearerToken).EncodeToString()
|
||||
}
|
||||
|
||||
p.Time = layer.TimeNow(ctx)
|
||||
|
||||
topics := filterSubjects(conf, p.Event, p.NotificationInfo.Name)
|
||||
|
||||
return h.notificator.SendNotifications(topics, p)
|
||||
}
|
||||
|
||||
// checkBucketConfiguration checks notification configuration and generates an ID for configurations with empty ids.
|
||||
func (h *handler) checkBucketConfiguration(conf *data.NotificationConfiguration, r *api.ReqInfo) (completed bool, err error) {
|
||||
func (h *handler) checkBucketConfiguration(ctx context.Context, conf *data.NotificationConfiguration, r *api.ReqInfo) (completed bool, err error) {
|
||||
if conf == nil {
|
||||
return
|
||||
}
|
||||
|
@ -189,7 +193,7 @@ func (h *handler) checkBucketConfiguration(conf *data.NotificationConfiguration,
|
|||
}
|
||||
|
||||
if h.cfg.NotificatorEnabled {
|
||||
if err = h.notificator.SendTestNotification(q.QueueArn, r.BucketName, r.RequestID, r.Host); err != nil {
|
||||
if err = h.notificator.SendTestNotification(q.QueueArn, r.BucketName, r.RequestID, r.Host, layer.TimeNow(ctx)); err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
|
|
|
@ -3,8 +3,8 @@ package handler
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
|
|
@ -6,11 +6,11 @@ import (
|
|||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
// ListObjectsV1Handler handles objects listing requests for API version 1.
|
||||
|
|
|
@ -6,7 +6,7 @@ import (
|
|||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
|
|
@ -16,15 +16,15 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -218,7 +218,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
encryption, err := h.formEncryptionParams(r.Header)
|
||||
encryptionParams, err := formEncryptionParams(r)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
|
||||
return
|
||||
|
@ -230,7 +230,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
Reader: r.Body,
|
||||
Size: r.ContentLength,
|
||||
Header: metadata,
|
||||
Encryption: encryption,
|
||||
Encryption: encryptionParams,
|
||||
CopiesNumber: copiesNumber,
|
||||
}
|
||||
|
||||
|
@ -240,7 +240,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
params.Lock, err = formObjectLock(bktInfo, settings.LockConfiguration, r.Header)
|
||||
params.Lock, err = formObjectLock(r.Context(), bktInfo, settings.LockConfiguration, r.Header)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not form object lock", reqInfo, err)
|
||||
return
|
||||
|
@ -304,7 +304,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
if settings.VersioningEnabled() {
|
||||
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
||||
}
|
||||
if encryption.Enabled() {
|
||||
if encryptionParams.Enabled() {
|
||||
addSSECHeaders(w.Header(), r.Header)
|
||||
}
|
||||
|
||||
|
@ -313,7 +313,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
func getCopiesNumberOrDefault(metadata map[string]string, defaultCopiesNumber uint32) (uint32, error) {
|
||||
copiesNumberStr, ok := metadata[layer.AttributeNeofsCopiesNumber]
|
||||
copiesNumberStr, ok := metadata[layer.AttributeFrostfsCopiesNumber]
|
||||
if !ok {
|
||||
return defaultCopiesNumber, nil
|
||||
}
|
||||
|
@ -326,16 +326,16 @@ func getCopiesNumberOrDefault(metadata map[string]string, defaultCopiesNumber ui
|
|||
return uint32(copiesNumber), nil
|
||||
}
|
||||
|
||||
func (h handler) formEncryptionParams(header http.Header) (enc encryption.Params, err error) {
|
||||
sseCustomerAlgorithm := header.Get(api.AmzServerSideEncryptionCustomerAlgorithm)
|
||||
sseCustomerKey := header.Get(api.AmzServerSideEncryptionCustomerKey)
|
||||
sseCustomerKeyMD5 := header.Get(api.AmzServerSideEncryptionCustomerKeyMD5)
|
||||
func formEncryptionParams(r *http.Request) (enc encryption.Params, err error) {
|
||||
sseCustomerAlgorithm := r.Header.Get(api.AmzServerSideEncryptionCustomerAlgorithm)
|
||||
sseCustomerKey := r.Header.Get(api.AmzServerSideEncryptionCustomerKey)
|
||||
sseCustomerKeyMD5 := r.Header.Get(api.AmzServerSideEncryptionCustomerKeyMD5)
|
||||
|
||||
if len(sseCustomerAlgorithm) == 0 && len(sseCustomerKey) == 0 && len(sseCustomerKeyMD5) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if !h.cfg.TLSEnabled {
|
||||
if r.TLS == nil {
|
||||
return enc, errorsStd.New("encryption available only when TLS is enabled")
|
||||
}
|
||||
|
||||
|
@ -667,12 +667,10 @@ func parseMetadata(r *http.Request) map[string]string {
|
|||
}
|
||||
|
||||
func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
reqInfo = api.GetReqInfo(r.Context())
|
||||
p = layer.CreateBucketParams{
|
||||
Name: reqInfo.BucketName,
|
||||
}
|
||||
)
|
||||
reqInfo := api.GetReqInfo(r.Context())
|
||||
p := &layer.CreateBucketParams{
|
||||
Name: reqInfo.BucketName,
|
||||
}
|
||||
|
||||
if err := checkBucketName(reqInfo.BucketName); err != nil {
|
||||
h.logAndSendError(w, "invalid bucket name", reqInfo, err)
|
||||
|
@ -722,29 +720,22 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
useDefaultPolicy := true
|
||||
if createParams.LocationConstraint != "" {
|
||||
for _, placementPolicy := range policies {
|
||||
if placementPolicy.LocationConstraint == createParams.LocationConstraint {
|
||||
p.Policy = placementPolicy.Policy
|
||||
p.LocationConstraint = createParams.LocationConstraint
|
||||
useDefaultPolicy = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if useDefaultPolicy {
|
||||
p.Policy = h.cfg.DefaultPolicy
|
||||
if err = h.setPolicy(p, createParams.LocationConstraint, policies); err != nil {
|
||||
h.logAndSendError(w, "couldn't set placement policy", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
p.ObjectLockEnabled = isLockEnabled(r.Header)
|
||||
|
||||
bktInfo, err := h.obj.CreateBucket(r.Context(), &p)
|
||||
bktInfo, err := h.obj.CreateBucket(r.Context(), p)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not create bucket", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
h.log.Info("bucket is created", zap.String("reqId", reqInfo.RequestID),
|
||||
zap.String("bucket", reqInfo.BucketName), zap.Stringer("container_id", bktInfo.CID))
|
||||
|
||||
if p.ObjectLockEnabled {
|
||||
sp := &layer.PutSettingsParams{
|
||||
BktInfo: bktInfo,
|
||||
|
@ -757,11 +748,32 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
}
|
||||
|
||||
h.log.Info("bucket is created", zap.Stringer("container_id", bktInfo.CID))
|
||||
|
||||
api.WriteSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
func (h handler) setPolicy(prm *layer.CreateBucketParams, locationConstraint string, userPolicies []*accessbox.ContainerPolicy) error {
|
||||
prm.Policy = h.cfg.Policy.Default()
|
||||
prm.LocationConstraint = locationConstraint
|
||||
|
||||
if locationConstraint == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, placementPolicy := range userPolicies {
|
||||
if placementPolicy.LocationConstraint == locationConstraint {
|
||||
prm.Policy = placementPolicy.Policy
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if policy, ok := h.cfg.Policy.Get(locationConstraint); ok {
|
||||
prm.Policy = policy
|
||||
return nil
|
||||
}
|
||||
|
||||
return errors.GetAPIError(errors.ErrInvalidLocationConstraint)
|
||||
}
|
||||
|
||||
func isLockEnabled(header http.Header) bool {
|
||||
lockEnabledStr := header.Get(api.AmzBucketObjectLockEnabled)
|
||||
lockEnabled, _ := strconv.ParseBool(lockEnabledStr)
|
||||
|
|
|
@ -8,8 +8,8 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -114,7 +114,7 @@ func TestPutObjectOverrideCopiesNumber(t *testing.T) {
|
|||
bktInfo := createTestBucket(tc, bktName)
|
||||
|
||||
w, r := prepareTestRequest(tc, bktName, objName, nil)
|
||||
r.Header.Set(api.MetadataPrefix+strings.ToUpper(layer.AttributeNeofsCopiesNumber), "1")
|
||||
r.Header.Set(api.MetadataPrefix+strings.ToUpper(layer.AttributeFrostfsCopiesNumber), "1")
|
||||
tc.Handler().PutObjectHandler(w, r)
|
||||
|
||||
p := &layer.HeadObjectParams{
|
||||
|
@ -124,5 +124,5 @@ func TestPutObjectOverrideCopiesNumber(t *testing.T) {
|
|||
|
||||
objInfo, err := tc.Layer().GetObjectInfo(tc.Context(), p)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "1", objInfo.Headers[layer.AttributeNeofsCopiesNumber])
|
||||
require.Equal(t, "1", objInfo.Headers[layer.AttributeFrostfsCopiesNumber])
|
||||
}
|
||||
|
|
|
@ -8,10 +8,10 @@ import (
|
|||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
|
|
@ -3,8 +3,8 @@ package handler
|
|||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
)
|
||||
|
||||
func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
|
@ -7,11 +7,11 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -29,6 +29,19 @@ func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo
|
|||
h.log.Error("call method", fields...)
|
||||
}
|
||||
|
||||
func (h *handler) logAndSendErrorNoHeader(w http.ResponseWriter, logText string, reqInfo *api.ReqInfo, err error, additional ...zap.Field) {
|
||||
api.WriteErrorResponseNoHeader(w, reqInfo, transformToS3Error(err))
|
||||
fields := []zap.Field{
|
||||
zap.String("request_id", reqInfo.RequestID),
|
||||
zap.String("method", reqInfo.API),
|
||||
zap.String("bucket", reqInfo.BucketName),
|
||||
zap.String("object", reqInfo.ObjectName),
|
||||
zap.String("description", logText),
|
||||
zap.Error(err)}
|
||||
fields = append(fields, additional...)
|
||||
h.log.Error("call method", fields...)
|
||||
}
|
||||
|
||||
func transformToS3Error(err error) error {
|
||||
if _, ok := err.(errors.Error); ok {
|
||||
return err
|
||||
|
@ -42,6 +55,10 @@ func transformToS3Error(err error) error {
|
|||
return errors.GetAPIError(errors.ErrInternalError)
|
||||
}
|
||||
|
||||
func (h *handler) ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error) {
|
||||
return h.obj.GetBucketInfo(ctx, bucket)
|
||||
}
|
||||
|
||||
func (h *handler) getBucketAndCheckOwner(r *http.Request, bucket string, header ...string) (*data.BucketInfo, error) {
|
||||
bktInfo, err := h.obj.GetBucketInfo(r.Context(), bucket)
|
||||
if err != nil {
|
||||
|
|
|
@ -4,10 +4,10 @@ import (
|
|||
"encoding/xml"
|
||||
"net/http"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
)
|
||||
|
||||
func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
|
@ -2,17 +2,17 @@ package api
|
|||
|
||||
// Standard S3 HTTP request/response constants.
|
||||
const (
|
||||
MetadataPrefix = "X-Amz-Meta-"
|
||||
NeoFSSystemMetadataPrefix = "S3-"
|
||||
AmzMetadataDirective = "X-Amz-Metadata-Directive"
|
||||
AmzTaggingDirective = "X-Amz-Tagging-Directive"
|
||||
AmzVersionID = "X-Amz-Version-Id"
|
||||
AmzTaggingCount = "X-Amz-Tagging-Count"
|
||||
AmzTagging = "X-Amz-Tagging"
|
||||
AmzDeleteMarker = "X-Amz-Delete-Marker"
|
||||
AmzCopySource = "X-Amz-Copy-Source"
|
||||
AmzCopySourceRange = "X-Amz-Copy-Source-Range"
|
||||
AmzDate = "X-Amz-Date"
|
||||
MetadataPrefix = "X-Amz-Meta-"
|
||||
FrostFSSystemMetadataPrefix = "S3-"
|
||||
AmzMetadataDirective = "X-Amz-Metadata-Directive"
|
||||
AmzTaggingDirective = "X-Amz-Tagging-Directive"
|
||||
AmzVersionID = "X-Amz-Version-Id"
|
||||
AmzTaggingCount = "X-Amz-Tagging-Count"
|
||||
AmzTagging = "X-Amz-Tagging"
|
||||
AmzDeleteMarker = "X-Amz-Delete-Marker"
|
||||
AmzCopySource = "X-Amz-Copy-Source"
|
||||
AmzCopySourceRange = "X-Amz-Copy-Source-Range"
|
||||
AmzDate = "X-Amz-Date"
|
||||
|
||||
LastModified = "Last-Modified"
|
||||
Date = "Date"
|
||||
|
@ -62,7 +62,9 @@ const (
|
|||
AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"
|
||||
AmzServerSideEncryptionCustomerKeyMD5 = "x-amz-server-side-encryption-customer-key-MD5"
|
||||
|
||||
ContainerID = "X-Container-Id"
|
||||
ContainerID = "X-Container-Id"
|
||||
ContainerName = "X-Container-Name"
|
||||
ContainerZone = "X-Container-Zone"
|
||||
|
||||
AccessControlAllowOrigin = "Access-Control-Allow-Origin"
|
||||
AccessControlAllowMethods = "Access-Control-Allow-Methods"
|
||||
|
|
|
@ -1,11 +1,11 @@
|
|||
package layer
|
||||
|
||||
import (
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
|
|
@ -4,8 +4,8 @@ import (
|
|||
"context"
|
||||
errorsStd "errors"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
)
|
||||
|
||||
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
|
||||
|
|
|
@ -4,16 +4,16 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/client"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/container"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -42,14 +42,14 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
|
|||
Name: idCnr.EncodeToString(),
|
||||
}
|
||||
)
|
||||
res, err = n.neoFS.Container(ctx, idCnr)
|
||||
res, err = n.frostFS.Container(ctx, idCnr)
|
||||
if err != nil {
|
||||
log.Error("could not fetch container", zap.Error(err))
|
||||
|
||||
if client.IsErrContainerNotFound(err) {
|
||||
return nil, errors.GetAPIError(errors.ErrNoSuchBucket)
|
||||
}
|
||||
return nil, fmt.Errorf("get neofs container: %w", err)
|
||||
return nil, fmt.Errorf("get frostfs container: %w", err)
|
||||
}
|
||||
|
||||
cnr := *res
|
||||
|
@ -57,6 +57,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
|
|||
info.Owner = cnr.Owner()
|
||||
if domain := container.ReadDomain(cnr); domain.Name() != "" {
|
||||
info.Name = domain.Name()
|
||||
info.Zone = domain.Zone()
|
||||
}
|
||||
info.Created = container.CreatedAt(cnr)
|
||||
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
|
||||
|
@ -84,7 +85,7 @@ func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
|||
res []cid.ID
|
||||
rid = api.GetRequestID(ctx)
|
||||
)
|
||||
res, err = n.neoFS.UserContainers(ctx, own)
|
||||
res, err = n.frostFS.UserContainers(ctx, own)
|
||||
if err != nil {
|
||||
n.log.Error("could not list user containers",
|
||||
zap.String("request_id", rid),
|
||||
|
@ -115,8 +116,9 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
|||
}
|
||||
bktInfo := &data.BucketInfo{
|
||||
Name: p.Name,
|
||||
Zone: v2container.SysAttributeZoneDefault,
|
||||
Owner: ownerID,
|
||||
Created: time.Now(), // this can be a little incorrect since the real time is set later
|
||||
Created: TimeNow(ctx),
|
||||
LocationConstraint: p.LocationConstraint,
|
||||
ObjectLockEnabled: p.ObjectLockEnabled,
|
||||
}
|
||||
|
@ -133,11 +135,12 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
|||
})
|
||||
}
|
||||
|
||||
idCnr, err := n.neoFS.CreateContainer(ctx, PrmContainerCreate{
|
||||
idCnr, err := n.frostFS.CreateContainer(ctx, PrmContainerCreate{
|
||||
Creator: bktInfo.Owner,
|
||||
Policy: p.Policy,
|
||||
Name: p.Name,
|
||||
SessionToken: p.SessionContainerCreation,
|
||||
CreationTime: bktInfo.Created,
|
||||
AdditionalAttributes: attributes,
|
||||
})
|
||||
if err != nil {
|
||||
|
@ -158,9 +161,9 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
|
|||
func (n *layer) setContainerEACLTable(ctx context.Context, idCnr cid.ID, table *eacl.Table, sessionToken *session.Container) error {
|
||||
table.SetCID(idCnr)
|
||||
|
||||
return n.neoFS.SetContainerEACL(ctx, *table, sessionToken)
|
||||
return n.frostFS.SetContainerEACL(ctx, *table, sessionToken)
|
||||
}
|
||||
|
||||
func (n *layer) GetContainerEACL(ctx context.Context, idCnr cid.ID) (*eacl.Table, error) {
|
||||
return n.neoFS.ContainerEACL(ctx, idCnr)
|
||||
return n.frostFS.ContainerEACL(ctx, idCnr)
|
||||
}
|
||||
|
|
|
@ -8,8 +8,8 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -39,8 +39,9 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
|||
prm := PrmObjectCreate{
|
||||
Container: p.BktInfo.CID,
|
||||
Creator: p.BktInfo.Owner,
|
||||
Payload: p.Reader,
|
||||
Payload: &buf,
|
||||
Filepath: p.BktInfo.CORSObjectName(),
|
||||
CreationTime: TimeNow(ctx),
|
||||
CopiesNumber: p.CopiesNumber,
|
||||
}
|
||||
|
||||
|
|
|
@ -7,21 +7,21 @@ import (
|
|||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/container"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/container/acl"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
)
|
||||
|
||||
// PrmContainerCreate groups parameters of NeoFS.CreateContainer operation.
|
||||
// PrmContainerCreate groups parameters of FrostFS.CreateContainer operation.
|
||||
type PrmContainerCreate struct {
|
||||
// NeoFS identifier of the container creator.
|
||||
// FrostFS identifier of the container creator.
|
||||
Creator user.ID
|
||||
|
||||
// Container placement policy.
|
||||
|
@ -30,6 +30,9 @@ type PrmContainerCreate struct {
|
|||
// Name for the container.
|
||||
Name string
|
||||
|
||||
// CreationTime value for Timestamp attribute
|
||||
CreationTime time.Time
|
||||
|
||||
// Token of the container's creation session. Nil means session absence.
|
||||
SessionToken *session.Container
|
||||
|
||||
|
@ -40,7 +43,7 @@ type PrmContainerCreate struct {
|
|||
AdditionalAttributes [][2]string
|
||||
}
|
||||
|
||||
// PrmAuth groups authentication parameters for the NeoFS operation.
|
||||
// PrmAuth groups authentication parameters for the FrostFS operation.
|
||||
type PrmAuth struct {
|
||||
// Bearer token to be used for the operation. Overlaps PrivateKey. Optional.
|
||||
BearerToken *bearer.Token
|
||||
|
@ -49,7 +52,7 @@ type PrmAuth struct {
|
|||
PrivateKey *ecdsa.PrivateKey
|
||||
}
|
||||
|
||||
// PrmObjectRead groups parameters of NeoFS.ReadObject operation.
|
||||
// PrmObjectRead groups parameters of FrostFS.ReadObject operation.
|
||||
type PrmObjectRead struct {
|
||||
// Authentication parameters.
|
||||
PrmAuth
|
||||
|
@ -70,7 +73,7 @@ type PrmObjectRead struct {
|
|||
PayloadRange [2]uint64
|
||||
}
|
||||
|
||||
// ObjectPart represents partially read NeoFS object.
|
||||
// ObjectPart represents partially read FrostFS object.
|
||||
type ObjectPart struct {
|
||||
// Object header with optional in-memory payload part.
|
||||
Head *object.Object
|
||||
|
@ -80,7 +83,7 @@ type ObjectPart struct {
|
|||
Payload io.ReadCloser
|
||||
}
|
||||
|
||||
// PrmObjectCreate groups parameters of NeoFS.CreateObject operation.
|
||||
// PrmObjectCreate groups parameters of FrostFS.CreateObject operation.
|
||||
type PrmObjectCreate struct {
|
||||
// Authentication parameters.
|
||||
PrmAuth
|
||||
|
@ -88,12 +91,15 @@ type PrmObjectCreate struct {
|
|||
// Container to store the object.
|
||||
Container cid.ID
|
||||
|
||||
// NeoFS identifier of the object creator.
|
||||
// FrostFS identifier of the object creator.
|
||||
Creator user.ID
|
||||
|
||||
// Key-value object attributes.
|
||||
Attributes [][2]string
|
||||
|
||||
// Value for Timestamp attribute (optional).
|
||||
CreationTime time.Time
|
||||
|
||||
// List of ids to lock (optional).
|
||||
Locks []oid.ID
|
||||
|
||||
|
@ -110,7 +116,7 @@ type PrmObjectCreate struct {
|
|||
CopiesNumber uint32
|
||||
}
|
||||
|
||||
// PrmObjectDelete groups parameters of NeoFS.DeleteObject operation.
|
||||
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
|
||||
type PrmObjectDelete struct {
|
||||
// Authentication parameters.
|
||||
PrmAuth
|
||||
|
@ -122,12 +128,12 @@ type PrmObjectDelete struct {
|
|||
Object oid.ID
|
||||
}
|
||||
|
||||
// ErrAccessDenied is returned from NeoFS in case of access violation.
|
||||
// ErrAccessDenied is returned from FrostFS in case of access violation.
|
||||
var ErrAccessDenied = errors.New("access denied")
|
||||
|
||||
// NeoFS represents virtual connection to NeoFS network.
|
||||
type NeoFS interface {
|
||||
// CreateContainer creates and saves parameterized container in NeoFS.
|
||||
// FrostFS represents virtual connection to FrostFS network.
|
||||
type FrostFS interface {
|
||||
// CreateContainer creates and saves parameterized container in FrostFS.
|
||||
// It sets 'Timestamp' attribute to the current time.
|
||||
// It returns the ID of the saved container.
|
||||
//
|
||||
|
@ -137,7 +143,7 @@ type NeoFS interface {
|
|||
// prevented the container from being created.
|
||||
CreateContainer(context.Context, PrmContainerCreate) (cid.ID, error)
|
||||
|
||||
// Container reads a container from NeoFS by ID.
|
||||
// Container reads a container from FrostFS by ID.
|
||||
//
|
||||
// It returns exactly one non-nil value. It returns any error encountered which
|
||||
// prevented the container from being read.
|
||||
|
@ -149,26 +155,26 @@ type NeoFS interface {
|
|||
// prevented the containers from being listed.
|
||||
UserContainers(context.Context, user.ID) ([]cid.ID, error)
|
||||
|
||||
// SetContainerEACL saves the eACL table of the container in NeoFS. The
|
||||
// SetContainerEACL saves the eACL table of the container in FrostFS. The
|
||||
// extended ACL is modified within session if session token is not nil.
|
||||
//
|
||||
// It returns any error encountered which prevented the eACL from being saved.
|
||||
SetContainerEACL(context.Context, eacl.Table, *session.Container) error
|
||||
|
||||
// ContainerEACL reads the container eACL from NeoFS by the container ID.
|
||||
// ContainerEACL reads the container eACL from FrostFS by the container ID.
|
||||
//
|
||||
// It returns exactly one non-nil value. It returns any error encountered which
|
||||
// prevented the eACL from being read.
|
||||
ContainerEACL(context.Context, cid.ID) (*eacl.Table, error)
|
||||
|
||||
// DeleteContainer marks the container to be removed from NeoFS by ID.
|
||||
// DeleteContainer marks the container to be removed from FrostFS by ID.
|
||||
// Request is sent within session if the session token is specified.
|
||||
// Successful return does not guarantee actual removal.
|
||||
//
|
||||
// It returns any error encountered which prevented the removal request from being sent.
|
||||
DeleteContainer(context.Context, cid.ID, *session.Container) error
|
||||
|
||||
// ReadObject reads a part of the object from the NeoFS container by identifier.
|
||||
// ReadObject reads a part of the object from the FrostFS container by identifier.
|
||||
// Exact part is returned according to the parameters:
|
||||
// * with header only: empty payload (both in-mem and reader parts are nil);
|
||||
// * with payload only: header is nil (zero range means full payload);
|
||||
|
@ -184,7 +190,7 @@ type NeoFS interface {
|
|||
// prevented the object header from being read.
|
||||
ReadObject(context.Context, PrmObjectRead) (*ObjectPart, error)
|
||||
|
||||
// CreateObject creates and saves a parameterized object in the NeoFS container.
|
||||
// CreateObject creates and saves a parameterized object in the FrostFS container.
|
||||
// It sets 'Timestamp' attribute to the current time.
|
||||
// It returns the ID of the saved object.
|
||||
//
|
||||
|
@ -196,7 +202,7 @@ type NeoFS interface {
|
|||
// prevented the container from being created.
|
||||
CreateObject(context.Context, PrmObjectCreate) (oid.ID, error)
|
||||
|
||||
// DeleteObject marks the object to be removed from the NeoFS container by identifier.
|
||||
// DeleteObject marks the object to be removed from the FrostFS container by identifier.
|
||||
// Successful return does not guarantee actual removal.
|
||||
//
|
||||
// It returns ErrAccessDenied on remove access violation.
|
||||
|
@ -204,11 +210,11 @@ type NeoFS interface {
|
|||
// It returns any error encountered which prevented the removal request from being sent.
|
||||
DeleteObject(context.Context, PrmObjectDelete) error
|
||||
|
||||
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided time.
|
||||
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided now and future time.
|
||||
// Note:
|
||||
// * time must be in the future
|
||||
// * time will be ceil rounded to match epoch
|
||||
// * future time must be after the now
|
||||
// * future time will be ceil rounded to match epoch
|
||||
//
|
||||
// It returns any error encountered which prevented computing epochs.
|
||||
TimeToEpoch(context.Context, time.Time) (uint64, uint64, error)
|
||||
TimeToEpoch(ctx context.Context, now time.Time, future time.Time) (uint64, uint64, error)
|
||||
}
|
|
@ -11,20 +11,20 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -45,7 +45,7 @@ type (
|
|||
}
|
||||
|
||||
layer struct {
|
||||
neoFS NeoFS
|
||||
frostFS FrostFS
|
||||
log *zap.Logger
|
||||
anonKey AnonymousKey
|
||||
resolver BucketResolver
|
||||
|
@ -250,12 +250,12 @@ const (
|
|||
|
||||
AESEncryptionAlgorithm = "AES256"
|
||||
AESKeySize = 32
|
||||
AttributeEncryptionAlgorithm = api.NeoFSSystemMetadataPrefix + "Algorithm"
|
||||
AttributeDecryptedSize = api.NeoFSSystemMetadataPrefix + "Decrypted-Size"
|
||||
AttributeHMACSalt = api.NeoFSSystemMetadataPrefix + "HMAC-Salt"
|
||||
AttributeHMACKey = api.NeoFSSystemMetadataPrefix + "HMAC-Key"
|
||||
AttributeEncryptionAlgorithm = api.FrostFSSystemMetadataPrefix + "Algorithm"
|
||||
AttributeDecryptedSize = api.FrostFSSystemMetadataPrefix + "Decrypted-Size"
|
||||
AttributeHMACSalt = api.FrostFSSystemMetadataPrefix + "HMAC-Salt"
|
||||
AttributeHMACKey = api.FrostFSSystemMetadataPrefix + "HMAC-Key"
|
||||
|
||||
AttributeNeofsCopiesNumber = "neofs-copies-number" // such formate to match X-Amz-Meta-Neofs-Copies-Number header
|
||||
AttributeFrostfsCopiesNumber = "frostfs-copies-number" // such format to match X-Amz-Meta-Frostfs-Copies-Number header
|
||||
)
|
||||
|
||||
func (t *VersionedObject) String() string {
|
||||
|
@ -268,9 +268,9 @@ func (f MsgHandlerFunc) HandleMessage(ctx context.Context, msg *nats.Msg) error
|
|||
|
||||
// NewLayer creates an instance of a layer. It checks credentials
|
||||
// and establishes gRPC connection with the node.
|
||||
func NewLayer(log *zap.Logger, neoFS NeoFS, config *Config) Client {
|
||||
func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) Client {
|
||||
return &layer{
|
||||
neoFS: neoFS,
|
||||
frostFS: frostFS,
|
||||
log: log,
|
||||
anonKey: config.AnonKey,
|
||||
resolver: config.Resolver,
|
||||
|
@ -306,6 +306,15 @@ func IsAuthenticatedRequest(ctx context.Context) bool {
|
|||
return ok
|
||||
}
|
||||
|
||||
// TimeNow returns client time from request or time.Now().
|
||||
func TimeNow(ctx context.Context) time.Time {
|
||||
if now, ok := ctx.Value(api.ClientTime).(time.Time); ok {
|
||||
return now
|
||||
}
|
||||
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// Owner returns owner id from BearerToken (context) or from client owner.
|
||||
func (n *layer) Owner(ctx context.Context) user.ID {
|
||||
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
|
||||
|
@ -368,7 +377,7 @@ func (n *layer) PutBucketACL(ctx context.Context, param *PutBucketACLParams) err
|
|||
}
|
||||
|
||||
// ListBuckets returns all user containers. The name of the bucket is a container
|
||||
// id. Timestamp is omitted since it is not saved in neofs container.
|
||||
// id. Timestamp is omitted since it is not saved in frostfs container.
|
||||
func (n *layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||
return n.containerList(ctx)
|
||||
}
|
||||
|
@ -473,11 +482,27 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.O
|
|||
|
||||
// GetExtendedObjectInfo returns meta information and corresponding info from the tree service about the object.
|
||||
func (n *layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) {
|
||||
var objInfo *data.ExtendedObjectInfo
|
||||
var err error
|
||||
|
||||
if len(p.VersionID) == 0 {
|
||||
return n.headLastVersionIfNotDeleted(ctx, p.BktInfo, p.Object)
|
||||
objInfo, err = n.headLastVersionIfNotDeleted(ctx, p.BktInfo, p.Object)
|
||||
} else {
|
||||
objInfo, err = n.headVersion(ctx, p.BktInfo, p)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return n.headVersion(ctx, p.BktInfo, p)
|
||||
reqInfo := api.GetReqInfo(ctx)
|
||||
n.log.Debug("get object",
|
||||
zap.String("reqId", reqInfo.RequestID),
|
||||
zap.String("bucket", p.BktInfo.Name),
|
||||
zap.Stringer("cid", p.BktInfo.CID),
|
||||
zap.String("object", objInfo.ObjectInfo.Name),
|
||||
zap.Stringer("oid", objInfo.ObjectInfo.ID))
|
||||
|
||||
return objInfo, nil
|
||||
}
|
||||
|
||||
// CopyObject from one bucket into another bucket.
|
||||
|
@ -565,7 +590,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
|||
FilePath: obj.Name,
|
||||
},
|
||||
DeleteMarker: &data.DeleteMarkerInfo{
|
||||
Created: time.Now(),
|
||||
Created: TimeNow(ctx),
|
||||
Owner: n.Owner(ctx),
|
||||
},
|
||||
IsUnversioned: settings.VersioningSuspended(),
|
||||
|
@ -636,7 +661,12 @@ func (n *layer) CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.
|
|||
func (n *layer) ResolveBucket(ctx context.Context, name string) (cid.ID, error) {
|
||||
var cnrID cid.ID
|
||||
if err := cnrID.DecodeString(name); err != nil {
|
||||
return n.resolver.Resolve(ctx, name)
|
||||
if cnrID, err = n.resolver.Resolve(ctx, name); err != nil {
|
||||
return cid.ID{}, err
|
||||
}
|
||||
|
||||
reqInfo := api.GetReqInfo(ctx)
|
||||
n.log.Info("resolve bucket", zap.String("reqId", reqInfo.RequestID), zap.String("bucket", name), zap.Stringer("cid", cnrID))
|
||||
}
|
||||
|
||||
return cnrID, nil
|
||||
|
@ -652,5 +682,5 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
|||
}
|
||||
|
||||
n.cache.DeleteBucket(p.BktInfo.Name)
|
||||
return n.neoFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
||||
return n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
||||
}
|
||||
|
|
|
@ -4,7 +4,8 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -43,10 +44,10 @@ func TestObjectLockAttributes(t *testing.T) {
|
|||
|
||||
expEpoch := false
|
||||
for _, attr := range lockObj.Attributes() {
|
||||
if attr.Key() == AttributeExpirationEpoch {
|
||||
if attr.Key() == object.SysAttributeExpEpoch {
|
||||
expEpoch = true
|
||||
}
|
||||
}
|
||||
|
||||
require.Truef(t, expEpoch, "system header __NEOFS__EXPIRATION_EPOCH presence")
|
||||
require.Truef(t, expEpoch, "system header __SYSTEM__EXPIRATION_EPOCH presence")
|
||||
}
|
||||
|
|
|
@ -11,12 +11,13 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/minio/sio"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -143,7 +144,7 @@ func (n *layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
|
|||
Key: p.Info.Key,
|
||||
UploadID: p.Info.UploadID,
|
||||
Owner: n.Owner(ctx),
|
||||
Created: time.Now(),
|
||||
Created: TimeNow(ctx),
|
||||
Meta: make(map[string]string, metaSize),
|
||||
CopiesNumber: p.CopiesNumber,
|
||||
}
|
||||
|
@ -205,6 +206,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
|||
Creator: bktInfo.Owner,
|
||||
Attributes: make([][2]string, 2),
|
||||
Payload: p.Reader,
|
||||
CreationTime: TimeNow(ctx),
|
||||
CopiesNumber: multipartInfo.CopiesNumber,
|
||||
}
|
||||
|
||||
|
@ -227,6 +229,13 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
|||
return nil, err
|
||||
}
|
||||
|
||||
reqInfo := api.GetReqInfo(ctx)
|
||||
n.log.Debug("upload part",
|
||||
zap.String("reqId", reqInfo.RequestID),
|
||||
zap.String("bucket", bktInfo.Name), zap.Stringer("cid", bktInfo.CID),
|
||||
zap.String("multipart upload", p.Info.UploadID),
|
||||
zap.Int("part number", p.PartNumber), zap.String("object", p.Info.Key), zap.Stringer("oid", id))
|
||||
|
||||
partInfo := &data.PartInfo{
|
||||
Key: p.Info.Key,
|
||||
UploadID: p.Info.UploadID,
|
||||
|
@ -234,7 +243,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
|||
OID: id,
|
||||
Size: decSize,
|
||||
ETag: hex.EncodeToString(hash),
|
||||
Created: time.Now(),
|
||||
Created: prm.CreationTime,
|
||||
}
|
||||
|
||||
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)
|
||||
|
@ -310,7 +319,7 @@ func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
|
|||
return n.uploadPart(ctx, multipartInfo, params)
|
||||
}
|
||||
|
||||
// implements io.Reader of payloads of the object list stored in the NeoFS network.
|
||||
// implements io.Reader of payloads of the object list stored in the FrostFS network.
|
||||
type multiObjectReader struct {
|
||||
ctx context.Context
|
||||
|
||||
|
@ -608,10 +617,24 @@ func (n *layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
|
|||
}
|
||||
|
||||
res := make(map[int]*data.PartInfo, len(parts))
|
||||
for _, part := range parts {
|
||||
partsNumbers := make([]int, len(parts))
|
||||
oids := make([]string, len(parts))
|
||||
for i, part := range parts {
|
||||
res[part.Number] = part
|
||||
partsNumbers[i] = part.Number
|
||||
oids[i] = part.OID.EncodeToString()
|
||||
}
|
||||
|
||||
reqInfo := api.GetReqInfo(ctx)
|
||||
n.log.Debug("part details",
|
||||
zap.String("reqId", reqInfo.RequestID),
|
||||
zap.String("bucket", p.Bkt.Name),
|
||||
zap.Stringer("cid", p.Bkt.CID),
|
||||
zap.String("object", p.Key),
|
||||
zap.String("upload id", p.UploadID),
|
||||
zap.Ints("part numbers", partsNumbers),
|
||||
zap.Strings("oids", oids))
|
||||
|
||||
return multipartInfo, res, nil
|
||||
}
|
||||
|
||||
|
|
|
@ -10,22 +10,22 @@ import (
|
|||
"io"
|
||||
"time"
|
||||
|
||||
objectv2 "github.com/nspcc-dev/neofs-api-go/v2/object"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/checksum"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/container"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
)
|
||||
|
||||
type TestNeoFS struct {
|
||||
NeoFS
|
||||
type TestFrostFS struct {
|
||||
FrostFS
|
||||
|
||||
objects map[string]*object.Object
|
||||
containers map[string]*container.Container
|
||||
|
@ -33,19 +33,19 @@ type TestNeoFS struct {
|
|||
currentEpoch uint64
|
||||
}
|
||||
|
||||
func NewTestNeoFS() *TestNeoFS {
|
||||
return &TestNeoFS{
|
||||
func NewTestFrostFS() *TestFrostFS {
|
||||
return &TestFrostFS{
|
||||
objects: make(map[string]*object.Object),
|
||||
containers: make(map[string]*container.Container),
|
||||
eaclTables: make(map[string]*eacl.Table),
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) CurrentEpoch() uint64 {
|
||||
func (t *TestFrostFS) CurrentEpoch() uint64 {
|
||||
return t.currentEpoch
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) Objects() []*object.Object {
|
||||
func (t *TestFrostFS) Objects() []*object.Object {
|
||||
res := make([]*object.Object, 0, len(t.objects))
|
||||
|
||||
for _, obj := range t.objects {
|
||||
|
@ -55,11 +55,11 @@ func (t *TestNeoFS) Objects() []*object.Object {
|
|||
return res
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) AddObject(key string, obj *object.Object) {
|
||||
func (t *TestFrostFS) AddObject(key string, obj *object.Object) {
|
||||
t.objects[key] = obj
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) ContainerID(name string) (cid.ID, error) {
|
||||
func (t *TestFrostFS) ContainerID(name string) (cid.ID, error) {
|
||||
for id, cnr := range t.containers {
|
||||
if container.Name(*cnr) == name {
|
||||
var cnrID cid.ID
|
||||
|
@ -69,13 +69,18 @@ func (t *TestNeoFS) ContainerID(name string) (cid.ID, error) {
|
|||
return cid.ID{}, fmt.Errorf("not found")
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (cid.ID, error) {
|
||||
func (t *TestFrostFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (cid.ID, error) {
|
||||
var cnr container.Container
|
||||
cnr.Init()
|
||||
cnr.SetOwner(prm.Creator)
|
||||
cnr.SetPlacementPolicy(prm.Policy)
|
||||
cnr.SetBasicACL(prm.BasicACL)
|
||||
container.SetCreationTime(&cnr, time.Now())
|
||||
|
||||
creationTime := prm.CreationTime
|
||||
if creationTime.IsZero() {
|
||||
creationTime = time.Now()
|
||||
}
|
||||
container.SetCreationTime(&cnr, creationTime)
|
||||
|
||||
if prm.Name != "" {
|
||||
var d container.Domain
|
||||
|
@ -101,13 +106,13 @@ func (t *TestNeoFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (
|
|||
return id, nil
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) DeleteContainer(_ context.Context, cnrID cid.ID, _ *session.Container) error {
|
||||
func (t *TestFrostFS) DeleteContainer(_ context.Context, cnrID cid.ID, _ *session.Container) error {
|
||||
delete(t.containers, cnrID.EncodeToString())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) Container(_ context.Context, id cid.ID) (*container.Container, error) {
|
||||
func (t *TestFrostFS) Container(_ context.Context, id cid.ID) (*container.Container, error) {
|
||||
for k, v := range t.containers {
|
||||
if k == id.EncodeToString() {
|
||||
return v, nil
|
||||
|
@ -117,7 +122,7 @@ func (t *TestNeoFS) Container(_ context.Context, id cid.ID) (*container.Containe
|
|||
return nil, fmt.Errorf("container not found %s", id)
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) UserContainers(_ context.Context, _ user.ID) ([]cid.ID, error) {
|
||||
func (t *TestFrostFS) UserContainers(_ context.Context, _ user.ID) ([]cid.ID, error) {
|
||||
var res []cid.ID
|
||||
for k := range t.containers {
|
||||
var idCnr cid.ID
|
||||
|
@ -130,7 +135,7 @@ func (t *TestNeoFS) UserContainers(_ context.Context, _ user.ID) ([]cid.ID, erro
|
|||
return res, nil
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectPart, error) {
|
||||
func (t *TestFrostFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectPart, error) {
|
||||
var addr oid.Address
|
||||
addr.SetContainer(prm.Container)
|
||||
addr.SetObject(prm.Object)
|
||||
|
@ -139,7 +144,7 @@ func (t *TestNeoFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectP
|
|||
|
||||
if obj, ok := t.objects[sAddr]; ok {
|
||||
owner := getOwner(ctx)
|
||||
if !obj.OwnerID().Equals(owner) {
|
||||
if !obj.OwnerID().Equals(owner) && !t.isPublicRead(prm.Container) {
|
||||
return nil, ErrAccessDenied
|
||||
}
|
||||
|
||||
|
@ -159,7 +164,7 @@ func (t *TestNeoFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectP
|
|||
return nil, fmt.Errorf("object not found %s", addr)
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oid.ID, error) {
|
||||
func (t *TestFrostFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oid.ID, error) {
|
||||
b := make([]byte, 32)
|
||||
if _, err := io.ReadFull(rand.Reader, b); err != nil {
|
||||
return oid.ID{}, err
|
||||
|
@ -218,7 +223,7 @@ func (t *TestNeoFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oid.
|
|||
return objID, nil
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
|
||||
func (t *TestFrostFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
|
||||
var addr oid.Address
|
||||
addr.SetContainer(prm.Container)
|
||||
addr.SetObject(prm.Object)
|
||||
|
@ -235,11 +240,11 @@ func (t *TestNeoFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error
|
|||
return nil
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) TimeToEpoch(_ context.Context, futureTime time.Time) (uint64, uint64, error) {
|
||||
return t.currentEpoch, t.currentEpoch + uint64(futureTime.Second()), nil
|
||||
func (t *TestFrostFS) TimeToEpoch(_ context.Context, now, futureTime time.Time) (uint64, uint64, error) {
|
||||
return t.currentEpoch, t.currentEpoch + uint64(futureTime.Sub(now).Seconds()), nil
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) AllObjects(cnrID cid.ID) []oid.ID {
|
||||
func (t *TestFrostFS) AllObjects(cnrID cid.ID) []oid.ID {
|
||||
result := make([]oid.ID, 0, len(t.objects))
|
||||
|
||||
for _, val := range t.objects {
|
||||
|
@ -253,7 +258,7 @@ func (t *TestNeoFS) AllObjects(cnrID cid.ID) []oid.ID {
|
|||
return result
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *session.Container) error {
|
||||
func (t *TestFrostFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *session.Container) error {
|
||||
cnrID, ok := table.CID()
|
||||
if !ok {
|
||||
return errors.New("invalid cid")
|
||||
|
@ -268,7 +273,7 @@ func (t *TestNeoFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *ses
|
|||
return nil
|
||||
}
|
||||
|
||||
func (t *TestNeoFS) ContainerEACL(_ context.Context, cnrID cid.ID) (*eacl.Table, error) {
|
||||
func (t *TestFrostFS) ContainerEACL(_ context.Context, cnrID cid.ID) (*eacl.Table, error) {
|
||||
table, ok := t.eaclTables[cnrID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, errors.New("not found")
|
||||
|
@ -277,6 +282,25 @@ func (t *TestNeoFS) ContainerEACL(_ context.Context, cnrID cid.ID) (*eacl.Table,
|
|||
return table, nil
|
||||
}
|
||||
|
||||
func (t *TestFrostFS) isPublicRead(cnrID cid.ID) bool {
|
||||
table, ok := t.eaclTables[cnrID.EncodeToString()]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, rec := range table.Records() {
|
||||
if rec.Operation() == eacl.OperationGet && len(rec.Filters()) == 0 {
|
||||
for _, trgt := range rec.Targets() {
|
||||
if trgt.Role() == eacl.RoleOthers {
|
||||
return rec.Action() == eacl.ActionAllow
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func getOwner(ctx context.Context) user.ID {
|
||||
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
|
||||
return bearer.ResolveIssuer(*bd.Gate.BearerToken)
|
||||
|
|
|
@ -7,8 +7,8 @@ import (
|
|||
errorsStd "errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -30,6 +30,7 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
|
|||
Creator: p.BktInfo.Owner,
|
||||
Payload: bytes.NewReader(confXML),
|
||||
Filepath: p.BktInfo.NotificationConfigurationObjectName(),
|
||||
CreationTime: TimeNow(ctx),
|
||||
CopiesNumber: p.CopiesNumber,
|
||||
}
|
||||
|
||||
|
|
|
@ -13,17 +13,16 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/minio/sio"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/client"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/panjf2000/ants/v2"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -91,7 +90,7 @@ func (n *layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj
|
|||
|
||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||
|
||||
res, err := n.neoFS.ReadObject(ctx, prm)
|
||||
res, err := n.frostFS.ReadObject(ctx, prm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -99,7 +98,7 @@ func (n *layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj
|
|||
return res.Head, nil
|
||||
}
|
||||
|
||||
// initializes payload reader of the NeoFS object.
|
||||
// initializes payload reader of the FrostFS object.
|
||||
// Zero range corresponds to full payload (panics if only offset is set).
|
||||
func (n *layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Reader, error) {
|
||||
prm := PrmObjectRead{
|
||||
|
@ -111,7 +110,7 @@ func (n *layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Re
|
|||
|
||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, p.bktInfo.Owner)
|
||||
|
||||
res, err := n.neoFS.ReadObject(ctx, prm)
|
||||
res, err := n.frostFS.ReadObject(ctx, prm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -130,7 +129,7 @@ func (n *layer) objectGet(ctx context.Context, bktInfo *data.BucketInfo, objID o
|
|||
|
||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||
|
||||
res, err := n.neoFS.ReadObject(ctx, prm)
|
||||
res, err := n.frostFS.ReadObject(ctx, prm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -183,7 +182,7 @@ func ParseCompletedPartHeader(hdr string) (*Part, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
// PutObject stores object into NeoFS, took payload from io.Reader.
|
||||
// PutObject stores object into FrostFS, took payload from io.Reader.
|
||||
func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error) {
|
||||
owner := n.Owner(ctx)
|
||||
|
||||
|
@ -234,6 +233,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
|||
PayloadSize: uint64(p.Size),
|
||||
Filepath: p.Object,
|
||||
Payload: r,
|
||||
CreationTime: TimeNow(ctx),
|
||||
CopiesNumber: p.CopiesNumber,
|
||||
}
|
||||
|
||||
|
@ -248,6 +248,12 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
|||
return nil, err
|
||||
}
|
||||
|
||||
reqInfo := api.GetReqInfo(ctx)
|
||||
n.log.Debug("put object",
|
||||
zap.String("reqId", reqInfo.RequestID),
|
||||
zap.String("bucket", p.BktInfo.Name), zap.Stringer("cid", p.BktInfo.CID),
|
||||
zap.String("object", p.Object), zap.Stringer("oid", id))
|
||||
|
||||
newVersion.OID = id
|
||||
newVersion.ETag = hex.EncodeToString(hash)
|
||||
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
|
||||
|
@ -281,7 +287,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
|||
Bucket: p.BktInfo.Name,
|
||||
Name: p.Object,
|
||||
Size: p.Size,
|
||||
Created: time.Now(),
|
||||
Created: prm.CreationTime,
|
||||
Headers: p.Header,
|
||||
ContentType: p.Header[api.ContentType],
|
||||
HashSum: newVersion.ETag,
|
||||
|
@ -383,7 +389,7 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
|
|||
return extObjInfo, nil
|
||||
}
|
||||
|
||||
// objectDelete puts tombstone object into neofs.
|
||||
// objectDelete puts tombstone object into frostfs.
|
||||
func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) error {
|
||||
prm := PrmObjectDelete{
|
||||
Container: bktInfo.CID,
|
||||
|
@ -394,10 +400,10 @@ func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idOb
|
|||
|
||||
n.cache.DeleteObject(newAddress(bktInfo.CID, idObj))
|
||||
|
||||
return n.neoFS.DeleteObject(ctx, prm)
|
||||
return n.frostFS.DeleteObject(ctx, prm)
|
||||
}
|
||||
|
||||
// objectPutAndHash prepare auth parameters and invoke neofs.CreateObject.
|
||||
// objectPutAndHash prepare auth parameters and invoke frostfs.CreateObject.
|
||||
// Returns object ID and payload sha256 hash.
|
||||
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (oid.ID, []byte, error) {
|
||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||
|
@ -405,8 +411,11 @@ func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktIn
|
|||
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
|
||||
hash.Write(buf)
|
||||
})
|
||||
id, err := n.neoFS.CreateObject(ctx, prm)
|
||||
return id, hash.Sum(nil), err
|
||||
id, err := n.frostFS.CreateObject(ctx, prm)
|
||||
if err != nil {
|
||||
return oid.ID{}, nil, err
|
||||
}
|
||||
return id, hash.Sum(nil), nil
|
||||
}
|
||||
|
||||
// ListObjectsV1 returns objects in a bucket for requests of Version 1.
|
||||
|
@ -574,10 +583,10 @@ func (n *layer) initWorkerPool(ctx context.Context, size int, p allObjectParams,
|
|||
wg.Add(1)
|
||||
err = pool.Submit(func() {
|
||||
defer wg.Done()
|
||||
oi := n.objectInfoFromObjectsCacheOrNeoFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter)
|
||||
oi := n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter)
|
||||
if oi == nil {
|
||||
// try to get object again
|
||||
if oi = n.objectInfoFromObjectsCacheOrNeoFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter); oi == nil {
|
||||
if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter); oi == nil {
|
||||
// form object info with data that the tree node contains
|
||||
oi = getPartialObjectInfo(p.Bucket, node)
|
||||
}
|
||||
|
@ -643,14 +652,14 @@ func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo,
|
|||
for _, nodeVersion := range nodeVersions {
|
||||
oi := &data.ObjectInfo{}
|
||||
|
||||
if nodeVersion.IsDeleteMarker() { // delete marker does not match any object in NeoFS
|
||||
if nodeVersion.IsDeleteMarker() { // delete marker does not match any object in FrostFS
|
||||
oi.ID = nodeVersion.OID
|
||||
oi.Name = nodeVersion.FilePath
|
||||
oi.Owner = nodeVersion.DeleteMarker.Owner
|
||||
oi.Created = nodeVersion.DeleteMarker.Created
|
||||
oi.IsDeleteMarker = true
|
||||
} else {
|
||||
if oi = n.objectInfoFromObjectsCacheOrNeoFS(ctx, bkt, nodeVersion, prefix, delimiter); oi == nil {
|
||||
if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, bkt, nodeVersion, prefix, delimiter); oi == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
@ -674,7 +683,7 @@ func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo,
|
|||
|
||||
func IsSystemHeader(key string) bool {
|
||||
_, ok := api.SystemMetadata[key]
|
||||
return ok || strings.HasPrefix(key, api.NeoFSSystemMetadataPrefix)
|
||||
return ok || strings.HasPrefix(key, api.FrostFSSystemMetadataPrefix)
|
||||
}
|
||||
|
||||
func shouldSkip(node *data.NodeVersion, p allObjectParams, existed map[string]struct{}) bool {
|
||||
|
@ -731,7 +740,7 @@ func triageExtendedObjects(allObjects []*data.ExtendedObjectInfo) (prefixes []st
|
|||
return
|
||||
}
|
||||
|
||||
func (n *layer) objectInfoFromObjectsCacheOrNeoFS(ctx context.Context, bktInfo *data.BucketInfo, node *data.NodeVersion, prefix, delimiter string) (oi *data.ObjectInfo) {
|
||||
func (n *layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo *data.BucketInfo, node *data.NodeVersion, prefix, delimiter string) (oi *data.ObjectInfo) {
|
||||
if oiDir := tryDirectory(bktInfo, node, prefix, delimiter); oiDir != nil {
|
||||
return oiDir
|
||||
}
|
||||
|
|
|
@ -9,14 +9,14 @@ import (
|
|||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
const (
|
||||
AttributeComplianceMode = ".s3-compliance-mode"
|
||||
AttributeExpirationEpoch = "__NEOFS__EXPIRATION_EPOCH"
|
||||
AttributeComplianceMode = ".s3-compliance-mode"
|
||||
)
|
||||
|
||||
type PutLockInfoParams struct {
|
||||
|
@ -32,7 +32,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
|
|||
// sometimes node version can be provided from executing context
|
||||
// if not, then receive node version from tree service
|
||||
if versionNode == nil {
|
||||
versionNode, err = n.getNodeVersionFromCacheOrNeofs(ctx, p.ObjVersion)
|
||||
versionNode, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -100,7 +100,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
|
|||
return nil
|
||||
}
|
||||
|
||||
func (n *layer) getNodeVersionFromCacheOrNeofs(ctx context.Context, objVersion *ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
|
||||
func (n *layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion *ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
|
||||
// check cache if node version is stored inside extendedObjectVersion
|
||||
nodeVersion = n.getNodeVersionFromCache(n.Owner(ctx), objVersion)
|
||||
if nodeVersion == nil {
|
||||
|
@ -116,6 +116,7 @@ func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
|
|||
Container: bktInfo.CID,
|
||||
Creator: bktInfo.Owner,
|
||||
Locks: []oid.ID{objID},
|
||||
CreationTime: TimeNow(ctx),
|
||||
CopiesNumber: copiesNumber,
|
||||
}
|
||||
|
||||
|
@ -227,7 +228,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
|
|||
)
|
||||
|
||||
if lock.Retention != nil {
|
||||
if _, expEpoch, err = n.neoFS.TimeToEpoch(ctx, lock.Retention.Until); err != nil {
|
||||
if _, expEpoch, err = n.frostFS.TimeToEpoch(ctx, TimeNow(ctx), lock.Retention.Until); err != nil {
|
||||
return nil, fmt.Errorf("fetch time to epoch: %w", err)
|
||||
}
|
||||
|
||||
|
@ -237,7 +238,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
|
|||
}
|
||||
|
||||
if lock.LegalHold != nil && lock.LegalHold.Enabled {
|
||||
// todo: (@KirillovDenis) reconsider this when NeoFS will support Legal Hold https://github.com/nspcc-dev/neofs-contract/issues/247
|
||||
// todo: (@KirillovDenis) reconsider this when FrostFS will support Legal Hold https://git.frostfs.info/TrueCloudLab/frostfs-contract/issues/2
|
||||
// Currently lock object must have an expiration epoch.
|
||||
// Besides we need to override retention expiration epoch since legal hold cannot be deleted yet.
|
||||
expEpoch = math.MaxUint64
|
||||
|
@ -245,7 +246,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
|
|||
|
||||
if expEpoch != 0 {
|
||||
result = append(result, [2]string{
|
||||
AttributeExpirationEpoch, strconv.FormatUint(expEpoch, 10),
|
||||
object.SysAttributeExpEpoch, strconv.FormatUint(expEpoch, 10),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
@ -4,11 +4,13 @@ import (
|
|||
"context"
|
||||
errorsStd "errors"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type GetObjectTaggingParams struct {
|
||||
|
@ -38,7 +40,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
|
|||
|
||||
nodeVersion := p.NodeVersion
|
||||
if nodeVersion == nil {
|
||||
nodeVersion, err = n.getNodeVersionFromCacheOrNeofs(ctx, p.ObjectVersion)
|
||||
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
@ -65,7 +67,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
|
|||
func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (nodeVersion *data.NodeVersion, err error) {
|
||||
nodeVersion = p.NodeVersion
|
||||
if nodeVersion == nil {
|
||||
nodeVersion, err = n.getNodeVersionFromCacheOrNeofs(ctx, p.ObjectVersion)
|
||||
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -175,6 +177,14 @@ func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (
|
|||
return nil, errors.GetAPIError(errors.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
if err == nil && version != nil && !version.IsDeleteMarker() {
|
||||
reqInfo := api.GetReqInfo(ctx)
|
||||
n.log.Debug("target details",
|
||||
zap.String("reqId", reqInfo.RequestID),
|
||||
zap.String("bucket", objVersion.BktInfo.Name), zap.Stringer("cid", objVersion.BktInfo.CID),
|
||||
zap.String("object", objVersion.ObjectName), zap.Stringer("oid", version.OID))
|
||||
}
|
||||
|
||||
return version, err
|
||||
}
|
||||
|
||||
|
|
|
@ -6,8 +6,8 @@ import (
|
|||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
type TreeServiceMock struct {
|
||||
|
@ -109,11 +109,32 @@ func (t *TreeServiceMock) PutNotificationConfigurationNode(ctx context.Context,
|
|||
}
|
||||
|
||||
func (t *TreeServiceMock) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
|
||||
panic("implement me")
|
||||
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return oid.ID{}, nil
|
||||
}
|
||||
|
||||
node, ok := systemMap["cors"]
|
||||
if !ok {
|
||||
return oid.ID{}, nil
|
||||
}
|
||||
|
||||
return node.OID, nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) {
|
||||
panic("implement me")
|
||||
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
systemMap = make(map[string]*data.BaseNodeVersion)
|
||||
}
|
||||
|
||||
systemMap["cors"] = &data.BaseNodeVersion{
|
||||
OID: objID,
|
||||
}
|
||||
|
||||
t.system[bktInfo.CID.EncodeToString()] = systemMap
|
||||
|
||||
return oid.ID{}, ErrNoNodeToRemove
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
|
||||
|
|
|
@ -4,8 +4,8 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
// TreeService provide interface to interact with tree service using s3 data models.
|
||||
|
@ -24,7 +24,7 @@ type TreeService interface {
|
|||
GetNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
||||
|
||||
// PutNotificationConfigurationNode puts a node to a system tree
|
||||
// and returns objectID of a previous notif config which must be deleted in NeoFS.
|
||||
// and returns objectID of a previous notif config which must be deleted in FrostFS.
|
||||
//
|
||||
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||
PutNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
|
||||
|
@ -34,12 +34,12 @@ type TreeService interface {
|
|||
// If object id is not found returns ErrNodeNotFound error.
|
||||
GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
||||
|
||||
// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in NeoFS.
|
||||
// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in FrostFS.
|
||||
//
|
||||
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||
PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
|
||||
|
||||
// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in NeoFS.
|
||||
// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in FrostFS.
|
||||
//
|
||||
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||
DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
||||
|
@ -69,7 +69,7 @@ type TreeService interface {
|
|||
GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error)
|
||||
|
||||
// AddPart puts a node to a system tree as a child of appropriate multipart upload
|
||||
// and returns objectID of a previous part which must be deleted in NeoFS.
|
||||
// and returns objectID of a previous part which must be deleted in FrostFS.
|
||||
//
|
||||
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)
|
||||
|
|
|
@ -9,11 +9,11 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
)
|
||||
|
||||
type (
|
||||
|
|
|
@ -6,11 +6,11 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/checksum"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
|
|
@ -4,7 +4,7 @@ import (
|
|||
"context"
|
||||
"sort"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
)
|
||||
|
||||
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
||||
|
|
|
@ -5,14 +5,14 @@ import (
|
|||
"context"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
bearertest "github.com/nspcc-dev/neofs-sdk-go/bearer/test"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -113,7 +113,7 @@ func (tc *testContext) checkListObjects(ids ...oid.ID) {
|
|||
}
|
||||
|
||||
func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
|
||||
for _, obj := range tc.testNeoFS.Objects() {
|
||||
for _, obj := range tc.testFrostFS.Objects() {
|
||||
id, _ := obj.ID()
|
||||
if id.Equals(objID) {
|
||||
return obj
|
||||
|
@ -123,12 +123,12 @@ func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
|
|||
}
|
||||
|
||||
type testContext struct {
|
||||
t *testing.T
|
||||
ctx context.Context
|
||||
layer Client
|
||||
bktInfo *data.BucketInfo
|
||||
obj string
|
||||
testNeoFS *TestNeoFS
|
||||
t *testing.T
|
||||
ctx context.Context
|
||||
layer Client
|
||||
bktInfo *data.BucketInfo
|
||||
obj string
|
||||
testFrostFS *TestFrostFS
|
||||
}
|
||||
|
||||
func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
||||
|
@ -146,7 +146,7 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
|||
GateKey: key.PublicKey(),
|
||||
},
|
||||
})
|
||||
tp := NewTestNeoFS()
|
||||
tp := NewTestFrostFS()
|
||||
|
||||
bktName := "testbucket1"
|
||||
bktID, err := tp.CreateContainer(ctx, PrmContainerCreate{
|
||||
|
@ -176,9 +176,9 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
|||
Owner: owner,
|
||||
CID: bktID,
|
||||
},
|
||||
obj: "obj1",
|
||||
t: t,
|
||||
testNeoFS: tp,
|
||||
obj: "obj1",
|
||||
t: t,
|
||||
testFrostFS: tp,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -4,7 +4,7 @@ import (
|
|||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
)
|
||||
|
||||
type (
|
||||
|
|
305
api/metrics.go
Normal file
305
api/metrics.go
Normal file
|
@ -0,0 +1,305 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
type RequestType int
|
||||
|
||||
const (
|
||||
UNKNOWNRequest RequestType = iota
|
||||
HEADRequest RequestType = iota
|
||||
PUTRequest RequestType = iota
|
||||
LISTRequest RequestType = iota
|
||||
GETRequest RequestType = iota
|
||||
DELETERequest RequestType = iota
|
||||
)
|
||||
|
||||
func (t RequestType) String() string {
|
||||
switch t {
|
||||
case 1:
|
||||
return "HEAD"
|
||||
case 2:
|
||||
return "PUT"
|
||||
case 3:
|
||||
return "LIST"
|
||||
case 4:
|
||||
return "GET"
|
||||
case 5:
|
||||
return "DELETE"
|
||||
default:
|
||||
return "Unknown"
|
||||
}
|
||||
}
|
||||
|
||||
func RequestTypeFromAPI(api string) RequestType {
|
||||
switch api {
|
||||
case "Options", "HeadObject", "HeadBucket":
|
||||
return HEADRequest
|
||||
case "CreateMultipartUpload", "UploadPartCopy", "UploadPart", "CompleteMultipartUpload",
|
||||
"PutObjectACL", "PutObjectTagging", "CopyObject", "PutObjectRetention", "PutObjectLegalHold",
|
||||
"PutObject", "PutBucketCors", "PutBucketACL", "PutBucketLifecycle", "PutBucketEncryption",
|
||||
"PutBucketPolicy", "PutBucketObjectLockConfig", "PutBucketTagging", "PutBucketVersioning",
|
||||
"PutBucketNotification", "CreateBucket", "PostObject":
|
||||
return PUTRequest
|
||||
case "ListObjectParts", "ListMultipartUploads", "ListObjectsV2M", "ListObjectsV2", "ListBucketVersions",
|
||||
"ListObjectsV1", "ListBuckets":
|
||||
return LISTRequest
|
||||
case "GetObjectACL", "GetObjectTagging", "SelectObjectContent", "GetObjectRetention", "getobjectlegalhold",
|
||||
"GetObjectAttributes", "GetObject", "GetBucketLocation", "GetBucketPolicy",
|
||||
"GetBucketLifecycle", "GetBucketEncryption", "GetBucketCors", "GetBucketACL",
|
||||
"GetBucketWebsite", "GetBucketAccelerate", "GetBucketRequestPayment", "GetBucketLogging",
|
||||
"GetBucketReplication", "GetBucketTagging", "GetBucketObjectLockConfig",
|
||||
"GetBucketVersioning", "GetBucketNotification", "ListenBucketNotification":
|
||||
return GETRequest
|
||||
case "AbortMultipartUpload", "DeleteObjectTagging", "DeleteObject", "DeleteBucketCors",
|
||||
"DeleteBucketWebsite", "DeleteBucketTagging", "DeleteMultipleObjects", "DeleteBucketPolicy",
|
||||
"DeleteBucketLifecycle", "DeleteBucketEncryption", "DeleteBucket":
|
||||
return DELETERequest
|
||||
default:
|
||||
return UNKNOWNRequest
|
||||
}
|
||||
}
|
||||
|
||||
type (
|
||||
// HTTPAPIStats holds statistics information about
|
||||
// the API given in the requests.
|
||||
HTTPAPIStats struct {
|
||||
apiStats map[string]int
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
UsersStat interface {
|
||||
Update(user, bucket, cnrID string, reqType RequestType, in, out uint64)
|
||||
}
|
||||
|
||||
// HTTPStats holds statistics information about
|
||||
// HTTP requests made by all clients.
|
||||
HTTPStats struct {
|
||||
currentS3Requests HTTPAPIStats
|
||||
totalS3Requests HTTPAPIStats
|
||||
totalS3Errors HTTPAPIStats
|
||||
|
||||
totalInputBytes uint64
|
||||
totalOutputBytes uint64
|
||||
}
|
||||
|
||||
readCounter struct {
|
||||
io.ReadCloser
|
||||
countBytes uint64
|
||||
}
|
||||
|
||||
writeCounter struct {
|
||||
http.ResponseWriter
|
||||
countBytes uint64
|
||||
}
|
||||
|
||||
responseWrapper struct {
|
||||
sync.Once
|
||||
http.ResponseWriter
|
||||
|
||||
statusCode int
|
||||
startTime time.Time
|
||||
}
|
||||
)
|
||||
|
||||
const systemPath = "/system"
|
||||
|
||||
var (
|
||||
httpStatsMetric = new(HTTPStats)
|
||||
httpRequestsDuration = prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Name: "frostfs_s3_request_seconds",
|
||||
Help: "Time taken by requests served by current FrostFS S3 Gate instance",
|
||||
Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10},
|
||||
},
|
||||
[]string{"api"},
|
||||
)
|
||||
)
|
||||
|
||||
// Collects HTTP metrics for FrostFS S3 Gate in Prometheus specific format
|
||||
// and sends to the given channel.
|
||||
func collectHTTPMetrics(ch chan<- prometheus.Metric) {
|
||||
for api, value := range httpStatsMetric.currentS3Requests.Load() {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName("frostfs_s3", "requests", "current"),
|
||||
"Total number of running s3 requests in current FrostFS S3 Gate instance",
|
||||
[]string{"api"}, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(value),
|
||||
api,
|
||||
)
|
||||
}
|
||||
|
||||
for api, value := range httpStatsMetric.totalS3Requests.Load() {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName("frostfs_s3", "requests", "total"),
|
||||
"Total number of s3 requests in current FrostFS S3 Gate instance",
|
||||
[]string{"api"}, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(value),
|
||||
api,
|
||||
)
|
||||
}
|
||||
|
||||
for api, value := range httpStatsMetric.totalS3Errors.Load() {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName("frostfs_s3", "errors", "total"),
|
||||
"Total number of s3 errors in current FrostFS S3 Gate instance",
|
||||
[]string{"api"}, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(value),
|
||||
api,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// CIDResolveFunc is a func to resolve CID in Stats handler.
|
||||
type CIDResolveFunc func(ctx context.Context, reqInfo *ReqInfo) (cnrID string)
|
||||
|
||||
// Stats is a handler that update metrics.
|
||||
func Stats(f http.HandlerFunc, resolveCID CIDResolveFunc, usersStat UsersStat) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
reqInfo := GetReqInfo(r.Context())
|
||||
|
||||
httpStatsMetric.currentS3Requests.Inc(reqInfo.API)
|
||||
defer httpStatsMetric.currentS3Requests.Dec(reqInfo.API)
|
||||
|
||||
in := &readCounter{ReadCloser: r.Body}
|
||||
out := &writeCounter{ResponseWriter: w}
|
||||
|
||||
r.Body = in
|
||||
|
||||
statsWriter := &responseWrapper{
|
||||
ResponseWriter: out,
|
||||
startTime: time.Now(),
|
||||
}
|
||||
|
||||
f(statsWriter, r)
|
||||
|
||||
// Time duration in secs since the call started.
|
||||
// We don't need to do nanosecond precision here
|
||||
// simply for the fact that it is not human-readable.
|
||||
durationSecs := time.Since(statsWriter.startTime).Seconds()
|
||||
|
||||
user := resolveUser(r.Context())
|
||||
cnrID := resolveCID(r.Context(), reqInfo)
|
||||
usersStat.Update(user, reqInfo.BucketName, cnrID, RequestTypeFromAPI(reqInfo.API), in.countBytes, out.countBytes)
|
||||
|
||||
code := statsWriter.statusCode
|
||||
// A successful request has a 2xx response code
|
||||
successReq := code >= http.StatusOK && code < http.StatusMultipleChoices
|
||||
if !strings.HasSuffix(r.URL.Path, systemPath) {
|
||||
httpStatsMetric.totalS3Requests.Inc(reqInfo.API)
|
||||
if !successReq && code != 0 {
|
||||
httpStatsMetric.totalS3Errors.Inc(reqInfo.API)
|
||||
}
|
||||
}
|
||||
|
||||
if r.Method == http.MethodGet {
|
||||
// Increment the prometheus http request response histogram with appropriate label
|
||||
httpRequestsDuration.With(prometheus.Labels{"api": reqInfo.API}).Observe(durationSecs)
|
||||
}
|
||||
|
||||
atomic.AddUint64(&httpStatsMetric.totalInputBytes, in.countBytes)
|
||||
atomic.AddUint64(&httpStatsMetric.totalOutputBytes, out.countBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func resolveUser(ctx context.Context) string {
|
||||
user := "anon"
|
||||
if bd, ok := ctx.Value(BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
|
||||
user = bearer.ResolveIssuer(*bd.Gate.BearerToken).String()
|
||||
}
|
||||
return user
|
||||
}
|
||||
|
||||
// Inc increments the api stats counter.
|
||||
func (stats *HTTPAPIStats) Inc(api string) {
|
||||
if stats == nil {
|
||||
return
|
||||
}
|
||||
stats.Lock()
|
||||
defer stats.Unlock()
|
||||
if stats.apiStats == nil {
|
||||
stats.apiStats = make(map[string]int)
|
||||
}
|
||||
stats.apiStats[api]++
|
||||
}
|
||||
|
||||
// Dec increments the api stats counter.
|
||||
func (stats *HTTPAPIStats) Dec(api string) {
|
||||
if stats == nil {
|
||||
return
|
||||
}
|
||||
stats.Lock()
|
||||
defer stats.Unlock()
|
||||
if val, ok := stats.apiStats[api]; ok && val > 0 {
|
||||
stats.apiStats[api]--
|
||||
}
|
||||
}
|
||||
|
||||
// Load returns the recorded stats.
|
||||
func (stats *HTTPAPIStats) Load() map[string]int {
|
||||
stats.Lock()
|
||||
defer stats.Unlock()
|
||||
var apiStats = make(map[string]int, len(stats.apiStats))
|
||||
for k, v := range stats.apiStats {
|
||||
apiStats[k] = v
|
||||
}
|
||||
return apiStats
|
||||
}
|
||||
|
||||
func (st *HTTPStats) getInputBytes() uint64 {
|
||||
return atomic.LoadUint64(&st.totalInputBytes)
|
||||
}
|
||||
|
||||
func (st *HTTPStats) getOutputBytes() uint64 {
|
||||
return atomic.LoadUint64(&st.totalOutputBytes)
|
||||
}
|
||||
|
||||
// WriteHeader -- writes http status code.
|
||||
func (w *responseWrapper) WriteHeader(code int) {
|
||||
w.Do(func() {
|
||||
w.statusCode = code
|
||||
w.ResponseWriter.WriteHeader(code)
|
||||
})
|
||||
}
|
||||
|
||||
// Flush -- calls the underlying Flush.
|
||||
func (w *responseWrapper) Flush() {
|
||||
if f, ok := w.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *writeCounter) Flush() {
|
||||
if f, ok := w.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *writeCounter) Write(p []byte) (int, error) {
|
||||
n, err := w.ResponseWriter.Write(p)
|
||||
atomic.AddUint64(&w.countBytes, uint64(n))
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *readCounter) Read(p []byte) (int, error) {
|
||||
n, err := r.ReadCloser.Read(p)
|
||||
atomic.AddUint64(&r.countBytes, uint64(n))
|
||||
return n, err
|
||||
}
|
|
@ -1,229 +0,0 @@
|
|||
package metrics
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
type (
|
||||
// HTTPAPIStats holds statistics information about
|
||||
// the API given in the requests.
|
||||
HTTPAPIStats struct {
|
||||
apiStats map[string]int
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
// HTTPStats holds statistics information about
|
||||
// HTTP requests made by all clients.
|
||||
HTTPStats struct {
|
||||
currentS3Requests HTTPAPIStats
|
||||
totalS3Requests HTTPAPIStats
|
||||
totalS3Errors HTTPAPIStats
|
||||
|
||||
totalInputBytes uint64
|
||||
totalOutputBytes uint64
|
||||
}
|
||||
|
||||
readCounter struct {
|
||||
io.ReadCloser
|
||||
countBytes uint64
|
||||
}
|
||||
|
||||
writeCounter struct {
|
||||
http.ResponseWriter
|
||||
countBytes uint64
|
||||
}
|
||||
|
||||
responseWrapper struct {
|
||||
sync.Once
|
||||
http.ResponseWriter
|
||||
|
||||
statusCode int
|
||||
startTime time.Time
|
||||
}
|
||||
)
|
||||
|
||||
const systemPath = "/system"
|
||||
|
||||
var (
|
||||
httpStatsMetric = new(HTTPStats)
|
||||
httpRequestsDuration = prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Name: "neofs_s3_request_seconds",
|
||||
Help: "Time taken by requests served by current NeoFS S3 Gate instance",
|
||||
Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10},
|
||||
},
|
||||
[]string{"api"},
|
||||
)
|
||||
)
|
||||
|
||||
// Collects HTTP metrics for NeoFS S3 Gate in Prometheus specific format
|
||||
// and sends to the given channel.
|
||||
func collectHTTPMetrics(ch chan<- prometheus.Metric) {
|
||||
for api, value := range httpStatsMetric.currentS3Requests.Load() {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName("neofs_s3", "requests", "current"),
|
||||
"Total number of running s3 requests in current NeoFS S3 Gate instance",
|
||||
[]string{"api"}, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(value),
|
||||
api,
|
||||
)
|
||||
}
|
||||
|
||||
for api, value := range httpStatsMetric.totalS3Requests.Load() {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName("neofs_s3", "requests", "total"),
|
||||
"Total number of s3 requests in current NeoFS S3 Gate instance",
|
||||
[]string{"api"}, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(value),
|
||||
api,
|
||||
)
|
||||
}
|
||||
|
||||
for api, value := range httpStatsMetric.totalS3Errors.Load() {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName("neofs_s3", "errors", "total"),
|
||||
"Total number of s3 errors in current NeoFS S3 Gate instance",
|
||||
[]string{"api"}, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(value),
|
||||
api,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// APIStats wraps http handler for api with basic statistics collection.
|
||||
func APIStats(api string, f http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
httpStatsMetric.currentS3Requests.Inc(api)
|
||||
defer httpStatsMetric.currentS3Requests.Dec(api)
|
||||
|
||||
in := &readCounter{ReadCloser: r.Body}
|
||||
out := &writeCounter{ResponseWriter: w}
|
||||
|
||||
r.Body = in
|
||||
|
||||
statsWriter := &responseWrapper{
|
||||
ResponseWriter: out,
|
||||
startTime: time.Now(),
|
||||
}
|
||||
|
||||
f.ServeHTTP(statsWriter, r)
|
||||
|
||||
// Time duration in secs since the call started.
|
||||
// We don't need to do nanosecond precision here
|
||||
// simply for the fact that it is not human readable.
|
||||
durationSecs := time.Since(statsWriter.startTime).Seconds()
|
||||
|
||||
httpStatsMetric.updateStats(api, statsWriter, r, durationSecs)
|
||||
|
||||
atomic.AddUint64(&httpStatsMetric.totalInputBytes, in.countBytes)
|
||||
atomic.AddUint64(&httpStatsMetric.totalOutputBytes, out.countBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// Inc increments the api stats counter.
|
||||
func (stats *HTTPAPIStats) Inc(api string) {
|
||||
if stats == nil {
|
||||
return
|
||||
}
|
||||
stats.Lock()
|
||||
defer stats.Unlock()
|
||||
if stats.apiStats == nil {
|
||||
stats.apiStats = make(map[string]int)
|
||||
}
|
||||
stats.apiStats[api]++
|
||||
}
|
||||
|
||||
// Dec increments the api stats counter.
|
||||
func (stats *HTTPAPIStats) Dec(api string) {
|
||||
if stats == nil {
|
||||
return
|
||||
}
|
||||
stats.Lock()
|
||||
defer stats.Unlock()
|
||||
if val, ok := stats.apiStats[api]; ok && val > 0 {
|
||||
stats.apiStats[api]--
|
||||
}
|
||||
}
|
||||
|
||||
// Load returns the recorded stats.
|
||||
func (stats *HTTPAPIStats) Load() map[string]int {
|
||||
stats.Lock()
|
||||
defer stats.Unlock()
|
||||
var apiStats = make(map[string]int, len(stats.apiStats))
|
||||
for k, v := range stats.apiStats {
|
||||
apiStats[k] = v
|
||||
}
|
||||
return apiStats
|
||||
}
|
||||
|
||||
func (st *HTTPStats) getInputBytes() uint64 {
|
||||
return atomic.LoadUint64(&st.totalInputBytes)
|
||||
}
|
||||
|
||||
func (st *HTTPStats) getOutputBytes() uint64 {
|
||||
return atomic.LoadUint64(&st.totalOutputBytes)
|
||||
}
|
||||
|
||||
// Update statistics from http request and response data.
|
||||
func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) {
|
||||
var code int
|
||||
|
||||
if res, ok := w.(*responseWrapper); ok {
|
||||
code = res.statusCode
|
||||
}
|
||||
|
||||
// A successful request has a 2xx response code
|
||||
successReq := code >= http.StatusOK && code < http.StatusMultipleChoices
|
||||
|
||||
if !strings.HasSuffix(r.URL.Path, systemPath) {
|
||||
st.totalS3Requests.Inc(api)
|
||||
if !successReq && code != 0 {
|
||||
st.totalS3Errors.Inc(api)
|
||||
}
|
||||
}
|
||||
|
||||
if r.Method == http.MethodGet {
|
||||
// Increment the prometheus http request response histogram with appropriate label
|
||||
httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(durationSecs)
|
||||
}
|
||||
}
|
||||
|
||||
// WriteHeader -- writes http status code.
|
||||
func (w *responseWrapper) WriteHeader(code int) {
|
||||
w.Do(func() {
|
||||
w.statusCode = code
|
||||
w.ResponseWriter.WriteHeader(code)
|
||||
})
|
||||
}
|
||||
|
||||
// Flush -- calls the underlying Flush.
|
||||
func (w *responseWrapper) Flush() {
|
||||
if f, ok := w.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *writeCounter) Write(p []byte) (int, error) {
|
||||
n, err := w.ResponseWriter.Write(p)
|
||||
atomic.AddUint64(&w.countBytes, uint64(n))
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *readCounter) Read(p []byte) (int, error) {
|
||||
n, err := r.ReadCloser.Read(p)
|
||||
atomic.AddUint64(&r.countBytes, uint64(n))
|
||||
return n, err
|
||||
}
|
|
@ -1,7 +1,7 @@
|
|||
package metrics
|
||||
package api
|
||||
|
||||
import (
|
||||
"github.com/nspcc-dev/neofs-s3-gw/internal/version"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
|
@ -12,9 +12,9 @@ type stats struct {
|
|||
var (
|
||||
versionInfo = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "neofs_s3",
|
||||
Namespace: "frostfs_s3",
|
||||
Name: "version_info",
|
||||
Help: "Version of current NeoFS S3 Gate instance",
|
||||
Help: "Version of current FrostFS S3 Gate instance",
|
||||
},
|
||||
[]string{
|
||||
// current version
|
||||
|
@ -23,7 +23,7 @@ var (
|
|||
)
|
||||
|
||||
statsMetrics = &stats{
|
||||
desc: prometheus.NewDesc("neofs_s3_stats", "Statistics exposed by NeoFS S3 Gate instance", nil, nil),
|
||||
desc: prometheus.NewDesc("frostfs_s3_stats", "Statistics exposed by FrostFS S3 Gate instance", nil, nil),
|
||||
}
|
||||
)
|
||||
|
||||
|
@ -37,8 +37,8 @@ func collectNetworkMetrics(ch chan<- prometheus.Metric) {
|
|||
// Network Sent/Received Bytes (Outbound)
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName("neofs_s3", "tx", "bytes_total"),
|
||||
"Total number of bytes sent by current NeoFS S3 Gate instance",
|
||||
prometheus.BuildFQName("frostfs_s3", "tx", "bytes_total"),
|
||||
"Total number of bytes sent by current FrostFS S3 Gate instance",
|
||||
nil, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(httpStatsMetric.getInputBytes()),
|
||||
|
@ -46,8 +46,8 @@ func collectNetworkMetrics(ch chan<- prometheus.Metric) {
|
|||
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName("neofs_s3", "rx", "bytes_total"),
|
||||
"Total number of bytes received by current NeoFS S3 Gate instance",
|
||||
prometheus.BuildFQName("frostfs_s3", "rx", "bytes_total"),
|
||||
"Total number of bytes received by current FrostFS S3 Gate instance",
|
||||
nil, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(httpStatsMetric.getOutputBytes()),
|
|
@ -7,9 +7,9 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/handler"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -61,7 +61,7 @@ type (
|
|||
|
||||
EventRecord struct {
|
||||
EventVersion string `json:"eventVersion"`
|
||||
EventSource string `json:"eventSource"` // neofs:s3
|
||||
EventSource string `json:"eventSource"` // frostfs:s3
|
||||
AWSRegion string `json:"awsRegion,omitempty"` // empty
|
||||
EventTime time.Time `json:"eventTime"`
|
||||
EventName string `json:"eventName"`
|
||||
|
@ -197,11 +197,11 @@ func (c *Controller) SendNotifications(topics map[string]string, p *handler.Send
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c *Controller) SendTestNotification(topic, bucketName, requestID, HostID string) error {
|
||||
func (c *Controller) SendTestNotification(topic, bucketName, requestID, HostID string, now time.Time) error {
|
||||
event := &TestEvent{
|
||||
Service: "NeoFS S3",
|
||||
Service: "FrostFS S3",
|
||||
Event: "s3:TestEvent",
|
||||
Time: time.Now(),
|
||||
Time: now,
|
||||
Bucket: bucketName,
|
||||
RequestID: requestID,
|
||||
HostID: HostID,
|
||||
|
@ -220,9 +220,9 @@ func prepareEvent(p *handler.SendNotificationParams) *Event {
|
|||
Records: []EventRecord{
|
||||
{
|
||||
EventVersion: EventVersion21,
|
||||
EventSource: "neofs:s3",
|
||||
EventSource: "frostfs:s3",
|
||||
AWSRegion: "",
|
||||
EventTime: time.Now(),
|
||||
EventTime: p.Time,
|
||||
EventName: p.Event,
|
||||
UserIdentity: UserIdentity{
|
||||
PrincipalID: p.User,
|
||||
|
|
|
@ -45,7 +45,7 @@ type (
|
|||
// Key used for Get/SetReqInfo.
|
||||
type contextKeyType string
|
||||
|
||||
const ctxRequestInfo = contextKeyType("NeoFS-S3-GW")
|
||||
const ctxRequestInfo = contextKeyType("FrostFS-S3-GW")
|
||||
|
||||
var (
|
||||
// De-facto standard header keys.
|
||||
|
|
|
@ -6,8 +6,9 @@ import (
|
|||
"fmt"
|
||||
"sync"
|
||||
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/ns"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -18,9 +19,9 @@ const (
|
|||
// ErrNoResolvers returns when trying to resolve container without any resolver.
|
||||
var ErrNoResolvers = errors.New("no resolvers")
|
||||
|
||||
// NeoFS represents virtual connection to the NeoFS network.
|
||||
type NeoFS interface {
|
||||
// SystemDNS reads system DNS network parameters of the NeoFS.
|
||||
// FrostFS represents virtual connection to the FrostFS network.
|
||||
type FrostFS interface {
|
||||
// SystemDNS reads system DNS network parameters of the FrostFS.
|
||||
//
|
||||
// It returns exactly on non-zero value. It returns any error encountered
|
||||
// which prevented the parameter from being read.
|
||||
|
@ -28,7 +29,7 @@ type NeoFS interface {
|
|||
}
|
||||
|
||||
type Config struct {
|
||||
NeoFS NeoFS
|
||||
FrostFS FrostFS
|
||||
RPCAddress string
|
||||
}
|
||||
|
||||
|
@ -133,7 +134,7 @@ func (r *BucketResolver) equals(resolverNames []string) bool {
|
|||
func newResolver(name string, cfg *Config) (*Resolver, error) {
|
||||
switch name {
|
||||
case DNSResolver:
|
||||
return NewDNSResolver(cfg.NeoFS)
|
||||
return NewDNSResolver(cfg.FrostFS)
|
||||
case NNSResolver:
|
||||
return NewNNSResolver(cfg.RPCAddress)
|
||||
default:
|
||||
|
@ -141,17 +142,17 @@ func newResolver(name string, cfg *Config) (*Resolver, error) {
|
|||
}
|
||||
}
|
||||
|
||||
func NewDNSResolver(neoFS NeoFS) (*Resolver, error) {
|
||||
if neoFS == nil {
|
||||
func NewDNSResolver(frostFS FrostFS) (*Resolver, error) {
|
||||
if frostFS == nil {
|
||||
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
|
||||
}
|
||||
|
||||
var dns ns.DNS
|
||||
|
||||
resolveFunc := func(ctx context.Context, name string) (cid.ID, error) {
|
||||
domain, err := neoFS.SystemDNS(ctx)
|
||||
domain, err := frostFS.SystemDNS(ctx)
|
||||
if err != nil {
|
||||
return cid.ID{}, fmt.Errorf("read system DNS parameter of the NeoFS: %w", err)
|
||||
return cid.ID{}, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err)
|
||||
}
|
||||
|
||||
domain = name + "." + domain
|
||||
|
@ -180,7 +181,10 @@ func NewNNSResolver(address string) (*Resolver, error) {
|
|||
}
|
||||
|
||||
resolveFunc := func(_ context.Context, name string) (cid.ID, error) {
|
||||
cnrID, err := nns.ResolveContainerName(name)
|
||||
var d container.Domain
|
||||
d.SetName(name)
|
||||
|
||||
cnrID, err := nns.ResolveContainerDomain(d)
|
||||
if err != nil {
|
||||
return cid.ID{}, fmt.Errorf("couldn't resolve container '%s': %w", name, err)
|
||||
}
|
||||
|
|
|
@ -7,9 +7,9 @@ import (
|
|||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||
"github.com/google/uuid"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/internal/version"
|
||||
)
|
||||
|
||||
type (
|
||||
|
@ -117,7 +117,7 @@ func WriteErrorResponse(w http.ResponseWriter, reqInfo *ReqInfo, err error) int
|
|||
code = e.HTTPStatusCode
|
||||
|
||||
switch e.Code {
|
||||
case "SlowDown", "XNeoFSServerNotInitialized", "XNeoFSReadQuorum", "XNeoFSWriteQuorum":
|
||||
case "SlowDown", "XFrostFSServerNotInitialized", "XFrostFSReadQuorum", "XFrostFSWriteQuorum":
|
||||
// Set retry-after header to indicate user-agents to retry request after 120secs.
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
||||
w.Header().Set(hdrRetryAfter, "120")
|
||||
|
@ -133,6 +133,13 @@ func WriteErrorResponse(w http.ResponseWriter, reqInfo *ReqInfo, err error) int
|
|||
return code
|
||||
}
|
||||
|
||||
// WriteErrorResponseNoHeader writes XML encoded error to the response body.
|
||||
func WriteErrorResponseNoHeader(w http.ResponseWriter, reqInfo *ReqInfo, err error) {
|
||||
errorResponse := getAPIErrorResponse(reqInfo, err)
|
||||
encodedErrorResponse := EncodeResponse(errorResponse)
|
||||
WriteResponseBody(w, encodedErrorResponse)
|
||||
}
|
||||
|
||||
// If none of the http routes match respond with appropriate errors.
|
||||
func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
||||
desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)
|
||||
|
@ -172,6 +179,11 @@ func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType
|
|||
return
|
||||
}
|
||||
|
||||
WriteResponseBody(w, response)
|
||||
}
|
||||
|
||||
// WriteResponseBody writes response into w.
|
||||
func WriteResponseBody(w http.ResponseWriter, response []byte) {
|
||||
_, _ = w.Write(response)
|
||||
if flusher, ok := w.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
|
@ -188,13 +200,30 @@ func EncodeResponse(response interface{}) []byte {
|
|||
return bytesBuffer.Bytes()
|
||||
}
|
||||
|
||||
// EncodeResponseNoHeader encodes response without setting xml.Header.
|
||||
// Should be used with periodicXMLWriter which sends xml.Header to the client
|
||||
// with whitespaces to keep connection alive.
|
||||
func EncodeResponseNoHeader(response interface{}) []byte {
|
||||
var bytesBuffer bytes.Buffer
|
||||
_ = xml.NewEncoder(&bytesBuffer).Encode(response)
|
||||
return bytesBuffer.Bytes()
|
||||
}
|
||||
|
||||
// EncodeToResponse encodes the response into ResponseWriter.
|
||||
func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
if _, err := w.Write(xmlHeader); err != nil {
|
||||
return fmt.Errorf("write headers: %w", err)
|
||||
} else if err = xml.NewEncoder(w).Encode(response); err != nil {
|
||||
}
|
||||
|
||||
return EncodeToResponseNoHeader(w, response)
|
||||
}
|
||||
|
||||
// EncodeToResponseNoHeader encodes the response into ResponseWriter without
|
||||
// header status.
|
||||
func EncodeToResponseNoHeader(w http.ResponseWriter, response interface{}) error {
|
||||
if err := xml.NewEncoder(w).Encode(response); err != nil {
|
||||
return fmt.Errorf("encode xml response: %w", err)
|
||||
}
|
||||
|
||||
|
|
281
api/router.go
281
api/router.go
|
@ -5,10 +5,10 @@ import (
|
|||
"net/http"
|
||||
"sync"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"github.com/google/uuid"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/metrics"
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
|
@ -82,6 +82,8 @@ type (
|
|||
AbortMultipartUploadHandler(http.ResponseWriter, *http.Request)
|
||||
ListPartsHandler(w http.ResponseWriter, r *http.Request)
|
||||
ListMultipartUploadsHandler(http.ResponseWriter, *http.Request)
|
||||
|
||||
ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error)
|
||||
}
|
||||
|
||||
// mimeType represents various MIME types used in API responses.
|
||||
|
@ -106,7 +108,7 @@ const (
|
|||
MimeXML mimeType = "application/xml"
|
||||
)
|
||||
|
||||
var _ = logErrorResponse
|
||||
var _ = logSuccessResponse
|
||||
|
||||
func (lrw *logResponseWriter) WriteHeader(code int) {
|
||||
lrw.Do(func() {
|
||||
|
@ -115,6 +117,12 @@ func (lrw *logResponseWriter) WriteHeader(code int) {
|
|||
})
|
||||
}
|
||||
|
||||
func (lrw *logResponseWriter) Flush() {
|
||||
if f, ok := lrw.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
func setRequestID(h http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// generate random UUIDv4
|
||||
|
@ -145,7 +153,37 @@ func appendCORS(handler Handler) mux.MiddlewareFunc {
|
|||
}
|
||||
}
|
||||
|
||||
func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc {
|
||||
// BucketResolveFunc is a func to resolve bucket info by name.
|
||||
type BucketResolveFunc func(ctx context.Context, bucket string) (*data.BucketInfo, error)
|
||||
|
||||
// metricsMiddleware wraps http handler for api with basic statistics collection.
|
||||
func metricsMiddleware(log *zap.Logger, resolveBucket BucketResolveFunc, usersStat UsersStat) mux.MiddlewareFunc {
|
||||
return func(h http.Handler) http.Handler {
|
||||
return Stats(h.ServeHTTP, resolveCID(log, resolveBucket), usersStat)
|
||||
}
|
||||
}
|
||||
|
||||
// resolveCID forms CIDResolveFunc using BucketResolveFunc.
|
||||
func resolveCID(log *zap.Logger, resolveBucket BucketResolveFunc) CIDResolveFunc {
|
||||
return func(ctx context.Context, reqInfo *ReqInfo) (cnrID string) {
|
||||
if reqInfo.BucketName == "" || reqInfo.API == "CreateBucket" || reqInfo.API == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
bktInfo, err := resolveBucket(ctx, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
log.Debug("failed to resolve CID",
|
||||
zap.String("request_id", reqInfo.RequestID), zap.String("method", reqInfo.API),
|
||||
zap.String("bucket", reqInfo.BucketName), zap.String("object", reqInfo.ObjectName),
|
||||
zap.Error(err))
|
||||
return ""
|
||||
}
|
||||
|
||||
return bktInfo.CID.EncodeToString()
|
||||
}
|
||||
}
|
||||
|
||||
func logSuccessResponse(l *zap.Logger) mux.MiddlewareFunc {
|
||||
return func(h http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
lw := &logResponseWriter{ResponseWriter: w}
|
||||
|
@ -161,6 +199,7 @@ func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc {
|
|||
|
||||
l.Info("call method",
|
||||
zap.Int("status", lw.statusCode),
|
||||
zap.String("host", r.Host),
|
||||
zap.String("request_id", GetRequestID(r.Context())),
|
||||
zap.String("method", mux.CurrentRoute(r).GetName()),
|
||||
zap.String("bucket", reqInfo.BucketName),
|
||||
|
@ -182,21 +221,49 @@ func GetRequestID(v interface{}) string {
|
|||
}
|
||||
}
|
||||
|
||||
func setErrorAPI(apiName string, h http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := SetReqInfo(r.Context(), &ReqInfo{API: apiName})
|
||||
h.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
}
|
||||
|
||||
// attachErrorHandler set NotFoundHandler and MethodNotAllowedHandler for mux.Router.
|
||||
func attachErrorHandler(api *mux.Router, log *zap.Logger, h Handler, center auth.Center, usersStat UsersStat) {
|
||||
middlewares := []mux.MiddlewareFunc{
|
||||
AuthMiddleware(log, center),
|
||||
metricsMiddleware(log, h.ResolveBucket, usersStat),
|
||||
}
|
||||
|
||||
var errorHandler http.Handler = http.HandlerFunc(errorResponseHandler)
|
||||
for i := len(middlewares) - 1; i >= 0; i-- {
|
||||
errorHandler = middlewares[i](errorHandler)
|
||||
}
|
||||
|
||||
// If none of the routes match, add default error handler routes
|
||||
api.NotFoundHandler = setErrorAPI("NotFound", errorHandler)
|
||||
api.MethodNotAllowedHandler = setErrorAPI("MethodNotAllowed", errorHandler)
|
||||
}
|
||||
|
||||
// Attach adds S3 API handlers from h to r for domains with m client limit using
|
||||
// center authentication and log logger.
|
||||
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) {
|
||||
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger, usersStat UsersStat) {
|
||||
api := r.PathPrefix(SlashSeparator).Subrouter()
|
||||
|
||||
api.Use(
|
||||
// -- prepare request
|
||||
setRequestID,
|
||||
|
||||
// Attach user authentication for all S3 routes.
|
||||
AuthMiddleware(log, center),
|
||||
|
||||
metricsMiddleware(log, h.ResolveBucket, usersStat),
|
||||
|
||||
// -- logging error requests
|
||||
logErrorResponse(log),
|
||||
logSuccessResponse(log),
|
||||
)
|
||||
|
||||
// Attach user authentication for all S3 routes.
|
||||
AttachUserAuth(api, center, log)
|
||||
attachErrorHandler(api, log, h, center, usersStat)
|
||||
|
||||
buckets := make([]*mux.Router, 0, len(domains)+1)
|
||||
buckets = append(buckets, api.PathPrefix("/{bucket}").Subrouter())
|
||||
|
@ -212,277 +279,327 @@ func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center aut
|
|||
// -- append CORS headers to a response for
|
||||
appendCORS(h),
|
||||
)
|
||||
bucket.Methods(http.MethodOptions).HandlerFunc(m.Handle(metrics.APIStats("preflight", h.Preflight))).Name("Options")
|
||||
bucket.Methods(http.MethodOptions).HandlerFunc(
|
||||
m.Handle(h.Preflight)).
|
||||
Name("Options")
|
||||
bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("headobject", h.HeadObjectHandler))).Name("HeadObject")
|
||||
m.Handle(h.HeadObjectHandler)).
|
||||
Name("HeadObject")
|
||||
// CopyObjectPart
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(metrics.APIStats("uploadpartcopy", h.UploadPartCopy))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(
|
||||
m.Handle(h.UploadPartCopy)).
|
||||
Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
|
||||
Name("UploadPartCopy")
|
||||
// PutObjectPart
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("uploadpart", h.UploadPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
|
||||
m.Handle(h.UploadPartHandler)).
|
||||
Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
|
||||
Name("UploadPart")
|
||||
// ListParts
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("listobjectparts", h.ListPartsHandler))).Queries("uploadId", "{uploadId:.*}").
|
||||
m.Handle(h.ListPartsHandler)).
|
||||
Queries("uploadId", "{uploadId:.*}").
|
||||
Name("ListObjectParts")
|
||||
// CompleteMultipartUpload
|
||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("completemutipartupload", h.CompleteMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}").
|
||||
m.Handle(h.CompleteMultipartUploadHandler)).
|
||||
Queries("uploadId", "{uploadId:.*}").
|
||||
Name("CompleteMultipartUpload")
|
||||
// CreateMultipartUpload
|
||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("createmultipartupload", h.CreateMultipartUploadHandler))).Queries("uploads", "").
|
||||
m.Handle(h.CreateMultipartUploadHandler)).
|
||||
Queries("uploads", "").
|
||||
Name("CreateMultipartUpload")
|
||||
// AbortMultipartUpload
|
||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("abortmultipartupload", h.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}").
|
||||
m.Handle(h.AbortMultipartUploadHandler)).
|
||||
Queries("uploadId", "{uploadId:.*}").
|
||||
Name("AbortMultipartUpload")
|
||||
// ListMultipartUploads
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("listmultipartuploads", h.ListMultipartUploadsHandler))).Queries("uploads", "").
|
||||
m.Handle(h.ListMultipartUploadsHandler)).
|
||||
Queries("uploads", "").
|
||||
Name("ListMultipartUploads")
|
||||
// GetObjectACL -- this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getobjectacl", h.GetObjectACLHandler))).Queries("acl", "").
|
||||
m.Handle(h.GetObjectACLHandler)).
|
||||
Queries("acl", "").
|
||||
Name("GetObjectACL")
|
||||
// PutObjectACL -- this is a dummy call.
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putobjectacl", h.PutObjectACLHandler))).Queries("acl", "").
|
||||
m.Handle(h.PutObjectACLHandler)).
|
||||
Queries("acl", "").
|
||||
Name("PutObjectACL")
|
||||
// GetObjectTagging
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getobjecttagging", h.GetObjectTaggingHandler))).Queries("tagging", "").
|
||||
m.Handle(h.GetObjectTaggingHandler)).
|
||||
Queries("tagging", "").
|
||||
Name("GetObjectTagging")
|
||||
// PutObjectTagging
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putobjecttagging", h.PutObjectTaggingHandler))).Queries("tagging", "").
|
||||
m.Handle(h.PutObjectTaggingHandler)).
|
||||
Queries("tagging", "").
|
||||
Name("PutObjectTagging")
|
||||
// DeleteObjectTagging
|
||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("deleteobjecttagging", h.DeleteObjectTaggingHandler))).Queries("tagging", "").
|
||||
m.Handle(h.DeleteObjectTaggingHandler)).
|
||||
Queries("tagging", "").
|
||||
Name("DeleteObjectTagging")
|
||||
// SelectObjectContent
|
||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("selectobjectcontent", h.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2").
|
||||
m.Handle(h.SelectObjectContentHandler)).
|
||||
Queries("select", "").Queries("select-type", "2").
|
||||
Name("SelectObjectContent")
|
||||
// GetObjectRetention
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getobjectretention", h.GetObjectRetentionHandler))).Queries("retention", "").
|
||||
m.Handle(h.GetObjectRetentionHandler)).
|
||||
Queries("retention", "").
|
||||
Name("GetObjectRetention")
|
||||
// GetObjectLegalHold
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getobjectlegalhold", h.GetObjectLegalHoldHandler))).Queries("legal-hold", "").
|
||||
m.Handle(h.GetObjectLegalHoldHandler)).
|
||||
Queries("legal-hold", "").
|
||||
Name("GetObjectLegalHold")
|
||||
// GetObjectAttributes
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getobjectattributes", h.GetObjectAttributesHandler))).Queries("attributes", "").
|
||||
m.Handle(h.GetObjectAttributesHandler)).
|
||||
Queries("attributes", "").
|
||||
Name("GetObjectAttributes")
|
||||
// GetObject
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getobject", h.GetObjectHandler))).
|
||||
m.Handle(h.GetObjectHandler)).
|
||||
Name("GetObject")
|
||||
// CopyObject
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(metrics.APIStats("copyobject", h.CopyObjectHandler))).
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(
|
||||
m.Handle(h.CopyObjectHandler)).
|
||||
Name("CopyObject")
|
||||
// PutObjectRetention
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putobjectretention", h.PutObjectRetentionHandler))).Queries("retention", "").
|
||||
m.Handle(h.PutObjectRetentionHandler)).
|
||||
Queries("retention", "").
|
||||
Name("PutObjectRetention")
|
||||
// PutObjectLegalHold
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putobjectlegalhold", h.PutObjectLegalHoldHandler))).Queries("legal-hold", "").
|
||||
m.Handle(h.PutObjectLegalHoldHandler)).
|
||||
Queries("legal-hold", "").
|
||||
Name("PutObjectLegalHold")
|
||||
|
||||
// PutObject
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putobject", h.PutObjectHandler))).
|
||||
m.Handle(h.PutObjectHandler)).
|
||||
Name("PutObject")
|
||||
// DeleteObject
|
||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("deleteobject", h.DeleteObjectHandler))).
|
||||
m.Handle(h.DeleteObjectHandler)).
|
||||
Name("DeleteObject")
|
||||
|
||||
// Bucket operations
|
||||
// GetBucketLocation
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketlocation", h.GetBucketLocationHandler))).Queries("location", "").
|
||||
m.Handle(h.GetBucketLocationHandler)).
|
||||
Queries("location", "").
|
||||
Name("GetBucketLocation")
|
||||
// GetBucketPolicy
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketpolicy", h.GetBucketPolicyHandler))).Queries("policy", "").
|
||||
m.Handle(h.GetBucketPolicyHandler)).
|
||||
Queries("policy", "").
|
||||
Name("GetBucketPolicy")
|
||||
// GetBucketLifecycle
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", "").
|
||||
m.Handle(h.GetBucketLifecycleHandler)).
|
||||
Queries("lifecycle", "").
|
||||
Name("GetBucketLifecycle")
|
||||
// GetBucketEncryption
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketencryption", h.GetBucketEncryptionHandler))).Queries("encryption", "").
|
||||
m.Handle(h.GetBucketEncryptionHandler)).
|
||||
Queries("encryption", "").
|
||||
Name("GetBucketEncryption")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketcors", h.GetBucketCorsHandler))).Queries("cors", "").
|
||||
m.Handle(h.GetBucketCorsHandler)).
|
||||
Queries("cors", "").
|
||||
Name("GetBucketCors")
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putbucketcors", h.PutBucketCorsHandler))).Queries("cors", "").
|
||||
m.Handle(h.PutBucketCorsHandler)).
|
||||
Queries("cors", "").
|
||||
Name("PutBucketCors")
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("deletebucketcors", h.DeleteBucketCorsHandler))).Queries("cors", "").
|
||||
m.Handle(h.DeleteBucketCorsHandler)).
|
||||
Queries("cors", "").
|
||||
Name("DeleteBucketCors")
|
||||
// Dummy Bucket Calls
|
||||
// GetBucketACL -- this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketacl", h.GetBucketACLHandler))).Queries("acl", "").
|
||||
m.Handle(h.GetBucketACLHandler)).
|
||||
Queries("acl", "").
|
||||
Name("GetBucketACL")
|
||||
// PutBucketACL -- this is a dummy call.
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putbucketacl", h.PutBucketACLHandler))).Queries("acl", "").
|
||||
m.Handle(h.PutBucketACLHandler)).
|
||||
Queries("acl", "").
|
||||
Name("PutBucketACL")
|
||||
// GetBucketWebsiteHandler -- this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketwebsite", h.GetBucketWebsiteHandler))).Queries("website", "").
|
||||
m.Handle(h.GetBucketWebsiteHandler)).
|
||||
Queries("website", "").
|
||||
Name("GetBucketWebsite")
|
||||
// GetBucketAccelerateHandler -- this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketaccelerate", h.GetBucketAccelerateHandler))).Queries("accelerate", "").
|
||||
m.Handle(h.GetBucketAccelerateHandler)).
|
||||
Queries("accelerate", "").
|
||||
Name("GetBucketAccelerate")
|
||||
// GetBucketRequestPaymentHandler -- this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketrequestpayment", h.GetBucketRequestPaymentHandler))).Queries("requestPayment", "").
|
||||
m.Handle(h.GetBucketRequestPaymentHandler)).
|
||||
Queries("requestPayment", "").
|
||||
Name("GetBucketRequestPayment")
|
||||
// GetBucketLoggingHandler -- this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketlogging", h.GetBucketLoggingHandler))).Queries("logging", "").
|
||||
m.Handle(h.GetBucketLoggingHandler)).
|
||||
Queries("logging", "").
|
||||
Name("GetBucketLogging")
|
||||
// GetBucketLifecycleHandler -- this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", "").
|
||||
Name("GetBucketLifecycle")
|
||||
// GetBucketReplicationHandler -- this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketreplication", h.GetBucketReplicationHandler))).Queries("replication", "").
|
||||
m.Handle(h.GetBucketReplicationHandler)).
|
||||
Queries("replication", "").
|
||||
Name("GetBucketReplication")
|
||||
// GetBucketTaggingHandler
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbuckettagging", h.GetBucketTaggingHandler))).Queries("tagging", "").
|
||||
m.Handle(h.GetBucketTaggingHandler)).
|
||||
Queries("tagging", "").
|
||||
Name("GetBucketTagging")
|
||||
// DeleteBucketWebsiteHandler
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("deletebucketwebsite", h.DeleteBucketWebsiteHandler))).Queries("website", "").
|
||||
m.Handle(h.DeleteBucketWebsiteHandler)).
|
||||
Queries("website", "").
|
||||
Name("DeleteBucketWebsite")
|
||||
// DeleteBucketTaggingHandler
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("deletebuckettagging", h.DeleteBucketTaggingHandler))).Queries("tagging", "").
|
||||
m.Handle(h.DeleteBucketTaggingHandler)).
|
||||
Queries("tagging", "").
|
||||
Name("DeleteBucketTagging")
|
||||
|
||||
// GetBucketObjectLockConfig
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketobjectlockconfiguration", h.GetBucketObjectLockConfigHandler))).Queries("object-lock", "").
|
||||
m.Handle(h.GetBucketObjectLockConfigHandler)).
|
||||
Queries("object-lock", "").
|
||||
Name("GetBucketObjectLockConfig")
|
||||
// GetBucketVersioning
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketversioning", h.GetBucketVersioningHandler))).Queries("versioning", "").
|
||||
m.Handle(h.GetBucketVersioningHandler)).
|
||||
Queries("versioning", "").
|
||||
Name("GetBucketVersioning")
|
||||
// GetBucketNotification
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("getbucketnotification", h.GetBucketNotificationHandler))).Queries("notification", "").
|
||||
m.Handle(h.GetBucketNotificationHandler)).
|
||||
Queries("notification", "").
|
||||
Name("GetBucketNotification")
|
||||
// ListenBucketNotification
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(metrics.APIStats("listenbucketnotification", h.ListenBucketNotificationHandler)).Queries("events", "{events:.*}").
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(h.ListenBucketNotificationHandler).
|
||||
Queries("events", "{events:.*}").
|
||||
Name("ListenBucketNotification")
|
||||
// ListObjectsV2M
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("listobjectsv2M", h.ListObjectsV2MHandler))).Queries("list-type", "2", "metadata", "true").
|
||||
m.Handle(h.ListObjectsV2MHandler)).
|
||||
Queries("list-type", "2", "metadata", "true").
|
||||
Name("ListObjectsV2M")
|
||||
// ListObjectsV2
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("listobjectsv2", h.ListObjectsV2Handler))).Queries("list-type", "2").
|
||||
m.Handle(h.ListObjectsV2Handler)).
|
||||
Queries("list-type", "2").
|
||||
Name("ListObjectsV2")
|
||||
// ListBucketVersions
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("listbucketversions", h.ListBucketObjectVersionsHandler))).Queries("versions", "").
|
||||
m.Handle(h.ListBucketObjectVersionsHandler)).
|
||||
Queries("versions", "").
|
||||
Name("ListBucketVersions")
|
||||
// ListObjectsV1 (Legacy)
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("listobjectsv1", h.ListObjectsV1Handler))).
|
||||
m.Handle(h.ListObjectsV1Handler)).
|
||||
Name("ListObjectsV1")
|
||||
// PutBucketLifecycle
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putbucketlifecycle", h.PutBucketLifecycleHandler))).Queries("lifecycle", "").
|
||||
m.Handle(h.PutBucketLifecycleHandler)).
|
||||
Queries("lifecycle", "").
|
||||
Name("PutBucketLifecycle")
|
||||
// PutBucketEncryption
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putbucketencryption", h.PutBucketEncryptionHandler))).Queries("encryption", "").
|
||||
m.Handle(h.PutBucketEncryptionHandler)).
|
||||
Queries("encryption", "").
|
||||
Name("PutBucketEncryption")
|
||||
|
||||
// PutBucketPolicy
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putbucketpolicy", h.PutBucketPolicyHandler))).Queries("policy", "").
|
||||
m.Handle(h.PutBucketPolicyHandler)).
|
||||
Queries("policy", "").
|
||||
Name("PutBucketPolicy")
|
||||
|
||||
// PutBucketObjectLockConfig
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putbucketobjectlockconfig", h.PutBucketObjectLockConfigHandler))).Queries("object-lock", "").
|
||||
m.Handle(h.PutBucketObjectLockConfigHandler)).
|
||||
Queries("object-lock", "").
|
||||
Name("PutBucketObjectLockConfig")
|
||||
// PutBucketTaggingHandler
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putbuckettagging", h.PutBucketTaggingHandler))).Queries("tagging", "").
|
||||
m.Handle(h.PutBucketTaggingHandler)).
|
||||
Queries("tagging", "").
|
||||
Name("PutBucketTagging")
|
||||
// PutBucketVersioning
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putbucketversioning", h.PutBucketVersioningHandler))).Queries("versioning", "").
|
||||
m.Handle(h.PutBucketVersioningHandler)).
|
||||
Queries("versioning", "").
|
||||
Name("PutBucketVersioning")
|
||||
// PutBucketNotification
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("putbucketnotification", h.PutBucketNotificationHandler))).Queries("notification", "").
|
||||
m.Handle(h.PutBucketNotificationHandler)).
|
||||
Queries("notification", "").
|
||||
Name("PutBucketNotification")
|
||||
// CreateBucket
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("createbucket", h.CreateBucketHandler))).
|
||||
m.Handle(h.CreateBucketHandler)).
|
||||
Name("CreateBucket")
|
||||
// HeadBucket
|
||||
bucket.Methods(http.MethodHead).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("headbucket", h.HeadBucketHandler))).
|
||||
m.Handle(h.HeadBucketHandler)).
|
||||
Name("HeadBucket")
|
||||
// PostPolicy
|
||||
bucket.Methods(http.MethodPost).HeadersRegexp(hdrContentType, "multipart/form-data*").HandlerFunc(
|
||||
m.Handle(metrics.APIStats("postobject", h.PostObject))).
|
||||
m.Handle(h.PostObject)).
|
||||
Name("PostObject")
|
||||
// DeleteMultipleObjects
|
||||
bucket.Methods(http.MethodPost).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("deletemultipleobjects", h.DeleteMultipleObjectsHandler))).Queries("delete", "").
|
||||
m.Handle(h.DeleteMultipleObjectsHandler)).
|
||||
Queries("delete", "").
|
||||
Name("DeleteMultipleObjects")
|
||||
// DeleteBucketPolicy
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("deletebucketpolicy", h.DeleteBucketPolicyHandler))).Queries("policy", "").
|
||||
m.Handle(h.DeleteBucketPolicyHandler)).
|
||||
Queries("policy", "").
|
||||
Name("DeleteBucketPolicy")
|
||||
// DeleteBucketLifecycle
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("deletebucketlifecycle", h.DeleteBucketLifecycleHandler))).Queries("lifecycle", "").
|
||||
m.Handle(h.DeleteBucketLifecycleHandler)).
|
||||
Queries("lifecycle", "").
|
||||
Name("DeleteBucketLifecycle")
|
||||
// DeleteBucketEncryption
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("deletebucketencryption", h.DeleteBucketEncryptionHandler))).Queries("encryption", "").
|
||||
m.Handle(h.DeleteBucketEncryptionHandler)).
|
||||
Queries("encryption", "").
|
||||
Name("DeleteBucketEncryption")
|
||||
// DeleteBucket
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("deletebucket", h.DeleteBucketHandler))).
|
||||
m.Handle(h.DeleteBucketHandler)).
|
||||
Name("DeleteBucket")
|
||||
}
|
||||
// Root operation
|
||||
|
||||
// ListBuckets
|
||||
api.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("listbuckets", h.ListBucketsHandler))).
|
||||
m.Handle(h.ListBucketsHandler)).
|
||||
Name("ListBuckets")
|
||||
|
||||
// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
|
||||
// than failing with UnknownAPIRequest we simply handle it for now.
|
||||
api.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
|
||||
m.Handle(metrics.APIStats("listbuckets", h.ListBucketsHandler))).
|
||||
m.Handle(h.ListBucketsHandler)).
|
||||
Name("ListBuckets")
|
||||
|
||||
// If none of the routes match, add default error handler routes
|
||||
api.NotFoundHandler = metrics.APIStats("notfound", errorResponseHandler)
|
||||
api.MethodNotAllowedHandler = metrics.APIStats("methodnotallowed", errorResponseHandler)
|
||||
}
|
||||
|
|
|
@ -4,9 +4,9 @@ import (
|
|||
"context"
|
||||
"net/http"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -16,9 +16,12 @@ type KeyWrapper string
|
|||
// BoxData is an ID used to store accessbox.Box in a context.
|
||||
var BoxData = KeyWrapper("__context_box_key")
|
||||
|
||||
// AttachUserAuth adds user authentication via center to router using log for logging.
|
||||
func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
|
||||
router.Use(func(h http.Handler) http.Handler {
|
||||
// ClientTime is an ID used to store client time.Time in a context.
|
||||
var ClientTime = KeyWrapper("__context_client_time")
|
||||
|
||||
// AuthMiddleware adds user authentication via center to router using log for logging.
|
||||
func AuthMiddleware(log *zap.Logger, center auth.Center) mux.MiddlewareFunc {
|
||||
return func(h http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var ctx context.Context
|
||||
box, err := center.Authenticate(r)
|
||||
|
@ -35,10 +38,13 @@ func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
|
|||
return
|
||||
}
|
||||
} else {
|
||||
ctx = context.WithValue(r.Context(), BoxData, box)
|
||||
ctx = context.WithValue(r.Context(), BoxData, box.AccessBox)
|
||||
if !box.ClientTime.IsZero() {
|
||||
ctx = context.WithValue(ctx, ClientTime, box.ClientTime)
|
||||
}
|
||||
}
|
||||
|
||||
h.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -11,25 +11,25 @@ import (
|
|||
"os"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/google/uuid"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
|
||||
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
neofsecdsa "github.com/nspcc-dev/neofs-sdk-go/crypto/ecdsa"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/eacl"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// PrmContainerCreate groups parameters of containers created by authmate.
|
||||
type PrmContainerCreate struct {
|
||||
// NeoFS identifier of the container creator.
|
||||
// FrostFS identifier of the container creator.
|
||||
Owner user.ID
|
||||
|
||||
// Container placement policy.
|
||||
|
@ -39,26 +39,26 @@ type PrmContainerCreate struct {
|
|||
FriendlyName string
|
||||
}
|
||||
|
||||
// NetworkState represents NeoFS network state which is needed for authmate processing.
|
||||
// NetworkState represents FrostFS network state which is needed for authmate processing.
|
||||
type NetworkState struct {
|
||||
// Current NeoFS time.
|
||||
// Current FrostFS time.
|
||||
Epoch uint64
|
||||
// Duration of the Morph chain block in ms.
|
||||
BlockDuration int64
|
||||
// Duration of the NeoFS epoch in Morph chain blocks.
|
||||
// Duration of the FrostFS epoch in Morph chain blocks.
|
||||
EpochDuration uint64
|
||||
}
|
||||
|
||||
// NeoFS represents virtual connection to NeoFS network.
|
||||
type NeoFS interface {
|
||||
// NeoFS interface required by credential tool.
|
||||
tokens.NeoFS
|
||||
// FrostFS represents virtual connection to FrostFS network.
|
||||
type FrostFS interface {
|
||||
// FrostFS interface required by credential tool.
|
||||
tokens.FrostFS
|
||||
|
||||
// ContainerExists checks container presence in NeoFS by identifier.
|
||||
// ContainerExists checks container presence in FrostFS by identifier.
|
||||
// Returns nil if container exists.
|
||||
ContainerExists(context.Context, cid.ID) error
|
||||
|
||||
// CreateContainer creates and saves parameterized container in NeoFS.
|
||||
// CreateContainer creates and saves parameterized container in FrostFS.
|
||||
// It sets 'Timestamp' attribute to the current time.
|
||||
// It returns the ID of the saved container.
|
||||
//
|
||||
|
@ -78,25 +78,25 @@ type NeoFS interface {
|
|||
TimeToEpoch(context.Context, time.Time) (uint64, uint64, error)
|
||||
}
|
||||
|
||||
// Agent contains client communicating with NeoFS and logger.
|
||||
// Agent contains client communicating with FrostFS and logger.
|
||||
type Agent struct {
|
||||
neoFS NeoFS
|
||||
log *zap.Logger
|
||||
frostFS FrostFS
|
||||
log *zap.Logger
|
||||
}
|
||||
|
||||
// New creates an object of type Agent that consists of Client and logger.
|
||||
func New(log *zap.Logger, neoFS NeoFS) *Agent {
|
||||
return &Agent{log: log, neoFS: neoFS}
|
||||
func New(log *zap.Logger, frostFS FrostFS) *Agent {
|
||||
return &Agent{log: log, frostFS: frostFS}
|
||||
}
|
||||
|
||||
type (
|
||||
// ContainerPolicies contains mapping of aws LocationConstraint to neofs PlacementPolicy.
|
||||
// ContainerPolicies contains mapping of aws LocationConstraint to frostfs PlacementPolicy.
|
||||
ContainerPolicies map[string]string
|
||||
|
||||
// IssueSecretOptions contains options for passing to Agent.IssueSecret method.
|
||||
IssueSecretOptions struct {
|
||||
Container ContainerOptions
|
||||
NeoFSKey *keys.PrivateKey
|
||||
FrostFSKey *keys.PrivateKey
|
||||
GatesPublicKeys []*keys.PublicKey
|
||||
EACLRules []byte
|
||||
SessionTokenRules []byte
|
||||
|
@ -120,7 +120,7 @@ type (
|
|||
}
|
||||
)
|
||||
|
||||
// lifetimeOptions holds NeoFS epochs, iat -- epoch which the token was issued at, exp -- epoch when the token expires.
|
||||
// lifetimeOptions holds FrostFS epochs, iat -- epoch which the token was issued at, exp -- epoch when the token expires.
|
||||
type lifetimeOptions struct {
|
||||
Iat uint64
|
||||
Exp uint64
|
||||
|
@ -143,7 +143,7 @@ type (
|
|||
|
||||
func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwner user.ID) (cid.ID, error) {
|
||||
if !opts.ID.Equals(cid.ID{}) {
|
||||
return opts.ID, a.neoFS.ContainerExists(ctx, opts.ID)
|
||||
return opts.ID, a.frostFS.ContainerExists(ctx, opts.ID)
|
||||
}
|
||||
|
||||
var prm PrmContainerCreate
|
||||
|
@ -156,9 +156,9 @@ func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwn
|
|||
prm.Owner = idOwner
|
||||
prm.FriendlyName = opts.FriendlyName
|
||||
|
||||
cnrID, err := a.neoFS.CreateContainer(ctx, prm)
|
||||
cnrID, err := a.frostFS.CreateContainer(ctx, prm)
|
||||
if err != nil {
|
||||
return cid.ID{}, fmt.Errorf("create container in NeoFS: %w", err)
|
||||
return cid.ID{}, fmt.Errorf("create container in FrostFS: %w", err)
|
||||
}
|
||||
|
||||
return cnrID, nil
|
||||
|
@ -200,7 +200,7 @@ func preparePolicy(policy ContainerPolicies) ([]*accessbox.AccessBox_ContainerPo
|
|||
return result, nil
|
||||
}
|
||||
|
||||
// IssueSecret creates an auth token, puts it in the NeoFS network and writes to io.Writer a new secret access key.
|
||||
// IssueSecret creates an auth token, puts it in the FrostFS network and writes to io.Writer a new secret access key.
|
||||
func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecretOptions) error {
|
||||
var (
|
||||
err error
|
||||
|
@ -213,7 +213,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
|||
return fmt.Errorf("prepare policies: %w", err)
|
||||
}
|
||||
|
||||
lifetime.Iat, lifetime.Exp, err = a.neoFS.TimeToEpoch(ctx, time.Now().Add(options.Lifetime))
|
||||
lifetime.Iat, lifetime.Exp, err = a.frostFS.TimeToEpoch(ctx, time.Now().Add(options.Lifetime))
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch time to epoch: %w", err)
|
||||
}
|
||||
|
@ -231,7 +231,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
|||
box.ContainerPolicy = policies
|
||||
|
||||
var idOwner user.ID
|
||||
user.IDFromKey(&idOwner, options.NeoFSKey.PrivateKey.PublicKey)
|
||||
user.IDFromKey(&idOwner, options.FrostFSKey.PrivateKey.PublicKey)
|
||||
|
||||
a.log.Info("check container or create", zap.Stringer("cid", options.Container.ID),
|
||||
zap.String("friendly_name", options.Container.FriendlyName),
|
||||
|
@ -241,11 +241,11 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
|||
return fmt.Errorf("check container: %w", err)
|
||||
}
|
||||
|
||||
a.log.Info("store bearer token into NeoFS",
|
||||
a.log.Info("store bearer token into FrostFS",
|
||||
zap.Stringer("owner_tkn", idOwner))
|
||||
|
||||
addr, err := tokens.
|
||||
New(a.neoFS, secrets.EphemeralKey, cache.DefaultAccessBoxConfig(a.log)).
|
||||
New(a.frostFS, secrets.EphemeralKey, cache.DefaultAccessBoxConfig(a.log)).
|
||||
Put(ctx, id, idOwner, box, lifetime.Exp, options.GatesPublicKeys...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to put bearer token: %w", err)
|
||||
|
@ -260,7 +260,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
|||
AccessKeyID: accessKeyID,
|
||||
SecretAccessKey: secrets.AccessKey,
|
||||
OwnerPrivateKey: hex.EncodeToString(secrets.EphemeralKey.Bytes()),
|
||||
WalletPublicKey: hex.EncodeToString(options.NeoFSKey.PublicKey().Bytes()),
|
||||
WalletPublicKey: hex.EncodeToString(options.FrostFSKey.PublicKey().Bytes()),
|
||||
ContainerID: id.EncodeToString(),
|
||||
}
|
||||
|
||||
|
@ -288,10 +288,10 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
|||
return nil
|
||||
}
|
||||
|
||||
// ObtainSecret receives an existing secret access key from NeoFS and
|
||||
// ObtainSecret receives an existing secret access key from FrostFS and
|
||||
// writes to io.Writer the secret access key.
|
||||
func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSecretOptions) error {
|
||||
bearerCreds := tokens.New(a.neoFS, options.GatePrivateKey, cache.DefaultAccessBoxConfig(a.log))
|
||||
bearerCreds := tokens.New(a.frostFS, options.GatePrivateKey, cache.DefaultAccessBoxConfig(a.log))
|
||||
|
||||
var addr oid.Address
|
||||
if err := addr.DecodeString(options.SecretAddress); err != nil {
|
||||
|
@ -381,7 +381,7 @@ func buildSessionToken(key *keys.PrivateKey, lifetime lifetimeOptions, ctx sessi
|
|||
tok.AppliedTo(ctx.containerID)
|
||||
|
||||
tok.SetID(uuid.New())
|
||||
tok.SetAuthKey((*neofsecdsa.PublicKey)(gateKey))
|
||||
tok.SetAuthKey((*frostfsecdsa.PublicKey)(gateKey))
|
||||
|
||||
tok.SetIat(lifetime.Iat)
|
||||
tok.SetNbf(lifetime.Iat)
|
||||
|
@ -413,7 +413,7 @@ func createTokens(options *IssueSecretOptions, lifetime lifetimeOptions) ([]*acc
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build eacl table: %w", err)
|
||||
}
|
||||
bearerTokens, err := buildBearerTokens(options.NeoFSKey, table, lifetime, options.GatesPublicKeys)
|
||||
bearerTokens, err := buildBearerTokens(options.FrostFSKey, table, lifetime, options.GatesPublicKeys)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build bearer tokens: %w", err)
|
||||
}
|
||||
|
@ -427,7 +427,7 @@ func createTokens(options *IssueSecretOptions, lifetime lifetimeOptions) ([]*acc
|
|||
return nil, fmt.Errorf("failed to build context for session token: %w", err)
|
||||
}
|
||||
|
||||
sessionTokens, err := buildSessionTokens(options.NeoFSKey, lifetime, sessionRules, options.GatesPublicKeys)
|
||||
sessionTokens, err := buildSessionTokens(options.FrostFSKey, lifetime, sessionRules, options.GatesPublicKeys)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to biuild session token: %w", err)
|
||||
}
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue