Compare commits

..

8 commits

Author SHA1 Message Date
2c2540b134 Release v0.25.1
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2022-11-30 16:41:25 +03:00
Angira Kekteeva
9c8d629f2b [#741] Fix data race
Signed-off-by: Angira Kekteeva <kira@nspcc.ru>
2022-11-16 18:02:15 +03:00
Denis Kirillov
9c795c8639 [#750] Update SDK to support timeout for stream
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-15 18:00:45 +03:00
Denis Kirillov
3556fec9c1 [#749] Update SDK
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-14 18:06:09 +03:00
Denis Kirillov
dfdfb913d5 [#740] Update docs
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-10 15:31:15 +03:00
Denis Kirillov
80fc3b4ac3 [#740] Fix forming policy by ast
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
2022-11-10 15:31:15 +03:00
459a848c84 [#737] Debian packaging
Debian package includes:
 - user creation;
 - directories and permissions;
 - unit file for systemd

Signed-off-by: Dmitriy Zabolotskiy <d.zabolotskiy@yadro.com>
2022-11-10 15:29:38 +03:00
2f93844417 [#738] Run CI on support branch PR
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2022-11-02 14:02:10 +03:00
156 changed files with 2721 additions and 5302 deletions

View file

@ -1,7 +1,7 @@
FROM golang:1.19 as builder
ARG BUILD=now
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
ARG REPO=github.com/nspcc-dev/neofs-s3-gw
ARG VERSION=dev
WORKDIR /src
@ -10,12 +10,12 @@ COPY . /src
RUN make
# Executable image
FROM alpine AS frostfs-s3-gw
FROM alpine AS neofs-s3-gw
RUN apk add --no-cache bash ca-certificates
WORKDIR /
COPY --from=builder /src/bin/frostfs-s3-gw /bin/frostfs-s3-gw
COPY --from=builder /src/bin/frostfs-s3-authmate /bin/frostfs-s3-authmate
COPY --from=builder /src/bin/neofs-s3-gw /bin/neofs-s3-gw
COPY --from=builder /src/bin/neofs-s3-authmate /bin/neofs-s3-authmate
ENTRYPOINT ["/bin/frostfs-s3-gw"]
ENTRYPOINT ["/bin/neofs-s3-gw"]

View file

@ -1,9 +1,9 @@
FROM alpine AS frostfs-s3-gw
FROM alpine AS neofs-s3-gw
RUN apk add --no-cache bash ca-certificates
WORKDIR /
COPY /bin/frostfs-s3-gw /bin/frostfs-s3-gw
COPY /bin/frostfs-s3-authmate /bin/frostfs-s3-authmate
COPY /bin/neofs-s3-gw /bin/neofs-s3-gw
COPY /bin/neofs-s3-authmate /bin/neofs-s3-authmate
ENTRYPOINT ["/bin/frostfs-s3-gw"]
ENTRYPOINT ["/bin/neofs-s3-gw"]

2
.github/CODEOWNERS vendored
View file

@ -1 +1 @@
* @alexvanin @KirillovDenis
* @alexvanin @masterSplinter01 @KirillovDenis

View file

@ -1,45 +0,0 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: community, triage, bug
assignees: ''
---
<!--- Provide a general summary of the issue in the Title above -->
## Expected Behavior
<!--- If you're describing a bug, tell us what should happen -->
<!--- If you're suggesting a change/improvement, tell us how it should work -->
## Current Behavior
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
## Possible Solution
<!--- Not obligatory -->
<!--- If no reason/fix/additions for the bug can be suggested, -->
<!--- uncomment the following phrase: -->
<!--- No fix can be suggested by a QA engineer. Further solutions shall be up to developers. -->
## Steps to Reproduce (for bugs)
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
<!--- reproduce this bug. -->
1.
## Context
<!--- How has this issue affected you? What are you trying to accomplish? -->
<!--- Providing context helps us come up with a solution that is most useful in the real world -->
## Regression
<!-- Is this issue a regression? (Yes / No) -->
<!-- If Yes, optionally please include version or commit id or PR# that caused this regression, if you have these details. -->
## Your Environment
<!--- Include as many relevant details about the environment you experienced the bug in -->
* Version used:
* Server setup and configuration:
* Operating System and version (`uname -a`):

View file

@ -1 +0,0 @@
blank_issues_enabled: false

View file

@ -1,20 +0,0 @@
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: community, triage
assignees: ''
---
## Is your feature request related to a problem? Please describe.
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
## Describe the solution you'd like
<!--- A clear and concise description of what you want to happen. -->
## Describe alternatives you've considered
<!--- A clear and concise description of any alternative solutions or features you've considered. -->
## Additional context
<!--- Add any other context or screenshots about the feature request here. -->

70
.github/logo.svg vendored
View file

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 25.0.1, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="Слой_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 184.2 51.8" style="enable-background:new 0 0 184.2 51.8;" xml:space="preserve">
<style type="text/css">
.st0{display:none;}
.st1{display:inline;}
.st2{fill:#01E397;}
.st3{display:inline;fill:#010032;}
.st4{display:inline;fill:#00E599;}
.st5{display:inline;fill:#00AF92;}
.st6{fill:#00C3E5;}
</style>
<g id="Layer_2">
<g id="Layer_1-2" class="st0">
<g class="st1">
<path class="st2" d="M146.6,18.3v7.2h10.9V29h-10.9v10.7h-4V14.8h18v3.5H146.6z"/>
<path class="st2" d="M180,15.7c1.7,0.9,3,2.2,4,3.8l-3,2.7c-0.6-1.3-1.5-2.4-2.6-3.3c-1.3-0.7-2.8-1-4.3-1
c-1.4-0.1-2.8,0.3-4,1.1c-0.9,0.5-1.5,1.5-1.4,2.6c0,1,0.5,1.9,1.4,2.4c1.5,0.8,3.2,1.3,4.9,1.5c1.9,0.3,3.7,0.8,5.4,1.6
c1.2,0.5,2.2,1.3,2.9,2.3c0.6,1,1,2.2,0.9,3.4c0,1.4-0.5,2.7-1.3,3.8c-0.9,1.2-2.1,2.1-3.5,2.6c-1.7,0.6-3.4,0.9-5.2,0.8
c-5,0-8.6-1.6-10.7-5l2.9-2.8c0.7,1.4,1.8,2.5,3.1,3.3c1.5,0.7,3.1,1.1,4.7,1c1.5,0.1,2.9-0.2,4.2-0.9c0.9-0.5,1.5-1.5,1.5-2.6
c0-0.9-0.5-1.8-1.3-2.2c-1.5-0.7-3.1-1.2-4.8-1.5c-1.9-0.3-3.7-0.8-5.5-1.5c-1.2-0.5-2.2-1.4-3-2.4c-0.6-1-1-2.2-0.9-3.4
c0-1.4,0.4-2.7,1.2-3.8c0.8-1.2,2-2.2,3.3-2.8c1.6-0.7,3.4-1.1,5.2-1C176.1,14.3,178.2,14.8,180,15.7z"/>
</g>
<path class="st3" d="M73.3,16.3c1.9,1.9,2.9,4.5,2.7,7.1v15.9h-4V24.8c0-2.6-0.5-4.5-1.6-5.7c-1.2-1.2-2.8-1.8-4.5-1.7
c-1.3,0-2.5,0.3-3.7,0.8c-1.2,0.7-2.2,1.7-2.9,2.9c-0.8,1.5-1.1,3.2-1.1,4.9v13.3h-4V15.1l3.6,1.5v1.7c0.8-1.5,2.1-2.6,3.6-3.3
c1.5-0.8,3.2-1.2,4.9-1.1C68.9,13.8,71.3,14.7,73.3,16.3z"/>
<path class="st3" d="M104.4,28.3H85.6c0.1,2.2,1,4.3,2.5,5.9c1.5,1.4,3.5,2.2,5.6,2.1c1.6,0.1,3.2-0.2,4.6-0.9
c1.1-0.6,2-1.6,2.5-2.8l3.3,1.8c-0.9,1.7-2.3,3.1-4,4c-2,1-4.2,1.5-6.4,1.4c-3.7,0-6.7-1.1-8.8-3.4s-3.2-5.5-3.2-9.6s1-7.2,3-9.5
s5-3.4,8.7-3.4c2.1-0.1,4.2,0.5,6.1,1.5c1.6,1,3,2.5,3.8,4.2c0.9,1.8,1.3,3.9,1.3,5.9C104.6,26.4,104.6,27.4,104.4,28.3z
M88.1,19.3c-1.4,1.5-2.2,3.4-2.4,5.5h15.1c-0.2-2-1-3.9-2.3-5.5c-1.4-1.3-3.2-2-5.1-1.9C91.5,17.3,89.6,18,88.1,19.3z"/>
<path class="st3" d="M131,17.3c2.2,2.3,3.2,5.5,3.2,9.5s-1,7.3-3.2,9.6s-5.1,3.4-8.8,3.4s-6.7-1.1-8.9-3.4s-3.2-5.5-3.2-9.6
s1.1-7.2,3.2-9.5s5.1-3.4,8.9-3.4S128.9,15,131,17.3z M116.2,19.9c-1.5,2-2.2,4.4-2.1,6.9c-0.2,2.5,0.6,5,2.1,7
c1.5,1.7,3.7,2.7,6,2.6c2.3,0.1,4.4-0.9,5.9-2.6c1.5-2,2.3-4.5,2.1-7c0.1-2.5-0.6-4.9-2.1-6.9c-1.5-1.7-3.6-2.7-5.9-2.6
C119.9,17.2,117.7,18.2,116.2,19.9z"/>
<polygon class="st4" points="0,9.1 0,43.7 22.5,51.8 22.5,16.9 46.8,7.9 24.8,0 "/>
<polygon class="st5" points="24.3,17.9 24.3,36.8 46.8,44.9 46.8,9.6 "/>
</g>
<g>
<g>
<path class="st6" d="M41.6,17.5H28.2v6.9h10.4v3.3H28.2v10.2h-3.9V14.2h17.2V17.5z"/>
<path class="st6" d="M45.8,37.9v-18h3.3l0.4,3.2c0.5-1.2,1.2-2.1,2.1-2.7c0.9-0.6,2.1-0.9,3.5-0.9c0.4,0,0.7,0,1.1,0.1
c0.4,0.1,0.7,0.2,0.9,0.3l-0.5,3.4c-0.3-0.1-0.6-0.2-0.9-0.2C55.4,23,54.9,23,54.4,23c-0.7,0-1.5,0.2-2.2,0.6
c-0.7,0.4-1.3,1-1.8,1.8s-0.7,1.8-0.7,3v9.5H45.8z"/>
<path class="st6" d="M68.6,19.6c1.8,0,3.3,0.4,4.6,1.1c1.3,0.7,2.4,1.8,3.1,3.2s1.1,3.1,1.1,5c0,1.9-0.4,3.6-1.1,5
c-0.8,1.4-1.8,2.5-3.1,3.2c-1.3,0.7-2.9,1.1-4.6,1.1s-3.3-0.4-4.6-1.1c-1.3-0.7-2.4-1.8-3.2-3.2c-0.8-1.4-1.2-3.1-1.2-5
c0-1.9,0.4-3.6,1.2-5s1.8-2.5,3.2-3.2C65.3,19.9,66.8,19.6,68.6,19.6z M68.6,22.6c-1.1,0-2,0.2-2.8,0.7c-0.8,0.5-1.3,1.2-1.7,2.1
s-0.6,2.1-0.6,3.5c0,1.3,0.2,2.5,0.6,3.4s1,1.7,1.7,2.2s1.7,0.7,2.8,0.7c1.1,0,2-0.2,2.7-0.7c0.7-0.5,1.3-1.2,1.7-2.2
s0.6-2.1,0.6-3.4c0-1.4-0.2-2.5-0.6-3.5s-1-1.6-1.7-2.1C70.6,22.8,69.6,22.6,68.6,22.6z"/>
<path class="st6" d="M89.2,38.3c-1.8,0-3.4-0.3-4.9-1c-1.5-0.7-2.7-1.7-3.5-3l2.7-2.3c0.5,1,1.3,1.8,2.3,2.4
c1,0.6,2.2,0.9,3.6,0.9c1.1,0,2-0.2,2.6-0.6c0.6-0.4,1-0.9,1-1.6c0-0.5-0.2-0.9-0.5-1.2s-0.9-0.6-1.7-0.8l-3.8-0.8
c-1.9-0.4-3.3-1-4.1-1.9c-0.8-0.9-1.2-1.9-1.2-3.3c0-1,0.3-1.9,0.9-2.7c0.6-0.8,1.4-1.5,2.5-2s2.5-0.8,4-0.8c1.8,0,3.3,0.3,4.6,1
c1.3,0.6,2.2,1.5,2.9,2.7l-2.7,2.2c-0.5-1-1.1-1.7-2-2.1c-0.9-0.5-1.8-0.7-2.8-0.7c-0.8,0-1.4,0.1-2,0.3c-0.6,0.2-1,0.5-1.3,0.8
c-0.3,0.3-0.4,0.7-0.4,1.2c0,0.5,0.2,0.9,0.5,1.3s1,0.6,1.9,0.8l4.1,0.9c1.7,0.3,2.9,0.9,3.7,1.7c0.7,0.8,1.1,1.8,1.1,2.9
c0,1.2-0.3,2.2-0.9,3c-0.6,0.9-1.5,1.6-2.6,2C92.1,38.1,90.7,38.3,89.2,38.3z"/>
<path class="st6" d="M112.8,19.9v3H99.3v-3H112.8z M106.6,14.6v17.9c0,0.9,0.2,1.5,0.7,1.9c0.5,0.4,1.1,0.6,1.9,0.6
c0.6,0,1.2-0.1,1.7-0.3c0.5-0.2,0.9-0.5,1.3-0.8l0.9,2.8c-0.6,0.5-1.2,0.9-2,1.1c-0.8,0.3-1.7,0.4-2.7,0.4c-1,0-2-0.2-2.8-0.5
s-1.5-0.9-2-1.6c-0.5-0.8-0.7-1.7-0.8-3V15.7L106.6,14.6z"/>
<path d="M137.9,17.5h-13.3v6.9h10.4v3.3h-10.4v10.2h-3.9V14.2h17.2V17.5z"/>
<path d="M150.9,13.8c2.1,0,4,0.4,5.5,1.2c1.6,0.8,2.9,2,4,3.5l-2.6,2.5c-0.9-1.4-1.9-2.4-3.1-3c-1.1-0.6-2.5-0.9-4-0.9
c-1.2,0-2.1,0.2-2.8,0.5c-0.7,0.3-1.3,0.7-1.6,1.2c-0.3,0.5-0.5,1.1-0.5,1.7c0,0.7,0.3,1.4,0.8,1.9c0.5,0.6,1.5,1,2.9,1.3
l4.8,1.1c2.3,0.5,3.9,1.3,4.9,2.3c1,1,1.4,2.3,1.4,3.9c0,1.5-0.4,2.7-1.2,3.8c-0.8,1.1-1.9,1.9-3.3,2.5s-3.1,0.9-5,0.9
c-1.7,0-3.2-0.2-4.5-0.6c-1.3-0.4-2.5-1-3.5-1.8c-1-0.7-1.8-1.6-2.5-2.6l2.7-2.7c0.5,0.8,1.1,1.6,1.9,2.2
c0.8,0.7,1.7,1.2,2.7,1.5c1,0.4,2.2,0.5,3.4,0.5c1.1,0,2.1-0.1,2.9-0.4c0.8-0.3,1.4-0.7,1.8-1.2c0.4-0.5,0.6-1.1,0.6-1.9
c0-0.7-0.2-1.3-0.7-1.8c-0.5-0.5-1.3-0.9-2.6-1.2l-5.2-1.2c-1.4-0.3-2.6-0.8-3.6-1.3c-0.9-0.6-1.6-1.3-2.1-2.1s-0.7-1.8-0.7-2.8
c0-1.3,0.4-2.6,1.1-3.7c0.7-1.1,1.8-2,3.2-2.6C147.3,14.1,148.9,13.8,150.9,13.8z"/>
</g>
</g>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 5.5 KiB

View file

@ -68,7 +68,7 @@ jobs:
runs-on: ubuntu-20.04
strategy:
matrix:
go_versions: [ '1.18.x', '1.19.x' ]
go_versions: [ '1.17', '1.18.x', '1.19.x' ]
fail-fast: false
steps:
- uses: actions/checkout@v2

10
.gitignore vendored
View file

@ -3,7 +3,7 @@
.vscode
# Tree service
internal/frostfs/services/tree/
internal/neofs/services/tree/
# Vendoring
vendor
@ -21,10 +21,4 @@ coverage.txt
coverage.html
# debhelpers
**/*debhelper*
# debian package build files
debian/files
debian/*.log
debian/*.substvars
debian/frostfs-s3-gw/
**/.debhelper

View file

@ -1,11 +0,0 @@
[general]
fail-without-commits=True
regex-style-search=True
contrib=CC1
[title-match-regex]
regex=^\[\#[0-9Xx]+\]\s
[ignore-by-title]
regex=^Release(.*)
ignore=title-match-regex

View file

@ -32,12 +32,15 @@ linters:
- revive
# some default golangci-lint linters
- deadcode
- errcheck
- gosimple
- ineffassign
- staticcheck
- structcheck
- typecheck
- unused
- varcheck
# extra linters
- exhaustive

View file

@ -1,45 +0,0 @@
ci:
autofix_prs: false
repos:
- repo: https://github.com/jorisroovers/gitlint
rev: v0.19.1
hooks:
- id: gitlint
stages: [commit-msg]
- id: gitlint-ci
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: check-added-large-files
- id: check-case-conflict
- id: check-executables-have-shebangs
- id: check-shebang-scripts-are-executable
- id: check-merge-conflict
- id: check-json
- id: check-xml
- id: check-yaml
- id: trailing-whitespace
args: [--markdown-linebreak-ext=md]
- id: end-of-file-fixer
exclude: ".key$"
- repo: https://github.com/shellcheck-py/shellcheck-py
rev: v0.9.0.2
hooks:
- id: shellcheck
- repo: https://github.com/golangci/golangci-lint
rev: v1.51.2
hooks:
- id: golangci-lint
- repo: local
hooks:
- id: go-unit-tests
name: go unit tests
entry: make test
pass_filenames: false
types: [go]
language: system

View file

@ -4,70 +4,10 @@ This document outlines major changes between releases.
## [Unreleased]
### Fixed
- Get empty bucket CORS from frostfs (TrueCloudLab#36)
- Don't count pool error on client abort (#35)
### Added
- Return container name in `head-bucket` response (TrueCloudLab#18)
- Billing metrics (TrueCloudLab#5)
- Multiple configs support (TrueCloudLab#21)
- Bucket name resolving policy (TrueCloudLab#25)
- Support string `Action` and `Resource` fields in `bucketPolicy.Statement` (TrueCloudLab#32)
- Add new `kludge.use_default_xmlns_for_complete_multipart` config param (TrueCloudLab#40)
### Changed
- Update neo-go to v0.101.0 (#14)
- Update viper to v1.15.0 (#14)
- Using multiple servers require only one healthy (TrueCloudLab#12)
- Update go version to go1.18 (TrueCloudLab#16)
- Return error on invalid LocationConstraint (TrueCloudLab#23)
- Place billing metrics to separate url path (TrueCloudLab#26)
- Add generated deb builder files to .gitignore, and fix typo (TrueCloudLab#28)
- Limit number of objects to delete at one time (TrueCloudLab#37)
- CompleteMultipartUpload handler now sends whitespace characters to keep alive client's connection (#60)
- Support new system attributes (#64)
## [0.26.0] - 2022-12-28
### Added
- Use client time as `now` in some requests (#726)
- Reload policies on SIGHUP (#747)
- Authmate flags for pool timeouts (#760)
- Multiple server listeners (#742)
### Changed
- Placement policy configuration (#568)
- Improved debug logging of CID and OID values (#754)
### Removed
- Deprecated linters (#755)
### Updating from v0.25.1
New config parameters were added. And old one `defaul_policy` were changed.
```yaml
placement_policy:
default: "REP 3"
region_mapping: /path/to/container/policies.json
```
Make sure you update the config accordingly:
If you configure application using environment variables change:
* `S3_GW_DEFAULT_POLICY` -> `S3_GW_PLACEMENT_POLICY_DEFAULT_POLICY`
* `S3_GW_LISTEN_ADDRESS` -> `S3_GW_SERVER_0_ADDRESS`
* `S3_GW_TLS_CERT_FILE` -> `S3_GW_SERVER_0_TLS_CERT_FILE` (and set `S3_GW_SERVER_0_TLS_ENABLED=true`)
* `S3_GW_TLS_KEY_FILE` -> `S3_GW_SERVER_0_TLS_KEY_FILE` (and set `S3_GW_SERVER_0_TLS_ENABLED=true`)
If you configure application using `.yaml` file change:
* `defaul_policy` -> `placement_policy.default`
* `listen_address` -> `server.0.address`
* `tls.cert_file` -> `server.0.tls.cert_file` (and set `server.0.tls.enabled: true`)
* `tls.key_file` -> `server.0.tls.key_file` (and set `server.0.tls.enabled: true`)
## [0.25.1] - 2022-10-30
### Fixed
- Empty bucket policy (#740)
- Empty bucket policy (#740)
- Big object removal (#749)
- Checksum panic (#741)
@ -91,7 +31,7 @@ If you configure application using `.yaml` file change:
### Changed
- GitHub actions update (#710)
- Makefile help (#725)
- Optimized object tags setting (#669)
- Optimized object tags setting (#669)
- Improved logging (#728)
- Unified unit test names (#617)
- Improved docs (#732)
@ -156,7 +96,7 @@ If you configure application using environment variables change:
* `S3_GW_LISTEN_DOMAINS_N` -> `S3_GW_LISTEN_DOMAINS` (use it as array variable)
If you configure application using `.yaml` file change:
* `wallet` -> `wallet.path`
* `wallet` -> `wallet.path`
* `address` -> `wallet.address`
* `listen_domains.n` -> `listen_domains` (use it as array param)
@ -179,8 +119,8 @@ If you configure application using `.yaml` file change:
- Rely on string sanitizing from zap (#498)
### Updating from v0.22.0
1. To enable pprof use `pprof.enabled` instead of `pprof` in config.
To enable prometheus metrics use `prometheus.enabled` instead of `metrics` in config.
1. To enable pprof use `pprof.enabled` instead of `pprof` in config.
To enable prometheus metrics use `prometheus.enabled` instead of `metrics` in config.
If you are using the command line flags you can skip this step.
## [0.22.0] - 2022-07-25
@ -202,7 +142,7 @@ Tree service support
- Cache type cast error logging (#465)
- `docker/*` target in Makefile (#471)
- Pre signed requests (#529)
- Tagging and ACL notifications (#361)
- Tagging and ACL notifications (#361)
- AWSv4 signer package to improve compatibility with S3 clients (#528)
- Extension mimetype detector (#289)
- Default params documentation (#592)
@ -236,7 +176,7 @@ Tree service support
- Obtainment of ETag value (#431)
### Changed
- Authmate doesn't parse session context anymore, now it accepts application defined
- Authmate doesn't parse session context anymore, now it accepts application defined
flexible structure with container ID in human-readable format (#428)
## [0.20.0] - 2022-04-29
@ -246,19 +186,19 @@ Tree service support
- Support of basic notifications (#357, #358, #359)
### Changed
- Logger behavior: now it writes to stderr instead of stdout, app name and
version are always presented and fixed, all user options except of `level` are
- Logger behavior: now it writes to stderr instead of stdout, app name and
version are always presented and fixed, all user options except of `level` are
dropped (#380)
- Improved docs, added config examples (#396, #398)
- Updated NeoFS SDK (#365, #409)
### Fixed
- Added check of `SetEACL` tokens before processing of requests (#347)
- Authmate: returned lost session tokens when a parameter `--session-token` is
- Authmate: returned lost session tokens when a parameter `--session-token` is
omitted (#387)
- Error when a bucket hasn't a settings file (#389)
- Response to a request to delete not existing object (#392)
- Replaced gate key in ACL Grantee by key of bearer token issuer (#395)
- Replaced gate key in ACL Grantee by key of bearer token issuer (#395)
- Missing attach of bearer token to requests to put system object (#399)
- Deletion of system object while CompleteMultipartUpload (#400)
- Improved English in docs and comments (#405)
@ -285,7 +225,7 @@ Tree service support
- Updated NeoFS SDK to v1.0.0-rc.3 (#297, #333, #346, #376)
- Authmate: changed session token rules handling (#329, #336, #338, #352)
- Changed status code for some failed requests (#308)
- GetBucketLocation returns policy name used at bucket creation (#301)
- GetBucketLocation returns policy name used at bucket creation (#301)
### Fixed
- Waiting for bucket to be deleted (#366)
@ -305,7 +245,7 @@ Tree service support
## [0.18.0] - 2021-12-16
### Added
- Support for MultipartUpload (#186, #187)
- Support for MultipartUpload (#186, #187)
- CORS support (#217)
- Authmate supports setting of tokens lifetime in a more convenient format (duration) (#258)
- Generation of a random key for `--no-sign-request` (#276)
@ -334,10 +274,10 @@ With this release we introduce [ceph-based](https://github.com/ceph/s3-tests) S3
* POST uploading support (#190)
* Delete marker support (#248)
* Expiration for access box (#255)
* AWS CLI credential generating by authmate (#241)
* AWS CLI credential generating by authmate (#241)
### Changed
* Default placement policy is now configurable (#218)
* Default placement policy is now configurable (#218)
* README is split into different files (#210)
* Unified error handling (#89, #149, #184)
* Authmate issue-secret response contains container id (#163)
@ -382,7 +322,7 @@ cryptography with NEP-6 wallet support.
* Support of time-based conditional CopyObject and GetObject (#94)
### Changed
* Accesskey format: now `0` used as a delimiter between container ID and object
* Accesskey format: now `0` used as a delimiter between container ID and object
ID instead of `_` (#164)
* Accessbox is encoded in protobuf format (#48)
* Authentication uses secp256r1 instead of ed25519 (#75)
@ -431,4 +371,5 @@ releases.
[0.23.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.22.0...v0.23.0
[0.24.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.23.0...v0.24.0
[0.25.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.24.0...v0.25.0
[Unreleased]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.25.0...master
[0.25.0]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.25.0...v0.25.1
[Unreleased]: https://github.com/nspcc-dev/neofs-s3-gw/compare/v0.25.1...master

View file

@ -3,8 +3,8 @@
First, thank you for contributing! We love and encourage pull requests from
everyone. Please follow the guidelines:
- Check the open [issues](https://github.com/TrueCloudLab/frostfs-s3-gw/issues) and
[pull requests](https://github.com/TrueCloudLab/frostfs-s3-gw/pulls) for existing
- Check the open [issues](https://github.com/nspcc-dev/neofs-s3-gw/issues) and
[pull requests](https://github.com/nspcc-dev/neofs-s3-gw/pulls) for existing
discussions.
- Open an issue first, to discuss a new feature or enhancement.
@ -23,24 +23,24 @@ everyone. Please follow the guidelines:
## Development Workflow
Start by forking the `frostfs-s3-gw` repository, make changes in a branch and then
Start by forking the `neofs-s3-gw` repository, make changes in a branch and then
send a pull request. We encourage pull requests to discuss code changes. Here
are the steps in details:
### Set up your GitHub Repository
Fork [FrostFS S3 Gateway
upstream](https://github.com/TrueCloudLab/frostfs-s3-gw/fork) source repository
Fork [NeoFS S3 Gateway
upstream](https://github.com/nspcc-dev/neofs-s3-gw/fork) source repository
to your own personal repository. Copy the URL of your fork (you will need it for
the `git clone` command below).
```sh
$ git clone https://github.com/TrueCloudLab/frostfs-s3-gw
$ git clone https://github.com/nspcc-dev/neofs-s3-gw
```
### Set up git remote as ``upstream``
```sh
$ cd frostfs-s3-gw
$ git remote add upstream https://github.com/TrueCloudLab/frostfs-s3-gw
$ cd neofs-s3-gw
$ git remote add upstream https://github.com/nspcc-dev/neofs-s3-gw
$ git fetch upstream
$ git merge upstream/master
...
@ -107,7 +107,7 @@ contributors".
To sign your work, just add a line like this at the end of your commit message:
```
Signed-off-by: Samii Sakisaka <samii@frostfs.info>
Signed-off-by: Samii Sakisaka <samii@nspcc.ru>
```
This can be easily done with the `--signoff` option to `git commit`.

View file

@ -16,4 +16,4 @@ In chronological order:
- Elizaveta Chichindaeva
- Stanislav Bogatyrev
- Anastasia Prasolova
- Leonard Liubich
- Leonard Liubich

26
Makefile Executable file → Normal file
View file

@ -8,22 +8,22 @@ LINT_VERSION ?= 1.49.0
BINDIR = bin
# Binaries to build
CMDS = $(addprefix frostfs-, $(notdir $(wildcard cmd/*)))
CMDS = $(addprefix neofs-, $(notdir $(wildcard cmd/*)))
BINS = $(addprefix $(BINDIR)/, $(CMDS))
# Variables for docker
REPO_BASENAME = $(shell basename `go list -m`)
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
HUB_IMAGE ?= "nspccdev/$(REPO_BASENAME)"
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint version clean protoc
# .deb package versioning
OS_RELEASE = $(shell lsb_release -cs)
PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \
sed "s/-/~/")-${OS_RELEASE}
.PHONY: debpackage debclean
.PHONY: debpackage debclean
# Make all binaries
all: $(BINS)
@ -33,7 +33,7 @@ $(BINS): sync-tree $(BINDIR) dep
CGO_ENABLED=0 \
go build -v -trimpath \
-ldflags "-X $(REPO)/internal/version.Version=$(VERSION)" \
-o $@ ./cmd/$(subst frostfs-,,$(notdir $@))
-o $@ ./cmd/$(subst neofs-,,$(notdir $@))
$(BINDIR):
@echo "⇒ Ensure dir: $@"
@ -80,7 +80,7 @@ format:
# Build clean Docker image
image:
@echo "⇒ Build FrostFS S3 Gateway docker image "
@echo "⇒ Build NeoFS S3 Gateway docker image "
@docker build \
--build-arg REPO=$(REPO) \
--build-arg VERSION=$(VERSION) \
@ -95,7 +95,7 @@ image-push:
# Build dirty Docker image
dirty-image:
@echo "⇒ Build FrostFS S3 Gateway dirty docker image "
@echo "⇒ Build NeoFS S3 Gateway dirty docker image "
@docker build \
--build-arg REPO=$(REPO) \
--build-arg VERSION=$(VERSION) \
@ -115,14 +115,6 @@ docker/lint:
--env HOME=/src \
golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint'
# Activate pre-commit hooks
pre-commit:
pre-commit install -t pre-commit -t commit-msg
# Deactivate pre-commit hooks
unpre-commit:
pre-commit uninstall -t pre-commit -t commit-msg
# Show current version
version:
@echo $(VERSION)
@ -143,7 +135,7 @@ protoc:
# Package for Debian
debpackage:
dch --package frostfs-s3-gw \
dch --package neofs-s3-gw \
--controlmaint \
--newversion $(PKG_VERSION) \
--distribution $(OS_RELEASE) \
@ -151,6 +143,6 @@ debpackage:
dpkg-buildpackage --no-sign -b
debclean:
dh clean
dh clean
include help.mk

View file

@ -1,23 +1,13 @@
<p align="center">
<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
</p>
<p align="center">
<a href="https://frostfs.info">FrostFS</a> is a decentralized distributed object storage integrated with the <a href="https://neo.org">NEO Blockchain</a>.
</p>
# NeoFS S3 Gateway
---
[![Report](https://goreportcard.com/badge/git.frostfs.info/TrueCloudLab/frostfs-s3-gw)](https://goreportcard.com/report/git.frostfs.info/TrueCloudLab/frostfs-s3-gw)
# FrostFS S3 Gateway
FrostFS S3 gateway provides API compatible with Amazon S3 cloud storage service.
NeoFS S3 gateway provides API compatible with Amazon S3 cloud storage service.
## Installation
```go get -u git.frostfs.info/TrueCloudLab/frostfs-s3-gw```
```go get -u github.com/nspcc-dev/neofs-s3-gw```
Or you can call `make` to build it from the cloned repository (the binary will
end up in `bin/frostfs-s3-gw` with authmate helper in `bin/frostfs-s3-authmate`).
end up in `bin/neofs-s3-gw` with authmate helper in `bin/neofs-s3-authmate`).
To build binaries in clean docker environment, call `make docker/all`.
Other notable make targets:
@ -32,45 +22,45 @@ version Show current version
```
Or you can also use a [Docker
image](https://hub.docker.com/r/truecloudlab/frostfs-s3-gw) provided for released
image](https://hub.docker.com/r/nspccdev/neofs-s3-gw) provided for released
(and occasionally unreleased) versions of gateway (`:latest` points to the
latest stable release).
## Execution
Minimalistic S3 gateway setup needs:
* FrostFS node(s) address (S3 gateway itself is not a FrostFS node)
* NeoFS node(s) address (S3 gateway itself is not a NeoFS node)
Passed via `-p` parameter or via `S3_GW_PEERS_<N>_ADDRESS` and
`S3_GW_PEERS_<N>_WEIGHT` environment variables (gateway supports multiple
FrostFS nodes with weighted load balancing).
* a wallet used to fetch key and communicate with FrostFS nodes
NeoFS nodes with weighted load balancing).
* a wallet used to fetch key and communicate with NeoFS nodes
Passed via `--wallet` parameter or `S3_GW_WALLET_PATH` environment variable.
These two commands are functionally equivalent, they run the gate with one
backend node, some keys and otherwise default settings:
```
$ frostfs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json
$ neofs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json
$ S3_GW_PEERS_0_ADDRESS=192.168.130.72:8080 \
S3_GW_WALLET=wallet.json \
frostfs-s3-gw
neofs-s3-gw
```
It's also possible to specify uri scheme (grpc or grpcs) when using `-p` or environment variables:
```
$ frostfs-s3-gw -p grpc://192.168.130.72:8080 --wallet wallet.json
$ neofs-s3-gw -p grpc://192.168.130.72:8080 --wallet wallet.json
$ S3_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 \
S3_GW_WALLET=wallet.json \
frostfs-s3-gw
neofs-s3-gw
```
## Domains
By default, s3-gw enable only `path-style access`.
By default, s3-gw enable only `path-style access`.
To be able to use both: `virtual-hosted-style` and `path-style` access you must configure `listen_domains`:
```shell
$ frostfs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json --listen_domains your.first.domain --listen_domains your.second.domain
$ neofs-s3-gw -p 192.168.130.72:8080 --wallet wallet.json --listen_domains your.first.domain --listen_domains your.second.domain
```
So now you can use (e.g. `HeadBucket`. Make sure DNS is properly configured):
@ -94,12 +84,12 @@ Also, you can configure domains using `.env` variables or `yaml` file.
## Documentation
- [Configuration](./docs/configuration.md)
- [FrostFS S3 AuthMate](./docs/authmate.md)
- [FrostFS Tree service](./docs/tree_service.md)
- [NeoFS S3 AuthMate](./docs/authmate.md)
- [NeoFS Tree service](./docs/tree_service.md)
- [AWS CLI basic usage](./docs/aws_cli.md)
- [AWS S3 API compatibility](./docs/aws_s3_compat.md)
- [AWS S3 Compatibility test results](./docs/s3_test_results.md)
## Credits
## Credits
Please see [CREDITS](CREDITS.md) for details.

View file

@ -1 +1 @@
v0.26.0
v0.25.1

View file

@ -14,14 +14,14 @@ import (
"strings"
"time"
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
v4 "github.com/nspcc-dev/neofs-s3-gw/api/auth/signer/v4"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
@ -33,13 +33,7 @@ var postPolicyCredentialRegexp = regexp.MustCompile(`(?P<access_key_id>[^/]+)/(?
type (
// Center is a user authentication interface.
Center interface {
Authenticate(request *http.Request) (*Box, error)
}
// Box contains access box and additional info.
Box struct {
AccessBox *accessbox.Box
ClientTime time.Time
Authenticate(request *http.Request) (*accessbox.Box, error)
}
center struct {
@ -92,9 +86,9 @@ func (p prs) Seek(_ int64, _ int) (int64, error) {
var _ io.ReadSeeker = prs(0)
// New creates an instance of AuthCenter.
func New(frostFS tokens.FrostFS, key *keys.PrivateKey, prefixes []string, config *cache.Config) Center {
func New(neoFS tokens.NeoFS, key *keys.PrivateKey, prefixes []string, config *cache.Config) Center {
return &center{
cli: tokens.New(frostFS, key, config),
cli: tokens.New(neoFS, key, config),
reg: NewRegexpMatcher(authorizationFieldRegexp),
postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
allowedAccessKeyIDPrefixes: prefixes,
@ -132,12 +126,11 @@ func (a *authHeader) getAddress() (oid.Address, error) {
return addr, nil
}
func (c *center) Authenticate(r *http.Request) (*Box, error) {
func (c *center) Authenticate(r *http.Request) (*accessbox.Box, error) {
var (
err error
authHdr *authHeader
signatureDateTimeStr string
needClientTime bool
)
queryValues := r.URL.Query()
@ -173,7 +166,6 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) {
return nil, err
}
signatureDateTimeStr = r.Header.Get(AmzDate)
needClientTime = true
}
signatureDateTime, err := time.Parse("20060102T150405Z", signatureDateTimeStr)
@ -200,12 +192,7 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) {
return nil, err
}
result := &Box{AccessBox: box}
if needClientTime {
result.ClientTime = signatureDateTime
}
return result, nil
return box, nil
}
func (c center) checkAccessKeyID(accessKeyID string) error {
@ -222,7 +209,7 @@ func (c center) checkAccessKeyID(accessKeyID string) error {
return apiErrors.GetAPIError(apiErrors.ErrAccessDenied)
}
func (c *center) checkFormData(r *http.Request) (*Box, error) {
func (c *center) checkFormData(r *http.Request) (*accessbox.Box, error) {
if err := r.ParseMultipartForm(maxFormSizeMemory); err != nil {
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidArgument)
}
@ -264,7 +251,7 @@ func (c *center) checkFormData(r *http.Request) (*Box, error) {
return nil, apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch)
}
return &Box{AccessBox: box}, nil
return box, nil
}
func cloneRequest(r *http.Request, authHeader *authHeader) *http.Request {

View file

@ -5,7 +5,7 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/stretchr/testify/require"
)

View file

@ -4,8 +4,8 @@ import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-sdk-go/user"
"go.uber.org/zap"
)

View file

@ -4,9 +4,9 @@ import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)

View file

@ -4,8 +4,8 @@ import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"go.uber.org/zap"
)

View file

@ -3,10 +3,10 @@ package cache
import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
"go.uber.org/zap/zaptest/observer"

2
api/cache/names.go vendored
View file

@ -4,8 +4,8 @@ import (
"fmt"
"time"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/bluele/gcache"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)

View file

@ -4,9 +4,9 @@ import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)

View file

@ -4,9 +4,9 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)

View file

@ -6,9 +6,9 @@ import (
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"go.uber.org/zap"
)
@ -20,7 +20,7 @@ import (
After putting a record, it lives for a while (default value is 60 seconds).
When we receive a request from a user, we try to find the suitable and non-expired cache entry, go through the list
and get ObjectInfos from common object cache or with a request to FrostFS.
and get ObjectInfos from common object cache or with a request to NeoFS.
When we put an object into a container, we invalidate entries with prefixes that are prefixes of the object's name.
*/

View file

@ -4,9 +4,9 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)

2
api/cache/system.go vendored
View file

@ -4,8 +4,8 @@ import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"go.uber.org/zap"
)

View file

@ -4,9 +4,9 @@ import (
"encoding/xml"
"time"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
)
const (
@ -22,8 +22,7 @@ const (
type (
// BucketInfo stores basic bucket data.
BucketInfo struct {
Name string // container name from system attribute
Zone string // container zone from system attribute
Name string
CID cid.ID
Owner user.ID
Created time.Time

View file

@ -4,9 +4,9 @@ import (
"strconv"
"time"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
)
const (
@ -25,7 +25,7 @@ func (v NodeVersion) IsDeleteMarker() bool {
}
// DeleteMarkerInfo is used to save object info if node in the tree service is delete marker.
// We need this information because the "delete marker" object is no longer stored in FrostFS.
// We need this information because the "delete marker" object is no longer stored in NeoFS.
type DeleteMarkerInfo struct {
Created time.Time
Owner user.ID

View file

@ -90,7 +90,6 @@ const (
ErrMissingFields
ErrMissingCredTag
ErrCredMalformed
ErrInvalidLocationConstraint
ErrInvalidRegion
ErrInvalidServiceS3
ErrInvalidServiceSTS
@ -681,12 +680,6 @@ var errorCodes = errorCodeMap{
Description: "Error parsing the X-Amz-Credential parameter; the region is wrong;",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidLocationConstraint: {
ErrCode: ErrInvalidLocationConstraint,
Code: "InvalidLocationConstraint",
Description: "The specified location (Region) constraint is not valid.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidRegion: {
ErrCode: ErrInvalidRegion,
Code: "InvalidRegion",
@ -993,7 +986,7 @@ var errorCodes = errorCodeMap{
ErrNotSupported: {
ErrCode: ErrNotSupported,
Code: "BadRequest",
Description: "Not supported by FrostFS S3 Gateway",
Description: "Not supported by NeoFS S3 Gateway",
HTTPStatusCode: http.StatusNotImplemented,
},
ErrInvalidEncryptionMethod: {

View file

@ -14,16 +14,16 @@ import (
"strconv"
"strings"
v2acl "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
v2acl "github.com/nspcc-dev/neofs-api-go/v2/acl"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"go.uber.org/zap"
)
@ -158,90 +158,6 @@ func (s ServiceRecord) ToEACLRecord() *eacl.Record {
return serviceRecord
}
var (
errInvalidStatement = stderrors.New("invalid statement")
errInvalidPrincipal = stderrors.New("invalid principal")
)
func (s *statement) UnmarshalJSON(data []byte) error {
var statementMap map[string]interface{}
if err := json.Unmarshal(data, &statementMap); err != nil {
return err
}
sidField, ok := statementMap["Sid"]
if ok {
if s.Sid, ok = sidField.(string); !ok {
return errInvalidStatement
}
}
effectField, ok := statementMap["Effect"]
if ok {
if s.Effect, ok = effectField.(string); !ok {
return errInvalidStatement
}
}
principalField, ok := statementMap["Principal"]
if ok {
principalMap, ok := principalField.(map[string]interface{})
if !ok {
return errInvalidPrincipal
}
awsField, ok := principalMap["AWS"]
if ok {
if s.Principal.AWS, ok = awsField.(string); !ok {
return fmt.Errorf("%w: 'AWS' field must be string", errInvalidPrincipal)
}
}
canonicalUserField, ok := principalMap["CanonicalUser"]
if ok {
if s.Principal.CanonicalUser, ok = canonicalUserField.(string); !ok {
return errInvalidPrincipal
}
}
}
actionField, ok := statementMap["Action"]
if ok {
switch actionField := actionField.(type) {
case []interface{}:
s.Action = make([]string, len(actionField))
for i, action := range actionField {
if s.Action[i], ok = action.(string); !ok {
return errInvalidStatement
}
}
case string:
s.Action = []string{actionField}
default:
return errInvalidStatement
}
}
resourceField, ok := statementMap["Resource"]
if ok {
switch resourceField := resourceField.(type) {
case []interface{}:
s.Resource = make([]string, len(resourceField))
for i, action := range resourceField {
if s.Resource[i], ok = action.(string); !ok {
return errInvalidStatement
}
}
case string:
s.Resource = []string{resourceField}
default:
return errInvalidStatement
}
}
return nil
}
func (h *handler) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())

View file

@ -13,15 +13,15 @@ import (
"net/http"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/stretchr/testify/require"
)
@ -1352,85 +1352,6 @@ func TestBucketPolicy(t *testing.T) {
}
}
func TestBucketPolicyUnmarshal(t *testing.T) {
for _, tc := range []struct {
name string
policy string
}{
{
name: "action/resource array",
policy: `
{
"Version": "2012-10-17",
"Statement": [{
"Principal": {
"AWS": "arn:aws:iam::111122223333:role/JohnDoe"
},
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:GetObjectVersion"
],
"Resource": [
"arn:aws:s3:::DOC-EXAMPLE-BUCKET/*",
"arn:aws:s3:::DOC-EXAMPLE-BUCKET2/*"
]
}]
}
`,
},
{
name: "action/resource string",
policy: `
{
"Version": "2012-10-17",
"Statement": [{
"Principal": {
"AWS": "arn:aws:iam::111122223333:role/JohnDoe"
},
"Effect": "Deny",
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::DOC-EXAMPLE-BUCKET/*"
}]
}
`,
},
} {
t.Run(tc.name, func(t *testing.T) {
bktPolicy := &bucketPolicy{}
err := json.Unmarshal([]byte(tc.policy), bktPolicy)
require.NoError(t, err)
})
}
}
func TestPutBucketPolicy(t *testing.T) {
bktPolicy := `
{
"Version": "2012-10-17",
"Statement": [{
"Principal": {
"AWS": "*"
},
"Effect": "Deny",
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::bucket-for-policy/*"
}]
}
`
hc := prepareHandlerContext(t)
bktName := "bucket-for-policy"
box, _ := createAccessBox(t)
createBucket(t, hc, bktName, box)
w, r := prepareTestPayloadRequest(hc, bktName, "", bytes.NewReader([]byte(bktPolicy)))
ctx := context.WithValue(r.Context(), api.BoxData, box)
r = r.WithContext(ctx)
hc.Handler().PutBucketPolicyHandler(w, r)
assertStatus(hc.t, w, http.StatusOK)
}
func getBucketPolicy(hc *handlerContext, bktName string) *bucketPolicy {
w, r := prepareTestRequest(hc, bktName, "", nil)
hc.Handler().GetBucketPolicyHandler(w, r)

View file

@ -1,14 +1,11 @@
package handler
import (
"encoding/xml"
"errors"
"io"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
"go.uber.org/zap"
)
@ -22,33 +19,21 @@ type (
Notificator interface {
SendNotifications(topics map[string]string, p *SendNotificationParams) error
SendTestNotification(topic, bucketName, requestID, HostID string, now time.Time) error
SendTestNotification(topic, bucketName, requestID, HostID string) error
}
// Config contains data which handler needs to keep.
Config struct {
Policy PlacementPolicy
XMLDecoder XMLDecoderProvider
DefaultMaxAge int
NotificatorEnabled bool
CopiesNumber uint32
ResolveZoneList []string
IsResolveListAllow bool // True if ResolveZoneList contains allowed zones
CompleteMultipartKeepalive time.Duration
}
PlacementPolicy interface {
Default() netmap.PlacementPolicy
Get(string) (netmap.PlacementPolicy, bool)
}
XMLDecoderProvider interface {
NewCompleteMultipartDecoder(io.Reader) *xml.Decoder
DefaultPolicy netmap.PlacementPolicy
DefaultMaxAge int
NotificatorEnabled bool
TLSEnabled bool
CopiesNumber uint32
}
)
const (
// DefaultPolicy is a default policy of placing containers in FrostFS if it's not set at the request.
// DefaultPolicy is a default policy of placing containers in NeoFS if it's not set at the request.
DefaultPolicy = "REP 3"
// DefaultCopiesNumber is a default number of object copies that is enough to consider put successful if it's not set in config.
DefaultCopiesNumber uint32 = 0
@ -60,7 +45,7 @@ var _ api.Handler = (*handler)(nil)
func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg *Config) (api.Handler, error) {
switch {
case obj == nil:
return nil, errors.New("empty FrostFS Object Layer")
return nil, errors.New("empty NeoFS Object Layer")
case log == nil:
return nil, errors.New("empty logger")
}

View file

@ -6,10 +6,10 @@ import (
"strconv"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"go.uber.org/zap"
)
@ -94,7 +94,7 @@ func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Requ
}
info := extendedInfo.ObjectInfo
encryptionParams, err := formEncryptionParams(r)
encryptionParams, err := h.formEncryptionParams(r.Header)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return

View file

@ -4,7 +4,7 @@ import (
"strings"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/stretchr/testify/require"
)

View file

@ -6,12 +6,12 @@ import (
"regexp"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-sdk-go/session"
"go.uber.org/zap"
)
@ -142,7 +142,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
}
}
encryptionParams, err := formEncryptionParams(r)
encryptionParams, err := h.formEncryptionParams(r.Header)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -184,7 +184,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
CopiesNuber: copiesNumber,
}
params.Lock, err = formObjectLock(r.Context(), dstBktInfo, settings.LockConfiguration, r.Header)
params.Lock, err = formObjectLock(dstBktInfo, settings.LockConfiguration, r.Header)
if err != nil {
h.logAndSendError(w, "could not form object lock", reqInfo, err)
return

View file

@ -6,7 +6,7 @@ import (
"net/url"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/stretchr/testify/require"
)

View file

@ -5,9 +5,9 @@ import (
"strconv"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"go.uber.org/zap"
)

View file

@ -1,41 +0,0 @@
package handler
import (
"context"
"net/http"
"strings"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
)
func TestCORSOriginWildcard(t *testing.T) {
body := `
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<CORSRule>
<AllowedMethod>GET</AllowedMethod>
<AllowedOrigin>*</AllowedOrigin>
</CORSRule>
</CORSConfiguration>
`
hc := prepareHandlerContext(t)
bktName := "bucket-for-cors"
box, _ := createAccessBox(t)
w, r := prepareTestRequest(hc, bktName, "", nil)
ctx := context.WithValue(r.Context(), api.BoxData, box)
r = r.WithContext(ctx)
r.Header.Add(api.AmzACL, "public-read")
hc.Handler().CreateBucketHandler(w, r)
assertStatus(t, w, http.StatusOK)
w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
ctx = context.WithValue(r.Context(), api.BoxData, box)
r = r.WithContext(ctx)
hc.Handler().PutBucketCorsHandler(w, r)
assertStatus(t, w, http.StatusOK)
w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
hc.Handler().GetBucketCorsHandler(w, r)
assertStatus(t, w, http.StatusOK)
}

View file

@ -6,20 +6,17 @@ import (
"strconv"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
// limitation of AWS https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
const maxObjectsToDelete = 1000
// DeleteObjectsRequest -- xml carrying the object key names which should be deleted.
type DeleteObjectsRequest struct {
// Element to enable quiet mode for the request
@ -179,11 +176,6 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
return
}
if len(requested.Objects) == 0 || len(requested.Objects) > maxObjectsToDelete {
h.logAndSendError(w, "number of objects to delete must be greater than 0 and less or equal to 1000", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
return
}
removed := make(map[string]*layer.VersionedObject)
toRemove := make([]*layer.VersionedObject, 0, len(requested.Objects))
for _, obj := range requested.Objects {

View file

@ -6,8 +6,8 @@ import (
"net/url"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/stretchr/testify/require"
)
@ -41,7 +41,7 @@ func TestDeleteObject(t *testing.T) {
deleteObject(t, tc, bktName, objName, emptyVersion)
checkNotFound(t, tc, bktName, objName, emptyVersion)
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo))
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo))
}
func TestDeleteObjectFromSuspended(t *testing.T) {
@ -109,7 +109,7 @@ func TestDeleteObjectVersioned(t *testing.T) {
deleteObject(t, tc, bktName, objName, objInfo.VersionID())
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object exists but shouldn't")
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object exists but shouldn't")
}
func TestDeleteObjectUnversioned(t *testing.T) {
@ -126,7 +126,7 @@ func TestDeleteObjectUnversioned(t *testing.T) {
require.Len(t, versions.DeleteMarker, 0, "delete markers must be empty")
require.Len(t, versions.Version, 0, "versions must be empty")
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object exists but shouldn't")
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object exists but shouldn't")
}
func TestRemoveDeleteMarker(t *testing.T) {
@ -144,7 +144,7 @@ func TestRemoveDeleteMarker(t *testing.T) {
deleteObject(t, tc, bktName, objName, deleteMarkerVersion)
checkFound(t, tc, bktName, objName, emptyVersion)
require.True(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object doesn't exist but should")
require.True(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object doesn't exist but should")
}
func TestDeleteObjectCombined(t *testing.T) {
@ -161,7 +161,7 @@ func TestDeleteObjectCombined(t *testing.T) {
checkFound(t, tc, bktName, objName, objInfo.VersionID())
require.True(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object doesn't exist but should")
require.True(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object doesn't exist but should")
}
func TestDeleteObjectSuspended(t *testing.T) {
@ -181,7 +181,7 @@ func TestDeleteObjectSuspended(t *testing.T) {
deleteObject(t, tc, bktName, objName, emptyVersion)
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo), "object exists but shouldn't")
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo), "object exists but shouldn't")
}
func TestDeleteMarkers(t *testing.T) {
@ -200,7 +200,7 @@ func TestDeleteMarkers(t *testing.T) {
require.Len(t, versions.DeleteMarker, 3, "invalid delete markers length")
require.Len(t, versions.Version, 0, "versions must be empty")
require.Len(t, listOIDsFromMockedFrostFS(t, tc, bktName), 0, "shouldn't be any object in frostfs")
require.Len(t, listOIDsFromMockedNeoFS(t, tc, bktName), 0, "shouldn't be any object in neofs")
}
func TestDeleteObjectFromListCache(t *testing.T) {
@ -220,7 +220,7 @@ func TestDeleteObjectFromListCache(t *testing.T) {
versions = listObjectsV1(t, tc, bktName, "", "", "", -1)
require.Len(t, versions.Contents, 0)
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo))
require.False(t, existInMockedNeoFS(tc, bktInfo, objInfo))
}
func TestDeleteObjectCheckMarkerReturn(t *testing.T) {

View file

@ -3,7 +3,6 @@ package handler
import (
"bytes"
"crypto/rand"
"crypto/tls"
"fmt"
"io"
"net/http"
@ -12,8 +11,8 @@ import (
"strings"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/stretchr/testify/require"
)
@ -285,7 +284,6 @@ func getEncryptedObjectRange(t *testing.T, tc *handlerContext, bktName, objName
}
func setEncryptHeaders(r *http.Request) {
r.TLS = &tls.ConnectionState{}
r.Header.Set(api.AmzServerSideEncryptionCustomerAlgorithm, layer.AESEncryptionAlgorithm)
r.Header.Set(api.AmzServerSideEncryptionCustomerKey, aes256Key)
r.Header.Set(api.AmzServerSideEncryptionCustomerKeyMD5, aes256KeyMD5)

View file

@ -8,10 +8,10 @@ import (
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"go.uber.org/zap"
)
@ -150,7 +150,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
encryptionParams, err := formEncryptionParams(r)
encryptionParams, err := h.formEncryptionParams(r.Header)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return

View file

@ -8,9 +8,9 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/stretchr/testify/require"
)

View file

@ -13,17 +13,15 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api/resolver"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
@ -32,7 +30,7 @@ type handlerContext struct {
owner user.ID
t *testing.T
h *handler
tp *layer.TestFrostFS
tp *layer.TestNeoFS
context context.Context
}
@ -40,7 +38,7 @@ func (hc *handlerContext) Handler() *handler {
return hc.h
}
func (hc *handlerContext) MockedPool() *layer.TestFrostFS {
func (hc *handlerContext) MockedPool() *layer.TestNeoFS {
return hc.tp
}
@ -52,30 +50,12 @@ func (hc *handlerContext) Context() context.Context {
return hc.context
}
type placementPolicyMock struct {
defaultPolicy netmap.PlacementPolicy
}
func (p *placementPolicyMock) Default() netmap.PlacementPolicy {
return p.defaultPolicy
}
func (p *placementPolicyMock) Get(string) (netmap.PlacementPolicy, bool) {
return netmap.PlacementPolicy{}, false
}
type xmlDecoderProviderMock struct{}
func (p *xmlDecoderProviderMock) NewCompleteMultipartDecoder(r io.Reader) *xml.Decoder {
return xml.NewDecoder(r)
}
func prepareHandlerContext(t *testing.T) *handlerContext {
key, err := keys.NewPrivateKey()
require.NoError(t, err)
l := zap.NewExample()
tp := layer.NewTestFrostFS()
tp := layer.NewTestNeoFS()
testResolver := &resolver.Resolver{Name: "test_resolver"}
testResolver.SetResolveFunc(func(_ context.Context, name string) (cid.ID, error) {
@ -89,19 +69,14 @@ func prepareHandlerContext(t *testing.T) *handlerContext {
Caches: layer.DefaultCachesConfigs(zap.NewExample()),
AnonKey: layer.AnonymousKey{Key: key},
Resolver: testResolver,
TreeService: NewTreeServiceMock(t),
TreeService: layer.NewTreeService(),
}
var pp netmap.PlacementPolicy
err = pp.DecodeString("REP 1")
require.NoError(t, err)
h := &handler{
log: l,
obj: layer.NewLayer(l, tp, layerCfg),
cfg: &Config{
Policy: &placementPolicyMock{defaultPolicy: pp},
XMLDecoder: &xmlDecoderProviderMock{},
TLSEnabled: true,
},
}
@ -114,12 +89,6 @@ func prepareHandlerContext(t *testing.T) *handlerContext {
}
}
func NewTreeServiceMock(t *testing.T) *tree.Tree {
memCli, err := tree.NewTreeServiceClientMemory()
require.NoError(t, err)
return tree.NewTree(memCli)
}
func createTestBucket(hc *handlerContext, bktName string) *data.BucketInfo {
_, err := hc.MockedPool().CreateContainer(hc.Context(), layer.PrmContainerCreate{
Creator: hc.owner,
@ -222,7 +191,7 @@ func parseTestResponse(t *testing.T, response *httptest.ResponseRecorder, body i
require.NoError(t, err)
}
func existInMockedFrostFS(tc *handlerContext, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo) bool {
func existInMockedNeoFS(tc *handlerContext, bktInfo *data.BucketInfo, objInfo *data.ObjectInfo) bool {
p := &layer.GetObjectParams{
BucketInfo: bktInfo,
ObjectInfo: objInfo,
@ -232,7 +201,7 @@ func existInMockedFrostFS(tc *handlerContext, bktInfo *data.BucketInfo, objInfo
return tc.Layer().GetObject(tc.Context(), p) == nil
}
func listOIDsFromMockedFrostFS(t *testing.T, tc *handlerContext, bktName string) []oid.ID {
func listOIDsFromMockedNeoFS(t *testing.T, tc *handlerContext, bktName string) []oid.ID {
bktInfo, err := tc.Layer().GetBucketInfo(tc.Context(), bktName)
require.NoError(t, err)

View file

@ -4,10 +4,10 @@ import (
"bytes"
"net/http"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"go.uber.org/zap"
)
@ -53,7 +53,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
}
info := extendedInfo.ObjectInfo
encryptionParams, err := formEncryptionParams(r)
encryptionParams, err := h.formEncryptionParams(r.Header)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -123,13 +123,6 @@ func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
}
w.Header().Set(api.ContainerID, bktInfo.CID.EncodeToString())
w.Header().Set(api.AmzBucketRegion, bktInfo.LocationConstraint)
if isAvailableToResolve(bktInfo.Zone, h.cfg.ResolveZoneList, h.cfg.IsResolveListAllow) {
w.Header().Set(api.ContainerName, bktInfo.Name)
w.Header().Set(api.ContainerZone, bktInfo.Zone)
}
api.WriteResponse(w, http.StatusOK, nil, api.MimeNone)
}
@ -163,25 +156,3 @@ func writeLockHeaders(h http.Header, legalHold *data.LegalHold, retention *data.
h.Set(api.AmzObjectLockMode, retention.Mode)
}
}
func isAvailableToResolve(zone string, list []string, isAllowList bool) bool {
// empty zone means container doesn't have proper system name,
// so we don't have to resolve it
if len(zone) == 0 {
return false
}
var zoneInList bool
for _, t := range list {
if t == zone {
zoneInList = true
break
}
}
// InList | IsAllowList | Result
// 0 0 1
// 0 1 0
// 1 0 0
// 1 1 1
return zoneInList == isAllowList
}

View file

@ -6,11 +6,11 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/stretchr/testify/require"
)
@ -86,26 +86,6 @@ func TestInvalidAccessThroughCache(t *testing.T) {
assertStatus(t, w, http.StatusForbidden)
}
func TestIsAvailableToResolve(t *testing.T) {
list := []string{"container", "s3"}
for i, testCase := range [...]struct {
isAllowList bool
list []string
zone string
expected bool
}{
{isAllowList: true, list: list, zone: "container", expected: true},
{isAllowList: true, list: list, zone: "sftp", expected: false},
{isAllowList: false, list: list, zone: "s3", expected: false},
{isAllowList: false, list: list, zone: "system", expected: true},
{isAllowList: true, list: list, zone: "", expected: false},
} {
result := isAvailableToResolve(testCase.zone, testCase.list, testCase.isAllowList)
require.Equal(t, testCase.expected, result, "case %d", i+1)
}
}
func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box {
var err error
if key == nil {

View file

@ -3,7 +3,7 @@ package handler
import (
"net/http"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api"
)
func (h *handler) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {

View file

@ -4,8 +4,8 @@ import (
"net/http"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-sdk-go/user"
)
const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.

View file

@ -1,17 +1,16 @@
package handler
import (
"context"
"encoding/xml"
"fmt"
"net/http"
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
)
const (
@ -209,7 +208,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
return
}
lock, err := formObjectLockFromRetention(r.Context(), retention, r.Header)
lock, err := formObjectLockFromRetention(retention, r.Header)
if err != nil {
h.logAndSendError(w, "invalid retention configuration", reqInfo, err)
return
@ -301,7 +300,7 @@ func checkLockConfiguration(conf *data.ObjectLockConfiguration) error {
return nil
}
func formObjectLock(ctx context.Context, bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConfiguration, header http.Header) (*data.ObjectLock, error) {
func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConfiguration, header http.Header) (*data.ObjectLock, error) {
if !bktInfo.ObjectLockEnabled {
if existLockHeaders(header) {
return nil, apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound)
@ -319,7 +318,7 @@ func formObjectLock(ctx context.Context, bktInfo *data.BucketInfo, defaultConfig
retention := &data.RetentionLock{}
defaultRetention := defaultConfig.Rule.DefaultRetention
retention.IsCompliance = defaultRetention.Mode == complianceMode
now := layer.TimeNow(ctx)
now := time.Now()
if defaultRetention.Days != 0 {
retention.Until = now.Add(time.Duration(defaultRetention.Days) * dayDuration)
} else {
@ -371,7 +370,7 @@ func formObjectLock(ctx context.Context, bktInfo *data.BucketInfo, defaultConfig
objectLock.Retention.ByPassedGovernance = bypass
}
if objectLock.Retention.Until.Before(layer.TimeNow(ctx)) {
if objectLock.Retention.Until.Before(time.Now()) {
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
}
}
@ -385,7 +384,7 @@ func existLockHeaders(header http.Header) bool {
header.Get(api.AmzObjectLockRetainUntilDate) != ""
}
func formObjectLockFromRetention(ctx context.Context, retention *data.Retention, header http.Header) (*data.ObjectLock, error) {
func formObjectLockFromRetention(retention *data.Retention, header http.Header) (*data.ObjectLock, error) {
if retention.Mode != governanceMode && retention.Mode != complianceMode {
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
}
@ -395,7 +394,7 @@ func formObjectLockFromRetention(ctx context.Context, retention *data.Retention,
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
}
if retentionDate.Before(layer.TimeNow(ctx)) {
if retentionDate.Before(time.Now()) {
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
}

View file

@ -10,17 +10,15 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/stretchr/testify/require"
)
const defaultURL = "http://localhost/"
func TestFormObjectLock(t *testing.T) {
ctx := context.Background()
for _, tc := range []struct {
name string
bktInfo *data.BucketInfo
@ -75,7 +73,7 @@ func TestFormObjectLock(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
actualObjLock, err := formObjectLock(ctx, tc.bktInfo, tc.config, tc.header)
actualObjLock, err := formObjectLock(tc.bktInfo, tc.config, tc.header)
if tc.expectedError {
require.Error(t, err)
return
@ -88,8 +86,6 @@ func TestFormObjectLock(t *testing.T) {
}
func TestFormObjectLockFromRetention(t *testing.T) {
ctx := context.Background()
for _, tc := range []struct {
name string
retention *data.Retention
@ -136,7 +132,7 @@ func TestFormObjectLockFromRetention(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
actualObjLock, err := formObjectLockFromRetention(ctx, tc.retention, tc.header)
actualObjLock, err := formObjectLockFromRetention(tc.retention, tc.header)
if tc.expectedError {
require.Error(t, err)
return

View file

@ -2,18 +2,17 @@ package handler
import (
"encoding/xml"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/google/uuid"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-sdk-go/session"
"go.uber.org/zap"
)
@ -138,7 +137,7 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
}
}
p.Info.Encryption, err = formEncryptionParams(r)
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -227,7 +226,7 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
Reader: r.Body,
}
p.Info.Encryption, err = formEncryptionParams(r)
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -332,7 +331,7 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
Range: srcRange,
}
p.Info.Encryption, err = formEncryptionParams(r)
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -373,6 +372,8 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
}
var (
sessionTokenSetEACL *session.Container
uploadID = r.URL.Query().Get(uploadIDHeaderName)
uploadInfo = &layer.UploadInfoParams{
UploadID: uploadID,
@ -383,7 +384,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
)
reqBody := new(CompleteMultipartUpload)
if err = h.cfg.XMLDecoder.NewCompleteMultipartDecoder(r.Body).Decode(reqBody); err != nil {
if err = xml.NewDecoder(r.Body).Decode(reqBody); err != nil {
h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
errors.GetAPIError(errors.ErrMalformedXML), additional...)
return
@ -398,51 +399,10 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
Parts: reqBody.Parts,
}
// Next operations might take some time, so we want to keep client's
// connection alive. To do so, gateway sends periodic white spaces
// back to the client the same way as Amazon S3 service does.
stopPeriodicResponseWriter := periodicXMLWriter(w, h.cfg.CompleteMultipartKeepalive)
// Start complete multipart upload which may take some time to fetch object
// and re-upload it part by part.
objInfo, err := h.completeMultipartUpload(r, c, bktInfo, reqInfo)
// Stop periodic writer as complete multipart upload is finished
// successfully or not.
headerIsWritten := stopPeriodicResponseWriter()
responseWriter := api.EncodeToResponse
errLogger := h.logAndSendError
// Do not send XML and HTTP headers if periodic writer was invoked at this point.
if headerIsWritten {
responseWriter = api.EncodeToResponseNoHeader
errLogger = h.logAndSendErrorNoHeader
}
if err != nil {
errLogger(w, "complete multipart error", reqInfo, err, additional...)
return
}
response := CompleteMultipartUploadResponse{
Bucket: objInfo.Bucket,
ETag: objInfo.HashSum,
Key: objInfo.Name,
}
// Here we previously set api.AmzVersionID header for versioned bucket.
// It is not possible after #60, because of periodic white
// space XML writer to keep connection with the client.
if err = responseWriter(w, response); err != nil {
errLogger(w, "something went wrong", reqInfo, err)
}
}
func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMultipartParams, bktInfo *data.BucketInfo, reqInfo *api.ReqInfo) (*data.ObjectInfo, error) {
uploadData, extendedObjInfo, err := h.obj.CompleteMultipartUpload(r.Context(), c)
if err != nil {
return nil, fmt.Errorf("could not complete multipart upload: %w", err)
h.logAndSendError(w, "could not complete multipart upload", reqInfo, err, additional...)
return
}
objInfo := extendedObjInfo.ObjectInfo
@ -457,22 +417,21 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
NodeVersion: extendedObjInfo.NodeVersion,
}
if _, err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
return nil, fmt.Errorf("could not put tagging file of completed multipart upload: %w", err)
h.logAndSendError(w, "could not put tagging file of completed multipart upload", reqInfo, err, additional...)
return
}
}
if len(uploadData.ACLHeaders) != 0 {
sessionTokenSetEACL, err := getSessionTokenSetEACL(r.Context())
if err != nil {
return nil, fmt.Errorf("couldn't get eacl token: %w", err)
}
key, err := h.bearerTokenIssuerKey(r.Context())
if err != nil {
return nil, fmt.Errorf("couldn't get gate key: %w", err)
h.logAndSendError(w, "couldn't get gate key", reqInfo, err)
return
}
acl, err := parseACLHeaders(r.Header, key)
if err != nil {
return nil, fmt.Errorf("could not parse acl: %w", err)
h.logAndSendError(w, "could not parse acl", reqInfo, err)
return
}
resInfo := &resourceInfo{
@ -481,10 +440,12 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
}
astObject, err := aclToAst(acl, resInfo)
if err != nil {
return nil, fmt.Errorf("could not translate acl of completed multipart upload to ast: %w", err)
h.logAndSendError(w, "could not translate acl of completed multipart upload to ast", reqInfo, err, additional...)
return
}
if _, err = h.updateBucketACL(r, astObject, bktInfo, sessionTokenSetEACL); err != nil {
return nil, fmt.Errorf("could not update bucket acl while completing multipart upload: %w", err)
h.logAndSendError(w, "could not update bucket acl while completing multipart upload", reqInfo, err, additional...)
return
}
}
@ -498,7 +459,24 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
h.log.Error("couldn't send notification: %w", zap.Error(err))
}
return objInfo, nil
bktSettings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
if err != nil {
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
}
response := CompleteMultipartUploadResponse{
Bucket: objInfo.Bucket,
ETag: objInfo.HashSum,
Key: objInfo.Name,
}
if bktSettings.VersioningEnabled() {
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
}
if err = api.EncodeToResponse(w, response); err != nil {
h.logAndSendError(w, "something went wrong", reqInfo, err)
}
}
func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
@ -595,7 +573,7 @@ func (h *handler) ListPartsHandler(w http.ResponseWriter, r *http.Request) {
PartNumberMarker: partNumberMarker,
}
p.Info.Encryption, err = formEncryptionParams(r)
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -630,7 +608,7 @@ func (h *handler) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Req
Key: reqInfo.ObjectName,
}
p.Encryption, err = formEncryptionParams(r)
p.Encryption, err = h.formEncryptionParams(r.Header)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -703,53 +681,3 @@ func encodeListPartsToResponse(info *layer.ListPartsInfo, params *layer.ListPart
Parts: info.Parts,
}
}
// periodicXMLWriter creates go routine to write xml header and whitespaces
// over time to avoid connection drop from the client. To work properly,
// pass `http.ResponseWriter` with implemented `http.Flusher` interface.
// Returns stop function which returns boolean if writer has been used
// during goroutine execution. To disable writer, pass 0 duration value.
func periodicXMLWriter(w io.Writer, dur time.Duration) (stop func() bool) {
if dur == 0 { // 0 duration disables periodic writer
return func() bool { return false }
}
whitespaceChar := []byte(" ")
closer := make(chan struct{})
done := make(chan struct{})
headerWritten := false
go func() {
defer close(done)
tick := time.NewTicker(dur)
defer tick.Stop()
for {
select {
case <-tick.C:
if !headerWritten {
_, err := w.Write([]byte(xml.Header))
headerWritten = err == nil
}
_, err := w.Write(whitespaceChar)
if err != nil {
return // is there anything we can do better than ignore error?
}
if buffered, ok := w.(http.Flusher); ok {
buffered.Flush()
}
case <-closer:
return
}
}
}()
stop = func() bool {
close(closer)
<-done // wait for goroutine to stop
return headerWritten
}
return stop
}

View file

@ -1,48 +0,0 @@
package handler
import (
"bytes"
"encoding/xml"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestPeriodicWriter(t *testing.T) {
const dur = 100 * time.Millisecond
const whitespaces = 8
expected := []byte(xml.Header)
for i := 0; i < whitespaces; i++ {
expected = append(expected, []byte(" ")...)
}
t.Run("writes data", func(t *testing.T) {
buf := bytes.NewBuffer(nil)
stop := periodicXMLWriter(buf, dur)
// N number of whitespaces + half durations to guarantee at least N writes in buffer
time.Sleep(whitespaces*dur + dur/2)
require.True(t, stop())
require.Equal(t, expected, buf.Bytes())
t.Run("no additional data after stop", func(t *testing.T) {
time.Sleep(2 * dur)
require.Equal(t, expected, buf.Bytes())
})
})
t.Run("does not write data", func(t *testing.T) {
buf := bytes.NewBuffer(nil)
stop := periodicXMLWriter(buf, dur)
time.Sleep(dur / 2)
require.False(t, stop())
require.Empty(t, buf.Bytes())
t.Run("disabled", func(t *testing.T) {
stop = periodicXMLWriter(buf, 0)
require.False(t, stop())
require.Empty(t, buf.Bytes())
})
})
}

View file

@ -3,8 +3,8 @@ package handler
import (
"net/http"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
)
func (h *handler) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {

View file

@ -6,14 +6,13 @@ import (
"fmt"
"net/http"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"github.com/google/uuid"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
)
type (
@ -23,7 +22,6 @@ type (
BktInfo *data.BucketInfo
ReqInfo *api.ReqInfo
User string
Time time.Time
}
NotificationConfiguration struct {
@ -109,7 +107,7 @@ func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Re
return
}
if _, err = h.checkBucketConfiguration(r.Context(), conf, reqInfo); err != nil {
if _, err = h.checkBucketConfiguration(conf, reqInfo); err != nil {
h.logAndSendError(w, "couldn't check bucket configuration", reqInfo, err)
return
}
@ -166,15 +164,13 @@ func (h *handler) sendNotifications(ctx context.Context, p *SendNotificationPara
p.User = bearer.ResolveIssuer(*box.Gate.BearerToken).EncodeToString()
}
p.Time = layer.TimeNow(ctx)
topics := filterSubjects(conf, p.Event, p.NotificationInfo.Name)
return h.notificator.SendNotifications(topics, p)
}
// checkBucketConfiguration checks notification configuration and generates an ID for configurations with empty ids.
func (h *handler) checkBucketConfiguration(ctx context.Context, conf *data.NotificationConfiguration, r *api.ReqInfo) (completed bool, err error) {
func (h *handler) checkBucketConfiguration(conf *data.NotificationConfiguration, r *api.ReqInfo) (completed bool, err error) {
if conf == nil {
return
}
@ -193,7 +189,7 @@ func (h *handler) checkBucketConfiguration(ctx context.Context, conf *data.Notif
}
if h.cfg.NotificatorEnabled {
if err = h.notificator.SendTestNotification(q.QueueArn, r.BucketName, r.RequestID, r.Host, layer.TimeNow(ctx)); err != nil {
if err = h.notificator.SendTestNotification(q.QueueArn, r.BucketName, r.RequestID, r.Host); err != nil {
return
}
} else {

View file

@ -3,8 +3,8 @@ package handler
import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/stretchr/testify/require"
)

View file

@ -6,11 +6,11 @@ import (
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
// ListObjectsV1Handler handles objects listing requests for API version 1.

View file

@ -6,7 +6,7 @@ import (
"strconv"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/stretchr/testify/require"
)

View file

@ -16,15 +16,15 @@ import (
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/session"
"go.uber.org/zap"
)
@ -218,7 +218,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
encryptionParams, err := formEncryptionParams(r)
encryption, err := h.formEncryptionParams(r.Header)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
return
@ -230,7 +230,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
Reader: r.Body,
Size: r.ContentLength,
Header: metadata,
Encryption: encryptionParams,
Encryption: encryption,
CopiesNumber: copiesNumber,
}
@ -240,7 +240,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
params.Lock, err = formObjectLock(r.Context(), bktInfo, settings.LockConfiguration, r.Header)
params.Lock, err = formObjectLock(bktInfo, settings.LockConfiguration, r.Header)
if err != nil {
h.logAndSendError(w, "could not form object lock", reqInfo, err)
return
@ -304,7 +304,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
if settings.VersioningEnabled() {
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
}
if encryptionParams.Enabled() {
if encryption.Enabled() {
addSSECHeaders(w.Header(), r.Header)
}
@ -313,7 +313,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
}
func getCopiesNumberOrDefault(metadata map[string]string, defaultCopiesNumber uint32) (uint32, error) {
copiesNumberStr, ok := metadata[layer.AttributeFrostfsCopiesNumber]
copiesNumberStr, ok := metadata[layer.AttributeNeofsCopiesNumber]
if !ok {
return defaultCopiesNumber, nil
}
@ -326,16 +326,16 @@ func getCopiesNumberOrDefault(metadata map[string]string, defaultCopiesNumber ui
return uint32(copiesNumber), nil
}
func formEncryptionParams(r *http.Request) (enc encryption.Params, err error) {
sseCustomerAlgorithm := r.Header.Get(api.AmzServerSideEncryptionCustomerAlgorithm)
sseCustomerKey := r.Header.Get(api.AmzServerSideEncryptionCustomerKey)
sseCustomerKeyMD5 := r.Header.Get(api.AmzServerSideEncryptionCustomerKeyMD5)
func (h handler) formEncryptionParams(header http.Header) (enc encryption.Params, err error) {
sseCustomerAlgorithm := header.Get(api.AmzServerSideEncryptionCustomerAlgorithm)
sseCustomerKey := header.Get(api.AmzServerSideEncryptionCustomerKey)
sseCustomerKeyMD5 := header.Get(api.AmzServerSideEncryptionCustomerKeyMD5)
if len(sseCustomerAlgorithm) == 0 && len(sseCustomerKey) == 0 && len(sseCustomerKeyMD5) == 0 {
return
}
if r.TLS == nil {
if !h.cfg.TLSEnabled {
return enc, errorsStd.New("encryption available only when TLS is enabled")
}
@ -667,10 +667,12 @@ func parseMetadata(r *http.Request) map[string]string {
}
func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := api.GetReqInfo(r.Context())
p := &layer.CreateBucketParams{
Name: reqInfo.BucketName,
}
var (
reqInfo = api.GetReqInfo(r.Context())
p = layer.CreateBucketParams{
Name: reqInfo.BucketName,
}
)
if err := checkBucketName(reqInfo.BucketName); err != nil {
h.logAndSendError(w, "invalid bucket name", reqInfo, err)
@ -720,22 +722,29 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
return
}
if err = h.setPolicy(p, createParams.LocationConstraint, policies); err != nil {
h.logAndSendError(w, "couldn't set placement policy", reqInfo, err)
return
useDefaultPolicy := true
if createParams.LocationConstraint != "" {
for _, placementPolicy := range policies {
if placementPolicy.LocationConstraint == createParams.LocationConstraint {
p.Policy = placementPolicy.Policy
p.LocationConstraint = createParams.LocationConstraint
useDefaultPolicy = false
break
}
}
}
if useDefaultPolicy {
p.Policy = h.cfg.DefaultPolicy
}
p.ObjectLockEnabled = isLockEnabled(r.Header)
bktInfo, err := h.obj.CreateBucket(r.Context(), p)
bktInfo, err := h.obj.CreateBucket(r.Context(), &p)
if err != nil {
h.logAndSendError(w, "could not create bucket", reqInfo, err)
return
}
h.log.Info("bucket is created", zap.String("reqId", reqInfo.RequestID),
zap.String("bucket", reqInfo.BucketName), zap.Stringer("container_id", bktInfo.CID))
if p.ObjectLockEnabled {
sp := &layer.PutSettingsParams{
BktInfo: bktInfo,
@ -748,32 +757,11 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
}
}
h.log.Info("bucket is created", zap.Stringer("container_id", bktInfo.CID))
api.WriteSuccessResponseHeadersOnly(w)
}
func (h handler) setPolicy(prm *layer.CreateBucketParams, locationConstraint string, userPolicies []*accessbox.ContainerPolicy) error {
prm.Policy = h.cfg.Policy.Default()
prm.LocationConstraint = locationConstraint
if locationConstraint == "" {
return nil
}
for _, placementPolicy := range userPolicies {
if placementPolicy.LocationConstraint == locationConstraint {
prm.Policy = placementPolicy.Policy
return nil
}
}
if policy, ok := h.cfg.Policy.Get(locationConstraint); ok {
prm.Policy = policy
return nil
}
return errors.GetAPIError(errors.ErrInvalidLocationConstraint)
}
func isLockEnabled(header http.Header) bool {
lockEnabledStr := header.Get(api.AmzBucketObjectLockEnabled)
lockEnabled, _ := strconv.ParseBool(lockEnabledStr)

View file

@ -8,8 +8,8 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/stretchr/testify/require"
)
@ -114,7 +114,7 @@ func TestPutObjectOverrideCopiesNumber(t *testing.T) {
bktInfo := createTestBucket(tc, bktName)
w, r := prepareTestRequest(tc, bktName, objName, nil)
r.Header.Set(api.MetadataPrefix+strings.ToUpper(layer.AttributeFrostfsCopiesNumber), "1")
r.Header.Set(api.MetadataPrefix+strings.ToUpper(layer.AttributeNeofsCopiesNumber), "1")
tc.Handler().PutObjectHandler(w, r)
p := &layer.HeadObjectParams{
@ -124,5 +124,5 @@ func TestPutObjectOverrideCopiesNumber(t *testing.T) {
objInfo, err := tc.Layer().GetObjectInfo(tc.Context(), p)
require.NoError(t, err)
require.Equal(t, "1", objInfo.Headers[layer.AttributeFrostfsCopiesNumber])
require.Equal(t, "1", objInfo.Headers[layer.AttributeNeofsCopiesNumber])
}

View file

@ -8,10 +8,10 @@ import (
"strings"
"unicode"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"go.uber.org/zap"
)

View file

@ -3,8 +3,8 @@ package handler
import (
"net/http"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
)
func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {

View file

@ -7,11 +7,11 @@ import (
"strconv"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-sdk-go/session"
"go.uber.org/zap"
)
@ -29,19 +29,6 @@ func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo
h.log.Error("call method", fields...)
}
func (h *handler) logAndSendErrorNoHeader(w http.ResponseWriter, logText string, reqInfo *api.ReqInfo, err error, additional ...zap.Field) {
api.WriteErrorResponseNoHeader(w, reqInfo, transformToS3Error(err))
fields := []zap.Field{
zap.String("request_id", reqInfo.RequestID),
zap.String("method", reqInfo.API),
zap.String("bucket", reqInfo.BucketName),
zap.String("object", reqInfo.ObjectName),
zap.String("description", logText),
zap.Error(err)}
fields = append(fields, additional...)
h.log.Error("call method", fields...)
}
func transformToS3Error(err error) error {
if _, ok := err.(errors.Error); ok {
return err
@ -55,10 +42,6 @@ func transformToS3Error(err error) error {
return errors.GetAPIError(errors.ErrInternalError)
}
func (h *handler) ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error) {
return h.obj.GetBucketInfo(ctx, bucket)
}
func (h *handler) getBucketAndCheckOwner(r *http.Request, bucket string, header ...string) (*data.BucketInfo, error) {
bktInfo, err := h.obj.GetBucketInfo(r.Context(), bucket)
if err != nil {

View file

@ -4,10 +4,10 @@ import (
"encoding/xml"
"net/http"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
)
func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {

View file

@ -2,17 +2,17 @@ package api
// Standard S3 HTTP request/response constants.
const (
MetadataPrefix = "X-Amz-Meta-"
FrostFSSystemMetadataPrefix = "S3-"
AmzMetadataDirective = "X-Amz-Metadata-Directive"
AmzTaggingDirective = "X-Amz-Tagging-Directive"
AmzVersionID = "X-Amz-Version-Id"
AmzTaggingCount = "X-Amz-Tagging-Count"
AmzTagging = "X-Amz-Tagging"
AmzDeleteMarker = "X-Amz-Delete-Marker"
AmzCopySource = "X-Amz-Copy-Source"
AmzCopySourceRange = "X-Amz-Copy-Source-Range"
AmzDate = "X-Amz-Date"
MetadataPrefix = "X-Amz-Meta-"
NeoFSSystemMetadataPrefix = "S3-"
AmzMetadataDirective = "X-Amz-Metadata-Directive"
AmzTaggingDirective = "X-Amz-Tagging-Directive"
AmzVersionID = "X-Amz-Version-Id"
AmzTaggingCount = "X-Amz-Tagging-Count"
AmzTagging = "X-Amz-Tagging"
AmzDeleteMarker = "X-Amz-Delete-Marker"
AmzCopySource = "X-Amz-Copy-Source"
AmzCopySourceRange = "X-Amz-Copy-Source-Range"
AmzDate = "X-Amz-Date"
LastModified = "Last-Modified"
Date = "Date"
@ -62,9 +62,7 @@ const (
AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"
AmzServerSideEncryptionCustomerKeyMD5 = "x-amz-server-side-encryption-customer-key-MD5"
ContainerID = "X-Container-Id"
ContainerName = "X-Container-Name"
ContainerZone = "X-Container-Zone"
ContainerID = "X-Container-Id"
AccessControlAllowOrigin = "Access-Control-Allow-Origin"
AccessControlAllowMethods = "Access-Control-Allow-Methods"

View file

@ -1,11 +1,11 @@
package layer
import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
"go.uber.org/zap"
)

View file

@ -4,8 +4,8 @@ import (
"context"
errorsStd "errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
)
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {

View file

@ -4,16 +4,16 @@ import (
"context"
"fmt"
"strconv"
"time"
v2container "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-sdk-go/client"
"github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/session"
"go.uber.org/zap"
)
@ -42,14 +42,14 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
Name: idCnr.EncodeToString(),
}
)
res, err = n.frostFS.Container(ctx, idCnr)
res, err = n.neoFS.Container(ctx, idCnr)
if err != nil {
log.Error("could not fetch container", zap.Error(err))
if client.IsErrContainerNotFound(err) {
return nil, errors.GetAPIError(errors.ErrNoSuchBucket)
}
return nil, fmt.Errorf("get frostfs container: %w", err)
return nil, fmt.Errorf("get neofs container: %w", err)
}
cnr := *res
@ -57,7 +57,6 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
info.Owner = cnr.Owner()
if domain := container.ReadDomain(cnr); domain.Name() != "" {
info.Name = domain.Name()
info.Zone = domain.Zone()
}
info.Created = container.CreatedAt(cnr)
info.LocationConstraint = cnr.Attribute(attributeLocationConstraint)
@ -85,7 +84,7 @@ func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
res []cid.ID
rid = api.GetRequestID(ctx)
)
res, err = n.frostFS.UserContainers(ctx, own)
res, err = n.neoFS.UserContainers(ctx, own)
if err != nil {
n.log.Error("could not list user containers",
zap.String("request_id", rid),
@ -116,9 +115,8 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
}
bktInfo := &data.BucketInfo{
Name: p.Name,
Zone: v2container.SysAttributeZoneDefault,
Owner: ownerID,
Created: TimeNow(ctx),
Created: time.Now(), // this can be a little incorrect since the real time is set later
LocationConstraint: p.LocationConstraint,
ObjectLockEnabled: p.ObjectLockEnabled,
}
@ -135,12 +133,11 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
})
}
idCnr, err := n.frostFS.CreateContainer(ctx, PrmContainerCreate{
idCnr, err := n.neoFS.CreateContainer(ctx, PrmContainerCreate{
Creator: bktInfo.Owner,
Policy: p.Policy,
Name: p.Name,
SessionToken: p.SessionContainerCreation,
CreationTime: bktInfo.Created,
AdditionalAttributes: attributes,
})
if err != nil {
@ -161,9 +158,9 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
func (n *layer) setContainerEACLTable(ctx context.Context, idCnr cid.ID, table *eacl.Table, sessionToken *session.Container) error {
table.SetCID(idCnr)
return n.frostFS.SetContainerEACL(ctx, *table, sessionToken)
return n.neoFS.SetContainerEACL(ctx, *table, sessionToken)
}
func (n *layer) GetContainerEACL(ctx context.Context, idCnr cid.ID) (*eacl.Table, error) {
return n.frostFS.ContainerEACL(ctx, idCnr)
return n.neoFS.ContainerEACL(ctx, idCnr)
}

View file

@ -8,8 +8,8 @@ import (
"fmt"
"io"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"go.uber.org/zap"
)
@ -39,9 +39,8 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
prm := PrmObjectCreate{
Container: p.BktInfo.CID,
Creator: p.BktInfo.Owner,
Payload: &buf,
Payload: p.Reader,
Filepath: p.BktInfo.CORSObjectName(),
CreationTime: TimeNow(ctx),
CopiesNumber: p.CopiesNumber,
}

View file

@ -11,20 +11,20 @@ import (
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nats-io/nats.go"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/user"
"go.uber.org/zap"
)
@ -45,7 +45,7 @@ type (
}
layer struct {
frostFS FrostFS
neoFS NeoFS
log *zap.Logger
anonKey AnonymousKey
resolver BucketResolver
@ -250,12 +250,12 @@ const (
AESEncryptionAlgorithm = "AES256"
AESKeySize = 32
AttributeEncryptionAlgorithm = api.FrostFSSystemMetadataPrefix + "Algorithm"
AttributeDecryptedSize = api.FrostFSSystemMetadataPrefix + "Decrypted-Size"
AttributeHMACSalt = api.FrostFSSystemMetadataPrefix + "HMAC-Salt"
AttributeHMACKey = api.FrostFSSystemMetadataPrefix + "HMAC-Key"
AttributeEncryptionAlgorithm = api.NeoFSSystemMetadataPrefix + "Algorithm"
AttributeDecryptedSize = api.NeoFSSystemMetadataPrefix + "Decrypted-Size"
AttributeHMACSalt = api.NeoFSSystemMetadataPrefix + "HMAC-Salt"
AttributeHMACKey = api.NeoFSSystemMetadataPrefix + "HMAC-Key"
AttributeFrostfsCopiesNumber = "frostfs-copies-number" // such format to match X-Amz-Meta-Frostfs-Copies-Number header
AttributeNeofsCopiesNumber = "neofs-copies-number" // such formate to match X-Amz-Meta-Neofs-Copies-Number header
)
func (t *VersionedObject) String() string {
@ -268,9 +268,9 @@ func (f MsgHandlerFunc) HandleMessage(ctx context.Context, msg *nats.Msg) error
// NewLayer creates an instance of a layer. It checks credentials
// and establishes gRPC connection with the node.
func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) Client {
func NewLayer(log *zap.Logger, neoFS NeoFS, config *Config) Client {
return &layer{
frostFS: frostFS,
neoFS: neoFS,
log: log,
anonKey: config.AnonKey,
resolver: config.Resolver,
@ -306,15 +306,6 @@ func IsAuthenticatedRequest(ctx context.Context) bool {
return ok
}
// TimeNow returns client time from request or time.Now().
func TimeNow(ctx context.Context) time.Time {
if now, ok := ctx.Value(api.ClientTime).(time.Time); ok {
return now
}
return time.Now()
}
// Owner returns owner id from BearerToken (context) or from client owner.
func (n *layer) Owner(ctx context.Context) user.ID {
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
@ -377,7 +368,7 @@ func (n *layer) PutBucketACL(ctx context.Context, param *PutBucketACLParams) err
}
// ListBuckets returns all user containers. The name of the bucket is a container
// id. Timestamp is omitted since it is not saved in frostfs container.
// id. Timestamp is omitted since it is not saved in neofs container.
func (n *layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) {
return n.containerList(ctx)
}
@ -482,27 +473,11 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.O
// GetExtendedObjectInfo returns meta information and corresponding info from the tree service about the object.
func (n *layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error) {
var objInfo *data.ExtendedObjectInfo
var err error
if len(p.VersionID) == 0 {
objInfo, err = n.headLastVersionIfNotDeleted(ctx, p.BktInfo, p.Object)
} else {
objInfo, err = n.headVersion(ctx, p.BktInfo, p)
}
if err != nil {
return nil, err
return n.headLastVersionIfNotDeleted(ctx, p.BktInfo, p.Object)
}
reqInfo := api.GetReqInfo(ctx)
n.log.Debug("get object",
zap.String("reqId", reqInfo.RequestID),
zap.String("bucket", p.BktInfo.Name),
zap.Stringer("cid", p.BktInfo.CID),
zap.String("object", objInfo.ObjectInfo.Name),
zap.Stringer("oid", objInfo.ObjectInfo.ID))
return objInfo, nil
return n.headVersion(ctx, p.BktInfo, p)
}
// CopyObject from one bucket into another bucket.
@ -590,7 +565,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
FilePath: obj.Name,
},
DeleteMarker: &data.DeleteMarkerInfo{
Created: TimeNow(ctx),
Created: time.Now(),
Owner: n.Owner(ctx),
},
IsUnversioned: settings.VersioningSuspended(),
@ -661,12 +636,7 @@ func (n *layer) CreateBucket(ctx context.Context, p *CreateBucketParams) (*data.
func (n *layer) ResolveBucket(ctx context.Context, name string) (cid.ID, error) {
var cnrID cid.ID
if err := cnrID.DecodeString(name); err != nil {
if cnrID, err = n.resolver.Resolve(ctx, name); err != nil {
return cid.ID{}, err
}
reqInfo := api.GetReqInfo(ctx)
n.log.Info("resolve bucket", zap.String("reqId", reqInfo.RequestID), zap.String("bucket", name), zap.Stringer("cid", cnrID))
return n.resolver.Resolve(ctx, name)
}
return cnrID, nil
@ -682,5 +652,5 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
}
n.cache.DeleteBucket(p.BktInfo.Name)
return n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
return n.neoFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
}

View file

@ -4,8 +4,7 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/stretchr/testify/require"
)
@ -44,10 +43,10 @@ func TestObjectLockAttributes(t *testing.T) {
expEpoch := false
for _, attr := range lockObj.Attributes() {
if attr.Key() == object.SysAttributeExpEpoch {
if attr.Key() == AttributeExpirationEpoch {
expEpoch = true
}
}
require.Truef(t, expEpoch, "system header __SYSTEM__EXPIRATION_EPOCH presence")
require.Truef(t, expEpoch, "system header __NEOFS__EXPIRATION_EPOCH presence")
}

View file

@ -11,13 +11,12 @@ import (
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/minio/sio"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
"go.uber.org/zap"
)
@ -144,7 +143,7 @@ func (n *layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
Key: p.Info.Key,
UploadID: p.Info.UploadID,
Owner: n.Owner(ctx),
Created: TimeNow(ctx),
Created: time.Now(),
Meta: make(map[string]string, metaSize),
CopiesNumber: p.CopiesNumber,
}
@ -206,7 +205,6 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
Creator: bktInfo.Owner,
Attributes: make([][2]string, 2),
Payload: p.Reader,
CreationTime: TimeNow(ctx),
CopiesNumber: multipartInfo.CopiesNumber,
}
@ -229,13 +227,6 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
return nil, err
}
reqInfo := api.GetReqInfo(ctx)
n.log.Debug("upload part",
zap.String("reqId", reqInfo.RequestID),
zap.String("bucket", bktInfo.Name), zap.Stringer("cid", bktInfo.CID),
zap.String("multipart upload", p.Info.UploadID),
zap.Int("part number", p.PartNumber), zap.String("object", p.Info.Key), zap.Stringer("oid", id))
partInfo := &data.PartInfo{
Key: p.Info.Key,
UploadID: p.Info.UploadID,
@ -243,7 +234,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
OID: id,
Size: decSize,
ETag: hex.EncodeToString(hash),
Created: prm.CreationTime,
Created: time.Now(),
}
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)
@ -319,7 +310,7 @@ func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
return n.uploadPart(ctx, multipartInfo, params)
}
// implements io.Reader of payloads of the object list stored in the FrostFS network.
// implements io.Reader of payloads of the object list stored in the NeoFS network.
type multiObjectReader struct {
ctx context.Context
@ -617,24 +608,10 @@ func (n *layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
}
res := make(map[int]*data.PartInfo, len(parts))
partsNumbers := make([]int, len(parts))
oids := make([]string, len(parts))
for i, part := range parts {
for _, part := range parts {
res[part.Number] = part
partsNumbers[i] = part.Number
oids[i] = part.OID.EncodeToString()
}
reqInfo := api.GetReqInfo(ctx)
n.log.Debug("part details",
zap.String("reqId", reqInfo.RequestID),
zap.String("bucket", p.Bkt.Name),
zap.Stringer("cid", p.Bkt.CID),
zap.String("object", p.Key),
zap.String("upload id", p.UploadID),
zap.Ints("part numbers", partsNumbers),
zap.Strings("oids", oids))
return multipartInfo, res, nil
}

View file

@ -7,21 +7,21 @@ import (
"io"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
"github.com/nspcc-dev/neofs-sdk-go/container"
"github.com/nspcc-dev/neofs-sdk-go/container/acl"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/user"
)
// PrmContainerCreate groups parameters of FrostFS.CreateContainer operation.
// PrmContainerCreate groups parameters of NeoFS.CreateContainer operation.
type PrmContainerCreate struct {
// FrostFS identifier of the container creator.
// NeoFS identifier of the container creator.
Creator user.ID
// Container placement policy.
@ -30,9 +30,6 @@ type PrmContainerCreate struct {
// Name for the container.
Name string
// CreationTime value for Timestamp attribute
CreationTime time.Time
// Token of the container's creation session. Nil means session absence.
SessionToken *session.Container
@ -43,7 +40,7 @@ type PrmContainerCreate struct {
AdditionalAttributes [][2]string
}
// PrmAuth groups authentication parameters for the FrostFS operation.
// PrmAuth groups authentication parameters for the NeoFS operation.
type PrmAuth struct {
// Bearer token to be used for the operation. Overlaps PrivateKey. Optional.
BearerToken *bearer.Token
@ -52,7 +49,7 @@ type PrmAuth struct {
PrivateKey *ecdsa.PrivateKey
}
// PrmObjectRead groups parameters of FrostFS.ReadObject operation.
// PrmObjectRead groups parameters of NeoFS.ReadObject operation.
type PrmObjectRead struct {
// Authentication parameters.
PrmAuth
@ -73,7 +70,7 @@ type PrmObjectRead struct {
PayloadRange [2]uint64
}
// ObjectPart represents partially read FrostFS object.
// ObjectPart represents partially read NeoFS object.
type ObjectPart struct {
// Object header with optional in-memory payload part.
Head *object.Object
@ -83,7 +80,7 @@ type ObjectPart struct {
Payload io.ReadCloser
}
// PrmObjectCreate groups parameters of FrostFS.CreateObject operation.
// PrmObjectCreate groups parameters of NeoFS.CreateObject operation.
type PrmObjectCreate struct {
// Authentication parameters.
PrmAuth
@ -91,15 +88,12 @@ type PrmObjectCreate struct {
// Container to store the object.
Container cid.ID
// FrostFS identifier of the object creator.
// NeoFS identifier of the object creator.
Creator user.ID
// Key-value object attributes.
Attributes [][2]string
// Value for Timestamp attribute (optional).
CreationTime time.Time
// List of ids to lock (optional).
Locks []oid.ID
@ -116,7 +110,7 @@ type PrmObjectCreate struct {
CopiesNumber uint32
}
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
// PrmObjectDelete groups parameters of NeoFS.DeleteObject operation.
type PrmObjectDelete struct {
// Authentication parameters.
PrmAuth
@ -128,12 +122,12 @@ type PrmObjectDelete struct {
Object oid.ID
}
// ErrAccessDenied is returned from FrostFS in case of access violation.
// ErrAccessDenied is returned from NeoFS in case of access violation.
var ErrAccessDenied = errors.New("access denied")
// FrostFS represents virtual connection to FrostFS network.
type FrostFS interface {
// CreateContainer creates and saves parameterized container in FrostFS.
// NeoFS represents virtual connection to NeoFS network.
type NeoFS interface {
// CreateContainer creates and saves parameterized container in NeoFS.
// It sets 'Timestamp' attribute to the current time.
// It returns the ID of the saved container.
//
@ -143,7 +137,7 @@ type FrostFS interface {
// prevented the container from being created.
CreateContainer(context.Context, PrmContainerCreate) (cid.ID, error)
// Container reads a container from FrostFS by ID.
// Container reads a container from NeoFS by ID.
//
// It returns exactly one non-nil value. It returns any error encountered which
// prevented the container from being read.
@ -155,26 +149,26 @@ type FrostFS interface {
// prevented the containers from being listed.
UserContainers(context.Context, user.ID) ([]cid.ID, error)
// SetContainerEACL saves the eACL table of the container in FrostFS. The
// SetContainerEACL saves the eACL table of the container in NeoFS. The
// extended ACL is modified within session if session token is not nil.
//
// It returns any error encountered which prevented the eACL from being saved.
SetContainerEACL(context.Context, eacl.Table, *session.Container) error
// ContainerEACL reads the container eACL from FrostFS by the container ID.
// ContainerEACL reads the container eACL from NeoFS by the container ID.
//
// It returns exactly one non-nil value. It returns any error encountered which
// prevented the eACL from being read.
ContainerEACL(context.Context, cid.ID) (*eacl.Table, error)
// DeleteContainer marks the container to be removed from FrostFS by ID.
// DeleteContainer marks the container to be removed from NeoFS by ID.
// Request is sent within session if the session token is specified.
// Successful return does not guarantee actual removal.
//
// It returns any error encountered which prevented the removal request from being sent.
DeleteContainer(context.Context, cid.ID, *session.Container) error
// ReadObject reads a part of the object from the FrostFS container by identifier.
// ReadObject reads a part of the object from the NeoFS container by identifier.
// Exact part is returned according to the parameters:
// * with header only: empty payload (both in-mem and reader parts are nil);
// * with payload only: header is nil (zero range means full payload);
@ -190,7 +184,7 @@ type FrostFS interface {
// prevented the object header from being read.
ReadObject(context.Context, PrmObjectRead) (*ObjectPart, error)
// CreateObject creates and saves a parameterized object in the FrostFS container.
// CreateObject creates and saves a parameterized object in the NeoFS container.
// It sets 'Timestamp' attribute to the current time.
// It returns the ID of the saved object.
//
@ -202,7 +196,7 @@ type FrostFS interface {
// prevented the container from being created.
CreateObject(context.Context, PrmObjectCreate) (oid.ID, error)
// DeleteObject marks the object to be removed from the FrostFS container by identifier.
// DeleteObject marks the object to be removed from the NeoFS container by identifier.
// Successful return does not guarantee actual removal.
//
// It returns ErrAccessDenied on remove access violation.
@ -210,11 +204,11 @@ type FrostFS interface {
// It returns any error encountered which prevented the removal request from being sent.
DeleteObject(context.Context, PrmObjectDelete) error
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided now and future time.
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided time.
// Note:
// * future time must be after the now
// * future time will be ceil rounded to match epoch
// * time must be in the future
// * time will be ceil rounded to match epoch
//
// It returns any error encountered which prevented computing epochs.
TimeToEpoch(ctx context.Context, now time.Time, future time.Time) (uint64, uint64, error)
TimeToEpoch(context.Context, time.Time) (uint64, uint64, error)
}

View file

@ -10,22 +10,22 @@ import (
"io"
"time"
objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
objectv2 "github.com/nspcc-dev/neofs-api-go/v2/object"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
"github.com/nspcc-dev/neofs-sdk-go/checksum"
"github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/user"
)
type TestFrostFS struct {
FrostFS
type TestNeoFS struct {
NeoFS
objects map[string]*object.Object
containers map[string]*container.Container
@ -33,19 +33,19 @@ type TestFrostFS struct {
currentEpoch uint64
}
func NewTestFrostFS() *TestFrostFS {
return &TestFrostFS{
func NewTestNeoFS() *TestNeoFS {
return &TestNeoFS{
objects: make(map[string]*object.Object),
containers: make(map[string]*container.Container),
eaclTables: make(map[string]*eacl.Table),
}
}
func (t *TestFrostFS) CurrentEpoch() uint64 {
func (t *TestNeoFS) CurrentEpoch() uint64 {
return t.currentEpoch
}
func (t *TestFrostFS) Objects() []*object.Object {
func (t *TestNeoFS) Objects() []*object.Object {
res := make([]*object.Object, 0, len(t.objects))
for _, obj := range t.objects {
@ -55,11 +55,11 @@ func (t *TestFrostFS) Objects() []*object.Object {
return res
}
func (t *TestFrostFS) AddObject(key string, obj *object.Object) {
func (t *TestNeoFS) AddObject(key string, obj *object.Object) {
t.objects[key] = obj
}
func (t *TestFrostFS) ContainerID(name string) (cid.ID, error) {
func (t *TestNeoFS) ContainerID(name string) (cid.ID, error) {
for id, cnr := range t.containers {
if container.Name(*cnr) == name {
var cnrID cid.ID
@ -69,18 +69,13 @@ func (t *TestFrostFS) ContainerID(name string) (cid.ID, error) {
return cid.ID{}, fmt.Errorf("not found")
}
func (t *TestFrostFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (cid.ID, error) {
func (t *TestNeoFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (cid.ID, error) {
var cnr container.Container
cnr.Init()
cnr.SetOwner(prm.Creator)
cnr.SetPlacementPolicy(prm.Policy)
cnr.SetBasicACL(prm.BasicACL)
creationTime := prm.CreationTime
if creationTime.IsZero() {
creationTime = time.Now()
}
container.SetCreationTime(&cnr, creationTime)
container.SetCreationTime(&cnr, time.Now())
if prm.Name != "" {
var d container.Domain
@ -106,13 +101,13 @@ func (t *TestFrostFS) CreateContainer(_ context.Context, prm PrmContainerCreate)
return id, nil
}
func (t *TestFrostFS) DeleteContainer(_ context.Context, cnrID cid.ID, _ *session.Container) error {
func (t *TestNeoFS) DeleteContainer(_ context.Context, cnrID cid.ID, _ *session.Container) error {
delete(t.containers, cnrID.EncodeToString())
return nil
}
func (t *TestFrostFS) Container(_ context.Context, id cid.ID) (*container.Container, error) {
func (t *TestNeoFS) Container(_ context.Context, id cid.ID) (*container.Container, error) {
for k, v := range t.containers {
if k == id.EncodeToString() {
return v, nil
@ -122,7 +117,7 @@ func (t *TestFrostFS) Container(_ context.Context, id cid.ID) (*container.Contai
return nil, fmt.Errorf("container not found %s", id)
}
func (t *TestFrostFS) UserContainers(_ context.Context, _ user.ID) ([]cid.ID, error) {
func (t *TestNeoFS) UserContainers(_ context.Context, _ user.ID) ([]cid.ID, error) {
var res []cid.ID
for k := range t.containers {
var idCnr cid.ID
@ -135,7 +130,7 @@ func (t *TestFrostFS) UserContainers(_ context.Context, _ user.ID) ([]cid.ID, er
return res, nil
}
func (t *TestFrostFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectPart, error) {
func (t *TestNeoFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*ObjectPart, error) {
var addr oid.Address
addr.SetContainer(prm.Container)
addr.SetObject(prm.Object)
@ -144,7 +139,7 @@ func (t *TestFrostFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*Objec
if obj, ok := t.objects[sAddr]; ok {
owner := getOwner(ctx)
if !obj.OwnerID().Equals(owner) && !t.isPublicRead(prm.Container) {
if !obj.OwnerID().Equals(owner) {
return nil, ErrAccessDenied
}
@ -164,7 +159,7 @@ func (t *TestFrostFS) ReadObject(ctx context.Context, prm PrmObjectRead) (*Objec
return nil, fmt.Errorf("object not found %s", addr)
}
func (t *TestFrostFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oid.ID, error) {
func (t *TestNeoFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oid.ID, error) {
b := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, b); err != nil {
return oid.ID{}, err
@ -223,7 +218,7 @@ func (t *TestFrostFS) CreateObject(ctx context.Context, prm PrmObjectCreate) (oi
return objID, nil
}
func (t *TestFrostFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
func (t *TestNeoFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
var addr oid.Address
addr.SetContainer(prm.Container)
addr.SetObject(prm.Object)
@ -240,11 +235,11 @@ func (t *TestFrostFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) err
return nil
}
func (t *TestFrostFS) TimeToEpoch(_ context.Context, now, futureTime time.Time) (uint64, uint64, error) {
return t.currentEpoch, t.currentEpoch + uint64(futureTime.Sub(now).Seconds()), nil
func (t *TestNeoFS) TimeToEpoch(_ context.Context, futureTime time.Time) (uint64, uint64, error) {
return t.currentEpoch, t.currentEpoch + uint64(futureTime.Second()), nil
}
func (t *TestFrostFS) AllObjects(cnrID cid.ID) []oid.ID {
func (t *TestNeoFS) AllObjects(cnrID cid.ID) []oid.ID {
result := make([]oid.ID, 0, len(t.objects))
for _, val := range t.objects {
@ -258,7 +253,7 @@ func (t *TestFrostFS) AllObjects(cnrID cid.ID) []oid.ID {
return result
}
func (t *TestFrostFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *session.Container) error {
func (t *TestNeoFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *session.Container) error {
cnrID, ok := table.CID()
if !ok {
return errors.New("invalid cid")
@ -273,7 +268,7 @@ func (t *TestFrostFS) SetContainerEACL(_ context.Context, table eacl.Table, _ *s
return nil
}
func (t *TestFrostFS) ContainerEACL(_ context.Context, cnrID cid.ID) (*eacl.Table, error) {
func (t *TestNeoFS) ContainerEACL(_ context.Context, cnrID cid.ID) (*eacl.Table, error) {
table, ok := t.eaclTables[cnrID.EncodeToString()]
if !ok {
return nil, errors.New("not found")
@ -282,25 +277,6 @@ func (t *TestFrostFS) ContainerEACL(_ context.Context, cnrID cid.ID) (*eacl.Tabl
return table, nil
}
func (t *TestFrostFS) isPublicRead(cnrID cid.ID) bool {
table, ok := t.eaclTables[cnrID.EncodeToString()]
if !ok {
return false
}
for _, rec := range table.Records() {
if rec.Operation() == eacl.OperationGet && len(rec.Filters()) == 0 {
for _, trgt := range rec.Targets() {
if trgt.Role() == eacl.RoleOthers {
return rec.Action() == eacl.ActionAllow
}
}
}
}
return false
}
func getOwner(ctx context.Context) user.ID {
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
return bearer.ResolveIssuer(*bd.Gate.BearerToken)

View file

@ -7,8 +7,8 @@ import (
errorsStd "errors"
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"go.uber.org/zap"
)
@ -30,7 +30,6 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
Creator: p.BktInfo.Owner,
Payload: bytes.NewReader(confXML),
Filepath: p.BktInfo.NotificationConfigurationObjectName(),
CreationTime: TimeNow(ctx),
CopiesNumber: p.CopiesNumber,
}

View file

@ -13,16 +13,17 @@ import (
"strconv"
"strings"
"sync"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/minio/sio"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-sdk-go/client"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/panjf2000/ants/v2"
"go.uber.org/zap"
)
@ -90,7 +91,7 @@ func (n *layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
res, err := n.frostFS.ReadObject(ctx, prm)
res, err := n.neoFS.ReadObject(ctx, prm)
if err != nil {
return nil, err
}
@ -98,7 +99,7 @@ func (n *layer) objectHead(ctx context.Context, bktInfo *data.BucketInfo, idObj
return res.Head, nil
}
// initializes payload reader of the FrostFS object.
// initializes payload reader of the NeoFS object.
// Zero range corresponds to full payload (panics if only offset is set).
func (n *layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Reader, error) {
prm := PrmObjectRead{
@ -110,7 +111,7 @@ func (n *layer) initObjectPayloadReader(ctx context.Context, p getParams) (io.Re
n.prepareAuthParameters(ctx, &prm.PrmAuth, p.bktInfo.Owner)
res, err := n.frostFS.ReadObject(ctx, prm)
res, err := n.neoFS.ReadObject(ctx, prm)
if err != nil {
return nil, err
}
@ -129,7 +130,7 @@ func (n *layer) objectGet(ctx context.Context, bktInfo *data.BucketInfo, objID o
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
res, err := n.frostFS.ReadObject(ctx, prm)
res, err := n.neoFS.ReadObject(ctx, prm)
if err != nil {
return nil, err
}
@ -182,7 +183,7 @@ func ParseCompletedPartHeader(hdr string) (*Part, error) {
}, nil
}
// PutObject stores object into FrostFS, took payload from io.Reader.
// PutObject stores object into NeoFS, took payload from io.Reader.
func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.ExtendedObjectInfo, error) {
owner := n.Owner(ctx)
@ -233,7 +234,6 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
PayloadSize: uint64(p.Size),
Filepath: p.Object,
Payload: r,
CreationTime: TimeNow(ctx),
CopiesNumber: p.CopiesNumber,
}
@ -248,12 +248,6 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
return nil, err
}
reqInfo := api.GetReqInfo(ctx)
n.log.Debug("put object",
zap.String("reqId", reqInfo.RequestID),
zap.String("bucket", p.BktInfo.Name), zap.Stringer("cid", p.BktInfo.CID),
zap.String("object", p.Object), zap.Stringer("oid", id))
newVersion.OID = id
newVersion.ETag = hex.EncodeToString(hash)
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
@ -287,7 +281,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
Bucket: p.BktInfo.Name,
Name: p.Object,
Size: p.Size,
Created: prm.CreationTime,
Created: time.Now(),
Headers: p.Header,
ContentType: p.Header[api.ContentType],
HashSum: newVersion.ETag,
@ -389,7 +383,7 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
return extObjInfo, nil
}
// objectDelete puts tombstone object into frostfs.
// objectDelete puts tombstone object into neofs.
func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idObj oid.ID) error {
prm := PrmObjectDelete{
Container: bktInfo.CID,
@ -400,10 +394,10 @@ func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idOb
n.cache.DeleteObject(newAddress(bktInfo.CID, idObj))
return n.frostFS.DeleteObject(ctx, prm)
return n.neoFS.DeleteObject(ctx, prm)
}
// objectPutAndHash prepare auth parameters and invoke frostfs.CreateObject.
// objectPutAndHash prepare auth parameters and invoke neofs.CreateObject.
// Returns object ID and payload sha256 hash.
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (oid.ID, []byte, error) {
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
@ -411,7 +405,7 @@ func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktIn
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
hash.Write(buf)
})
id, err := n.frostFS.CreateObject(ctx, prm)
id, err := n.neoFS.CreateObject(ctx, prm)
if err != nil {
return oid.ID{}, nil, err
}
@ -583,10 +577,10 @@ func (n *layer) initWorkerPool(ctx context.Context, size int, p allObjectParams,
wg.Add(1)
err = pool.Submit(func() {
defer wg.Done()
oi := n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter)
oi := n.objectInfoFromObjectsCacheOrNeoFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter)
if oi == nil {
// try to get object again
if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter); oi == nil {
if oi = n.objectInfoFromObjectsCacheOrNeoFS(ctx, p.Bucket, node, p.Prefix, p.Delimiter); oi == nil {
// form object info with data that the tree node contains
oi = getPartialObjectInfo(p.Bucket, node)
}
@ -652,14 +646,14 @@ func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo,
for _, nodeVersion := range nodeVersions {
oi := &data.ObjectInfo{}
if nodeVersion.IsDeleteMarker() { // delete marker does not match any object in FrostFS
if nodeVersion.IsDeleteMarker() { // delete marker does not match any object in NeoFS
oi.ID = nodeVersion.OID
oi.Name = nodeVersion.FilePath
oi.Owner = nodeVersion.DeleteMarker.Owner
oi.Created = nodeVersion.DeleteMarker.Created
oi.IsDeleteMarker = true
} else {
if oi = n.objectInfoFromObjectsCacheOrFrostFS(ctx, bkt, nodeVersion, prefix, delimiter); oi == nil {
if oi = n.objectInfoFromObjectsCacheOrNeoFS(ctx, bkt, nodeVersion, prefix, delimiter); oi == nil {
continue
}
}
@ -683,7 +677,7 @@ func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo,
func IsSystemHeader(key string) bool {
_, ok := api.SystemMetadata[key]
return ok || strings.HasPrefix(key, api.FrostFSSystemMetadataPrefix)
return ok || strings.HasPrefix(key, api.NeoFSSystemMetadataPrefix)
}
func shouldSkip(node *data.NodeVersion, p allObjectParams, existed map[string]struct{}) bool {
@ -740,7 +734,7 @@ func triageExtendedObjects(allObjects []*data.ExtendedObjectInfo) (prefixes []st
return
}
func (n *layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo *data.BucketInfo, node *data.NodeVersion, prefix, delimiter string) (oi *data.ObjectInfo) {
func (n *layer) objectInfoFromObjectsCacheOrNeoFS(ctx context.Context, bktInfo *data.BucketInfo, node *data.NodeVersion, prefix, delimiter string) (oi *data.ObjectInfo) {
if oiDir := tryDirectory(bktInfo, node, prefix, delimiter); oiDir != nil {
return oiDir
}

View file

@ -9,14 +9,14 @@ import (
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
const (
AttributeComplianceMode = ".s3-compliance-mode"
AttributeComplianceMode = ".s3-compliance-mode"
AttributeExpirationEpoch = "__NEOFS__EXPIRATION_EPOCH"
)
type PutLockInfoParams struct {
@ -32,7 +32,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
// sometimes node version can be provided from executing context
// if not, then receive node version from tree service
if versionNode == nil {
versionNode, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjVersion)
versionNode, err = n.getNodeVersionFromCacheOrNeofs(ctx, p.ObjVersion)
if err != nil {
return err
}
@ -100,7 +100,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
return nil
}
func (n *layer) getNodeVersionFromCacheOrFrostfs(ctx context.Context, objVersion *ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
func (n *layer) getNodeVersionFromCacheOrNeofs(ctx context.Context, objVersion *ObjectVersion) (nodeVersion *data.NodeVersion, err error) {
// check cache if node version is stored inside extendedObjectVersion
nodeVersion = n.getNodeVersionFromCache(n.Owner(ctx), objVersion)
if nodeVersion == nil {
@ -116,7 +116,6 @@ func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
Container: bktInfo.CID,
Creator: bktInfo.Owner,
Locks: []oid.ID{objID},
CreationTime: TimeNow(ctx),
CopiesNumber: copiesNumber,
}
@ -228,7 +227,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
)
if lock.Retention != nil {
if _, expEpoch, err = n.frostFS.TimeToEpoch(ctx, TimeNow(ctx), lock.Retention.Until); err != nil {
if _, expEpoch, err = n.neoFS.TimeToEpoch(ctx, lock.Retention.Until); err != nil {
return nil, fmt.Errorf("fetch time to epoch: %w", err)
}
@ -238,7 +237,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
}
if lock.LegalHold != nil && lock.LegalHold.Enabled {
// todo: (@KirillovDenis) reconsider this when FrostFS will support Legal Hold https://git.frostfs.info/TrueCloudLab/frostfs-contract/issues/2
// todo: (@KirillovDenis) reconsider this when NeoFS will support Legal Hold https://github.com/nspcc-dev/neofs-contract/issues/247
// Currently lock object must have an expiration epoch.
// Besides we need to override retention expiration epoch since legal hold cannot be deleted yet.
expEpoch = math.MaxUint64
@ -246,7 +245,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
if expEpoch != 0 {
result = append(result, [2]string{
object.SysAttributeExpEpoch, strconv.FormatUint(expEpoch, 10),
AttributeExpirationEpoch, strconv.FormatUint(expEpoch, 10),
})
}

View file

@ -4,13 +4,11 @@ import (
"context"
errorsStd "errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"go.uber.org/zap"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
)
type GetObjectTaggingParams struct {
@ -40,7 +38,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
nodeVersion := p.NodeVersion
if nodeVersion == nil {
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
nodeVersion, err = n.getNodeVersionFromCacheOrNeofs(ctx, p.ObjectVersion)
if err != nil {
return "", nil, err
}
@ -67,7 +65,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams)
func (n *layer) PutObjectTagging(ctx context.Context, p *PutObjectTaggingParams) (nodeVersion *data.NodeVersion, err error) {
nodeVersion = p.NodeVersion
if nodeVersion == nil {
nodeVersion, err = n.getNodeVersionFromCacheOrFrostfs(ctx, p.ObjectVersion)
nodeVersion, err = n.getNodeVersionFromCacheOrNeofs(ctx, p.ObjectVersion)
if err != nil {
return nil, err
}
@ -177,14 +175,6 @@ func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (
return nil, errors.GetAPIError(errors.ErrNoSuchKey)
}
if err == nil && version != nil && !version.IsDeleteMarker() {
reqInfo := api.GetReqInfo(ctx)
n.log.Debug("target details",
zap.String("reqId", reqInfo.RequestID),
zap.String("bucket", objVersion.BktInfo.Name), zap.Stringer("cid", objVersion.BktInfo.CID),
zap.String("object", objVersion.ObjectName), zap.Stringer("oid", version.OID))
}
return version, err
}

View file

@ -6,8 +6,8 @@ import (
"sort"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
type TreeServiceMock struct {
@ -109,32 +109,11 @@ func (t *TreeServiceMock) PutNotificationConfigurationNode(ctx context.Context,
}
func (t *TreeServiceMock) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
if !ok {
return oid.ID{}, nil
}
node, ok := systemMap["cors"]
if !ok {
return oid.ID{}, nil
}
return node.OID, nil
panic("implement me")
}
func (t *TreeServiceMock) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) {
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
if !ok {
systemMap = make(map[string]*data.BaseNodeVersion)
}
systemMap["cors"] = &data.BaseNodeVersion{
OID: objID,
}
t.system[bktInfo.CID.EncodeToString()] = systemMap
return oid.ID{}, ErrNoNodeToRemove
panic("implement me")
}
func (t *TreeServiceMock) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {

View file

@ -4,8 +4,8 @@ import (
"context"
"errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
// TreeService provide interface to interact with tree service using s3 data models.
@ -24,7 +24,7 @@ type TreeService interface {
GetNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
// PutNotificationConfigurationNode puts a node to a system tree
// and returns objectID of a previous notif config which must be deleted in FrostFS.
// and returns objectID of a previous notif config which must be deleted in NeoFS.
//
// If object id to remove is not found returns ErrNoNodeToRemove error.
PutNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
@ -34,12 +34,12 @@ type TreeService interface {
// If object id is not found returns ErrNodeNotFound error.
GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in FrostFS.
// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in NeoFS.
//
// If object id to remove is not found returns ErrNoNodeToRemove error.
PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in FrostFS.
// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in NeoFS.
//
// If object id to remove is not found returns ErrNoNodeToRemove error.
DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
@ -69,7 +69,7 @@ type TreeService interface {
GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error)
// AddPart puts a node to a system tree as a child of appropriate multipart upload
// and returns objectID of a previous part which must be deleted in FrostFS.
// and returns objectID of a previous part which must be deleted in NeoFS.
//
// If object id to remove is not found returns ErrNoNodeToRemove error.
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)

View file

@ -9,11 +9,11 @@ import (
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/object"
)
type (

View file

@ -6,11 +6,11 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-sdk-go/checksum"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
"github.com/stretchr/testify/require"
)

View file

@ -4,7 +4,7 @@ import (
"context"
"sort"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
)
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {

View file

@ -5,14 +5,14 @@ import (
"context"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
bearertest "github.com/nspcc-dev/neofs-sdk-go/bearer/test"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
@ -113,7 +113,7 @@ func (tc *testContext) checkListObjects(ids ...oid.ID) {
}
func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
for _, obj := range tc.testFrostFS.Objects() {
for _, obj := range tc.testNeoFS.Objects() {
id, _ := obj.ID()
if id.Equals(objID) {
return obj
@ -123,12 +123,12 @@ func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
}
type testContext struct {
t *testing.T
ctx context.Context
layer Client
bktInfo *data.BucketInfo
obj string
testFrostFS *TestFrostFS
t *testing.T
ctx context.Context
layer Client
bktInfo *data.BucketInfo
obj string
testNeoFS *TestNeoFS
}
func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
@ -146,7 +146,7 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
GateKey: key.PublicKey(),
},
})
tp := NewTestFrostFS()
tp := NewTestNeoFS()
bktName := "testbucket1"
bktID, err := tp.CreateContainer(ctx, PrmContainerCreate{
@ -176,9 +176,9 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
Owner: owner,
CID: bktID,
},
obj: "obj1",
t: t,
testFrostFS: tp,
obj: "obj1",
t: t,
testNeoFS: tp,
}
}

View file

@ -4,7 +4,7 @@ import (
"net/http"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
)
type (

View file

@ -1,305 +0,0 @@
package api
import (
"context"
"io"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"github.com/prometheus/client_golang/prometheus"
)
type RequestType int
const (
UNKNOWNRequest RequestType = iota
HEADRequest RequestType = iota
PUTRequest RequestType = iota
LISTRequest RequestType = iota
GETRequest RequestType = iota
DELETERequest RequestType = iota
)
func (t RequestType) String() string {
switch t {
case 1:
return "HEAD"
case 2:
return "PUT"
case 3:
return "LIST"
case 4:
return "GET"
case 5:
return "DELETE"
default:
return "Unknown"
}
}
func RequestTypeFromAPI(api string) RequestType {
switch api {
case "Options", "HeadObject", "HeadBucket":
return HEADRequest
case "CreateMultipartUpload", "UploadPartCopy", "UploadPart", "CompleteMultipartUpload",
"PutObjectACL", "PutObjectTagging", "CopyObject", "PutObjectRetention", "PutObjectLegalHold",
"PutObject", "PutBucketCors", "PutBucketACL", "PutBucketLifecycle", "PutBucketEncryption",
"PutBucketPolicy", "PutBucketObjectLockConfig", "PutBucketTagging", "PutBucketVersioning",
"PutBucketNotification", "CreateBucket", "PostObject":
return PUTRequest
case "ListObjectParts", "ListMultipartUploads", "ListObjectsV2M", "ListObjectsV2", "ListBucketVersions",
"ListObjectsV1", "ListBuckets":
return LISTRequest
case "GetObjectACL", "GetObjectTagging", "SelectObjectContent", "GetObjectRetention", "getobjectlegalhold",
"GetObjectAttributes", "GetObject", "GetBucketLocation", "GetBucketPolicy",
"GetBucketLifecycle", "GetBucketEncryption", "GetBucketCors", "GetBucketACL",
"GetBucketWebsite", "GetBucketAccelerate", "GetBucketRequestPayment", "GetBucketLogging",
"GetBucketReplication", "GetBucketTagging", "GetBucketObjectLockConfig",
"GetBucketVersioning", "GetBucketNotification", "ListenBucketNotification":
return GETRequest
case "AbortMultipartUpload", "DeleteObjectTagging", "DeleteObject", "DeleteBucketCors",
"DeleteBucketWebsite", "DeleteBucketTagging", "DeleteMultipleObjects", "DeleteBucketPolicy",
"DeleteBucketLifecycle", "DeleteBucketEncryption", "DeleteBucket":
return DELETERequest
default:
return UNKNOWNRequest
}
}
type (
// HTTPAPIStats holds statistics information about
// the API given in the requests.
HTTPAPIStats struct {
apiStats map[string]int
sync.RWMutex
}
UsersStat interface {
Update(user, bucket, cnrID string, reqType RequestType, in, out uint64)
}
// HTTPStats holds statistics information about
// HTTP requests made by all clients.
HTTPStats struct {
currentS3Requests HTTPAPIStats
totalS3Requests HTTPAPIStats
totalS3Errors HTTPAPIStats
totalInputBytes uint64
totalOutputBytes uint64
}
readCounter struct {
io.ReadCloser
countBytes uint64
}
writeCounter struct {
http.ResponseWriter
countBytes uint64
}
responseWrapper struct {
sync.Once
http.ResponseWriter
statusCode int
startTime time.Time
}
)
const systemPath = "/system"
var (
httpStatsMetric = new(HTTPStats)
httpRequestsDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "frostfs_s3_request_seconds",
Help: "Time taken by requests served by current FrostFS S3 Gate instance",
Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10},
},
[]string{"api"},
)
)
// Collects HTTP metrics for FrostFS S3 Gate in Prometheus specific format
// and sends to the given channel.
func collectHTTPMetrics(ch chan<- prometheus.Metric) {
for api, value := range httpStatsMetric.currentS3Requests.Load() {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("frostfs_s3", "requests", "current"),
"Total number of running s3 requests in current FrostFS S3 Gate instance",
[]string{"api"}, nil),
prometheus.CounterValue,
float64(value),
api,
)
}
for api, value := range httpStatsMetric.totalS3Requests.Load() {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("frostfs_s3", "requests", "total"),
"Total number of s3 requests in current FrostFS S3 Gate instance",
[]string{"api"}, nil),
prometheus.CounterValue,
float64(value),
api,
)
}
for api, value := range httpStatsMetric.totalS3Errors.Load() {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("frostfs_s3", "errors", "total"),
"Total number of s3 errors in current FrostFS S3 Gate instance",
[]string{"api"}, nil),
prometheus.CounterValue,
float64(value),
api,
)
}
}
// CIDResolveFunc is a func to resolve CID in Stats handler.
type CIDResolveFunc func(ctx context.Context, reqInfo *ReqInfo) (cnrID string)
// Stats is a handler that update metrics.
func Stats(f http.HandlerFunc, resolveCID CIDResolveFunc, usersStat UsersStat) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
reqInfo := GetReqInfo(r.Context())
httpStatsMetric.currentS3Requests.Inc(reqInfo.API)
defer httpStatsMetric.currentS3Requests.Dec(reqInfo.API)
in := &readCounter{ReadCloser: r.Body}
out := &writeCounter{ResponseWriter: w}
r.Body = in
statsWriter := &responseWrapper{
ResponseWriter: out,
startTime: time.Now(),
}
f(statsWriter, r)
// Time duration in secs since the call started.
// We don't need to do nanosecond precision here
// simply for the fact that it is not human-readable.
durationSecs := time.Since(statsWriter.startTime).Seconds()
user := resolveUser(r.Context())
cnrID := resolveCID(r.Context(), reqInfo)
usersStat.Update(user, reqInfo.BucketName, cnrID, RequestTypeFromAPI(reqInfo.API), in.countBytes, out.countBytes)
code := statsWriter.statusCode
// A successful request has a 2xx response code
successReq := code >= http.StatusOK && code < http.StatusMultipleChoices
if !strings.HasSuffix(r.URL.Path, systemPath) {
httpStatsMetric.totalS3Requests.Inc(reqInfo.API)
if !successReq && code != 0 {
httpStatsMetric.totalS3Errors.Inc(reqInfo.API)
}
}
if r.Method == http.MethodGet {
// Increment the prometheus http request response histogram with appropriate label
httpRequestsDuration.With(prometheus.Labels{"api": reqInfo.API}).Observe(durationSecs)
}
atomic.AddUint64(&httpStatsMetric.totalInputBytes, in.countBytes)
atomic.AddUint64(&httpStatsMetric.totalOutputBytes, out.countBytes)
}
}
func resolveUser(ctx context.Context) string {
user := "anon"
if bd, ok := ctx.Value(BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
user = bearer.ResolveIssuer(*bd.Gate.BearerToken).String()
}
return user
}
// Inc increments the api stats counter.
func (stats *HTTPAPIStats) Inc(api string) {
if stats == nil {
return
}
stats.Lock()
defer stats.Unlock()
if stats.apiStats == nil {
stats.apiStats = make(map[string]int)
}
stats.apiStats[api]++
}
// Dec increments the api stats counter.
func (stats *HTTPAPIStats) Dec(api string) {
if stats == nil {
return
}
stats.Lock()
defer stats.Unlock()
if val, ok := stats.apiStats[api]; ok && val > 0 {
stats.apiStats[api]--
}
}
// Load returns the recorded stats.
func (stats *HTTPAPIStats) Load() map[string]int {
stats.Lock()
defer stats.Unlock()
var apiStats = make(map[string]int, len(stats.apiStats))
for k, v := range stats.apiStats {
apiStats[k] = v
}
return apiStats
}
func (st *HTTPStats) getInputBytes() uint64 {
return atomic.LoadUint64(&st.totalInputBytes)
}
func (st *HTTPStats) getOutputBytes() uint64 {
return atomic.LoadUint64(&st.totalOutputBytes)
}
// WriteHeader -- writes http status code.
func (w *responseWrapper) WriteHeader(code int) {
w.Do(func() {
w.statusCode = code
w.ResponseWriter.WriteHeader(code)
})
}
// Flush -- calls the underlying Flush.
func (w *responseWrapper) Flush() {
if f, ok := w.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func (w *writeCounter) Flush() {
if f, ok := w.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func (w *writeCounter) Write(p []byte) (int, error) {
n, err := w.ResponseWriter.Write(p)
atomic.AddUint64(&w.countBytes, uint64(n))
return n, err
}
func (r *readCounter) Read(p []byte) (int, error) {
n, err := r.ReadCloser.Read(p)
atomic.AddUint64(&r.countBytes, uint64(n))
return n, err
}

229
api/metrics/api.go Normal file
View file

@ -0,0 +1,229 @@
package metrics
import (
"io"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/prometheus/client_golang/prometheus"
)
type (
// HTTPAPIStats holds statistics information about
// the API given in the requests.
HTTPAPIStats struct {
apiStats map[string]int
sync.RWMutex
}
// HTTPStats holds statistics information about
// HTTP requests made by all clients.
HTTPStats struct {
currentS3Requests HTTPAPIStats
totalS3Requests HTTPAPIStats
totalS3Errors HTTPAPIStats
totalInputBytes uint64
totalOutputBytes uint64
}
readCounter struct {
io.ReadCloser
countBytes uint64
}
writeCounter struct {
http.ResponseWriter
countBytes uint64
}
responseWrapper struct {
sync.Once
http.ResponseWriter
statusCode int
startTime time.Time
}
)
const systemPath = "/system"
var (
httpStatsMetric = new(HTTPStats)
httpRequestsDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "neofs_s3_request_seconds",
Help: "Time taken by requests served by current NeoFS S3 Gate instance",
Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10},
},
[]string{"api"},
)
)
// Collects HTTP metrics for NeoFS S3 Gate in Prometheus specific format
// and sends to the given channel.
func collectHTTPMetrics(ch chan<- prometheus.Metric) {
for api, value := range httpStatsMetric.currentS3Requests.Load() {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("neofs_s3", "requests", "current"),
"Total number of running s3 requests in current NeoFS S3 Gate instance",
[]string{"api"}, nil),
prometheus.CounterValue,
float64(value),
api,
)
}
for api, value := range httpStatsMetric.totalS3Requests.Load() {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("neofs_s3", "requests", "total"),
"Total number of s3 requests in current NeoFS S3 Gate instance",
[]string{"api"}, nil),
prometheus.CounterValue,
float64(value),
api,
)
}
for api, value := range httpStatsMetric.totalS3Errors.Load() {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("neofs_s3", "errors", "total"),
"Total number of s3 errors in current NeoFS S3 Gate instance",
[]string{"api"}, nil),
prometheus.CounterValue,
float64(value),
api,
)
}
}
// APIStats wraps http handler for api with basic statistics collection.
func APIStats(api string, f http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
httpStatsMetric.currentS3Requests.Inc(api)
defer httpStatsMetric.currentS3Requests.Dec(api)
in := &readCounter{ReadCloser: r.Body}
out := &writeCounter{ResponseWriter: w}
r.Body = in
statsWriter := &responseWrapper{
ResponseWriter: out,
startTime: time.Now(),
}
f.ServeHTTP(statsWriter, r)
// Time duration in secs since the call started.
// We don't need to do nanosecond precision here
// simply for the fact that it is not human readable.
durationSecs := time.Since(statsWriter.startTime).Seconds()
httpStatsMetric.updateStats(api, statsWriter, r, durationSecs)
atomic.AddUint64(&httpStatsMetric.totalInputBytes, in.countBytes)
atomic.AddUint64(&httpStatsMetric.totalOutputBytes, out.countBytes)
}
}
// Inc increments the api stats counter.
func (stats *HTTPAPIStats) Inc(api string) {
if stats == nil {
return
}
stats.Lock()
defer stats.Unlock()
if stats.apiStats == nil {
stats.apiStats = make(map[string]int)
}
stats.apiStats[api]++
}
// Dec increments the api stats counter.
func (stats *HTTPAPIStats) Dec(api string) {
if stats == nil {
return
}
stats.Lock()
defer stats.Unlock()
if val, ok := stats.apiStats[api]; ok && val > 0 {
stats.apiStats[api]--
}
}
// Load returns the recorded stats.
func (stats *HTTPAPIStats) Load() map[string]int {
stats.Lock()
defer stats.Unlock()
var apiStats = make(map[string]int, len(stats.apiStats))
for k, v := range stats.apiStats {
apiStats[k] = v
}
return apiStats
}
func (st *HTTPStats) getInputBytes() uint64 {
return atomic.LoadUint64(&st.totalInputBytes)
}
func (st *HTTPStats) getOutputBytes() uint64 {
return atomic.LoadUint64(&st.totalOutputBytes)
}
// Update statistics from http request and response data.
func (st *HTTPStats) updateStats(api string, w http.ResponseWriter, r *http.Request, durationSecs float64) {
var code int
if res, ok := w.(*responseWrapper); ok {
code = res.statusCode
}
// A successful request has a 2xx response code
successReq := code >= http.StatusOK && code < http.StatusMultipleChoices
if !strings.HasSuffix(r.URL.Path, systemPath) {
st.totalS3Requests.Inc(api)
if !successReq && code != 0 {
st.totalS3Errors.Inc(api)
}
}
if r.Method == http.MethodGet {
// Increment the prometheus http request response histogram with appropriate label
httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(durationSecs)
}
}
// WriteHeader -- writes http status code.
func (w *responseWrapper) WriteHeader(code int) {
w.Do(func() {
w.statusCode = code
w.ResponseWriter.WriteHeader(code)
})
}
// Flush -- calls the underlying Flush.
func (w *responseWrapper) Flush() {
if f, ok := w.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func (w *writeCounter) Write(p []byte) (int, error) {
n, err := w.ResponseWriter.Write(p)
atomic.AddUint64(&w.countBytes, uint64(n))
return n, err
}
func (r *readCounter) Read(p []byte) (int, error) {
n, err := r.ReadCloser.Read(p)
atomic.AddUint64(&r.countBytes, uint64(n))
return n, err
}

View file

@ -1,7 +1,7 @@
package api
package metrics
import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
"github.com/nspcc-dev/neofs-s3-gw/internal/version"
"github.com/prometheus/client_golang/prometheus"
)
@ -12,9 +12,9 @@ type stats struct {
var (
versionInfo = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: "frostfs_s3",
Namespace: "neofs_s3",
Name: "version_info",
Help: "Version of current FrostFS S3 Gate instance",
Help: "Version of current NeoFS S3 Gate instance",
},
[]string{
// current version
@ -23,7 +23,7 @@ var (
)
statsMetrics = &stats{
desc: prometheus.NewDesc("frostfs_s3_stats", "Statistics exposed by FrostFS S3 Gate instance", nil, nil),
desc: prometheus.NewDesc("neofs_s3_stats", "Statistics exposed by NeoFS S3 Gate instance", nil, nil),
}
)
@ -37,8 +37,8 @@ func collectNetworkMetrics(ch chan<- prometheus.Metric) {
// Network Sent/Received Bytes (Outbound)
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("frostfs_s3", "tx", "bytes_total"),
"Total number of bytes sent by current FrostFS S3 Gate instance",
prometheus.BuildFQName("neofs_s3", "tx", "bytes_total"),
"Total number of bytes sent by current NeoFS S3 Gate instance",
nil, nil),
prometheus.CounterValue,
float64(httpStatsMetric.getInputBytes()),
@ -46,8 +46,8 @@ func collectNetworkMetrics(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName("frostfs_s3", "rx", "bytes_total"),
"Total number of bytes received by current FrostFS S3 Gate instance",
prometheus.BuildFQName("neofs_s3", "rx", "bytes_total"),
"Total number of bytes received by current NeoFS S3 Gate instance",
nil, nil),
prometheus.CounterValue,
float64(httpStatsMetric.getOutputBytes()),

View file

@ -7,9 +7,9 @@ import (
"sync"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"github.com/nats-io/nats.go"
"github.com/nspcc-dev/neofs-s3-gw/api/handler"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"go.uber.org/zap"
)
@ -61,7 +61,7 @@ type (
EventRecord struct {
EventVersion string `json:"eventVersion"`
EventSource string `json:"eventSource"` // frostfs:s3
EventSource string `json:"eventSource"` // neofs:s3
AWSRegion string `json:"awsRegion,omitempty"` // empty
EventTime time.Time `json:"eventTime"`
EventName string `json:"eventName"`
@ -197,11 +197,11 @@ func (c *Controller) SendNotifications(topics map[string]string, p *handler.Send
return nil
}
func (c *Controller) SendTestNotification(topic, bucketName, requestID, HostID string, now time.Time) error {
func (c *Controller) SendTestNotification(topic, bucketName, requestID, HostID string) error {
event := &TestEvent{
Service: "FrostFS S3",
Service: "NeoFS S3",
Event: "s3:TestEvent",
Time: now,
Time: time.Now(),
Bucket: bucketName,
RequestID: requestID,
HostID: HostID,
@ -220,9 +220,9 @@ func prepareEvent(p *handler.SendNotificationParams) *Event {
Records: []EventRecord{
{
EventVersion: EventVersion21,
EventSource: "frostfs:s3",
EventSource: "neofs:s3",
AWSRegion: "",
EventTime: p.Time,
EventTime: time.Now(),
EventName: p.Event,
UserIdentity: UserIdentity{
PrincipalID: p.User,

View file

@ -45,7 +45,7 @@ type (
// Key used for Get/SetReqInfo.
type contextKeyType string
const ctxRequestInfo = contextKeyType("FrostFS-S3-GW")
const ctxRequestInfo = contextKeyType("NeoFS-S3-GW")
var (
// De-facto standard header keys.

View file

@ -6,9 +6,9 @@ import (
"fmt"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
"github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/ns"
)
const (
@ -19,9 +19,9 @@ const (
// ErrNoResolvers returns when trying to resolve container without any resolver.
var ErrNoResolvers = errors.New("no resolvers")
// FrostFS represents virtual connection to the FrostFS network.
type FrostFS interface {
// SystemDNS reads system DNS network parameters of the FrostFS.
// NeoFS represents virtual connection to the NeoFS network.
type NeoFS interface {
// SystemDNS reads system DNS network parameters of the NeoFS.
//
// It returns exactly on non-zero value. It returns any error encountered
// which prevented the parameter from being read.
@ -29,7 +29,7 @@ type FrostFS interface {
}
type Config struct {
FrostFS FrostFS
NeoFS NeoFS
RPCAddress string
}
@ -134,7 +134,7 @@ func (r *BucketResolver) equals(resolverNames []string) bool {
func newResolver(name string, cfg *Config) (*Resolver, error) {
switch name {
case DNSResolver:
return NewDNSResolver(cfg.FrostFS)
return NewDNSResolver(cfg.NeoFS)
case NNSResolver:
return NewNNSResolver(cfg.RPCAddress)
default:
@ -142,17 +142,17 @@ func newResolver(name string, cfg *Config) (*Resolver, error) {
}
}
func NewDNSResolver(frostFS FrostFS) (*Resolver, error) {
if frostFS == nil {
func NewDNSResolver(neoFS NeoFS) (*Resolver, error) {
if neoFS == nil {
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
}
var dns ns.DNS
resolveFunc := func(ctx context.Context, name string) (cid.ID, error) {
domain, err := frostFS.SystemDNS(ctx)
domain, err := neoFS.SystemDNS(ctx)
if err != nil {
return cid.ID{}, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err)
return cid.ID{}, fmt.Errorf("read system DNS parameter of the NeoFS: %w", err)
}
domain = name + "." + domain

View file

@ -7,9 +7,9 @@ import (
"net/http"
"strconv"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
"github.com/google/uuid"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"github.com/nspcc-dev/neofs-s3-gw/internal/version"
)
type (
@ -117,7 +117,7 @@ func WriteErrorResponse(w http.ResponseWriter, reqInfo *ReqInfo, err error) int
code = e.HTTPStatusCode
switch e.Code {
case "SlowDown", "XFrostFSServerNotInitialized", "XFrostFSReadQuorum", "XFrostFSWriteQuorum":
case "SlowDown", "XNeoFSServerNotInitialized", "XNeoFSReadQuorum", "XNeoFSWriteQuorum":
// Set retry-after header to indicate user-agents to retry request after 120secs.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
w.Header().Set(hdrRetryAfter, "120")
@ -133,13 +133,6 @@ func WriteErrorResponse(w http.ResponseWriter, reqInfo *ReqInfo, err error) int
return code
}
// WriteErrorResponseNoHeader writes XML encoded error to the response body.
func WriteErrorResponseNoHeader(w http.ResponseWriter, reqInfo *ReqInfo, err error) {
errorResponse := getAPIErrorResponse(reqInfo, err)
encodedErrorResponse := EncodeResponse(errorResponse)
WriteResponseBody(w, encodedErrorResponse)
}
// If none of the http routes match respond with appropriate errors.
func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)
@ -179,11 +172,6 @@ func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType
return
}
WriteResponseBody(w, response)
}
// WriteResponseBody writes response into w.
func WriteResponseBody(w http.ResponseWriter, response []byte) {
_, _ = w.Write(response)
if flusher, ok := w.(http.Flusher); ok {
flusher.Flush()
@ -200,30 +188,13 @@ func EncodeResponse(response interface{}) []byte {
return bytesBuffer.Bytes()
}
// EncodeResponseNoHeader encodes response without setting xml.Header.
// Should be used with periodicXMLWriter which sends xml.Header to the client
// with whitespaces to keep connection alive.
func EncodeResponseNoHeader(response interface{}) []byte {
var bytesBuffer bytes.Buffer
_ = xml.NewEncoder(&bytesBuffer).Encode(response)
return bytesBuffer.Bytes()
}
// EncodeToResponse encodes the response into ResponseWriter.
func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
w.WriteHeader(http.StatusOK)
if _, err := w.Write(xmlHeader); err != nil {
return fmt.Errorf("write headers: %w", err)
}
return EncodeToResponseNoHeader(w, response)
}
// EncodeToResponseNoHeader encodes the response into ResponseWriter without
// header status.
func EncodeToResponseNoHeader(w http.ResponseWriter, response interface{}) error {
if err := xml.NewEncoder(w).Encode(response); err != nil {
} else if err = xml.NewEncoder(w).Encode(response); err != nil {
return fmt.Errorf("encode xml response: %w", err)
}

View file

@ -5,10 +5,10 @@ import (
"net/http"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"github.com/google/uuid"
"github.com/gorilla/mux"
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
"github.com/nspcc-dev/neofs-s3-gw/api/metrics"
"go.uber.org/zap"
"google.golang.org/grpc/metadata"
)
@ -82,8 +82,6 @@ type (
AbortMultipartUploadHandler(http.ResponseWriter, *http.Request)
ListPartsHandler(w http.ResponseWriter, r *http.Request)
ListMultipartUploadsHandler(http.ResponseWriter, *http.Request)
ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error)
}
// mimeType represents various MIME types used in API responses.
@ -108,7 +106,7 @@ const (
MimeXML mimeType = "application/xml"
)
var _ = logSuccessResponse
var _ = logErrorResponse
func (lrw *logResponseWriter) WriteHeader(code int) {
lrw.Do(func() {
@ -117,12 +115,6 @@ func (lrw *logResponseWriter) WriteHeader(code int) {
})
}
func (lrw *logResponseWriter) Flush() {
if f, ok := lrw.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func setRequestID(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// generate random UUIDv4
@ -153,37 +145,7 @@ func appendCORS(handler Handler) mux.MiddlewareFunc {
}
}
// BucketResolveFunc is a func to resolve bucket info by name.
type BucketResolveFunc func(ctx context.Context, bucket string) (*data.BucketInfo, error)
// metricsMiddleware wraps http handler for api with basic statistics collection.
func metricsMiddleware(log *zap.Logger, resolveBucket BucketResolveFunc, usersStat UsersStat) mux.MiddlewareFunc {
return func(h http.Handler) http.Handler {
return Stats(h.ServeHTTP, resolveCID(log, resolveBucket), usersStat)
}
}
// resolveCID forms CIDResolveFunc using BucketResolveFunc.
func resolveCID(log *zap.Logger, resolveBucket BucketResolveFunc) CIDResolveFunc {
return func(ctx context.Context, reqInfo *ReqInfo) (cnrID string) {
if reqInfo.BucketName == "" || reqInfo.API == "CreateBucket" || reqInfo.API == "" {
return ""
}
bktInfo, err := resolveBucket(ctx, reqInfo.BucketName)
if err != nil {
log.Debug("failed to resolve CID",
zap.String("request_id", reqInfo.RequestID), zap.String("method", reqInfo.API),
zap.String("bucket", reqInfo.BucketName), zap.String("object", reqInfo.ObjectName),
zap.Error(err))
return ""
}
return bktInfo.CID.EncodeToString()
}
}
func logSuccessResponse(l *zap.Logger) mux.MiddlewareFunc {
func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
lw := &logResponseWriter{ResponseWriter: w}
@ -199,7 +161,6 @@ func logSuccessResponse(l *zap.Logger) mux.MiddlewareFunc {
l.Info("call method",
zap.Int("status", lw.statusCode),
zap.String("host", r.Host),
zap.String("request_id", GetRequestID(r.Context())),
zap.String("method", mux.CurrentRoute(r).GetName()),
zap.String("bucket", reqInfo.BucketName),
@ -221,49 +182,21 @@ func GetRequestID(v interface{}) string {
}
}
func setErrorAPI(apiName string, h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := SetReqInfo(r.Context(), &ReqInfo{API: apiName})
h.ServeHTTP(w, r.WithContext(ctx))
})
}
// attachErrorHandler set NotFoundHandler and MethodNotAllowedHandler for mux.Router.
func attachErrorHandler(api *mux.Router, log *zap.Logger, h Handler, center auth.Center, usersStat UsersStat) {
middlewares := []mux.MiddlewareFunc{
AuthMiddleware(log, center),
metricsMiddleware(log, h.ResolveBucket, usersStat),
}
var errorHandler http.Handler = http.HandlerFunc(errorResponseHandler)
for i := len(middlewares) - 1; i >= 0; i-- {
errorHandler = middlewares[i](errorHandler)
}
// If none of the routes match, add default error handler routes
api.NotFoundHandler = setErrorAPI("NotFound", errorHandler)
api.MethodNotAllowedHandler = setErrorAPI("MethodNotAllowed", errorHandler)
}
// Attach adds S3 API handlers from h to r for domains with m client limit using
// center authentication and log logger.
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger, usersStat UsersStat) {
func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center auth.Center, log *zap.Logger) {
api := r.PathPrefix(SlashSeparator).Subrouter()
api.Use(
// -- prepare request
setRequestID,
// Attach user authentication for all S3 routes.
AuthMiddleware(log, center),
metricsMiddleware(log, h.ResolveBucket, usersStat),
// -- logging error requests
logSuccessResponse(log),
logErrorResponse(log),
)
attachErrorHandler(api, log, h, center, usersStat)
// Attach user authentication for all S3 routes.
AttachUserAuth(api, center, log)
buckets := make([]*mux.Router, 0, len(domains)+1)
buckets = append(buckets, api.PathPrefix("/{bucket}").Subrouter())
@ -279,327 +212,277 @@ func Attach(r *mux.Router, domains []string, m MaxClients, h Handler, center aut
// -- append CORS headers to a response for
appendCORS(h),
)
bucket.Methods(http.MethodOptions).HandlerFunc(
m.Handle(h.Preflight)).
Name("Options")
bucket.Methods(http.MethodOptions).HandlerFunc(m.Handle(metrics.APIStats("preflight", h.Preflight))).Name("Options")
bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
m.Handle(h.HeadObjectHandler)).
Name("HeadObject")
m.Handle(metrics.APIStats("headobject", h.HeadObjectHandler))).Name("HeadObject")
// CopyObjectPart
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(
m.Handle(h.UploadPartCopy)).
Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(metrics.APIStats("uploadpartcopy", h.UploadPartCopy))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
Name("UploadPartCopy")
// PutObjectPart
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
m.Handle(h.UploadPartHandler)).
Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
m.Handle(metrics.APIStats("uploadpart", h.UploadPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").
Name("UploadPart")
// ListParts
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(h.ListPartsHandler)).
Queries("uploadId", "{uploadId:.*}").
m.Handle(metrics.APIStats("listobjectparts", h.ListPartsHandler))).Queries("uploadId", "{uploadId:.*}").
Name("ListObjectParts")
// CompleteMultipartUpload
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
m.Handle(h.CompleteMultipartUploadHandler)).
Queries("uploadId", "{uploadId:.*}").
m.Handle(metrics.APIStats("completemutipartupload", h.CompleteMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}").
Name("CompleteMultipartUpload")
// CreateMultipartUpload
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
m.Handle(h.CreateMultipartUploadHandler)).
Queries("uploads", "").
m.Handle(metrics.APIStats("createmultipartupload", h.CreateMultipartUploadHandler))).Queries("uploads", "").
Name("CreateMultipartUpload")
// AbortMultipartUpload
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
m.Handle(h.AbortMultipartUploadHandler)).
Queries("uploadId", "{uploadId:.*}").
m.Handle(metrics.APIStats("abortmultipartupload", h.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}").
Name("AbortMultipartUpload")
// ListMultipartUploads
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.ListMultipartUploadsHandler)).
Queries("uploads", "").
m.Handle(metrics.APIStats("listmultipartuploads", h.ListMultipartUploadsHandler))).Queries("uploads", "").
Name("ListMultipartUploads")
// GetObjectACL -- this is a dummy call.
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(h.GetObjectACLHandler)).
Queries("acl", "").
m.Handle(metrics.APIStats("getobjectacl", h.GetObjectACLHandler))).Queries("acl", "").
Name("GetObjectACL")
// PutObjectACL -- this is a dummy call.
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
m.Handle(h.PutObjectACLHandler)).
Queries("acl", "").
m.Handle(metrics.APIStats("putobjectacl", h.PutObjectACLHandler))).Queries("acl", "").
Name("PutObjectACL")
// GetObjectTagging
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(h.GetObjectTaggingHandler)).
Queries("tagging", "").
m.Handle(metrics.APIStats("getobjecttagging", h.GetObjectTaggingHandler))).Queries("tagging", "").
Name("GetObjectTagging")
// PutObjectTagging
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
m.Handle(h.PutObjectTaggingHandler)).
Queries("tagging", "").
m.Handle(metrics.APIStats("putobjecttagging", h.PutObjectTaggingHandler))).Queries("tagging", "").
Name("PutObjectTagging")
// DeleteObjectTagging
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
m.Handle(h.DeleteObjectTaggingHandler)).
Queries("tagging", "").
m.Handle(metrics.APIStats("deleteobjecttagging", h.DeleteObjectTaggingHandler))).Queries("tagging", "").
Name("DeleteObjectTagging")
// SelectObjectContent
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
m.Handle(h.SelectObjectContentHandler)).
Queries("select", "").Queries("select-type", "2").
m.Handle(metrics.APIStats("selectobjectcontent", h.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2").
Name("SelectObjectContent")
// GetObjectRetention
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(h.GetObjectRetentionHandler)).
Queries("retention", "").
m.Handle(metrics.APIStats("getobjectretention", h.GetObjectRetentionHandler))).Queries("retention", "").
Name("GetObjectRetention")
// GetObjectLegalHold
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(h.GetObjectLegalHoldHandler)).
Queries("legal-hold", "").
m.Handle(metrics.APIStats("getobjectlegalhold", h.GetObjectLegalHoldHandler))).Queries("legal-hold", "").
Name("GetObjectLegalHold")
// GetObjectAttributes
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(h.GetObjectAttributesHandler)).
Queries("attributes", "").
m.Handle(metrics.APIStats("getobjectattributes", h.GetObjectAttributesHandler))).Queries("attributes", "").
Name("GetObjectAttributes")
// GetObject
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
m.Handle(h.GetObjectHandler)).
m.Handle(metrics.APIStats("getobject", h.GetObjectHandler))).
Name("GetObject")
// CopyObject
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(
m.Handle(h.CopyObjectHandler)).
bucket.Methods(http.MethodPut).Path("/{object:.+}").Headers(hdrAmzCopySource, "").HandlerFunc(m.Handle(metrics.APIStats("copyobject", h.CopyObjectHandler))).
Name("CopyObject")
// PutObjectRetention
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
m.Handle(h.PutObjectRetentionHandler)).
Queries("retention", "").
m.Handle(metrics.APIStats("putobjectretention", h.PutObjectRetentionHandler))).Queries("retention", "").
Name("PutObjectRetention")
// PutObjectLegalHold
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
m.Handle(h.PutObjectLegalHoldHandler)).
Queries("legal-hold", "").
m.Handle(metrics.APIStats("putobjectlegalhold", h.PutObjectLegalHoldHandler))).Queries("legal-hold", "").
Name("PutObjectLegalHold")
// PutObject
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
m.Handle(h.PutObjectHandler)).
m.Handle(metrics.APIStats("putobject", h.PutObjectHandler))).
Name("PutObject")
// DeleteObject
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
m.Handle(h.DeleteObjectHandler)).
m.Handle(metrics.APIStats("deleteobject", h.DeleteObjectHandler))).
Name("DeleteObject")
// Bucket operations
// GetBucketLocation
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketLocationHandler)).
Queries("location", "").
m.Handle(metrics.APIStats("getbucketlocation", h.GetBucketLocationHandler))).Queries("location", "").
Name("GetBucketLocation")
// GetBucketPolicy
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketPolicyHandler)).
Queries("policy", "").
m.Handle(metrics.APIStats("getbucketpolicy", h.GetBucketPolicyHandler))).Queries("policy", "").
Name("GetBucketPolicy")
// GetBucketLifecycle
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketLifecycleHandler)).
Queries("lifecycle", "").
m.Handle(metrics.APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", "").
Name("GetBucketLifecycle")
// GetBucketEncryption
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketEncryptionHandler)).
Queries("encryption", "").
m.Handle(metrics.APIStats("getbucketencryption", h.GetBucketEncryptionHandler))).Queries("encryption", "").
Name("GetBucketEncryption")
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketCorsHandler)).
Queries("cors", "").
m.Handle(metrics.APIStats("getbucketcors", h.GetBucketCorsHandler))).Queries("cors", "").
Name("GetBucketCors")
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(h.PutBucketCorsHandler)).
Queries("cors", "").
m.Handle(metrics.APIStats("putbucketcors", h.PutBucketCorsHandler))).Queries("cors", "").
Name("PutBucketCors")
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(h.DeleteBucketCorsHandler)).
Queries("cors", "").
m.Handle(metrics.APIStats("deletebucketcors", h.DeleteBucketCorsHandler))).Queries("cors", "").
Name("DeleteBucketCors")
// Dummy Bucket Calls
// GetBucketACL -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketACLHandler)).
Queries("acl", "").
m.Handle(metrics.APIStats("getbucketacl", h.GetBucketACLHandler))).Queries("acl", "").
Name("GetBucketACL")
// PutBucketACL -- this is a dummy call.
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(h.PutBucketACLHandler)).
Queries("acl", "").
m.Handle(metrics.APIStats("putbucketacl", h.PutBucketACLHandler))).Queries("acl", "").
Name("PutBucketACL")
// GetBucketWebsiteHandler -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketWebsiteHandler)).
Queries("website", "").
m.Handle(metrics.APIStats("getbucketwebsite", h.GetBucketWebsiteHandler))).Queries("website", "").
Name("GetBucketWebsite")
// GetBucketAccelerateHandler -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketAccelerateHandler)).
Queries("accelerate", "").
m.Handle(metrics.APIStats("getbucketaccelerate", h.GetBucketAccelerateHandler))).Queries("accelerate", "").
Name("GetBucketAccelerate")
// GetBucketRequestPaymentHandler -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketRequestPaymentHandler)).
Queries("requestPayment", "").
m.Handle(metrics.APIStats("getbucketrequestpayment", h.GetBucketRequestPaymentHandler))).Queries("requestPayment", "").
Name("GetBucketRequestPayment")
// GetBucketLoggingHandler -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketLoggingHandler)).
Queries("logging", "").
m.Handle(metrics.APIStats("getbucketlogging", h.GetBucketLoggingHandler))).Queries("logging", "").
Name("GetBucketLogging")
// GetBucketLifecycleHandler -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(metrics.APIStats("getbucketlifecycle", h.GetBucketLifecycleHandler))).Queries("lifecycle", "").
Name("GetBucketLifecycle")
// GetBucketReplicationHandler -- this is a dummy call.
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketReplicationHandler)).
Queries("replication", "").
m.Handle(metrics.APIStats("getbucketreplication", h.GetBucketReplicationHandler))).Queries("replication", "").
Name("GetBucketReplication")
// GetBucketTaggingHandler
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketTaggingHandler)).
Queries("tagging", "").
m.Handle(metrics.APIStats("getbuckettagging", h.GetBucketTaggingHandler))).Queries("tagging", "").
Name("GetBucketTagging")
// DeleteBucketWebsiteHandler
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(h.DeleteBucketWebsiteHandler)).
Queries("website", "").
m.Handle(metrics.APIStats("deletebucketwebsite", h.DeleteBucketWebsiteHandler))).Queries("website", "").
Name("DeleteBucketWebsite")
// DeleteBucketTaggingHandler
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(h.DeleteBucketTaggingHandler)).
Queries("tagging", "").
m.Handle(metrics.APIStats("deletebuckettagging", h.DeleteBucketTaggingHandler))).Queries("tagging", "").
Name("DeleteBucketTagging")
// GetBucketObjectLockConfig
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketObjectLockConfigHandler)).
Queries("object-lock", "").
m.Handle(metrics.APIStats("getbucketobjectlockconfiguration", h.GetBucketObjectLockConfigHandler))).Queries("object-lock", "").
Name("GetBucketObjectLockConfig")
// GetBucketVersioning
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketVersioningHandler)).
Queries("versioning", "").
m.Handle(metrics.APIStats("getbucketversioning", h.GetBucketVersioningHandler))).Queries("versioning", "").
Name("GetBucketVersioning")
// GetBucketNotification
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.GetBucketNotificationHandler)).
Queries("notification", "").
m.Handle(metrics.APIStats("getbucketnotification", h.GetBucketNotificationHandler))).Queries("notification", "").
Name("GetBucketNotification")
// ListenBucketNotification
bucket.Methods(http.MethodGet).HandlerFunc(h.ListenBucketNotificationHandler).
Queries("events", "{events:.*}").
bucket.Methods(http.MethodGet).HandlerFunc(metrics.APIStats("listenbucketnotification", h.ListenBucketNotificationHandler)).Queries("events", "{events:.*}").
Name("ListenBucketNotification")
// ListObjectsV2M
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.ListObjectsV2MHandler)).
Queries("list-type", "2", "metadata", "true").
m.Handle(metrics.APIStats("listobjectsv2M", h.ListObjectsV2MHandler))).Queries("list-type", "2", "metadata", "true").
Name("ListObjectsV2M")
// ListObjectsV2
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.ListObjectsV2Handler)).
Queries("list-type", "2").
m.Handle(metrics.APIStats("listobjectsv2", h.ListObjectsV2Handler))).Queries("list-type", "2").
Name("ListObjectsV2")
// ListBucketVersions
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.ListBucketObjectVersionsHandler)).
Queries("versions", "").
m.Handle(metrics.APIStats("listbucketversions", h.ListBucketObjectVersionsHandler))).Queries("versions", "").
Name("ListBucketVersions")
// ListObjectsV1 (Legacy)
bucket.Methods(http.MethodGet).HandlerFunc(
m.Handle(h.ListObjectsV1Handler)).
m.Handle(metrics.APIStats("listobjectsv1", h.ListObjectsV1Handler))).
Name("ListObjectsV1")
// PutBucketLifecycle
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(h.PutBucketLifecycleHandler)).
Queries("lifecycle", "").
m.Handle(metrics.APIStats("putbucketlifecycle", h.PutBucketLifecycleHandler))).Queries("lifecycle", "").
Name("PutBucketLifecycle")
// PutBucketEncryption
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(h.PutBucketEncryptionHandler)).
Queries("encryption", "").
m.Handle(metrics.APIStats("putbucketencryption", h.PutBucketEncryptionHandler))).Queries("encryption", "").
Name("PutBucketEncryption")
// PutBucketPolicy
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(h.PutBucketPolicyHandler)).
Queries("policy", "").
m.Handle(metrics.APIStats("putbucketpolicy", h.PutBucketPolicyHandler))).Queries("policy", "").
Name("PutBucketPolicy")
// PutBucketObjectLockConfig
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(h.PutBucketObjectLockConfigHandler)).
Queries("object-lock", "").
m.Handle(metrics.APIStats("putbucketobjectlockconfig", h.PutBucketObjectLockConfigHandler))).Queries("object-lock", "").
Name("PutBucketObjectLockConfig")
// PutBucketTaggingHandler
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(h.PutBucketTaggingHandler)).
Queries("tagging", "").
m.Handle(metrics.APIStats("putbuckettagging", h.PutBucketTaggingHandler))).Queries("tagging", "").
Name("PutBucketTagging")
// PutBucketVersioning
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(h.PutBucketVersioningHandler)).
Queries("versioning", "").
m.Handle(metrics.APIStats("putbucketversioning", h.PutBucketVersioningHandler))).Queries("versioning", "").
Name("PutBucketVersioning")
// PutBucketNotification
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(h.PutBucketNotificationHandler)).
Queries("notification", "").
m.Handle(metrics.APIStats("putbucketnotification", h.PutBucketNotificationHandler))).Queries("notification", "").
Name("PutBucketNotification")
// CreateBucket
bucket.Methods(http.MethodPut).HandlerFunc(
m.Handle(h.CreateBucketHandler)).
m.Handle(metrics.APIStats("createbucket", h.CreateBucketHandler))).
Name("CreateBucket")
// HeadBucket
bucket.Methods(http.MethodHead).HandlerFunc(
m.Handle(h.HeadBucketHandler)).
m.Handle(metrics.APIStats("headbucket", h.HeadBucketHandler))).
Name("HeadBucket")
// PostPolicy
bucket.Methods(http.MethodPost).HeadersRegexp(hdrContentType, "multipart/form-data*").HandlerFunc(
m.Handle(h.PostObject)).
m.Handle(metrics.APIStats("postobject", h.PostObject))).
Name("PostObject")
// DeleteMultipleObjects
bucket.Methods(http.MethodPost).HandlerFunc(
m.Handle(h.DeleteMultipleObjectsHandler)).
Queries("delete", "").
m.Handle(metrics.APIStats("deletemultipleobjects", h.DeleteMultipleObjectsHandler))).Queries("delete", "").
Name("DeleteMultipleObjects")
// DeleteBucketPolicy
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(h.DeleteBucketPolicyHandler)).
Queries("policy", "").
m.Handle(metrics.APIStats("deletebucketpolicy", h.DeleteBucketPolicyHandler))).Queries("policy", "").
Name("DeleteBucketPolicy")
// DeleteBucketLifecycle
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(h.DeleteBucketLifecycleHandler)).
Queries("lifecycle", "").
m.Handle(metrics.APIStats("deletebucketlifecycle", h.DeleteBucketLifecycleHandler))).Queries("lifecycle", "").
Name("DeleteBucketLifecycle")
// DeleteBucketEncryption
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(h.DeleteBucketEncryptionHandler)).
Queries("encryption", "").
m.Handle(metrics.APIStats("deletebucketencryption", h.DeleteBucketEncryptionHandler))).Queries("encryption", "").
Name("DeleteBucketEncryption")
// DeleteBucket
bucket.Methods(http.MethodDelete).HandlerFunc(
m.Handle(h.DeleteBucketHandler)).
m.Handle(metrics.APIStats("deletebucket", h.DeleteBucketHandler))).
Name("DeleteBucket")
}
// Root operation
// ListBuckets
api.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
m.Handle(h.ListBucketsHandler)).
m.Handle(metrics.APIStats("listbuckets", h.ListBucketsHandler))).
Name("ListBuckets")
// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
// than failing with UnknownAPIRequest we simply handle it for now.
api.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
m.Handle(h.ListBucketsHandler)).
m.Handle(metrics.APIStats("listbuckets", h.ListBucketsHandler))).
Name("ListBuckets")
// If none of the routes match, add default error handler routes
api.NotFoundHandler = metrics.APIStats("notfound", errorResponseHandler)
api.MethodNotAllowedHandler = metrics.APIStats("methodnotallowed", errorResponseHandler)
}

View file

@ -4,9 +4,9 @@ import (
"context"
"net/http"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/gorilla/mux"
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
"go.uber.org/zap"
)
@ -16,12 +16,9 @@ type KeyWrapper string
// BoxData is an ID used to store accessbox.Box in a context.
var BoxData = KeyWrapper("__context_box_key")
// ClientTime is an ID used to store client time.Time in a context.
var ClientTime = KeyWrapper("__context_client_time")
// AuthMiddleware adds user authentication via center to router using log for logging.
func AuthMiddleware(log *zap.Logger, center auth.Center) mux.MiddlewareFunc {
return func(h http.Handler) http.Handler {
// AttachUserAuth adds user authentication via center to router using log for logging.
func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
router.Use(func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var ctx context.Context
box, err := center.Authenticate(r)
@ -38,13 +35,10 @@ func AuthMiddleware(log *zap.Logger, center auth.Center) mux.MiddlewareFunc {
return
}
} else {
ctx = context.WithValue(r.Context(), BoxData, box.AccessBox)
if !box.ClientTime.IsZero() {
ctx = context.WithValue(ctx, ClientTime, box.ClientTime)
}
ctx = context.WithValue(r.Context(), BoxData, box)
}
h.ServeHTTP(w, r.WithContext(ctx))
})
}
})
}

View file

@ -11,25 +11,25 @@ import (
"os"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/google/uuid"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-s3-gw/creds/tokens"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
neofsecdsa "github.com/nspcc-dev/neofs-sdk-go/crypto/ecdsa"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/user"
"go.uber.org/zap"
)
// PrmContainerCreate groups parameters of containers created by authmate.
type PrmContainerCreate struct {
// FrostFS identifier of the container creator.
// NeoFS identifier of the container creator.
Owner user.ID
// Container placement policy.
@ -39,26 +39,26 @@ type PrmContainerCreate struct {
FriendlyName string
}
// NetworkState represents FrostFS network state which is needed for authmate processing.
// NetworkState represents NeoFS network state which is needed for authmate processing.
type NetworkState struct {
// Current FrostFS time.
// Current NeoFS time.
Epoch uint64
// Duration of the Morph chain block in ms.
BlockDuration int64
// Duration of the FrostFS epoch in Morph chain blocks.
// Duration of the NeoFS epoch in Morph chain blocks.
EpochDuration uint64
}
// FrostFS represents virtual connection to FrostFS network.
type FrostFS interface {
// FrostFS interface required by credential tool.
tokens.FrostFS
// NeoFS represents virtual connection to NeoFS network.
type NeoFS interface {
// NeoFS interface required by credential tool.
tokens.NeoFS
// ContainerExists checks container presence in FrostFS by identifier.
// ContainerExists checks container presence in NeoFS by identifier.
// Returns nil if container exists.
ContainerExists(context.Context, cid.ID) error
// CreateContainer creates and saves parameterized container in FrostFS.
// CreateContainer creates and saves parameterized container in NeoFS.
// It sets 'Timestamp' attribute to the current time.
// It returns the ID of the saved container.
//
@ -78,25 +78,25 @@ type FrostFS interface {
TimeToEpoch(context.Context, time.Time) (uint64, uint64, error)
}
// Agent contains client communicating with FrostFS and logger.
// Agent contains client communicating with NeoFS and logger.
type Agent struct {
frostFS FrostFS
log *zap.Logger
neoFS NeoFS
log *zap.Logger
}
// New creates an object of type Agent that consists of Client and logger.
func New(log *zap.Logger, frostFS FrostFS) *Agent {
return &Agent{log: log, frostFS: frostFS}
func New(log *zap.Logger, neoFS NeoFS) *Agent {
return &Agent{log: log, neoFS: neoFS}
}
type (
// ContainerPolicies contains mapping of aws LocationConstraint to frostfs PlacementPolicy.
// ContainerPolicies contains mapping of aws LocationConstraint to neofs PlacementPolicy.
ContainerPolicies map[string]string
// IssueSecretOptions contains options for passing to Agent.IssueSecret method.
IssueSecretOptions struct {
Container ContainerOptions
FrostFSKey *keys.PrivateKey
NeoFSKey *keys.PrivateKey
GatesPublicKeys []*keys.PublicKey
EACLRules []byte
SessionTokenRules []byte
@ -120,7 +120,7 @@ type (
}
)
// lifetimeOptions holds FrostFS epochs, iat -- epoch which the token was issued at, exp -- epoch when the token expires.
// lifetimeOptions holds NeoFS epochs, iat -- epoch which the token was issued at, exp -- epoch when the token expires.
type lifetimeOptions struct {
Iat uint64
Exp uint64
@ -143,7 +143,7 @@ type (
func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwner user.ID) (cid.ID, error) {
if !opts.ID.Equals(cid.ID{}) {
return opts.ID, a.frostFS.ContainerExists(ctx, opts.ID)
return opts.ID, a.neoFS.ContainerExists(ctx, opts.ID)
}
var prm PrmContainerCreate
@ -156,9 +156,9 @@ func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwn
prm.Owner = idOwner
prm.FriendlyName = opts.FriendlyName
cnrID, err := a.frostFS.CreateContainer(ctx, prm)
cnrID, err := a.neoFS.CreateContainer(ctx, prm)
if err != nil {
return cid.ID{}, fmt.Errorf("create container in FrostFS: %w", err)
return cid.ID{}, fmt.Errorf("create container in NeoFS: %w", err)
}
return cnrID, nil
@ -200,7 +200,7 @@ func preparePolicy(policy ContainerPolicies) ([]*accessbox.AccessBox_ContainerPo
return result, nil
}
// IssueSecret creates an auth token, puts it in the FrostFS network and writes to io.Writer a new secret access key.
// IssueSecret creates an auth token, puts it in the NeoFS network and writes to io.Writer a new secret access key.
func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecretOptions) error {
var (
err error
@ -213,7 +213,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
return fmt.Errorf("prepare policies: %w", err)
}
lifetime.Iat, lifetime.Exp, err = a.frostFS.TimeToEpoch(ctx, time.Now().Add(options.Lifetime))
lifetime.Iat, lifetime.Exp, err = a.neoFS.TimeToEpoch(ctx, time.Now().Add(options.Lifetime))
if err != nil {
return fmt.Errorf("fetch time to epoch: %w", err)
}
@ -231,7 +231,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
box.ContainerPolicy = policies
var idOwner user.ID
user.IDFromKey(&idOwner, options.FrostFSKey.PrivateKey.PublicKey)
user.IDFromKey(&idOwner, options.NeoFSKey.PrivateKey.PublicKey)
a.log.Info("check container or create", zap.Stringer("cid", options.Container.ID),
zap.String("friendly_name", options.Container.FriendlyName),
@ -241,11 +241,11 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
return fmt.Errorf("check container: %w", err)
}
a.log.Info("store bearer token into FrostFS",
a.log.Info("store bearer token into NeoFS",
zap.Stringer("owner_tkn", idOwner))
addr, err := tokens.
New(a.frostFS, secrets.EphemeralKey, cache.DefaultAccessBoxConfig(a.log)).
New(a.neoFS, secrets.EphemeralKey, cache.DefaultAccessBoxConfig(a.log)).
Put(ctx, id, idOwner, box, lifetime.Exp, options.GatesPublicKeys...)
if err != nil {
return fmt.Errorf("failed to put bearer token: %w", err)
@ -260,7 +260,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
AccessKeyID: accessKeyID,
SecretAccessKey: secrets.AccessKey,
OwnerPrivateKey: hex.EncodeToString(secrets.EphemeralKey.Bytes()),
WalletPublicKey: hex.EncodeToString(options.FrostFSKey.PublicKey().Bytes()),
WalletPublicKey: hex.EncodeToString(options.NeoFSKey.PublicKey().Bytes()),
ContainerID: id.EncodeToString(),
}
@ -288,10 +288,10 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
return nil
}
// ObtainSecret receives an existing secret access key from FrostFS and
// ObtainSecret receives an existing secret access key from NeoFS and
// writes to io.Writer the secret access key.
func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSecretOptions) error {
bearerCreds := tokens.New(a.frostFS, options.GatePrivateKey, cache.DefaultAccessBoxConfig(a.log))
bearerCreds := tokens.New(a.neoFS, options.GatePrivateKey, cache.DefaultAccessBoxConfig(a.log))
var addr oid.Address
if err := addr.DecodeString(options.SecretAddress); err != nil {
@ -381,7 +381,7 @@ func buildSessionToken(key *keys.PrivateKey, lifetime lifetimeOptions, ctx sessi
tok.AppliedTo(ctx.containerID)
tok.SetID(uuid.New())
tok.SetAuthKey((*frostfsecdsa.PublicKey)(gateKey))
tok.SetAuthKey((*neofsecdsa.PublicKey)(gateKey))
tok.SetIat(lifetime.Iat)
tok.SetNbf(lifetime.Iat)
@ -413,7 +413,7 @@ func createTokens(options *IssueSecretOptions, lifetime lifetimeOptions) ([]*acc
if err != nil {
return nil, fmt.Errorf("failed to build eacl table: %w", err)
}
bearerTokens, err := buildBearerTokens(options.FrostFSKey, table, lifetime, options.GatesPublicKeys)
bearerTokens, err := buildBearerTokens(options.NeoFSKey, table, lifetime, options.GatesPublicKeys)
if err != nil {
return nil, fmt.Errorf("failed to build bearer tokens: %w", err)
}
@ -427,7 +427,7 @@ func createTokens(options *IssueSecretOptions, lifetime lifetimeOptions) ([]*acc
return nil, fmt.Errorf("failed to build context for session token: %w", err)
}
sessionTokens, err := buildSessionTokens(options.FrostFSKey, lifetime, sessionRules, options.GatesPublicKeys)
sessionTokens, err := buildSessionTokens(options.NeoFSKey, lifetime, sessionRules, options.GatesPublicKeys)
if err != nil {
return nil, fmt.Errorf("failed to biuild session token: %w", err)
}

View file

@ -4,9 +4,9 @@ import (
"encoding/json"
"fmt"
apisession "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
apisession "github.com/nspcc-dev/neofs-api-go/v2/session"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/session"
)
type (

View file

@ -3,7 +3,7 @@ package authmate
import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"github.com/nspcc-dev/neofs-sdk-go/session"
"github.com/stretchr/testify/require"
)

Some files were not shown because too many files have changed in this diff Show more