Compare commits

...

28 commits

Author SHA1 Message Date
52be32e4ba Release v0.27.0
Some checks failed
Tests / Lint (pull_request) Failing after 5s
Tests / Tests (1.18) (pull_request) Failing after 4s
Builds / Build CLI (pull_request) Has started running
Tests / Coverage (pull_request) Failing after 10s
Tests / Tests (1.19) (pull_request) Failing after 6s
Builds / Build Docker image (pull_request) Has been cancelled
CodeQL / Analyze (go) (pull_request) Failing after 2s
CodeQL / Analyze (go) (push) Failing after 2s
DCO check / Commits Check (pull_request) Failing after 2s
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-07-12 11:34:00 +03:00
4791428b6e [#61] Update api-go version to latest support/v2.15
Some checks failed
Tests / Lint (pull_request) Has been cancelled
Tests / Coverage (pull_request) Has been cancelled
Tests / Tests (1.18) (pull_request) Has been cancelled
Tests / Tests (1.19) (pull_request) Has been cancelled
Builds / Build CLI (pull_request) Has been cancelled
Builds / Build Docker image (pull_request) Has been cancelled
CodeQL / Analyze (go) (pull_request) Failing after 31s
DCO check / Commits Check (pull_request) Failing after 32s
CodeQL / Analyze (go) (push) Failing after 52s
Contains fix for keepalive parameter of RPC client.

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-06-19 11:46:18 +03:00
2888151e40 [#1] Update comments
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-05-16 17:41:33 +03:00
048c626750 [#1] Use FrostFS AIO image in integration test
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-05-16 17:41:27 +03:00
168b67dc31 [#38] Enabling gate metrics
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-04-26 09:53:36 +03:00
959213520e [#32] Update health metric values
Now values are:
0 - undefined
1 - starting
2 - ready
3 - shutting down

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-04-17 16:28:27 +03:00
162738e771 [#27] Update SDK to fix handling request canceling
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-30 15:59:33 +03:00
7c16ffa250 [#26] Fix pre-commit issues
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-24 15:35:42 +03:00
6f35d7198d [#24] Use build tags to run integration tests
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-24 12:27:17 +03:00
81f7168a16 [#22] Update CHANGELOG.md
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-24 11:33:48 +03:00
a8ec09e76a [#22] Update system attributes prefix
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-24 11:33:01 +03:00
1f66149316 [#17] Add pre-commit config 2023-03-24 08:13:56 +00:00
e2059a8926 [#21] Add bug label
Add bug label in the bug report template

Signed-off-by: Liza <e.chichindaeva@yadro.com>
2023-03-24 06:23:39 +00:00
53ee124b19 [#18] Fix typo in error message
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-22 10:43:34 +03:00
8f6be59e23 [#18] Extract error details
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-03-13 16:14:04 +03:00
e02ee50d7b Rename package name
Due to source code relocation from GitHub.

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
2023-03-07 17:08:53 +03:00
93ec4c444d [TrueCloudLab#12] Update CHANGELOG.md
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-02-06 13:56:55 +03:00
6be8d47d92 [TrueCloudLab#12] Support multiple configs
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-02-06 13:56:55 +03:00
ed983f8ad0 [TrueCloudLab#11] Update SDK to renew tokens beforehand
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-02-03 17:01:01 +03:00
5f01abf300 [#8] Update neo-go and viper
Signed-off-by: Artem Tataurov <a.tataurov@yadro.com>
2023-01-25 15:23:50 +03:00
38aa6db041 [TrueCloudLab#9] Fix tests
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-25 11:24:41 +03:00
5a98df9d2d [TrueCloudLab#9] Update CHANGELOG.md
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-25 11:24:41 +03:00
361acacf07 [TrueCloudLab#9] Update go version to 1.18
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-25 11:24:41 +03:00
6909ef5382 [TrueCloudLab#7] Fix tests
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-24 17:09:14 +03:00
148b1aa7f5 [TrueCloudLab#7] Require only one healthy server
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-24 17:09:14 +03:00
7df26d9181 [#6] Update SDK
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2023-01-10 11:25:14 +03:00
913644d64a Change logo
Signed-off-by: Stanislav Bogatyrev <s.bogatyrev@yadro.com>
2023-01-09 11:05:04 +03:00
4c30ff6638 Change logo
Signed-off-by: Stanislav Bogatyrev <s.bogatyrev@yadro.com>
2023-01-08 10:37:07 +03:00
47 changed files with 1252 additions and 1254 deletions

View file

@ -2,7 +2,7 @@
name: Bug report
about: Create a report to help us improve
title: ''
labels: community, triage
labels: community, triage, bug
assignees: ''
---
@ -18,17 +18,17 @@ assignees: ''
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
## Possible Solution
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
<!--- or ideas how to implement the addition or change -->
<!-- Not obligatory
If no reason/fix/additions for the bug can be suggested,
uncomment the following phrase:
<-- No fix can be suggested by a QA engineer. Further solutions shall be up to developers. -->
## Steps to Reproduce (for bugs)
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
<!--- reproduce this bug. -->
1.
2.
3.
4.
## Context
<!--- How has this issue affected you? What are you trying to accomplish? -->

View file

@ -7,14 +7,14 @@ assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
## Is your feature request related to a problem? Please describe.
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
## Describe the solution you'd like
<!--- A clear and concise description of what you want to happen. -->
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
## Describe alternatives you've considered
<!--- A clear and concise description of any alternative solutions or features you've considered. -->
**Additional context**
Add any other context or screenshots about the feature request here.
## Additional context
<!--- Add any other context or screenshots about the feature request here. -->

70
.github/logo.svg vendored Normal file
View file

@ -0,0 +1,70 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 25.0.1, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="Слой_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 184.2 51.8" style="enable-background:new 0 0 184.2 51.8;" xml:space="preserve">
<style type="text/css">
.st0{display:none;}
.st1{display:inline;}
.st2{fill:#01E397;}
.st3{display:inline;fill:#010032;}
.st4{display:inline;fill:#00E599;}
.st5{display:inline;fill:#00AF92;}
.st6{fill:#00C3E5;}
</style>
<g id="Layer_2">
<g id="Layer_1-2" class="st0">
<g class="st1">
<path class="st2" d="M146.6,18.3v7.2h10.9V29h-10.9v10.7h-4V14.8h18v3.5H146.6z"/>
<path class="st2" d="M180,15.7c1.7,0.9,3,2.2,4,3.8l-3,2.7c-0.6-1.3-1.5-2.4-2.6-3.3c-1.3-0.7-2.8-1-4.3-1
c-1.4-0.1-2.8,0.3-4,1.1c-0.9,0.5-1.5,1.5-1.4,2.6c0,1,0.5,1.9,1.4,2.4c1.5,0.8,3.2,1.3,4.9,1.5c1.9,0.3,3.7,0.8,5.4,1.6
c1.2,0.5,2.2,1.3,2.9,2.3c0.6,1,1,2.2,0.9,3.4c0,1.4-0.5,2.7-1.3,3.8c-0.9,1.2-2.1,2.1-3.5,2.6c-1.7,0.6-3.4,0.9-5.2,0.8
c-5,0-8.6-1.6-10.7-5l2.9-2.8c0.7,1.4,1.8,2.5,3.1,3.3c1.5,0.7,3.1,1.1,4.7,1c1.5,0.1,2.9-0.2,4.2-0.9c0.9-0.5,1.5-1.5,1.5-2.6
c0-0.9-0.5-1.8-1.3-2.2c-1.5-0.7-3.1-1.2-4.8-1.5c-1.9-0.3-3.7-0.8-5.5-1.5c-1.2-0.5-2.2-1.4-3-2.4c-0.6-1-1-2.2-0.9-3.4
c0-1.4,0.4-2.7,1.2-3.8c0.8-1.2,2-2.2,3.3-2.8c1.6-0.7,3.4-1.1,5.2-1C176.1,14.3,178.2,14.8,180,15.7z"/>
</g>
<path class="st3" d="M73.3,16.3c1.9,1.9,2.9,4.5,2.7,7.1v15.9h-4V24.8c0-2.6-0.5-4.5-1.6-5.7c-1.2-1.2-2.8-1.8-4.5-1.7
c-1.3,0-2.5,0.3-3.7,0.8c-1.2,0.7-2.2,1.7-2.9,2.9c-0.8,1.5-1.1,3.2-1.1,4.9v13.3h-4V15.1l3.6,1.5v1.7c0.8-1.5,2.1-2.6,3.6-3.3
c1.5-0.8,3.2-1.2,4.9-1.1C68.9,13.8,71.3,14.7,73.3,16.3z"/>
<path class="st3" d="M104.4,28.3H85.6c0.1,2.2,1,4.3,2.5,5.9c1.5,1.4,3.5,2.2,5.6,2.1c1.6,0.1,3.2-0.2,4.6-0.9
c1.1-0.6,2-1.6,2.5-2.8l3.3,1.8c-0.9,1.7-2.3,3.1-4,4c-2,1-4.2,1.5-6.4,1.4c-3.7,0-6.7-1.1-8.8-3.4s-3.2-5.5-3.2-9.6s1-7.2,3-9.5
s5-3.4,8.7-3.4c2.1-0.1,4.2,0.5,6.1,1.5c1.6,1,3,2.5,3.8,4.2c0.9,1.8,1.3,3.9,1.3,5.9C104.6,26.4,104.6,27.4,104.4,28.3z
M88.1,19.3c-1.4,1.5-2.2,3.4-2.4,5.5h15.1c-0.2-2-1-3.9-2.3-5.5c-1.4-1.3-3.2-2-5.1-1.9C91.5,17.3,89.6,18,88.1,19.3z"/>
<path class="st3" d="M131,17.3c2.2,2.3,3.2,5.5,3.2,9.5s-1,7.3-3.2,9.6s-5.1,3.4-8.8,3.4s-6.7-1.1-8.9-3.4s-3.2-5.5-3.2-9.6
s1.1-7.2,3.2-9.5s5.1-3.4,8.9-3.4S128.9,15,131,17.3z M116.2,19.9c-1.5,2-2.2,4.4-2.1,6.9c-0.2,2.5,0.6,5,2.1,7
c1.5,1.7,3.7,2.7,6,2.6c2.3,0.1,4.4-0.9,5.9-2.6c1.5-2,2.3-4.5,2.1-7c0.1-2.5-0.6-4.9-2.1-6.9c-1.5-1.7-3.6-2.7-5.9-2.6
C119.9,17.2,117.7,18.2,116.2,19.9z"/>
<polygon class="st4" points="0,9.1 0,43.7 22.5,51.8 22.5,16.9 46.8,7.9 24.8,0 "/>
<polygon class="st5" points="24.3,17.9 24.3,36.8 46.8,44.9 46.8,9.6 "/>
</g>
<g>
<g>
<path class="st6" d="M41.6,17.5H28.2v6.9h10.4v3.3H28.2v10.2h-3.9V14.2h17.2V17.5z"/>
<path class="st6" d="M45.8,37.9v-18h3.3l0.4,3.2c0.5-1.2,1.2-2.1,2.1-2.7c0.9-0.6,2.1-0.9,3.5-0.9c0.4,0,0.7,0,1.1,0.1
c0.4,0.1,0.7,0.2,0.9,0.3l-0.5,3.4c-0.3-0.1-0.6-0.2-0.9-0.2C55.4,23,54.9,23,54.4,23c-0.7,0-1.5,0.2-2.2,0.6
c-0.7,0.4-1.3,1-1.8,1.8s-0.7,1.8-0.7,3v9.5H45.8z"/>
<path class="st6" d="M68.6,19.6c1.8,0,3.3,0.4,4.6,1.1c1.3,0.7,2.4,1.8,3.1,3.2s1.1,3.1,1.1,5c0,1.9-0.4,3.6-1.1,5
c-0.8,1.4-1.8,2.5-3.1,3.2c-1.3,0.7-2.9,1.1-4.6,1.1s-3.3-0.4-4.6-1.1c-1.3-0.7-2.4-1.8-3.2-3.2c-0.8-1.4-1.2-3.1-1.2-5
c0-1.9,0.4-3.6,1.2-5s1.8-2.5,3.2-3.2C65.3,19.9,66.8,19.6,68.6,19.6z M68.6,22.6c-1.1,0-2,0.2-2.8,0.7c-0.8,0.5-1.3,1.2-1.7,2.1
s-0.6,2.1-0.6,3.5c0,1.3,0.2,2.5,0.6,3.4s1,1.7,1.7,2.2s1.7,0.7,2.8,0.7c1.1,0,2-0.2,2.7-0.7c0.7-0.5,1.3-1.2,1.7-2.2
s0.6-2.1,0.6-3.4c0-1.4-0.2-2.5-0.6-3.5s-1-1.6-1.7-2.1C70.6,22.8,69.6,22.6,68.6,22.6z"/>
<path class="st6" d="M89.2,38.3c-1.8,0-3.4-0.3-4.9-1c-1.5-0.7-2.7-1.7-3.5-3l2.7-2.3c0.5,1,1.3,1.8,2.3,2.4
c1,0.6,2.2,0.9,3.6,0.9c1.1,0,2-0.2,2.6-0.6c0.6-0.4,1-0.9,1-1.6c0-0.5-0.2-0.9-0.5-1.2s-0.9-0.6-1.7-0.8l-3.8-0.8
c-1.9-0.4-3.3-1-4.1-1.9c-0.8-0.9-1.2-1.9-1.2-3.3c0-1,0.3-1.9,0.9-2.7c0.6-0.8,1.4-1.5,2.5-2s2.5-0.8,4-0.8c1.8,0,3.3,0.3,4.6,1
c1.3,0.6,2.2,1.5,2.9,2.7l-2.7,2.2c-0.5-1-1.1-1.7-2-2.1c-0.9-0.5-1.8-0.7-2.8-0.7c-0.8,0-1.4,0.1-2,0.3c-0.6,0.2-1,0.5-1.3,0.8
c-0.3,0.3-0.4,0.7-0.4,1.2c0,0.5,0.2,0.9,0.5,1.3s1,0.6,1.9,0.8l4.1,0.9c1.7,0.3,2.9,0.9,3.7,1.7c0.7,0.8,1.1,1.8,1.1,2.9
c0,1.2-0.3,2.2-0.9,3c-0.6,0.9-1.5,1.6-2.6,2C92.1,38.1,90.7,38.3,89.2,38.3z"/>
<path class="st6" d="M112.8,19.9v3H99.3v-3H112.8z M106.6,14.6v17.9c0,0.9,0.2,1.5,0.7,1.9c0.5,0.4,1.1,0.6,1.9,0.6
c0.6,0,1.2-0.1,1.7-0.3c0.5-0.2,0.9-0.5,1.3-0.8l0.9,2.8c-0.6,0.5-1.2,0.9-2,1.1c-0.8,0.3-1.7,0.4-2.7,0.4c-1,0-2-0.2-2.8-0.5
s-1.5-0.9-2-1.6c-0.5-0.8-0.7-1.7-0.8-3V15.7L106.6,14.6z"/>
<path d="M137.9,17.5h-13.3v6.9h10.4v3.3h-10.4v10.2h-3.9V14.2h17.2V17.5z"/>
<path d="M150.9,13.8c2.1,0,4,0.4,5.5,1.2c1.6,0.8,2.9,2,4,3.5l-2.6,2.5c-0.9-1.4-1.9-2.4-3.1-3c-1.1-0.6-2.5-0.9-4-0.9
c-1.2,0-2.1,0.2-2.8,0.5c-0.7,0.3-1.3,0.7-1.6,1.2c-0.3,0.5-0.5,1.1-0.5,1.7c0,0.7,0.3,1.4,0.8,1.9c0.5,0.6,1.5,1,2.9,1.3
l4.8,1.1c2.3,0.5,3.9,1.3,4.9,2.3c1,1,1.4,2.3,1.4,3.9c0,1.5-0.4,2.7-1.2,3.8c-0.8,1.1-1.9,1.9-3.3,2.5s-3.1,0.9-5,0.9
c-1.7,0-3.2-0.2-4.5-0.6c-1.3-0.4-2.5-1-3.5-1.8c-1-0.7-1.8-1.6-2.5-2.6l2.7-2.7c0.5,0.8,1.1,1.6,1.9,2.2
c0.8,0.7,1.7,1.2,2.7,1.5c1,0.4,2.2,0.5,3.4,0.5c1.1,0,2.1-0.1,2.9-0.4c0.8-0.3,1.4-0.7,1.8-1.2c0.4-0.5,0.6-1.1,0.6-1.9
c0-0.7-0.2-1.3-0.7-1.8c-0.5-0.5-1.3-0.9-2.6-1.2l-5.2-1.2c-1.4-0.3-2.6-0.8-3.6-1.3c-0.9-0.6-1.6-1.3-2.1-2.1s-0.7-1.8-0.7-2.8
c0-1.3,0.4-2.6,1.1-3.7c0.7-1.1,1.8-2,3.2-2.6C147.3,14.1,148.9,13.8,150.9,13.8z"/>
</g>
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 5.5 KiB

View file

@ -61,7 +61,7 @@ jobs:
runs-on: ubuntu-20.04
strategy:
matrix:
go_versions: [ '1.17', '1.18', '1.19' ]
go_versions: [ '1.18', '1.19' ]
fail-fast: false
steps:
- uses: actions/checkout@v2

11
.gitlint Normal file
View file

@ -0,0 +1,11 @@
[general]
fail-without-commits=True
regex-style-search=True
contrib=CC1
[title-match-regex]
regex=^\[\#[0-9Xx]+\]\s
[ignore-by-title]
regex=^Release(.*)
ignore=title-match-regex

45
.pre-commit-config.yaml Normal file
View file

@ -0,0 +1,45 @@
ci:
autofix_prs: false
repos:
- repo: https://github.com/jorisroovers/gitlint
rev: v0.19.1
hooks:
- id: gitlint
stages: [commit-msg]
- id: gitlint-ci
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: check-added-large-files
- id: check-case-conflict
- id: check-executables-have-shebangs
- id: check-shebang-scripts-are-executable
- id: check-merge-conflict
- id: check-json
- id: check-xml
- id: check-yaml
- id: trailing-whitespace
args: [--markdown-linebreak-ext=md]
- id: end-of-file-fixer
exclude: ".key$"
- repo: https://github.com/shellcheck-py/shellcheck-py
rev: v0.9.0.2
hooks:
- id: shellcheck
- repo: https://github.com/golangci/golangci-lint
rev: v1.51.2
hooks:
- id: golangci-lint
- repo: local
hooks:
- id: go-unit-tests
name: go unit tests
entry: make test
pass_filenames: false
types: [go]
language: system

View file

@ -4,276 +4,42 @@ This document outlines major changes between releases.
## [Unreleased]
## [0.26.0] - 2022-12-28
## [0.27.0] - Karpinsky - 2023-07-12
This is a first FrostFS HTTP Gateway release named after
[Karpinsky glacier](https://en.wikipedia.org/wiki/Karpinsky_Glacier).
### Fixed
- ENV config example (#236)
- Require only one healthy storage server to start (#7)
- Enable gate metrics (#38)
- `Too many pings` error (#61)
### Added
- Support the `Date` header on upload (#214)
- Available routes specification (#216)
- Mention caching strategy in docs (#215)
- Add error response on attribute duplicates (#221)
- Multiple server listeners (#228)
### Removed
- Deprecated linters (#239)
### Updating from v0.25.1
Make sure your configuration is valid:
If you configure application using environment variables change:
* `HTTP_GW_LISTEN_ADDRESS` -> `HTTP_GW_SERVER_0_ADDRESS`
* `HTTP_GW_TLS_CERT_FILE` -> `HTTP_GW_SERVER_0_TLS_CERT_FILE` (and set `HTTP_GW_SERVER_0_TLS_ENABLED=true`)
* `HTTP_GW_TLS_KEY_FILE` -> `HTTP_GW_SERVER_0_TLS_KEY_FILE` (and set `HTTP_GW_SERVER_0_TLS_ENABLED=true`)
If you configure application using `.yaml` file change:
* `listen_address` -> `server.0.address`
* `tls.cert_file` -> `server.0.tls.cert_file` (and set `server.0.tls.enabled: true`)
* `tls.key_file` -> `server.0.tls.key_file` (and set `server.0.tls.enabled: true`)
## [0.25.1] - 2022-11-30
### Fixed
- Download zip archive when `FilePath` is invalid (#222)
- Only one peer must be healthy to init pool (#233)
### Added
- Debian packaging (#223)
- Timeout for individual operations in streaming RPC (#234)
## [0.25.0] - 2022-10-31
### Added
- Config reloading on SIGHUP (#200, #208)
- Stop pool dial on SIGINT (#212)
- Makefile help (#213)
- Multiple configs support (#12)
### Changed
- Update NeoFS error handling (#206)
- GitHub actions updates (#205, #209)
- Unified system attribute format for GET and HEAD (#213)
- Repository rebranding (#1)
- Update neo-go to v0.101.0 (#8)
- Update viper to v1.15.0 (#8)
- Update go version to 1.18 (#9)
- Errors have become more detailed (#18)
- Update system attribute names (#22)
- Separate integration tests with build tags (#24)
- Changed values for `frostfs_http_gw_state_health` metric (#32)
## [0.24.0] - 2022-09-14
### Updating from v0.26.0
### Fixed
- Fix expiration epoch calculation (#198)
- Fix panic on go1.19 (#188)
To set system attributes use updated headers
(you can use old ones for now, but their support will be dropped in the future releases):
### Added
- Exposure of pool metrics (#179, #194)
### Changed
- Help doesn't print empty parameters (#186)
- Update version calculation (#190, #199)
- Update neofs-sdk-go (#196)
- Update go version in CI and docker (#197, #202)
## [0.23.0] - 2022-08-02
### Added
- New param to configure pool error threshold (#184)
### Changed
- Pprof and prometheus metrics configuration (#171)
- Drop GO111MODULES from builds (#182)
### Updating from v0.22.0
1. To enable pprof use `pprof.enabled` instead of `pprof` in config.
To enable prometheus metrics use `prometheus.enabled` instead of `metrics` in config.
If you are using the command line flags you can skip this step.
## [0.22.0] - 2022-07-25
### Added
- Default params documentation (#172)
- Health metric (#175)
### Changed
- Version output (#169)
- Updated SDK Version (#178)
## [0.21.0] - 2022-06-20
### Fixed
- Downloading ZIP archive using streaming (#163)
### Added
- New make target to build app in docker (#159)
### Changed
- Increased buffer size for file uploading (#148)
- Updated linter version to v1.46.2 (#161)
- Updated CodeQL version to v2 (#158)
## [0.20.0] - 2022-04-29
### Fixed
- Get rid of data race on server shutdown (#145)
- Improved English in docs and comments (#153)
- Use `FilePath` to download zip (#150)
### Added
- Support container name NNS resolving (#142)
### Changed
- Updated docs (#133, #140)
- Increased default read/write timeouts (#154)
- Updated SDK (#137, #139)
- Updated go version to 1.17 (#143)
- Improved error messages (#144)
## [0.19.0] - 2022-03-16
### Fixed
- Uploading object with zero payload (#122)
- Different headers format in GET and HEAD (#125)
- Fixed project name in docs (#120)
### Added
- Support object attributes with spaces (#123)
### Changed
- Updated fasthttp to v1.34.0 (#129)
- Updated NeoFS SDK to v1.0.0-rc.3 (#126, #132)
- Refactored content type detecting (#128)
## [0.18.0] - 2021-12-10
### Fixed
- System headers format (#111)
### Added
- Different formats to set object's expiration: in epoch, duration, timestamp,
RFC3339 (#108)
- Support of nodes priority (#115)
### Changed
- Updated testcontainers dependency (#100)
## [0.17.0] - 2021-11-15
Support of bulk file download with zip streams and various bug fixes.
### Fixed
- Allow canonical `X-Attribute-Neofs-*` headers (#87)
- Responses with error message now end with `\n` character (#105)
- Application does not require all neofs endpoints to be healthy at start now
(#103)
- Application now tracks session token errors and recreates tokens in runtime
(#95)
### Added
- Integration tests with [all-in-one](https://github.com/nspcc-dev/neofs-aio/)
test containers (#85, #94)
- Bulk download support with zip streams (#92, #96)
## 0.16.1 (28 Jul 2021)
New features:
* logging requests (#77)
* HEAD methods for download routes (#76)
Improvements:
* updated sdk-go dependency (#82)
Bugs fixed:
* wrong NotFound status was used (#30)
## 0.16.0 (29 Jun 2021)
We update HTTP gateway with NEP-6 wallets support, YAML configuration files
and small fixes.
New features:
* YAML configuration file (#71)
Behavior changes:
* gateway key needs to be stored in a proper NEP-6 wallet now, `-k` option is
no longer available, see `-w` and `-a` (#68)
Bugs fixed:
* downloads were not streamed leading to excessive memory usage (#67)
* Last-Modified header incorrectly used local time (#75)
## 0.15.2 (22 Jun 2021)
New features:
* Content-Type returned for object GET requests can now be taken from
attributes (overriding autodetection, #65)
Behavior changes:
* grpc keepalive options can no longer be changed (#60)
Improvements:
* code refactoring (more reuse between different gateways, moved some code to
sdk-go, #47, #46, #51, #62, #63)
* documentation updates and fixes (#53, #49, #55, #59)
* updated api-go dependency (#57)
Bugs fixed:
* `-k` option wasn't accepted for key although it was documented (#50)
## 0.15.1 (24 May 2021)
This important release makes HTTP gateway compatible with NeoFS node version
0.20.0.
Behavior changes:
* neofs-api-go was updated to 1.26.1, which contains some incompatible
changes in underlying components (#39, #44)
* `neofs-http-gw` is consistently used now for repository, binary and image
names (#43)
Improvements:
* minor code cleanups based on stricter set of linters (#41)
* updated README (#42)
## 0.15.0 (30 Apr 2021)
This is the first public release incorporating latest NeoFS protocol support
and fixing some bugs.
New features:
* upload support (#14, #13, #29)
* ephemeral keys (#26)
* TLS server support (#28)
Behavior changes:
* node weights can now be specified as simple numbers instead of percentages
and gateway will calculate the proportion automatically (#27)
* attributes are converted now to `X-Attribute-*` headers when retrieving
object from gate instead of `X-*` (#29)
Improvements:
* better Makefile (#16, #24, #33, #34)
* updated documentation (#16, #29, #35, #36)
* updated neofs-api-go to v1.25.0 (#17, #20)
* updated fasthttp to v1.23.0+ (#17, #29)
* refactoring, eliminating some dependencies (#20, #29)
Bugs fixed:
* gateway attempted to work with no NeoFS peers configured (#29)
* some invalid headers could be sent for attributes using non-ASCII or
non-printable characters (#29)
* `X-Attribute-Neofs-*` -> `X-Attribute-System-*`
* `X-Attribute-NEOFS-*` -> `X-Attribute-SYSTEM-*`
* `X-Attribute-neofs-*` -> `X-Attribute-system-*`
## Older versions
Please refer to [Github
releases](https://github.com/nspcc-dev/neofs-http-gw/releases/) for older
releases.
This project is a fork of [NeoFS HTTP Gateway](https://github.com/nspcc-dev/neofs-http-gw) from version v0.26.0.
To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs-http-gw/blob/master/CHANGELOG.md.
[0.17.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.16.1...v0.17.0
[0.18.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.17.0...v0.18.0
[0.19.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.18.0...v0.19.0
[0.20.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.19.0...v0.20.0
[0.21.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.20.0...v0.21.0
[0.22.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.21.0...v0.22.0
[0.23.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.22.0...v0.23.0
[0.24.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.23.0...v0.24.0
[0.25.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.24.0...v0.25.0
[0.25.1]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.25.0...v0.25.1
[0.26.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.25.1...v0.26.0
[Unreleased]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.26.0...master
[0.27.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/72734ab4...v0.27.0
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.27.0...master

15
Makefile Normal file → Executable file
View file

@ -14,7 +14,7 @@ BINDIR = bin
DIRS = $(BINDIR)
BINS = $(BINDIR)/frostfs-http-gw
.PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint version clean
.PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean
# .deb package versioning
OS_RELEASE = $(shell lsb_release -cs)
@ -62,6 +62,11 @@ docker/%:
test:
@go test ./... -cover
# Run integration tests
.PHONY: integration-test
integration-test:
@go test ./... -cover --tags=integration
# Run tests with race detection and produce coverage output
cover:
@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
@ -109,6 +114,14 @@ docker/lint:
--env HOME=/src \
golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint'
# Activate pre-commit hooks
pre-commit:
pre-commit install -t pre-commit -t commit-msg
# Deactivate pre-commit hooks
unpre-commit:
pre-commit uninstall -t pre-commit -t commit-msg
# Print version
version:
@echo $(VERSION)

View file

@ -1,11 +1,12 @@
<p align="center">
<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
</p>
<p align="center">
<a href="https://frostfs.info">FrostFS</a> is a decentralized distributed object storage integrated with the <a href="https://neo.org">NEO Blockchain</a>.
</p>
---
[![Report](https://goreportcard.com/badge/github.com/TrueCloudLab/frostfs-http-gw)](https://goreportcard.com/report/github.com/TrueCloudLab/frostfs-http-gw)
![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/TrueCloudLab/frostfs-http-gw?sort=semver)
![License](https://img.shields.io/github/license/TrueCloudLab/frostfs-http-gw.svg?style=popout)
[![Report](https://goreportcard.com/badge/git.frostfs.info/TrueCloudLab/frostfs-http-gw)](https://goreportcard.com/report/git.frostfs.info/TrueCloudLab/frostfs-http-gw)
# FrostFS HTTP Gateway
@ -17,7 +18,7 @@ See available routes in [specification](./docs/api.md).
## Installation
```go install github.com/TrueCloudLab/frostfs-http-gw```
```go install git.frostfs.info/TrueCloudLab/frostfs-http-gw```
Or you can call `make` to build it from the cloned repository (the binary will
end up in `bin/frostfs-http-gw`). To build frostfs-http-gw binary in clean docker
@ -47,7 +48,7 @@ can be done either via `-p` parameter or via `HTTP_GW_PEERS_<N>_ADDRESS` and
`HTTP_GW_PEERS_<N>_WEIGHT` environment variables (the gate supports multiple
FrostFS nodes with weighted load balancing).
If you launch HTTP gateway in bundle with [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env),
If you launch HTTP gateway in bundle with [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env),
you can get the IP address of the node in the output of `make hosts` command
(with s0*.frostfs.devenv name).
@ -179,6 +180,23 @@ $ frostfs-http-gw --config your-config.yaml
See [config](./config/config.yaml) and [defaults](./docs/gate-configuration.md) for example.
#### Multiple configs
You can use several config files when running application. It allows you to split configuration into parts.
For example, you can use separate yaml file for pprof and prometheus section in config (see [config examples](./config)).
You can either provide several files with repeating `--config` flag or provide path to the dir that contains all configs using `--config-dir` flag.
Also, you can combine these flags:
```shell
$ frostfs-http-gw --config ./config/config.yaml --config /your/partial/config.yaml --config-dir ./config/dir
```
**Note:** next file in `--config` flag overwrites values from the previous one.
Files from `--config-dir` directory overwrite values from `--config` files.
So the command above run `frostfs-http-gw` to listen on `0.0.0.0:8080` address (value from `./config/config.yaml`),
applies parameters from `/your/partial/config.yaml`,
enable pprof (value from `./config/dir/pprof.yaml`) and prometheus (value from `./config/dir/prometheus.yaml`).
## HTTP API provided
This gateway intentionally provides limited feature set and doesn't try to
@ -208,7 +226,7 @@ resolve_order:
- nns
```
2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env)
2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env)
you can check if your container (e.g. with `container-name` name) is registered in NNS:
```shell
@ -235,7 +253,7 @@ $ curl http://localhost:8082/get_by_attribute/container-name/FileName/object-nam
#### Create a container
You can create a container via [frostfs-cli](https://github.com/TrueCloudLab/frostfs-node/releases):
You can create a container via [frostfs-cli](https://git.frostfs.info/TrueCloudLab/frostfs-node/releases):
```
$ frostfs-cli -r $FROSTFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL
```
@ -248,13 +266,13 @@ For example:
$ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json container create --policy "REP 3" --basic-acl public --await
```
If you have launched nodes via [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env),
If you have launched nodes via [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env),
you can get the key value from `wallets/wallet.json` or write the path to
the file `wallets/wallet.key`.
#### Prepare a file in a container
To create a file via [frostfs-cli](https://github.com/TrueCloudLab/frostfs-node/releases), run a command below:
To create a file via [frostfs-cli](https://git.frostfs.info/TrueCloudLab/frostfs-node/releases), run a command below:
```
$ frostfs-cli -r $FROSTFS_NODE -k $KEY object put --file $FILENAME --cid $CID
```
@ -406,12 +424,12 @@ You can also add some attributes to your file using the following rules:
"X-Attribute-" prefix stripped, that is if you add "X-Attribute-Ololo:
100500" header to your request the resulting object will get "Ololo:
100500" attribute
* "X-Attribute-NEOFS-*" headers are special
(`-NEOFS-` part can also be `-neofs-` or`-Neofs-`), they're used to set internal
NeoFS attributes starting with `__NEOFS__` prefix, for these attributes all
* "X-Attribute-SYSTEM-*" headers are special
(`-SYSTEM-` part can also be `-system-` or`-System-` (and even legacy `-Neofs-` for some next releases)), they're used to set internal
FrostFS attributes starting with `__SYSTEM__` prefix, for these attributes all
dashes get converted to underscores and all letters are capitalized. For
example, you can use "X-Attribute-NEOFS-Expiration-Epoch" header to set
`__NEOFS__EXPIRATION_EPOCH` attribute
example, you can use "X-Attribute-SYSTEM-Expiration-Epoch" header to set
`__SYSTEM__EXPIRATION_EPOCH` attribute
* `FileName` attribute is set from multipart's `filename` if not set
explicitly via `X-Attribute-FileName` header
* `Timestamp` attribute can be set using gateway local time if using
@ -421,13 +439,13 @@ You can also add some attributes to your file using the following rules:
---
**NOTE**
There are some reserved headers type of `X-Attribute-NEOFS-*` (headers are arranged in descending order of priority):
1. `X-Attribute-Neofs-Expiration-Epoch: 100`
2. `X-Attribute-Neofs-Expiration-Duration: 24h30m`
3. `X-Attribute-Neofs-Expiration-Timestamp: 1637574797`
4. `X-Attribute-Neofs-Expiration-RFC3339: 2021-11-22T09:55:49Z`
There are some reserved headers type of `X-Attribute-SYSTEM-*` (headers are arranged in descending order of priority):
1. `X-Attribute-System-Expiration-Epoch: 100`
2. `X-Attribute-System-Expiration-Duration: 24h30m`
3. `X-Attribute-System-Expiration-Timestamp: 1637574797`
4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z`
which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide expiration any convenient way.
which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way.
---

View file

@ -1 +1 @@
v0.26.0
v0.27.0

88
app.go
View file

@ -11,14 +11,14 @@ import (
"sync"
"syscall"
"github.com/TrueCloudLab/frostfs-http-gw/downloader"
"github.com/TrueCloudLab/frostfs-http-gw/metrics"
"github.com/TrueCloudLab/frostfs-http-gw/resolver"
"github.com/TrueCloudLab/frostfs-http-gw/response"
"github.com/TrueCloudLab/frostfs-http-gw/uploader"
"github.com/TrueCloudLab/frostfs-http-gw/utils"
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/TrueCloudLab/frostfs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/downloader"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/fasthttp/router"
"github.com/nspcc-dev/neo-go/cli/flags"
"github.com/nspcc-dev/neo-go/cli/input"
@ -62,15 +62,10 @@ type (
gateMetrics struct {
logger *zap.Logger
provider GateMetricsProvider
provider *metrics.GateMetrics
mu sync.RWMutex
enabled bool
}
GateMetricsProvider interface {
SetHealth(int32)
Unregister()
}
)
// WithLogger returns Option to set a specific logger.
@ -214,15 +209,17 @@ func (a *app) getResolverConfig() ([]string, *resolver.Config) {
func (a *app) initMetrics() {
gateMetricsProvider := metrics.NewGateMetrics(a.pool)
a.metrics = newGateMetrics(a.log, gateMetricsProvider, a.cfg.GetBool(cfgPrometheusEnabled))
a.metrics.SetHealth(metrics.HealthStatusStarting)
}
func newGateMetrics(logger *zap.Logger, provider GateMetricsProvider, enabled bool) *gateMetrics {
func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled bool) *gateMetrics {
if !enabled {
logger.Warn("metrics are disabled")
}
return &gateMetrics{
logger: logger,
provider: provider,
enabled: enabled,
}
}
@ -236,7 +233,7 @@ func (m *gateMetrics) SetEnabled(enabled bool) {
m.mu.Unlock()
}
func (m *gateMetrics) SetHealth(status int32) {
func (m *gateMetrics) SetHealth(status metrics.HealthStatus) {
m.mu.RLock()
if !m.enabled {
m.mu.RUnlock()
@ -250,7 +247,7 @@ func (m *gateMetrics) SetHealth(status int32) {
func (m *gateMetrics) Shutdown() {
m.mu.Lock()
if m.enabled {
m.provider.SetHealth(0)
m.provider.SetHealth(metrics.HealthStatusShuttingDown)
m.enabled = false
}
m.provider.Unregister()
@ -335,7 +332,7 @@ func (a *app) Wait() {
}
func (a *app) setHealthStatus() {
a.metrics.SetHealth(1)
a.metrics.SetHealth(metrics.HealthStatusReady)
}
func (a *app) Serve(ctx context.Context) {
@ -380,14 +377,15 @@ LOOP:
func (a *app) configReload() {
a.log.Info("SIGHUP config reload started")
if !a.cfg.IsSet(cmdConfig) {
if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) {
a.log.Warn("failed to reload config because it's missed")
return
}
if err := readConfig(a.cfg); err != nil {
if err := readInConfig(a.cfg); err != nil {
a.log.Warn("failed to reload config", zap.Error(err))
return
}
if lvl, err := getLogLevel(a.cfg); err != nil {
a.log.Warn("log level won't be updated", zap.Error(err))
} else {
@ -485,33 +483,57 @@ func (a *app) AppParams() *utils.AppParams {
func (a *app) initServers(ctx context.Context) {
serversInfo := fetchServers(a.cfg)
a.servers = make([]Server, len(serversInfo))
for i, serverInfo := range serversInfo {
a.log.Info("added server",
a.servers = make([]Server, 0, len(serversInfo))
for _, serverInfo := range serversInfo {
fields := []zap.Field{
zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled),
zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile))
a.servers[i] = newServer(ctx, serverInfo, a.log)
zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile),
}
srv, err := newServer(ctx, serverInfo)
if err != nil {
a.log.Warn("failed to add server", append(fields, zap.Error(err))...)
continue
}
a.servers = append(a.servers, srv)
a.log.Info("add server", fields...)
}
if len(a.servers) == 0 {
a.log.Fatal("no healthy servers")
}
}
func (a *app) updateServers() error {
serversInfo := fetchServers(a.cfg)
if len(serversInfo) != len(a.servers) {
return fmt.Errorf("invalid servers configuration: length mismatch: old '%d', new '%d", len(a.servers), len(serversInfo))
}
for i, serverInfo := range serversInfo {
if serverInfo.Address != a.servers[i].Address() {
return fmt.Errorf("invalid servers configuration: addresses mismatch: old '%s', new '%s", a.servers[i].Address(), serverInfo.Address)
var found bool
for _, serverInfo := range serversInfo {
index := a.serverIndex(serverInfo.Address)
if index == -1 {
continue
}
if serverInfo.TLS.Enabled {
if err := a.servers[i].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
if err := a.servers[index].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
return fmt.Errorf("failed to update tls certs: %w", err)
}
}
found = true
}
if !found {
return fmt.Errorf("invalid servers configuration: no known server found")
}
return nil
}
func (a *app) serverIndex(address string) int {
for i := range a.servers {
if a.servers[i].Address() == address {
return i
}
}
return -1
}

View file

@ -4,10 +4,10 @@ wallet:
passphrase: pwd # Passphrase to decrypt wallet. If you're using a wallet without a password, place '' here.
pprof:
enabled: true # Enable pprof.
enabled: false # Enable pprof.
address: localhost:8083
prometheus:
enabled: true # Enable metrics.
enabled: false # Enable metrics.
address: localhost:8084
logger:

3
config/dir/pprof.yaml Normal file
View file

@ -0,0 +1,3 @@
pprof:
enabled: true
address: localhost:8083

View file

@ -0,0 +1,3 @@
prometheus:
enabled: true
address: localhost:8084

5
debian/control vendored
View file

@ -5,11 +5,10 @@ Maintainer: TrueCloudLab <tech@frostfs.info>
Build-Depends: debhelper-compat (= 13), dh-sysuser, git, devscripts
Standards-Version: 4.5.1
Homepage: https://frostfs.info/
Vcs-Git: https://github.com/TrueCloudLab/frostfs-http-gw.git
Vcs-Browser: https://github.com/TrueCloudLab/frostfs-http-gw
Vcs-Git: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw.git
Vcs-Browser: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw
Package: frostfs-http-gw
Architecture: any
Depends: ${misc:Depends}
Description: FrostFS HTTP Gateway bridges FrostFS internal protocol and HTTP standard.

4
debian/copyright vendored
View file

@ -1,13 +1,13 @@
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: frostfs-http-gw
Upstream-Contact: tech@frostfs.info
Source: https://github.com/TrueCloudLab/frostfs-http-gw
Source: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw
Files: *
Copyright: 2018-2022 NeoSPCC (@nspcc-dev), contributors of neofs-http-gw project
(https://github.com/nspcc-dev/neofs-http-gw/blob/master/CREDITS.md)
2022 True Cloud Lab (@TrueCloudLab), contributors of frostfs-http-gw project
(https://github.com/TrueCloudLab/frostfs-http-gw/blob/master/CREDITS.md)
(https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/src/branch/master/CREDITS.md)
License: GPL-3

4
debian/frostfs-http-gw.postinst vendored Normal file → Executable file
View file

@ -29,8 +29,8 @@ case "$1" in
chmod -f 0640 /etc/frostfs/$USERNAME/config.yaml || true
fi
USERDIR=$(getent passwd "frostfs-$USERNAME" | cut -d: -f6)
if ! dpkg-statoverride --list frostfs-$USERDIR >/dev/null; then
chown -f frostfs-$USERNAME: $USERDIR
if ! dpkg-statoverride --list frostfs-"$USERDIR" >/dev/null; then
chown -f frostfs-$USERNAME: "$USERDIR"
fi
;;

0
debian/frostfs-http-gw.postrm vendored Normal file → Executable file
View file

0
debian/frostfs-http-gw.preinst vendored Normal file → Executable file
View file

0
debian/frostfs-http-gw.prerm vendored Normal file → Executable file
View file

2
debian/rules vendored
View file

@ -12,5 +12,3 @@ override_dh_installsystemd:
override_dh_installchangelogs:
dh_installchangelogs -k CHANGELOG.md

View file

@ -57,20 +57,20 @@ Upload file as object with attributes to FrostFS.
###### Headers
| Header | Description |
|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|
|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|
| Common headers | See [bearer token](#bearer-token). |
| `X-Attribute-Neofs-*` | Used to set system NeoFS object attributes <br/> (e.g. use "X-Attribute-Neofs-Expiration-Epoch" to set `__NEOFS__EXPIRATION_EPOCH` attribute). |
| `X-Attribute-System-*` | Used to set system FrostFS object attributes <br/> (e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). |
| `X-Attribute-*` | Used to set regular object attributes <br/> (e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). |
| `Date` | This header is used to calculate the right `__NEOFS__EXPIRATION` attribute for object. If the header is missing, the current server time is used. |
| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. |
There are some reserved headers type of `X-Attribute-NEOFS-*` (headers are arranged in descending order of priority):
There are some reserved headers type of `X-Attribute-FROSTFS-*` (headers are arranged in descending order of priority):
1. `X-Attribute-Neofs-Expiration-Epoch: 100`
2. `X-Attribute-Neofs-Expiration-Duration: 24h30m`
3. `X-Attribute-Neofs-Expiration-Timestamp: 1637574797`
4. `X-Attribute-Neofs-Expiration-RFC3339: 2021-11-22T09:55:49Z`
1. `X-Attribute-System-Expiration-Epoch: 100`
2. `X-Attribute-System-Expiration-Duration: 24h30m`
3. `X-Attribute-System-Expiration-Timestamp: 1637574797`
4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z`
which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide expiration any convenient way.
which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way.
If you don't specify the `X-Attribute-Timestamp` header the `Timestamp` attribute can be set anyway
(see http-gw [configuration](gate-configuration.md#upload-header-section)).
@ -122,8 +122,8 @@ Get an object (payload and attributes) by an address.
###### Headers
| Header | Description |
|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
| `X-Attribute-Neofs-*` | System NeoFS object attributes <br/> (e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). |
|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
| `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
| `Content-Disposition` | Indicate how to browsers should treat file. <br/> Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). |
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
@ -158,8 +158,8 @@ Get an object attributes by an address.
###### Headers
| Header | Description |
|-----------------------|--------------------------------------------------------------------------------------------------------------------------|
| `X-Attribute-Neofs-*` | System NeoFS object attributes <br/> (e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). |
|------------------------|------------------------------------------------------------------------------------------------------------------------------|
| `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
| `Content-Length` | Size of object payload. |
@ -207,8 +207,8 @@ If more than one object is found, an arbitrary one will be returned.
###### Headers
| Header | Description |
|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
| `X-Attribute-Neofs-*` | System NeoFS object attributes <br/> (e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). |
|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
| `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
| `Content-Disposition` | Indicate how to browsers should treat file. <br/> Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). |
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
@ -244,8 +244,8 @@ If more than one object is found, an arbitrary one will be used to get attribute
###### Headers
| Header | Description |
|-----------------------|--------------------------------------------------------------------------------------------------------------------------|
| `X-Attribute-Neofs-*` | System NeoFS object attributes <br/> (e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). |
|------------------------|------------------------------------------------------------------------------------------------------------------------------|
| `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
| `Content-Length` | Size of object payload. |

View file

@ -14,20 +14,18 @@ import (
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/TrueCloudLab/frostfs-http-gw/resolver"
"github.com/TrueCloudLab/frostfs-http-gw/response"
"github.com/TrueCloudLab/frostfs-http-gw/tokens"
"github.com/TrueCloudLab/frostfs-http-gw/utils"
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
"github.com/TrueCloudLab/frostfs-sdk-go/client"
"github.com/TrueCloudLab/frostfs-sdk-go/container"
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
"github.com/TrueCloudLab/frostfs-sdk-go/object"
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/valyala/fasthttp"
"go.uber.org/atomic"
"go.uber.org/zap"
@ -131,9 +129,9 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) {
if !isValidToken(key) || !isValidValue(val) {
continue
}
if strings.HasPrefix(key, utils.SystemAttributePrefix) {
key = systemBackwardTranslator(key)
}
key = utils.BackwardTransformIfSystem(key)
r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
switch key {
case object.AttributeFileName:
@ -187,36 +185,6 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) {
r.Response.SetBodyStream(rObj.Payload, int(payloadSize))
}
// systemBackwardTranslator is used to convert headers looking like '__NEOFS__ATTR_NAME' to 'Neofs-Attr-Name'.
func systemBackwardTranslator(key string) string {
// trim specified prefix '__NEOFS__'
key = strings.TrimPrefix(key, utils.SystemAttributePrefix)
var res strings.Builder
res.WriteString("Neofs-")
strs := strings.Split(key, "_")
for i, s := range strs {
s = title(strings.ToLower(s))
res.WriteString(s)
if i != len(strs)-1 {
res.WriteString("-")
}
}
return res.String()
}
func title(str string) string {
if str == "" {
return ""
}
r, size := utf8.DecodeRuneInString(str)
r0 := unicode.ToTitle(r)
return string(r0) + str[size:]
}
func bearerToken(ctx context.Context) *bearer.Token {
if tkn, err := tokens.LoadBearerToken(ctx); err == nil {
return tkn
@ -225,19 +193,15 @@ func bearerToken(ctx context.Context) *bearer.Token {
}
func (r *request) handleFrostFSErr(err error, start time.Time) {
r.log.Error(
"could not receive object",
logFields := []zap.Field{
zap.Stringer("elapsed", time.Since(start)),
zap.Error(err),
)
if client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err) {
response.Error(r.RequestCtx, "Not Found", fasthttp.StatusNotFound)
return
}
statusCode, msg, additionalFields := response.FormErrorResponse("could not receive object", err)
logFields = append(logFields, additionalFields...)
msg := fmt.Sprintf("could not receive object: %v", err)
response.Error(r.RequestCtx, msg, fasthttp.StatusBadRequest)
r.log.Error("could not receive object", logFields...)
response.Error(r.RequestCtx, msg, statusCode)
}
// Downloader is a download request handler.

View file

@ -1,23 +0,0 @@
package downloader
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestSystemBackwardTranslator(t *testing.T) {
input := []string{
"__NEOFS__EXPIRATION_EPOCH",
"__NEOFS__RANDOM_ATTR",
}
expected := []string{
"Neofs-Expiration-Epoch",
"Neofs-Random-Attr",
}
for i, str := range input {
res := systemBackwardTranslator(str)
require.Equal(t, expected[i], res)
}
}

View file

@ -4,15 +4,14 @@ import (
"io"
"net/http"
"strconv"
"strings"
"time"
"github.com/TrueCloudLab/frostfs-http-gw/response"
"github.com/TrueCloudLab/frostfs-http-gw/tokens"
"github.com/TrueCloudLab/frostfs-http-gw/utils"
"github.com/TrueCloudLab/frostfs-sdk-go/object"
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
@ -56,9 +55,9 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) {
if !isValidToken(key) || !isValidValue(val) {
continue
}
if strings.HasPrefix(key, utils.SystemAttributePrefix) {
key = systemBackwardTranslator(key)
}
key = utils.BackwardTransformIfSystem(key)
r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
switch key {
case object.AttributeTimestamp:

View file

@ -1,3 +1,5 @@
//go:build !integration
package downloader
import (

89
go.mod
View file

@ -1,71 +1,74 @@
module github.com/TrueCloudLab/frostfs-http-gw
module git.frostfs.info/TrueCloudLab/frostfs-http-gw
go 1.17
go 1.18
require (
github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68
github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221214065929-4c779423f556
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142024-4cb0068ddef0
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230329125804-552219b8e130
github.com/fasthttp/router v1.4.1
github.com/nspcc-dev/neo-go v0.99.4
github.com/nspcc-dev/neo-go v0.101.0
github.com/prometheus/client_golang v1.13.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.8.1
github.com/stretchr/testify v1.8.0
github.com/spf13/viper v1.15.0
github.com/stretchr/testify v1.8.2
github.com/testcontainers/testcontainers-go v0.13.0
github.com/valyala/fasthttp v1.34.0
go.uber.org/atomic v1.10.0
go.uber.org/zap v1.23.0
go.uber.org/zap v1.24.0
)
require (
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb // indirect
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect
git.frostfs.info/TrueCloudLab/hrw v1.2.0 // indirect
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Microsoft/hcsshim v0.9.2 // indirect
github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42 // indirect
github.com/TrueCloudLab/frostfs-crypto v0.5.0 // indirect
github.com/TrueCloudLab/hrw v1.1.0 // indirect
github.com/TrueCloudLab/rfc6979 v0.3.0 // indirect
github.com/TrueCloudLab/tzhash v1.7.0 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btcd v0.22.0-beta // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cenkalti/backoff/v4 v4.2.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/containerd/cgroups v1.0.3 // indirect
github.com/containerd/containerd v1.6.2 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/docker v20.10.14+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/hashicorp/golang-lru v0.6.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/klauspost/compress v1.15.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/sys/mount v0.3.2 // indirect
github.com/moby/sys/mountinfo v0.6.1 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262 // indirect
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/opencontainers/runc v1.1.1 // indirect
github.com/pelletier/go-toml v1.9.3 // indirect
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
@ -75,26 +78,34 @@ require (
github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.6.0 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/afero v1.9.3 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
github.com/urfave/cli v1.22.5 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
go.opencensus.io v0.23.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect
google.golang.org/grpc v1.48.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.14.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect
go.opentelemetry.io/otel/sdk v1.14.0 // indirect
go.opentelemetry.io/otel/trace v1.14.0 // indirect
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
golang.org/x/crypto v0.4.0 // indirect
golang.org/x/exp v0.0.0-20221227203929-1b447090c38c // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/term v0.5.0 // indirect
golang.org/x/text v0.7.0 // indirect
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
google.golang.org/grpc v1.53.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

409
go.sum

File diff suppressed because it is too large Load diff

View file

@ -1,3 +1,5 @@
//go:build integration
package main
import (
@ -13,14 +15,15 @@ import (
"testing"
"time"
"github.com/TrueCloudLab/frostfs-sdk-go/container"
"github.com/TrueCloudLab/frostfs-sdk-go/container/acl"
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
"github.com/TrueCloudLab/frostfs-sdk-go/netmap"
"github.com/TrueCloudLab/frostfs-sdk-go/object"
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/TrueCloudLab/frostfs-sdk-go/user"
containerv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
@ -35,21 +38,15 @@ type putResponse struct {
const (
testContainerName = "friendly"
versionWithNativeNames = "0.27.5"
testListenAddress = "localhost:8082"
testHost = "http://" + testListenAddress
)
func TestIntegration(t *testing.T) {
rootCtx := context.Background()
aioImage := "nspccdev/neofs-aio-testcontainer:"
aioImage := "truecloudlab/frostfs-aio:"
versions := []string{
"0.27.5",
"0.28.1",
"0.29.0",
"0.30.0",
"0.32.0",
"latest",
"1.2.5", // frostfs-storage v0.36.0 RC
}
key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb")
require.NoError(t, err)
@ -95,11 +92,9 @@ func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID cid.ID, vers
url := testHost + "/upload/" + CID.String()
makePutRequestAndCheck(ctx, t, p, CID, url)
if version >= versionWithNativeNames {
url = testHost + "/upload/" + testContainerName
makePutRequestAndCheck(ctx, t, p, CID, url)
}
}
func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnrID cid.ID, url string) {
content := "content of file"
@ -221,12 +216,10 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID
require.NoError(t, err)
checkGetResponse(t, resp, content, attributes)
if version >= versionWithNativeNames {
resp, err = http.Get(testHost + "/get/" + testContainerName + "/" + id.String())
require.NoError(t, err)
checkGetResponse(t, resp, content, attributes)
}
}
func checkGetResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) {
defer func() {
@ -275,12 +268,10 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID
require.NoError(t, err)
checkGetByAttrResponse(t, resp, content, expectedAttr)
if version >= versionWithNativeNames {
resp, err = http.Get(testHost + "/get_by_attribute/" + testContainerName + "/" + keyAttr + "/" + valAttr)
require.NoError(t, err)
checkGetByAttrResponse(t, resp, content, expectedAttr)
}
}
func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) {
names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"}
@ -294,11 +285,9 @@ func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID us
baseURL := testHost + "/zip/" + CID.String()
makeZipTest(t, baseURL, names, contents)
if version >= versionWithNativeNames {
baseURL = testHost + "/zip/" + testContainerName
makeZipTest(t, baseURL, names, contents)
}
}
func makeZipTest(t *testing.T, baseURL string, names, contents []string) {
url := baseURL + "/zipfolder"
@ -405,11 +394,11 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o
container.SetCreationTime(&cnr, time.Now())
if version >= versionWithNativeNames {
var domain container.Domain
domain.SetName(testContainerName)
container.WriteDomain(&cnr, domain)
}
cnr.SetAttribute(containerv2.SysAttributeName, domain.Name())
cnr.SetAttribute(containerv2.SysAttributeZone, domain.Zone())
var waitPrm pool.WaitParams
waitPrm.SetTimeout(15 * time.Second)

View file

@ -3,7 +3,7 @@ package metrics
import (
"net/http"
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.uber.org/zap"
@ -31,6 +31,16 @@ const (
methodCreateSession = "create_session"
)
// HealthStatus of the gate application.
type HealthStatus int32
const (
HealthStatusUndefined HealthStatus = 0
HealthStatusStarting HealthStatus = 1
HealthStatusReady HealthStatus = 2
HealthStatusShuttingDown HealthStatus = 3
)
type GateMetrics struct {
stateMetrics
poolMetricsCollector
@ -87,7 +97,7 @@ func (m stateMetrics) unregister() {
prometheus.Unregister(m.healthCheck)
}
func (m stateMetrics) SetHealth(s int32) {
func (m stateMetrics) SetHealth(s HealthStatus) {
m.healthCheck.Set(float64(s))
}

View file

@ -5,7 +5,7 @@ import (
"errors"
"fmt"
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
)
// FrostFSResolver represents virtual connection to the FrostFS network.

View file

@ -6,9 +6,9 @@ import (
"fmt"
"sync"
"github.com/TrueCloudLab/frostfs-sdk-go/container"
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
"github.com/TrueCloudLab/frostfs-sdk-go/ns"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
)
const (

View file

@ -1,7 +1,41 @@
package response
import "github.com/valyala/fasthttp"
import (
"errors"
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
sdkstatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
func Error(r *fasthttp.RequestCtx, msg string, code int) {
r.Error(msg+"\n", code)
}
func FormErrorResponse(message string, err error) (int, string, []zap.Field) {
var (
msg string
statusCode int
logFields []zap.Field
)
st := new(sdkstatus.ObjectAccessDenied)
switch {
case errors.As(err, &st):
statusCode = fasthttp.StatusForbidden
reason := st.Reason()
msg = fmt.Sprintf("%s: %v: %s", message, err, reason)
logFields = append(logFields, zap.String("error_detail", reason))
case client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err):
statusCode = fasthttp.StatusNotFound
msg = "Not Found"
default:
statusCode = fasthttp.StatusBadRequest
msg = fmt.Sprintf("%s: %v", message, err)
}
return statusCode, msg, logFields
}

View file

@ -7,8 +7,6 @@ import (
"fmt"
"net"
"sync"
"go.uber.org/zap"
)
type (
@ -57,11 +55,11 @@ func (s *server) UpdateCert(certFile, keyFile string) error {
return s.tlsProvider.UpdateCert(certFile, keyFile)
}
func newServer(ctx context.Context, serverInfo ServerInfo, logger *zap.Logger) *server {
func newServer(ctx context.Context, serverInfo ServerInfo) (*server, error) {
var lic net.ListenConfig
ln, err := lic.Listen(ctx, "tcp", serverInfo.Address)
if err != nil {
logger.Fatal("could not prepare listener", zap.String("address", serverInfo.Address), zap.Error(err))
return nil, fmt.Errorf("could not prepare listener: %w", err)
}
tlsProvider := &certProvider{
@ -70,7 +68,7 @@ func newServer(ctx context.Context, serverInfo ServerInfo, logger *zap.Logger) *
if serverInfo.TLS.Enabled {
if err = tlsProvider.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
logger.Fatal("failed to update cert", zap.Error(err))
return nil, fmt.Errorf("failed to update cert: %w", err)
}
ln = tls.NewListener(ln, &tls.Config{
@ -82,7 +80,7 @@ func newServer(ctx context.Context, serverInfo ServerInfo, logger *zap.Logger) *
address: serverInfo.Address,
listener: ln,
tlsProvider: tlsProvider,
}
}, nil
}
func (p *certProvider) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {

View file

@ -3,13 +3,14 @@ package main
import (
"fmt"
"os"
"path"
"runtime"
"sort"
"strconv"
"strings"
"time"
"github.com/TrueCloudLab/frostfs-http-gw/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
"github.com/spf13/pflag"
"github.com/spf13/viper"
"github.com/valyala/fasthttp"
@ -84,6 +85,7 @@ const (
cmdWallet = "wallet"
cmdAddress = "address"
cmdConfig = "config"
cmdConfigDir = "config-dir"
cmdListenAddress = "listen_address"
)
@ -114,7 +116,8 @@ func settings() *viper.Viper {
flags.StringP(cmdWallet, "w", "", `path to the wallet`)
flags.String(cmdAddress, "", `address of wallet account`)
flags.String(cmdConfig, "", "config path")
flags.StringArray(cmdConfig, nil, "config paths")
flags.String(cmdConfigDir, "", "config dir path")
flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout")
flags.Duration(cfgStreamTimeout, defaultStreamTimeout, "gRPC individual message timeout")
flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout")
@ -233,11 +236,9 @@ func settings() *viper.Viper {
os.Exit(0)
}
if v.IsSet(cmdConfig) {
if err := readConfig(v); err != nil {
if err := readInConfig(v); err != nil {
panic(err)
}
}
if peers != nil && len(*peers) > 0 {
for i := range *peers {
@ -250,17 +251,72 @@ func settings() *viper.Viper {
return v
}
func readConfig(v *viper.Viper) error {
cfgFileName := v.GetString(cmdConfig)
cfgFile, err := os.Open(cfgFileName)
func readInConfig(v *viper.Viper) error {
if v.IsSet(cmdConfig) {
if err := readConfig(v); err != nil {
return err
}
}
if v.IsSet(cmdConfigDir) {
if err := readConfigDir(v); err != nil {
return err
}
}
return nil
}
func readConfigDir(v *viper.Viper) error {
cfgSubConfigDir := v.GetString(cmdConfigDir)
entries, err := os.ReadDir(cfgSubConfigDir)
if err != nil {
return err
}
if err = v.ReadConfig(cfgFile); err != nil {
for _, entry := range entries {
if entry.IsDir() {
continue
}
ext := path.Ext(entry.Name())
if ext != ".yaml" && ext != ".yml" {
continue
}
if err = mergeConfig(v, path.Join(cfgSubConfigDir, entry.Name())); err != nil {
return err
}
}
return nil
}
func readConfig(v *viper.Viper) error {
for _, fileName := range v.GetStringSlice(cmdConfig) {
if err := mergeConfig(v, fileName); err != nil {
return err
}
}
return nil
}
func mergeConfig(v *viper.Viper, fileName string) error {
cfgFile, err := os.Open(fileName)
if err != nil {
return err
}
return cfgFile.Close()
defer func() {
if errClose := cfgFile.Close(); errClose != nil {
panic(errClose)
}
}()
if err = v.MergeConfig(cfgFile); err != nil {
return err
}
return nil
}
// newLogger constructs a zap.Logger instance for current application.

View file

@ -7,7 +7,7 @@ import (
"errors"
"fmt"
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"github.com/valyala/fasthttp"
)

View file

@ -1,11 +1,13 @@
//go:build !integration
package tokens
import (
"encoding/base64"
"testing"
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
"github.com/TrueCloudLab/frostfs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/stretchr/testify/require"
"github.com/valyala/fasthttp"

View file

@ -3,29 +3,12 @@ package uploader
import (
"bytes"
"fmt"
"math"
"strconv"
"time"
"github.com/TrueCloudLab/frostfs-api-go/v2/object"
"github.com/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
var frostfsAttributeHeaderPrefixes = [...][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")}
func systemTranslator(key, prefix []byte) []byte {
// replace the specified prefix with `__NEOFS__`
key = bytes.Replace(key, prefix, []byte(utils.SystemAttributePrefix), 1)
// replace `-` with `_`
key = bytes.ReplaceAll(key, []byte("-"), []byte("_"))
// replace with uppercase
return bytes.ToUpper(key)
}
func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]string, error) {
var err error
result := make(map[string]string)
@ -45,13 +28,7 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st
// removing attribute prefix
clearKey := bytes.TrimPrefix(key, prefix)
// checks that it's a system NeoFS header
for _, system := range frostfsAttributeHeaderPrefixes {
if bytes.HasPrefix(clearKey, system) {
clearKey = systemTranslator(clearKey, system)
break
}
}
clearKey = utils.TransformIfSystem(clearKey)
// checks that the attribute key is not empty
if len(clearKey) == 0 {
@ -77,69 +54,3 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st
return result, err
}
func prepareExpirationHeader(headers map[string]string, epochDurations *epochDurations, now time.Time) error {
expirationInEpoch := headers[object.SysAttributeExpEpoch]
if timeRFC3339, ok := headers[utils.ExpirationRFC3339Attr]; ok {
expTime, err := time.Parse(time.RFC3339, timeRFC3339)
if err != nil {
return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, utils.ExpirationRFC3339Attr)
}
if expTime.Before(now) {
return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, utils.ExpirationRFC3339Attr)
}
updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
delete(headers, utils.ExpirationRFC3339Attr)
}
if timestamp, ok := headers[utils.ExpirationTimestampAttr]; ok {
value, err := strconv.ParseInt(timestamp, 10, 64)
if err != nil {
return fmt.Errorf("couldn't parse value %s of header %s", timestamp, utils.ExpirationTimestampAttr)
}
expTime := time.Unix(value, 0)
if expTime.Before(now) {
return fmt.Errorf("value %s of header %s must be in the future", timestamp, utils.ExpirationTimestampAttr)
}
updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
delete(headers, utils.ExpirationTimestampAttr)
}
if duration, ok := headers[utils.ExpirationDurationAttr]; ok {
expDuration, err := time.ParseDuration(duration)
if err != nil {
return fmt.Errorf("couldn't parse value %s of header %s", duration, utils.ExpirationDurationAttr)
}
if expDuration <= 0 {
return fmt.Errorf("value %s of header %s must be positive", expDuration, utils.ExpirationDurationAttr)
}
updateExpirationHeader(headers, epochDurations, expDuration)
delete(headers, utils.ExpirationDurationAttr)
}
if expirationInEpoch != "" {
headers[object.SysAttributeExpEpoch] = expirationInEpoch
}
return nil
}
func updateExpirationHeader(headers map[string]string, durations *epochDurations, expDuration time.Duration) {
epochDuration := uint64(durations.msPerBlock) * durations.blockPerEpoch
currentEpoch := durations.currentEpoch
numEpoch := uint64(expDuration.Milliseconds()) / epochDuration
if uint64(expDuration.Milliseconds())%epochDuration != 0 {
numEpoch++
}
expirationEpoch := uint64(math.MaxUint64)
if numEpoch < math.MaxUint64-currentEpoch {
expirationEpoch = currentEpoch + numEpoch
}
headers[object.SysAttributeExpEpoch] = strconv.FormatUint(expirationEpoch, 10)
}

View file

@ -1,13 +1,10 @@
//go:build !integration
package uploader
import (
"math"
"strconv"
"testing"
"time"
"github.com/TrueCloudLab/frostfs-api-go/v2/object"
"github.com/TrueCloudLab/frostfs-http-gw/utils"
"github.com/stretchr/testify/require"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
@ -28,8 +25,8 @@ func TestFilter(t *testing.T) {
t.Run("duplicate system keys error", func(t *testing.T) {
req := &fasthttp.RequestHeader{}
req.DisableNormalizing()
req.Add("X-Attribute-Neofs-DupKey", "first-value")
req.Add("X-Attribute-Neofs-DupKey", "second-value")
req.Add("X-Attribute-System-DupKey", "first-value")
req.Add("X-Attribute-System-DupKey", "second-value")
_, err := filterHeaders(log, req)
require.Error(t, err)
})
@ -37,16 +34,16 @@ func TestFilter(t *testing.T) {
req := &fasthttp.RequestHeader{}
req.DisableNormalizing()
req.Set("X-Attribute-Neofs-Expiration-Epoch1", "101")
req.Set("X-Attribute-NEOFS-Expiration-Epoch2", "102")
req.Set("X-Attribute-neofs-Expiration-Epoch3", "103")
req.Set("X-Attribute-System-Expiration-Epoch1", "101")
req.Set("X-Attribute-SYSTEM-Expiration-Epoch2", "102")
req.Set("X-Attribute-system-Expiration-Epoch3", "103")
req.Set("X-Attribute-MyAttribute", "value")
expected := map[string]string{
"__NEOFS__EXPIRATION_EPOCH1": "101",
"__SYSTEM__EXPIRATION_EPOCH1": "101",
"MyAttribute": "value",
"__NEOFS__EXPIRATION_EPOCH3": "103",
"__NEOFS__EXPIRATION_EPOCH2": "102",
"__SYSTEM__EXPIRATION_EPOCH3": "103",
"__SYSTEM__EXPIRATION_EPOCH2": "102",
}
result, err := filterHeaders(log, req)
@ -54,157 +51,3 @@ func TestFilter(t *testing.T) {
require.Equal(t, expected, result)
}
func TestPrepareExpirationHeader(t *testing.T) {
tomorrow := time.Now().Add(24 * time.Hour)
tomorrowUnix := tomorrow.Unix()
tomorrowUnixNano := tomorrow.UnixNano()
tomorrowUnixMilli := tomorrowUnixNano / 1e6
epoch := "100"
duration := "24h"
timestampSec := strconv.FormatInt(tomorrowUnix, 10)
timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10)
timestampNano := strconv.FormatInt(tomorrowUnixNano, 10)
defaultDurations := &epochDurations{
currentEpoch: 10,
msPerBlock: 1000,
blockPerEpoch: 101,
}
msPerBlock := defaultDurations.blockPerEpoch * uint64(defaultDurations.msPerBlock)
epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock
if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 {
epochPerDay++
}
defaultExpEpoch := strconv.FormatUint(defaultDurations.currentEpoch+epochPerDay, 10)
for _, tc := range []struct {
name string
headers map[string]string
durations *epochDurations
err bool
expected map[string]string
}{
{
name: "valid epoch",
headers: map[string]string{object.SysAttributeExpEpoch: epoch},
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
},
{
name: "valid epoch, valid duration",
headers: map[string]string{
object.SysAttributeExpEpoch: epoch,
utils.ExpirationDurationAttr: duration,
},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
},
{
name: "valid epoch, valid rfc3339",
headers: map[string]string{
object.SysAttributeExpEpoch: epoch,
utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339),
},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
},
{
name: "valid epoch, valid timestamp sec",
headers: map[string]string{
object.SysAttributeExpEpoch: epoch,
utils.ExpirationTimestampAttr: timestampSec,
},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
},
{
name: "valid epoch, valid timestamp milli",
headers: map[string]string{
object.SysAttributeExpEpoch: epoch,
utils.ExpirationTimestampAttr: timestampMilli,
},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
},
{
name: "valid epoch, valid timestamp nano",
headers: map[string]string{
object.SysAttributeExpEpoch: epoch,
utils.ExpirationTimestampAttr: timestampNano,
},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
},
{
name: "valid timestamp sec",
headers: map[string]string{utils.ExpirationTimestampAttr: timestampSec},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch},
},
{
name: "valid duration",
headers: map[string]string{utils.ExpirationDurationAttr: duration},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch},
},
{
name: "valid rfc3339",
headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)},
durations: defaultDurations,
expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch},
},
{
name: "valid max uint 64",
headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)},
durations: &epochDurations{
currentEpoch: math.MaxUint64 - 1,
msPerBlock: defaultDurations.msPerBlock,
blockPerEpoch: defaultDurations.blockPerEpoch,
},
expected: map[string]string{object.SysAttributeExpEpoch: strconv.FormatUint(uint64(math.MaxUint64), 10)},
},
{
name: "invalid timestamp sec",
headers: map[string]string{utils.ExpirationTimestampAttr: "abc"},
err: true,
},
{
name: "invalid timestamp sec zero",
headers: map[string]string{utils.ExpirationTimestampAttr: "0"},
err: true,
},
{
name: "invalid duration",
headers: map[string]string{utils.ExpirationDurationAttr: "1d"},
err: true,
},
{
name: "invalid duration negative",
headers: map[string]string{utils.ExpirationDurationAttr: "-5h"},
err: true,
},
{
name: "invalid rfc3339",
headers: map[string]string{utils.ExpirationRFC3339Attr: "abc"},
err: true,
},
{
name: "invalid rfc3339 zero",
headers: map[string]string{utils.ExpirationRFC3339Attr: time.RFC3339},
err: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
err := prepareExpirationHeader(tc.headers, tc.durations, time.Now())
if tc.err {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, tc.expected, tc.headers)
}
})
}
}

View file

@ -3,7 +3,7 @@ package uploader
import (
"io"
"github.com/TrueCloudLab/frostfs-http-gw/uploader/multipart"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader/multipart"
"go.uber.org/zap"
)
@ -16,7 +16,8 @@ type MultipartFile interface {
func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartFile, error) {
// To have a custom buffer (3mb) the custom multipart reader is used.
// https://github.com/nspcc-dev/neofs-http-gw/issues/148
// Default reader uses 4KiB chunks, which slow down upload speed up to 400%
// https://github.com/golang/go/blob/91b9915d3f6f8cd2e9e9fda63f67772803adfa03/src/mime/multipart/multipart.go#L32
reader := multipart.NewReader(r, boundary)
for {

View file

@ -1,3 +1,5 @@
//go:build !integration
package uploader
import (

View file

@ -3,21 +3,20 @@ package uploader
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strconv"
"time"
"github.com/TrueCloudLab/frostfs-http-gw/resolver"
"github.com/TrueCloudLab/frostfs-http-gw/response"
"github.com/TrueCloudLab/frostfs-http-gw/tokens"
"github.com/TrueCloudLab/frostfs-http-gw/utils"
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
"github.com/TrueCloudLab/frostfs-sdk-go/object"
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/TrueCloudLab/frostfs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/valyala/fasthttp"
"go.uber.org/atomic"
"go.uber.org/zap"
@ -38,12 +37,6 @@ type Uploader struct {
containerResolver *resolver.ContainerResolver
}
type epochDurations struct {
currentEpoch uint64
msPerBlock int64
blockPerEpoch uint64
}
// Settings stores reloading parameters, so it has to provide atomic getters and setters.
type Settings struct {
defaultTimestamp atomic.Bool
@ -120,13 +113,6 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
response.Error(c, err.Error(), fasthttp.StatusBadRequest)
return
}
if needParseExpiration(filtered) {
epochDuration, err := getEpochDurations(c, u.pool)
if err != nil {
log.Error("could not get epoch durations from network info", zap.Error(err))
response.Error(c, "could not get epoch durations from network info: "+err.Error(), fasthttp.StatusBadRequest)
return
}
now := time.Now()
if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil {
@ -137,12 +123,11 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
}
}
if err = prepareExpirationHeader(filtered, epochDuration, now); err != nil {
log.Error("could not parse expiration header", zap.Error(err))
response.Error(c, "could not parse expiration header: "+err.Error(), fasthttp.StatusBadRequest)
if err = utils.PrepareExpirationHeader(c, u.pool, filtered, now); err != nil {
log.Error("could not prepare expiration header", zap.Error(err))
response.Error(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest)
return
}
}
attributes := make([]object.Attribute, 0, len(filtered))
// prepares attributes from filtered headers
@ -182,8 +167,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
}
if idObj, err = u.pool.PutObject(u.appCtx, prm); err != nil {
log.Error("could not store file in frostfs", zap.Error(err))
response.Error(c, "could not store file in frostfs: "+err.Error(), fasthttp.StatusBadRequest)
u.handlePutFrostFSErr(c, err)
return
}
@ -214,6 +198,14 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
c.Response.Header.SetContentType(jsonHeader)
}
func (u *Uploader) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) {
statusCode, msg, additionalFields := response.FormErrorResponse("could not store file in frostfs", err)
logFields := append([]zap.Field{zap.Error(err)}, additionalFields...)
u.log.Error("could not store file in frostfs", logFields...)
response.Error(r, msg, statusCode)
}
func (u *Uploader) fetchOwnerAndBearerToken(ctx context.Context) (*user.ID, *bearer.Token) {
if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil {
issuer := bearer.ResolveIssuer(*tkn)
@ -239,28 +231,3 @@ func (pr *putResponse) encode(w io.Writer) error {
enc.SetIndent("", "\t")
return enc.Encode(pr)
}
func getEpochDurations(ctx context.Context, p *pool.Pool) (*epochDurations, error) {
networkInfo, err := p.NetworkInfo(ctx)
if err != nil {
return nil, err
}
res := &epochDurations{
currentEpoch: networkInfo.CurrentEpoch(),
msPerBlock: networkInfo.MsPerBlock(),
blockPerEpoch: networkInfo.EpochDuration(),
}
if res.blockPerEpoch == 0 {
return nil, fmt.Errorf("EpochDuration is empty")
}
return res, nil
}
func needParseExpiration(headers map[string]string) bool {
_, ok1 := headers[utils.ExpirationDurationAttr]
_, ok2 := headers[utils.ExpirationRFC3339Attr]
_, ok3 := headers[utils.ExpirationTimestampAttr]
return ok1 || ok2 || ok3
}

View file

@ -1,10 +1,250 @@
package utils
import (
"bytes"
"context"
"errors"
"fmt"
"math"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
)
const (
UserAttributeHeaderPrefix = "X-Attribute-"
SystemAttributePrefix = "__NEOFS__"
ExpirationDurationAttr = SystemAttributePrefix + "EXPIRATION_DURATION"
ExpirationTimestampAttr = SystemAttributePrefix + "EXPIRATION_TIMESTAMP"
ExpirationRFC3339Attr = SystemAttributePrefix + "EXPIRATION_RFC3339"
)
const (
systemAttributePrefix = "__SYSTEM__"
// deprecated: use systemAttributePrefix
systemAttributePrefixNeoFS = "__NEOFS__"
)
type systemTransformer struct {
prefix string
backwardPrefix string
xAttrPrefixes [][]byte
}
var transformers = []systemTransformer{
{
prefix: systemAttributePrefix,
backwardPrefix: "System-",
xAttrPrefixes: [][]byte{[]byte("System-"), []byte("SYSTEM-"), []byte("system-")},
},
{
prefix: systemAttributePrefixNeoFS,
backwardPrefix: "Neofs-",
xAttrPrefixes: [][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")},
},
}
func (t systemTransformer) existsExpirationAttributes(headers map[string]string) bool {
_, ok0 := headers[t.expirationEpochAttr()]
_, ok1 := headers[t.expirationDurationAttr()]
_, ok2 := headers[t.expirationTimestampAttr()]
_, ok3 := headers[t.expirationRFC3339Attr()]
return ok0 || ok1 || ok2 || ok3
}
func (t systemTransformer) expirationEpochAttr() string {
return t.prefix + "EXPIRATION_EPOCH"
}
func (t systemTransformer) expirationDurationAttr() string {
return t.prefix + "EXPIRATION_DURATION"
}
func (t systemTransformer) expirationTimestampAttr() string {
return t.prefix + "EXPIRATION_TIMESTAMP"
}
func (t systemTransformer) expirationRFC3339Attr() string {
return t.prefix + "EXPIRATION_RFC3339"
}
func (t systemTransformer) systemTranslator(key, prefix []byte) []byte {
// replace the specified prefix with system prefix
key = bytes.Replace(key, prefix, []byte(t.prefix), 1)
// replace `-` with `_`
key = bytes.ReplaceAll(key, []byte("-"), []byte("_"))
// replace with uppercase
return bytes.ToUpper(key)
}
func (t systemTransformer) transformIfSystem(key []byte) ([]byte, bool) {
// checks that it's a system FrostFS header
for _, system := range t.xAttrPrefixes {
if bytes.HasPrefix(key, system) {
return t.systemTranslator(key, system), true
}
}
return key, false
}
// systemBackwardTranslator is used to convert headers looking like '__PREFIX__ATTR_NAME' to 'Prefix-Attr-Name'.
func (t systemTransformer) systemBackwardTranslator(key string) string {
// trim specified prefix '__PREFIX__'
key = strings.TrimPrefix(key, t.prefix)
var res strings.Builder
res.WriteString(t.backwardPrefix)
strs := strings.Split(key, "_")
for i, s := range strs {
s = title(strings.ToLower(s))
res.WriteString(s)
if i != len(strs)-1 {
res.WriteString("-")
}
}
return res.String()
}
func (t systemTransformer) backwardTransformIfSystem(key string) (string, bool) {
if strings.HasPrefix(key, t.prefix) {
return t.systemBackwardTranslator(key), true
}
return key, false
}
func TransformIfSystem(key []byte) []byte {
for _, transformer := range transformers {
key, transformed := transformer.transformIfSystem(key)
if transformed {
return key
}
}
return key
}
func BackwardTransformIfSystem(key string) string {
for _, transformer := range transformers {
key, transformed := transformer.backwardTransformIfSystem(key)
if transformed {
return key
}
}
return key
}
func title(str string) string {
if str == "" {
return ""
}
r, size := utf8.DecodeRuneInString(str)
r0 := unicode.ToTitle(r)
return string(r0) + str[size:]
}
func PrepareExpirationHeader(ctx context.Context, p *pool.Pool, headers map[string]string, now time.Time) error {
formatsNum := 0
index := -1
for i, transformer := range transformers {
if transformer.existsExpirationAttributes(headers) {
formatsNum++
index = i
}
}
switch formatsNum {
case 0:
return nil
case 1:
epochDuration, err := GetEpochDurations(ctx, p)
if err != nil {
return fmt.Errorf("couldn't get epoch durations from network info: %w", err)
}
return transformers[index].prepareExpirationHeader(headers, epochDuration, now)
default:
return errors.New("both deprecated and new system attributes formats are used, please use only one")
}
}
func (t systemTransformer) prepareExpirationHeader(headers map[string]string, epochDurations *EpochDurations, now time.Time) error {
expirationInEpoch := headers[t.expirationEpochAttr()]
if timeRFC3339, ok := headers[t.expirationRFC3339Attr()]; ok {
expTime, err := time.Parse(time.RFC3339, timeRFC3339)
if err != nil {
return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, t.expirationRFC3339Attr())
}
if expTime.Before(now) {
return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, t.expirationRFC3339Attr())
}
t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
delete(headers, t.expirationRFC3339Attr())
}
if timestamp, ok := headers[t.expirationTimestampAttr()]; ok {
value, err := strconv.ParseInt(timestamp, 10, 64)
if err != nil {
return fmt.Errorf("couldn't parse value %s of header %s", timestamp, t.expirationTimestampAttr())
}
expTime := time.Unix(value, 0)
if expTime.Before(now) {
return fmt.Errorf("value %s of header %s must be in the future", timestamp, t.expirationTimestampAttr())
}
t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
delete(headers, t.expirationTimestampAttr())
}
if duration, ok := headers[t.expirationDurationAttr()]; ok {
expDuration, err := time.ParseDuration(duration)
if err != nil {
return fmt.Errorf("couldn't parse value %s of header %s", duration, t.expirationDurationAttr())
}
if expDuration <= 0 {
return fmt.Errorf("value %s of header %s must be positive", expDuration, t.expirationDurationAttr())
}
t.updateExpirationHeader(headers, epochDurations, expDuration)
delete(headers, t.expirationDurationAttr())
}
if expirationInEpoch != "" {
expEpoch, err := strconv.ParseUint(expirationInEpoch, 10, 64)
if err != nil {
return fmt.Errorf("parse expiration epoch '%s': %w", expirationInEpoch, err)
}
if expEpoch < epochDurations.CurrentEpoch {
return fmt.Errorf("expiration epoch '%d' must be greater than current epoch '%d'", expEpoch, epochDurations.CurrentEpoch)
}
headers[t.expirationEpochAttr()] = expirationInEpoch
}
return nil
}
func (t systemTransformer) updateExpirationHeader(headers map[string]string, durations *EpochDurations, expDuration time.Duration) {
epochDuration := uint64(durations.MsPerBlock) * durations.BlockPerEpoch
currentEpoch := durations.CurrentEpoch
numEpoch := uint64(expDuration.Milliseconds()) / epochDuration
if uint64(expDuration.Milliseconds())%epochDuration != 0 {
numEpoch++
}
expirationEpoch := uint64(math.MaxUint64)
if numEpoch < math.MaxUint64-currentEpoch {
expirationEpoch = currentEpoch + numEpoch
}
headers[t.expirationEpochAttr()] = strconv.FormatUint(expirationEpoch, 10)
}

189
utils/attributes_test.go Normal file
View file

@ -0,0 +1,189 @@
//go:build !integration
package utils
import (
"math"
"strconv"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestPrepareExpirationHeader(t *testing.T) {
tomorrow := time.Now().Add(24 * time.Hour)
tomorrowUnix := tomorrow.Unix()
tomorrowUnixNano := tomorrow.UnixNano()
tomorrowUnixMilli := tomorrowUnixNano / 1e6
epoch := "100"
duration := "24h"
timestampSec := strconv.FormatInt(tomorrowUnix, 10)
timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10)
timestampNano := strconv.FormatInt(tomorrowUnixNano, 10)
defaultDurations := &EpochDurations{
CurrentEpoch: 10,
MsPerBlock: 1000,
BlockPerEpoch: 101,
}
msPerBlock := defaultDurations.BlockPerEpoch * uint64(defaultDurations.MsPerBlock)
epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock
if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 {
epochPerDay++
}
defaultExpEpoch := strconv.FormatUint(defaultDurations.CurrentEpoch+epochPerDay, 10)
for _, transformer := range transformers {
for _, tc := range []struct {
name string
headers map[string]string
durations *EpochDurations
err bool
expected map[string]string
}{
{
name: "valid epoch",
headers: map[string]string{transformer.expirationEpochAttr(): epoch},
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
durations: defaultDurations,
},
{
name: "valid epoch, valid duration",
headers: map[string]string{
transformer.expirationEpochAttr(): epoch,
transformer.expirationDurationAttr(): duration,
},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
},
{
name: "valid epoch, valid rfc3339",
headers: map[string]string{
transformer.expirationEpochAttr(): epoch,
transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339),
},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
},
{
name: "valid epoch, valid timestamp sec",
headers: map[string]string{
transformer.expirationEpochAttr(): epoch,
transformer.expirationTimestampAttr(): timestampSec,
},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
},
{
name: "valid epoch, valid timestamp milli",
headers: map[string]string{
transformer.expirationEpochAttr(): epoch,
transformer.expirationTimestampAttr(): timestampMilli,
},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
},
{
name: "valid epoch, valid timestamp nano",
headers: map[string]string{
transformer.expirationEpochAttr(): epoch,
transformer.expirationTimestampAttr(): timestampNano,
},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
},
{
name: "valid timestamp sec",
headers: map[string]string{transformer.expirationTimestampAttr(): timestampSec},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
},
{
name: "valid duration",
headers: map[string]string{transformer.expirationDurationAttr(): duration},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
},
{
name: "valid rfc3339",
headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)},
durations: defaultDurations,
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
},
{
name: "valid max uint 64",
headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)},
durations: &EpochDurations{
CurrentEpoch: math.MaxUint64 - 1,
MsPerBlock: defaultDurations.MsPerBlock,
BlockPerEpoch: defaultDurations.BlockPerEpoch,
},
expected: map[string]string{transformer.expirationEpochAttr(): strconv.FormatUint(uint64(math.MaxUint64), 10)},
},
{
name: "invalid timestamp sec",
headers: map[string]string{transformer.expirationTimestampAttr(): "abc"},
err: true,
},
{
name: "invalid timestamp sec zero",
headers: map[string]string{transformer.expirationTimestampAttr(): "0"},
err: true,
},
{
name: "invalid duration",
headers: map[string]string{transformer.expirationDurationAttr(): "1d"},
err: true,
},
{
name: "invalid duration negative",
headers: map[string]string{transformer.expirationDurationAttr(): "-5h"},
err: true,
},
{
name: "invalid rfc3339",
headers: map[string]string{transformer.expirationRFC3339Attr(): "abc"},
err: true,
},
{
name: "invalid rfc3339 zero",
headers: map[string]string{transformer.expirationRFC3339Attr(): time.RFC3339},
err: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
err := transformer.prepareExpirationHeader(tc.headers, tc.durations, time.Now())
if tc.err {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, tc.expected, tc.headers)
}
})
}
}
}
func TestSystemBackwardTranslator(t *testing.T) {
input := []string{
"__SYSTEM__EXPIRATION_EPOCH",
"__SYSTEM__RANDOM_ATTR",
"__NEOFS__EXPIRATION_EPOCH",
"__NEOFS__RANDOM_ATTR",
}
expected := []string{
"System-Expiration-Epoch",
"System-Random-Attr",
"Neofs-Expiration-Epoch",
"Neofs-Random-Attr",
}
for i, str := range input {
res := BackwardTransformIfSystem(str)
require.Equal(t, expected[i], res)
}
}

View file

@ -1,9 +1,9 @@
package utils
import (
"github.com/TrueCloudLab/frostfs-http-gw/resolver"
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/TrueCloudLab/frostfs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"go.uber.org/zap"
)

View file

@ -2,9 +2,11 @@ package utils
import (
"context"
"fmt"
"github.com/TrueCloudLab/frostfs-http-gw/resolver"
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
)
// GetContainerID decode container id, if it's not a valid container id
@ -17,3 +19,27 @@ func GetContainerID(ctx context.Context, containerID string, resolver *resolver.
}
return cnrID, err
}
type EpochDurations struct {
CurrentEpoch uint64
MsPerBlock int64
BlockPerEpoch uint64
}
func GetEpochDurations(ctx context.Context, p *pool.Pool) (*EpochDurations, error) {
networkInfo, err := p.NetworkInfo(ctx)
if err != nil {
return nil, err
}
res := &EpochDurations{
CurrentEpoch: networkInfo.CurrentEpoch(),
MsPerBlock: networkInfo.MsPerBlock(),
BlockPerEpoch: networkInfo.EpochDuration(),
}
if res.BlockPerEpoch == 0 {
return nil, fmt.Errorf("EpochDuration is empty")
}
return res, nil
}