Compare commits
28 commits
Author | SHA1 | Date | |
---|---|---|---|
52be32e4ba | |||
4791428b6e | |||
2888151e40 | |||
048c626750 | |||
168b67dc31 | |||
959213520e | |||
162738e771 | |||
7c16ffa250 | |||
6f35d7198d | |||
81f7168a16 | |||
a8ec09e76a | |||
1f66149316 | |||
e2059a8926 | |||
53ee124b19 | |||
8f6be59e23 | |||
e02ee50d7b | |||
93ec4c444d | |||
6be8d47d92 | |||
ed983f8ad0 | |||
5f01abf300 | |||
38aa6db041 | |||
5a98df9d2d | |||
361acacf07 | |||
6909ef5382 | |||
148b1aa7f5 | |||
7df26d9181 | |||
913644d64a | |||
4c30ff6638 |
47 changed files with 1252 additions and 1254 deletions
12
.github/ISSUE_TEMPLATE/bug_report.md
vendored
12
.github/ISSUE_TEMPLATE/bug_report.md
vendored
|
@ -2,7 +2,7 @@
|
||||||
name: Bug report
|
name: Bug report
|
||||||
about: Create a report to help us improve
|
about: Create a report to help us improve
|
||||||
title: ''
|
title: ''
|
||||||
labels: community, triage
|
labels: community, triage, bug
|
||||||
assignees: ''
|
assignees: ''
|
||||||
|
|
||||||
---
|
---
|
||||||
|
@ -18,17 +18,17 @@ assignees: ''
|
||||||
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
|
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
|
||||||
|
|
||||||
## Possible Solution
|
## Possible Solution
|
||||||
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
|
<!-- Not obligatory
|
||||||
<!--- or ideas how to implement the addition or change -->
|
If no reason/fix/additions for the bug can be suggested,
|
||||||
|
uncomment the following phrase:
|
||||||
|
|
||||||
|
<-- No fix can be suggested by a QA engineer. Further solutions shall be up to developers. -->
|
||||||
|
|
||||||
## Steps to Reproduce (for bugs)
|
## Steps to Reproduce (for bugs)
|
||||||
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
||||||
<!--- reproduce this bug. -->
|
<!--- reproduce this bug. -->
|
||||||
|
|
||||||
1.
|
1.
|
||||||
2.
|
|
||||||
3.
|
|
||||||
4.
|
|
||||||
|
|
||||||
## Context
|
## Context
|
||||||
<!--- How has this issue affected you? What are you trying to accomplish? -->
|
<!--- How has this issue affected you? What are you trying to accomplish? -->
|
||||||
|
|
16
.github/ISSUE_TEMPLATE/feature_request.md
vendored
16
.github/ISSUE_TEMPLATE/feature_request.md
vendored
|
@ -7,14 +7,14 @@ assignees: ''
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
**Is your feature request related to a problem? Please describe.**
|
## Is your feature request related to a problem? Please describe.
|
||||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
|
||||||
|
|
||||||
**Describe the solution you'd like**
|
## Describe the solution you'd like
|
||||||
A clear and concise description of what you want to happen.
|
<!--- A clear and concise description of what you want to happen. -->
|
||||||
|
|
||||||
**Describe alternatives you've considered**
|
## Describe alternatives you've considered
|
||||||
A clear and concise description of any alternative solutions or features you've considered.
|
<!--- A clear and concise description of any alternative solutions or features you've considered. -->
|
||||||
|
|
||||||
**Additional context**
|
## Additional context
|
||||||
Add any other context or screenshots about the feature request here.
|
<!--- Add any other context or screenshots about the feature request here. -->
|
||||||
|
|
70
.github/logo.svg
vendored
Normal file
70
.github/logo.svg
vendored
Normal file
|
@ -0,0 +1,70 @@
|
||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<!-- Generator: Adobe Illustrator 25.0.1, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||||
|
<svg version="1.1" id="Слой_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||||
|
viewBox="0 0 184.2 51.8" style="enable-background:new 0 0 184.2 51.8;" xml:space="preserve">
|
||||||
|
<style type="text/css">
|
||||||
|
.st0{display:none;}
|
||||||
|
.st1{display:inline;}
|
||||||
|
.st2{fill:#01E397;}
|
||||||
|
.st3{display:inline;fill:#010032;}
|
||||||
|
.st4{display:inline;fill:#00E599;}
|
||||||
|
.st5{display:inline;fill:#00AF92;}
|
||||||
|
.st6{fill:#00C3E5;}
|
||||||
|
</style>
|
||||||
|
<g id="Layer_2">
|
||||||
|
<g id="Layer_1-2" class="st0">
|
||||||
|
<g class="st1">
|
||||||
|
<path class="st2" d="M146.6,18.3v7.2h10.9V29h-10.9v10.7h-4V14.8h18v3.5H146.6z"/>
|
||||||
|
<path class="st2" d="M180,15.7c1.7,0.9,3,2.2,4,3.8l-3,2.7c-0.6-1.3-1.5-2.4-2.6-3.3c-1.3-0.7-2.8-1-4.3-1
|
||||||
|
c-1.4-0.1-2.8,0.3-4,1.1c-0.9,0.5-1.5,1.5-1.4,2.6c0,1,0.5,1.9,1.4,2.4c1.5,0.8,3.2,1.3,4.9,1.5c1.9,0.3,3.7,0.8,5.4,1.6
|
||||||
|
c1.2,0.5,2.2,1.3,2.9,2.3c0.6,1,1,2.2,0.9,3.4c0,1.4-0.5,2.7-1.3,3.8c-0.9,1.2-2.1,2.1-3.5,2.6c-1.7,0.6-3.4,0.9-5.2,0.8
|
||||||
|
c-5,0-8.6-1.6-10.7-5l2.9-2.8c0.7,1.4,1.8,2.5,3.1,3.3c1.5,0.7,3.1,1.1,4.7,1c1.5,0.1,2.9-0.2,4.2-0.9c0.9-0.5,1.5-1.5,1.5-2.6
|
||||||
|
c0-0.9-0.5-1.8-1.3-2.2c-1.5-0.7-3.1-1.2-4.8-1.5c-1.9-0.3-3.7-0.8-5.5-1.5c-1.2-0.5-2.2-1.4-3-2.4c-0.6-1-1-2.2-0.9-3.4
|
||||||
|
c0-1.4,0.4-2.7,1.2-3.8c0.8-1.2,2-2.2,3.3-2.8c1.6-0.7,3.4-1.1,5.2-1C176.1,14.3,178.2,14.8,180,15.7z"/>
|
||||||
|
</g>
|
||||||
|
<path class="st3" d="M73.3,16.3c1.9,1.9,2.9,4.5,2.7,7.1v15.9h-4V24.8c0-2.6-0.5-4.5-1.6-5.7c-1.2-1.2-2.8-1.8-4.5-1.7
|
||||||
|
c-1.3,0-2.5,0.3-3.7,0.8c-1.2,0.7-2.2,1.7-2.9,2.9c-0.8,1.5-1.1,3.2-1.1,4.9v13.3h-4V15.1l3.6,1.5v1.7c0.8-1.5,2.1-2.6,3.6-3.3
|
||||||
|
c1.5-0.8,3.2-1.2,4.9-1.1C68.9,13.8,71.3,14.7,73.3,16.3z"/>
|
||||||
|
<path class="st3" d="M104.4,28.3H85.6c0.1,2.2,1,4.3,2.5,5.9c1.5,1.4,3.5,2.2,5.6,2.1c1.6,0.1,3.2-0.2,4.6-0.9
|
||||||
|
c1.1-0.6,2-1.6,2.5-2.8l3.3,1.8c-0.9,1.7-2.3,3.1-4,4c-2,1-4.2,1.5-6.4,1.4c-3.7,0-6.7-1.1-8.8-3.4s-3.2-5.5-3.2-9.6s1-7.2,3-9.5
|
||||||
|
s5-3.4,8.7-3.4c2.1-0.1,4.2,0.5,6.1,1.5c1.6,1,3,2.5,3.8,4.2c0.9,1.8,1.3,3.9,1.3,5.9C104.6,26.4,104.6,27.4,104.4,28.3z
|
||||||
|
M88.1,19.3c-1.4,1.5-2.2,3.4-2.4,5.5h15.1c-0.2-2-1-3.9-2.3-5.5c-1.4-1.3-3.2-2-5.1-1.9C91.5,17.3,89.6,18,88.1,19.3z"/>
|
||||||
|
<path class="st3" d="M131,17.3c2.2,2.3,3.2,5.5,3.2,9.5s-1,7.3-3.2,9.6s-5.1,3.4-8.8,3.4s-6.7-1.1-8.9-3.4s-3.2-5.5-3.2-9.6
|
||||||
|
s1.1-7.2,3.2-9.5s5.1-3.4,8.9-3.4S128.9,15,131,17.3z M116.2,19.9c-1.5,2-2.2,4.4-2.1,6.9c-0.2,2.5,0.6,5,2.1,7
|
||||||
|
c1.5,1.7,3.7,2.7,6,2.6c2.3,0.1,4.4-0.9,5.9-2.6c1.5-2,2.3-4.5,2.1-7c0.1-2.5-0.6-4.9-2.1-6.9c-1.5-1.7-3.6-2.7-5.9-2.6
|
||||||
|
C119.9,17.2,117.7,18.2,116.2,19.9z"/>
|
||||||
|
<polygon class="st4" points="0,9.1 0,43.7 22.5,51.8 22.5,16.9 46.8,7.9 24.8,0 "/>
|
||||||
|
<polygon class="st5" points="24.3,17.9 24.3,36.8 46.8,44.9 46.8,9.6 "/>
|
||||||
|
</g>
|
||||||
|
<g>
|
||||||
|
<g>
|
||||||
|
<path class="st6" d="M41.6,17.5H28.2v6.9h10.4v3.3H28.2v10.2h-3.9V14.2h17.2V17.5z"/>
|
||||||
|
<path class="st6" d="M45.8,37.9v-18h3.3l0.4,3.2c0.5-1.2,1.2-2.1,2.1-2.7c0.9-0.6,2.1-0.9,3.5-0.9c0.4,0,0.7,0,1.1,0.1
|
||||||
|
c0.4,0.1,0.7,0.2,0.9,0.3l-0.5,3.4c-0.3-0.1-0.6-0.2-0.9-0.2C55.4,23,54.9,23,54.4,23c-0.7,0-1.5,0.2-2.2,0.6
|
||||||
|
c-0.7,0.4-1.3,1-1.8,1.8s-0.7,1.8-0.7,3v9.5H45.8z"/>
|
||||||
|
<path class="st6" d="M68.6,19.6c1.8,0,3.3,0.4,4.6,1.1c1.3,0.7,2.4,1.8,3.1,3.2s1.1,3.1,1.1,5c0,1.9-0.4,3.6-1.1,5
|
||||||
|
c-0.8,1.4-1.8,2.5-3.1,3.2c-1.3,0.7-2.9,1.1-4.6,1.1s-3.3-0.4-4.6-1.1c-1.3-0.7-2.4-1.8-3.2-3.2c-0.8-1.4-1.2-3.1-1.2-5
|
||||||
|
c0-1.9,0.4-3.6,1.2-5s1.8-2.5,3.2-3.2C65.3,19.9,66.8,19.6,68.6,19.6z M68.6,22.6c-1.1,0-2,0.2-2.8,0.7c-0.8,0.5-1.3,1.2-1.7,2.1
|
||||||
|
s-0.6,2.1-0.6,3.5c0,1.3,0.2,2.5,0.6,3.4s1,1.7,1.7,2.2s1.7,0.7,2.8,0.7c1.1,0,2-0.2,2.7-0.7c0.7-0.5,1.3-1.2,1.7-2.2
|
||||||
|
s0.6-2.1,0.6-3.4c0-1.4-0.2-2.5-0.6-3.5s-1-1.6-1.7-2.1C70.6,22.8,69.6,22.6,68.6,22.6z"/>
|
||||||
|
<path class="st6" d="M89.2,38.3c-1.8,0-3.4-0.3-4.9-1c-1.5-0.7-2.7-1.7-3.5-3l2.7-2.3c0.5,1,1.3,1.8,2.3,2.4
|
||||||
|
c1,0.6,2.2,0.9,3.6,0.9c1.1,0,2-0.2,2.6-0.6c0.6-0.4,1-0.9,1-1.6c0-0.5-0.2-0.9-0.5-1.2s-0.9-0.6-1.7-0.8l-3.8-0.8
|
||||||
|
c-1.9-0.4-3.3-1-4.1-1.9c-0.8-0.9-1.2-1.9-1.2-3.3c0-1,0.3-1.9,0.9-2.7c0.6-0.8,1.4-1.5,2.5-2s2.5-0.8,4-0.8c1.8,0,3.3,0.3,4.6,1
|
||||||
|
c1.3,0.6,2.2,1.5,2.9,2.7l-2.7,2.2c-0.5-1-1.1-1.7-2-2.1c-0.9-0.5-1.8-0.7-2.8-0.7c-0.8,0-1.4,0.1-2,0.3c-0.6,0.2-1,0.5-1.3,0.8
|
||||||
|
c-0.3,0.3-0.4,0.7-0.4,1.2c0,0.5,0.2,0.9,0.5,1.3s1,0.6,1.9,0.8l4.1,0.9c1.7,0.3,2.9,0.9,3.7,1.7c0.7,0.8,1.1,1.8,1.1,2.9
|
||||||
|
c0,1.2-0.3,2.2-0.9,3c-0.6,0.9-1.5,1.6-2.6,2C92.1,38.1,90.7,38.3,89.2,38.3z"/>
|
||||||
|
<path class="st6" d="M112.8,19.9v3H99.3v-3H112.8z M106.6,14.6v17.9c0,0.9,0.2,1.5,0.7,1.9c0.5,0.4,1.1,0.6,1.9,0.6
|
||||||
|
c0.6,0,1.2-0.1,1.7-0.3c0.5-0.2,0.9-0.5,1.3-0.8l0.9,2.8c-0.6,0.5-1.2,0.9-2,1.1c-0.8,0.3-1.7,0.4-2.7,0.4c-1,0-2-0.2-2.8-0.5
|
||||||
|
s-1.5-0.9-2-1.6c-0.5-0.8-0.7-1.7-0.8-3V15.7L106.6,14.6z"/>
|
||||||
|
<path d="M137.9,17.5h-13.3v6.9h10.4v3.3h-10.4v10.2h-3.9V14.2h17.2V17.5z"/>
|
||||||
|
<path d="M150.9,13.8c2.1,0,4,0.4,5.5,1.2c1.6,0.8,2.9,2,4,3.5l-2.6,2.5c-0.9-1.4-1.9-2.4-3.1-3c-1.1-0.6-2.5-0.9-4-0.9
|
||||||
|
c-1.2,0-2.1,0.2-2.8,0.5c-0.7,0.3-1.3,0.7-1.6,1.2c-0.3,0.5-0.5,1.1-0.5,1.7c0,0.7,0.3,1.4,0.8,1.9c0.5,0.6,1.5,1,2.9,1.3
|
||||||
|
l4.8,1.1c2.3,0.5,3.9,1.3,4.9,2.3c1,1,1.4,2.3,1.4,3.9c0,1.5-0.4,2.7-1.2,3.8c-0.8,1.1-1.9,1.9-3.3,2.5s-3.1,0.9-5,0.9
|
||||||
|
c-1.7,0-3.2-0.2-4.5-0.6c-1.3-0.4-2.5-1-3.5-1.8c-1-0.7-1.8-1.6-2.5-2.6l2.7-2.7c0.5,0.8,1.1,1.6,1.9,2.2
|
||||||
|
c0.8,0.7,1.7,1.2,2.7,1.5c1,0.4,2.2,0.5,3.4,0.5c1.1,0,2.1-0.1,2.9-0.4c0.8-0.3,1.4-0.7,1.8-1.2c0.4-0.5,0.6-1.1,0.6-1.9
|
||||||
|
c0-0.7-0.2-1.3-0.7-1.8c-0.5-0.5-1.3-0.9-2.6-1.2l-5.2-1.2c-1.4-0.3-2.6-0.8-3.6-1.3c-0.9-0.6-1.6-1.3-2.1-2.1s-0.7-1.8-0.7-2.8
|
||||||
|
c0-1.3,0.4-2.6,1.1-3.7c0.7-1.1,1.8-2,3.2-2.6C147.3,14.1,148.9,13.8,150.9,13.8z"/>
|
||||||
|
</g>
|
||||||
|
</g>
|
||||||
|
</g>
|
||||||
|
</svg>
|
After Width: | Height: | Size: 5.5 KiB |
2
.github/workflows/tests.yml
vendored
2
.github/workflows/tests.yml
vendored
|
@ -61,7 +61,7 @@ jobs:
|
||||||
runs-on: ubuntu-20.04
|
runs-on: ubuntu-20.04
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go_versions: [ '1.17', '1.18', '1.19' ]
|
go_versions: [ '1.18', '1.19' ]
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v2
|
||||||
|
|
11
.gitlint
Normal file
11
.gitlint
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
[general]
|
||||||
|
fail-without-commits=True
|
||||||
|
regex-style-search=True
|
||||||
|
contrib=CC1
|
||||||
|
|
||||||
|
[title-match-regex]
|
||||||
|
regex=^\[\#[0-9Xx]+\]\s
|
||||||
|
|
||||||
|
[ignore-by-title]
|
||||||
|
regex=^Release(.*)
|
||||||
|
ignore=title-match-regex
|
45
.pre-commit-config.yaml
Normal file
45
.pre-commit-config.yaml
Normal file
|
@ -0,0 +1,45 @@
|
||||||
|
ci:
|
||||||
|
autofix_prs: false
|
||||||
|
|
||||||
|
repos:
|
||||||
|
- repo: https://github.com/jorisroovers/gitlint
|
||||||
|
rev: v0.19.1
|
||||||
|
hooks:
|
||||||
|
- id: gitlint
|
||||||
|
stages: [commit-msg]
|
||||||
|
- id: gitlint-ci
|
||||||
|
|
||||||
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
|
rev: v4.4.0
|
||||||
|
hooks:
|
||||||
|
- id: check-added-large-files
|
||||||
|
- id: check-case-conflict
|
||||||
|
- id: check-executables-have-shebangs
|
||||||
|
- id: check-shebang-scripts-are-executable
|
||||||
|
- id: check-merge-conflict
|
||||||
|
- id: check-json
|
||||||
|
- id: check-xml
|
||||||
|
- id: check-yaml
|
||||||
|
- id: trailing-whitespace
|
||||||
|
args: [--markdown-linebreak-ext=md]
|
||||||
|
- id: end-of-file-fixer
|
||||||
|
exclude: ".key$"
|
||||||
|
|
||||||
|
- repo: https://github.com/shellcheck-py/shellcheck-py
|
||||||
|
rev: v0.9.0.2
|
||||||
|
hooks:
|
||||||
|
- id: shellcheck
|
||||||
|
|
||||||
|
- repo: https://github.com/golangci/golangci-lint
|
||||||
|
rev: v1.51.2
|
||||||
|
hooks:
|
||||||
|
- id: golangci-lint
|
||||||
|
|
||||||
|
- repo: local
|
||||||
|
hooks:
|
||||||
|
- id: go-unit-tests
|
||||||
|
name: go unit tests
|
||||||
|
entry: make test
|
||||||
|
pass_filenames: false
|
||||||
|
types: [go]
|
||||||
|
language: system
|
286
CHANGELOG.md
286
CHANGELOG.md
|
@ -4,276 +4,42 @@ This document outlines major changes between releases.
|
||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
## [0.26.0] - 2022-12-28
|
## [0.27.0] - Karpinsky - 2023-07-12
|
||||||
|
|
||||||
|
This is a first FrostFS HTTP Gateway release named after
|
||||||
|
[Karpinsky glacier](https://en.wikipedia.org/wiki/Karpinsky_Glacier).
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
- ENV config example (#236)
|
- Require only one healthy storage server to start (#7)
|
||||||
|
- Enable gate metrics (#38)
|
||||||
|
- `Too many pings` error (#61)
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
- Support the `Date` header on upload (#214)
|
- Multiple configs support (#12)
|
||||||
- Available routes specification (#216)
|
|
||||||
- Mention caching strategy in docs (#215)
|
|
||||||
- Add error response on attribute duplicates (#221)
|
|
||||||
- Multiple server listeners (#228)
|
|
||||||
|
|
||||||
### Removed
|
|
||||||
- Deprecated linters (#239)
|
|
||||||
|
|
||||||
### Updating from v0.25.1
|
|
||||||
Make sure your configuration is valid:
|
|
||||||
|
|
||||||
If you configure application using environment variables change:
|
|
||||||
* `HTTP_GW_LISTEN_ADDRESS` -> `HTTP_GW_SERVER_0_ADDRESS`
|
|
||||||
* `HTTP_GW_TLS_CERT_FILE` -> `HTTP_GW_SERVER_0_TLS_CERT_FILE` (and set `HTTP_GW_SERVER_0_TLS_ENABLED=true`)
|
|
||||||
* `HTTP_GW_TLS_KEY_FILE` -> `HTTP_GW_SERVER_0_TLS_KEY_FILE` (and set `HTTP_GW_SERVER_0_TLS_ENABLED=true`)
|
|
||||||
|
|
||||||
If you configure application using `.yaml` file change:
|
|
||||||
* `listen_address` -> `server.0.address`
|
|
||||||
* `tls.cert_file` -> `server.0.tls.cert_file` (and set `server.0.tls.enabled: true`)
|
|
||||||
* `tls.key_file` -> `server.0.tls.key_file` (and set `server.0.tls.enabled: true`)
|
|
||||||
|
|
||||||
## [0.25.1] - 2022-11-30
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Download zip archive when `FilePath` is invalid (#222)
|
|
||||||
- Only one peer must be healthy to init pool (#233)
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- Debian packaging (#223)
|
|
||||||
- Timeout for individual operations in streaming RPC (#234)
|
|
||||||
|
|
||||||
## [0.25.0] - 2022-10-31
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- Config reloading on SIGHUP (#200, #208)
|
|
||||||
- Stop pool dial on SIGINT (#212)
|
|
||||||
- Makefile help (#213)
|
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
- Update NeoFS error handling (#206)
|
- Repository rebranding (#1)
|
||||||
- GitHub actions updates (#205, #209)
|
- Update neo-go to v0.101.0 (#8)
|
||||||
- Unified system attribute format for GET and HEAD (#213)
|
- Update viper to v1.15.0 (#8)
|
||||||
|
- Update go version to 1.18 (#9)
|
||||||
|
- Errors have become more detailed (#18)
|
||||||
|
- Update system attribute names (#22)
|
||||||
|
- Separate integration tests with build tags (#24)
|
||||||
|
- Changed values for `frostfs_http_gw_state_health` metric (#32)
|
||||||
|
|
||||||
## [0.24.0] - 2022-09-14
|
### Updating from v0.26.0
|
||||||
|
|
||||||
### Fixed
|
To set system attributes use updated headers
|
||||||
- Fix expiration epoch calculation (#198)
|
(you can use old ones for now, but their support will be dropped in the future releases):
|
||||||
- Fix panic on go1.19 (#188)
|
|
||||||
|
|
||||||
### Added
|
* `X-Attribute-Neofs-*` -> `X-Attribute-System-*`
|
||||||
- Exposure of pool metrics (#179, #194)
|
* `X-Attribute-NEOFS-*` -> `X-Attribute-SYSTEM-*`
|
||||||
|
* `X-Attribute-neofs-*` -> `X-Attribute-system-*`
|
||||||
### Changed
|
|
||||||
- Help doesn't print empty parameters (#186)
|
|
||||||
- Update version calculation (#190, #199)
|
|
||||||
- Update neofs-sdk-go (#196)
|
|
||||||
- Update go version in CI and docker (#197, #202)
|
|
||||||
|
|
||||||
## [0.23.0] - 2022-08-02
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- New param to configure pool error threshold (#184)
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
- Pprof and prometheus metrics configuration (#171)
|
|
||||||
- Drop GO111MODULES from builds (#182)
|
|
||||||
|
|
||||||
### Updating from v0.22.0
|
|
||||||
1. To enable pprof use `pprof.enabled` instead of `pprof` in config.
|
|
||||||
To enable prometheus metrics use `prometheus.enabled` instead of `metrics` in config.
|
|
||||||
If you are using the command line flags you can skip this step.
|
|
||||||
|
|
||||||
## [0.22.0] - 2022-07-25
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- Default params documentation (#172)
|
|
||||||
- Health metric (#175)
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
- Version output (#169)
|
|
||||||
- Updated SDK Version (#178)
|
|
||||||
|
|
||||||
## [0.21.0] - 2022-06-20
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Downloading ZIP archive using streaming (#163)
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- New make target to build app in docker (#159)
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
- Increased buffer size for file uploading (#148)
|
|
||||||
- Updated linter version to v1.46.2 (#161)
|
|
||||||
- Updated CodeQL version to v2 (#158)
|
|
||||||
|
|
||||||
|
|
||||||
## [0.20.0] - 2022-04-29
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Get rid of data race on server shutdown (#145)
|
|
||||||
- Improved English in docs and comments (#153)
|
|
||||||
- Use `FilePath` to download zip (#150)
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- Support container name NNS resolving (#142)
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
- Updated docs (#133, #140)
|
|
||||||
- Increased default read/write timeouts (#154)
|
|
||||||
- Updated SDK (#137, #139)
|
|
||||||
- Updated go version to 1.17 (#143)
|
|
||||||
- Improved error messages (#144)
|
|
||||||
|
|
||||||
## [0.19.0] - 2022-03-16
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Uploading object with zero payload (#122)
|
|
||||||
- Different headers format in GET and HEAD (#125)
|
|
||||||
- Fixed project name in docs (#120)
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- Support object attributes with spaces (#123)
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
- Updated fasthttp to v1.34.0 (#129)
|
|
||||||
- Updated NeoFS SDK to v1.0.0-rc.3 (#126, #132)
|
|
||||||
- Refactored content type detecting (#128)
|
|
||||||
|
|
||||||
|
|
||||||
## [0.18.0] - 2021-12-10
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- System headers format (#111)
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- Different formats to set object's expiration: in epoch, duration, timestamp,
|
|
||||||
RFC3339 (#108)
|
|
||||||
- Support of nodes priority (#115)
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
- Updated testcontainers dependency (#100)
|
|
||||||
|
|
||||||
## [0.17.0] - 2021-11-15
|
|
||||||
|
|
||||||
Support of bulk file download with zip streams and various bug fixes.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Allow canonical `X-Attribute-Neofs-*` headers (#87)
|
|
||||||
- Responses with error message now end with `\n` character (#105)
|
|
||||||
- Application does not require all neofs endpoints to be healthy at start now
|
|
||||||
(#103)
|
|
||||||
- Application now tracks session token errors and recreates tokens in runtime
|
|
||||||
(#95)
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- Integration tests with [all-in-one](https://github.com/nspcc-dev/neofs-aio/)
|
|
||||||
test containers (#85, #94)
|
|
||||||
- Bulk download support with zip streams (#92, #96)
|
|
||||||
|
|
||||||
## 0.16.1 (28 Jul 2021)
|
|
||||||
|
|
||||||
New features:
|
|
||||||
* logging requests (#77)
|
|
||||||
* HEAD methods for download routes (#76)
|
|
||||||
|
|
||||||
Improvements:
|
|
||||||
* updated sdk-go dependency (#82)
|
|
||||||
|
|
||||||
Bugs fixed:
|
|
||||||
* wrong NotFound status was used (#30)
|
|
||||||
|
|
||||||
## 0.16.0 (29 Jun 2021)
|
|
||||||
|
|
||||||
We update HTTP gateway with NEP-6 wallets support, YAML configuration files
|
|
||||||
and small fixes.
|
|
||||||
|
|
||||||
New features:
|
|
||||||
* YAML configuration file (#71)
|
|
||||||
|
|
||||||
Behavior changes:
|
|
||||||
* gateway key needs to be stored in a proper NEP-6 wallet now, `-k` option is
|
|
||||||
no longer available, see `-w` and `-a` (#68)
|
|
||||||
|
|
||||||
Bugs fixed:
|
|
||||||
* downloads were not streamed leading to excessive memory usage (#67)
|
|
||||||
* Last-Modified header incorrectly used local time (#75)
|
|
||||||
|
|
||||||
## 0.15.2 (22 Jun 2021)
|
|
||||||
|
|
||||||
New features:
|
|
||||||
* Content-Type returned for object GET requests can now be taken from
|
|
||||||
attributes (overriding autodetection, #65)
|
|
||||||
|
|
||||||
Behavior changes:
|
|
||||||
* grpc keepalive options can no longer be changed (#60)
|
|
||||||
|
|
||||||
Improvements:
|
|
||||||
* code refactoring (more reuse between different gateways, moved some code to
|
|
||||||
sdk-go, #47, #46, #51, #62, #63)
|
|
||||||
* documentation updates and fixes (#53, #49, #55, #59)
|
|
||||||
* updated api-go dependency (#57)
|
|
||||||
|
|
||||||
Bugs fixed:
|
|
||||||
* `-k` option wasn't accepted for key although it was documented (#50)
|
|
||||||
|
|
||||||
## 0.15.1 (24 May 2021)
|
|
||||||
|
|
||||||
This important release makes HTTP gateway compatible with NeoFS node version
|
|
||||||
0.20.0.
|
|
||||||
|
|
||||||
Behavior changes:
|
|
||||||
* neofs-api-go was updated to 1.26.1, which contains some incompatible
|
|
||||||
changes in underlying components (#39, #44)
|
|
||||||
* `neofs-http-gw` is consistently used now for repository, binary and image
|
|
||||||
names (#43)
|
|
||||||
|
|
||||||
Improvements:
|
|
||||||
* minor code cleanups based on stricter set of linters (#41)
|
|
||||||
* updated README (#42)
|
|
||||||
|
|
||||||
## 0.15.0 (30 Apr 2021)
|
|
||||||
|
|
||||||
This is the first public release incorporating latest NeoFS protocol support
|
|
||||||
and fixing some bugs.
|
|
||||||
|
|
||||||
New features:
|
|
||||||
* upload support (#14, #13, #29)
|
|
||||||
* ephemeral keys (#26)
|
|
||||||
* TLS server support (#28)
|
|
||||||
|
|
||||||
Behavior changes:
|
|
||||||
* node weights can now be specified as simple numbers instead of percentages
|
|
||||||
and gateway will calculate the proportion automatically (#27)
|
|
||||||
* attributes are converted now to `X-Attribute-*` headers when retrieving
|
|
||||||
object from gate instead of `X-*` (#29)
|
|
||||||
|
|
||||||
Improvements:
|
|
||||||
* better Makefile (#16, #24, #33, #34)
|
|
||||||
* updated documentation (#16, #29, #35, #36)
|
|
||||||
* updated neofs-api-go to v1.25.0 (#17, #20)
|
|
||||||
* updated fasthttp to v1.23.0+ (#17, #29)
|
|
||||||
* refactoring, eliminating some dependencies (#20, #29)
|
|
||||||
|
|
||||||
Bugs fixed:
|
|
||||||
* gateway attempted to work with no NeoFS peers configured (#29)
|
|
||||||
* some invalid headers could be sent for attributes using non-ASCII or
|
|
||||||
non-printable characters (#29)
|
|
||||||
|
|
||||||
## Older versions
|
## Older versions
|
||||||
|
|
||||||
Please refer to [Github
|
This project is a fork of [NeoFS HTTP Gateway](https://github.com/nspcc-dev/neofs-http-gw) from version v0.26.0.
|
||||||
releases](https://github.com/nspcc-dev/neofs-http-gw/releases/) for older
|
To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs-http-gw/blob/master/CHANGELOG.md.
|
||||||
releases.
|
|
||||||
|
|
||||||
[0.17.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.16.1...v0.17.0
|
[0.27.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/72734ab4...v0.27.0
|
||||||
[0.18.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.17.0...v0.18.0
|
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.27.0...master
|
||||||
[0.19.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.18.0...v0.19.0
|
|
||||||
[0.20.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.19.0...v0.20.0
|
|
||||||
[0.21.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.20.0...v0.21.0
|
|
||||||
[0.22.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.21.0...v0.22.0
|
|
||||||
[0.23.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.22.0...v0.23.0
|
|
||||||
[0.24.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.23.0...v0.24.0
|
|
||||||
[0.25.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.24.0...v0.25.0
|
|
||||||
[0.25.1]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.25.0...v0.25.1
|
|
||||||
[0.26.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.25.1...v0.26.0
|
|
||||||
[Unreleased]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.26.0...master
|
|
||||||
|
|
15
Makefile
Normal file → Executable file
15
Makefile
Normal file → Executable file
|
@ -14,7 +14,7 @@ BINDIR = bin
|
||||||
DIRS = $(BINDIR)
|
DIRS = $(BINDIR)
|
||||||
BINS = $(BINDIR)/frostfs-http-gw
|
BINS = $(BINDIR)/frostfs-http-gw
|
||||||
|
|
||||||
.PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint version clean
|
.PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean
|
||||||
|
|
||||||
# .deb package versioning
|
# .deb package versioning
|
||||||
OS_RELEASE = $(shell lsb_release -cs)
|
OS_RELEASE = $(shell lsb_release -cs)
|
||||||
|
@ -62,6 +62,11 @@ docker/%:
|
||||||
test:
|
test:
|
||||||
@go test ./... -cover
|
@go test ./... -cover
|
||||||
|
|
||||||
|
# Run integration tests
|
||||||
|
.PHONY: integration-test
|
||||||
|
integration-test:
|
||||||
|
@go test ./... -cover --tags=integration
|
||||||
|
|
||||||
# Run tests with race detection and produce coverage output
|
# Run tests with race detection and produce coverage output
|
||||||
cover:
|
cover:
|
||||||
@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
|
@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
|
||||||
|
@ -109,6 +114,14 @@ docker/lint:
|
||||||
--env HOME=/src \
|
--env HOME=/src \
|
||||||
golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint'
|
golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint'
|
||||||
|
|
||||||
|
# Activate pre-commit hooks
|
||||||
|
pre-commit:
|
||||||
|
pre-commit install -t pre-commit -t commit-msg
|
||||||
|
|
||||||
|
# Deactivate pre-commit hooks
|
||||||
|
unpre-commit:
|
||||||
|
pre-commit uninstall -t pre-commit -t commit-msg
|
||||||
|
|
||||||
# Print version
|
# Print version
|
||||||
version:
|
version:
|
||||||
@echo $(VERSION)
|
@echo $(VERSION)
|
||||||
|
|
58
README.md
58
README.md
|
@ -1,11 +1,12 @@
|
||||||
|
<p align="center">
|
||||||
|
<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
|
||||||
|
</p>
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<a href="https://frostfs.info">FrostFS</a> is a decentralized distributed object storage integrated with the <a href="https://neo.org">NEO Blockchain</a>.
|
<a href="https://frostfs.info">FrostFS</a> is a decentralized distributed object storage integrated with the <a href="https://neo.org">NEO Blockchain</a>.
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
---
|
---
|
||||||
[](https://goreportcard.com/report/github.com/TrueCloudLab/frostfs-http-gw)
|
[](https://goreportcard.com/report/git.frostfs.info/TrueCloudLab/frostfs-http-gw)
|
||||||

|
|
||||||

|
|
||||||
|
|
||||||
# FrostFS HTTP Gateway
|
# FrostFS HTTP Gateway
|
||||||
|
|
||||||
|
@ -17,7 +18,7 @@ See available routes in [specification](./docs/api.md).
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
```go install github.com/TrueCloudLab/frostfs-http-gw```
|
```go install git.frostfs.info/TrueCloudLab/frostfs-http-gw```
|
||||||
|
|
||||||
Or you can call `make` to build it from the cloned repository (the binary will
|
Or you can call `make` to build it from the cloned repository (the binary will
|
||||||
end up in `bin/frostfs-http-gw`). To build frostfs-http-gw binary in clean docker
|
end up in `bin/frostfs-http-gw`). To build frostfs-http-gw binary in clean docker
|
||||||
|
@ -47,7 +48,7 @@ can be done either via `-p` parameter or via `HTTP_GW_PEERS_<N>_ADDRESS` and
|
||||||
`HTTP_GW_PEERS_<N>_WEIGHT` environment variables (the gate supports multiple
|
`HTTP_GW_PEERS_<N>_WEIGHT` environment variables (the gate supports multiple
|
||||||
FrostFS nodes with weighted load balancing).
|
FrostFS nodes with weighted load balancing).
|
||||||
|
|
||||||
If you launch HTTP gateway in bundle with [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env),
|
If you launch HTTP gateway in bundle with [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env),
|
||||||
you can get the IP address of the node in the output of `make hosts` command
|
you can get the IP address of the node in the output of `make hosts` command
|
||||||
(with s0*.frostfs.devenv name).
|
(with s0*.frostfs.devenv name).
|
||||||
|
|
||||||
|
@ -179,6 +180,23 @@ $ frostfs-http-gw --config your-config.yaml
|
||||||
|
|
||||||
See [config](./config/config.yaml) and [defaults](./docs/gate-configuration.md) for example.
|
See [config](./config/config.yaml) and [defaults](./docs/gate-configuration.md) for example.
|
||||||
|
|
||||||
|
#### Multiple configs
|
||||||
|
|
||||||
|
You can use several config files when running application. It allows you to split configuration into parts.
|
||||||
|
For example, you can use separate yaml file for pprof and prometheus section in config (see [config examples](./config)).
|
||||||
|
You can either provide several files with repeating `--config` flag or provide path to the dir that contains all configs using `--config-dir` flag.
|
||||||
|
Also, you can combine these flags:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
$ frostfs-http-gw --config ./config/config.yaml --config /your/partial/config.yaml --config-dir ./config/dir
|
||||||
|
```
|
||||||
|
|
||||||
|
**Note:** next file in `--config` flag overwrites values from the previous one.
|
||||||
|
Files from `--config-dir` directory overwrite values from `--config` files.
|
||||||
|
So the command above run `frostfs-http-gw` to listen on `0.0.0.0:8080` address (value from `./config/config.yaml`),
|
||||||
|
applies parameters from `/your/partial/config.yaml`,
|
||||||
|
enable pprof (value from `./config/dir/pprof.yaml`) and prometheus (value from `./config/dir/prometheus.yaml`).
|
||||||
|
|
||||||
## HTTP API provided
|
## HTTP API provided
|
||||||
|
|
||||||
This gateway intentionally provides limited feature set and doesn't try to
|
This gateway intentionally provides limited feature set and doesn't try to
|
||||||
|
@ -208,7 +226,7 @@ resolve_order:
|
||||||
- nns
|
- nns
|
||||||
```
|
```
|
||||||
|
|
||||||
2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env)
|
2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env)
|
||||||
you can check if your container (e.g. with `container-name` name) is registered in NNS:
|
you can check if your container (e.g. with `container-name` name) is registered in NNS:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
|
@ -235,7 +253,7 @@ $ curl http://localhost:8082/get_by_attribute/container-name/FileName/object-nam
|
||||||
|
|
||||||
#### Create a container
|
#### Create a container
|
||||||
|
|
||||||
You can create a container via [frostfs-cli](https://github.com/TrueCloudLab/frostfs-node/releases):
|
You can create a container via [frostfs-cli](https://git.frostfs.info/TrueCloudLab/frostfs-node/releases):
|
||||||
```
|
```
|
||||||
$ frostfs-cli -r $FROSTFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL
|
$ frostfs-cli -r $FROSTFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL
|
||||||
```
|
```
|
||||||
|
@ -248,13 +266,13 @@ For example:
|
||||||
$ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json container create --policy "REP 3" --basic-acl public --await
|
$ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json container create --policy "REP 3" --basic-acl public --await
|
||||||
```
|
```
|
||||||
|
|
||||||
If you have launched nodes via [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env),
|
If you have launched nodes via [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env),
|
||||||
you can get the key value from `wallets/wallet.json` or write the path to
|
you can get the key value from `wallets/wallet.json` or write the path to
|
||||||
the file `wallets/wallet.key`.
|
the file `wallets/wallet.key`.
|
||||||
|
|
||||||
#### Prepare a file in a container
|
#### Prepare a file in a container
|
||||||
|
|
||||||
To create a file via [frostfs-cli](https://github.com/TrueCloudLab/frostfs-node/releases), run a command below:
|
To create a file via [frostfs-cli](https://git.frostfs.info/TrueCloudLab/frostfs-node/releases), run a command below:
|
||||||
```
|
```
|
||||||
$ frostfs-cli -r $FROSTFS_NODE -k $KEY object put --file $FILENAME --cid $CID
|
$ frostfs-cli -r $FROSTFS_NODE -k $KEY object put --file $FILENAME --cid $CID
|
||||||
```
|
```
|
||||||
|
@ -406,12 +424,12 @@ You can also add some attributes to your file using the following rules:
|
||||||
"X-Attribute-" prefix stripped, that is if you add "X-Attribute-Ololo:
|
"X-Attribute-" prefix stripped, that is if you add "X-Attribute-Ololo:
|
||||||
100500" header to your request the resulting object will get "Ololo:
|
100500" header to your request the resulting object will get "Ololo:
|
||||||
100500" attribute
|
100500" attribute
|
||||||
* "X-Attribute-NEOFS-*" headers are special
|
* "X-Attribute-SYSTEM-*" headers are special
|
||||||
(`-NEOFS-` part can also be `-neofs-` or`-Neofs-`), they're used to set internal
|
(`-SYSTEM-` part can also be `-system-` or`-System-` (and even legacy `-Neofs-` for some next releases)), they're used to set internal
|
||||||
NeoFS attributes starting with `__NEOFS__` prefix, for these attributes all
|
FrostFS attributes starting with `__SYSTEM__` prefix, for these attributes all
|
||||||
dashes get converted to underscores and all letters are capitalized. For
|
dashes get converted to underscores and all letters are capitalized. For
|
||||||
example, you can use "X-Attribute-NEOFS-Expiration-Epoch" header to set
|
example, you can use "X-Attribute-SYSTEM-Expiration-Epoch" header to set
|
||||||
`__NEOFS__EXPIRATION_EPOCH` attribute
|
`__SYSTEM__EXPIRATION_EPOCH` attribute
|
||||||
* `FileName` attribute is set from multipart's `filename` if not set
|
* `FileName` attribute is set from multipart's `filename` if not set
|
||||||
explicitly via `X-Attribute-FileName` header
|
explicitly via `X-Attribute-FileName` header
|
||||||
* `Timestamp` attribute can be set using gateway local time if using
|
* `Timestamp` attribute can be set using gateway local time if using
|
||||||
|
@ -421,13 +439,13 @@ You can also add some attributes to your file using the following rules:
|
||||||
---
|
---
|
||||||
**NOTE**
|
**NOTE**
|
||||||
|
|
||||||
There are some reserved headers type of `X-Attribute-NEOFS-*` (headers are arranged in descending order of priority):
|
There are some reserved headers type of `X-Attribute-SYSTEM-*` (headers are arranged in descending order of priority):
|
||||||
1. `X-Attribute-Neofs-Expiration-Epoch: 100`
|
1. `X-Attribute-System-Expiration-Epoch: 100`
|
||||||
2. `X-Attribute-Neofs-Expiration-Duration: 24h30m`
|
2. `X-Attribute-System-Expiration-Duration: 24h30m`
|
||||||
3. `X-Attribute-Neofs-Expiration-Timestamp: 1637574797`
|
3. `X-Attribute-System-Expiration-Timestamp: 1637574797`
|
||||||
4. `X-Attribute-Neofs-Expiration-RFC3339: 2021-11-22T09:55:49Z`
|
4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z`
|
||||||
|
|
||||||
which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide expiration any convenient way.
|
which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
2
VERSION
2
VERSION
|
@ -1 +1 @@
|
||||||
v0.26.0
|
v0.27.0
|
||||||
|
|
88
app.go
88
app.go
|
@ -11,14 +11,14 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/downloader"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/downloader"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/metrics"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/resolver"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/response"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/uploader"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/utils"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/fasthttp/router"
|
"github.com/fasthttp/router"
|
||||||
"github.com/nspcc-dev/neo-go/cli/flags"
|
"github.com/nspcc-dev/neo-go/cli/flags"
|
||||||
"github.com/nspcc-dev/neo-go/cli/input"
|
"github.com/nspcc-dev/neo-go/cli/input"
|
||||||
|
@ -62,15 +62,10 @@ type (
|
||||||
|
|
||||||
gateMetrics struct {
|
gateMetrics struct {
|
||||||
logger *zap.Logger
|
logger *zap.Logger
|
||||||
provider GateMetricsProvider
|
provider *metrics.GateMetrics
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
enabled bool
|
enabled bool
|
||||||
}
|
}
|
||||||
|
|
||||||
GateMetricsProvider interface {
|
|
||||||
SetHealth(int32)
|
|
||||||
Unregister()
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// WithLogger returns Option to set a specific logger.
|
// WithLogger returns Option to set a specific logger.
|
||||||
|
@ -214,15 +209,17 @@ func (a *app) getResolverConfig() ([]string, *resolver.Config) {
|
||||||
func (a *app) initMetrics() {
|
func (a *app) initMetrics() {
|
||||||
gateMetricsProvider := metrics.NewGateMetrics(a.pool)
|
gateMetricsProvider := metrics.NewGateMetrics(a.pool)
|
||||||
a.metrics = newGateMetrics(a.log, gateMetricsProvider, a.cfg.GetBool(cfgPrometheusEnabled))
|
a.metrics = newGateMetrics(a.log, gateMetricsProvider, a.cfg.GetBool(cfgPrometheusEnabled))
|
||||||
|
a.metrics.SetHealth(metrics.HealthStatusStarting)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newGateMetrics(logger *zap.Logger, provider GateMetricsProvider, enabled bool) *gateMetrics {
|
func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled bool) *gateMetrics {
|
||||||
if !enabled {
|
if !enabled {
|
||||||
logger.Warn("metrics are disabled")
|
logger.Warn("metrics are disabled")
|
||||||
}
|
}
|
||||||
return &gateMetrics{
|
return &gateMetrics{
|
||||||
logger: logger,
|
logger: logger,
|
||||||
provider: provider,
|
provider: provider,
|
||||||
|
enabled: enabled,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -236,7 +233,7 @@ func (m *gateMetrics) SetEnabled(enabled bool) {
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *gateMetrics) SetHealth(status int32) {
|
func (m *gateMetrics) SetHealth(status metrics.HealthStatus) {
|
||||||
m.mu.RLock()
|
m.mu.RLock()
|
||||||
if !m.enabled {
|
if !m.enabled {
|
||||||
m.mu.RUnlock()
|
m.mu.RUnlock()
|
||||||
|
@ -250,7 +247,7 @@ func (m *gateMetrics) SetHealth(status int32) {
|
||||||
func (m *gateMetrics) Shutdown() {
|
func (m *gateMetrics) Shutdown() {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
if m.enabled {
|
if m.enabled {
|
||||||
m.provider.SetHealth(0)
|
m.provider.SetHealth(metrics.HealthStatusShuttingDown)
|
||||||
m.enabled = false
|
m.enabled = false
|
||||||
}
|
}
|
||||||
m.provider.Unregister()
|
m.provider.Unregister()
|
||||||
|
@ -335,7 +332,7 @@ func (a *app) Wait() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *app) setHealthStatus() {
|
func (a *app) setHealthStatus() {
|
||||||
a.metrics.SetHealth(1)
|
a.metrics.SetHealth(metrics.HealthStatusReady)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *app) Serve(ctx context.Context) {
|
func (a *app) Serve(ctx context.Context) {
|
||||||
|
@ -380,14 +377,15 @@ LOOP:
|
||||||
|
|
||||||
func (a *app) configReload() {
|
func (a *app) configReload() {
|
||||||
a.log.Info("SIGHUP config reload started")
|
a.log.Info("SIGHUP config reload started")
|
||||||
if !a.cfg.IsSet(cmdConfig) {
|
if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) {
|
||||||
a.log.Warn("failed to reload config because it's missed")
|
a.log.Warn("failed to reload config because it's missed")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := readConfig(a.cfg); err != nil {
|
if err := readInConfig(a.cfg); err != nil {
|
||||||
a.log.Warn("failed to reload config", zap.Error(err))
|
a.log.Warn("failed to reload config", zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if lvl, err := getLogLevel(a.cfg); err != nil {
|
if lvl, err := getLogLevel(a.cfg); err != nil {
|
||||||
a.log.Warn("log level won't be updated", zap.Error(err))
|
a.log.Warn("log level won't be updated", zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
|
@ -485,33 +483,57 @@ func (a *app) AppParams() *utils.AppParams {
|
||||||
func (a *app) initServers(ctx context.Context) {
|
func (a *app) initServers(ctx context.Context) {
|
||||||
serversInfo := fetchServers(a.cfg)
|
serversInfo := fetchServers(a.cfg)
|
||||||
|
|
||||||
a.servers = make([]Server, len(serversInfo))
|
a.servers = make([]Server, 0, len(serversInfo))
|
||||||
for i, serverInfo := range serversInfo {
|
for _, serverInfo := range serversInfo {
|
||||||
a.log.Info("added server",
|
fields := []zap.Field{
|
||||||
zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled),
|
zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled),
|
||||||
zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile))
|
zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile),
|
||||||
a.servers[i] = newServer(ctx, serverInfo, a.log)
|
}
|
||||||
|
srv, err := newServer(ctx, serverInfo)
|
||||||
|
if err != nil {
|
||||||
|
a.log.Warn("failed to add server", append(fields, zap.Error(err))...)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
a.servers = append(a.servers, srv)
|
||||||
|
a.log.Info("add server", fields...)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(a.servers) == 0 {
|
||||||
|
a.log.Fatal("no healthy servers")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *app) updateServers() error {
|
func (a *app) updateServers() error {
|
||||||
serversInfo := fetchServers(a.cfg)
|
serversInfo := fetchServers(a.cfg)
|
||||||
|
|
||||||
if len(serversInfo) != len(a.servers) {
|
var found bool
|
||||||
return fmt.Errorf("invalid servers configuration: length mismatch: old '%d', new '%d", len(a.servers), len(serversInfo))
|
for _, serverInfo := range serversInfo {
|
||||||
}
|
index := a.serverIndex(serverInfo.Address)
|
||||||
|
if index == -1 {
|
||||||
for i, serverInfo := range serversInfo {
|
continue
|
||||||
if serverInfo.Address != a.servers[i].Address() {
|
|
||||||
return fmt.Errorf("invalid servers configuration: addresses mismatch: old '%s', new '%s", a.servers[i].Address(), serverInfo.Address)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if serverInfo.TLS.Enabled {
|
if serverInfo.TLS.Enabled {
|
||||||
if err := a.servers[i].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
|
if err := a.servers[index].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
|
||||||
return fmt.Errorf("failed to update tls certs: %w", err)
|
return fmt.Errorf("failed to update tls certs: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("invalid servers configuration: no known server found")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (a *app) serverIndex(address string) int {
|
||||||
|
for i := range a.servers {
|
||||||
|
if a.servers[i].Address() == address {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
|
@ -4,10 +4,10 @@ wallet:
|
||||||
passphrase: pwd # Passphrase to decrypt wallet. If you're using a wallet without a password, place '' here.
|
passphrase: pwd # Passphrase to decrypt wallet. If you're using a wallet without a password, place '' here.
|
||||||
|
|
||||||
pprof:
|
pprof:
|
||||||
enabled: true # Enable pprof.
|
enabled: false # Enable pprof.
|
||||||
address: localhost:8083
|
address: localhost:8083
|
||||||
prometheus:
|
prometheus:
|
||||||
enabled: true # Enable metrics.
|
enabled: false # Enable metrics.
|
||||||
address: localhost:8084
|
address: localhost:8084
|
||||||
|
|
||||||
logger:
|
logger:
|
||||||
|
|
3
config/dir/pprof.yaml
Normal file
3
config/dir/pprof.yaml
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
pprof:
|
||||||
|
enabled: true
|
||||||
|
address: localhost:8083
|
3
config/dir/prometheus.yaml
Normal file
3
config/dir/prometheus.yaml
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
prometheus:
|
||||||
|
enabled: true
|
||||||
|
address: localhost:8084
|
5
debian/control
vendored
5
debian/control
vendored
|
@ -5,11 +5,10 @@ Maintainer: TrueCloudLab <tech@frostfs.info>
|
||||||
Build-Depends: debhelper-compat (= 13), dh-sysuser, git, devscripts
|
Build-Depends: debhelper-compat (= 13), dh-sysuser, git, devscripts
|
||||||
Standards-Version: 4.5.1
|
Standards-Version: 4.5.1
|
||||||
Homepage: https://frostfs.info/
|
Homepage: https://frostfs.info/
|
||||||
Vcs-Git: https://github.com/TrueCloudLab/frostfs-http-gw.git
|
Vcs-Git: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw.git
|
||||||
Vcs-Browser: https://github.com/TrueCloudLab/frostfs-http-gw
|
Vcs-Browser: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw
|
||||||
|
|
||||||
Package: frostfs-http-gw
|
Package: frostfs-http-gw
|
||||||
Architecture: any
|
Architecture: any
|
||||||
Depends: ${misc:Depends}
|
Depends: ${misc:Depends}
|
||||||
Description: FrostFS HTTP Gateway bridges FrostFS internal protocol and HTTP standard.
|
Description: FrostFS HTTP Gateway bridges FrostFS internal protocol and HTTP standard.
|
||||||
|
|
||||||
|
|
4
debian/copyright
vendored
4
debian/copyright
vendored
|
@ -1,13 +1,13 @@
|
||||||
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
|
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
|
||||||
Upstream-Name: frostfs-http-gw
|
Upstream-Name: frostfs-http-gw
|
||||||
Upstream-Contact: tech@frostfs.info
|
Upstream-Contact: tech@frostfs.info
|
||||||
Source: https://github.com/TrueCloudLab/frostfs-http-gw
|
Source: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw
|
||||||
|
|
||||||
Files: *
|
Files: *
|
||||||
Copyright: 2018-2022 NeoSPCC (@nspcc-dev), contributors of neofs-http-gw project
|
Copyright: 2018-2022 NeoSPCC (@nspcc-dev), contributors of neofs-http-gw project
|
||||||
(https://github.com/nspcc-dev/neofs-http-gw/blob/master/CREDITS.md)
|
(https://github.com/nspcc-dev/neofs-http-gw/blob/master/CREDITS.md)
|
||||||
2022 True Cloud Lab (@TrueCloudLab), contributors of frostfs-http-gw project
|
2022 True Cloud Lab (@TrueCloudLab), contributors of frostfs-http-gw project
|
||||||
(https://github.com/TrueCloudLab/frostfs-http-gw/blob/master/CREDITS.md)
|
(https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/src/branch/master/CREDITS.md)
|
||||||
|
|
||||||
|
|
||||||
License: GPL-3
|
License: GPL-3
|
||||||
|
|
4
debian/frostfs-http-gw.postinst
vendored
Normal file → Executable file
4
debian/frostfs-http-gw.postinst
vendored
Normal file → Executable file
|
@ -29,8 +29,8 @@ case "$1" in
|
||||||
chmod -f 0640 /etc/frostfs/$USERNAME/config.yaml || true
|
chmod -f 0640 /etc/frostfs/$USERNAME/config.yaml || true
|
||||||
fi
|
fi
|
||||||
USERDIR=$(getent passwd "frostfs-$USERNAME" | cut -d: -f6)
|
USERDIR=$(getent passwd "frostfs-$USERNAME" | cut -d: -f6)
|
||||||
if ! dpkg-statoverride --list frostfs-$USERDIR >/dev/null; then
|
if ! dpkg-statoverride --list frostfs-"$USERDIR" >/dev/null; then
|
||||||
chown -f frostfs-$USERNAME: $USERDIR
|
chown -f frostfs-$USERNAME: "$USERDIR"
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
|
0
debian/frostfs-http-gw.postrm
vendored
Normal file → Executable file
0
debian/frostfs-http-gw.postrm
vendored
Normal file → Executable file
0
debian/frostfs-http-gw.preinst
vendored
Normal file → Executable file
0
debian/frostfs-http-gw.preinst
vendored
Normal file → Executable file
0
debian/frostfs-http-gw.prerm
vendored
Normal file → Executable file
0
debian/frostfs-http-gw.prerm
vendored
Normal file → Executable file
2
debian/rules
vendored
2
debian/rules
vendored
|
@ -12,5 +12,3 @@ override_dh_installsystemd:
|
||||||
|
|
||||||
override_dh_installchangelogs:
|
override_dh_installchangelogs:
|
||||||
dh_installchangelogs -k CHANGELOG.md
|
dh_installchangelogs -k CHANGELOG.md
|
||||||
|
|
||||||
|
|
||||||
|
|
34
docs/api.md
34
docs/api.md
|
@ -57,20 +57,20 @@ Upload file as object with attributes to FrostFS.
|
||||||
###### Headers
|
###### Headers
|
||||||
|
|
||||||
| Header | Description |
|
| Header | Description |
|
||||||
|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|
|
|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| Common headers | See [bearer token](#bearer-token). |
|
| Common headers | See [bearer token](#bearer-token). |
|
||||||
| `X-Attribute-Neofs-*` | Used to set system NeoFS object attributes <br/> (e.g. use "X-Attribute-Neofs-Expiration-Epoch" to set `__NEOFS__EXPIRATION_EPOCH` attribute). |
|
| `X-Attribute-System-*` | Used to set system FrostFS object attributes <br/> (e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). |
|
||||||
| `X-Attribute-*` | Used to set regular object attributes <br/> (e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). |
|
| `X-Attribute-*` | Used to set regular object attributes <br/> (e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). |
|
||||||
| `Date` | This header is used to calculate the right `__NEOFS__EXPIRATION` attribute for object. If the header is missing, the current server time is used. |
|
| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. |
|
||||||
|
|
||||||
There are some reserved headers type of `X-Attribute-NEOFS-*` (headers are arranged in descending order of priority):
|
There are some reserved headers type of `X-Attribute-FROSTFS-*` (headers are arranged in descending order of priority):
|
||||||
|
|
||||||
1. `X-Attribute-Neofs-Expiration-Epoch: 100`
|
1. `X-Attribute-System-Expiration-Epoch: 100`
|
||||||
2. `X-Attribute-Neofs-Expiration-Duration: 24h30m`
|
2. `X-Attribute-System-Expiration-Duration: 24h30m`
|
||||||
3. `X-Attribute-Neofs-Expiration-Timestamp: 1637574797`
|
3. `X-Attribute-System-Expiration-Timestamp: 1637574797`
|
||||||
4. `X-Attribute-Neofs-Expiration-RFC3339: 2021-11-22T09:55:49Z`
|
4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z`
|
||||||
|
|
||||||
which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide expiration any convenient way.
|
which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way.
|
||||||
|
|
||||||
If you don't specify the `X-Attribute-Timestamp` header the `Timestamp` attribute can be set anyway
|
If you don't specify the `X-Attribute-Timestamp` header the `Timestamp` attribute can be set anyway
|
||||||
(see http-gw [configuration](gate-configuration.md#upload-header-section)).
|
(see http-gw [configuration](gate-configuration.md#upload-header-section)).
|
||||||
|
@ -122,8 +122,8 @@ Get an object (payload and attributes) by an address.
|
||||||
###### Headers
|
###### Headers
|
||||||
|
|
||||||
| Header | Description |
|
| Header | Description |
|
||||||
|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
|
|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| `X-Attribute-Neofs-*` | System NeoFS object attributes <br/> (e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). |
|
| `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
|
||||||
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
|
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
|
||||||
| `Content-Disposition` | Indicate how to browsers should treat file. <br/> Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). |
|
| `Content-Disposition` | Indicate how to browsers should treat file. <br/> Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). |
|
||||||
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
|
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
|
||||||
|
@ -158,8 +158,8 @@ Get an object attributes by an address.
|
||||||
###### Headers
|
###### Headers
|
||||||
|
|
||||||
| Header | Description |
|
| Header | Description |
|
||||||
|-----------------------|--------------------------------------------------------------------------------------------------------------------------|
|
|------------------------|------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| `X-Attribute-Neofs-*` | System NeoFS object attributes <br/> (e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). |
|
| `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
|
||||||
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
|
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
|
||||||
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
|
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
|
||||||
| `Content-Length` | Size of object payload. |
|
| `Content-Length` | Size of object payload. |
|
||||||
|
@ -207,8 +207,8 @@ If more than one object is found, an arbitrary one will be returned.
|
||||||
###### Headers
|
###### Headers
|
||||||
|
|
||||||
| Header | Description |
|
| Header | Description |
|
||||||
|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
|
|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| `X-Attribute-Neofs-*` | System NeoFS object attributes <br/> (e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). |
|
| `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
|
||||||
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
|
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
|
||||||
| `Content-Disposition` | Indicate how to browsers should treat file. <br/> Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). |
|
| `Content-Disposition` | Indicate how to browsers should treat file. <br/> Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). |
|
||||||
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
|
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
|
||||||
|
@ -244,8 +244,8 @@ If more than one object is found, an arbitrary one will be used to get attribute
|
||||||
###### Headers
|
###### Headers
|
||||||
|
|
||||||
| Header | Description |
|
| Header | Description |
|
||||||
|-----------------------|--------------------------------------------------------------------------------------------------------------------------|
|
|------------------------|------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| `X-Attribute-Neofs-*` | System NeoFS object attributes <br/> (e.g. `__NEOFS__EXPIRATION_EPOCH` set "X-Attribute-Neofs-Expiration-Epoch" header). |
|
| `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
|
||||||
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
|
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
|
||||||
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
|
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
|
||||||
| `Content-Length` | Size of object payload. |
|
| `Content-Length` | Size of object payload. |
|
||||||
|
|
|
@ -14,20 +14,18 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"unicode"
|
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/resolver"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/response"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/tokens"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/utils"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/client"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
"go.uber.org/atomic"
|
"go.uber.org/atomic"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -131,9 +129,9 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) {
|
||||||
if !isValidToken(key) || !isValidValue(val) {
|
if !isValidToken(key) || !isValidValue(val) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(key, utils.SystemAttributePrefix) {
|
|
||||||
key = systemBackwardTranslator(key)
|
key = utils.BackwardTransformIfSystem(key)
|
||||||
}
|
|
||||||
r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
|
r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
|
||||||
switch key {
|
switch key {
|
||||||
case object.AttributeFileName:
|
case object.AttributeFileName:
|
||||||
|
@ -187,36 +185,6 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) {
|
||||||
r.Response.SetBodyStream(rObj.Payload, int(payloadSize))
|
r.Response.SetBodyStream(rObj.Payload, int(payloadSize))
|
||||||
}
|
}
|
||||||
|
|
||||||
// systemBackwardTranslator is used to convert headers looking like '__NEOFS__ATTR_NAME' to 'Neofs-Attr-Name'.
|
|
||||||
func systemBackwardTranslator(key string) string {
|
|
||||||
// trim specified prefix '__NEOFS__'
|
|
||||||
key = strings.TrimPrefix(key, utils.SystemAttributePrefix)
|
|
||||||
|
|
||||||
var res strings.Builder
|
|
||||||
res.WriteString("Neofs-")
|
|
||||||
|
|
||||||
strs := strings.Split(key, "_")
|
|
||||||
for i, s := range strs {
|
|
||||||
s = title(strings.ToLower(s))
|
|
||||||
res.WriteString(s)
|
|
||||||
if i != len(strs)-1 {
|
|
||||||
res.WriteString("-")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return res.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func title(str string) string {
|
|
||||||
if str == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
r, size := utf8.DecodeRuneInString(str)
|
|
||||||
r0 := unicode.ToTitle(r)
|
|
||||||
return string(r0) + str[size:]
|
|
||||||
}
|
|
||||||
|
|
||||||
func bearerToken(ctx context.Context) *bearer.Token {
|
func bearerToken(ctx context.Context) *bearer.Token {
|
||||||
if tkn, err := tokens.LoadBearerToken(ctx); err == nil {
|
if tkn, err := tokens.LoadBearerToken(ctx); err == nil {
|
||||||
return tkn
|
return tkn
|
||||||
|
@ -225,19 +193,15 @@ func bearerToken(ctx context.Context) *bearer.Token {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *request) handleFrostFSErr(err error, start time.Time) {
|
func (r *request) handleFrostFSErr(err error, start time.Time) {
|
||||||
r.log.Error(
|
logFields := []zap.Field{
|
||||||
"could not receive object",
|
|
||||||
zap.Stringer("elapsed", time.Since(start)),
|
zap.Stringer("elapsed", time.Since(start)),
|
||||||
zap.Error(err),
|
zap.Error(err),
|
||||||
)
|
|
||||||
|
|
||||||
if client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err) {
|
|
||||||
response.Error(r.RequestCtx, "Not Found", fasthttp.StatusNotFound)
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
statusCode, msg, additionalFields := response.FormErrorResponse("could not receive object", err)
|
||||||
|
logFields = append(logFields, additionalFields...)
|
||||||
|
|
||||||
msg := fmt.Sprintf("could not receive object: %v", err)
|
r.log.Error("could not receive object", logFields...)
|
||||||
response.Error(r.RequestCtx, msg, fasthttp.StatusBadRequest)
|
response.Error(r.RequestCtx, msg, statusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Downloader is a download request handler.
|
// Downloader is a download request handler.
|
||||||
|
|
|
@ -1,23 +0,0 @@
|
||||||
package downloader
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestSystemBackwardTranslator(t *testing.T) {
|
|
||||||
input := []string{
|
|
||||||
"__NEOFS__EXPIRATION_EPOCH",
|
|
||||||
"__NEOFS__RANDOM_ATTR",
|
|
||||||
}
|
|
||||||
expected := []string{
|
|
||||||
"Neofs-Expiration-Epoch",
|
|
||||||
"Neofs-Random-Attr",
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, str := range input {
|
|
||||||
res := systemBackwardTranslator(str)
|
|
||||||
require.Equal(t, expected[i], res)
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -4,15 +4,14 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/response"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/tokens"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/utils"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -56,9 +55,9 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) {
|
||||||
if !isValidToken(key) || !isValidValue(val) {
|
if !isValidToken(key) || !isValidValue(val) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(key, utils.SystemAttributePrefix) {
|
|
||||||
key = systemBackwardTranslator(key)
|
key = utils.BackwardTransformIfSystem(key)
|
||||||
}
|
|
||||||
r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
|
r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
|
||||||
switch key {
|
switch key {
|
||||||
case object.AttributeTimestamp:
|
case object.AttributeTimestamp:
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
//go:build !integration
|
||||||
|
|
||||||
package downloader
|
package downloader
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
89
go.mod
89
go.mod
|
@ -1,71 +1,74 @@
|
||||||
module github.com/TrueCloudLab/frostfs-http-gw
|
module git.frostfs.info/TrueCloudLab/frostfs-http-gw
|
||||||
|
|
||||||
go 1.17
|
go 1.18
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/TrueCloudLab/frostfs-api-go/v2 v2.0.0-20221212144048-1351b6656d68
|
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142024-4cb0068ddef0
|
||||||
github.com/TrueCloudLab/frostfs-sdk-go v0.0.0-20221214065929-4c779423f556
|
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230329125804-552219b8e130
|
||||||
github.com/fasthttp/router v1.4.1
|
github.com/fasthttp/router v1.4.1
|
||||||
github.com/nspcc-dev/neo-go v0.99.4
|
github.com/nspcc-dev/neo-go v0.101.0
|
||||||
github.com/prometheus/client_golang v1.13.0
|
github.com/prometheus/client_golang v1.13.0
|
||||||
github.com/spf13/pflag v1.0.5
|
github.com/spf13/pflag v1.0.5
|
||||||
github.com/spf13/viper v1.8.1
|
github.com/spf13/viper v1.15.0
|
||||||
github.com/stretchr/testify v1.8.0
|
github.com/stretchr/testify v1.8.2
|
||||||
github.com/testcontainers/testcontainers-go v0.13.0
|
github.com/testcontainers/testcontainers-go v0.13.0
|
||||||
github.com/valyala/fasthttp v1.34.0
|
github.com/valyala/fasthttp v1.34.0
|
||||||
go.uber.org/atomic v1.10.0
|
go.uber.org/atomic v1.10.0
|
||||||
go.uber.org/zap v1.23.0
|
go.uber.org/zap v1.24.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb // indirect
|
||||||
|
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect
|
||||||
|
git.frostfs.info/TrueCloudLab/hrw v1.2.0 // indirect
|
||||||
|
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
|
||||||
|
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect
|
||||||
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
|
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
|
||||||
github.com/Microsoft/go-winio v0.5.2 // indirect
|
github.com/Microsoft/go-winio v0.5.2 // indirect
|
||||||
github.com/Microsoft/hcsshim v0.9.2 // indirect
|
github.com/Microsoft/hcsshim v0.9.2 // indirect
|
||||||
github.com/TrueCloudLab/frostfs-contract v0.0.0-20221213081248-6c805c1b4e42 // indirect
|
|
||||||
github.com/TrueCloudLab/frostfs-crypto v0.5.0 // indirect
|
|
||||||
github.com/TrueCloudLab/hrw v1.1.0 // indirect
|
|
||||||
github.com/TrueCloudLab/rfc6979 v0.3.0 // indirect
|
|
||||||
github.com/TrueCloudLab/tzhash v1.7.0 // indirect
|
|
||||||
github.com/andybalholm/brotli v1.0.4 // indirect
|
github.com/andybalholm/brotli v1.0.4 // indirect
|
||||||
github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
|
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect
|
||||||
github.com/beorn7/perks v1.0.1 // indirect
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
github.com/btcsuite/btcd v0.22.0-beta // indirect
|
github.com/cenkalti/backoff/v4 v4.2.0 // indirect
|
||||||
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
|
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
|
||||||
github.com/containerd/cgroups v1.0.3 // indirect
|
github.com/containerd/cgroups v1.0.3 // indirect
|
||||||
github.com/containerd/containerd v1.6.2 // indirect
|
github.com/containerd/containerd v1.6.2 // indirect
|
||||||
github.com/coreos/go-semver v0.3.0 // indirect
|
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
|
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
|
||||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
|
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
|
||||||
github.com/docker/distribution v2.8.1+incompatible // indirect
|
github.com/docker/distribution v2.8.1+incompatible // indirect
|
||||||
github.com/docker/docker v20.10.14+incompatible // indirect
|
github.com/docker/docker v20.10.14+incompatible // indirect
|
||||||
github.com/docker/go-connections v0.4.0 // indirect
|
github.com/docker/go-connections v0.4.0 // indirect
|
||||||
github.com/docker/go-units v0.4.0 // indirect
|
github.com/docker/go-units v0.4.0 // indirect
|
||||||
github.com/fsnotify/fsnotify v1.4.9 // indirect
|
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||||
|
github.com/go-logr/logr v1.2.3 // indirect
|
||||||
|
github.com/go-logr/stdr v1.2.2 // indirect
|
||||||
github.com/gogo/protobuf v1.3.2 // indirect
|
github.com/gogo/protobuf v1.3.2 // indirect
|
||||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||||
github.com/golang/protobuf v1.5.2 // indirect
|
github.com/golang/protobuf v1.5.2 // indirect
|
||||||
github.com/google/uuid v1.3.0 // indirect
|
github.com/google/uuid v1.3.0 // indirect
|
||||||
github.com/gorilla/mux v1.8.0 // indirect
|
github.com/gorilla/mux v1.8.0 // indirect
|
||||||
github.com/gorilla/websocket v1.4.2 // indirect
|
github.com/gorilla/websocket v1.4.2 // indirect
|
||||||
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
|
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
|
||||||
|
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||||
|
github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
|
||||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||||
github.com/klauspost/compress v1.15.0 // indirect
|
github.com/klauspost/compress v1.15.0 // indirect
|
||||||
github.com/magiconair/properties v1.8.6 // indirect
|
github.com/magiconair/properties v1.8.7 // indirect
|
||||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||||
github.com/mitchellh/mapstructure v1.4.1 // indirect
|
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||||
github.com/moby/sys/mount v0.3.2 // indirect
|
github.com/moby/sys/mount v0.3.2 // indirect
|
||||||
github.com/moby/sys/mountinfo v0.6.1 // indirect
|
github.com/moby/sys/mountinfo v0.6.1 // indirect
|
||||||
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
|
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
|
||||||
github.com/morikuni/aec v1.0.0 // indirect
|
github.com/morikuni/aec v1.0.0 // indirect
|
||||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect
|
github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect
|
||||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262 // indirect
|
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect
|
||||||
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
|
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
|
||||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||||
github.com/opencontainers/image-spec v1.0.2 // indirect
|
github.com/opencontainers/image-spec v1.0.2 // indirect
|
||||||
github.com/opencontainers/runc v1.1.1 // indirect
|
github.com/opencontainers/runc v1.1.1 // indirect
|
||||||
github.com/pelletier/go-toml v1.9.3 // indirect
|
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
|
||||||
github.com/pkg/errors v0.9.1 // indirect
|
github.com/pkg/errors v0.9.1 // indirect
|
||||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
github.com/prometheus/client_model v0.2.0 // indirect
|
github.com/prometheus/client_model v0.2.0 // indirect
|
||||||
|
@ -75,26 +78,34 @@ require (
|
||||||
github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect
|
github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect
|
||||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||||
github.com/spf13/afero v1.6.0 // indirect
|
github.com/spf13/afero v1.9.3 // indirect
|
||||||
github.com/spf13/cast v1.3.1 // indirect
|
github.com/spf13/cast v1.5.0 // indirect
|
||||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||||
github.com/subosito/gotenv v1.2.0 // indirect
|
github.com/subosito/gotenv v1.4.2 // indirect
|
||||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
|
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
|
||||||
github.com/urfave/cli v1.22.5 // indirect
|
github.com/urfave/cli v1.22.5 // indirect
|
||||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||||
go.opencensus.io v0.23.0 // indirect
|
go.opencensus.io v0.24.0 // indirect
|
||||||
go.uber.org/multierr v1.8.0 // indirect
|
go.opentelemetry.io/otel v1.14.0 // indirect
|
||||||
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
|
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
|
||||||
golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
|
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect
|
||||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
|
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect
|
||||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
|
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect
|
||||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
go.opentelemetry.io/otel/sdk v1.14.0 // indirect
|
||||||
golang.org/x/text v0.3.7 // indirect
|
go.opentelemetry.io/otel/trace v1.14.0 // indirect
|
||||||
google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect
|
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
|
||||||
google.golang.org/grpc v1.48.0 // indirect
|
go.uber.org/multierr v1.9.0 // indirect
|
||||||
|
golang.org/x/crypto v0.4.0 // indirect
|
||||||
|
golang.org/x/exp v0.0.0-20221227203929-1b447090c38c // indirect
|
||||||
|
golang.org/x/net v0.7.0 // indirect
|
||||||
|
golang.org/x/sync v0.1.0 // indirect
|
||||||
|
golang.org/x/sys v0.5.0 // indirect
|
||||||
|
golang.org/x/term v0.5.0 // indirect
|
||||||
|
golang.org/x/text v0.7.0 // indirect
|
||||||
|
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
|
||||||
|
google.golang.org/grpc v1.53.0 // indirect
|
||||||
google.golang.org/protobuf v1.28.1 // indirect
|
google.golang.org/protobuf v1.28.1 // indirect
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||||
gopkg.in/ini.v1 v1.62.0 // indirect
|
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
)
|
)
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
//go:build integration
|
||||||
|
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
@ -13,14 +15,15 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/container"
|
containerv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/container/acl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/netmap"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
@ -35,21 +38,15 @@ type putResponse struct {
|
||||||
|
|
||||||
const (
|
const (
|
||||||
testContainerName = "friendly"
|
testContainerName = "friendly"
|
||||||
versionWithNativeNames = "0.27.5"
|
|
||||||
testListenAddress = "localhost:8082"
|
testListenAddress = "localhost:8082"
|
||||||
testHost = "http://" + testListenAddress
|
testHost = "http://" + testListenAddress
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestIntegration(t *testing.T) {
|
func TestIntegration(t *testing.T) {
|
||||||
rootCtx := context.Background()
|
rootCtx := context.Background()
|
||||||
aioImage := "nspccdev/neofs-aio-testcontainer:"
|
aioImage := "truecloudlab/frostfs-aio:"
|
||||||
versions := []string{
|
versions := []string{
|
||||||
"0.27.5",
|
"1.2.5", // frostfs-storage v0.36.0 RC
|
||||||
"0.28.1",
|
|
||||||
"0.29.0",
|
|
||||||
"0.30.0",
|
|
||||||
"0.32.0",
|
|
||||||
"latest",
|
|
||||||
}
|
}
|
||||||
key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb")
|
key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -95,11 +92,9 @@ func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID cid.ID, vers
|
||||||
url := testHost + "/upload/" + CID.String()
|
url := testHost + "/upload/" + CID.String()
|
||||||
makePutRequestAndCheck(ctx, t, p, CID, url)
|
makePutRequestAndCheck(ctx, t, p, CID, url)
|
||||||
|
|
||||||
if version >= versionWithNativeNames {
|
|
||||||
url = testHost + "/upload/" + testContainerName
|
url = testHost + "/upload/" + testContainerName
|
||||||
makePutRequestAndCheck(ctx, t, p, CID, url)
|
makePutRequestAndCheck(ctx, t, p, CID, url)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnrID cid.ID, url string) {
|
func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnrID cid.ID, url string) {
|
||||||
content := "content of file"
|
content := "content of file"
|
||||||
|
@ -221,12 +216,10 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
checkGetResponse(t, resp, content, attributes)
|
checkGetResponse(t, resp, content, attributes)
|
||||||
|
|
||||||
if version >= versionWithNativeNames {
|
|
||||||
resp, err = http.Get(testHost + "/get/" + testContainerName + "/" + id.String())
|
resp, err = http.Get(testHost + "/get/" + testContainerName + "/" + id.String())
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
checkGetResponse(t, resp, content, attributes)
|
checkGetResponse(t, resp, content, attributes)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
func checkGetResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) {
|
func checkGetResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) {
|
||||||
defer func() {
|
defer func() {
|
||||||
|
@ -275,12 +268,10 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
checkGetByAttrResponse(t, resp, content, expectedAttr)
|
checkGetByAttrResponse(t, resp, content, expectedAttr)
|
||||||
|
|
||||||
if version >= versionWithNativeNames {
|
|
||||||
resp, err = http.Get(testHost + "/get_by_attribute/" + testContainerName + "/" + keyAttr + "/" + valAttr)
|
resp, err = http.Get(testHost + "/get_by_attribute/" + testContainerName + "/" + keyAttr + "/" + valAttr)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
checkGetByAttrResponse(t, resp, content, expectedAttr)
|
checkGetByAttrResponse(t, resp, content, expectedAttr)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) {
|
func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) {
|
||||||
names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"}
|
names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"}
|
||||||
|
@ -294,11 +285,9 @@ func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID us
|
||||||
baseURL := testHost + "/zip/" + CID.String()
|
baseURL := testHost + "/zip/" + CID.String()
|
||||||
makeZipTest(t, baseURL, names, contents)
|
makeZipTest(t, baseURL, names, contents)
|
||||||
|
|
||||||
if version >= versionWithNativeNames {
|
|
||||||
baseURL = testHost + "/zip/" + testContainerName
|
baseURL = testHost + "/zip/" + testContainerName
|
||||||
makeZipTest(t, baseURL, names, contents)
|
makeZipTest(t, baseURL, names, contents)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
func makeZipTest(t *testing.T, baseURL string, names, contents []string) {
|
func makeZipTest(t *testing.T, baseURL string, names, contents []string) {
|
||||||
url := baseURL + "/zipfolder"
|
url := baseURL + "/zipfolder"
|
||||||
|
@ -405,11 +394,11 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o
|
||||||
|
|
||||||
container.SetCreationTime(&cnr, time.Now())
|
container.SetCreationTime(&cnr, time.Now())
|
||||||
|
|
||||||
if version >= versionWithNativeNames {
|
|
||||||
var domain container.Domain
|
var domain container.Domain
|
||||||
domain.SetName(testContainerName)
|
domain.SetName(testContainerName)
|
||||||
container.WriteDomain(&cnr, domain)
|
|
||||||
}
|
cnr.SetAttribute(containerv2.SysAttributeName, domain.Name())
|
||||||
|
cnr.SetAttribute(containerv2.SysAttributeZone, domain.Zone())
|
||||||
|
|
||||||
var waitPrm pool.WaitParams
|
var waitPrm pool.WaitParams
|
||||||
waitPrm.SetTimeout(15 * time.Second)
|
waitPrm.SetTimeout(15 * time.Second)
|
||||||
|
|
|
@ -3,7 +3,7 @@ package metrics
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -31,6 +31,16 @@ const (
|
||||||
methodCreateSession = "create_session"
|
methodCreateSession = "create_session"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// HealthStatus of the gate application.
|
||||||
|
type HealthStatus int32
|
||||||
|
|
||||||
|
const (
|
||||||
|
HealthStatusUndefined HealthStatus = 0
|
||||||
|
HealthStatusStarting HealthStatus = 1
|
||||||
|
HealthStatusReady HealthStatus = 2
|
||||||
|
HealthStatusShuttingDown HealthStatus = 3
|
||||||
|
)
|
||||||
|
|
||||||
type GateMetrics struct {
|
type GateMetrics struct {
|
||||||
stateMetrics
|
stateMetrics
|
||||||
poolMetricsCollector
|
poolMetricsCollector
|
||||||
|
@ -87,7 +97,7 @@ func (m stateMetrics) unregister() {
|
||||||
prometheus.Unregister(m.healthCheck)
|
prometheus.Unregister(m.healthCheck)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m stateMetrics) SetHealth(s int32) {
|
func (m stateMetrics) SetHealth(s HealthStatus) {
|
||||||
m.healthCheck.Set(float64(s))
|
m.healthCheck.Set(float64(s))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -5,7 +5,7 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FrostFSResolver represents virtual connection to the FrostFS network.
|
// FrostFSResolver represents virtual connection to the FrostFS network.
|
||||||
|
|
|
@ -6,9 +6,9 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/ns"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
|
@ -1,7 +1,41 @@
|
||||||
package response
|
package response
|
||||||
|
|
||||||
import "github.com/valyala/fasthttp"
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
|
sdkstatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||||
|
"github.com/valyala/fasthttp"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
func Error(r *fasthttp.RequestCtx, msg string, code int) {
|
func Error(r *fasthttp.RequestCtx, msg string, code int) {
|
||||||
r.Error(msg+"\n", code)
|
r.Error(msg+"\n", code)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func FormErrorResponse(message string, err error) (int, string, []zap.Field) {
|
||||||
|
var (
|
||||||
|
msg string
|
||||||
|
statusCode int
|
||||||
|
logFields []zap.Field
|
||||||
|
)
|
||||||
|
|
||||||
|
st := new(sdkstatus.ObjectAccessDenied)
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case errors.As(err, &st):
|
||||||
|
statusCode = fasthttp.StatusForbidden
|
||||||
|
reason := st.Reason()
|
||||||
|
msg = fmt.Sprintf("%s: %v: %s", message, err, reason)
|
||||||
|
logFields = append(logFields, zap.String("error_detail", reason))
|
||||||
|
case client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err):
|
||||||
|
statusCode = fasthttp.StatusNotFound
|
||||||
|
msg = "Not Found"
|
||||||
|
default:
|
||||||
|
statusCode = fasthttp.StatusBadRequest
|
||||||
|
msg = fmt.Sprintf("%s: %v", message, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return statusCode, msg, logFields
|
||||||
|
}
|
||||||
|
|
10
server.go
10
server.go
|
@ -7,8 +7,6 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"go.uber.org/zap"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
@ -57,11 +55,11 @@ func (s *server) UpdateCert(certFile, keyFile string) error {
|
||||||
return s.tlsProvider.UpdateCert(certFile, keyFile)
|
return s.tlsProvider.UpdateCert(certFile, keyFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newServer(ctx context.Context, serverInfo ServerInfo, logger *zap.Logger) *server {
|
func newServer(ctx context.Context, serverInfo ServerInfo) (*server, error) {
|
||||||
var lic net.ListenConfig
|
var lic net.ListenConfig
|
||||||
ln, err := lic.Listen(ctx, "tcp", serverInfo.Address)
|
ln, err := lic.Listen(ctx, "tcp", serverInfo.Address)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Fatal("could not prepare listener", zap.String("address", serverInfo.Address), zap.Error(err))
|
return nil, fmt.Errorf("could not prepare listener: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
tlsProvider := &certProvider{
|
tlsProvider := &certProvider{
|
||||||
|
@ -70,7 +68,7 @@ func newServer(ctx context.Context, serverInfo ServerInfo, logger *zap.Logger) *
|
||||||
|
|
||||||
if serverInfo.TLS.Enabled {
|
if serverInfo.TLS.Enabled {
|
||||||
if err = tlsProvider.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
|
if err = tlsProvider.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
|
||||||
logger.Fatal("failed to update cert", zap.Error(err))
|
return nil, fmt.Errorf("failed to update cert: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ln = tls.NewListener(ln, &tls.Config{
|
ln = tls.NewListener(ln, &tls.Config{
|
||||||
|
@ -82,7 +80,7 @@ func newServer(ctx context.Context, serverInfo ServerInfo, logger *zap.Logger) *
|
||||||
address: serverInfo.Address,
|
address: serverInfo.Address,
|
||||||
listener: ln,
|
listener: ln,
|
||||||
tlsProvider: tlsProvider,
|
tlsProvider: tlsProvider,
|
||||||
}
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *certProvider) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
|
func (p *certProvider) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||||
|
|
76
settings.go
76
settings.go
|
@ -3,13 +3,14 @@ package main
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
"runtime"
|
"runtime"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/resolver"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
|
@ -84,6 +85,7 @@ const (
|
||||||
cmdWallet = "wallet"
|
cmdWallet = "wallet"
|
||||||
cmdAddress = "address"
|
cmdAddress = "address"
|
||||||
cmdConfig = "config"
|
cmdConfig = "config"
|
||||||
|
cmdConfigDir = "config-dir"
|
||||||
cmdListenAddress = "listen_address"
|
cmdListenAddress = "listen_address"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -114,7 +116,8 @@ func settings() *viper.Viper {
|
||||||
|
|
||||||
flags.StringP(cmdWallet, "w", "", `path to the wallet`)
|
flags.StringP(cmdWallet, "w", "", `path to the wallet`)
|
||||||
flags.String(cmdAddress, "", `address of wallet account`)
|
flags.String(cmdAddress, "", `address of wallet account`)
|
||||||
flags.String(cmdConfig, "", "config path")
|
flags.StringArray(cmdConfig, nil, "config paths")
|
||||||
|
flags.String(cmdConfigDir, "", "config dir path")
|
||||||
flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout")
|
flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout")
|
||||||
flags.Duration(cfgStreamTimeout, defaultStreamTimeout, "gRPC individual message timeout")
|
flags.Duration(cfgStreamTimeout, defaultStreamTimeout, "gRPC individual message timeout")
|
||||||
flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout")
|
flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout")
|
||||||
|
@ -233,11 +236,9 @@ func settings() *viper.Viper {
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
if v.IsSet(cmdConfig) {
|
if err := readInConfig(v); err != nil {
|
||||||
if err := readConfig(v); err != nil {
|
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if peers != nil && len(*peers) > 0 {
|
if peers != nil && len(*peers) > 0 {
|
||||||
for i := range *peers {
|
for i := range *peers {
|
||||||
|
@ -250,17 +251,72 @@ func settings() *viper.Viper {
|
||||||
return v
|
return v
|
||||||
}
|
}
|
||||||
|
|
||||||
func readConfig(v *viper.Viper) error {
|
func readInConfig(v *viper.Viper) error {
|
||||||
cfgFileName := v.GetString(cmdConfig)
|
if v.IsSet(cmdConfig) {
|
||||||
cfgFile, err := os.Open(cfgFileName)
|
if err := readConfig(v); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if v.IsSet(cmdConfigDir) {
|
||||||
|
if err := readConfigDir(v); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readConfigDir(v *viper.Viper) error {
|
||||||
|
cfgSubConfigDir := v.GetString(cmdConfigDir)
|
||||||
|
entries, err := os.ReadDir(cfgSubConfigDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err = v.ReadConfig(cfgFile); err != nil {
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ext := path.Ext(entry.Name())
|
||||||
|
if ext != ".yaml" && ext != ".yml" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = mergeConfig(v, path.Join(cfgSubConfigDir, entry.Name())); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readConfig(v *viper.Viper) error {
|
||||||
|
for _, fileName := range v.GetStringSlice(cmdConfig) {
|
||||||
|
if err := mergeConfig(v, fileName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeConfig(v *viper.Viper, fileName string) error {
|
||||||
|
cfgFile, err := os.Open(fileName)
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return cfgFile.Close()
|
defer func() {
|
||||||
|
if errClose := cfgFile.Close(); errClose != nil {
|
||||||
|
panic(errClose)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if err = v.MergeConfig(cfgFile); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// newLogger constructs a zap.Logger instance for current application.
|
// newLogger constructs a zap.Logger instance for current application.
|
||||||
|
|
|
@ -7,7 +7,7 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -1,11 +1,13 @@
|
||||||
|
//go:build !integration
|
||||||
|
|
||||||
package tokens
|
package tokens
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
|
|
|
@ -3,29 +3,12 @@ package uploader
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-api-go/v2/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/utils"
|
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
var frostfsAttributeHeaderPrefixes = [...][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")}
|
|
||||||
|
|
||||||
func systemTranslator(key, prefix []byte) []byte {
|
|
||||||
// replace the specified prefix with `__NEOFS__`
|
|
||||||
key = bytes.Replace(key, prefix, []byte(utils.SystemAttributePrefix), 1)
|
|
||||||
|
|
||||||
// replace `-` with `_`
|
|
||||||
key = bytes.ReplaceAll(key, []byte("-"), []byte("_"))
|
|
||||||
|
|
||||||
// replace with uppercase
|
|
||||||
return bytes.ToUpper(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]string, error) {
|
func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]string, error) {
|
||||||
var err error
|
var err error
|
||||||
result := make(map[string]string)
|
result := make(map[string]string)
|
||||||
|
@ -45,13 +28,7 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st
|
||||||
// removing attribute prefix
|
// removing attribute prefix
|
||||||
clearKey := bytes.TrimPrefix(key, prefix)
|
clearKey := bytes.TrimPrefix(key, prefix)
|
||||||
|
|
||||||
// checks that it's a system NeoFS header
|
clearKey = utils.TransformIfSystem(clearKey)
|
||||||
for _, system := range frostfsAttributeHeaderPrefixes {
|
|
||||||
if bytes.HasPrefix(clearKey, system) {
|
|
||||||
clearKey = systemTranslator(clearKey, system)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checks that the attribute key is not empty
|
// checks that the attribute key is not empty
|
||||||
if len(clearKey) == 0 {
|
if len(clearKey) == 0 {
|
||||||
|
@ -77,69 +54,3 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st
|
||||||
|
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func prepareExpirationHeader(headers map[string]string, epochDurations *epochDurations, now time.Time) error {
|
|
||||||
expirationInEpoch := headers[object.SysAttributeExpEpoch]
|
|
||||||
|
|
||||||
if timeRFC3339, ok := headers[utils.ExpirationRFC3339Attr]; ok {
|
|
||||||
expTime, err := time.Parse(time.RFC3339, timeRFC3339)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, utils.ExpirationRFC3339Attr)
|
|
||||||
}
|
|
||||||
|
|
||||||
if expTime.Before(now) {
|
|
||||||
return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, utils.ExpirationRFC3339Attr)
|
|
||||||
}
|
|
||||||
updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
|
|
||||||
delete(headers, utils.ExpirationRFC3339Attr)
|
|
||||||
}
|
|
||||||
|
|
||||||
if timestamp, ok := headers[utils.ExpirationTimestampAttr]; ok {
|
|
||||||
value, err := strconv.ParseInt(timestamp, 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("couldn't parse value %s of header %s", timestamp, utils.ExpirationTimestampAttr)
|
|
||||||
}
|
|
||||||
expTime := time.Unix(value, 0)
|
|
||||||
|
|
||||||
if expTime.Before(now) {
|
|
||||||
return fmt.Errorf("value %s of header %s must be in the future", timestamp, utils.ExpirationTimestampAttr)
|
|
||||||
}
|
|
||||||
updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
|
|
||||||
delete(headers, utils.ExpirationTimestampAttr)
|
|
||||||
}
|
|
||||||
|
|
||||||
if duration, ok := headers[utils.ExpirationDurationAttr]; ok {
|
|
||||||
expDuration, err := time.ParseDuration(duration)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("couldn't parse value %s of header %s", duration, utils.ExpirationDurationAttr)
|
|
||||||
}
|
|
||||||
if expDuration <= 0 {
|
|
||||||
return fmt.Errorf("value %s of header %s must be positive", expDuration, utils.ExpirationDurationAttr)
|
|
||||||
}
|
|
||||||
updateExpirationHeader(headers, epochDurations, expDuration)
|
|
||||||
delete(headers, utils.ExpirationDurationAttr)
|
|
||||||
}
|
|
||||||
|
|
||||||
if expirationInEpoch != "" {
|
|
||||||
headers[object.SysAttributeExpEpoch] = expirationInEpoch
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func updateExpirationHeader(headers map[string]string, durations *epochDurations, expDuration time.Duration) {
|
|
||||||
epochDuration := uint64(durations.msPerBlock) * durations.blockPerEpoch
|
|
||||||
currentEpoch := durations.currentEpoch
|
|
||||||
numEpoch := uint64(expDuration.Milliseconds()) / epochDuration
|
|
||||||
|
|
||||||
if uint64(expDuration.Milliseconds())%epochDuration != 0 {
|
|
||||||
numEpoch++
|
|
||||||
}
|
|
||||||
|
|
||||||
expirationEpoch := uint64(math.MaxUint64)
|
|
||||||
if numEpoch < math.MaxUint64-currentEpoch {
|
|
||||||
expirationEpoch = currentEpoch + numEpoch
|
|
||||||
}
|
|
||||||
|
|
||||||
headers[object.SysAttributeExpEpoch] = strconv.FormatUint(expirationEpoch, 10)
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,13 +1,10 @@
|
||||||
|
//go:build !integration
|
||||||
|
|
||||||
package uploader
|
package uploader
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"math"
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-api-go/v2/object"
|
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/utils"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -28,8 +25,8 @@ func TestFilter(t *testing.T) {
|
||||||
t.Run("duplicate system keys error", func(t *testing.T) {
|
t.Run("duplicate system keys error", func(t *testing.T) {
|
||||||
req := &fasthttp.RequestHeader{}
|
req := &fasthttp.RequestHeader{}
|
||||||
req.DisableNormalizing()
|
req.DisableNormalizing()
|
||||||
req.Add("X-Attribute-Neofs-DupKey", "first-value")
|
req.Add("X-Attribute-System-DupKey", "first-value")
|
||||||
req.Add("X-Attribute-Neofs-DupKey", "second-value")
|
req.Add("X-Attribute-System-DupKey", "second-value")
|
||||||
_, err := filterHeaders(log, req)
|
_, err := filterHeaders(log, req)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
})
|
})
|
||||||
|
@ -37,16 +34,16 @@ func TestFilter(t *testing.T) {
|
||||||
req := &fasthttp.RequestHeader{}
|
req := &fasthttp.RequestHeader{}
|
||||||
req.DisableNormalizing()
|
req.DisableNormalizing()
|
||||||
|
|
||||||
req.Set("X-Attribute-Neofs-Expiration-Epoch1", "101")
|
req.Set("X-Attribute-System-Expiration-Epoch1", "101")
|
||||||
req.Set("X-Attribute-NEOFS-Expiration-Epoch2", "102")
|
req.Set("X-Attribute-SYSTEM-Expiration-Epoch2", "102")
|
||||||
req.Set("X-Attribute-neofs-Expiration-Epoch3", "103")
|
req.Set("X-Attribute-system-Expiration-Epoch3", "103")
|
||||||
req.Set("X-Attribute-MyAttribute", "value")
|
req.Set("X-Attribute-MyAttribute", "value")
|
||||||
|
|
||||||
expected := map[string]string{
|
expected := map[string]string{
|
||||||
"__NEOFS__EXPIRATION_EPOCH1": "101",
|
"__SYSTEM__EXPIRATION_EPOCH1": "101",
|
||||||
"MyAttribute": "value",
|
"MyAttribute": "value",
|
||||||
"__NEOFS__EXPIRATION_EPOCH3": "103",
|
"__SYSTEM__EXPIRATION_EPOCH3": "103",
|
||||||
"__NEOFS__EXPIRATION_EPOCH2": "102",
|
"__SYSTEM__EXPIRATION_EPOCH2": "102",
|
||||||
}
|
}
|
||||||
|
|
||||||
result, err := filterHeaders(log, req)
|
result, err := filterHeaders(log, req)
|
||||||
|
@ -54,157 +51,3 @@ func TestFilter(t *testing.T) {
|
||||||
|
|
||||||
require.Equal(t, expected, result)
|
require.Equal(t, expected, result)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPrepareExpirationHeader(t *testing.T) {
|
|
||||||
tomorrow := time.Now().Add(24 * time.Hour)
|
|
||||||
tomorrowUnix := tomorrow.Unix()
|
|
||||||
tomorrowUnixNano := tomorrow.UnixNano()
|
|
||||||
tomorrowUnixMilli := tomorrowUnixNano / 1e6
|
|
||||||
|
|
||||||
epoch := "100"
|
|
||||||
duration := "24h"
|
|
||||||
timestampSec := strconv.FormatInt(tomorrowUnix, 10)
|
|
||||||
timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10)
|
|
||||||
timestampNano := strconv.FormatInt(tomorrowUnixNano, 10)
|
|
||||||
|
|
||||||
defaultDurations := &epochDurations{
|
|
||||||
currentEpoch: 10,
|
|
||||||
msPerBlock: 1000,
|
|
||||||
blockPerEpoch: 101,
|
|
||||||
}
|
|
||||||
|
|
||||||
msPerBlock := defaultDurations.blockPerEpoch * uint64(defaultDurations.msPerBlock)
|
|
||||||
epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock
|
|
||||||
if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 {
|
|
||||||
epochPerDay++
|
|
||||||
}
|
|
||||||
|
|
||||||
defaultExpEpoch := strconv.FormatUint(defaultDurations.currentEpoch+epochPerDay, 10)
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
headers map[string]string
|
|
||||||
durations *epochDurations
|
|
||||||
err bool
|
|
||||||
expected map[string]string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "valid epoch",
|
|
||||||
headers: map[string]string{object.SysAttributeExpEpoch: epoch},
|
|
||||||
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid epoch, valid duration",
|
|
||||||
headers: map[string]string{
|
|
||||||
object.SysAttributeExpEpoch: epoch,
|
|
||||||
utils.ExpirationDurationAttr: duration,
|
|
||||||
},
|
|
||||||
durations: defaultDurations,
|
|
||||||
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid epoch, valid rfc3339",
|
|
||||||
headers: map[string]string{
|
|
||||||
object.SysAttributeExpEpoch: epoch,
|
|
||||||
utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339),
|
|
||||||
},
|
|
||||||
durations: defaultDurations,
|
|
||||||
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid epoch, valid timestamp sec",
|
|
||||||
headers: map[string]string{
|
|
||||||
object.SysAttributeExpEpoch: epoch,
|
|
||||||
utils.ExpirationTimestampAttr: timestampSec,
|
|
||||||
},
|
|
||||||
durations: defaultDurations,
|
|
||||||
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid epoch, valid timestamp milli",
|
|
||||||
headers: map[string]string{
|
|
||||||
object.SysAttributeExpEpoch: epoch,
|
|
||||||
utils.ExpirationTimestampAttr: timestampMilli,
|
|
||||||
},
|
|
||||||
durations: defaultDurations,
|
|
||||||
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid epoch, valid timestamp nano",
|
|
||||||
headers: map[string]string{
|
|
||||||
object.SysAttributeExpEpoch: epoch,
|
|
||||||
utils.ExpirationTimestampAttr: timestampNano,
|
|
||||||
},
|
|
||||||
durations: defaultDurations,
|
|
||||||
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid timestamp sec",
|
|
||||||
headers: map[string]string{utils.ExpirationTimestampAttr: timestampSec},
|
|
||||||
durations: defaultDurations,
|
|
||||||
expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid duration",
|
|
||||||
headers: map[string]string{utils.ExpirationDurationAttr: duration},
|
|
||||||
durations: defaultDurations,
|
|
||||||
expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid rfc3339",
|
|
||||||
headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)},
|
|
||||||
durations: defaultDurations,
|
|
||||||
expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "valid max uint 64",
|
|
||||||
headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)},
|
|
||||||
durations: &epochDurations{
|
|
||||||
currentEpoch: math.MaxUint64 - 1,
|
|
||||||
msPerBlock: defaultDurations.msPerBlock,
|
|
||||||
blockPerEpoch: defaultDurations.blockPerEpoch,
|
|
||||||
},
|
|
||||||
expected: map[string]string{object.SysAttributeExpEpoch: strconv.FormatUint(uint64(math.MaxUint64), 10)},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid timestamp sec",
|
|
||||||
headers: map[string]string{utils.ExpirationTimestampAttr: "abc"},
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid timestamp sec zero",
|
|
||||||
headers: map[string]string{utils.ExpirationTimestampAttr: "0"},
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid duration",
|
|
||||||
headers: map[string]string{utils.ExpirationDurationAttr: "1d"},
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid duration negative",
|
|
||||||
headers: map[string]string{utils.ExpirationDurationAttr: "-5h"},
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid rfc3339",
|
|
||||||
headers: map[string]string{utils.ExpirationRFC3339Attr: "abc"},
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid rfc3339 zero",
|
|
||||||
headers: map[string]string{utils.ExpirationRFC3339Attr: time.RFC3339},
|
|
||||||
err: true,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
err := prepareExpirationHeader(tc.headers, tc.durations, time.Now())
|
|
||||||
if tc.err {
|
|
||||||
require.Error(t, err)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, tc.expected, tc.headers)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -3,7 +3,7 @@ package uploader
|
||||||
import (
|
import (
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/uploader/multipart"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader/multipart"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -16,7 +16,8 @@ type MultipartFile interface {
|
||||||
|
|
||||||
func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartFile, error) {
|
func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartFile, error) {
|
||||||
// To have a custom buffer (3mb) the custom multipart reader is used.
|
// To have a custom buffer (3mb) the custom multipart reader is used.
|
||||||
// https://github.com/nspcc-dev/neofs-http-gw/issues/148
|
// Default reader uses 4KiB chunks, which slow down upload speed up to 400%
|
||||||
|
// https://github.com/golang/go/blob/91b9915d3f6f8cd2e9e9fda63f67772803adfa03/src/mime/multipart/multipart.go#L32
|
||||||
reader := multipart.NewReader(r, boundary)
|
reader := multipart.NewReader(r, boundary)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
//go:build !integration
|
||||||
|
|
||||||
package uploader
|
package uploader
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
|
@ -3,21 +3,20 @@ package uploader
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/resolver"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/response"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/tokens"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/utils"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "github.com/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
"go.uber.org/atomic"
|
"go.uber.org/atomic"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -38,12 +37,6 @@ type Uploader struct {
|
||||||
containerResolver *resolver.ContainerResolver
|
containerResolver *resolver.ContainerResolver
|
||||||
}
|
}
|
||||||
|
|
||||||
type epochDurations struct {
|
|
||||||
currentEpoch uint64
|
|
||||||
msPerBlock int64
|
|
||||||
blockPerEpoch uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// Settings stores reloading parameters, so it has to provide atomic getters and setters.
|
// Settings stores reloading parameters, so it has to provide atomic getters and setters.
|
||||||
type Settings struct {
|
type Settings struct {
|
||||||
defaultTimestamp atomic.Bool
|
defaultTimestamp atomic.Bool
|
||||||
|
@ -120,13 +113,6 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
|
||||||
response.Error(c, err.Error(), fasthttp.StatusBadRequest)
|
response.Error(c, err.Error(), fasthttp.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if needParseExpiration(filtered) {
|
|
||||||
epochDuration, err := getEpochDurations(c, u.pool)
|
|
||||||
if err != nil {
|
|
||||||
log.Error("could not get epoch durations from network info", zap.Error(err))
|
|
||||||
response.Error(c, "could not get epoch durations from network info: "+err.Error(), fasthttp.StatusBadRequest)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil {
|
if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil {
|
||||||
|
@ -137,12 +123,11 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = prepareExpirationHeader(filtered, epochDuration, now); err != nil {
|
if err = utils.PrepareExpirationHeader(c, u.pool, filtered, now); err != nil {
|
||||||
log.Error("could not parse expiration header", zap.Error(err))
|
log.Error("could not prepare expiration header", zap.Error(err))
|
||||||
response.Error(c, "could not parse expiration header: "+err.Error(), fasthttp.StatusBadRequest)
|
response.Error(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
attributes := make([]object.Attribute, 0, len(filtered))
|
attributes := make([]object.Attribute, 0, len(filtered))
|
||||||
// prepares attributes from filtered headers
|
// prepares attributes from filtered headers
|
||||||
|
@ -182,8 +167,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if idObj, err = u.pool.PutObject(u.appCtx, prm); err != nil {
|
if idObj, err = u.pool.PutObject(u.appCtx, prm); err != nil {
|
||||||
log.Error("could not store file in frostfs", zap.Error(err))
|
u.handlePutFrostFSErr(c, err)
|
||||||
response.Error(c, "could not store file in frostfs: "+err.Error(), fasthttp.StatusBadRequest)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -214,6 +198,14 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
|
||||||
c.Response.Header.SetContentType(jsonHeader)
|
c.Response.Header.SetContentType(jsonHeader)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (u *Uploader) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) {
|
||||||
|
statusCode, msg, additionalFields := response.FormErrorResponse("could not store file in frostfs", err)
|
||||||
|
logFields := append([]zap.Field{zap.Error(err)}, additionalFields...)
|
||||||
|
|
||||||
|
u.log.Error("could not store file in frostfs", logFields...)
|
||||||
|
response.Error(r, msg, statusCode)
|
||||||
|
}
|
||||||
|
|
||||||
func (u *Uploader) fetchOwnerAndBearerToken(ctx context.Context) (*user.ID, *bearer.Token) {
|
func (u *Uploader) fetchOwnerAndBearerToken(ctx context.Context) (*user.ID, *bearer.Token) {
|
||||||
if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil {
|
if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil {
|
||||||
issuer := bearer.ResolveIssuer(*tkn)
|
issuer := bearer.ResolveIssuer(*tkn)
|
||||||
|
@ -239,28 +231,3 @@ func (pr *putResponse) encode(w io.Writer) error {
|
||||||
enc.SetIndent("", "\t")
|
enc.SetIndent("", "\t")
|
||||||
return enc.Encode(pr)
|
return enc.Encode(pr)
|
||||||
}
|
}
|
||||||
|
|
||||||
func getEpochDurations(ctx context.Context, p *pool.Pool) (*epochDurations, error) {
|
|
||||||
networkInfo, err := p.NetworkInfo(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
res := &epochDurations{
|
|
||||||
currentEpoch: networkInfo.CurrentEpoch(),
|
|
||||||
msPerBlock: networkInfo.MsPerBlock(),
|
|
||||||
blockPerEpoch: networkInfo.EpochDuration(),
|
|
||||||
}
|
|
||||||
|
|
||||||
if res.blockPerEpoch == 0 {
|
|
||||||
return nil, fmt.Errorf("EpochDuration is empty")
|
|
||||||
}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func needParseExpiration(headers map[string]string) bool {
|
|
||||||
_, ok1 := headers[utils.ExpirationDurationAttr]
|
|
||||||
_, ok2 := headers[utils.ExpirationRFC3339Attr]
|
|
||||||
_, ok3 := headers[utils.ExpirationTimestampAttr]
|
|
||||||
return ok1 || ok2 || ok3
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,10 +1,250 @@
|
||||||
package utils
|
package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
"unicode"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
UserAttributeHeaderPrefix = "X-Attribute-"
|
UserAttributeHeaderPrefix = "X-Attribute-"
|
||||||
SystemAttributePrefix = "__NEOFS__"
|
|
||||||
|
|
||||||
ExpirationDurationAttr = SystemAttributePrefix + "EXPIRATION_DURATION"
|
|
||||||
ExpirationTimestampAttr = SystemAttributePrefix + "EXPIRATION_TIMESTAMP"
|
|
||||||
ExpirationRFC3339Attr = SystemAttributePrefix + "EXPIRATION_RFC3339"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
systemAttributePrefix = "__SYSTEM__"
|
||||||
|
|
||||||
|
// deprecated: use systemAttributePrefix
|
||||||
|
systemAttributePrefixNeoFS = "__NEOFS__"
|
||||||
|
)
|
||||||
|
|
||||||
|
type systemTransformer struct {
|
||||||
|
prefix string
|
||||||
|
backwardPrefix string
|
||||||
|
xAttrPrefixes [][]byte
|
||||||
|
}
|
||||||
|
|
||||||
|
var transformers = []systemTransformer{
|
||||||
|
{
|
||||||
|
prefix: systemAttributePrefix,
|
||||||
|
backwardPrefix: "System-",
|
||||||
|
xAttrPrefixes: [][]byte{[]byte("System-"), []byte("SYSTEM-"), []byte("system-")},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
prefix: systemAttributePrefixNeoFS,
|
||||||
|
backwardPrefix: "Neofs-",
|
||||||
|
xAttrPrefixes: [][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t systemTransformer) existsExpirationAttributes(headers map[string]string) bool {
|
||||||
|
_, ok0 := headers[t.expirationEpochAttr()]
|
||||||
|
_, ok1 := headers[t.expirationDurationAttr()]
|
||||||
|
_, ok2 := headers[t.expirationTimestampAttr()]
|
||||||
|
_, ok3 := headers[t.expirationRFC3339Attr()]
|
||||||
|
return ok0 || ok1 || ok2 || ok3
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t systemTransformer) expirationEpochAttr() string {
|
||||||
|
return t.prefix + "EXPIRATION_EPOCH"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t systemTransformer) expirationDurationAttr() string {
|
||||||
|
return t.prefix + "EXPIRATION_DURATION"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t systemTransformer) expirationTimestampAttr() string {
|
||||||
|
return t.prefix + "EXPIRATION_TIMESTAMP"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t systemTransformer) expirationRFC3339Attr() string {
|
||||||
|
return t.prefix + "EXPIRATION_RFC3339"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t systemTransformer) systemTranslator(key, prefix []byte) []byte {
|
||||||
|
// replace the specified prefix with system prefix
|
||||||
|
key = bytes.Replace(key, prefix, []byte(t.prefix), 1)
|
||||||
|
|
||||||
|
// replace `-` with `_`
|
||||||
|
key = bytes.ReplaceAll(key, []byte("-"), []byte("_"))
|
||||||
|
|
||||||
|
// replace with uppercase
|
||||||
|
return bytes.ToUpper(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t systemTransformer) transformIfSystem(key []byte) ([]byte, bool) {
|
||||||
|
// checks that it's a system FrostFS header
|
||||||
|
for _, system := range t.xAttrPrefixes {
|
||||||
|
if bytes.HasPrefix(key, system) {
|
||||||
|
return t.systemTranslator(key, system), true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return key, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// systemBackwardTranslator is used to convert headers looking like '__PREFIX__ATTR_NAME' to 'Prefix-Attr-Name'.
|
||||||
|
func (t systemTransformer) systemBackwardTranslator(key string) string {
|
||||||
|
// trim specified prefix '__PREFIX__'
|
||||||
|
key = strings.TrimPrefix(key, t.prefix)
|
||||||
|
|
||||||
|
var res strings.Builder
|
||||||
|
res.WriteString(t.backwardPrefix)
|
||||||
|
|
||||||
|
strs := strings.Split(key, "_")
|
||||||
|
for i, s := range strs {
|
||||||
|
s = title(strings.ToLower(s))
|
||||||
|
res.WriteString(s)
|
||||||
|
if i != len(strs)-1 {
|
||||||
|
res.WriteString("-")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t systemTransformer) backwardTransformIfSystem(key string) (string, bool) {
|
||||||
|
if strings.HasPrefix(key, t.prefix) {
|
||||||
|
return t.systemBackwardTranslator(key), true
|
||||||
|
}
|
||||||
|
|
||||||
|
return key, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func TransformIfSystem(key []byte) []byte {
|
||||||
|
for _, transformer := range transformers {
|
||||||
|
key, transformed := transformer.transformIfSystem(key)
|
||||||
|
if transformed {
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
func BackwardTransformIfSystem(key string) string {
|
||||||
|
for _, transformer := range transformers {
|
||||||
|
key, transformed := transformer.backwardTransformIfSystem(key)
|
||||||
|
if transformed {
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
func title(str string) string {
|
||||||
|
if str == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
r, size := utf8.DecodeRuneInString(str)
|
||||||
|
r0 := unicode.ToTitle(r)
|
||||||
|
return string(r0) + str[size:]
|
||||||
|
}
|
||||||
|
|
||||||
|
func PrepareExpirationHeader(ctx context.Context, p *pool.Pool, headers map[string]string, now time.Time) error {
|
||||||
|
formatsNum := 0
|
||||||
|
index := -1
|
||||||
|
for i, transformer := range transformers {
|
||||||
|
if transformer.existsExpirationAttributes(headers) {
|
||||||
|
formatsNum++
|
||||||
|
index = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
switch formatsNum {
|
||||||
|
case 0:
|
||||||
|
return nil
|
||||||
|
case 1:
|
||||||
|
epochDuration, err := GetEpochDurations(ctx, p)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("couldn't get epoch durations from network info: %w", err)
|
||||||
|
}
|
||||||
|
return transformers[index].prepareExpirationHeader(headers, epochDuration, now)
|
||||||
|
default:
|
||||||
|
return errors.New("both deprecated and new system attributes formats are used, please use only one")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t systemTransformer) prepareExpirationHeader(headers map[string]string, epochDurations *EpochDurations, now time.Time) error {
|
||||||
|
expirationInEpoch := headers[t.expirationEpochAttr()]
|
||||||
|
|
||||||
|
if timeRFC3339, ok := headers[t.expirationRFC3339Attr()]; ok {
|
||||||
|
expTime, err := time.Parse(time.RFC3339, timeRFC3339)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, t.expirationRFC3339Attr())
|
||||||
|
}
|
||||||
|
|
||||||
|
if expTime.Before(now) {
|
||||||
|
return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, t.expirationRFC3339Attr())
|
||||||
|
}
|
||||||
|
t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
|
||||||
|
delete(headers, t.expirationRFC3339Attr())
|
||||||
|
}
|
||||||
|
|
||||||
|
if timestamp, ok := headers[t.expirationTimestampAttr()]; ok {
|
||||||
|
value, err := strconv.ParseInt(timestamp, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("couldn't parse value %s of header %s", timestamp, t.expirationTimestampAttr())
|
||||||
|
}
|
||||||
|
expTime := time.Unix(value, 0)
|
||||||
|
|
||||||
|
if expTime.Before(now) {
|
||||||
|
return fmt.Errorf("value %s of header %s must be in the future", timestamp, t.expirationTimestampAttr())
|
||||||
|
}
|
||||||
|
t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
|
||||||
|
delete(headers, t.expirationTimestampAttr())
|
||||||
|
}
|
||||||
|
|
||||||
|
if duration, ok := headers[t.expirationDurationAttr()]; ok {
|
||||||
|
expDuration, err := time.ParseDuration(duration)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("couldn't parse value %s of header %s", duration, t.expirationDurationAttr())
|
||||||
|
}
|
||||||
|
if expDuration <= 0 {
|
||||||
|
return fmt.Errorf("value %s of header %s must be positive", expDuration, t.expirationDurationAttr())
|
||||||
|
}
|
||||||
|
t.updateExpirationHeader(headers, epochDurations, expDuration)
|
||||||
|
delete(headers, t.expirationDurationAttr())
|
||||||
|
}
|
||||||
|
|
||||||
|
if expirationInEpoch != "" {
|
||||||
|
expEpoch, err := strconv.ParseUint(expirationInEpoch, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("parse expiration epoch '%s': %w", expirationInEpoch, err)
|
||||||
|
}
|
||||||
|
if expEpoch < epochDurations.CurrentEpoch {
|
||||||
|
return fmt.Errorf("expiration epoch '%d' must be greater than current epoch '%d'", expEpoch, epochDurations.CurrentEpoch)
|
||||||
|
}
|
||||||
|
|
||||||
|
headers[t.expirationEpochAttr()] = expirationInEpoch
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t systemTransformer) updateExpirationHeader(headers map[string]string, durations *EpochDurations, expDuration time.Duration) {
|
||||||
|
epochDuration := uint64(durations.MsPerBlock) * durations.BlockPerEpoch
|
||||||
|
currentEpoch := durations.CurrentEpoch
|
||||||
|
numEpoch := uint64(expDuration.Milliseconds()) / epochDuration
|
||||||
|
|
||||||
|
if uint64(expDuration.Milliseconds())%epochDuration != 0 {
|
||||||
|
numEpoch++
|
||||||
|
}
|
||||||
|
|
||||||
|
expirationEpoch := uint64(math.MaxUint64)
|
||||||
|
if numEpoch < math.MaxUint64-currentEpoch {
|
||||||
|
expirationEpoch = currentEpoch + numEpoch
|
||||||
|
}
|
||||||
|
|
||||||
|
headers[t.expirationEpochAttr()] = strconv.FormatUint(expirationEpoch, 10)
|
||||||
|
}
|
||||||
|
|
189
utils/attributes_test.go
Normal file
189
utils/attributes_test.go
Normal file
|
@ -0,0 +1,189 @@
|
||||||
|
//go:build !integration
|
||||||
|
|
||||||
|
package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPrepareExpirationHeader(t *testing.T) {
|
||||||
|
tomorrow := time.Now().Add(24 * time.Hour)
|
||||||
|
tomorrowUnix := tomorrow.Unix()
|
||||||
|
tomorrowUnixNano := tomorrow.UnixNano()
|
||||||
|
tomorrowUnixMilli := tomorrowUnixNano / 1e6
|
||||||
|
|
||||||
|
epoch := "100"
|
||||||
|
duration := "24h"
|
||||||
|
timestampSec := strconv.FormatInt(tomorrowUnix, 10)
|
||||||
|
timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10)
|
||||||
|
timestampNano := strconv.FormatInt(tomorrowUnixNano, 10)
|
||||||
|
|
||||||
|
defaultDurations := &EpochDurations{
|
||||||
|
CurrentEpoch: 10,
|
||||||
|
MsPerBlock: 1000,
|
||||||
|
BlockPerEpoch: 101,
|
||||||
|
}
|
||||||
|
|
||||||
|
msPerBlock := defaultDurations.BlockPerEpoch * uint64(defaultDurations.MsPerBlock)
|
||||||
|
epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock
|
||||||
|
if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 {
|
||||||
|
epochPerDay++
|
||||||
|
}
|
||||||
|
|
||||||
|
defaultExpEpoch := strconv.FormatUint(defaultDurations.CurrentEpoch+epochPerDay, 10)
|
||||||
|
|
||||||
|
for _, transformer := range transformers {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
headers map[string]string
|
||||||
|
durations *EpochDurations
|
||||||
|
err bool
|
||||||
|
expected map[string]string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "valid epoch",
|
||||||
|
headers: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||||
|
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||||
|
durations: defaultDurations,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid epoch, valid duration",
|
||||||
|
headers: map[string]string{
|
||||||
|
transformer.expirationEpochAttr(): epoch,
|
||||||
|
transformer.expirationDurationAttr(): duration,
|
||||||
|
},
|
||||||
|
durations: defaultDurations,
|
||||||
|
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid epoch, valid rfc3339",
|
||||||
|
headers: map[string]string{
|
||||||
|
transformer.expirationEpochAttr(): epoch,
|
||||||
|
transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339),
|
||||||
|
},
|
||||||
|
durations: defaultDurations,
|
||||||
|
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid epoch, valid timestamp sec",
|
||||||
|
headers: map[string]string{
|
||||||
|
transformer.expirationEpochAttr(): epoch,
|
||||||
|
transformer.expirationTimestampAttr(): timestampSec,
|
||||||
|
},
|
||||||
|
durations: defaultDurations,
|
||||||
|
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid epoch, valid timestamp milli",
|
||||||
|
headers: map[string]string{
|
||||||
|
transformer.expirationEpochAttr(): epoch,
|
||||||
|
transformer.expirationTimestampAttr(): timestampMilli,
|
||||||
|
},
|
||||||
|
durations: defaultDurations,
|
||||||
|
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid epoch, valid timestamp nano",
|
||||||
|
headers: map[string]string{
|
||||||
|
transformer.expirationEpochAttr(): epoch,
|
||||||
|
transformer.expirationTimestampAttr(): timestampNano,
|
||||||
|
},
|
||||||
|
durations: defaultDurations,
|
||||||
|
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid timestamp sec",
|
||||||
|
headers: map[string]string{transformer.expirationTimestampAttr(): timestampSec},
|
||||||
|
durations: defaultDurations,
|
||||||
|
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid duration",
|
||||||
|
headers: map[string]string{transformer.expirationDurationAttr(): duration},
|
||||||
|
durations: defaultDurations,
|
||||||
|
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid rfc3339",
|
||||||
|
headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)},
|
||||||
|
durations: defaultDurations,
|
||||||
|
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid max uint 64",
|
||||||
|
headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)},
|
||||||
|
durations: &EpochDurations{
|
||||||
|
CurrentEpoch: math.MaxUint64 - 1,
|
||||||
|
MsPerBlock: defaultDurations.MsPerBlock,
|
||||||
|
BlockPerEpoch: defaultDurations.BlockPerEpoch,
|
||||||
|
},
|
||||||
|
expected: map[string]string{transformer.expirationEpochAttr(): strconv.FormatUint(uint64(math.MaxUint64), 10)},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid timestamp sec",
|
||||||
|
headers: map[string]string{transformer.expirationTimestampAttr(): "abc"},
|
||||||
|
err: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid timestamp sec zero",
|
||||||
|
headers: map[string]string{transformer.expirationTimestampAttr(): "0"},
|
||||||
|
err: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid duration",
|
||||||
|
headers: map[string]string{transformer.expirationDurationAttr(): "1d"},
|
||||||
|
err: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid duration negative",
|
||||||
|
headers: map[string]string{transformer.expirationDurationAttr(): "-5h"},
|
||||||
|
err: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid rfc3339",
|
||||||
|
headers: map[string]string{transformer.expirationRFC3339Attr(): "abc"},
|
||||||
|
err: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid rfc3339 zero",
|
||||||
|
headers: map[string]string{transformer.expirationRFC3339Attr(): time.RFC3339},
|
||||||
|
err: true,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
err := transformer.prepareExpirationHeader(tc.headers, tc.durations, time.Now())
|
||||||
|
if tc.err {
|
||||||
|
require.Error(t, err)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, tc.expected, tc.headers)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSystemBackwardTranslator(t *testing.T) {
|
||||||
|
input := []string{
|
||||||
|
"__SYSTEM__EXPIRATION_EPOCH",
|
||||||
|
"__SYSTEM__RANDOM_ATTR",
|
||||||
|
"__NEOFS__EXPIRATION_EPOCH",
|
||||||
|
"__NEOFS__RANDOM_ATTR",
|
||||||
|
}
|
||||||
|
expected := []string{
|
||||||
|
"System-Expiration-Epoch",
|
||||||
|
"System-Random-Attr",
|
||||||
|
"Neofs-Expiration-Epoch",
|
||||||
|
"Neofs-Random-Attr",
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, str := range input {
|
||||||
|
res := BackwardTransformIfSystem(str)
|
||||||
|
require.Equal(t, expected[i], res)
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,9 +1,9 @@
|
||||||
package utils
|
package utils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/resolver"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
"github.com/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -2,9 +2,11 @@ package utils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
"github.com/TrueCloudLab/frostfs-http-gw/resolver"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||||
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetContainerID decode container id, if it's not a valid container id
|
// GetContainerID decode container id, if it's not a valid container id
|
||||||
|
@ -17,3 +19,27 @@ func GetContainerID(ctx context.Context, containerID string, resolver *resolver.
|
||||||
}
|
}
|
||||||
return cnrID, err
|
return cnrID, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type EpochDurations struct {
|
||||||
|
CurrentEpoch uint64
|
||||||
|
MsPerBlock int64
|
||||||
|
BlockPerEpoch uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetEpochDurations(ctx context.Context, p *pool.Pool) (*EpochDurations, error) {
|
||||||
|
networkInfo, err := p.NetworkInfo(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
res := &EpochDurations{
|
||||||
|
CurrentEpoch: networkInfo.CurrentEpoch(),
|
||||||
|
MsPerBlock: networkInfo.MsPerBlock(),
|
||||||
|
BlockPerEpoch: networkInfo.EpochDuration(),
|
||||||
|
}
|
||||||
|
|
||||||
|
if res.BlockPerEpoch == 0 {
|
||||||
|
return nil, fmt.Errorf("EpochDuration is empty")
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
Loading…
Add table
Reference in a new issue