forked from TrueCloudLab/frostfs-http-gw
Compare commits
7 commits
master
...
support/v0
| Author | SHA1 | Date | |
|---|---|---|---|
| 71b40185a9 | |||
|
e1de48ff67 | ||
|
|
14377dff9a | ||
| af73636125 | |||
|
|
ec921e75dd | ||
|
|
f27e2e10e1 | ||
| b4c1600fd3 |
64 changed files with 1266 additions and 1944 deletions
2
.github/CODEOWNERS
vendored
2
.github/CODEOWNERS
vendored
|
|
@ -1 +1 @@
|
|||
* @alexvanin @KirillovDenis
|
||||
* @alexvanin @masterSplinter01 @KirillovDenis
|
||||
|
|
|
|||
12
.github/ISSUE_TEMPLATE/bug_report.md
vendored
12
.github/ISSUE_TEMPLATE/bug_report.md
vendored
|
|
@ -2,7 +2,7 @@
|
|||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: community, triage, bug
|
||||
labels: community, triage
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
|
@ -18,17 +18,17 @@ assignees: ''
|
|||
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
|
||||
|
||||
## Possible Solution
|
||||
<!-- Not obligatory
|
||||
If no reason/fix/additions for the bug can be suggested,
|
||||
uncomment the following phrase:
|
||||
|
||||
<-- No fix can be suggested by a QA engineer. Further solutions shall be up to developers. -->
|
||||
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
|
||||
<!--- or ideas how to implement the addition or change -->
|
||||
|
||||
## Steps to Reproduce (for bugs)
|
||||
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
||||
<!--- reproduce this bug. -->
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
4.
|
||||
|
||||
## Context
|
||||
<!--- How has this issue affected you? What are you trying to accomplish? -->
|
||||
|
|
|
|||
16
.github/ISSUE_TEMPLATE/feature_request.md
vendored
16
.github/ISSUE_TEMPLATE/feature_request.md
vendored
|
|
@ -7,14 +7,14 @@ assignees: ''
|
|||
|
||||
---
|
||||
|
||||
## Is your feature request related to a problem? Please describe.
|
||||
<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
## Describe the solution you'd like
|
||||
<!--- A clear and concise description of what you want to happen. -->
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
## Describe alternatives you've considered
|
||||
<!--- A clear and concise description of any alternative solutions or features you've considered. -->
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
## Additional context
|
||||
<!--- Add any other context or screenshots about the feature request here. -->
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
||||
|
|
|
|||
197
.github/logo.svg
vendored
197
.github/logo.svg
vendored
|
|
@ -1,70 +1,129 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Generator: Adobe Illustrator 25.0.1, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||
<svg version="1.1" id="Слой_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||
viewBox="0 0 184.2 51.8" style="enable-background:new 0 0 184.2 51.8;" xml:space="preserve">
|
||||
<style type="text/css">
|
||||
.st0{display:none;}
|
||||
.st1{display:inline;}
|
||||
.st2{fill:#01E397;}
|
||||
.st3{display:inline;fill:#010032;}
|
||||
.st4{display:inline;fill:#00E599;}
|
||||
.st5{display:inline;fill:#00AF92;}
|
||||
.st6{fill:#00C3E5;}
|
||||
</style>
|
||||
<g id="Layer_2">
|
||||
<g id="Layer_1-2" class="st0">
|
||||
<g class="st1">
|
||||
<path class="st2" d="M146.6,18.3v7.2h10.9V29h-10.9v10.7h-4V14.8h18v3.5H146.6z"/>
|
||||
<path class="st2" d="M180,15.7c1.7,0.9,3,2.2,4,3.8l-3,2.7c-0.6-1.3-1.5-2.4-2.6-3.3c-1.3-0.7-2.8-1-4.3-1
|
||||
c-1.4-0.1-2.8,0.3-4,1.1c-0.9,0.5-1.5,1.5-1.4,2.6c0,1,0.5,1.9,1.4,2.4c1.5,0.8,3.2,1.3,4.9,1.5c1.9,0.3,3.7,0.8,5.4,1.6
|
||||
c1.2,0.5,2.2,1.3,2.9,2.3c0.6,1,1,2.2,0.9,3.4c0,1.4-0.5,2.7-1.3,3.8c-0.9,1.2-2.1,2.1-3.5,2.6c-1.7,0.6-3.4,0.9-5.2,0.8
|
||||
c-5,0-8.6-1.6-10.7-5l2.9-2.8c0.7,1.4,1.8,2.5,3.1,3.3c1.5,0.7,3.1,1.1,4.7,1c1.5,0.1,2.9-0.2,4.2-0.9c0.9-0.5,1.5-1.5,1.5-2.6
|
||||
c0-0.9-0.5-1.8-1.3-2.2c-1.5-0.7-3.1-1.2-4.8-1.5c-1.9-0.3-3.7-0.8-5.5-1.5c-1.2-0.5-2.2-1.4-3-2.4c-0.6-1-1-2.2-0.9-3.4
|
||||
c0-1.4,0.4-2.7,1.2-3.8c0.8-1.2,2-2.2,3.3-2.8c1.6-0.7,3.4-1.1,5.2-1C176.1,14.3,178.2,14.8,180,15.7z"/>
|
||||
</g>
|
||||
<path class="st3" d="M73.3,16.3c1.9,1.9,2.9,4.5,2.7,7.1v15.9h-4V24.8c0-2.6-0.5-4.5-1.6-5.7c-1.2-1.2-2.8-1.8-4.5-1.7
|
||||
c-1.3,0-2.5,0.3-3.7,0.8c-1.2,0.7-2.2,1.7-2.9,2.9c-0.8,1.5-1.1,3.2-1.1,4.9v13.3h-4V15.1l3.6,1.5v1.7c0.8-1.5,2.1-2.6,3.6-3.3
|
||||
c1.5-0.8,3.2-1.2,4.9-1.1C68.9,13.8,71.3,14.7,73.3,16.3z"/>
|
||||
<path class="st3" d="M104.4,28.3H85.6c0.1,2.2,1,4.3,2.5,5.9c1.5,1.4,3.5,2.2,5.6,2.1c1.6,0.1,3.2-0.2,4.6-0.9
|
||||
c1.1-0.6,2-1.6,2.5-2.8l3.3,1.8c-0.9,1.7-2.3,3.1-4,4c-2,1-4.2,1.5-6.4,1.4c-3.7,0-6.7-1.1-8.8-3.4s-3.2-5.5-3.2-9.6s1-7.2,3-9.5
|
||||
s5-3.4,8.7-3.4c2.1-0.1,4.2,0.5,6.1,1.5c1.6,1,3,2.5,3.8,4.2c0.9,1.8,1.3,3.9,1.3,5.9C104.6,26.4,104.6,27.4,104.4,28.3z
|
||||
M88.1,19.3c-1.4,1.5-2.2,3.4-2.4,5.5h15.1c-0.2-2-1-3.9-2.3-5.5c-1.4-1.3-3.2-2-5.1-1.9C91.5,17.3,89.6,18,88.1,19.3z"/>
|
||||
<path class="st3" d="M131,17.3c2.2,2.3,3.2,5.5,3.2,9.5s-1,7.3-3.2,9.6s-5.1,3.4-8.8,3.4s-6.7-1.1-8.9-3.4s-3.2-5.5-3.2-9.6
|
||||
s1.1-7.2,3.2-9.5s5.1-3.4,8.9-3.4S128.9,15,131,17.3z M116.2,19.9c-1.5,2-2.2,4.4-2.1,6.9c-0.2,2.5,0.6,5,2.1,7
|
||||
c1.5,1.7,3.7,2.7,6,2.6c2.3,0.1,4.4-0.9,5.9-2.6c1.5-2,2.3-4.5,2.1-7c0.1-2.5-0.6-4.9-2.1-6.9c-1.5-1.7-3.6-2.7-5.9-2.6
|
||||
C119.9,17.2,117.7,18.2,116.2,19.9z"/>
|
||||
<polygon class="st4" points="0,9.1 0,43.7 22.5,51.8 22.5,16.9 46.8,7.9 24.8,0 "/>
|
||||
<polygon class="st5" points="24.3,17.9 24.3,36.8 46.8,44.9 46.8,9.6 "/>
|
||||
</g>
|
||||
<g>
|
||||
<g>
|
||||
<path class="st6" d="M41.6,17.5H28.2v6.9h10.4v3.3H28.2v10.2h-3.9V14.2h17.2V17.5z"/>
|
||||
<path class="st6" d="M45.8,37.9v-18h3.3l0.4,3.2c0.5-1.2,1.2-2.1,2.1-2.7c0.9-0.6,2.1-0.9,3.5-0.9c0.4,0,0.7,0,1.1,0.1
|
||||
c0.4,0.1,0.7,0.2,0.9,0.3l-0.5,3.4c-0.3-0.1-0.6-0.2-0.9-0.2C55.4,23,54.9,23,54.4,23c-0.7,0-1.5,0.2-2.2,0.6
|
||||
c-0.7,0.4-1.3,1-1.8,1.8s-0.7,1.8-0.7,3v9.5H45.8z"/>
|
||||
<path class="st6" d="M68.6,19.6c1.8,0,3.3,0.4,4.6,1.1c1.3,0.7,2.4,1.8,3.1,3.2s1.1,3.1,1.1,5c0,1.9-0.4,3.6-1.1,5
|
||||
c-0.8,1.4-1.8,2.5-3.1,3.2c-1.3,0.7-2.9,1.1-4.6,1.1s-3.3-0.4-4.6-1.1c-1.3-0.7-2.4-1.8-3.2-3.2c-0.8-1.4-1.2-3.1-1.2-5
|
||||
c0-1.9,0.4-3.6,1.2-5s1.8-2.5,3.2-3.2C65.3,19.9,66.8,19.6,68.6,19.6z M68.6,22.6c-1.1,0-2,0.2-2.8,0.7c-0.8,0.5-1.3,1.2-1.7,2.1
|
||||
s-0.6,2.1-0.6,3.5c0,1.3,0.2,2.5,0.6,3.4s1,1.7,1.7,2.2s1.7,0.7,2.8,0.7c1.1,0,2-0.2,2.7-0.7c0.7-0.5,1.3-1.2,1.7-2.2
|
||||
s0.6-2.1,0.6-3.4c0-1.4-0.2-2.5-0.6-3.5s-1-1.6-1.7-2.1C70.6,22.8,69.6,22.6,68.6,22.6z"/>
|
||||
<path class="st6" d="M89.2,38.3c-1.8,0-3.4-0.3-4.9-1c-1.5-0.7-2.7-1.7-3.5-3l2.7-2.3c0.5,1,1.3,1.8,2.3,2.4
|
||||
c1,0.6,2.2,0.9,3.6,0.9c1.1,0,2-0.2,2.6-0.6c0.6-0.4,1-0.9,1-1.6c0-0.5-0.2-0.9-0.5-1.2s-0.9-0.6-1.7-0.8l-3.8-0.8
|
||||
c-1.9-0.4-3.3-1-4.1-1.9c-0.8-0.9-1.2-1.9-1.2-3.3c0-1,0.3-1.9,0.9-2.7c0.6-0.8,1.4-1.5,2.5-2s2.5-0.8,4-0.8c1.8,0,3.3,0.3,4.6,1
|
||||
c1.3,0.6,2.2,1.5,2.9,2.7l-2.7,2.2c-0.5-1-1.1-1.7-2-2.1c-0.9-0.5-1.8-0.7-2.8-0.7c-0.8,0-1.4,0.1-2,0.3c-0.6,0.2-1,0.5-1.3,0.8
|
||||
c-0.3,0.3-0.4,0.7-0.4,1.2c0,0.5,0.2,0.9,0.5,1.3s1,0.6,1.9,0.8l4.1,0.9c1.7,0.3,2.9,0.9,3.7,1.7c0.7,0.8,1.1,1.8,1.1,2.9
|
||||
c0,1.2-0.3,2.2-0.9,3c-0.6,0.9-1.5,1.6-2.6,2C92.1,38.1,90.7,38.3,89.2,38.3z"/>
|
||||
<path class="st6" d="M112.8,19.9v3H99.3v-3H112.8z M106.6,14.6v17.9c0,0.9,0.2,1.5,0.7,1.9c0.5,0.4,1.1,0.6,1.9,0.6
|
||||
c0.6,0,1.2-0.1,1.7-0.3c0.5-0.2,0.9-0.5,1.3-0.8l0.9,2.8c-0.6,0.5-1.2,0.9-2,1.1c-0.8,0.3-1.7,0.4-2.7,0.4c-1,0-2-0.2-2.8-0.5
|
||||
s-1.5-0.9-2-1.6c-0.5-0.8-0.7-1.7-0.8-3V15.7L106.6,14.6z"/>
|
||||
<path d="M137.9,17.5h-13.3v6.9h10.4v3.3h-10.4v10.2h-3.9V14.2h17.2V17.5z"/>
|
||||
<path d="M150.9,13.8c2.1,0,4,0.4,5.5,1.2c1.6,0.8,2.9,2,4,3.5l-2.6,2.5c-0.9-1.4-1.9-2.4-3.1-3c-1.1-0.6-2.5-0.9-4-0.9
|
||||
c-1.2,0-2.1,0.2-2.8,0.5c-0.7,0.3-1.3,0.7-1.6,1.2c-0.3,0.5-0.5,1.1-0.5,1.7c0,0.7,0.3,1.4,0.8,1.9c0.5,0.6,1.5,1,2.9,1.3
|
||||
l4.8,1.1c2.3,0.5,3.9,1.3,4.9,2.3c1,1,1.4,2.3,1.4,3.9c0,1.5-0.4,2.7-1.2,3.8c-0.8,1.1-1.9,1.9-3.3,2.5s-3.1,0.9-5,0.9
|
||||
c-1.7,0-3.2-0.2-4.5-0.6c-1.3-0.4-2.5-1-3.5-1.8c-1-0.7-1.8-1.6-2.5-2.6l2.7-2.7c0.5,0.8,1.1,1.6,1.9,2.2
|
||||
c0.8,0.7,1.7,1.2,2.7,1.5c1,0.4,2.2,0.5,3.4,0.5c1.1,0,2.1-0.1,2.9-0.4c0.8-0.3,1.4-0.7,1.8-1.2c0.4-0.5,0.6-1.1,0.6-1.9
|
||||
c0-0.7-0.2-1.3-0.7-1.8c-0.5-0.5-1.3-0.9-2.6-1.2l-5.2-1.2c-1.4-0.3-2.6-0.8-3.6-1.3c-0.9-0.6-1.6-1.3-2.1-2.1s-0.7-1.8-0.7-2.8
|
||||
c0-1.3,0.4-2.6,1.1-3.7c0.7-1.1,1.8-2,3.2-2.6C147.3,14.1,148.9,13.8,150.9,13.8z"/>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<svg
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:svg="http://www.w3.org/2000/svg"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
sodipodi:docname="logo_fs.svg"
|
||||
inkscape:version="1.0 (4035a4fb49, 2020-05-01)"
|
||||
id="svg57"
|
||||
version="1.1"
|
||||
viewBox="0 0 105 25"
|
||||
height="25mm"
|
||||
width="105mm">
|
||||
<defs
|
||||
id="defs51">
|
||||
<clipPath
|
||||
clipPathUnits="userSpaceOnUse"
|
||||
id="clipPath434">
|
||||
<path
|
||||
d="M 0,0 H 1366 V 768 H 0 Z"
|
||||
id="path432" />
|
||||
</clipPath>
|
||||
</defs>
|
||||
<sodipodi:namedview
|
||||
inkscape:window-maximized="0"
|
||||
inkscape:window-y="0"
|
||||
inkscape:window-x="130"
|
||||
inkscape:window-height="1040"
|
||||
inkscape:window-width="1274"
|
||||
height="50mm"
|
||||
units="mm"
|
||||
showgrid="false"
|
||||
inkscape:document-rotation="0"
|
||||
inkscape:current-layer="layer1"
|
||||
inkscape:document-units="mm"
|
||||
inkscape:cy="344.49897"
|
||||
inkscape:cx="468.64708"
|
||||
inkscape:zoom="0.7"
|
||||
inkscape:pageshadow="2"
|
||||
inkscape:pageopacity="0.0"
|
||||
borderopacity="1.0"
|
||||
bordercolor="#666666"
|
||||
pagecolor="#ffffff"
|
||||
id="base" />
|
||||
<metadata
|
||||
id="metadata54">
|
||||
<rdf:RDF>
|
||||
<cc:Work
|
||||
rdf:about="">
|
||||
<dc:format>image/svg+xml</dc:format>
|
||||
<dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
||||
<dc:title></dc:title>
|
||||
</cc:Work>
|
||||
</rdf:RDF>
|
||||
</metadata>
|
||||
<g
|
||||
id="layer1"
|
||||
inkscape:groupmode="layer"
|
||||
inkscape:label="Layer 1">
|
||||
<g
|
||||
id="g424"
|
||||
transform="matrix(0.35277777,0,0,-0.35277777,63.946468,10.194047)">
|
||||
<path
|
||||
d="m 0,0 v -8.093 h 12.287 v -3.94 H 0 V -24.067 H -4.534 V 3.898 H 15.677 V 0 Z"
|
||||
style="fill:#00e396;fill-opacity:1;fill-rule:nonzero;stroke:none"
|
||||
id="path426" />
|
||||
</g>
|
||||
<g
|
||||
transform="matrix(0.35277777,0,0,-0.35277777,-315.43002,107.34005)"
|
||||
id="g428">
|
||||
<g
|
||||
id="g430"
|
||||
clip-path="url(#clipPath434)">
|
||||
<g
|
||||
id="g436"
|
||||
transform="translate(1112.874,278.2981)">
|
||||
<path
|
||||
d="M 0,0 C 1.822,-0.932 3.354,-2.359 4.597,-4.28 L 1.165,-7.373 c -0.791,1.695 -1.779,2.924 -2.966,3.686 -1.186,0.763 -2.768,1.145 -4.745,1.145 -1.949,0 -3.461,-0.389 -4.534,-1.166 -1.074,-0.777 -1.61,-1.772 -1.61,-2.987 0,-1.13 0.523,-2.027 1.568,-2.69 1.045,-0.664 2.909,-1.236 5.593,-1.716 2.514,-0.452 4.512,-1.024 5.995,-1.716 1.483,-0.693 2.564,-1.554 3.242,-2.585 0.677,-1.031 1.016,-2.309 1.016,-3.834 0,-1.639 -0.466,-3.079 -1.398,-4.322 -0.932,-1.243 -2.239,-2.197 -3.919,-2.86 -1.681,-0.664 -3.623,-0.996 -5.826,-0.996 -5.678,0 -9.689,1.892 -12.033,5.678 l 3.178,3.178 c 0.903,-1.695 2.068,-2.939 3.495,-3.729 1.426,-0.791 3.199,-1.186 5.318,-1.186 2.005,0 3.58,0.345 4.724,1.038 1.144,0.692 1.716,1.674 1.716,2.945 0,1.017 -0.516,1.835 -1.547,2.457 -1.031,0.621 -2.832,1.172 -5.402,1.653 -2.571,0.479 -4.618,1.073 -6.143,1.779 -1.526,0.706 -2.635,1.582 -3.326,2.627 -0.693,1.045 -1.039,2.316 -1.039,3.813 0,1.582 0.438,3.023 1.314,4.322 0.875,1.299 2.14,2.33 3.792,3.093 1.653,0.763 3.58,1.144 5.783,1.144 C -4.018,1.398 -1.822,0.932 0,0"
|
||||
style="fill:#00e396;fill-opacity:1;fill-rule:nonzero;stroke:none"
|
||||
id="path438" />
|
||||
</g>
|
||||
<g
|
||||
id="g440"
|
||||
transform="translate(993.0239,277.5454)">
|
||||
<path
|
||||
d="m 0,0 c 2.054,-1.831 3.083,-4.465 3.083,-7.902 v -17.935 h -4.484 v 16.366 c 0,2.914 -0.626,5.024 -1.877,6.332 -1.253,1.308 -2.924,1.962 -5.016,1.962 -1.495,0 -2.896,-0.327 -4.204,-0.981 -1.308,-0.654 -2.381,-1.719 -3.222,-3.194 -0.841,-1.477 -1.261,-3.335 -1.261,-5.576 v -14.909 h -4.484 V 1.328 l 4.086,-1.674 0.118,-1.84 c 0.933,1.681 2.222,2.923 3.867,3.727 1.643,0.803 3.493,1.205 5.548,1.205 C -4.671,2.746 -2.055,1.83 0,0"
|
||||
style="fill:#000033;fill-opacity:1;fill-rule:nonzero;stroke:none"
|
||||
id="path442" />
|
||||
</g>
|
||||
<g
|
||||
id="g444"
|
||||
transform="translate(1027.9968,264.0386)">
|
||||
<path
|
||||
d="m 0,0 h -21.128 c 0.261,-2.84 1.205,-5.044 2.83,-6.613 1.625,-1.57 3.727,-2.355 6.305,-2.355 2.054,0 3.763,0.356 5.128,1.065 1.363,0.71 2.288,1.738 2.774,3.083 l 3.755,-1.961 c -1.121,-1.981 -2.616,-3.495 -4.484,-4.54 -1.868,-1.046 -4.259,-1.569 -7.173,-1.569 -4.223,0 -7.538,1.289 -9.948,3.867 -2.41,2.578 -3.615,6.146 -3.615,10.704 0,4.558 1.149,8.127 3.447,10.705 2.298,2.578 5.557,3.867 9.779,3.867 2.615,0 4.876,-0.58 6.782,-1.738 1.905,-1.158 3.343,-2.728 4.315,-4.707 C -0.262,7.827 0.224,5.605 0.224,3.139 0.224,2.092 0.149,1.046 0,0 m -18.298,10.144 c -1.513,-1.457 -2.438,-3.512 -2.775,-6.165 h 16.982 c -0.3,2.615 -1.159,4.661 -2.578,6.137 -1.42,1.476 -3.307,2.214 -5.661,2.214 -2.466,0 -4.455,-0.728 -5.968,-2.186"
|
||||
style="fill:#000033;fill-opacity:1;fill-rule:nonzero;stroke:none"
|
||||
id="path446" />
|
||||
</g>
|
||||
<g
|
||||
id="g448"
|
||||
transform="translate(1057.8818,276.4246)">
|
||||
<path
|
||||
d="m 0,0 c 2.41,-2.578 3.615,-6.147 3.615,-10.705 0,-4.558 -1.205,-8.126 -3.615,-10.704 -2.41,-2.578 -5.726,-3.867 -9.948,-3.867 -4.222,0 -7.537,1.289 -9.947,3.867 -2.41,2.578 -3.615,6.146 -3.615,10.704 0,4.558 1.205,8.127 3.615,10.705 2.41,2.578 5.725,3.867 9.947,3.867 C -5.726,3.867 -2.41,2.578 0,0 m -16.617,-2.858 c -1.607,-1.906 -2.41,-4.522 -2.41,-7.847 0,-3.326 0.803,-5.94 2.41,-7.846 1.607,-1.905 3.83,-2.858 6.669,-2.858 2.839,0 5.063,0.953 6.67,2.858 1.606,1.906 2.41,4.52 2.41,7.846 0,3.325 -0.804,5.941 -2.41,7.847 C -4.885,-0.953 -7.109,0 -9.948,0 c -2.839,0 -5.062,-0.953 -6.669,-2.858"
|
||||
style="fill:#000033;fill-opacity:1;fill-rule:nonzero;stroke:none"
|
||||
id="path450" />
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
<g
|
||||
id="g452"
|
||||
transform="matrix(0.35277777,0,0,-0.35277777,5.8329581,6.5590171)">
|
||||
<path
|
||||
d="m 0,0 0.001,-38.946 25.286,-9.076 V -8.753 L 52.626,1.321 27.815,10.207 Z"
|
||||
style="fill:#00e599;fill-opacity:1;fill-rule:nonzero;stroke:none"
|
||||
id="path454" />
|
||||
</g>
|
||||
<g
|
||||
id="g456"
|
||||
transform="matrix(0.35277777,0,0,-0.35277777,15.479008,10.041927)">
|
||||
<path
|
||||
d="M 0,0 V -21.306 L 25.293,-30.364 25.282,9.347 Z"
|
||||
style="fill:#00b091;fill-opacity:1;fill-rule:nonzero;stroke:none"
|
||||
id="path458" />
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 5.5 KiB After Width: | Height: | Size: 6.5 KiB |
2
.github/workflows/tests.yml
vendored
2
.github/workflows/tests.yml
vendored
|
|
@ -61,7 +61,7 @@ jobs:
|
|||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
go_versions: [ '1.18', '1.19' ]
|
||||
go_versions: [ '1.17', '1.18', '1.19' ]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
|
|||
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -10,7 +10,7 @@ temp
|
|||
test.sh
|
||||
testfile
|
||||
.blast.yml
|
||||
.frostfs-cli.yml
|
||||
.neofs-cli.yml
|
||||
|
||||
.cache
|
||||
|
||||
|
|
|
|||
11
.gitlint
11
.gitlint
|
|
@ -1,11 +0,0 @@
|
|||
[general]
|
||||
fail-without-commits=True
|
||||
regex-style-search=True
|
||||
contrib=CC1
|
||||
|
||||
[title-match-regex]
|
||||
regex=^\[\#[0-9Xx]+\]\s
|
||||
|
||||
[ignore-by-title]
|
||||
regex=^Release(.*)
|
||||
ignore=title-match-regex
|
||||
|
|
@ -32,12 +32,15 @@ linters:
|
|||
- revive
|
||||
|
||||
# some default golangci-lint linters
|
||||
- deadcode
|
||||
- errcheck
|
||||
- gosimple
|
||||
- ineffassign
|
||||
- staticcheck
|
||||
- structcheck
|
||||
- typecheck
|
||||
- unused
|
||||
- varcheck
|
||||
|
||||
# extra linters
|
||||
- exhaustive
|
||||
|
|
|
|||
|
|
@ -1,45 +0,0 @@
|
|||
ci:
|
||||
autofix_prs: false
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/jorisroovers/gitlint
|
||||
rev: v0.19.1
|
||||
hooks:
|
||||
- id: gitlint
|
||||
stages: [commit-msg]
|
||||
- id: gitlint-ci
|
||||
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.4.0
|
||||
hooks:
|
||||
- id: check-added-large-files
|
||||
- id: check-case-conflict
|
||||
- id: check-executables-have-shebangs
|
||||
- id: check-shebang-scripts-are-executable
|
||||
- id: check-merge-conflict
|
||||
- id: check-json
|
||||
- id: check-xml
|
||||
- id: check-yaml
|
||||
- id: trailing-whitespace
|
||||
args: [--markdown-linebreak-ext=md]
|
||||
- id: end-of-file-fixer
|
||||
exclude: ".key$"
|
||||
|
||||
- repo: https://github.com/shellcheck-py/shellcheck-py
|
||||
rev: v0.9.0.2
|
||||
hooks:
|
||||
- id: shellcheck
|
||||
|
||||
- repo: https://github.com/golangci/golangci-lint
|
||||
rev: v1.51.2
|
||||
hooks:
|
||||
- id: golangci-lint
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: go-unit-tests
|
||||
name: go unit tests
|
||||
entry: make test
|
||||
pass_filenames: false
|
||||
types: [go]
|
||||
language: system
|
||||
51
CHANGELOG.md
51
CHANGELOG.md
|
|
@ -4,54 +4,6 @@ This document outlines major changes between releases.
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- Multiple configs support (TrueCloudLab#12)
|
||||
|
||||
### Changed
|
||||
- Update go version to 1.18 (TrueCloudLab#9)
|
||||
- Update neo-go to v0.101.0 (#8)
|
||||
- Update viper to v1.15.0 (#8)
|
||||
- Errors have become more detailed (#18)
|
||||
- Update system attribute names (#22)
|
||||
- Separate integration tests with build tags (#24)
|
||||
|
||||
### Updating from v0.26.0
|
||||
|
||||
To set system attributes use updated headers
|
||||
(you can use old ones for now, but their support will be dropped in the future releases):
|
||||
|
||||
* `X-Attribute-Neofs-*` -> `X-Attribute-System-*`
|
||||
* `X-Attribute-NEOFS-*` -> `X-Attribute-SYSTEM-*`
|
||||
* `X-Attribute-neofs-*` -> `X-Attribute-system-*`
|
||||
|
||||
## [0.26.0] - 2022-12-28
|
||||
|
||||
### Fixed
|
||||
- ENV config example (#236)
|
||||
|
||||
### Added
|
||||
- Support the `Date` header on upload (#214)
|
||||
- Available routes specification (#216)
|
||||
- Mention caching strategy in docs (#215)
|
||||
- Add error response on attribute duplicates (#221)
|
||||
- Multiple server listeners (#228)
|
||||
|
||||
### Removed
|
||||
- Deprecated linters (#239)
|
||||
|
||||
### Updating from v0.25.1
|
||||
Make sure your configuration is valid:
|
||||
|
||||
If you configure application using environment variables change:
|
||||
* `HTTP_GW_LISTEN_ADDRESS` -> `HTTP_GW_SERVER_0_ADDRESS`
|
||||
* `HTTP_GW_TLS_CERT_FILE` -> `HTTP_GW_SERVER_0_TLS_CERT_FILE` (and set `HTTP_GW_SERVER_0_TLS_ENABLED=true`)
|
||||
* `HTTP_GW_TLS_KEY_FILE` -> `HTTP_GW_SERVER_0_TLS_KEY_FILE` (and set `HTTP_GW_SERVER_0_TLS_ENABLED=true`)
|
||||
|
||||
If you configure application using `.yaml` file change:
|
||||
* `listen_address` -> `server.0.address`
|
||||
* `tls.cert_file` -> `server.0.tls.cert_file` (and set `server.0.tls.enabled: true`)
|
||||
* `tls.key_file` -> `server.0.tls.key_file` (and set `server.0.tls.enabled: true`)
|
||||
|
||||
## [0.25.1] - 2022-11-30
|
||||
|
||||
### Fixed
|
||||
|
|
@ -295,5 +247,4 @@ releases.
|
|||
[0.24.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.23.0...v0.24.0
|
||||
[0.25.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.24.0...v0.25.0
|
||||
[0.25.1]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.25.0...v0.25.1
|
||||
[0.26.0]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.25.1...v0.26.0
|
||||
[Unreleased]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.26.0...master
|
||||
[Unreleased]: https://github.com/nspcc-dev/neofs-http-gw/compare/v0.25.1...master
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
First, thank you for contributing! We love and encourage pull requests from
|
||||
everyone. Please follow the guidelines:
|
||||
|
||||
- Check the open [issues](https://github.com/TrueCloudLab/frostfs-http-gw/issues) and
|
||||
[pull requests](https://github.com/TrueCloudLab/frostfs-http-gw/pulls) for existing
|
||||
- Check the open [issues](https://github.com/nspcc-dev/neofs-http-gw/issues) and
|
||||
[pull requests](https://github.com/nspcc-dev/neofs-http-gw/pulls) for existing
|
||||
discussions.
|
||||
|
||||
- Open an issue first, to discuss a new feature or enhancement.
|
||||
|
|
@ -23,24 +23,24 @@ everyone. Please follow the guidelines:
|
|||
|
||||
## Development Workflow
|
||||
|
||||
Start by forking the `frostfs-http-gw` repository, make changes in a branch and then
|
||||
Start by forking the `neofs-http-gw` repository, make changes in a branch and then
|
||||
send a pull request. We encourage pull requests to discuss code changes. Here
|
||||
are the steps in details:
|
||||
|
||||
### Set up your GitHub Repository
|
||||
Fork [FrostFS HTTP Gateway
|
||||
upstream](https://github.com/TrueCloudLab/frostfs-http-gw/fork) source repository
|
||||
Fork [NeoFS HTTP Gateway
|
||||
upstream](https://github.com/nspcc-dev/neofs-http-gw/fork) source repository
|
||||
to your own personal repository. Copy the URL of your fork (you will need it for
|
||||
the `git clone` command below).
|
||||
|
||||
```sh
|
||||
$ git clone https://github.com/TrueCloudLab/frostfs-http-gw
|
||||
$ git clone https://github.com/nspcc-dev/neofs-http-gw
|
||||
```
|
||||
|
||||
### Set up git remote as ``upstream``
|
||||
```sh
|
||||
$ cd frostfs-http-gw
|
||||
$ git remote add upstream https://github.com/TrueCloudLab/frostfs-http-gw
|
||||
$ cd neofs-http-gw
|
||||
$ git remote add upstream https://github.com/nspcc-dev/neofs-http-gw
|
||||
$ git fetch upstream
|
||||
$ git merge upstream/master
|
||||
...
|
||||
|
|
@ -107,7 +107,7 @@ contributors".
|
|||
To sign your work, just add a line like this at the end of your commit message:
|
||||
|
||||
```
|
||||
Signed-off-by: Samii Sakisaka <samii@frostfs.info>
|
||||
Signed-off-by: Samii Sakisaka <samii@nspcc.ru>
|
||||
```
|
||||
|
||||
This can be easily done with the `--signoff` option to `git commit`.
|
||||
|
|
|
|||
|
|
@ -18,6 +18,6 @@ FROM scratch
|
|||
WORKDIR /
|
||||
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=builder /src/bin/frostfs-http-gw /bin/frostfs-http-gw
|
||||
COPY --from=builder /src/bin/neofs-http-gw /bin/neofs-http-gw
|
||||
|
||||
ENTRYPOINT ["/bin/frostfs-http-gw"]
|
||||
ENTRYPOINT ["/bin/neofs-http-gw"]
|
||||
|
|
|
|||
|
|
@ -3,6 +3,6 @@ RUN apk add --update --no-cache bash ca-certificates
|
|||
|
||||
WORKDIR /
|
||||
|
||||
COPY bin/frostfs-http-gw /bin/frostfs-http-gw
|
||||
COPY bin/neofs-http-gw /bin/neofs-http-gw
|
||||
|
||||
CMD ["frostfs-http-gw"]
|
||||
CMD ["neofs-http-gw"]
|
||||
|
|
|
|||
25
Makefile
25
Makefile
|
|
@ -6,15 +6,15 @@ GO_VERSION ?= 1.19
|
|||
LINT_VERSION ?= 1.49.0
|
||||
BUILD ?= $(shell date -u --iso=seconds)
|
||||
|
||||
HUB_IMAGE ?= truecloudlab/frostfs-http-gw
|
||||
HUB_IMAGE ?= nspccdev/neofs-http-gw
|
||||
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
||||
|
||||
# List of binaries to build. For now just one.
|
||||
BINDIR = bin
|
||||
DIRS = $(BINDIR)
|
||||
BINS = $(BINDIR)/frostfs-http-gw
|
||||
BINS = $(BINDIR)/neofs-http-gw
|
||||
|
||||
.PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean
|
||||
.PHONY: all $(BINS) $(DIRS) dep docker/ test cover fmt image image-push dirty-image lint docker/lint version clean
|
||||
|
||||
# .deb package versioning
|
||||
OS_RELEASE = $(shell lsb_release -cs)
|
||||
|
|
@ -62,11 +62,6 @@ docker/%:
|
|||
test:
|
||||
@go test ./... -cover
|
||||
|
||||
# Run integration tests
|
||||
.PHONY: integration-test
|
||||
integration-test:
|
||||
@go test ./... -cover --tags=integration
|
||||
|
||||
# Run tests with race detection and produce coverage output
|
||||
cover:
|
||||
@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
|
||||
|
|
@ -79,7 +74,7 @@ fmt:
|
|||
|
||||
# Build clean Docker image
|
||||
image:
|
||||
@echo "⇒ Build FrostFS HTTP Gateway docker image "
|
||||
@echo "⇒ Build NeoFS HTTP Gateway docker image "
|
||||
@docker build \
|
||||
--build-arg REPO=$(REPO) \
|
||||
--build-arg VERSION=$(VERSION) \
|
||||
|
|
@ -94,7 +89,7 @@ image-push:
|
|||
|
||||
# Build dirty Docker image
|
||||
dirty-image:
|
||||
@echo "⇒ Build FrostFS HTTP Gateway dirty docker image "
|
||||
@echo "⇒ Build NeoFS HTTP Gateway dirty docker image "
|
||||
@docker build \
|
||||
--build-arg REPO=$(REPO) \
|
||||
--build-arg VERSION=$(VERSION) \
|
||||
|
|
@ -114,14 +109,6 @@ docker/lint:
|
|||
--env HOME=/src \
|
||||
golangci/golangci-lint:v$(LINT_VERSION) bash -c 'cd /src/ && make lint'
|
||||
|
||||
# Activate pre-commit hooks
|
||||
pre-commit:
|
||||
pre-commit install -t pre-commit -t commit-msg
|
||||
|
||||
# Deactivate pre-commit hooks
|
||||
unpre-commit:
|
||||
pre-commit uninstall -t pre-commit -t commit-msg
|
||||
|
||||
# Print version
|
||||
version:
|
||||
@echo $(VERSION)
|
||||
|
|
@ -133,7 +120,7 @@ clean:
|
|||
|
||||
# Package for Debian
|
||||
debpackage:
|
||||
dch --package frostfs-http-gw \
|
||||
dch --package neofs-http-gw \
|
||||
--controlmaint \
|
||||
--newversion $(PKG_VERSION) \
|
||||
--distribution $(OS_RELEASE) \
|
||||
|
|
|
|||
153
README.md
153
README.md
|
|
@ -1,28 +1,28 @@
|
|||
<p align="center">
|
||||
<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
|
||||
<img src="./.github/logo.svg" width="500px" alt="NeoFS">
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="https://frostfs.info">FrostFS</a> is a decentralized distributed object storage integrated with the <a href="https://neo.org">NEO Blockchain</a>.
|
||||
<a href="https://fs.neo.org">NeoFS</a> is a decentralized distributed object storage integrated with the <a href="https://neo.org">NEO Blockchain</a>.
|
||||
</p>
|
||||
|
||||
---
|
||||
[](https://goreportcard.com/report/git.frostfs.info/TrueCloudLab/frostfs-http-gw)
|
||||
[](https://goreportcard.com/report/github.com/nspcc-dev/neofs-http-gw)
|
||||
|
||||
|
||||
|
||||
# FrostFS HTTP Gateway
|
||||
# NeoFS HTTP Gateway
|
||||
|
||||
FrostFS HTTP Gateway bridges FrostFS internal protocol and HTTP standard.
|
||||
- you can download one file per request from the FrostFS Network
|
||||
- you can upload one file per request into the FrostFS Network
|
||||
|
||||
See available routes in [specification](./docs/api.md).
|
||||
NeoFS HTTP Gateway bridges NeoFS internal protocol and HTTP standard.
|
||||
- you can download one file per request from the NeoFS Network
|
||||
- you can upload one file per request into the NeoFS Network
|
||||
|
||||
## Installation
|
||||
|
||||
```go install git.frostfs.info/TrueCloudLab/frostfs-http-gw```
|
||||
```go install github.com/nspcc-dev/neofs-http-gw```
|
||||
|
||||
Or you can call `make` to build it from the cloned repository (the binary will
|
||||
end up in `bin/frostfs-http-gw`). To build frostfs-http-gw binary in clean docker
|
||||
environment, call `make docker/bin/frostfs-http-gw`.
|
||||
end up in `bin/neofs-http-gw`). To build neofs-http-gw binary in clean docker
|
||||
environment, call `make docker/bin/neofs-http-gw`.
|
||||
|
||||
Other notable make targets:
|
||||
|
||||
|
|
@ -36,32 +36,32 @@ version Show current version
|
|||
```
|
||||
|
||||
Or you can also use a [Docker
|
||||
image](https://hub.docker.com/r/truecloudlab/frostfs-http-gw) provided for the released
|
||||
image](https://hub.docker.com/r/nspccdev/neofs-http-gw) provided for the released
|
||||
(and occasionally unreleased) versions of the gateway (`:latest` points to the
|
||||
latest stable release).
|
||||
|
||||
## Execution
|
||||
|
||||
HTTP gateway itself is not a FrostFS node, so to access FrostFS it uses node's
|
||||
HTTP gateway itself is not a NeoFS node, so to access NeoFS it uses node's
|
||||
gRPC interface and you need to provide some node that it will connect to. This
|
||||
can be done either via `-p` parameter or via `HTTP_GW_PEERS_<N>_ADDRESS` and
|
||||
`HTTP_GW_PEERS_<N>_WEIGHT` environment variables (the gate supports multiple
|
||||
FrostFS nodes with weighted load balancing).
|
||||
NeoFS nodes with weighted load balancing).
|
||||
|
||||
If you launch HTTP gateway in bundle with [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env),
|
||||
If you launch HTTP gateway in bundle with [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env),
|
||||
you can get the IP address of the node in the output of `make hosts` command
|
||||
(with s0*.frostfs.devenv name).
|
||||
(with s0*.neofs.devenv name).
|
||||
|
||||
These two commands are functionally equivalent, they run the gate with one
|
||||
backend node (and otherwise default settings):
|
||||
```
|
||||
$ frostfs-http-gw -p 192.168.130.72:8080
|
||||
$ HTTP_GW_PEERS_0_ADDRESS=192.168.130.72:8080 frostfs-http-gw
|
||||
$ neofs-http-gw -p 192.168.130.72:8080
|
||||
$ HTTP_GW_PEERS_0_ADDRESS=192.168.130.72:8080 neofs-http-gw
|
||||
```
|
||||
It's also possible to specify uri scheme (grpc or grpcs) when using `-p`:
|
||||
```
|
||||
$ frostfs-http-gw -p grpc://192.168.130.72:8080
|
||||
$ HTTP_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 frostfs-http-gw
|
||||
$ neofs-http-gw -p grpc://192.168.130.72:8080
|
||||
$ HTTP_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 neofs-http-gw
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
|
@ -72,11 +72,11 @@ environment variables (see [example](./config/config.env)), so they're not speci
|
|||
|
||||
### Nodes: weights and priorities
|
||||
|
||||
You can specify multiple `-p` options to add more FrostFS nodes, this will make
|
||||
You can specify multiple `-p` options to add more NeoFS nodes, this will make
|
||||
gateway spread requests equally among them (using weight 1 and priority 1 for every node):
|
||||
|
||||
```
|
||||
$ frostfs-http-gw -p 192.168.130.72:8080 -p 192.168.130.71:8080
|
||||
$ neofs-http-gw -p 192.168.130.72:8080 -p 192.168.130.71:8080
|
||||
```
|
||||
If you want some specific load distribution proportions, use weights and priorities:
|
||||
|
||||
|
|
@ -84,7 +84,7 @@ If you want some specific load distribution proportions, use weights and priorit
|
|||
$ HTTP_GW_PEERS_0_ADDRESS=192.168.130.71:8080 HTTP_GW_PEERS_0_WEIGHT=1 HTTP_GW_PEERS_0_PRIORITY=1 \
|
||||
HTTP_GW_PEERS_1_ADDRESS=192.168.130.72:8080 HTTP_GW_PEERS_1_WEIGHT=9 HTTP_GW_PEERS_1_PRIORITY=2 \
|
||||
HTTP_GW_PEERS_2_ADDRESS=192.168.130.73:8080 HTTP_GW_PEERS_2_WEIGHT=1 HTTP_GW_PEERS_2_PRIORITY=2 \
|
||||
frostfs-http-gw
|
||||
neofs-http-gw
|
||||
```
|
||||
This command will make gateway use 192.168.130.71 while it is healthy. Otherwise, it will make the gateway use
|
||||
192.168.130.72 for 90% of requests and 192.168.130.73 for remaining 10%.
|
||||
|
|
@ -92,13 +92,13 @@ This command will make gateway use 192.168.130.71 while it is healthy. Otherwise
|
|||
### Keys
|
||||
You can provide a wallet via `--wallet` or `-w` flag. You can also specify the account address using `--address`
|
||||
(if no address provided default one will be used). If wallet is used, you need to set `HTTP_GW_WALLET_PASSPHRASE` variable to decrypt the wallet.
|
||||
If no wallet provided, the gateway autogenerates a key pair it will use for FrostFS requests.
|
||||
If no wallet provided, the gateway autogenerates a key pair it will use for NeoFS requests.
|
||||
```
|
||||
$ frostfs-http-gw -p $FROSTFS_NODE -w $WALLET_PATH --address $ACCOUNT_ADDRESS
|
||||
$ neofs-http-gw -p $NEOFS_NODE -w $WALLET_PATH --address $ACCOUNT_ADDRESS
|
||||
```
|
||||
Example:
|
||||
```
|
||||
$ frostfs-http-gw -p 192.168.130.72:8080 -w wallet.json --address NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP
|
||||
$ neofs-http-gw -p 192.168.130.72:8080 -w wallet.json --address NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP
|
||||
```
|
||||
|
||||
### Binding and TLS
|
||||
|
|
@ -114,7 +114,7 @@ external redirecting solution.
|
|||
Example to bind to `192.168.130.130:443` and serve TLS there:
|
||||
|
||||
```
|
||||
$ frostfs-http-gw -p 192.168.130.72:8080 --listen_address 192.168.130.130:443 \
|
||||
$ neofs-http-gw -p 192.168.130.72:8080 --listen_address 192.168.130.130:443 \
|
||||
--tls_key=key.pem --tls_certificate=cert.pem
|
||||
```
|
||||
|
||||
|
|
@ -132,12 +132,12 @@ request with data stream after timeout.
|
|||
|
||||
`HTTP_GW_WEB_STREAM_REQUEST_BODY` environment variable can be used to disable
|
||||
request body streaming (effectively it'll make the gateway accept the file completely
|
||||
first and only then try sending it to FrostFS).
|
||||
first and only then try sending it to NeoFS).
|
||||
|
||||
`HTTP_GW_WEB_MAX_REQUEST_BODY_SIZE` controls maximum request body size
|
||||
limiting uploads to files slightly lower than this limit.
|
||||
|
||||
### FrostFS parameters
|
||||
### NeoFS parameters
|
||||
|
||||
Gateway can automatically set timestamps for uploaded files based on local
|
||||
time source, use `HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP` environment
|
||||
|
|
@ -175,32 +175,15 @@ HTTP_GW_LOGGER_LEVEL=debug
|
|||
Configuration file is optional and can be used instead of environment variables/other parameters.
|
||||
It can be specified with `--config` parameter:
|
||||
```
|
||||
$ frostfs-http-gw --config your-config.yaml
|
||||
$ neofs-http-gw --config your-config.yaml
|
||||
```
|
||||
|
||||
See [config](./config/config.yaml) and [defaults](./docs/gate-configuration.md) for example.
|
||||
|
||||
#### Multiple configs
|
||||
|
||||
You can use several config files when running application. It allows you to split configuration into parts.
|
||||
For example, you can use separate yaml file for pprof and prometheus section in config (see [config examples](./config)).
|
||||
You can either provide several files with repeating `--config` flag or provide path to the dir that contains all configs using `--config-dir` flag.
|
||||
Also, you can combine these flags:
|
||||
|
||||
```shell
|
||||
$ frostfs-http-gw --config ./config/config.yaml --config /your/partial/config.yaml --config-dir ./config/dir
|
||||
```
|
||||
|
||||
**Note:** next file in `--config` flag overwrites values from the previous one.
|
||||
Files from `--config-dir` directory overwrite values from `--config` files.
|
||||
So the command above run `frostfs-http-gw` to listen on `0.0.0.0:8080` address (value from `./config/config.yaml`),
|
||||
applies parameters from `/your/partial/config.yaml`,
|
||||
enable pprof (value from `./config/dir/pprof.yaml`) and prometheus (value from `./config/dir/prometheus.yaml`).
|
||||
|
||||
## HTTP API provided
|
||||
|
||||
This gateway intentionally provides limited feature set and doesn't try to
|
||||
substitute (or completely wrap) regular gRPC FrostFS interface. You can download
|
||||
substitute (or completely wrap) regular gRPC NeoFS interface. You can download
|
||||
and upload objects with it, but deleting, searching, managing ACLs, creating
|
||||
containers and other activities are not supported and not planned to be
|
||||
supported.
|
||||
|
|
@ -221,23 +204,23 @@ Steps to start using name resolving:
|
|||
1. Enable NNS resolving in config (`rpc_endpoint` must be a valid neo rpc node, see [configs](./config) for other examples):
|
||||
|
||||
```yaml
|
||||
rpc_endpoint: http://morph-chain.frostfs.devenv:30333
|
||||
rpc_endpoint: http://morph-chain.neofs.devenv:30333
|
||||
resolve_order:
|
||||
- nns
|
||||
```
|
||||
|
||||
2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env)
|
||||
2. Make sure your container is registered in NNS contract. If you use [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env)
|
||||
you can check if your container (e.g. with `container-name` name) is registered in NNS:
|
||||
|
||||
```shell
|
||||
$ curl -s --data '{"id":1,"jsonrpc":"2.0","method":"getcontractstate","params":[1]}' \
|
||||
http://morph-chain.frostfs.devenv:30333 | jq -r '.result.hash'
|
||||
http://morph-chain.neofs.devenv:30333 | jq -r '.result.hash'
|
||||
|
||||
0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667
|
||||
|
||||
$ docker exec -it morph_chain neo-go \
|
||||
contract testinvokefunction \
|
||||
-r http://morph-chain.frostfs.devenv:30333 0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 \
|
||||
-r http://morph-chain.neofs.devenv:30333 0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 \
|
||||
resolve string:container-name.container int:16 \
|
||||
| jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \
|
||||
| base64 -d && echo
|
||||
|
|
@ -253,9 +236,9 @@ $ curl http://localhost:8082/get_by_attribute/container-name/FileName/object-nam
|
|||
|
||||
#### Create a container
|
||||
|
||||
You can create a container via [frostfs-cli](https://git.frostfs.info/TrueCloudLab/frostfs-node/releases):
|
||||
You can create a container via [neofs-cli](https://github.com/nspcc-dev/neofs-node/releases):
|
||||
```
|
||||
$ frostfs-cli -r $FROSTFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL
|
||||
$ neofs-cli -r $NEOFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL
|
||||
```
|
||||
where `$WALLET` is a path to user wallet,
|
||||
`$ACL` -- hex encoded basic ACL value or keywords 'private, 'public-read', 'public-read-write' and
|
||||
|
|
@ -263,18 +246,18 @@ where `$WALLET` is a path to user wallet,
|
|||
|
||||
For example:
|
||||
```
|
||||
$ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json container create --policy "REP 3" --basic-acl public --await
|
||||
$ neofs-cli -r 192.168.130.72:8080 -w ./wallet.json container create --policy "REP 3" --basic-acl public --await
|
||||
```
|
||||
|
||||
If you have launched nodes via [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env),
|
||||
If you have launched nodes via [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env),
|
||||
you can get the key value from `wallets/wallet.json` or write the path to
|
||||
the file `wallets/wallet.key`.
|
||||
|
||||
#### Prepare a file in a container
|
||||
|
||||
To create a file via [frostfs-cli](https://git.frostfs.info/TrueCloudLab/frostfs-node/releases), run a command below:
|
||||
To create a file via [neofs-cli](https://github.com/nspcc-dev/neofs-node/releases), run a command below:
|
||||
```
|
||||
$ frostfs-cli -r $FROSTFS_NODE -k $KEY object put --file $FILENAME --cid $CID
|
||||
$ neofs-cli -r $NEOFS_NODE -k $KEY object put --file $FILENAME --cid $CID
|
||||
```
|
||||
where
|
||||
`$KEY` -- the key, please read the information [above](#create-a-container),
|
||||
|
|
@ -282,7 +265,7 @@ where
|
|||
|
||||
For example:
|
||||
```
|
||||
$ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json object put --file cat.png --cid Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ --attributes img_type=cat,my_attr=cute
|
||||
$ neofs-cli -r 192.168.130.72:8080 -w ./wallet.json object put --file cat.png --cid Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ --attributes img_type=cat,my_attr=cute
|
||||
```
|
||||
|
||||
|
||||
|
|
@ -322,9 +305,8 @@ where
|
|||
`$ATTRIBUTE_NAME` is the name of the attribute we want to use,
|
||||
`$ATTRIBUTE_VALUE` is the value of this attribute that the target object should have.
|
||||
|
||||
**NB!** The attribute key and value should be url encoded, i.e., if you want to download an object with the attribute value
|
||||
`a cat`, the value in the request must be `a+cat`. In the same way with the attribute key. If you don't escape such values
|
||||
everything can still work (for example you can use `d@ta` without encoding) but it's HIGHLY RECOMMENDED to encode all your attributes.
|
||||
**NB!** The attribute key and value must be url encoded, i.e., if you want to download an object with the attribute value
|
||||
`a cat`, the value in the request must be `a+cat`. In the same way with the attribute key.
|
||||
|
||||
If multiple objects have specified attribute with specified value, then the
|
||||
first one of them is returned (and you can't get others via this interface).
|
||||
|
|
@ -387,15 +369,10 @@ set of reply headers generated using the following rules:
|
|||
* `x-container-id` contains container ID
|
||||
* `x-object-id` contains object ID
|
||||
* `x-owner-id` contains owner address
|
||||
* all the other FrostFS attributes are converted to `X-Attribute-*` headers (but only
|
||||
* all the other NeoFS attributes are converted to `X-Attribute-*` headers (but only
|
||||
if they can be safely represented in HTTP header), for example `FileName`
|
||||
attribute becomes `X-Attribute-FileName` header
|
||||
|
||||
##### Caching strategy
|
||||
|
||||
HTTP Gateway doesn't control caching (doesn't anything with the `Cache-Control` header). Caching strategy strictly
|
||||
depends on application use case. So it should be carefully done by proxy server.
|
||||
|
||||
### Uploading
|
||||
|
||||
You can POST files to `/upload/$CID` path where `$CID` is a container ID or its name if NNS is enabled. The
|
||||
|
|
@ -424,12 +401,12 @@ You can also add some attributes to your file using the following rules:
|
|||
"X-Attribute-" prefix stripped, that is if you add "X-Attribute-Ololo:
|
||||
100500" header to your request the resulting object will get "Ololo:
|
||||
100500" attribute
|
||||
* "X-Attribute-SYSTEM-*" headers are special
|
||||
(`-SYSTEM-` part can also be `-system-` or`-System-` (and even legacy `-Neofs-` for some next releases)), they're used to set internal
|
||||
FrostFS attributes starting with `__SYSTEM__` prefix, for these attributes all
|
||||
* "X-Attribute-NEOFS-*" headers are special
|
||||
(`-NEOFS-` part can also be `-neofs-` or`-Neofs-`), they're used to set internal
|
||||
NeoFS attributes starting with `__NEOFS__` prefix, for these attributes all
|
||||
dashes get converted to underscores and all letters are capitalized. For
|
||||
example, you can use "X-Attribute-SYSTEM-Expiration-Epoch" header to set
|
||||
`__SYSTEM__EXPIRATION_EPOCH` attribute
|
||||
example, you can use "X-Attribute-NEOFS-Expiration-Epoch" header to set
|
||||
`__NEOFS__EXPIRATION_EPOCH` attribute
|
||||
* `FileName` attribute is set from multipart's `filename` if not set
|
||||
explicitly via `X-Attribute-FileName` header
|
||||
* `Timestamp` attribute can be set using gateway local time if using
|
||||
|
|
@ -439,13 +416,13 @@ You can also add some attributes to your file using the following rules:
|
|||
---
|
||||
**NOTE**
|
||||
|
||||
There are some reserved headers type of `X-Attribute-SYSTEM-*` (headers are arranged in descending order of priority):
|
||||
1. `X-Attribute-System-Expiration-Epoch: 100`
|
||||
2. `X-Attribute-System-Expiration-Duration: 24h30m`
|
||||
3. `X-Attribute-System-Expiration-Timestamp: 1637574797`
|
||||
4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z`
|
||||
There are some reserved headers type of `X-Attribute-NEOFS-*` (headers are arranged in descending order of priority):
|
||||
1. `X-Attribute-Neofs-Expiration-Epoch: 100`
|
||||
2. `X-Attribute-Neofs-Expiration-Duration: 24h30m`
|
||||
3. `X-Attribute-Neofs-Expiration-Timestamp: 1637574797`
|
||||
4. `X-Attribute-Neofs-Expiration-RFC3339: 2021-11-22T09:55:49Z`
|
||||
|
||||
which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way.
|
||||
which transforms to `X-Attribute-Neofs-Expiration-Epoch`. So you can provide expiration any convenient way.
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -467,25 +444,25 @@ operations for a request signed with your HTTP Gateway keys.
|
|||
If your don't want to manage gateway's secret keys and adjust eACL rules when
|
||||
gateway configuration changes (new gate, key rotation, etc) or you plan to use
|
||||
public services, there is an option to let your application backend (or you) to
|
||||
issue Bearer Tokens ans pass them from the client via gate down to FrostFS level
|
||||
issue Bearer Tokens ans pass them from the client via gate down to NeoFS level
|
||||
to grant access.
|
||||
|
||||
FrostFS Bearer Token basically is a container owner-signed ACL data (refer to FrostFS
|
||||
NeoFS Bearer Token basically is a container owner-signed ACL data (refer to NeoFS
|
||||
documentation for more details). There are two options to pass them to gateway:
|
||||
* "Authorization" header with "Bearer" type and base64-encoded token in
|
||||
credentials field
|
||||
* "Bearer" cookie with base64-encoded token contents
|
||||
|
||||
For example, you have a mobile application frontend with a backend part storing
|
||||
data in FrostFS. When a user authorizes in the mobile app, the backend issues a FrostFS
|
||||
data in NeoFS. When a user authorizes in the mobile app, the backend issues a NeoFS
|
||||
Bearer token and provides it to the frontend. Then, the mobile app may generate
|
||||
some data and upload it via any available FrostFS HTTP Gateway by adding
|
||||
some data and upload it via any available NeoFS HTTP Gateway by adding
|
||||
the corresponding header to the upload request. Accessing the ACL protected data
|
||||
works the same way.
|
||||
|
||||
##### Example
|
||||
In order to generate a bearer token, you need to know the container owner key and
|
||||
the address of the sender who will do the request to FrostFS (in our case, it's a gateway wallet address).
|
||||
the address of the sender who will do the request to NeoFS (in our case, it's a gateway wallet address).
|
||||
|
||||
Suppose we have:
|
||||
* **KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr** (container owner key)
|
||||
|
|
@ -536,7 +513,7 @@ Now, we can form a Bearer token (10000 is liftetime expiration in epoch) and sav
|
|||
|
||||
Next, sign it with the container owner key:
|
||||
```
|
||||
$ frostfs-cli util sign bearer-token --from bearer.json --to signed.json -w ./wallet.json
|
||||
$ neofs-cli util sign bearer-token --from bearer.json --to signed.json -w ./wallet.json
|
||||
```
|
||||
Encoding to base64 to use via the header:
|
||||
```
|
||||
|
|
@ -563,12 +540,12 @@ For the token to work correctly, you need to create a container with a basic ACL
|
|||
|
||||
For example:
|
||||
```
|
||||
$ frostfs-cli -w ./wallet.json --basic-acl 0x0FFFCFFF -r 192.168.130.72:8080 container create --policy "REP 3" --await
|
||||
$ neofs-cli -w ./wallet.json --basic-acl 0x0FFFCFFF -r 192.168.130.72:8080 container create --policy "REP 3" --await
|
||||
```
|
||||
|
||||
To deny access to a container without a token, set the eACL rules:
|
||||
```
|
||||
$ frostfs-cli -w ./wallet.json -r 192.168.130.72:8080 container set-eacl --table eacl.json --await --cid BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K
|
||||
$ neofs-cli -w ./wallet.json -r 192.168.130.72:8080 container set-eacl --table eacl.json --await --cid BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K
|
||||
```
|
||||
|
||||
File **eacl.json**:
|
||||
|
|
|
|||
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
|||
v0.26.0
|
||||
v0.25.1
|
||||
|
|
|
|||
189
app.go
189
app.go
|
|
@ -3,28 +3,30 @@ package main
|
|||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/downloader"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/fasthttp/router"
|
||||
"github.com/nspcc-dev/neo-go/cli/flags"
|
||||
"github.com/nspcc-dev/neo-go/cli/input"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/nspcc-dev/neo-go/pkg/util"
|
||||
"github.com/nspcc-dev/neo-go/pkg/wallet"
|
||||
"github.com/nspcc-dev/neofs-http-gw/downloader"
|
||||
"github.com/nspcc-dev/neofs-http-gw/metrics"
|
||||
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||
"github.com/nspcc-dev/neofs-http-gw/response"
|
||||
"github.com/nspcc-dev/neofs-http-gw/uploader"
|
||||
"github.com/nspcc-dev/neofs-http-gw/utils"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
|
|
@ -43,12 +45,12 @@ type (
|
|||
metrics *gateMetrics
|
||||
services []*metrics.Service
|
||||
settings *appSettings
|
||||
servers []Server
|
||||
}
|
||||
|
||||
appSettings struct {
|
||||
Uploader *uploader.Settings
|
||||
Downloader *downloader.Settings
|
||||
Uploader *uploader.Settings
|
||||
Downloader *downloader.Settings
|
||||
TLSProvider *certProvider
|
||||
}
|
||||
|
||||
// App is an interface for the main gateway function.
|
||||
|
|
@ -111,7 +113,7 @@ func newApp(ctx context.Context, opt ...Option) App {
|
|||
}
|
||||
|
||||
// -- setup FastHTTP server --
|
||||
a.webServer.Name = "frost-http-gw"
|
||||
a.webServer.Name = "neofs-http-gw"
|
||||
a.webServer.ReadBufferSize = a.cfg.GetInt(cfgWebReadBufferSize)
|
||||
a.webServer.WriteBufferSize = a.cfg.GetInt(cfgWebWriteBufferSize)
|
||||
a.webServer.ReadTimeout = a.cfg.GetDuration(cfgWebReadTimeout)
|
||||
|
|
@ -123,9 +125,9 @@ func newApp(ctx context.Context, opt ...Option) App {
|
|||
a.webServer.DisablePreParseMultipartForm = true
|
||||
a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody)
|
||||
// -- -- -- -- -- -- -- -- -- -- -- -- -- --
|
||||
key, err = getFrostFSKey(a)
|
||||
key, err = getNeoFSKey(a)
|
||||
if err != nil {
|
||||
a.log.Fatal("failed to get frostfs credentials", zap.Error(err))
|
||||
a.log.Fatal("failed to get neofs credentials", zap.Error(err))
|
||||
}
|
||||
|
||||
var owner user.ID
|
||||
|
|
@ -177,8 +179,9 @@ func newApp(ctx context.Context, opt ...Option) App {
|
|||
|
||||
func (a *app) initAppSettings() {
|
||||
a.settings = &appSettings{
|
||||
Uploader: &uploader.Settings{},
|
||||
Downloader: &downloader.Settings{},
|
||||
Uploader: &uploader.Settings{},
|
||||
Downloader: &downloader.Settings{},
|
||||
TLSProvider: &certProvider{Enabled: a.cfg.IsSet(cfgTLSCertificate) || a.cfg.IsSet(cfgTLSKey)},
|
||||
}
|
||||
|
||||
a.updateSettings()
|
||||
|
|
@ -194,7 +197,7 @@ func (a *app) initResolver() {
|
|||
|
||||
func (a *app) getResolverConfig() ([]string, *resolver.Config) {
|
||||
resolveCfg := &resolver.Config{
|
||||
FrostFS: resolver.NewFrostFSResolver(a.pool),
|
||||
NeoFS: resolver.NewNeoFSResolver(a.pool),
|
||||
RPCAddress: a.cfg.GetString(cfgRPCEndpoint),
|
||||
}
|
||||
|
||||
|
|
@ -266,7 +269,7 @@ func remove(list []string, element string) []string {
|
|||
return list
|
||||
}
|
||||
|
||||
func getFrostFSKey(a *app) (*ecdsa.PrivateKey, error) {
|
||||
func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) {
|
||||
walletPath := a.cfg.GetString(cfgWalletPath)
|
||||
|
||||
if len(walletPath) == 0 {
|
||||
|
|
@ -327,7 +330,7 @@ func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*ecds
|
|||
}
|
||||
|
||||
func (a *app) Wait() {
|
||||
a.log.Info("starting application", zap.String("app_name", "frostfs-http-gw"), zap.String("version", Version))
|
||||
a.log.Info("starting application", zap.String("app_name", "neofs-http-gw"), zap.String("version", Version))
|
||||
|
||||
a.setHealthStatus()
|
||||
|
||||
|
|
@ -338,6 +341,43 @@ func (a *app) setHealthStatus() {
|
|||
a.metrics.SetHealth(1)
|
||||
}
|
||||
|
||||
type certProvider struct {
|
||||
Enabled bool
|
||||
|
||||
mu sync.RWMutex
|
||||
certPath string
|
||||
keyPath string
|
||||
cert *tls.Certificate
|
||||
}
|
||||
|
||||
func (p *certProvider) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
if !p.Enabled {
|
||||
return nil, errors.New("cert provider: disabled")
|
||||
}
|
||||
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return p.cert, nil
|
||||
}
|
||||
|
||||
func (p *certProvider) UpdateCert(certPath, keyPath string) error {
|
||||
if !p.Enabled {
|
||||
return fmt.Errorf("tls disabled")
|
||||
}
|
||||
|
||||
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot load TLS key pair from certFile '%s' and keyFile '%s': %w", certPath, keyPath, err)
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
p.certPath = certPath
|
||||
p.keyPath = keyPath
|
||||
p.cert = &cert
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *app) Serve(ctx context.Context) {
|
||||
uploadRoutes := uploader.New(ctx, a.AppParams(), a.settings.Uploader)
|
||||
downloadRoutes := downloader.New(ctx, a.AppParams(), a.settings.Downloader)
|
||||
|
|
@ -346,16 +386,38 @@ func (a *app) Serve(ctx context.Context) {
|
|||
a.configureRouter(uploadRoutes, downloadRoutes)
|
||||
|
||||
a.startServices()
|
||||
a.initServers(ctx)
|
||||
|
||||
for i := range a.servers {
|
||||
go func(i int) {
|
||||
a.log.Info("starting server", zap.String("address", a.servers[i].Address()))
|
||||
if err := a.webServer.Serve(a.servers[i].Listener()); err != nil && err != http.ErrServerClosed {
|
||||
a.log.Fatal("listen and serve", zap.Error(err))
|
||||
go func() {
|
||||
var err error
|
||||
defer func() {
|
||||
if err != nil {
|
||||
a.log.Fatal("could not start server", zap.Error(err))
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
}()
|
||||
|
||||
bind := a.cfg.GetString(cfgListenAddress)
|
||||
|
||||
if a.settings.TLSProvider.Enabled {
|
||||
if err = a.settings.TLSProvider.UpdateCert(a.cfg.GetString(cfgTLSCertificate), a.cfg.GetString(cfgTLSKey)); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var lnConf net.ListenConfig
|
||||
var ln net.Listener
|
||||
if ln, err = lnConf.Listen(ctx, "tcp4", bind); err != nil {
|
||||
return
|
||||
}
|
||||
lnTLS := tls.NewListener(ln, &tls.Config{
|
||||
GetCertificate: a.settings.TLSProvider.GetCertificate,
|
||||
})
|
||||
|
||||
a.log.Info("running web server (TLS-enabled)", zap.String("address", bind))
|
||||
err = a.webServer.Serve(lnTLS)
|
||||
} else {
|
||||
a.log.Info("running web server", zap.String("address", bind))
|
||||
err = a.webServer.ListenAndServe(bind)
|
||||
}
|
||||
}()
|
||||
|
||||
sigs := make(chan os.Signal, 1)
|
||||
signal.Notify(sigs, syscall.SIGHUP)
|
||||
|
|
@ -380,15 +442,14 @@ LOOP:
|
|||
|
||||
func (a *app) configReload() {
|
||||
a.log.Info("SIGHUP config reload started")
|
||||
if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) {
|
||||
if !a.cfg.IsSet(cmdConfig) {
|
||||
a.log.Warn("failed to reload config because it's missed")
|
||||
return
|
||||
}
|
||||
if err := readInConfig(a.cfg); err != nil {
|
||||
if err := readConfig(a.cfg); err != nil {
|
||||
a.log.Warn("failed to reload config", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
if lvl, err := getLogLevel(a.cfg); err != nil {
|
||||
a.log.Warn("log level won't be updated", zap.Error(err))
|
||||
} else {
|
||||
|
|
@ -399,10 +460,6 @@ func (a *app) configReload() {
|
|||
a.log.Warn("failed to update resolvers", zap.Error(err))
|
||||
}
|
||||
|
||||
if err := a.updateServers(); err != nil {
|
||||
a.log.Warn("failed to reload server parameters", zap.Error(err))
|
||||
}
|
||||
|
||||
a.stopServices()
|
||||
a.startServices()
|
||||
|
||||
|
|
@ -417,6 +474,10 @@ func (a *app) configReload() {
|
|||
func (a *app) updateSettings() {
|
||||
a.settings.Uploader.SetDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp))
|
||||
a.settings.Downloader.SetZipCompression(a.cfg.GetBool(cfgZipCompression))
|
||||
|
||||
if err := a.settings.TLSProvider.UpdateCert(a.cfg.GetString(cfgTLSCertificate), a.cfg.GetString(cfgTLSKey)); err != nil {
|
||||
a.log.Warn("failed to reload TLS certs", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (a *app) startServices() {
|
||||
|
|
@ -482,61 +543,3 @@ func (a *app) AppParams() *utils.AppParams {
|
|||
Resolver: a.resolver,
|
||||
}
|
||||
}
|
||||
|
||||
func (a *app) initServers(ctx context.Context) {
|
||||
serversInfo := fetchServers(a.cfg)
|
||||
|
||||
a.servers = make([]Server, 0, len(serversInfo))
|
||||
for _, serverInfo := range serversInfo {
|
||||
fields := []zap.Field{
|
||||
zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled),
|
||||
zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile),
|
||||
}
|
||||
srv, err := newServer(ctx, serverInfo)
|
||||
if err != nil {
|
||||
a.log.Warn("failed to add server", append(fields, zap.Error(err))...)
|
||||
continue
|
||||
}
|
||||
|
||||
a.servers = append(a.servers, srv)
|
||||
a.log.Info("add server", fields...)
|
||||
}
|
||||
|
||||
if len(a.servers) == 0 {
|
||||
a.log.Fatal("no healthy servers")
|
||||
}
|
||||
}
|
||||
|
||||
func (a *app) updateServers() error {
|
||||
serversInfo := fetchServers(a.cfg)
|
||||
|
||||
var found bool
|
||||
for _, serverInfo := range serversInfo {
|
||||
index := a.serverIndex(serverInfo.Address)
|
||||
if index == -1 {
|
||||
continue
|
||||
}
|
||||
|
||||
if serverInfo.TLS.Enabled {
|
||||
if err := a.servers[index].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
|
||||
return fmt.Errorf("failed to update tls certs: %w", err)
|
||||
}
|
||||
}
|
||||
found = true
|
||||
}
|
||||
|
||||
if !found {
|
||||
return fmt.Errorf("invalid servers configuration: no known server found")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *app) serverIndex(address string) int {
|
||||
for i := range a.servers {
|
||||
if a.servers[i].Address() == address {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,23 +17,21 @@ HTTP_GW_PROMETHEUS_ADDRESS=localhost:8084
|
|||
# Log level.
|
||||
HTTP_GW_LOGGER_LEVEL=debug
|
||||
|
||||
HTTP_GW_SERVER_0_ADDRESS=0.0.0.0:443
|
||||
HTTP_GW_SERVER_0_TLS_ENABLED=false
|
||||
HTTP_GW_SERVER_0_TLS_CERT_FILE=/path/to/tls/cert
|
||||
HTTP_GW_SERVER_0_TLS_KEY_FILE=/path/to/tls/key
|
||||
HTTP_GW_SERVER_1_ADDRESS=0.0.0.0:444
|
||||
HTTP_GW_SERVER_1_TLS_ENABLED=true
|
||||
HTTP_GW_SERVER_1_TLS_CERT_FILE=/path/to/tls/cert
|
||||
HTTP_GW_SERVER_1_TLS_KEY_FILE=/path/to/tls/key
|
||||
# Address to bind.
|
||||
HTTP_GW_LISTEN_ADDRESS=0.0.0.0:443
|
||||
# Provide cert to enable TLS.
|
||||
HTTP_GW_TLS_CERTIFICATE=/path/to/tls/cert
|
||||
# Provide key to enable TLS.
|
||||
HTTP_GW_TLS_KEY=/path/to/tls/key
|
||||
|
||||
# Nodes configuration.
|
||||
# This configuration make the gateway use the first node (grpc://s01.frostfs.devenv:8080)
|
||||
# while it's healthy. Otherwise, the gateway use the second node (grpc://s01.frostfs.devenv:8080)
|
||||
# This configuration make the gateway use the first node (grpc://s01.neofs.devenv:8080)
|
||||
# while it's healthy. Otherwise, the gateway use the second node (grpc://s01.neofs.devenv:8080)
|
||||
# for 10% of requests and the third node for 90% of requests.
|
||||
|
||||
# Peer 1.
|
||||
# Endpoint.
|
||||
HTTP_GW_PEERS_0_ADDRESS=grpc://s01.frostfs.devenv:8080
|
||||
HTTP_GW_PEERS_0_ADDRESS=grpc://s01.neofs.devenv:8080
|
||||
# Until nodes with the same priority level are healthy
|
||||
# nodes with other priority are not used.
|
||||
# The lower the value, the higher the priority.
|
||||
|
|
@ -41,11 +39,11 @@ HTTP_GW_PEERS_0_PRIORITY=1
|
|||
# Load distribution proportion for nodes with the same priority.
|
||||
HTTP_GW_PEERS_0_WEIGHT=1
|
||||
# Peer 2.
|
||||
HTTP_GW_PEERS_1_ADDRESS=grpc://s02.frostfs.devenv:8080
|
||||
HTTP_GW_PEERS_1_ADDRESS=grpc://s02.neofs.devenv:8080
|
||||
HTTP_GW_PEERS_1_PRIORITY=2
|
||||
HTTP_GW_PEERS_1_WEIGHT=1
|
||||
# Peer 3.
|
||||
HTTP_GW_PEERS_2_ADDRESS=grpc://s03.frostfs.devenv:8080
|
||||
HTTP_GW_PEERS_2_ADDRESS=grpc://s03.neofs.devenv:8080
|
||||
HTTP_GW_PEERS_2_PRIORITY=2
|
||||
HTTP_GW_PEERS_2_WEIGHT=9
|
||||
|
||||
|
|
@ -72,7 +70,7 @@ HTTP_GW_STREAM_REQUEST_BODY=true
|
|||
HTTP_GW_MAX_REQUEST_BODY_SIZE=4194304
|
||||
|
||||
# RPC endpoint to be able to use nns container resolving.
|
||||
HTTP_GW_RPC_ENDPOINT=http://morph-chain.frostfs.devenv:30333
|
||||
HTTP_GW_RPC_ENDPOINT=http://morph-chain.neofs.devenv:30333
|
||||
# The order in which resolvers are used to find an container id by name.
|
||||
HTTP_GW_RESOLVE_ORDER="nns dns"
|
||||
|
||||
|
|
@ -88,7 +86,7 @@ HTTP_GW_REQUEST_TIMEOUT=5s
|
|||
# Interval to check nodes health.
|
||||
HTTP_GW_REBALANCE_TIMER=30s
|
||||
# The number of errors on connection after which node is considered as unhealthy
|
||||
HTTP_GW_POOL_ERROR_THRESHOLD=100
|
||||
S3_GW_POOL_ERROR_THRESHOLD=100
|
||||
|
||||
# Enable zip compression to download files by common prefix.
|
||||
HTTP_GW_ZIP_COMPRESSION=false
|
||||
|
|
|
|||
|
|
@ -4,35 +4,27 @@ wallet:
|
|||
passphrase: pwd # Passphrase to decrypt wallet. If you're using a wallet without a password, place '' here.
|
||||
|
||||
pprof:
|
||||
enabled: false # Enable pprof.
|
||||
enabled: true # Enable pprof.
|
||||
address: localhost:8083
|
||||
prometheus:
|
||||
enabled: false # Enable metrics.
|
||||
enabled: true # Enable metrics.
|
||||
address: localhost:8084
|
||||
|
||||
logger:
|
||||
level: debug # Log level.
|
||||
|
||||
server:
|
||||
- address: 0.0.0.0:8080
|
||||
tls:
|
||||
enabled: false
|
||||
cert_file: /path/to/cert
|
||||
key_file: /path/to/key
|
||||
- address: 0.0.0.0:8081
|
||||
tls:
|
||||
enabled: false
|
||||
cert_file: /path/to/cert
|
||||
key_file: /path/to/key
|
||||
listen_address: 0.0.0.0:443 # Address to bind.
|
||||
tls_certificate: /path/to/tls/cert # Provide cert to enable TLS.
|
||||
tls_key: /path/to/tls/key # Provide key to enable TLS.
|
||||
|
||||
# Nodes configuration.
|
||||
# This configuration make the gateway use the first node (grpc://s01.frostfs.devenv:8080)
|
||||
# while it's healthy. Otherwise, the gateway use the second node (grpc://s01.frostfs.devenv:8080)
|
||||
# This configuration make the gateway use the first node (grpc://s01.neofs.devenv:8080)
|
||||
# while it's healthy. Otherwise, the gateway use the second node (grpc://s01.neofs.devenv:8080)
|
||||
# for 10% of requests and the third node for 90% of requests.
|
||||
peers:
|
||||
0:
|
||||
# Endpoint.
|
||||
address: grpc://s01.frostfs.devenv:8080
|
||||
address: grpc://s01.neofs.devenv:8080
|
||||
|
||||
# Until nodes with the same priority level are healthy
|
||||
# nodes with other priority are not used.
|
||||
|
|
@ -42,11 +34,11 @@ peers:
|
|||
# Load distribution proportion for nodes with the same priority.
|
||||
weight: 1
|
||||
1:
|
||||
address: grpc://s02.frostfs.devenv:8080
|
||||
address: grpc://s02.neofs.devenv:8080
|
||||
priority: 2
|
||||
weight: 1
|
||||
2:
|
||||
address: grpc://s03.frostfs.devenv:8080
|
||||
address: grpc://s03.neofs.devenv:8080
|
||||
priority: 2
|
||||
weight: 9
|
||||
|
||||
|
|
@ -80,7 +72,7 @@ web:
|
|||
max_request_body_size: 4194304
|
||||
|
||||
# RPC endpoint to be able to use nns container resolving.
|
||||
rpc_endpoint: http://morph-chain.frostfs.devenv:30333
|
||||
rpc_endpoint: http://morph-chain.neofs.devenv:30333
|
||||
# The order in which resolvers are used to find an container id by name.
|
||||
resolve_order:
|
||||
- nns
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
pprof:
|
||||
enabled: true
|
||||
address: localhost:8083
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
prometheus:
|
||||
enabled: true
|
||||
address: localhost:8084
|
||||
4
debian/changelog
vendored
4
debian/changelog
vendored
|
|
@ -1,5 +1,5 @@
|
|||
frostfs-http-gw (0.0.0) stable; urgency=medium
|
||||
neofs-http-gw (0.0.0) stable; urgency=medium
|
||||
|
||||
* Please see CHANGELOG.md
|
||||
|
||||
-- TrueCloudLab <tech@frostfs.info> Wed, 24 Aug 2022 18:29:49 +0300
|
||||
-- NeoSPCC <tech@nspcc.ru> Wed, 24 Aug 2022 18:29:49 +0300
|
||||
|
|
|
|||
16
debian/control
vendored
16
debian/control
vendored
|
|
@ -1,15 +1,15 @@
|
|||
Source: frostfs-http-gw
|
||||
Section: frostfs
|
||||
Source: neofs-http-gw
|
||||
Section: neofs
|
||||
Priority: optional
|
||||
Maintainer: TrueCloudLab <tech@frostfs.info>
|
||||
Maintainer: NeoSPCC <tech@nspcc.ru>
|
||||
Build-Depends: debhelper-compat (= 13), dh-sysuser, git, devscripts
|
||||
Standards-Version: 4.5.1
|
||||
Homepage: https://frostfs.info/
|
||||
Vcs-Git: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw.git
|
||||
Vcs-Browser: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw
|
||||
Homepage: https://fs.neo.org/
|
||||
Vcs-Git: https://github.com/nspcc-dev/neofs-http-gw.git
|
||||
Vcs-Browser: https://github.com/nspcc-dev/neofs-http-gw
|
||||
|
||||
Package: frostfs-http-gw
|
||||
Package: neofs-http-gw
|
||||
Architecture: any
|
||||
Depends: ${misc:Depends}
|
||||
Description: FrostFS HTTP Gateway bridges FrostFS internal protocol and HTTP standard.
|
||||
Description: NeoFS HTTP Gateway bridges NeoFS internal protocol and HTTP standard.
|
||||
|
||||
|
|
|
|||
10
debian/copyright
vendored
10
debian/copyright
vendored
|
|
@ -1,13 +1,11 @@
|
|||
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
|
||||
Upstream-Name: frostfs-http-gw
|
||||
Upstream-Contact: tech@frostfs.info
|
||||
Source: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw
|
||||
Upstream-Name: neofs-http-gw
|
||||
Upstream-Contact: tech@nspcc.ru
|
||||
Source: https://github.com/nspcc-dev/neofs-http-gw
|
||||
|
||||
Files: *
|
||||
Copyright: 2018-2022 NeoSPCC (@nspcc-dev), contributors of neofs-http-gw project
|
||||
Copyright: 2018-2022 NeoSPCC (@nspcc-dev), contributors of neofs-http-gw project
|
||||
(https://github.com/nspcc-dev/neofs-http-gw/blob/master/CREDITS.md)
|
||||
2022 True Cloud Lab (@TrueCloudLab), contributors of frostfs-http-gw project
|
||||
(https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/src/branch/master/CREDITS.md)
|
||||
|
||||
|
||||
License: GPL-3
|
||||
|
|
|
|||
2
debian/frostfs-http-gw.dirs
vendored
2
debian/frostfs-http-gw.dirs
vendored
|
|
@ -1,2 +0,0 @@
|
|||
etc/frostfs
|
||||
srv/frostfs_cache
|
||||
2
debian/frostfs-http-gw.install
vendored
2
debian/frostfs-http-gw.install
vendored
|
|
@ -1,2 +0,0 @@
|
|||
bin/frostfs-http-gw usr/bin
|
||||
config/config.yaml etc/frostfs/http
|
||||
16
debian/frostfs-http-gw.service
vendored
16
debian/frostfs-http-gw.service
vendored
|
|
@ -1,16 +0,0 @@
|
|||
[Unit]
|
||||
Description=FrostFS HTTP Gateway
|
||||
Requires=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/frostfs-http-gw --config /etc/frostfs/http/config.yaml
|
||||
User=frostfs-http
|
||||
Group=frostfs-http
|
||||
WorkingDirectory=/srv/frostfs_cache
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
PrivateTmp=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
2
debian/neofs-http-gw.dirs
vendored
Normal file
2
debian/neofs-http-gw.dirs
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
etc/neofs
|
||||
srv/neofs_cache
|
||||
2
debian/neofs-http-gw.install
vendored
Normal file
2
debian/neofs-http-gw.install
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
bin/neofs-http-gw usr/bin
|
||||
config/config.yaml etc/neofs/http
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
#!/bin/sh
|
||||
# postinst script for frostfs-http-gw
|
||||
# postinst script for neofs-http-gw
|
||||
#
|
||||
# see: dh_installdeb(1)
|
||||
|
||||
|
|
@ -21,16 +21,16 @@ set -e
|
|||
case "$1" in
|
||||
configure)
|
||||
USERNAME=http
|
||||
id -u frostfs-$USERNAME >/dev/null 2>&1 || useradd -s /usr/sbin/nologin -d /srv/frostfs_cache --system -M -U -c "FrostFS HTTP gateway" frostfs-$USERNAME
|
||||
if ! dpkg-statoverride --list /etc/frostfs/$USERNAME >/dev/null; then
|
||||
chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME
|
||||
chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME/config.yaml || true
|
||||
chmod -f 0750 /etc/frostfs/$USERNAME
|
||||
chmod -f 0640 /etc/frostfs/$USERNAME/config.yaml || true
|
||||
id -u neofs-$USERNAME >/dev/null 2>&1 || useradd -s /usr/sbin/nologin -d /srv/neofs_cache --system -M -U -c "NeoFS HTTP gateway" neofs-$USERNAME
|
||||
if ! dpkg-statoverride --list /etc/neofs/$USERNAME >/dev/null; then
|
||||
chown -f root:neofs-$USERNAME /etc/neofs/$USERNAME
|
||||
chown -f root:neofs-$USERNAME /etc/neofs/$USERNAME/config.yaml || true
|
||||
chmod -f 0750 /etc/neofs/$USERNAME
|
||||
chmod -f 0640 /etc/neofs/$USERNAME/config.yaml || true
|
||||
fi
|
||||
USERDIR=$(getent passwd "frostfs-$USERNAME" | cut -d: -f6)
|
||||
if ! dpkg-statoverride --list frostfs-$USERDIR >/dev/null; then
|
||||
chown -f frostfs-$USERNAME: $USERDIR
|
||||
USERDIR=$(getent passwd "neofs-$USERNAME" | cut -d: -f6)
|
||||
if ! dpkg-statoverride --list neofs-$USERDIR >/dev/null; then
|
||||
chown -f neofs-$USERNAME: $USERDIR
|
||||
fi
|
||||
;;
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
#!/bin/sh
|
||||
# postrm script for frostfs-http-gw
|
||||
# postrm script for neofs-http-gw
|
||||
#
|
||||
# see: dh_installdeb(1)
|
||||
|
||||
|
|
@ -21,7 +21,7 @@ set -e
|
|||
|
||||
case "$1" in
|
||||
purge)
|
||||
rm -rf /srv/frostfs_cache
|
||||
rm -rf /srv/neofs_cache
|
||||
;;
|
||||
|
||||
remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
#!/bin/sh
|
||||
# preinst script for frostfs-http-gw
|
||||
# preinst script for neofs-http-gw
|
||||
#
|
||||
# see: dh_installdeb(1)
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
#!/bin/sh
|
||||
# prerm script for frostfs-http-gw
|
||||
# prerm script for neofs-http-gw
|
||||
#
|
||||
# see: dh_installdeb(1)
|
||||
|
||||
16
debian/neofs-http-gw.service
vendored
Normal file
16
debian/neofs-http-gw.service
vendored
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
[Unit]
|
||||
Description=NeoFS HTTP Gateway
|
||||
Requires=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/neofs-http-gw --config /etc/neofs/http/config.yaml
|
||||
User=neofs-http
|
||||
Group=neofs-http
|
||||
WorkingDirectory=/srv/neofs_cache
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
PrivateTmp=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
2
debian/rules
vendored
2
debian/rules
vendored
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# Do not try to strip Go binaries and do not run test
|
||||
export DEB_BUILD_OPTIONS := nostrip nocheck
|
||||
SERVICE = frostfs-http-gw
|
||||
SERVICE = neofs-http-gw
|
||||
|
||||
%:
|
||||
dh $@
|
||||
|
|
|
|||
309
docs/api.md
309
docs/api.md
|
|
@ -1,309 +0,0 @@
|
|||
# HTTP Gateway Specification
|
||||
|
||||
| Route | Description |
|
||||
|-------------------------------------------------|----------------------------------------------|
|
||||
| `/upload/{cid}` | [Put object](#put-object) |
|
||||
| `/get/{cid}/{oid}` | [Get object](#get-object) |
|
||||
| `/get_by_attribute/{cid}/{attr_key}/{attr_val}` | [Search object](#search-object) |
|
||||
| `/zip/{cid}/{prefix}` | [Download objects in archive](#download-zip) |
|
||||
|
||||
**Note:** `cid` parameter can be base58 encoded container ID or container name
|
||||
(the name must be registered in NNS, see appropriate section in [README](../README.md#nns)).
|
||||
|
||||
Route parameters can be:
|
||||
|
||||
* `Single` - match a single path segment (cannot contain `/` and be empty)
|
||||
* `Catch-All` - match everything (such parameter usually the last one in routes)
|
||||
* `Query` - regular query parameter
|
||||
|
||||
### Bearer token
|
||||
|
||||
All routes can accept [bearer token](../README.md#authentication) from:
|
||||
|
||||
* `Authorization` header with `Bearer` type and base64-encoded token in
|
||||
credentials field
|
||||
* `Bearer` cookie with base64-encoded token contents
|
||||
|
||||
Example:
|
||||
|
||||
Header:
|
||||
|
||||
```
|
||||
Authorization: Bearer ChA5Gev0d8JI26tAtWyyQA3WEhsKGTVxfQ56a0uQeFmOO63mqykBS1HNpw1rxSgaBgiyEBjODyIhAyxcn89Bj5fwCfXlj5HjSYjonHSErZoXiSqeyh0ZQSb2MgQIARAB
|
||||
```
|
||||
|
||||
Cookie:
|
||||
|
||||
```
|
||||
cookie: Bearer=ChA5Gev0d8JI26tAtWyyQA3WEhsKGTVxfQ56a0uQeFmOO63mqykBS1HNpw1rxSgaBgiyEBjODyIhAyxcn89Bj5fwCfXlj5HjSYjonHSErZoXiSqeyh0ZQSb2MgQIARAB
|
||||
```
|
||||
|
||||
## Put object
|
||||
|
||||
Route: `/upload/{cid}`
|
||||
|
||||
| Route parameter | Type | Description |
|
||||
|-----------------|--------|---------------------------------------------------------|
|
||||
| `cid` | Single | Base58 encoded container ID or container name from NNS. |
|
||||
|
||||
### Methods
|
||||
|
||||
#### POST
|
||||
|
||||
Upload file as object with attributes to FrostFS.
|
||||
|
||||
##### Request
|
||||
|
||||
###### Headers
|
||||
|
||||
| Header | Description |
|
||||
|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| Common headers | See [bearer token](#bearer-token). |
|
||||
| `X-Attribute-System-*` | Used to set system FrostFS object attributes <br/> (e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). |
|
||||
| `X-Attribute-*` | Used to set regular object attributes <br/> (e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). |
|
||||
| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. |
|
||||
|
||||
There are some reserved headers type of `X-Attribute-FROSTFS-*` (headers are arranged in descending order of priority):
|
||||
|
||||
1. `X-Attribute-System-Expiration-Epoch: 100`
|
||||
2. `X-Attribute-System-Expiration-Duration: 24h30m`
|
||||
3. `X-Attribute-System-Expiration-Timestamp: 1637574797`
|
||||
4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z`
|
||||
|
||||
which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way.
|
||||
|
||||
If you don't specify the `X-Attribute-Timestamp` header the `Timestamp` attribute can be set anyway
|
||||
(see http-gw [configuration](gate-configuration.md#upload-header-section)).
|
||||
|
||||
The `X-Attribute-*` headers must be unique. If you provide several the same headers only one will be used.
|
||||
Attribute key and value must be valid utf8 string. All attributes in sum must not be greater than 3mb.
|
||||
|
||||
###### Body
|
||||
|
||||
Body must contain multipart form with file.
|
||||
The `filename` field from the multipart form will be set as `FileName` attribute of object
|
||||
(can be overriden by `X-Attribute-FileName` header).
|
||||
|
||||
##### Response
|
||||
|
||||
###### Status codes
|
||||
|
||||
| Status | Description |
|
||||
|--------|----------------------------------------------|
|
||||
| 200 | Object created successfully. |
|
||||
| 400 | Some error occurred during object uploading. |
|
||||
|
||||
## Get object
|
||||
|
||||
Route: `/get/{cid}/{oid}?[download=true]`
|
||||
|
||||
| Route parameter | Type | Description |
|
||||
|-----------------|--------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `cid` | Single | Base58 encoded container ID or container name from NNS. |
|
||||
| `oid` | Single | Base58 encoded object ID. |
|
||||
| `download` | Query | Set the `Content-Disposition` header as `attachment` in response.<br/> This make the browser to download object as file instead of showing it on the page. |
|
||||
|
||||
### Methods
|
||||
|
||||
#### GET
|
||||
|
||||
Get an object (payload and attributes) by an address.
|
||||
|
||||
##### Request
|
||||
|
||||
###### Headers
|
||||
|
||||
| Header | Description |
|
||||
|----------------|------------------------------------|
|
||||
| Common headers | See [bearer token](#bearer-token). |
|
||||
|
||||
##### Response
|
||||
|
||||
###### Headers
|
||||
|
||||
| Header | Description |
|
||||
|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
|
||||
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
|
||||
| `Content-Disposition` | Indicate how to browsers should treat file. <br/> Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). |
|
||||
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
|
||||
| `Content-Length` | Size of object payload. |
|
||||
| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). |
|
||||
| `X-Owner-Id` | Base58 encoded owner ID. |
|
||||
| `X-Container-Id` | Base58 encoded container ID. |
|
||||
| `X-Object-Id` | Base58 encoded object ID. |
|
||||
|
||||
###### Status codes
|
||||
|
||||
| Status | Description |
|
||||
|--------|------------------------------------------------|
|
||||
| 200 | Object got successfully. |
|
||||
| 400 | Some error occurred during object downloading. |
|
||||
| 404 | Container or object not found. |
|
||||
|
||||
#### HEAD
|
||||
|
||||
Get an object attributes by an address.
|
||||
|
||||
##### Request
|
||||
|
||||
###### Headers
|
||||
|
||||
| Header | Description |
|
||||
|----------------|------------------------------------|
|
||||
| Common headers | See [bearer token](#bearer-token). |
|
||||
|
||||
##### Response
|
||||
|
||||
###### Headers
|
||||
|
||||
| Header | Description |
|
||||
|------------------------|------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
|
||||
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
|
||||
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
|
||||
| `Content-Length` | Size of object payload. |
|
||||
| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). |
|
||||
| `X-Owner-Id` | Base58 encoded owner ID. |
|
||||
| `X-Container-Id` | Base58 encoded container ID. |
|
||||
| `X-Object-Id` | Base58 encoded object ID. |
|
||||
|
||||
###### Status codes
|
||||
|
||||
| Status | Description |
|
||||
|--------|---------------------------------------------------|
|
||||
| 200 | Object head successfully. |
|
||||
| 400 | Some error occurred during object HEAD operation. |
|
||||
| 404 | Container or object not found. |
|
||||
|
||||
## Search object
|
||||
|
||||
Route: `/get_by_attribute/{cid}/{attr_key}/{attr_val}?[download=true]`
|
||||
|
||||
| Route parameter | Type | Description |
|
||||
|-----------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `cid` | Single | Base58 encoded container ID or container name from NNS. |
|
||||
| `attr_key` | Single | Object attribute key to search. |
|
||||
| `attr_val` | Catch-All | Object attribute value to match. |
|
||||
| `download` | Query | Set the `Content-Disposition` header as `attachment` in response. This make the browser to download object as file instead of showing it on the page. |
|
||||
|
||||
### Methods
|
||||
|
||||
#### GET
|
||||
|
||||
Find and get an object (payload and attributes) by a specific attribute.
|
||||
If more than one object is found, an arbitrary one will be returned.
|
||||
|
||||
##### Request
|
||||
|
||||
###### Headers
|
||||
|
||||
| Header | Description |
|
||||
|----------------|------------------------------------|
|
||||
| Common headers | See [bearer token](#bearer-token). |
|
||||
|
||||
##### Response
|
||||
|
||||
###### Headers
|
||||
|
||||
| Header | Description |
|
||||
|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
|
||||
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
|
||||
| `Content-Disposition` | Indicate how to browsers should treat file. <br/> Set `filename` as base part of `FileName` object attribute (if it's set, empty otherwise). |
|
||||
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
|
||||
| `Content-Length` | Size of object payload. |
|
||||
| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). |
|
||||
| `X-Owner-Id` | Base58 encoded owner ID. |
|
||||
| `X-Container-Id` | Base58 encoded container ID. |
|
||||
| `X-Object-Id` | Base58 encoded object ID. |
|
||||
|
||||
###### Status codes
|
||||
|
||||
| Status | Description |
|
||||
|--------|------------------------------------------------|
|
||||
| 200 | Object got successfully. |
|
||||
| 400 | Some error occurred during object downloading. |
|
||||
| 404 | Container or object not found. |
|
||||
|
||||
#### HEAD
|
||||
|
||||
Get object attributes by a specific attribute.
|
||||
If more than one object is found, an arbitrary one will be used to get attributes.
|
||||
|
||||
##### Request
|
||||
|
||||
###### Headers
|
||||
|
||||
| Header | Description |
|
||||
|----------------|------------------------------------|
|
||||
| Common headers | See [bearer token](#bearer-token). |
|
||||
|
||||
##### Response
|
||||
|
||||
###### Headers
|
||||
|
||||
| Header | Description |
|
||||
|------------------------|------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `X-Attribute-System-*` | System FrostFS object attributes <br/> (e.g. `__SYSTEM__EXPIRATION_EPOCH` set "X-Attribute-System-Expiration-Epoch" header). |
|
||||
| `X-Attribute-*` | Regular object attributes <br/> (e.g. `My-Tag` set "X-Attribute-My-Tag" header). |
|
||||
| `Content-Type` | Indicate content type of object. Set from `Content-Type` attribute or detected using payload. |
|
||||
| `Content-Length` | Size of object payload. |
|
||||
| `Last-Modified` | Contains the `Timestamp` attribute (if exists) formatted as HTTP time (RFC7231,RFC1123). |
|
||||
| `X-Owner-Id` | Base58 encoded owner ID. |
|
||||
| `X-Container-Id` | Base58 encoded container ID. |
|
||||
| `X-Object-Id` | Base58 encoded object ID. |
|
||||
|
||||
###### Status codes
|
||||
|
||||
| Status | Description |
|
||||
|--------|---------------------------------------|
|
||||
| 200 | Object head successfully. |
|
||||
| 400 | Some error occurred during operation. |
|
||||
| 404 | Container or object not found. |
|
||||
|
||||
## Download zip
|
||||
|
||||
Route: `/zip/{cid}/{prefix}`
|
||||
|
||||
| Route parameter | Type | Description |
|
||||
|-----------------|-----------|---------------------------------------------------------|
|
||||
| `cid` | Single | Base58 encoded container ID or container name from NNS. |
|
||||
| `prefix` | Catch-All | Prefix for object attribute `FilePath` to match. |
|
||||
|
||||
### Methods
|
||||
|
||||
#### GET
|
||||
|
||||
Find objects by prefix for `FilePath` attributes. Return found objects in zip archive.
|
||||
Name of files in archive sets to `FilePath` attribute of objects.
|
||||
Time of files sets to time when object has started downloading.
|
||||
You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` route.
|
||||
|
||||
Archive can be compressed (see http-gw [configuration](gate-configuration.md#zip-section)).
|
||||
|
||||
##### Request
|
||||
|
||||
###### Headers
|
||||
|
||||
| Header | Description |
|
||||
|----------------|------------------------------------|
|
||||
| Common headers | See [bearer token](#bearer-token). |
|
||||
|
||||
##### Response
|
||||
|
||||
###### Headers
|
||||
|
||||
| Header | Description |
|
||||
|-----------------------|-------------------------------------------------------------------------------------------------------------------|
|
||||
| `Content-Disposition` | Indicate how to browsers should treat file (`attachment`). Set `filename` as `archive.zip`. |
|
||||
| `Content-Type` | Indicate content type of object. Set to `application/zip` |
|
||||
|
||||
###### Status codes
|
||||
|
||||
| Status | Description |
|
||||
|--------|-----------------------------------------------------|
|
||||
| 200 | Object got successfully. |
|
||||
| 400 | Some error occurred during object downloading. |
|
||||
| 404 | Container or objects not found. |
|
||||
| 500 | Some inner error (e.g. error on streaming objects). |
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
# FrostFS HTTP Gateway configuration file
|
||||
# NeoFS HTTP Gateway configuration file
|
||||
|
||||
This section contains detailed FrostFS HTTP Gateway configuration file description
|
||||
This section contains detailed NeoFS HTTP Gateway configuration file description
|
||||
including default config values and some tips to set up configurable values.
|
||||
|
||||
There are some custom types used for brevity:
|
||||
|
|
@ -23,19 +23,19 @@ $ kill -s SIGHUP <app_pid>
|
|||
Example:
|
||||
|
||||
```shell
|
||||
$ ./bin/frostfs-http-gw --config config.yaml &> http.log &
|
||||
$ ./bin/neofs-http-gw --config config.yaml &> http.log &
|
||||
[1] 998346
|
||||
|
||||
$ cat http.log
|
||||
# ...
|
||||
2022-10-03T09:37:25.826+0300 info frostfs-http-gw/app.go:332 starting application {"app_name": "frostfs-http-gw", "version": "v0.24.0"}
|
||||
2022-10-03T09:37:25.826+0300 info neofs-http-gw/app.go:332 starting application {"app_name": "neofs-http-gw", "version": "v0.24.0"}
|
||||
# ...
|
||||
|
||||
$ kill -s SIGHUP 998346
|
||||
|
||||
$ cat http.log
|
||||
# ...
|
||||
2022-10-03T09:38:16.205+0300 info frostfs-http-gw/app.go:470 SIGHUP config reload completed
|
||||
2022-10-03T09:38:16.205+0300 info neofs-http-gw/app.go:470 SIGHUP config reload completed
|
||||
```
|
||||
|
||||
# Structure
|
||||
|
|
@ -47,7 +47,6 @@ $ cat http.log
|
|||
| `peers` | [Nodes configuration](#peers-section) |
|
||||
| `logger` | [Logger configuration](#logger-section) |
|
||||
| `web` | [Web configuration](#web-section) |
|
||||
| `server` | [Server configuration](#server-section) |
|
||||
| `upload-header` | [Upload header configuration](#upload-header-section) |
|
||||
| `zip` | [ZIP configuration](#zip-section) |
|
||||
| `pprof` | [Pprof configuration](#pprof-section) |
|
||||
|
|
@ -57,7 +56,11 @@ $ cat http.log
|
|||
# General section
|
||||
|
||||
```yaml
|
||||
rpc_endpoint: http://morph-chain.frostfs.devenv:30333
|
||||
listen_address: 0.0.0.0:8082
|
||||
tls_certificate: /path/to/tls/cert
|
||||
tls_key: /path/to/tls/key
|
||||
|
||||
rpc_endpoint: http://morph-chain.neofs.devenv:30333
|
||||
resolve_order:
|
||||
- nns
|
||||
- dns
|
||||
|
|
@ -71,6 +74,9 @@ pool_error_threshold: 100
|
|||
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|------------------------|------------|---------------|----------------|------------------------------------------------------------------------------------|
|
||||
| `listen_address` | `string` | | `0.0.0.0:8082` | The address that the gateway is listening on. |
|
||||
| `tls_certificate` | `string` | yes | | Path to the TLS certificate. |
|
||||
| `tls_key` | `string` | yes | | Path to the TLS key. |
|
||||
| `rpc_endpoint` | `string` | yes | | The address of the RPC host to which the gateway connects to resolve bucket names. |
|
||||
| `resolve_order` | `[]string` | yes | `[nns, dns]` | Order of bucket name resolvers to use. |
|
||||
| `connect_timeout` | `duration` | | `10s` | Timeout to connect to a node. |
|
||||
|
|
@ -98,23 +104,23 @@ wallet:
|
|||
|
||||
```yaml
|
||||
# Nodes configuration
|
||||
# This configuration makes the gateway use the first node (node1.frostfs:8080)
|
||||
# while it's healthy. Otherwise, gateway uses the second node (node2.frostfs:8080)
|
||||
# for 10% of requests and the third node (node3.frostfs:8080) for 90% of requests.
|
||||
# This configuration makes the gateway use the first node (node1.neofs:8080)
|
||||
# while it's healthy. Otherwise, gateway uses the second node (node2.neofs:8080)
|
||||
# for 10% of requests and the third node (node3.neofs:8080) for 90% of requests.
|
||||
# Until nodes with the same priority level are healthy
|
||||
# nodes with other priority are not used.
|
||||
# The lower the value, the higher the priority.
|
||||
peers:
|
||||
0:
|
||||
address: node1.frostfs:8080
|
||||
address: node1.neofs:8080
|
||||
priority: 1
|
||||
weight: 1
|
||||
1:
|
||||
address: node2.frostfs:8080
|
||||
address: node2.neofs:8080
|
||||
priority: 2
|
||||
weight: 0.1
|
||||
2:
|
||||
address: node3.frostfs:8080
|
||||
address: node3.neofs:8080
|
||||
priority: 2
|
||||
weight: 0.9
|
||||
```
|
||||
|
|
@ -125,32 +131,6 @@ peers:
|
|||
| `priority` | `int` | `1` | It allows to group nodes and don't switch group until all nodes with the same priority will be unhealthy. The lower the value, the higher the priority. |
|
||||
| `weight` | `float` | `1` | Weight of node in the group with the same priority. Distribute requests to nodes proportionally to these values. |
|
||||
|
||||
# `server` section
|
||||
|
||||
You can specify several listeners for server. For example, for `http` and `https`.
|
||||
|
||||
```yaml
|
||||
server:
|
||||
- address: 0.0.0.0:8080
|
||||
tls:
|
||||
enabled: false
|
||||
cert_file: /path/to/cert
|
||||
key_file: /path/to/key
|
||||
- address: 0.0.0.0:8081
|
||||
tls:
|
||||
enabled: true
|
||||
cert_file: /path/to/another/cert
|
||||
key_file: /path/to/another/key
|
||||
```
|
||||
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|-----------------|----------|---------------|----------------|-----------------------------------------------|
|
||||
| `address` | `string` | | `0.0.0.0:8080` | The address that the gateway is listening on. |
|
||||
| `tls.enabled` | `bool` | | false | Enable TLS or not. |
|
||||
| `tls.cert_file` | `string` | yes | | Path to the TLS certificate. |
|
||||
| `tls.key_file` | `string` | yes | | Path to the key. |
|
||||
|
||||
|
||||
# `logger` section
|
||||
|
||||
```yaml
|
||||
|
|
|
|||
|
|
@ -14,18 +14,20 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||
"github.com/nspcc-dev/neofs-http-gw/response"
|
||||
"github.com/nspcc-dev/neofs-http-gw/tokens"
|
||||
"github.com/nspcc-dev/neofs-http-gw/utils"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/client"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/container"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/atomic"
|
||||
"go.uber.org/zap"
|
||||
|
|
@ -109,7 +111,7 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) {
|
|||
|
||||
rObj, err := clnt.GetObject(r.appCtx, prm)
|
||||
if err != nil {
|
||||
r.handleFrostFSErr(err, start)
|
||||
r.handleNeoFSErr(err, start)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -129,9 +131,9 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) {
|
|||
if !isValidToken(key) || !isValidValue(val) {
|
||||
continue
|
||||
}
|
||||
|
||||
key = utils.BackwardTransformIfSystem(key)
|
||||
|
||||
if strings.HasPrefix(key, utils.SystemAttributePrefix) {
|
||||
key = systemBackwardTranslator(key)
|
||||
}
|
||||
r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
|
||||
switch key {
|
||||
case object.AttributeFileName:
|
||||
|
|
@ -185,6 +187,36 @@ func (r request) receiveFile(clnt *pool.Pool, objectAddress oid.Address) {
|
|||
r.Response.SetBodyStream(rObj.Payload, int(payloadSize))
|
||||
}
|
||||
|
||||
// systemBackwardTranslator is used to convert headers looking like '__NEOFS__ATTR_NAME' to 'Neofs-Attr-Name'.
|
||||
func systemBackwardTranslator(key string) string {
|
||||
// trim specified prefix '__NEOFS__'
|
||||
key = strings.TrimPrefix(key, utils.SystemAttributePrefix)
|
||||
|
||||
var res strings.Builder
|
||||
res.WriteString("Neofs-")
|
||||
|
||||
strs := strings.Split(key, "_")
|
||||
for i, s := range strs {
|
||||
s = title(strings.ToLower(s))
|
||||
res.WriteString(s)
|
||||
if i != len(strs)-1 {
|
||||
res.WriteString("-")
|
||||
}
|
||||
}
|
||||
|
||||
return res.String()
|
||||
}
|
||||
|
||||
func title(str string) string {
|
||||
if str == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
r, size := utf8.DecodeRuneInString(str)
|
||||
r0 := unicode.ToTitle(r)
|
||||
return string(r0) + str[size:]
|
||||
}
|
||||
|
||||
func bearerToken(ctx context.Context) *bearer.Token {
|
||||
if tkn, err := tokens.LoadBearerToken(ctx); err == nil {
|
||||
return tkn
|
||||
|
|
@ -192,16 +224,20 @@ func bearerToken(ctx context.Context) *bearer.Token {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (r *request) handleFrostFSErr(err error, start time.Time) {
|
||||
logFields := []zap.Field{
|
||||
func (r *request) handleNeoFSErr(err error, start time.Time) {
|
||||
r.log.Error(
|
||||
"could not receive object",
|
||||
zap.Stringer("elapsed", time.Since(start)),
|
||||
zap.Error(err),
|
||||
}
|
||||
statusCode, msg, additionalFields := response.FormErrorResponse("could not receive object", err)
|
||||
logFields = append(logFields, additionalFields...)
|
||||
)
|
||||
|
||||
r.log.Error("could not receive object", logFields...)
|
||||
response.Error(r.RequestCtx, msg, statusCode)
|
||||
if client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err) {
|
||||
response.Error(r.RequestCtx, "Not Found", fasthttp.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
msg := fmt.Sprintf("could not receive object: %v", err)
|
||||
response.Error(r.RequestCtx, msg, fasthttp.StatusBadRequest)
|
||||
}
|
||||
|
||||
// Downloader is a download request handler.
|
||||
|
|
@ -464,7 +500,7 @@ func (d *Downloader) zipObject(zipWriter *zip.Writer, addr oid.Address, btoken *
|
|||
|
||||
resGet, err := d.pool.GetObject(d.appCtx, prm)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get FrostFS object: %v", err)
|
||||
return fmt.Errorf("get NeoFS object: %v", err)
|
||||
}
|
||||
|
||||
objWriter, err := d.addObjectToZip(zipWriter, &resGet.Header)
|
||||
|
|
|
|||
23
downloader/download_test.go
Normal file
23
downloader/download_test.go
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
package downloader
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSystemBackwardTranslator(t *testing.T) {
|
||||
input := []string{
|
||||
"__NEOFS__EXPIRATION_EPOCH",
|
||||
"__NEOFS__RANDOM_ATTR",
|
||||
}
|
||||
expected := []string{
|
||||
"Neofs-Expiration-Epoch",
|
||||
"Neofs-Random-Attr",
|
||||
}
|
||||
|
||||
for i, str := range input {
|
||||
res := systemBackwardTranslator(str)
|
||||
require.Equal(t, expected[i], res)
|
||||
}
|
||||
}
|
||||
|
|
@ -4,14 +4,15 @@ import (
|
|||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
"github.com/nspcc-dev/neofs-http-gw/response"
|
||||
"github.com/nspcc-dev/neofs-http-gw/tokens"
|
||||
"github.com/nspcc-dev/neofs-http-gw/utils"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
|
@ -43,7 +44,7 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) {
|
|||
|
||||
obj, err := clnt.HeadObject(r.appCtx, prm)
|
||||
if err != nil {
|
||||
r.handleFrostFSErr(err, start)
|
||||
r.handleNeoFSErr(err, start)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -55,9 +56,9 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) {
|
|||
if !isValidToken(key) || !isValidValue(val) {
|
||||
continue
|
||||
}
|
||||
|
||||
key = utils.BackwardTransformIfSystem(key)
|
||||
|
||||
if strings.HasPrefix(key, utils.SystemAttributePrefix) {
|
||||
key = systemBackwardTranslator(key)
|
||||
}
|
||||
r.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
|
||||
switch key {
|
||||
case object.AttributeTimestamp:
|
||||
|
|
@ -93,7 +94,7 @@ func (r request) headObject(clnt *pool.Pool, objectAddress oid.Address) {
|
|||
return &resObj, nil
|
||||
})
|
||||
if err != nil && err != io.EOF {
|
||||
r.handleFrostFSErr(err, start)
|
||||
r.handleNeoFSErr(err, start)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
//go:build !integration
|
||||
|
||||
package downloader
|
||||
|
||||
import (
|
||||
|
|
|
|||
73
go.mod
73
go.mod
|
|
@ -1,71 +1,70 @@
|
|||
module git.frostfs.info/TrueCloudLab/frostfs-http-gw
|
||||
module github.com/nspcc-dev/neofs-http-gw
|
||||
|
||||
go 1.18
|
||||
go 1.17
|
||||
|
||||
require (
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85
|
||||
github.com/fasthttp/router v1.4.1
|
||||
github.com/nspcc-dev/neo-go v0.101.0
|
||||
github.com/nspcc-dev/neo-go v0.99.4
|
||||
github.com/nspcc-dev/neofs-api-go/v2 v2.14.0
|
||||
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.7.0.20221115140820-b4b07a3c4e11
|
||||
github.com/prometheus/client_golang v1.13.0
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/spf13/viper v1.15.0
|
||||
github.com/stretchr/testify v1.8.1
|
||||
github.com/spf13/viper v1.8.1
|
||||
github.com/stretchr/testify v1.8.0
|
||||
github.com/testcontainers/testcontainers-go v0.13.0
|
||||
github.com/valyala/fasthttp v1.34.0
|
||||
go.uber.org/atomic v1.10.0
|
||||
go.uber.org/zap v1.24.0
|
||||
go.uber.org/zap v1.23.0
|
||||
)
|
||||
|
||||
require (
|
||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb // indirect
|
||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.0 // indirect
|
||||
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
|
||||
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
|
||||
github.com/Microsoft/go-winio v0.5.2 // indirect
|
||||
github.com/Microsoft/hcsshim v0.9.2 // indirect
|
||||
github.com/andybalholm/brotli v1.0.4 // indirect
|
||||
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect
|
||||
github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/btcsuite/btcd v0.22.0-beta // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/containerd/cgroups v1.0.3 // indirect
|
||||
github.com/containerd/containerd v1.6.2 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
|
||||
github.com/docker/distribution v2.8.1+incompatible // indirect
|
||||
github.com/docker/docker v20.10.14+incompatible // indirect
|
||||
github.com/docker/go-connections v0.4.0 // indirect
|
||||
github.com/docker/go-units v0.4.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.4.9 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/gorilla/mux v1.8.0 // indirect
|
||||
github.com/gorilla/websocket v1.4.2 // indirect
|
||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/klauspost/compress v1.15.0 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/magiconair/properties v1.8.6 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.1 // indirect
|
||||
github.com/moby/sys/mount v0.3.2 // indirect
|
||||
github.com/moby/sys/mountinfo v0.6.1 // indirect
|
||||
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect
|
||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect
|
||||
github.com/nspcc-dev/hrw v1.0.9 // indirect
|
||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262 // indirect
|
||||
github.com/nspcc-dev/neofs-contract v0.16.0 // indirect
|
||||
github.com/nspcc-dev/neofs-crypto v0.4.0 // indirect
|
||||
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
|
||||
github.com/nspcc-dev/tzhash v1.6.1 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.0.2 // indirect
|
||||
github.com/opencontainers/runc v1.1.1 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
|
||||
github.com/pelletier/go-toml v1.9.3 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_model v0.2.0 // indirect
|
||||
|
|
@ -75,26 +74,26 @@ require (
|
|||
github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect
|
||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/spf13/afero v1.9.3 // indirect
|
||||
github.com/spf13/cast v1.5.0 // indirect
|
||||
github.com/spf13/afero v1.6.0 // indirect
|
||||
github.com/spf13/cast v1.3.1 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/subosito/gotenv v1.4.2 // indirect
|
||||
github.com/subosito/gotenv v1.2.0 // indirect
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
|
||||
github.com/urfave/cli v1.22.5 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
golang.org/x/crypto v0.4.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20221227203929-1b447090c38c // indirect
|
||||
golang.org/x/net v0.4.0 // indirect
|
||||
golang.org/x/sync v0.1.0 // indirect
|
||||
golang.org/x/sys v0.3.0 // indirect
|
||||
golang.org/x/term v0.3.0 // indirect
|
||||
golang.org/x/text v0.5.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect
|
||||
google.golang.org/grpc v1.52.0 // indirect
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
go.uber.org/multierr v1.8.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
|
||||
golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect
|
||||
google.golang.org/grpc v1.48.0 // indirect
|
||||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/ini.v1 v1.62.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
|
@ -15,16 +13,15 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
containerv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/container"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/container/acl"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
|
|
@ -47,10 +44,11 @@ func TestIntegration(t *testing.T) {
|
|||
rootCtx := context.Background()
|
||||
aioImage := "nspccdev/neofs-aio-testcontainer:"
|
||||
versions := []string{
|
||||
"0.27.5",
|
||||
"0.28.1",
|
||||
"0.29.0",
|
||||
"0.30.0",
|
||||
"0.32.0",
|
||||
"0.34.0",
|
||||
"latest",
|
||||
}
|
||||
key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb")
|
||||
|
|
@ -69,7 +67,6 @@ func TestIntegration(t *testing.T) {
|
|||
require.NoError(t, err, version)
|
||||
|
||||
t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID, version) })
|
||||
t.Run("put with duplicate keys "+version, func(t *testing.T) { putWithDuplicateKeys(t, CID) })
|
||||
t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, ownerID, CID, version) })
|
||||
t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, ownerID, CID, version) })
|
||||
t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, ownerID, CID, version) })
|
||||
|
|
@ -174,43 +171,6 @@ func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnr
|
|||
}
|
||||
}
|
||||
|
||||
func putWithDuplicateKeys(t *testing.T, CID cid.ID) {
|
||||
url := testHost + "/upload/" + CID.String()
|
||||
|
||||
attr := "X-Attribute-User-Attribute"
|
||||
content := "content of file"
|
||||
valOne, valTwo := "first_value", "second_value"
|
||||
fileName := "newFile.txt"
|
||||
|
||||
var buff bytes.Buffer
|
||||
w := multipart.NewWriter(&buff)
|
||||
fw, err := w.CreateFormFile("file", fileName)
|
||||
require.NoError(t, err)
|
||||
_, err = io.Copy(fw, bytes.NewBufferString(content))
|
||||
require.NoError(t, err)
|
||||
err = w.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
request, err := http.NewRequest(http.MethodPost, url, &buff)
|
||||
require.NoError(t, err)
|
||||
request.Header.Set("Content-Type", w.FormDataContentType())
|
||||
request.Header.Add(attr, valOne)
|
||||
request.Header.Add(attr, valTwo)
|
||||
|
||||
resp, err := http.DefaultClient.Do(request)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
err := resp.Body.Close()
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "key duplication error: "+attr+"\n", string(body))
|
||||
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
|
||||
}
|
||||
|
||||
func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, CID cid.ID, version string) {
|
||||
content := "content of file"
|
||||
attributes := map[string]string{
|
||||
|
|
@ -375,7 +335,7 @@ func getDefaultConfig() *viper.Viper {
|
|||
v.SetDefault(cfgPeers+".0.priority", 1)
|
||||
|
||||
v.SetDefault(cfgRPCEndpoint, "http://localhost:30333")
|
||||
v.SetDefault("server.0.address", testListenAddress)
|
||||
v.SetDefault(cfgListenAddress, testListenAddress)
|
||||
|
||||
return v
|
||||
}
|
||||
|
|
@ -410,11 +370,7 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o
|
|||
if version >= versionWithNativeNames {
|
||||
var domain container.Domain
|
||||
domain.SetName(testContainerName)
|
||||
|
||||
// currently node in aio image knows nothing about new sys attributes
|
||||
// todo (@dkirillov): #2 use frostfs aio images that supports new attributes
|
||||
cnr.SetAttribute(containerv2.SysAttributeNameNeoFS, domain.Name())
|
||||
cnr.SetAttribute(containerv2.SysAttributeZoneNeoFS, domain.Zone())
|
||||
container.WriteDomain(&cnr, domain)
|
||||
}
|
||||
|
||||
var waitPrm pool.WaitParams
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@ package metrics
|
|||
import (
|
||||
"net/http"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const (
|
||||
namespace = "frostfs_http_gw"
|
||||
namespace = "neofs_http_gw"
|
||||
stateSubsystem = "state"
|
||||
poolSubsystem = "pool"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,35 +0,0 @@
|
|||
package resolver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
)
|
||||
|
||||
// FrostFSResolver represents virtual connection to the FrostFS network.
|
||||
// It implements resolver.FrostFS.
|
||||
type FrostFSResolver struct {
|
||||
pool *pool.Pool
|
||||
}
|
||||
|
||||
// NewFrostFSResolver creates new FrostFSResolver using provided pool.Pool.
|
||||
func NewFrostFSResolver(p *pool.Pool) *FrostFSResolver {
|
||||
return &FrostFSResolver{pool: p}
|
||||
}
|
||||
|
||||
// SystemDNS implements resolver.FrostFS interface method.
|
||||
func (x *FrostFSResolver) SystemDNS(ctx context.Context) (string, error) {
|
||||
networkInfo, err := x.pool.NetworkInfo(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read network info via client: %w", err)
|
||||
}
|
||||
|
||||
domain := networkInfo.RawNetworkParameter("SystemDNS")
|
||||
if domain == nil {
|
||||
return "", errors.New("system DNS parameter not found or empty")
|
||||
}
|
||||
|
||||
return string(domain), nil
|
||||
}
|
||||
35
resolver/neofs.go
Normal file
35
resolver/neofs.go
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
package resolver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
)
|
||||
|
||||
// NeoFSResolver represents virtual connection to the NeoFS network.
|
||||
// It implements resolver.NeoFS.
|
||||
type NeoFSResolver struct {
|
||||
pool *pool.Pool
|
||||
}
|
||||
|
||||
// NewNeoFSResolver creates new NeoFSResolver using provided pool.Pool.
|
||||
func NewNeoFSResolver(p *pool.Pool) *NeoFSResolver {
|
||||
return &NeoFSResolver{pool: p}
|
||||
}
|
||||
|
||||
// SystemDNS implements resolver.NeoFS interface method.
|
||||
func (x *NeoFSResolver) SystemDNS(ctx context.Context) (string, error) {
|
||||
networkInfo, err := x.pool.NetworkInfo(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read network info via client: %w", err)
|
||||
}
|
||||
|
||||
domain := networkInfo.RawNetworkParameter("SystemDNS")
|
||||
if domain == nil {
|
||||
return "", errors.New("system DNS parameter not found or empty")
|
||||
}
|
||||
|
||||
return string(domain), nil
|
||||
}
|
||||
|
|
@ -6,9 +6,9 @@ import (
|
|||
"fmt"
|
||||
"sync"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/container"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/ns"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -19,9 +19,9 @@ const (
|
|||
// ErrNoResolvers returns when trying to resolve container without any resolver.
|
||||
var ErrNoResolvers = errors.New("no resolvers")
|
||||
|
||||
// FrostFS represents virtual connection to the FrostFS network.
|
||||
type FrostFS interface {
|
||||
// SystemDNS reads system DNS network parameters of the FrostFS.
|
||||
// NeoFS represents virtual connection to the NeoFS network.
|
||||
type NeoFS interface {
|
||||
// SystemDNS reads system DNS network parameters of the NeoFS.
|
||||
//
|
||||
// Returns exactly on non-zero value. Returns any error encountered
|
||||
// which prevented the parameter to be read.
|
||||
|
|
@ -29,7 +29,7 @@ type FrostFS interface {
|
|||
}
|
||||
|
||||
type Config struct {
|
||||
FrostFS FrostFS
|
||||
NeoFS NeoFS
|
||||
RPCAddress string
|
||||
}
|
||||
|
||||
|
|
@ -135,7 +135,7 @@ func (r *ContainerResolver) equals(resolverNames []string) bool {
|
|||
func newResolver(name string, cfg *Config) (*Resolver, error) {
|
||||
switch name {
|
||||
case DNSResolver:
|
||||
return NewDNSResolver(cfg.FrostFS)
|
||||
return NewDNSResolver(cfg.NeoFS)
|
||||
case NNSResolver:
|
||||
return NewNNSResolver(cfg.RPCAddress)
|
||||
default:
|
||||
|
|
@ -143,17 +143,17 @@ func newResolver(name string, cfg *Config) (*Resolver, error) {
|
|||
}
|
||||
}
|
||||
|
||||
func NewDNSResolver(frostFS FrostFS) (*Resolver, error) {
|
||||
if frostFS == nil {
|
||||
func NewDNSResolver(neoFS NeoFS) (*Resolver, error) {
|
||||
if neoFS == nil {
|
||||
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
|
||||
}
|
||||
|
||||
var dns ns.DNS
|
||||
|
||||
resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) {
|
||||
domain, err := frostFS.SystemDNS(ctx)
|
||||
domain, err := neoFS.SystemDNS(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err)
|
||||
return nil, fmt.Errorf("read system DNS parameter of the NeoFS: %w", err)
|
||||
}
|
||||
|
||||
domain = name + "." + domain
|
||||
|
|
|
|||
|
|
@ -1,41 +1,7 @@
|
|||
package response
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
sdkstatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
import "github.com/valyala/fasthttp"
|
||||
|
||||
func Error(r *fasthttp.RequestCtx, msg string, code int) {
|
||||
r.Error(msg+"\n", code)
|
||||
}
|
||||
|
||||
func FormErrorResponse(message string, err error) (int, string, []zap.Field) {
|
||||
var (
|
||||
msg string
|
||||
statusCode int
|
||||
logFields []zap.Field
|
||||
)
|
||||
|
||||
st := new(sdkstatus.ObjectAccessDenied)
|
||||
|
||||
switch {
|
||||
case errors.As(err, &st):
|
||||
statusCode = fasthttp.StatusForbidden
|
||||
reason := st.Reason()
|
||||
msg = fmt.Sprintf("%s: %v: %s", message, err, reason)
|
||||
logFields = append(logFields, zap.String("error_detail", reason))
|
||||
case client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err):
|
||||
statusCode = fasthttp.StatusNotFound
|
||||
msg = "Not Found"
|
||||
default:
|
||||
statusCode = fasthttp.StatusBadRequest
|
||||
msg = fmt.Sprintf("%s: %v", message, err)
|
||||
}
|
||||
|
||||
return statusCode, msg, logFields
|
||||
}
|
||||
|
|
|
|||
122
server.go
122
server.go
|
|
@ -1,122 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type (
|
||||
ServerInfo struct {
|
||||
Address string
|
||||
TLS ServerTLSInfo
|
||||
}
|
||||
|
||||
ServerTLSInfo struct {
|
||||
Enabled bool
|
||||
CertFile string
|
||||
KeyFile string
|
||||
}
|
||||
|
||||
Server interface {
|
||||
Address() string
|
||||
Listener() net.Listener
|
||||
UpdateCert(certFile, keyFile string) error
|
||||
}
|
||||
|
||||
server struct {
|
||||
address string
|
||||
listener net.Listener
|
||||
tlsProvider *certProvider
|
||||
}
|
||||
|
||||
certProvider struct {
|
||||
Enabled bool
|
||||
|
||||
mu sync.RWMutex
|
||||
certPath string
|
||||
keyPath string
|
||||
cert *tls.Certificate
|
||||
}
|
||||
)
|
||||
|
||||
func (s *server) Address() string {
|
||||
return s.address
|
||||
}
|
||||
|
||||
func (s *server) Listener() net.Listener {
|
||||
return s.listener
|
||||
}
|
||||
|
||||
func (s *server) UpdateCert(certFile, keyFile string) error {
|
||||
return s.tlsProvider.UpdateCert(certFile, keyFile)
|
||||
}
|
||||
|
||||
func newServer(ctx context.Context, serverInfo ServerInfo) (*server, error) {
|
||||
var lic net.ListenConfig
|
||||
ln, err := lic.Listen(ctx, "tcp", serverInfo.Address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not prepare listener: %w", err)
|
||||
}
|
||||
|
||||
tlsProvider := &certProvider{
|
||||
Enabled: serverInfo.TLS.Enabled,
|
||||
}
|
||||
|
||||
if serverInfo.TLS.Enabled {
|
||||
if err = tlsProvider.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
|
||||
return nil, fmt.Errorf("failed to update cert: %w", err)
|
||||
}
|
||||
|
||||
ln = tls.NewListener(ln, &tls.Config{
|
||||
GetCertificate: tlsProvider.GetCertificate,
|
||||
})
|
||||
}
|
||||
|
||||
return &server{
|
||||
address: serverInfo.Address,
|
||||
listener: ln,
|
||||
tlsProvider: tlsProvider,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *certProvider) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
if !p.Enabled {
|
||||
return nil, errors.New("cert provider: disabled")
|
||||
}
|
||||
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return p.cert, nil
|
||||
}
|
||||
|
||||
func (p *certProvider) UpdateCert(certPath, keyPath string) error {
|
||||
if !p.Enabled {
|
||||
return fmt.Errorf("tls disabled")
|
||||
}
|
||||
|
||||
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot load TLS key pair from certFile '%s' and keyFile '%s': %w", certPath, keyPath, err)
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
p.certPath = certPath
|
||||
p.keyPath = keyPath
|
||||
p.cert = &cert
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *certProvider) FilePaths() (string, string) {
|
||||
if !p.Enabled {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return p.certPath, p.keyPath
|
||||
}
|
||||
146
settings.go
146
settings.go
|
|
@ -3,14 +3,13 @@ package main
|
|||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/valyala/fasthttp"
|
||||
|
|
@ -28,10 +27,9 @@ const (
|
|||
|
||||
defaultPoolErrorThreshold uint32 = 100
|
||||
|
||||
cfgServer = "server"
|
||||
cfgTLSEnabled = "tls.enabled"
|
||||
cfgTLSCertFile = "tls.cert_file"
|
||||
cfgTLSKeyFile = "tls.key_file"
|
||||
cfgListenAddress = "listen_address"
|
||||
cfgTLSCertificate = "tls_certificate"
|
||||
cfgTLSKey = "tls_key"
|
||||
|
||||
// Web.
|
||||
cfgWebReadBufferSize = "web.read_buffer_size"
|
||||
|
|
@ -78,15 +76,13 @@ const (
|
|||
cfgZipCompression = "zip.compression"
|
||||
|
||||
// Command line args.
|
||||
cmdHelp = "help"
|
||||
cmdVersion = "version"
|
||||
cmdPprof = "pprof"
|
||||
cmdMetrics = "metrics"
|
||||
cmdWallet = "wallet"
|
||||
cmdAddress = "address"
|
||||
cmdConfig = "config"
|
||||
cmdConfigDir = "config-dir"
|
||||
cmdListenAddress = "listen_address"
|
||||
cmdHelp = "help"
|
||||
cmdVersion = "version"
|
||||
cmdPprof = "pprof"
|
||||
cmdMetrics = "metrics"
|
||||
cmdWallet = "wallet"
|
||||
cmdAddress = "address"
|
||||
cmdConfig = "config"
|
||||
)
|
||||
|
||||
var ignore = map[string]struct{}{
|
||||
|
|
@ -116,17 +112,16 @@ func settings() *viper.Viper {
|
|||
|
||||
flags.StringP(cmdWallet, "w", "", `path to the wallet`)
|
||||
flags.String(cmdAddress, "", `address of wallet account`)
|
||||
flags.StringArray(cmdConfig, nil, "config paths")
|
||||
flags.String(cmdConfigDir, "", "config dir path")
|
||||
flags.String(cmdConfig, "", "config path")
|
||||
flags.Duration(cfgConTimeout, defaultConnectTimeout, "gRPC connect timeout")
|
||||
flags.Duration(cfgStreamTimeout, defaultStreamTimeout, "gRPC individual message timeout")
|
||||
flags.Duration(cfgReqTimeout, defaultRequestTimeout, "gRPC request timeout")
|
||||
flags.Duration(cfgRebalance, defaultRebalanceTimer, "gRPC connection rebalance timer")
|
||||
|
||||
flags.String(cmdListenAddress, "0.0.0.0:8080", "addresses to listen")
|
||||
flags.String(cfgTLSCertFile, "", "TLS certificate path")
|
||||
flags.String(cfgTLSKeyFile, "", "TLS key path")
|
||||
peers := flags.StringArrayP(cfgPeers, "p", nil, "FrostFS nodes")
|
||||
flags.String(cfgListenAddress, "0.0.0.0:8082", "address to listen")
|
||||
flags.String(cfgTLSCertificate, "", "TLS certificate path")
|
||||
flags.String(cfgTLSKey, "", "TLS key path")
|
||||
peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes")
|
||||
|
||||
resolveMethods := flags.StringSlice(cfgResolveOrder, []string{resolver.NNSResolver, resolver.DNSResolver}, "set container name resolve order")
|
||||
|
||||
|
|
@ -176,31 +171,17 @@ func settings() *viper.Viper {
|
|||
panic(err)
|
||||
}
|
||||
|
||||
if err := v.BindPFlag(cfgServer+".0.address", flags.Lookup(cmdListenAddress)); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := v.BindPFlag(cfgServer+".0."+cfgTLSKeyFile, flags.Lookup(cfgTLSKeyFile)); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := v.BindPFlag(cfgServer+".0."+cfgTLSCertFile, flags.Lookup(cfgTLSCertFile)); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if err := flags.Parse(os.Args); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if v.IsSet(cfgServer+".0."+cfgTLSKeyFile) && v.IsSet(cfgServer+".0."+cfgTLSCertFile) {
|
||||
v.Set(cfgServer+".0."+cfgTLSEnabled, true)
|
||||
}
|
||||
|
||||
if resolveMethods != nil {
|
||||
v.SetDefault(cfgResolveOrder, *resolveMethods)
|
||||
}
|
||||
|
||||
switch {
|
||||
case help != nil && *help:
|
||||
fmt.Printf("FrostFS HTTP Gateway %s\n", Version)
|
||||
fmt.Printf("NeoFS HTTP Gateway %s\n", Version)
|
||||
flags.PrintDefaults()
|
||||
|
||||
fmt.Println()
|
||||
|
|
@ -232,12 +213,14 @@ func settings() *viper.Viper {
|
|||
|
||||
os.Exit(0)
|
||||
case version != nil && *version:
|
||||
fmt.Printf("FrostFS HTTP Gateway\nVersion: %s\nGoVersion: %s\n", Version, runtime.Version())
|
||||
fmt.Printf("NeoFS HTTP Gateway\nVersion: %s\nGoVersion: %s\n", Version, runtime.Version())
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
if err := readInConfig(v); err != nil {
|
||||
panic(err)
|
||||
if v.IsSet(cmdConfig) {
|
||||
if err := readConfig(v); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
if peers != nil && len(*peers) > 0 {
|
||||
|
|
@ -251,72 +234,17 @@ func settings() *viper.Viper {
|
|||
return v
|
||||
}
|
||||
|
||||
func readInConfig(v *viper.Viper) error {
|
||||
if v.IsSet(cmdConfig) {
|
||||
if err := readConfig(v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if v.IsSet(cmdConfigDir) {
|
||||
if err := readConfigDir(v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func readConfigDir(v *viper.Viper) error {
|
||||
cfgSubConfigDir := v.GetString(cmdConfigDir)
|
||||
entries, err := os.ReadDir(cfgSubConfigDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
ext := path.Ext(entry.Name())
|
||||
if ext != ".yaml" && ext != ".yml" {
|
||||
continue
|
||||
}
|
||||
|
||||
if err = mergeConfig(v, path.Join(cfgSubConfigDir, entry.Name())); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func readConfig(v *viper.Viper) error {
|
||||
for _, fileName := range v.GetStringSlice(cmdConfig) {
|
||||
if err := mergeConfig(v, fileName); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeConfig(v *viper.Viper, fileName string) error {
|
||||
cfgFile, err := os.Open(fileName)
|
||||
cfgFileName := v.GetString(cmdConfig)
|
||||
cfgFile, err := os.Open(cfgFileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if errClose := cfgFile.Close(); errClose != nil {
|
||||
panic(errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
if err = v.MergeConfig(cfgFile); err != nil {
|
||||
if err = v.ReadConfig(cfgFile); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
return cfgFile.Close()
|
||||
}
|
||||
|
||||
// newLogger constructs a zap.Logger instance for current application.
|
||||
|
|
@ -369,25 +297,3 @@ func getLogLevel(v *viper.Viper) (zapcore.Level, error) {
|
|||
}
|
||||
return lvl, nil
|
||||
}
|
||||
|
||||
func fetchServers(v *viper.Viper) []ServerInfo {
|
||||
var servers []ServerInfo
|
||||
|
||||
for i := 0; ; i++ {
|
||||
key := cfgServer + "." + strconv.Itoa(i) + "."
|
||||
|
||||
var serverInfo ServerInfo
|
||||
serverInfo.Address = v.GetString(key + "address")
|
||||
serverInfo.TLS.Enabled = v.GetBool(key + cfgTLSEnabled)
|
||||
serverInfo.TLS.KeyFile = v.GetString(key + cfgTLSKeyFile)
|
||||
serverInfo.TLS.CertFile = v.GetString(key + cfgTLSCertFile)
|
||||
|
||||
if serverInfo.Address == "" {
|
||||
break
|
||||
}
|
||||
|
||||
servers = append(servers, serverInfo)
|
||||
}
|
||||
|
||||
return servers
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,12 @@
|
|||
//go:build !integration
|
||||
|
||||
package tokens
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,14 +3,30 @@ package uploader
|
|||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
"github.com/nspcc-dev/neofs-api-go/v2/object"
|
||||
"github.com/nspcc-dev/neofs-http-gw/utils"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]string, error) {
|
||||
var err error
|
||||
var neofsAttributeHeaderPrefixes = [...][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")}
|
||||
|
||||
func systemTranslator(key, prefix []byte) []byte {
|
||||
// replace the specified prefix with `__NEOFS__`
|
||||
key = bytes.Replace(key, prefix, []byte(utils.SystemAttributePrefix), 1)
|
||||
|
||||
// replace `-` with `_`
|
||||
key = bytes.ReplaceAll(key, []byte("-"), []byte("_"))
|
||||
|
||||
// replace with uppercase
|
||||
return bytes.ToUpper(key)
|
||||
}
|
||||
|
||||
func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) map[string]string {
|
||||
result := make(map[string]string)
|
||||
prefix := []byte(utils.UserAttributeHeaderPrefix)
|
||||
|
||||
|
|
@ -26,24 +42,23 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st
|
|||
}
|
||||
|
||||
// removing attribute prefix
|
||||
clearKey := bytes.TrimPrefix(key, prefix)
|
||||
key = bytes.TrimPrefix(key, prefix)
|
||||
|
||||
clearKey = utils.TransformIfSystem(clearKey)
|
||||
|
||||
// checks that the attribute key is not empty
|
||||
if len(clearKey) == 0 {
|
||||
return
|
||||
// checks that it's a system NeoFS header
|
||||
for _, system := range neofsAttributeHeaderPrefixes {
|
||||
if bytes.HasPrefix(key, system) {
|
||||
key = systemTranslator(key, system)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// check if key gets duplicated
|
||||
// return error containing full key name (with prefix)
|
||||
if _, ok := result[string(clearKey)]; ok {
|
||||
err = fmt.Errorf("key duplication error: %s", string(key))
|
||||
// checks that the attribute key is not empty
|
||||
if len(key) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// make string representation of key / val
|
||||
k, v := string(clearKey), string(val)
|
||||
k, v := string(key), string(val)
|
||||
|
||||
result[k] = v
|
||||
|
||||
|
|
@ -52,5 +67,73 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st
|
|||
zap.String("val", v))
|
||||
})
|
||||
|
||||
return result, err
|
||||
return result
|
||||
}
|
||||
|
||||
func prepareExpirationHeader(headers map[string]string, epochDurations *epochDurations) error {
|
||||
expirationInEpoch := headers[object.SysAttributeExpEpoch]
|
||||
|
||||
if timeRFC3339, ok := headers[utils.ExpirationRFC3339Attr]; ok {
|
||||
expTime, err := time.Parse(time.RFC3339, timeRFC3339)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, utils.ExpirationRFC3339Attr)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
if expTime.Before(now) {
|
||||
return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, utils.ExpirationRFC3339Attr)
|
||||
}
|
||||
updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
|
||||
delete(headers, utils.ExpirationRFC3339Attr)
|
||||
}
|
||||
|
||||
if timestamp, ok := headers[utils.ExpirationTimestampAttr]; ok {
|
||||
value, err := strconv.ParseInt(timestamp, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't parse value %s of header %s", timestamp, utils.ExpirationTimestampAttr)
|
||||
}
|
||||
expTime := time.Unix(value, 0)
|
||||
|
||||
now := time.Now()
|
||||
if expTime.Before(now) {
|
||||
return fmt.Errorf("value %s of header %s must be in the future", timestamp, utils.ExpirationTimestampAttr)
|
||||
}
|
||||
updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
|
||||
delete(headers, utils.ExpirationTimestampAttr)
|
||||
}
|
||||
|
||||
if duration, ok := headers[utils.ExpirationDurationAttr]; ok {
|
||||
expDuration, err := time.ParseDuration(duration)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't parse value %s of header %s", duration, utils.ExpirationDurationAttr)
|
||||
}
|
||||
if expDuration <= 0 {
|
||||
return fmt.Errorf("value %s of header %s must be positive", expDuration, utils.ExpirationDurationAttr)
|
||||
}
|
||||
updateExpirationHeader(headers, epochDurations, expDuration)
|
||||
delete(headers, utils.ExpirationDurationAttr)
|
||||
}
|
||||
|
||||
if expirationInEpoch != "" {
|
||||
headers[object.SysAttributeExpEpoch] = expirationInEpoch
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func updateExpirationHeader(headers map[string]string, durations *epochDurations, expDuration time.Duration) {
|
||||
epochDuration := uint64(durations.msPerBlock) * durations.blockPerEpoch
|
||||
currentEpoch := durations.currentEpoch
|
||||
numEpoch := uint64(expDuration.Milliseconds()) / epochDuration
|
||||
|
||||
if uint64(expDuration.Milliseconds())%epochDuration != 0 {
|
||||
numEpoch++
|
||||
}
|
||||
|
||||
expirationEpoch := uint64(math.MaxUint64)
|
||||
if numEpoch < math.MaxUint64-currentEpoch {
|
||||
expirationEpoch = currentEpoch + numEpoch
|
||||
}
|
||||
|
||||
headers[object.SysAttributeExpEpoch] = strconv.FormatUint(expirationEpoch, 10)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
//go:build !integration
|
||||
|
||||
package uploader
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-api-go/v2/object"
|
||||
"github.com/nspcc-dev/neofs-http-gw/utils"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
|
|
@ -13,41 +16,175 @@ import (
|
|||
func TestFilter(t *testing.T) {
|
||||
log := zap.NewNop()
|
||||
|
||||
t.Run("duplicate keys error", func(t *testing.T) {
|
||||
req := &fasthttp.RequestHeader{}
|
||||
req.DisableNormalizing()
|
||||
req.Add("X-Attribute-DupKey", "first-value")
|
||||
req.Add("X-Attribute-DupKey", "second-value")
|
||||
_, err := filterHeaders(log, req)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("duplicate system keys error", func(t *testing.T) {
|
||||
req := &fasthttp.RequestHeader{}
|
||||
req.DisableNormalizing()
|
||||
req.Add("X-Attribute-System-DupKey", "first-value")
|
||||
req.Add("X-Attribute-System-DupKey", "second-value")
|
||||
_, err := filterHeaders(log, req)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
req := &fasthttp.RequestHeader{}
|
||||
req.DisableNormalizing()
|
||||
|
||||
req.Set("X-Attribute-System-Expiration-Epoch1", "101")
|
||||
req.Set("X-Attribute-SYSTEM-Expiration-Epoch2", "102")
|
||||
req.Set("X-Attribute-system-Expiration-Epoch3", "103")
|
||||
req.Set("X-Attribute-Neofs-Expiration-Epoch1", "101")
|
||||
req.Set("X-Attribute-NEOFS-Expiration-Epoch2", "102")
|
||||
req.Set("X-Attribute-neofs-Expiration-Epoch3", "103")
|
||||
req.Set("X-Attribute-MyAttribute", "value")
|
||||
|
||||
expected := map[string]string{
|
||||
"__SYSTEM__EXPIRATION_EPOCH1": "101",
|
||||
"MyAttribute": "value",
|
||||
"__SYSTEM__EXPIRATION_EPOCH3": "103",
|
||||
"__SYSTEM__EXPIRATION_EPOCH2": "102",
|
||||
"__NEOFS__EXPIRATION_EPOCH1": "101",
|
||||
"MyAttribute": "value",
|
||||
"__NEOFS__EXPIRATION_EPOCH3": "103",
|
||||
"__NEOFS__EXPIRATION_EPOCH2": "102",
|
||||
}
|
||||
|
||||
result, err := filterHeaders(log, req)
|
||||
require.NoError(t, err)
|
||||
result := filterHeaders(log, req)
|
||||
|
||||
require.Equal(t, expected, result)
|
||||
}
|
||||
|
||||
func TestPrepareExpirationHeader(t *testing.T) {
|
||||
tomorrow := time.Now().Add(24 * time.Hour)
|
||||
tomorrowUnix := tomorrow.Unix()
|
||||
tomorrowUnixNano := tomorrow.UnixNano()
|
||||
tomorrowUnixMilli := tomorrowUnixNano / 1e6
|
||||
|
||||
epoch := "100"
|
||||
duration := "24h"
|
||||
timestampSec := strconv.FormatInt(tomorrowUnix, 10)
|
||||
timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10)
|
||||
timestampNano := strconv.FormatInt(tomorrowUnixNano, 10)
|
||||
|
||||
defaultDurations := &epochDurations{
|
||||
currentEpoch: 10,
|
||||
msPerBlock: 1000,
|
||||
blockPerEpoch: 101,
|
||||
}
|
||||
|
||||
msPerBlock := defaultDurations.blockPerEpoch * uint64(defaultDurations.msPerBlock)
|
||||
epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock
|
||||
if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 {
|
||||
epochPerDay++
|
||||
}
|
||||
|
||||
defaultExpEpoch := strconv.FormatUint(defaultDurations.currentEpoch+epochPerDay, 10)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
headers map[string]string
|
||||
durations *epochDurations
|
||||
err bool
|
||||
expected map[string]string
|
||||
}{
|
||||
{
|
||||
name: "valid epoch",
|
||||
headers: map[string]string{object.SysAttributeExpEpoch: epoch},
|
||||
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid duration",
|
||||
headers: map[string]string{
|
||||
object.SysAttributeExpEpoch: epoch,
|
||||
utils.ExpirationDurationAttr: duration,
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid rfc3339",
|
||||
headers: map[string]string{
|
||||
object.SysAttributeExpEpoch: epoch,
|
||||
utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339),
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid timestamp sec",
|
||||
headers: map[string]string{
|
||||
object.SysAttributeExpEpoch: epoch,
|
||||
utils.ExpirationTimestampAttr: timestampSec,
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid timestamp milli",
|
||||
headers: map[string]string{
|
||||
object.SysAttributeExpEpoch: epoch,
|
||||
utils.ExpirationTimestampAttr: timestampMilli,
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid timestamp nano",
|
||||
headers: map[string]string{
|
||||
object.SysAttributeExpEpoch: epoch,
|
||||
utils.ExpirationTimestampAttr: timestampNano,
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{object.SysAttributeExpEpoch: epoch},
|
||||
},
|
||||
{
|
||||
name: "valid timestamp sec",
|
||||
headers: map[string]string{utils.ExpirationTimestampAttr: timestampSec},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch},
|
||||
},
|
||||
{
|
||||
name: "valid duration",
|
||||
headers: map[string]string{utils.ExpirationDurationAttr: duration},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch},
|
||||
},
|
||||
{
|
||||
name: "valid rfc3339",
|
||||
headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{object.SysAttributeExpEpoch: defaultExpEpoch},
|
||||
},
|
||||
{
|
||||
name: "valid max uint 64",
|
||||
headers: map[string]string{utils.ExpirationRFC3339Attr: tomorrow.Format(time.RFC3339)},
|
||||
durations: &epochDurations{
|
||||
currentEpoch: math.MaxUint64 - 1,
|
||||
msPerBlock: defaultDurations.msPerBlock,
|
||||
blockPerEpoch: defaultDurations.blockPerEpoch,
|
||||
},
|
||||
expected: map[string]string{object.SysAttributeExpEpoch: strconv.FormatUint(uint64(math.MaxUint64), 10)},
|
||||
},
|
||||
{
|
||||
name: "invalid timestamp sec",
|
||||
headers: map[string]string{utils.ExpirationTimestampAttr: "abc"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid timestamp sec zero",
|
||||
headers: map[string]string{utils.ExpirationTimestampAttr: "0"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid duration",
|
||||
headers: map[string]string{utils.ExpirationDurationAttr: "1d"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid duration negative",
|
||||
headers: map[string]string{utils.ExpirationDurationAttr: "-5h"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid rfc3339",
|
||||
headers: map[string]string{utils.ExpirationRFC3339Attr: "abc"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid rfc3339 zero",
|
||||
headers: map[string]string{utils.ExpirationRFC3339Attr: time.RFC3339},
|
||||
err: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := prepareExpirationHeader(tc.headers, tc.durations)
|
||||
if tc.err {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.expected, tc.headers)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package uploader
|
|||
import (
|
||||
"io"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/uploader/multipart"
|
||||
"github.com/nspcc-dev/neofs-http-gw/uploader/multipart"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
//go:build !integration
|
||||
|
||||
package uploader
|
||||
|
||||
import (
|
||||
|
|
|
|||
|
|
@ -3,20 +3,20 @@ package uploader
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||
"github.com/nspcc-dev/neofs-http-gw/response"
|
||||
"github.com/nspcc-dev/neofs-http-gw/tokens"
|
||||
"github.com/nspcc-dev/neofs-http-gw/utils"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/atomic"
|
||||
"go.uber.org/zap"
|
||||
|
|
@ -37,6 +37,12 @@ type Uploader struct {
|
|||
containerResolver *resolver.ContainerResolver
|
||||
}
|
||||
|
||||
type epochDurations struct {
|
||||
currentEpoch uint64
|
||||
msPerBlock int64
|
||||
blockPerEpoch uint64
|
||||
}
|
||||
|
||||
// Settings stores reloading parameters, so it has to provide atomic getters and setters.
|
||||
type Settings struct {
|
||||
defaultTimestamp atomic.Bool
|
||||
|
|
@ -107,26 +113,19 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
|
|||
response.Error(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
filtered, err := filterHeaders(u.log, &c.Request.Header)
|
||||
if err != nil {
|
||||
log.Error("could not process headers", zap.Error(err))
|
||||
response.Error(c, err.Error(), fasthttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil {
|
||||
if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil {
|
||||
log.Warn("could not parse client time", zap.String("Date header", string(rawHeader)), zap.Error(err))
|
||||
} else {
|
||||
now = parsed
|
||||
filtered := filterHeaders(u.log, &c.Request.Header)
|
||||
if needParseExpiration(filtered) {
|
||||
epochDuration, err := getEpochDurations(c, u.pool)
|
||||
if err != nil {
|
||||
log.Error("could not get epoch durations from network info", zap.Error(err))
|
||||
response.Error(c, "could not get epoch durations from network info: "+err.Error(), fasthttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err = prepareExpirationHeader(filtered, epochDuration); err != nil {
|
||||
log.Error("could not parse expiration header", zap.Error(err))
|
||||
response.Error(c, "could not parse expiration header: "+err.Error(), fasthttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err = utils.PrepareExpirationHeader(c, u.pool, filtered, now); err != nil {
|
||||
log.Error("could not prepare expiration header", zap.Error(err))
|
||||
response.Error(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
attributes := make([]object.Attribute, 0, len(filtered))
|
||||
|
|
@ -167,7 +166,8 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
|
|||
}
|
||||
|
||||
if idObj, err = u.pool.PutObject(u.appCtx, prm); err != nil {
|
||||
u.handlePutFrostFSErr(c, err)
|
||||
log.Error("could not store file in neofs", zap.Error(err))
|
||||
response.Error(c, "could not store file in neofs: "+err.Error(), fasthttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -198,14 +198,6 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
|
|||
c.Response.Header.SetContentType(jsonHeader)
|
||||
}
|
||||
|
||||
func (u *Uploader) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) {
|
||||
statusCode, msg, additionalFields := response.FormErrorResponse("could not store file in frostfs", err)
|
||||
logFields := append([]zap.Field{zap.Error(err)}, additionalFields...)
|
||||
|
||||
u.log.Error("could not store file in frostfs", logFields...)
|
||||
response.Error(r, msg, statusCode)
|
||||
}
|
||||
|
||||
func (u *Uploader) fetchOwnerAndBearerToken(ctx context.Context) (*user.ID, *bearer.Token) {
|
||||
if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil {
|
||||
issuer := bearer.ResolveIssuer(*tkn)
|
||||
|
|
@ -231,3 +223,28 @@ func (pr *putResponse) encode(w io.Writer) error {
|
|||
enc.SetIndent("", "\t")
|
||||
return enc.Encode(pr)
|
||||
}
|
||||
|
||||
func getEpochDurations(ctx context.Context, p *pool.Pool) (*epochDurations, error) {
|
||||
networkInfo, err := p.NetworkInfo(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := &epochDurations{
|
||||
currentEpoch: networkInfo.CurrentEpoch(),
|
||||
msPerBlock: networkInfo.MsPerBlock(),
|
||||
blockPerEpoch: networkInfo.EpochDuration(),
|
||||
}
|
||||
|
||||
if res.blockPerEpoch == 0 {
|
||||
return nil, fmt.Errorf("EpochDuration is empty")
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func needParseExpiration(headers map[string]string) bool {
|
||||
_, ok1 := headers[utils.ExpirationDurationAttr]
|
||||
_, ok2 := headers[utils.ExpirationRFC3339Attr]
|
||||
_, ok3 := headers[utils.ExpirationTimestampAttr]
|
||||
return ok1 || ok2 || ok3
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,250 +1,10 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
)
|
||||
|
||||
const (
|
||||
UserAttributeHeaderPrefix = "X-Attribute-"
|
||||
SystemAttributePrefix = "__NEOFS__"
|
||||
|
||||
ExpirationDurationAttr = SystemAttributePrefix + "EXPIRATION_DURATION"
|
||||
ExpirationTimestampAttr = SystemAttributePrefix + "EXPIRATION_TIMESTAMP"
|
||||
ExpirationRFC3339Attr = SystemAttributePrefix + "EXPIRATION_RFC3339"
|
||||
)
|
||||
|
||||
const (
|
||||
systemAttributePrefix = "__SYSTEM__"
|
||||
|
||||
// deprecated: use systemAttributePrefix
|
||||
systemAttributePrefixNeoFS = "__NEOFS__"
|
||||
)
|
||||
|
||||
type systemTransformer struct {
|
||||
prefix string
|
||||
backwardPrefix string
|
||||
xAttrPrefixes [][]byte
|
||||
}
|
||||
|
||||
var transformers = []systemTransformer{
|
||||
{
|
||||
prefix: systemAttributePrefix,
|
||||
backwardPrefix: "System-",
|
||||
xAttrPrefixes: [][]byte{[]byte("System-"), []byte("SYSTEM-"), []byte("system-")},
|
||||
},
|
||||
{
|
||||
prefix: systemAttributePrefixNeoFS,
|
||||
backwardPrefix: "Neofs-",
|
||||
xAttrPrefixes: [][]byte{[]byte("Neofs-"), []byte("NEOFS-"), []byte("neofs-")},
|
||||
},
|
||||
}
|
||||
|
||||
func (t systemTransformer) existsExpirationAttributes(headers map[string]string) bool {
|
||||
_, ok0 := headers[t.expirationEpochAttr()]
|
||||
_, ok1 := headers[t.expirationDurationAttr()]
|
||||
_, ok2 := headers[t.expirationTimestampAttr()]
|
||||
_, ok3 := headers[t.expirationRFC3339Attr()]
|
||||
return ok0 || ok1 || ok2 || ok3
|
||||
}
|
||||
|
||||
func (t systemTransformer) expirationEpochAttr() string {
|
||||
return t.prefix + "EXPIRATION_EPOCH"
|
||||
}
|
||||
|
||||
func (t systemTransformer) expirationDurationAttr() string {
|
||||
return t.prefix + "EXPIRATION_DURATION"
|
||||
}
|
||||
|
||||
func (t systemTransformer) expirationTimestampAttr() string {
|
||||
return t.prefix + "EXPIRATION_TIMESTAMP"
|
||||
}
|
||||
|
||||
func (t systemTransformer) expirationRFC3339Attr() string {
|
||||
return t.prefix + "EXPIRATION_RFC3339"
|
||||
}
|
||||
|
||||
func (t systemTransformer) systemTranslator(key, prefix []byte) []byte {
|
||||
// replace the specified prefix with system prefix
|
||||
key = bytes.Replace(key, prefix, []byte(t.prefix), 1)
|
||||
|
||||
// replace `-` with `_`
|
||||
key = bytes.ReplaceAll(key, []byte("-"), []byte("_"))
|
||||
|
||||
// replace with uppercase
|
||||
return bytes.ToUpper(key)
|
||||
}
|
||||
|
||||
func (t systemTransformer) transformIfSystem(key []byte) ([]byte, bool) {
|
||||
// checks that it's a system FrostFS header
|
||||
for _, system := range t.xAttrPrefixes {
|
||||
if bytes.HasPrefix(key, system) {
|
||||
return t.systemTranslator(key, system), true
|
||||
}
|
||||
}
|
||||
|
||||
return key, false
|
||||
}
|
||||
|
||||
// systemBackwardTranslator is used to convert headers looking like '__PREFIX__ATTR_NAME' to 'Prefix-Attr-Name'.
|
||||
func (t systemTransformer) systemBackwardTranslator(key string) string {
|
||||
// trim specified prefix '__PREFIX__'
|
||||
key = strings.TrimPrefix(key, t.prefix)
|
||||
|
||||
var res strings.Builder
|
||||
res.WriteString(t.backwardPrefix)
|
||||
|
||||
strs := strings.Split(key, "_")
|
||||
for i, s := range strs {
|
||||
s = title(strings.ToLower(s))
|
||||
res.WriteString(s)
|
||||
if i != len(strs)-1 {
|
||||
res.WriteString("-")
|
||||
}
|
||||
}
|
||||
|
||||
return res.String()
|
||||
}
|
||||
|
||||
func (t systemTransformer) backwardTransformIfSystem(key string) (string, bool) {
|
||||
if strings.HasPrefix(key, t.prefix) {
|
||||
return t.systemBackwardTranslator(key), true
|
||||
}
|
||||
|
||||
return key, false
|
||||
}
|
||||
|
||||
func TransformIfSystem(key []byte) []byte {
|
||||
for _, transformer := range transformers {
|
||||
key, transformed := transformer.transformIfSystem(key)
|
||||
if transformed {
|
||||
return key
|
||||
}
|
||||
}
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
func BackwardTransformIfSystem(key string) string {
|
||||
for _, transformer := range transformers {
|
||||
key, transformed := transformer.backwardTransformIfSystem(key)
|
||||
if transformed {
|
||||
return key
|
||||
}
|
||||
}
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
func title(str string) string {
|
||||
if str == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
r, size := utf8.DecodeRuneInString(str)
|
||||
r0 := unicode.ToTitle(r)
|
||||
return string(r0) + str[size:]
|
||||
}
|
||||
|
||||
func PrepareExpirationHeader(ctx context.Context, p *pool.Pool, headers map[string]string, now time.Time) error {
|
||||
formatsNum := 0
|
||||
index := -1
|
||||
for i, transformer := range transformers {
|
||||
if transformer.existsExpirationAttributes(headers) {
|
||||
formatsNum++
|
||||
index = i
|
||||
}
|
||||
}
|
||||
|
||||
switch formatsNum {
|
||||
case 0:
|
||||
return nil
|
||||
case 1:
|
||||
epochDuration, err := GetEpochDurations(ctx, p)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't get epoch durations from network info: %w", err)
|
||||
}
|
||||
return transformers[index].prepareExpirationHeader(headers, epochDuration, now)
|
||||
default:
|
||||
return errors.New("both deprecated and new system attributes formats are used, please use only one")
|
||||
}
|
||||
}
|
||||
|
||||
func (t systemTransformer) prepareExpirationHeader(headers map[string]string, epochDurations *EpochDurations, now time.Time) error {
|
||||
expirationInEpoch := headers[t.expirationEpochAttr()]
|
||||
|
||||
if timeRFC3339, ok := headers[t.expirationRFC3339Attr()]; ok {
|
||||
expTime, err := time.Parse(time.RFC3339, timeRFC3339)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't parse value %s of header %s", timeRFC3339, t.expirationRFC3339Attr())
|
||||
}
|
||||
|
||||
if expTime.Before(now) {
|
||||
return fmt.Errorf("value %s of header %s must be in the future", timeRFC3339, t.expirationRFC3339Attr())
|
||||
}
|
||||
t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
|
||||
delete(headers, t.expirationRFC3339Attr())
|
||||
}
|
||||
|
||||
if timestamp, ok := headers[t.expirationTimestampAttr()]; ok {
|
||||
value, err := strconv.ParseInt(timestamp, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't parse value %s of header %s", timestamp, t.expirationTimestampAttr())
|
||||
}
|
||||
expTime := time.Unix(value, 0)
|
||||
|
||||
if expTime.Before(now) {
|
||||
return fmt.Errorf("value %s of header %s must be in the future", timestamp, t.expirationTimestampAttr())
|
||||
}
|
||||
t.updateExpirationHeader(headers, epochDurations, expTime.Sub(now))
|
||||
delete(headers, t.expirationTimestampAttr())
|
||||
}
|
||||
|
||||
if duration, ok := headers[t.expirationDurationAttr()]; ok {
|
||||
expDuration, err := time.ParseDuration(duration)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't parse value %s of header %s", duration, t.expirationDurationAttr())
|
||||
}
|
||||
if expDuration <= 0 {
|
||||
return fmt.Errorf("value %s of header %s must be positive", expDuration, t.expirationDurationAttr())
|
||||
}
|
||||
t.updateExpirationHeader(headers, epochDurations, expDuration)
|
||||
delete(headers, t.expirationDurationAttr())
|
||||
}
|
||||
|
||||
if expirationInEpoch != "" {
|
||||
expEpoch, err := strconv.ParseUint(expirationInEpoch, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse expiration epoch '%s': %w", expirationInEpoch, err)
|
||||
}
|
||||
if expEpoch < epochDurations.CurrentEpoch {
|
||||
return fmt.Errorf("expiration epoch '%d' must be greater than current epoch '%d'", expEpoch, epochDurations.CurrentEpoch)
|
||||
}
|
||||
|
||||
headers[t.expirationEpochAttr()] = expirationInEpoch
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t systemTransformer) updateExpirationHeader(headers map[string]string, durations *EpochDurations, expDuration time.Duration) {
|
||||
epochDuration := uint64(durations.MsPerBlock) * durations.BlockPerEpoch
|
||||
currentEpoch := durations.CurrentEpoch
|
||||
numEpoch := uint64(expDuration.Milliseconds()) / epochDuration
|
||||
|
||||
if uint64(expDuration.Milliseconds())%epochDuration != 0 {
|
||||
numEpoch++
|
||||
}
|
||||
|
||||
expirationEpoch := uint64(math.MaxUint64)
|
||||
if numEpoch < math.MaxUint64-currentEpoch {
|
||||
expirationEpoch = currentEpoch + numEpoch
|
||||
}
|
||||
|
||||
headers[t.expirationEpochAttr()] = strconv.FormatUint(expirationEpoch, 10)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,189 +0,0 @@
|
|||
//go:build !integration
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPrepareExpirationHeader(t *testing.T) {
|
||||
tomorrow := time.Now().Add(24 * time.Hour)
|
||||
tomorrowUnix := tomorrow.Unix()
|
||||
tomorrowUnixNano := tomorrow.UnixNano()
|
||||
tomorrowUnixMilli := tomorrowUnixNano / 1e6
|
||||
|
||||
epoch := "100"
|
||||
duration := "24h"
|
||||
timestampSec := strconv.FormatInt(tomorrowUnix, 10)
|
||||
timestampMilli := strconv.FormatInt(tomorrowUnixMilli, 10)
|
||||
timestampNano := strconv.FormatInt(tomorrowUnixNano, 10)
|
||||
|
||||
defaultDurations := &EpochDurations{
|
||||
CurrentEpoch: 10,
|
||||
MsPerBlock: 1000,
|
||||
BlockPerEpoch: 101,
|
||||
}
|
||||
|
||||
msPerBlock := defaultDurations.BlockPerEpoch * uint64(defaultDurations.MsPerBlock)
|
||||
epochPerDay := uint64((24 * time.Hour).Milliseconds()) / msPerBlock
|
||||
if uint64((24*time.Hour).Milliseconds())%msPerBlock != 0 {
|
||||
epochPerDay++
|
||||
}
|
||||
|
||||
defaultExpEpoch := strconv.FormatUint(defaultDurations.CurrentEpoch+epochPerDay, 10)
|
||||
|
||||
for _, transformer := range transformers {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
headers map[string]string
|
||||
durations *EpochDurations
|
||||
err bool
|
||||
expected map[string]string
|
||||
}{
|
||||
{
|
||||
name: "valid epoch",
|
||||
headers: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
durations: defaultDurations,
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid duration",
|
||||
headers: map[string]string{
|
||||
transformer.expirationEpochAttr(): epoch,
|
||||
transformer.expirationDurationAttr(): duration,
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid rfc3339",
|
||||
headers: map[string]string{
|
||||
transformer.expirationEpochAttr(): epoch,
|
||||
transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339),
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid timestamp sec",
|
||||
headers: map[string]string{
|
||||
transformer.expirationEpochAttr(): epoch,
|
||||
transformer.expirationTimestampAttr(): timestampSec,
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid timestamp milli",
|
||||
headers: map[string]string{
|
||||
transformer.expirationEpochAttr(): epoch,
|
||||
transformer.expirationTimestampAttr(): timestampMilli,
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
},
|
||||
{
|
||||
name: "valid epoch, valid timestamp nano",
|
||||
headers: map[string]string{
|
||||
transformer.expirationEpochAttr(): epoch,
|
||||
transformer.expirationTimestampAttr(): timestampNano,
|
||||
},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): epoch},
|
||||
},
|
||||
{
|
||||
name: "valid timestamp sec",
|
||||
headers: map[string]string{transformer.expirationTimestampAttr(): timestampSec},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
|
||||
},
|
||||
{
|
||||
name: "valid duration",
|
||||
headers: map[string]string{transformer.expirationDurationAttr(): duration},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
|
||||
},
|
||||
{
|
||||
name: "valid rfc3339",
|
||||
headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)},
|
||||
durations: defaultDurations,
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): defaultExpEpoch},
|
||||
},
|
||||
{
|
||||
name: "valid max uint 64",
|
||||
headers: map[string]string{transformer.expirationRFC3339Attr(): tomorrow.Format(time.RFC3339)},
|
||||
durations: &EpochDurations{
|
||||
CurrentEpoch: math.MaxUint64 - 1,
|
||||
MsPerBlock: defaultDurations.MsPerBlock,
|
||||
BlockPerEpoch: defaultDurations.BlockPerEpoch,
|
||||
},
|
||||
expected: map[string]string{transformer.expirationEpochAttr(): strconv.FormatUint(uint64(math.MaxUint64), 10)},
|
||||
},
|
||||
{
|
||||
name: "invalid timestamp sec",
|
||||
headers: map[string]string{transformer.expirationTimestampAttr(): "abc"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid timestamp sec zero",
|
||||
headers: map[string]string{transformer.expirationTimestampAttr(): "0"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid duration",
|
||||
headers: map[string]string{transformer.expirationDurationAttr(): "1d"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid duration negative",
|
||||
headers: map[string]string{transformer.expirationDurationAttr(): "-5h"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid rfc3339",
|
||||
headers: map[string]string{transformer.expirationRFC3339Attr(): "abc"},
|
||||
err: true,
|
||||
},
|
||||
{
|
||||
name: "invalid rfc3339 zero",
|
||||
headers: map[string]string{transformer.expirationRFC3339Attr(): time.RFC3339},
|
||||
err: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := transformer.prepareExpirationHeader(tc.headers, tc.durations, time.Now())
|
||||
if tc.err {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.expected, tc.headers)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSystemBackwardTranslator(t *testing.T) {
|
||||
input := []string{
|
||||
"__SYSTEM__EXPIRATION_EPOCH",
|
||||
"__SYSTEM__RANDOM_ATTR",
|
||||
"__NEOFS__EXPIRATION_EPOCH",
|
||||
"__NEOFS__RANDOM_ATTR",
|
||||
}
|
||||
expected := []string{
|
||||
"System-Expiration-Epoch",
|
||||
"System-Random-Attr",
|
||||
"Neofs-Expiration-Epoch",
|
||||
"Neofs-Random-Attr",
|
||||
}
|
||||
|
||||
for i, str := range input {
|
||||
res := BackwardTransformIfSystem(str)
|
||||
require.Equal(t, expected[i], res)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,11 +2,9 @@ package utils
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
)
|
||||
|
||||
// GetContainerID decode container id, if it's not a valid container id
|
||||
|
|
@ -19,27 +17,3 @@ func GetContainerID(ctx context.Context, containerID string, resolver *resolver.
|
|||
}
|
||||
return cnrID, err
|
||||
}
|
||||
|
||||
type EpochDurations struct {
|
||||
CurrentEpoch uint64
|
||||
MsPerBlock int64
|
||||
BlockPerEpoch uint64
|
||||
}
|
||||
|
||||
func GetEpochDurations(ctx context.Context, p *pool.Pool) (*EpochDurations, error) {
|
||||
networkInfo, err := p.NetworkInfo(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := &EpochDurations{
|
||||
CurrentEpoch: networkInfo.CurrentEpoch(),
|
||||
MsPerBlock: networkInfo.MsPerBlock(),
|
||||
BlockPerEpoch: networkInfo.EpochDuration(),
|
||||
}
|
||||
|
||||
if res.BlockPerEpoch == 0 {
|
||||
return nil, fmt.Errorf("EpochDuration is empty")
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue