Fix pre-commit issues #26
13 changed files with 51 additions and 54 deletions
2
.gitlint
2
.gitlint
|
@ -8,4 +8,4 @@ regex=^\[\#[0-9Xx]+\]\s
|
|||
|
||||
[ignore-by-title]
|
||||
regex=^Release(.*)
|
||||
ignore=title-match-regex
|
||||
ignore=title-match-regex
|
||||
|
|
|
@ -42,4 +42,4 @@ repos:
|
|||
entry: make test
|
||||
pass_filenames: false
|
||||
types: [go]
|
||||
language: system
|
||||
language: system
|
||||
|
|
|
@ -139,7 +139,7 @@ If you configure application using `.yaml` file change:
|
|||
|
||||
### Changed
|
||||
- Updated docs (#133, #140)
|
||||
- Increased default read/write timeouts (#154)
|
||||
- Increased default read/write timeouts (#154)
|
||||
- Updated SDK (#137, #139)
|
||||
- Updated go version to 1.17 (#143)
|
||||
- Improved error messages (#144)
|
||||
|
@ -166,11 +166,11 @@ If you configure application using `.yaml` file change:
|
|||
- System headers format (#111)
|
||||
|
||||
### Added
|
||||
- Different formats to set object's expiration: in epoch, duration, timestamp,
|
||||
- Different formats to set object's expiration: in epoch, duration, timestamp,
|
||||
RFC3339 (#108)
|
||||
- Support of nodes priority (#115)
|
||||
|
||||
### Changed
|
||||
### Changed
|
||||
- Updated testcontainers dependency (#100)
|
||||
|
||||
## [0.17.0] - 2021-11-15
|
||||
|
|
2
Makefile
Normal file → Executable file
2
Makefile
Normal file → Executable file
|
@ -21,7 +21,7 @@ OS_RELEASE = $(shell lsb_release -cs)
|
|||
PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
|
||||
sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \
|
||||
sed "s/-/~/")-${OS_RELEASE}
|
||||
.PHONY: debpackage debclean
|
||||
.PHONY: debpackage debclean
|
||||
|
||||
# Make all binaries
|
||||
all: $(BINS)
|
||||
|
|
64
README.md
64
README.md
|
@ -48,8 +48,8 @@ can be done either via `-p` parameter or via `HTTP_GW_PEERS_<N>_ADDRESS` and
|
|||
`HTTP_GW_PEERS_<N>_WEIGHT` environment variables (the gate supports multiple
|
||||
FrostFS nodes with weighted load balancing).
|
||||
|
||||
If you launch HTTP gateway in bundle with [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env),
|
||||
you can get the IP address of the node in the output of `make hosts` command
|
||||
If you launch HTTP gateway in bundle with [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env),
|
||||
you can get the IP address of the node in the output of `make hosts` command
|
||||
(with s0*.frostfs.devenv name).
|
||||
|
||||
These two commands are functionally equivalent, they run the gate with one
|
||||
|
@ -86,12 +86,12 @@ $ HTTP_GW_PEERS_0_ADDRESS=192.168.130.71:8080 HTTP_GW_PEERS_0_WEIGHT=1 HTTP_GW_P
|
|||
HTTP_GW_PEERS_2_ADDRESS=192.168.130.73:8080 HTTP_GW_PEERS_2_WEIGHT=1 HTTP_GW_PEERS_2_PRIORITY=2 \
|
||||
frostfs-http-gw
|
||||
```
|
||||
This command will make gateway use 192.168.130.71 while it is healthy. Otherwise, it will make the gateway use
|
||||
This command will make gateway use 192.168.130.71 while it is healthy. Otherwise, it will make the gateway use
|
||||
192.168.130.72 for 90% of requests and 192.168.130.73 for remaining 10%.
|
||||
|
||||
### Keys
|
||||
You can provide a wallet via `--wallet` or `-w` flag. You can also specify the account address using `--address`
|
||||
(if no address provided default one will be used). If wallet is used, you need to set `HTTP_GW_WALLET_PASSPHRASE` variable to decrypt the wallet.
|
||||
You can provide a wallet via `--wallet` or `-w` flag. You can also specify the account address using `--address`
|
||||
(if no address provided default one will be used). If wallet is used, you need to set `HTTP_GW_WALLET_PASSPHRASE` variable to decrypt the wallet.
|
||||
If no wallet provided, the gateway autogenerates a key pair it will use for FrostFS requests.
|
||||
```
|
||||
$ frostfs-http-gw -p $FROSTFS_NODE -w $WALLET_PATH --address $ACCOUNT_ADDRESS
|
||||
|
@ -162,7 +162,7 @@ All timing options accept values with suffixes, so "15s" is 15 seconds and
|
|||
"2m" is 2 minutes.
|
||||
|
||||
### Zip streaming
|
||||
The gateway supports downloading files by common prefix (like dir) in zip format. You can enable compression
|
||||
The gateway supports downloading files by common prefix (like dir) in zip format. You can enable compression
|
||||
using config or `HTTP_GW_ZIP_COMPRESSION=true` environment variable.
|
||||
|
||||
### Logging
|
||||
|
@ -172,7 +172,7 @@ HTTP_GW_LOGGER_LEVEL=debug
|
|||
```
|
||||
|
||||
### Yaml file
|
||||
Configuration file is optional and can be used instead of environment variables/other parameters.
|
||||
Configuration file is optional and can be used instead of environment variables/other parameters.
|
||||
It can be specified with `--config` parameter:
|
||||
```
|
||||
$ frostfs-http-gw --config your-config.yaml
|
||||
|
@ -207,7 +207,7 @@ supported.
|
|||
|
||||
### Preparation
|
||||
|
||||
Before uploading or downloading a file make sure you have a prepared container.
|
||||
Before uploading or downloading a file make sure you have a prepared container.
|
||||
You can create it with instructions below.
|
||||
|
||||
Also, in case of downloading, you need to have a file inside a container.
|
||||
|
@ -226,13 +226,13 @@ resolve_order:
|
|||
- nns
|
||||
```
|
||||
|
||||
2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env)
|
||||
2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env)
|
||||
you can check if your container (e.g. with `container-name` name) is registered in NNS:
|
||||
|
||||
```shell
|
||||
$ curl -s --data '{"id":1,"jsonrpc":"2.0","method":"getcontractstate","params":[1]}' \
|
||||
http://morph-chain.frostfs.devenv:30333 | jq -r '.result.hash'
|
||||
|
||||
|
||||
0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667
|
||||
|
||||
$ docker exec -it morph_chain neo-go \
|
||||
|
@ -241,7 +241,7 @@ $ docker exec -it morph_chain neo-go \
|
|||
resolve string:container-name.container int:16 \
|
||||
| jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \
|
||||
| base64 -d && echo
|
||||
|
||||
|
||||
7f3vvkw4iTiS5ZZbu5BQXEmJtETWbi3uUjLNaSs29xrL
|
||||
```
|
||||
|
||||
|
@ -257,7 +257,7 @@ You can create a container via [frostfs-cli](https://git.frostfs.info/TrueCloudL
|
|||
```
|
||||
$ frostfs-cli -r $FROSTFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL
|
||||
```
|
||||
where `$WALLET` is a path to user wallet,
|
||||
where `$WALLET` is a path to user wallet,
|
||||
`$ACL` -- hex encoded basic ACL value or keywords 'private, 'public-read', 'public-read-write' and
|
||||
`$POLICY` -- QL-encoded or JSON-encoded placement policy or path to file with it
|
||||
|
||||
|
@ -267,17 +267,17 @@ $ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json container create --policy
|
|||
```
|
||||
|
||||
If you have launched nodes via [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env),
|
||||
you can get the key value from `wallets/wallet.json` or write the path to
|
||||
you can get the key value from `wallets/wallet.json` or write the path to
|
||||
the file `wallets/wallet.key`.
|
||||
|
||||
#### Prepare a file in a container
|
||||
|
||||
To create a file via [frostfs-cli](https://git.frostfs.info/TrueCloudLab/frostfs-node/releases), run a command below:
|
||||
```
|
||||
$ frostfs-cli -r $FROSTFS_NODE -k $KEY object put --file $FILENAME --cid $CID
|
||||
$ frostfs-cli -r $FROSTFS_NODE -k $KEY object put --file $FILENAME --cid $CID
|
||||
```
|
||||
where
|
||||
`$KEY` -- the key, please read the information [above](#create-a-container),
|
||||
where
|
||||
`$KEY` -- the key, please read the information [above](#create-a-container),
|
||||
`$CID` -- container ID.
|
||||
|
||||
For example:
|
||||
|
@ -290,13 +290,13 @@ $ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json object put --file cat.png
|
|||
|
||||
#### Requests
|
||||
|
||||
The following requests support GET/HEAD methods.
|
||||
The following requests support GET/HEAD methods.
|
||||
|
||||
##### By IDs
|
||||
|
||||
Basic downloading involves container ID and object ID and is done via GET
|
||||
requests to `/get/$CID/$OID` path, where `$CID` is a container ID or its name if NNS is enabled,
|
||||
`$OID` is an object's (i.e. your file's) ID.
|
||||
requests to `/get/$CID/$OID` path, where `$CID` is a container ID or its name if NNS is enabled,
|
||||
`$OID` is an object's (i.e. your file's) ID.
|
||||
|
||||
For example:
|
||||
|
||||
|
@ -317,12 +317,12 @@ can be used as well. The generic syntax for it looks like this:
|
|||
|
||||
```/get_by_attribute/$CID/$ATTRIBUTE_NAME/$ATTRIBUTE_VALUE```
|
||||
|
||||
where
|
||||
`$CID` is a container ID or its name if NNS is enabled,
|
||||
where
|
||||
`$CID` is a container ID or its name if NNS is enabled,
|
||||
`$ATTRIBUTE_NAME` is the name of the attribute we want to use,
|
||||
`$ATTRIBUTE_VALUE` is the value of this attribute that the target object should have.
|
||||
|
||||
**NB!** The attribute key and value should be url encoded, i.e., if you want to download an object with the attribute value
|
||||
**NB!** The attribute key and value should be url encoded, i.e., if you want to download an object with the attribute value
|
||||
`a cat`, the value in the request must be `a+cat`. In the same way with the attribute key. If you don't escape such values
|
||||
everything can still work (for example you can use `d@ta` without encoding) but it's HIGHLY RECOMMENDED to encode all your attributes.
|
||||
|
||||
|
@ -346,7 +346,7 @@ Some other user-defined attributes:
|
|||
$ wget http://localhost:8082/get_by_attribute/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/Ololo/100500
|
||||
```
|
||||
|
||||
Or when the attribute includes special symbols:
|
||||
Or when the attribute includes special symbols:
|
||||
```
|
||||
$ wget http://localhost:8082/get_by_attribute/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/Olo%2Blo/100500 # means Olo+lo
|
||||
```
|
||||
|
@ -365,7 +365,7 @@ You can download some dir (files with the same prefix) in zip (it will be compre
|
|||
$ wget http://localhost:8082/zip/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/common/prefix
|
||||
```
|
||||
|
||||
**Note:** the objects must have a valid `FilePath` attribute (it should not contain trailing `/`),
|
||||
**Note:** the objects must have a valid `FilePath` attribute (it should not contain trailing `/`),
|
||||
otherwise they will not be in the zip archive. You can upload file with this attribute using `curl`:
|
||||
|
||||
```
|
||||
|
@ -393,7 +393,7 @@ set of reply headers generated using the following rules:
|
|||
|
||||
##### Caching strategy
|
||||
|
||||
HTTP Gateway doesn't control caching (doesn't anything with the `Cache-Control` header). Caching strategy strictly
|
||||
HTTP Gateway doesn't control caching (doesn't anything with the `Cache-Control` header). Caching strategy strictly
|
||||
depends on application use case. So it should be carefully done by proxy server.
|
||||
|
||||
### Uploading
|
||||
|
@ -424,7 +424,7 @@ You can also add some attributes to your file using the following rules:
|
|||
"X-Attribute-" prefix stripped, that is if you add "X-Attribute-Ololo:
|
||||
100500" header to your request the resulting object will get "Ololo:
|
||||
100500" attribute
|
||||
* "X-Attribute-SYSTEM-*" headers are special
|
||||
* "X-Attribute-SYSTEM-*" headers are special
|
||||
(`-SYSTEM-` part can also be `-system-` or`-System-` (and even legacy `-Neofs-` for some next releases)), they're used to set internal
|
||||
FrostFS attributes starting with `__SYSTEM__` prefix, for these attributes all
|
||||
dashes get converted to underscores and all letters are capitalized. For
|
||||
|
@ -445,7 +445,7 @@ There are some reserved headers type of `X-Attribute-SYSTEM-*` (headers are arra
|
|||
3. `X-Attribute-System-Expiration-Timestamp: 1637574797`
|
||||
4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z`
|
||||
|
||||
which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way.
|
||||
which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way.
|
||||
|
||||
---
|
||||
|
||||
|
@ -484,7 +484,7 @@ the corresponding header to the upload request. Accessing the ACL protected data
|
|||
works the same way.
|
||||
|
||||
##### Example
|
||||
In order to generate a bearer token, you need to know the container owner key and
|
||||
In order to generate a bearer token, you need to know the container owner key and
|
||||
the address of the sender who will do the request to FrostFS (in our case, it's a gateway wallet address).
|
||||
|
||||
Suppose we have:
|
||||
|
@ -492,7 +492,7 @@ Suppose we have:
|
|||
* **NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3** (token owner address)
|
||||
* **BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K** (container id)
|
||||
|
||||
Firstly, we need to encode the container id and the sender address to base64 (now it's base58).
|
||||
Firstly, we need to encode the container id and the sender address to base64 (now it's base58).
|
||||
So use **base58** and **base64** utils.
|
||||
|
||||
1. Encoding container id:
|
||||
|
@ -540,7 +540,7 @@ $ frostfs-cli util sign bearer-token --from bearer.json --to signed.json -w ./wa
|
|||
```
|
||||
Encoding to base64 to use via the header:
|
||||
```
|
||||
$ base64 -w 0 signed.json
|
||||
$ base64 -w 0 signed.json
|
||||
# output: Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw==
|
||||
```
|
||||
|
||||
|
@ -599,8 +599,8 @@ File **eacl.json**:
|
|||
|
||||
### Metrics and Pprof
|
||||
|
||||
If enabled, Prometheus metrics are available at `localhost:8084` endpoint
|
||||
and Pprof at `localhost:8083/debug/pprof` by default. Host and port can be configured.
|
||||
If enabled, Prometheus metrics are available at `localhost:8084` endpoint
|
||||
and Pprof at `localhost:8083/debug/pprof` by default. Host and port can be configured.
|
||||
See [configuration](./docs/gate-configuration.md).
|
||||
|
||||
## Credits
|
||||
|
|
1
debian/control
vendored
1
debian/control
vendored
|
@ -12,4 +12,3 @@ Package: frostfs-http-gw
|
|||
Architecture: any
|
||||
Depends: ${misc:Depends}
|
||||
Description: FrostFS HTTP Gateway bridges FrostFS internal protocol and HTTP standard.
|
||||
|
||||
|
|
8
debian/copyright
vendored
8
debian/copyright
vendored
|
@ -14,12 +14,12 @@ License: GPL-3
|
|||
This program is free software: you can redistribute it and/or modify it
|
||||
under the terms of the GNU General Public License as published
|
||||
by the Free Software Foundation; version 3.
|
||||
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program or at /usr/share/common-licenses/GPL-3.
|
||||
If not, see <http://www.gnu.org/licenses/>.
|
||||
along with this program or at /usr/share/common-licenses/GPL-3.
|
||||
If not, see <http://www.gnu.org/licenses/>.
|
||||
|
|
4
debian/frostfs-http-gw.postinst
vendored
Normal file → Executable file
4
debian/frostfs-http-gw.postinst
vendored
Normal file → Executable file
|
@ -29,8 +29,8 @@ case "$1" in
|
|||
chmod -f 0640 /etc/frostfs/$USERNAME/config.yaml || true
|
||||
fi
|
||||
USERDIR=$(getent passwd "frostfs-$USERNAME" | cut -d: -f6)
|
||||
if ! dpkg-statoverride --list frostfs-$USERDIR >/dev/null; then
|
||||
chown -f frostfs-$USERNAME: $USERDIR
|
||||
if ! dpkg-statoverride --list frostfs-"$USERDIR" >/dev/null; then
|
||||
chown -f frostfs-$USERNAME: "$USERDIR"
|
||||
fi
|
||||
;;
|
||||
|
||||
|
|
0
debian/frostfs-http-gw.postrm
vendored
Normal file → Executable file
0
debian/frostfs-http-gw.postrm
vendored
Normal file → Executable file
0
debian/frostfs-http-gw.preinst
vendored
Normal file → Executable file
0
debian/frostfs-http-gw.preinst
vendored
Normal file → Executable file
0
debian/frostfs-http-gw.prerm
vendored
Normal file → Executable file
0
debian/frostfs-http-gw.prerm
vendored
Normal file → Executable file
6
debian/rules
vendored
6
debian/rules
vendored
|
@ -8,9 +8,7 @@ SERVICE = frostfs-http-gw
|
|||
dh $@
|
||||
|
||||
override_dh_installsystemd:
|
||||
dh_installsystemd --no-enable --no-start $(SERVICE).service
|
||||
dh_installsystemd --no-enable --no-start $(SERVICE).service
|
||||
|
||||
override_dh_installchangelogs:
|
||||
dh_installchangelogs -k CHANGELOG.md
|
||||
|
||||
|
||||
dh_installchangelogs -k CHANGELOG.md
|
||||
|
|
|
@ -62,9 +62,9 @@ resolve_order:
|
|||
- nns
|
||||
- dns
|
||||
|
||||
connect_timeout: 5s
|
||||
connect_timeout: 5s
|
||||
stream_timeout: 10s
|
||||
request_timeout: 5s
|
||||
request_timeout: 5s
|
||||
rebalance_timer: 30s
|
||||
pool_error_threshold: 100
|
||||
```
|
||||
|
@ -83,8 +83,8 @@ pool_error_threshold: 100
|
|||
|
||||
```yaml
|
||||
wallet:
|
||||
path: /path/to/wallet.json
|
||||
address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP
|
||||
path: /path/to/wallet.json
|
||||
address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP
|
||||
passphrase: pwd
|
||||
```
|
||||
|
||||
|
@ -201,7 +201,7 @@ upload_header:
|
|||
|
||||
```yaml
|
||||
zip:
|
||||
compression: false
|
||||
compression: false
|
||||
```
|
||||
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|
|
Loading…
Reference in a new issue