diff --git a/.gitlint b/.gitlint index fda43fb..e7218ac 100644 --- a/.gitlint +++ b/.gitlint @@ -8,4 +8,4 @@ regex=^\[\#[0-9Xx]+\]\s [ignore-by-title] regex=^Release(.*) -ignore=title-match-regex \ No newline at end of file +ignore=title-match-regex diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1e89def..4fde2a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,4 +42,4 @@ repos: entry: make test pass_filenames: false types: [go] - language: system \ No newline at end of file + language: system diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dcd456..dee5545 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,7 +139,7 @@ If you configure application using `.yaml` file change: ### Changed - Updated docs (#133, #140) -- Increased default read/write timeouts (#154) +- Increased default read/write timeouts (#154) - Updated SDK (#137, #139) - Updated go version to 1.17 (#143) - Improved error messages (#144) @@ -166,11 +166,11 @@ If you configure application using `.yaml` file change: - System headers format (#111) ### Added -- Different formats to set object's expiration: in epoch, duration, timestamp, +- Different formats to set object's expiration: in epoch, duration, timestamp, RFC3339 (#108) - Support of nodes priority (#115) -### Changed +### Changed - Updated testcontainers dependency (#100) ## [0.17.0] - 2021-11-15 diff --git a/Makefile b/Makefile old mode 100644 new mode 100755 index a4bdb1e..c8a1c23 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ OS_RELEASE = $(shell lsb_release -cs) PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \ sed "s/-/~/")-${OS_RELEASE} -.PHONY: debpackage debclean +.PHONY: debpackage debclean # Make all binaries all: $(BINS) diff --git a/README.md b/README.md index 3acf639..504768f 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,8 @@ can be done either via `-p` parameter or via `HTTP_GW_PEERS__ADDRESS` and `HTTP_GW_PEERS__WEIGHT` environment variables (the gate supports multiple FrostFS nodes with weighted load balancing). -If you launch HTTP gateway in bundle with [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env), -you can get the IP address of the node in the output of `make hosts` command +If you launch HTTP gateway in bundle with [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env), +you can get the IP address of the node in the output of `make hosts` command (with s0*.frostfs.devenv name). These two commands are functionally equivalent, they run the gate with one @@ -86,12 +86,12 @@ $ HTTP_GW_PEERS_0_ADDRESS=192.168.130.71:8080 HTTP_GW_PEERS_0_WEIGHT=1 HTTP_GW_P HTTP_GW_PEERS_2_ADDRESS=192.168.130.73:8080 HTTP_GW_PEERS_2_WEIGHT=1 HTTP_GW_PEERS_2_PRIORITY=2 \ frostfs-http-gw ``` -This command will make gateway use 192.168.130.71 while it is healthy. Otherwise, it will make the gateway use +This command will make gateway use 192.168.130.71 while it is healthy. Otherwise, it will make the gateway use 192.168.130.72 for 90% of requests and 192.168.130.73 for remaining 10%. ### Keys -You can provide a wallet via `--wallet` or `-w` flag. You can also specify the account address using `--address` -(if no address provided default one will be used). If wallet is used, you need to set `HTTP_GW_WALLET_PASSPHRASE` variable to decrypt the wallet. +You can provide a wallet via `--wallet` or `-w` flag. You can also specify the account address using `--address` +(if no address provided default one will be used). If wallet is used, you need to set `HTTP_GW_WALLET_PASSPHRASE` variable to decrypt the wallet. If no wallet provided, the gateway autogenerates a key pair it will use for FrostFS requests. ``` $ frostfs-http-gw -p $FROSTFS_NODE -w $WALLET_PATH --address $ACCOUNT_ADDRESS @@ -162,7 +162,7 @@ All timing options accept values with suffixes, so "15s" is 15 seconds and "2m" is 2 minutes. ### Zip streaming -The gateway supports downloading files by common prefix (like dir) in zip format. You can enable compression +The gateway supports downloading files by common prefix (like dir) in zip format. You can enable compression using config or `HTTP_GW_ZIP_COMPRESSION=true` environment variable. ### Logging @@ -172,7 +172,7 @@ HTTP_GW_LOGGER_LEVEL=debug ``` ### Yaml file -Configuration file is optional and can be used instead of environment variables/other parameters. +Configuration file is optional and can be used instead of environment variables/other parameters. It can be specified with `--config` parameter: ``` $ frostfs-http-gw --config your-config.yaml @@ -207,7 +207,7 @@ supported. ### Preparation -Before uploading or downloading a file make sure you have a prepared container. +Before uploading or downloading a file make sure you have a prepared container. You can create it with instructions below. Also, in case of downloading, you need to have a file inside a container. @@ -226,13 +226,13 @@ resolve_order: - nns ``` -2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env) +2. Make sure your container is registered in NNS contract. If you use [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env) you can check if your container (e.g. with `container-name` name) is registered in NNS: ```shell $ curl -s --data '{"id":1,"jsonrpc":"2.0","method":"getcontractstate","params":[1]}' \ http://morph-chain.frostfs.devenv:30333 | jq -r '.result.hash' - + 0x8e6c3cd4b976b28e84a3788f6ea9e2676c15d667 $ docker exec -it morph_chain neo-go \ @@ -241,7 +241,7 @@ $ docker exec -it morph_chain neo-go \ resolve string:container-name.container int:16 \ | jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \ | base64 -d && echo - + 7f3vvkw4iTiS5ZZbu5BQXEmJtETWbi3uUjLNaSs29xrL ``` @@ -257,7 +257,7 @@ You can create a container via [frostfs-cli](https://git.frostfs.info/TrueCloudL ``` $ frostfs-cli -r $FROSTFS_NODE -w $WALLET container create --policy $POLICY --basic-acl $ACL ``` -where `$WALLET` is a path to user wallet, +where `$WALLET` is a path to user wallet, `$ACL` -- hex encoded basic ACL value or keywords 'private, 'public-read', 'public-read-write' and `$POLICY` -- QL-encoded or JSON-encoded placement policy or path to file with it @@ -267,17 +267,17 @@ $ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json container create --policy ``` If you have launched nodes via [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env), -you can get the key value from `wallets/wallet.json` or write the path to +you can get the key value from `wallets/wallet.json` or write the path to the file `wallets/wallet.key`. #### Prepare a file in a container To create a file via [frostfs-cli](https://git.frostfs.info/TrueCloudLab/frostfs-node/releases), run a command below: ``` -$ frostfs-cli -r $FROSTFS_NODE -k $KEY object put --file $FILENAME --cid $CID +$ frostfs-cli -r $FROSTFS_NODE -k $KEY object put --file $FILENAME --cid $CID ``` -where -`$KEY` -- the key, please read the information [above](#create-a-container), +where +`$KEY` -- the key, please read the information [above](#create-a-container), `$CID` -- container ID. For example: @@ -290,13 +290,13 @@ $ frostfs-cli -r 192.168.130.72:8080 -w ./wallet.json object put --file cat.png #### Requests -The following requests support GET/HEAD methods. +The following requests support GET/HEAD methods. ##### By IDs Basic downloading involves container ID and object ID and is done via GET -requests to `/get/$CID/$OID` path, where `$CID` is a container ID or its name if NNS is enabled, -`$OID` is an object's (i.e. your file's) ID. +requests to `/get/$CID/$OID` path, where `$CID` is a container ID or its name if NNS is enabled, +`$OID` is an object's (i.e. your file's) ID. For example: @@ -317,12 +317,12 @@ can be used as well. The generic syntax for it looks like this: ```/get_by_attribute/$CID/$ATTRIBUTE_NAME/$ATTRIBUTE_VALUE``` -where -`$CID` is a container ID or its name if NNS is enabled, +where +`$CID` is a container ID or its name if NNS is enabled, `$ATTRIBUTE_NAME` is the name of the attribute we want to use, `$ATTRIBUTE_VALUE` is the value of this attribute that the target object should have. -**NB!** The attribute key and value should be url encoded, i.e., if you want to download an object with the attribute value +**NB!** The attribute key and value should be url encoded, i.e., if you want to download an object with the attribute value `a cat`, the value in the request must be `a+cat`. In the same way with the attribute key. If you don't escape such values everything can still work (for example you can use `d@ta` without encoding) but it's HIGHLY RECOMMENDED to encode all your attributes. @@ -346,7 +346,7 @@ Some other user-defined attributes: $ wget http://localhost:8082/get_by_attribute/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/Ololo/100500 ``` -Or when the attribute includes special symbols: +Or when the attribute includes special symbols: ``` $ wget http://localhost:8082/get_by_attribute/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/Olo%2Blo/100500 # means Olo+lo ``` @@ -365,7 +365,7 @@ You can download some dir (files with the same prefix) in zip (it will be compre $ wget http://localhost:8082/zip/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/common/prefix ``` -**Note:** the objects must have a valid `FilePath` attribute (it should not contain trailing `/`), +**Note:** the objects must have a valid `FilePath` attribute (it should not contain trailing `/`), otherwise they will not be in the zip archive. You can upload file with this attribute using `curl`: ``` @@ -393,7 +393,7 @@ set of reply headers generated using the following rules: ##### Caching strategy -HTTP Gateway doesn't control caching (doesn't anything with the `Cache-Control` header). Caching strategy strictly +HTTP Gateway doesn't control caching (doesn't anything with the `Cache-Control` header). Caching strategy strictly depends on application use case. So it should be carefully done by proxy server. ### Uploading @@ -424,7 +424,7 @@ You can also add some attributes to your file using the following rules: "X-Attribute-" prefix stripped, that is if you add "X-Attribute-Ololo: 100500" header to your request the resulting object will get "Ololo: 100500" attribute - * "X-Attribute-SYSTEM-*" headers are special + * "X-Attribute-SYSTEM-*" headers are special (`-SYSTEM-` part can also be `-system-` or`-System-` (and even legacy `-Neofs-` for some next releases)), they're used to set internal FrostFS attributes starting with `__SYSTEM__` prefix, for these attributes all dashes get converted to underscores and all letters are capitalized. For @@ -445,7 +445,7 @@ There are some reserved headers type of `X-Attribute-SYSTEM-*` (headers are arra 3. `X-Attribute-System-Expiration-Timestamp: 1637574797` 4. `X-Attribute-System-Expiration-RFC3339: 2021-11-22T09:55:49Z` -which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way. +which transforms to `X-Attribute-System-Expiration-Epoch`. So you can provide expiration any convenient way. --- @@ -484,7 +484,7 @@ the corresponding header to the upload request. Accessing the ACL protected data works the same way. ##### Example -In order to generate a bearer token, you need to know the container owner key and +In order to generate a bearer token, you need to know the container owner key and the address of the sender who will do the request to FrostFS (in our case, it's a gateway wallet address). Suppose we have: @@ -492,7 +492,7 @@ Suppose we have: * **NhVtreTTCoqsMQV5Wp55fqnriiUCpEaKm3** (token owner address) * **BJeErH9MWmf52VsR1mLWKkgF3pRm3FkubYxM7TZkBP4K** (container id) -Firstly, we need to encode the container id and the sender address to base64 (now it's base58). +Firstly, we need to encode the container id and the sender address to base64 (now it's base58). So use **base58** and **base64** utils. 1. Encoding container id: @@ -540,7 +540,7 @@ $ frostfs-cli util sign bearer-token --from bearer.json --to signed.json -w ./wa ``` Encoding to base64 to use via the header: ``` -$ base64 -w 0 signed.json +$ base64 -w 0 signed.json # output: Ck4KKgoECAIQBhIiCiCZGdlbN7DPGPMg9rsWqV+p2XdMzUqknRiexewSFp8kmBIbChk17MUri6OJ0X5ftsHzy7NERDNFB4C92PcaGgMIkE4SZgohAxpsb7vfAso1F0X6hrm6WpRS14WsT3/Ct1SMoqRsT89KEkEEGxKi8GjKSf52YqhppgaOTQHbUsL3jn7SHLqS3ndAQ7NtAATnmRHleZw2V2xRRSRBQdjDC05KK83LhdSax72Fsw== ``` @@ -599,8 +599,8 @@ File **eacl.json**: ### Metrics and Pprof -If enabled, Prometheus metrics are available at `localhost:8084` endpoint -and Pprof at `localhost:8083/debug/pprof` by default. Host and port can be configured. +If enabled, Prometheus metrics are available at `localhost:8084` endpoint +and Pprof at `localhost:8083/debug/pprof` by default. Host and port can be configured. See [configuration](./docs/gate-configuration.md). ## Credits diff --git a/debian/control b/debian/control index 612be94..7924bc6 100644 --- a/debian/control +++ b/debian/control @@ -12,4 +12,3 @@ Package: frostfs-http-gw Architecture: any Depends: ${misc:Depends} Description: FrostFS HTTP Gateway bridges FrostFS internal protocol and HTTP standard. - diff --git a/debian/copyright b/debian/copyright index c278a3a..9ab3cb9 100644 --- a/debian/copyright +++ b/debian/copyright @@ -14,12 +14,12 @@ License: GPL-3 This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 3. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License - along with this program or at /usr/share/common-licenses/GPL-3. - If not, see . + along with this program or at /usr/share/common-licenses/GPL-3. + If not, see . diff --git a/debian/frostfs-http-gw.postinst b/debian/frostfs-http-gw.postinst old mode 100644 new mode 100755 index 85b6c0f..360ceef --- a/debian/frostfs-http-gw.postinst +++ b/debian/frostfs-http-gw.postinst @@ -29,8 +29,8 @@ case "$1" in chmod -f 0640 /etc/frostfs/$USERNAME/config.yaml || true fi USERDIR=$(getent passwd "frostfs-$USERNAME" | cut -d: -f6) - if ! dpkg-statoverride --list frostfs-$USERDIR >/dev/null; then - chown -f frostfs-$USERNAME: $USERDIR + if ! dpkg-statoverride --list frostfs-"$USERDIR" >/dev/null; then + chown -f frostfs-$USERNAME: "$USERDIR" fi ;; diff --git a/debian/frostfs-http-gw.postrm b/debian/frostfs-http-gw.postrm old mode 100644 new mode 100755 diff --git a/debian/frostfs-http-gw.preinst b/debian/frostfs-http-gw.preinst old mode 100644 new mode 100755 diff --git a/debian/frostfs-http-gw.prerm b/debian/frostfs-http-gw.prerm old mode 100644 new mode 100755 diff --git a/debian/rules b/debian/rules index 477128a..0554034 100755 --- a/debian/rules +++ b/debian/rules @@ -8,9 +8,7 @@ SERVICE = frostfs-http-gw dh $@ override_dh_installsystemd: - dh_installsystemd --no-enable --no-start $(SERVICE).service + dh_installsystemd --no-enable --no-start $(SERVICE).service override_dh_installchangelogs: - dh_installchangelogs -k CHANGELOG.md - - + dh_installchangelogs -k CHANGELOG.md diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index d955d2a..8323bdc 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -62,9 +62,9 @@ resolve_order: - nns - dns -connect_timeout: 5s +connect_timeout: 5s stream_timeout: 10s -request_timeout: 5s +request_timeout: 5s rebalance_timer: 30s pool_error_threshold: 100 ``` @@ -83,8 +83,8 @@ pool_error_threshold: 100 ```yaml wallet: - path: /path/to/wallet.json - address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP + path: /path/to/wallet.json + address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP passphrase: pwd ``` @@ -201,7 +201,7 @@ upload_header: ```yaml zip: - compression: false + compression: false ``` | Parameter | Type | SIGHUP reload | Default value | Description |