TrueCloudLab/frostfs-http-gw
18
1
Fork 18
Code Issues 11 Pull requests 7 Projects Releases 10 Activity Actions

[#170] Updated docs and configuration of archive section
All checks were successful
/ DCO (pull_request) Successful in 9m16s
Details
/ Vulncheck (pull_request) Successful in 9m28s
Details
/ Lint (pull_request) Successful in 3m0s
Details
/ Tests (pull_request) Successful in 2m4s
Details
/ Builds (pull_request) Successful in 2m0s
Details

Browse source 
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
Nikita Zinkevich 2024-12-06 15:01:16 +03:00
parent c9050c9daf
commit b353b2cd81
Signed by: nzinkevich
GPG key ID: 748EA1D0B2E6420A
9 changed files with 40 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
cmd/http-gw/app.go
View file

@ -97,7 +97,7 @@ type (
mu sync.RWMutex mu sync.RWMutex
defaultTimestamp bool defaultTimestamp bool
zipCompression bool archiveCompression bool
clientCut bool clientCut bool
returnIndexPage bool returnIndexPage bool
indexPageTemplate string indexPageTemplate string
@ -177,7 +177,7 @@ func (a *app) initAppSettings() {
func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
defaultTimestamp := v.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) defaultTimestamp := v.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)
zipCompression := v.GetBool(cfgZipCompression) archiveCompression := v.GetBool(cfgArchiveCompression)
returnIndexPage := v.GetBool(cfgIndexPageEnabled) returnIndexPage := v.GetBool(cfgIndexPageEnabled)
clientCut := v.GetBool(cfgClientCut) clientCut := v.GetBool(cfgClientCut)
bufferMaxSizeForPut := v.GetUint64(cfgBufferMaxSizeForPut) bufferMaxSizeForPut := v.GetUint64(cfgBufferMaxSizeForPut)
@ -196,7 +196,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
defer s.mu.Unlock() defer s.mu.Unlock()
s.defaultTimestamp = defaultTimestamp s.defaultTimestamp = defaultTimestamp
s.zipCompression = zipCompression s.archiveCompression = archiveCompression
s.returnIndexPage = returnIndexPage s.returnIndexPage = returnIndexPage
s.clientCut = clientCut s.clientCut = clientCut
s.bufferMaxSizeForPut = bufferMaxSizeForPut s.bufferMaxSizeForPut = bufferMaxSizeForPut
@ -235,10 +235,10 @@ func (s *appSettings) DefaultTimestamp() bool {
return s.defaultTimestamp return s.defaultTimestamp
} }
func (s *appSettings) ZipCompression() bool { func (s *appSettings) ArchiveCompression() bool {
s.mu.RLock() s.mu.RLock()
defer s.mu.RUnlock() defer s.mu.RUnlock()
return s.zipCompression return s.archiveCompression
} }
func (s *appSettings) IndexPageEnabled() bool { func (s *appSettings) IndexPageEnabled() bool {

6
cmd/http-gw/settings.go
View file

@ -127,8 +127,8 @@ const (
// Resolving. // Resolving.
cfgResolveOrder = "resolve_order" cfgResolveOrder = "resolve_order"
// Zip compression. // Archive compression.
cfgZipCompression = "zip.compression" cfgArchiveCompression = "archive.compression"
// Runtime. // Runtime.
cfgSoftMemoryLimit = "runtime.soft_memory_limit" cfgSoftMemoryLimit = "runtime.soft_memory_limit"
@ -254,7 +254,7 @@ func settings() *viper.Viper {
v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false)
// zip: // zip:
v.SetDefault(cfgZipCompression, false) v.SetDefault(cfgArchiveCompression, false)
// metrics // metrics
v.SetDefault(cfgPprofAddress, "localhost:8083") v.SetDefault(cfgPprofAddress, "localhost:8083")

4
config/config.env
View file

@ -97,8 +97,8 @@ HTTP_GW_REBALANCE_TIMER=30s
# The number of errors on connection after which node is considered as unhealthy # The number of errors on connection after which node is considered as unhealthy
HTTP_GW_POOL_ERROR_THRESHOLD=100 HTTP_GW_POOL_ERROR_THRESHOLD=100
# Enable zip compression to download files by common prefix. # Enable archive compression to download files by common prefix.
HTTP_GW_ZIP_COMPRESSION=false HTTP_GW_ARCHIVE_COMPRESSION=false
HTTP_GW_TRACING_ENABLED=true HTTP_GW_TRACING_ENABLED=true
HTTP_GW_TRACING_ENDPOINT="localhost:4317" HTTP_GW_TRACING_ENDPOINT="localhost:4317"

4
config/config.yaml
View file

@ -121,8 +121,8 @@ index_page:
enabled: false enabled: false
template_path: internal/handler/templates/index.gotmpl template_path: internal/handler/templates/index.gotmpl
zip: archive:
compression: false # Enable zip compression to download files by common prefix. compression: false # Enable archive compression to download files by common prefix.
runtime: runtime:
soft_memory_limit: 1gb soft_memory_limit: 1gb

36
docs/api.md
View file

@ -1,11 +1,11 @@
# HTTP Gateway Specification # HTTP Gateway Specification
| Route | Description | | Route | Description |
|-------------------------------------------------|----------------------------------------------| |-------------------------------------------------|--------------------------------------------------|
| `/upload/{cid}` | [Put object](#put-object) | | `/upload/{cid}` | [Put object](#put-object) |
| `/get/{cid}/{oid}` | [Get object](#get-object) | | `/get/{cid}/{oid}` | [Get object](#get-object) |
| `/get_by_attribute/{cid}/{attr_key}/{attr_val}` | [Search object](#search-object) | | `/get_by_attribute/{cid}/{attr_key}/{attr_val}` | [Search object](#search-object) |
| `/zip/{cid}/{prefix}` | [Download objects in archive](#download-zip) | | `/zip/{cid}/{prefix}`, `/tar/{cid}/{prefix}` | [Download objects in archive](#download-archive) |
**Note:** `cid` parameter can be base58 encoded container ID or container name **Note:** `cid` parameter can be base58 encoded container ID or container name
(the name must be registered in NNS, see appropriate section in [nns.md](./nns.md)). (the name must be registered in NNS, see appropriate section in [nns.md](./nns.md)).
@ -56,12 +56,13 @@ Upload file as object with attributes to FrostFS.
###### Headers ###### Headers
| Header | Description | | Header | Description |
|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| |------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Common headers | See [bearer token](#bearer-token). | | Common headers | See [bearer token](#bearer-token). |
| `X-Attribute-System-*` | Used to set system FrostFS object attributes <br/> (e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). | | `X-Attribute-System-*` | Used to set system FrostFS object attributes <br/> (e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). |
| `X-Attribute-*` | Used to set regular object attributes <br/> (e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). | | `X-Attribute-*` | Used to set regular object attributes <br/> (e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). |
| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. | | `X-Explode-Archive` | If set, gate tries to read files from uploading `tar` archive and creates object for each file in it. Uploading `tar` could be compressed via Gzip. Sets FilePath attribute as relative path from archive root. |
| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. |
There are some reserved headers type of `X-Attribute-FROSTFS-*` (headers are arranged in descending order of priority): There are some reserved headers type of `X-Attribute-FROSTFS-*` (headers are arranged in descending order of priority):
@ -269,9 +270,9 @@ If more than one object is found, an arbitrary one will be used to get attribute
| 400 | Some error occurred during operation. | | 400 | Some error occurred during operation. |
| 404 | Container or object not found. | | 404 | Container or object not found. |
## Download zip ## Download archive
Route: `/zip/{cid}/{prefix}` Route: `/zip/{cid}/{prefix}`, `/tar/{cid}/{prefix}`
| Route parameter | Type | Description | | Route parameter | Type | Description |
|-----------------|-----------|---------------------------------------------------------| |-----------------|-----------|---------------------------------------------------------|
@ -282,12 +283,13 @@ Route: `/zip/{cid}/{prefix}`
#### GET #### GET
Find objects by prefix for `FilePath` attributes. Return found objects in zip archive. Find objects by prefix for `FilePath` attributes. Return found objects in zip or tar archive.
Name of files in archive sets to `FilePath` attribute of objects. Name of files in archive sets to `FilePath` attribute of objects.
Time of files sets to time when object has started downloading. Time of files sets to time when object has started downloading.
You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` route. You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` or
`/tar/{cid}/` route.
Archive can be compressed (see http-gw [configuration](gate-configuration.md#zip-section)). Archive can be compressed (see http-gw [configuration](gate-configuration.md#archive-section)).
##### Request ##### Request

11
docs/gate-configuration.md
View file

@ -218,17 +218,16 @@ upload_header:
|-------------------------|--------|---------------|---------------|-------------------------------------------------------------| |-------------------------|--------|---------------|---------------|-------------------------------------------------------------|
| `use_default_timestamp` | `bool` | yes | `false` | Create timestamp for object if it isn't provided by header. | | `use_default_timestamp` | `bool` | yes | `false` | Create timestamp for object if it isn't provided by header. |
# `archive` section
# `zip` section
```yaml ```yaml
zip: archive:
compression: false compression: false
``` ```
| Parameter | Type | SIGHUP reload | Default value | Description | | Parameter | Type | SIGHUP reload | Default value | Description |
|---------------|--------|---------------|---------------|--------------------------------------------------------------| |---------------|--------|---------------|---------------|------------------------------------------------------------------|
| `compression` | `bool` | yes | `false` | Enable zip compression when download files by common prefix. | | `compression` | `bool` | yes | `false` | Enable archive compression when download files by common prefix. |
# `pprof` section # `pprof` section

4
internal/handler/download.go
View file

@ -143,7 +143,7 @@ func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, res
func (h *Handler) createZipFile(zw *zip.Writer, obj *object.Object) (io.Writer, error) { func (h *Handler) createZipFile(zw *zip.Writer, obj *object.Object) (io.Writer, error) {
method := zip.Store method := zip.Store
if h.config.ZipCompression() { if h.config.ArchiveCompression() {
method = zip.Deflate method = zip.Deflate
} }
@ -186,7 +186,7 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res
defer resSearch.Close() defer resSearch.Close()
compressionLevel := gzip.NoCompression compressionLevel := gzip.NoCompression
if h.config.ZipCompression() { if h.config.ArchiveCompression() {
compressionLevel = gzip.DefaultCompression compressionLevel = gzip.DefaultCompression
} }

2
internal/handler/handler.go
View file

@ -29,7 +29,7 @@ import (
type Config interface { type Config interface {
DefaultTimestamp() bool DefaultTimestamp() bool
ZipCompression() bool ArchiveCompression() bool
ClientCut() bool ClientCut() bool
IndexPageEnabled() bool IndexPageEnabled() bool
IndexPageTemplate() string IndexPageTemplate() string

2
internal/handler/handler_test.go
View file

@ -66,7 +66,7 @@ func (c *configMock) DefaultTimestamp() bool {
return false return false
} }
func (c *configMock) ZipCompression() bool { func (c *configMock) ArchiveCompression() bool {
return false return false
} }