From b353b2cd81aac520810cb98b736dceb45c3864e0 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 6 Dec 2024 15:01:16 +0300 Subject: [PATCH] [#170] Updated docs and configuration of archive section Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 10 ++++----- cmd/http-gw/settings.go | 6 +++--- config/config.env | 4 ++-- config/config.yaml | 4 ++-- docs/api.md | 36 +++++++++++++++++--------------- docs/gate-configuration.md | 11 +++++----- internal/handler/download.go | 4 ++-- internal/handler/handler.go | 2 +- internal/handler/handler_test.go | 2 +- 9 files changed, 40 insertions(+), 39 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index adbc8ae..ade8898 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -97,7 +97,7 @@ type ( mu sync.RWMutex defaultTimestamp bool - zipCompression bool + archiveCompression bool clientCut bool returnIndexPage bool indexPageTemplate string @@ -177,7 +177,7 @@ func (a *app) initAppSettings() { func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { defaultTimestamp := v.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) - zipCompression := v.GetBool(cfgZipCompression) + archiveCompression := v.GetBool(cfgArchiveCompression) returnIndexPage := v.GetBool(cfgIndexPageEnabled) clientCut := v.GetBool(cfgClientCut) bufferMaxSizeForPut := v.GetUint64(cfgBufferMaxSizeForPut) @@ -196,7 +196,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { defer s.mu.Unlock() s.defaultTimestamp = defaultTimestamp - s.zipCompression = zipCompression + s.archiveCompression = archiveCompression s.returnIndexPage = returnIndexPage s.clientCut = clientCut s.bufferMaxSizeForPut = bufferMaxSizeForPut @@ -235,10 +235,10 @@ func (s *appSettings) DefaultTimestamp() bool { return s.defaultTimestamp } -func (s *appSettings) ZipCompression() bool { +func (s *appSettings) ArchiveCompression() bool { s.mu.RLock() defer s.mu.RUnlock() - return s.zipCompression + return s.archiveCompression } func (s *appSettings) IndexPageEnabled() bool { diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 62ef83e..de03fac 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -127,8 +127,8 @@ const ( // Resolving. cfgResolveOrder = "resolve_order" - // Zip compression. - cfgZipCompression = "zip.compression" + // Archive compression. + cfgArchiveCompression = "archive.compression" // Runtime. cfgSoftMemoryLimit = "runtime.soft_memory_limit" @@ -254,7 +254,7 @@ func settings() *viper.Viper { v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) // zip: - v.SetDefault(cfgZipCompression, false) + v.SetDefault(cfgArchiveCompression, false) // metrics v.SetDefault(cfgPprofAddress, "localhost:8083") diff --git a/config/config.env b/config/config.env index 2822357..54053e6 100644 --- a/config/config.env +++ b/config/config.env @@ -97,8 +97,8 @@ HTTP_GW_REBALANCE_TIMER=30s # The number of errors on connection after which node is considered as unhealthy HTTP_GW_POOL_ERROR_THRESHOLD=100 -# Enable zip compression to download files by common prefix. -HTTP_GW_ZIP_COMPRESSION=false +# Enable archive compression to download files by common prefix. +HTTP_GW_ARCHIVE_COMPRESSION=false HTTP_GW_TRACING_ENABLED=true HTTP_GW_TRACING_ENDPOINT="localhost:4317" diff --git a/config/config.yaml b/config/config.yaml index 6296bd9..855e117 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -121,8 +121,8 @@ index_page: enabled: false template_path: internal/handler/templates/index.gotmpl -zip: - compression: false # Enable zip compression to download files by common prefix. +archive: + compression: false # Enable archive compression to download files by common prefix. runtime: soft_memory_limit: 1gb diff --git a/docs/api.md b/docs/api.md index e59956a..1400db8 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,11 +1,11 @@ # HTTP Gateway Specification -| Route | Description | -|-------------------------------------------------|----------------------------------------------| -| `/upload/{cid}` | [Put object](#put-object) | -| `/get/{cid}/{oid}` | [Get object](#get-object) | -| `/get_by_attribute/{cid}/{attr_key}/{attr_val}` | [Search object](#search-object) | -| `/zip/{cid}/{prefix}` | [Download objects in archive](#download-zip) | +| Route | Description | +|-------------------------------------------------|--------------------------------------------------| +| `/upload/{cid}` | [Put object](#put-object) | +| `/get/{cid}/{oid}` | [Get object](#get-object) | +| `/get_by_attribute/{cid}/{attr_key}/{attr_val}` | [Search object](#search-object) | +| `/zip/{cid}/{prefix}`, `/tar/{cid}/{prefix}` | [Download objects in archive](#download-archive) | **Note:** `cid` parameter can be base58 encoded container ID or container name (the name must be registered in NNS, see appropriate section in [nns.md](./nns.md)). @@ -56,12 +56,13 @@ Upload file as object with attributes to FrostFS. ###### Headers -| Header | Description | -|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| -| Common headers | See [bearer token](#bearer-token). | -| `X-Attribute-System-*` | Used to set system FrostFS object attributes
(e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). | -| `X-Attribute-*` | Used to set regular object attributes
(e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). | -| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. | +| Header | Description | +|------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Common headers | See [bearer token](#bearer-token). | +| `X-Attribute-System-*` | Used to set system FrostFS object attributes
(e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). | +| `X-Attribute-*` | Used to set regular object attributes
(e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). | +| `X-Explode-Archive` | If set, gate tries to read files from uploading `tar` archive and creates object for each file in it. Uploading `tar` could be compressed via Gzip. Sets FilePath attribute as relative path from archive root. | +| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. | There are some reserved headers type of `X-Attribute-FROSTFS-*` (headers are arranged in descending order of priority): @@ -269,9 +270,9 @@ If more than one object is found, an arbitrary one will be used to get attribute | 400 | Some error occurred during operation. | | 404 | Container or object not found. | -## Download zip +## Download archive -Route: `/zip/{cid}/{prefix}` +Route: `/zip/{cid}/{prefix}`, `/tar/{cid}/{prefix}` | Route parameter | Type | Description | |-----------------|-----------|---------------------------------------------------------| @@ -282,12 +283,13 @@ Route: `/zip/{cid}/{prefix}` #### GET -Find objects by prefix for `FilePath` attributes. Return found objects in zip archive. +Find objects by prefix for `FilePath` attributes. Return found objects in zip or tar archive. Name of files in archive sets to `FilePath` attribute of objects. Time of files sets to time when object has started downloading. -You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` route. +You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` or +`/tar/{cid}/` route. -Archive can be compressed (see http-gw [configuration](gate-configuration.md#zip-section)). +Archive can be compressed (see http-gw [configuration](gate-configuration.md#archive-section)). ##### Request diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 7476f5d..75d7f40 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -218,17 +218,16 @@ upload_header: |-------------------------|--------|---------------|---------------|-------------------------------------------------------------| | `use_default_timestamp` | `bool` | yes | `false` | Create timestamp for object if it isn't provided by header. | - -# `zip` section +# `archive` section ```yaml -zip: +archive: compression: false ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|---------------|--------|---------------|---------------|--------------------------------------------------------------| -| `compression` | `bool` | yes | `false` | Enable zip compression when download files by common prefix. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|---------------|--------|---------------|---------------|------------------------------------------------------------------| +| `compression` | `bool` | yes | `false` | Enable archive compression when download files by common prefix. | # `pprof` section diff --git a/internal/handler/download.go b/internal/handler/download.go index 3d34a30..191c0b1 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -143,7 +143,7 @@ func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, res func (h *Handler) createZipFile(zw *zip.Writer, obj *object.Object) (io.Writer, error) { method := zip.Store - if h.config.ZipCompression() { + if h.config.ArchiveCompression() { method = zip.Deflate } @@ -186,7 +186,7 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res defer resSearch.Close() compressionLevel := gzip.NoCompression - if h.config.ZipCompression() { + if h.config.ArchiveCompression() { compressionLevel = gzip.DefaultCompression } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 653c02d..558f83e 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -29,7 +29,7 @@ import ( type Config interface { DefaultTimestamp() bool - ZipCompression() bool + ArchiveCompression() bool ClientCut() bool IndexPageEnabled() bool IndexPageTemplate() string diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 14f9c98..ca8991c 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -66,7 +66,7 @@ func (c *configMock) DefaultTimestamp() bool { return false } -func (c *configMock) ZipCompression() bool { +func (c *configMock) ArchiveCompression() bool { return false }