nzinkevich/frostfs-http-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-http-gw
Code Pull requests Activity

[#170] Updated docs and configuration of archive section

Browse source 
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
Nikita Zinkevich 2024-12-06 15:01:16 +03:00
parent ed3c7aca60
commit 967ed1d6ae
Signed by: nzinkevich
GPG key ID: 748EA1D0B2E6420A
10 changed files with 41 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
cmd/http-gw/app.go
View file

@ -97,7 +97,7 @@ type (
mu sync.RWMutex
defaultTimestamp bool
zipCompression bool
archiveCompression bool
clientCut bool
returnIndexPage bool
indexPageTemplate string
@ -176,7 +176,7 @@ func (a *app) initAppSettings() {
func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
defaultTimestamp := v.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)
zipCompression := v.GetBool(cfgZipCompression)
archiveCompression := v.GetBool(cfgArchiveCompression)
returnIndexPage := v.GetBool(cfgIndexPageEnabled)
clientCut := v.GetBool(cfgClientCut)
bufferMaxSizeForPut := v.GetUint64(cfgBufferMaxSizeForPut)
@ -194,7 +194,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
defer s.mu.Unlock()
s.defaultTimestamp = defaultTimestamp
s.zipCompression = zipCompression
s.archiveCompression = archiveCompression
s.returnIndexPage = returnIndexPage
s.clientCut = clientCut
s.bufferMaxSizeForPut = bufferMaxSizeForPut
@ -232,10 +232,10 @@ func (s *appSettings) DefaultTimestamp() bool {
return s.defaultTimestamp
}
func (s *appSettings) ZipCompression() bool {
func (s *appSettings) ArchiveCompression() bool {
s.mu.RLock()
defer s.mu.RUnlock()
return s.zipCompression
return s.archiveCompression
}
func (s *appSettings) IndexPageEnabled() bool {

6
cmd/http-gw/settings.go
View file

@ -127,8 +127,8 @@ const (
// Resolving.
cfgResolveOrder = "resolve_order"
// Zip compression.
cfgZipCompression = "zip.compression"
// Archive compression.
cfgArchiveCompression = "archive.compression"
// Runtime.
cfgSoftMemoryLimit = "runtime.soft_memory_limit"
@ -251,7 +251,7 @@ func settings() *viper.Viper {
v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false)
// zip:
v.SetDefault(cfgZipCompression, false)
v.SetDefault(cfgArchiveCompression, false)
// metrics
v.SetDefault(cfgPprofAddress, "localhost:8083")

4
config/config.env
View file

@ -97,8 +97,8 @@ HTTP_GW_REBALANCE_TIMER=30s
# The number of errors on connection after which node is considered as unhealthy
HTTP_GW_POOL_ERROR_THRESHOLD=100
# Enable zip compression to download files by common prefix.
HTTP_GW_ZIP_COMPRESSION=false
# Enable archive compression to download files by common prefix.
HTTP_GW_ARCHIVE_COMPRESSION=false
HTTP_GW_TRACING_ENABLED=true
HTTP_GW_TRACING_ENDPOINT="localhost:4317"

4
config/config.yaml
View file

@ -121,8 +121,8 @@ index_page:
enabled: false
template_path: internal/handler/templates/index.gotmpl
zip:
compression: false # Enable zip compression to download files by common prefix.
archive:
compression: false # Enable archive compression to download files by common prefix.
runtime:
soft_memory_limit: 1gb

36
docs/api.md
View file

@ -1,11 +1,11 @@
# HTTP Gateway Specification
| Route | Description |
|-------------------------------------------------|----------------------------------------------|
| `/upload/{cid}` | [Put object](#put-object) |
| `/get/{cid}/{oid}` | [Get object](#get-object) |
| `/get_by_attribute/{cid}/{attr_key}/{attr_val}` | [Search object](#search-object) |
| `/zip/{cid}/{prefix}` | [Download objects in archive](#download-zip) |
| Route | Description |
|-------------------------------------------------|--------------------------------------------------|
| `/upload/{cid}` | [Put object](#put-object) |
| `/get/{cid}/{oid}` | [Get object](#get-object) |
| `/get_by_attribute/{cid}/{attr_key}/{attr_val}` | [Search object](#search-object) |
| `/zip/{cid}/{prefix}`, `/tar/{cid}/{prefix}` | [Download objects in archive](#download-archive) |
**Note:** `cid` parameter can be base58 encoded container ID or container name
(the name must be registered in NNS, see appropriate section in [README](../README.md#nns)).
@ -56,12 +56,13 @@ Upload file as object with attributes to FrostFS.
###### Headers
| Header | Description |
|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|
| Common headers | See [bearer token](#bearer-token). |
| `X-Attribute-System-*` | Used to set system FrostFS object attributes <br/> (e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). |
| `X-Attribute-*` | Used to set regular object attributes <br/> (e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). |
| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. |
| Header | Description |
|-------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Common headers | See [bearer token](#bearer-token). |
| `X-Attribute-System-*` | Used to set system FrostFS object attributes <br/> (e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). |
| `X-Attribute-*` | Used to set regular object attributes <br/> (e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). |
| `X-Attribute-Explode-Archive` | If this header is set, gate reads files from uploaded `tar.gz` archive and creates object for each file in it. Set FilePath attribute as relative path from archive root. |
| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. |
There are some reserved headers type of `X-Attribute-FROSTFS-*` (headers are arranged in descending order of priority):
@ -269,9 +270,9 @@ If more than one object is found, an arbitrary one will be used to get attribute
| 400 | Some error occurred during operation. |
| 404 | Container or object not found. |
## Download zip
## Download archive
Route: `/zip/{cid}/{prefix}`
Route: `/zip/{cid}/{prefix}`, `/tar/{cid}/{prefix}`
| Route parameter | Type | Description |
|-----------------|-----------|---------------------------------------------------------|
@ -282,12 +283,13 @@ Route: `/zip/{cid}/{prefix}`
#### GET
Find objects by prefix for `FilePath` attributes. Return found objects in zip archive.
Find objects by prefix for `FilePath` attributes. Return found objects in zip or tar archive.
Name of files in archive sets to `FilePath` attribute of objects.
Time of files sets to time when object has started downloading.
You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` route.
You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` or
`/tar/{cid}/` route.
Archive can be compressed (see http-gw [configuration](gate-configuration.md#zip-section)).
Archive can be compressed (see http-gw [configuration](gate-configuration.md#archive-section)).
##### Request

11
docs/gate-configuration.md
View file

@ -218,17 +218,16 @@ upload_header:
|-------------------------|--------|---------------|---------------|-------------------------------------------------------------|
| `use_default_timestamp` | `bool` | yes | `false` | Create timestamp for object if it isn't provided by header. |
# `zip` section
# `archive` section
```yaml
zip:
archive:
compression: false
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|---------------|--------|---------------|---------------|--------------------------------------------------------------|
| `compression` | `bool` | yes | `false` | Enable zip compression when download files by common prefix. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|---------------|--------|---------------|---------------|------------------------------------------------------------------|
| `compression` | `bool` | yes | `false` | Enable archive compression when download files by common prefix. |
# `pprof` section

4
internal/handler/download.go
View file

@ -105,7 +105,7 @@ func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, res
func (h *Handler) createZipFile(zw *zip.Writer, obj *object.Object) (io.Writer, error) {
method := zip.Store
if h.config.ZipCompression() {
if h.config.ArchiveCompression() {
method = zip.Deflate
}
@ -148,7 +148,7 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res
defer resSearch.Close()
var gzipWriter *gzip.Writer
if h.config.ZipCompression() {
if h.config.ArchiveCompression() {
gzipWriter, _ = gzip.NewWriterLevel(w, gzip.DefaultCompression)
} else {
gzipWriter, _ = gzip.NewWriterLevel(w, gzip.NoCompression)

2
internal/handler/handler.go
View file

@ -29,7 +29,7 @@ import (
type Config interface {
DefaultTimestamp() bool
ZipCompression() bool
ArchiveCompression() bool
ClientCut() bool
IndexPageEnabled() bool
IndexPageTemplate() string

2
internal/handler/handler_test.go
View file

@ -50,7 +50,7 @@ func (c *configMock) DefaultTimestamp() bool {
return false
}
func (c *configMock) ZipCompression() bool {
func (c *configMock) ArchiveCompression() bool {
return false
}

2
internal/handler/upload.go
View file

@ -197,7 +197,7 @@ func (h *Handler) uploadObject(c *fasthttp.RequestCtx, log *zap.Logger, bktInfo
}
// explodeGzip read files from tar.gz archive and creates objects for each of them.
// Sets FilePath attribute with name from tar.Header
// Sets FilePath attribute with Name from tar.Header.
func (h *Handler) explodeGzip(c *fasthttp.RequestCtx, log *zap.Logger, bktInfo *data.BucketInfo, file io.Reader) {
gzipReader, err := gzip.NewReader(file)
if err != nil {