[#170] Updated docs and configuration of archive section
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
parent
ed3c7aca60
commit
967ed1d6ae
GPG key ID:
748EA1D0B2E6420A
10 changed files with 41 additions and 40 deletions
|
|
@ -97,7 +97,7 @@ type (
|
|||
|
||||
mu sync.RWMutex
|
||||
defaultTimestamp bool
|
||||
zipCompression bool
|
||||
archiveCompression bool
|
||||
clientCut bool
|
||||
returnIndexPage bool
|
||||
indexPageTemplate string
|
||||
|
|
@ -176,7 +176,7 @@ func (a *app) initAppSettings() {
|
|||
|
||||
func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
|
||||
defaultTimestamp := v.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)
|
||||
zipCompression := v.GetBool(cfgZipCompression)
|
||||
archiveCompression := v.GetBool(cfgArchiveCompression)
|
||||
returnIndexPage := v.GetBool(cfgIndexPageEnabled)
|
||||
clientCut := v.GetBool(cfgClientCut)
|
||||
bufferMaxSizeForPut := v.GetUint64(cfgBufferMaxSizeForPut)
|
||||
|
|
@ -194,7 +194,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
|
|||
defer s.mu.Unlock()
|
||||
|
||||
s.defaultTimestamp = defaultTimestamp
|
||||
s.zipCompression = zipCompression
|
||||
s.archiveCompression = archiveCompression
|
||||
s.returnIndexPage = returnIndexPage
|
||||
s.clientCut = clientCut
|
||||
s.bufferMaxSizeForPut = bufferMaxSizeForPut
|
||||
|
|
@ -232,10 +232,10 @@ func (s *appSettings) DefaultTimestamp() bool {
|
|||
return s.defaultTimestamp
|
||||
}
|
||||
|
||||
func (s *appSettings) ZipCompression() bool {
|
||||
func (s *appSettings) ArchiveCompression() bool {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.zipCompression
|
||||
return s.archiveCompression
|
||||
}
|
||||
|
||||
func (s *appSettings) IndexPageEnabled() bool {
|
||||
|
|
|
|||
|
|
@ -127,8 +127,8 @@ const (
|
|||
// Resolving.
|
||||
cfgResolveOrder = "resolve_order"
|
||||
|
||||
// Zip compression.
|
||||
cfgZipCompression = "zip.compression"
|
||||
// Archive compression.
|
||||
cfgArchiveCompression = "archive.compression"
|
||||
|
||||
// Runtime.
|
||||
cfgSoftMemoryLimit = "runtime.soft_memory_limit"
|
||||
|
|
@ -251,7 +251,7 @@ func settings() *viper.Viper {
|
|||
v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false)
|
||||
|
||||
// zip:
|
||||
v.SetDefault(cfgZipCompression, false)
|
||||
v.SetDefault(cfgArchiveCompression, false)
|
||||
|
||||
// metrics
|
||||
v.SetDefault(cfgPprofAddress, "localhost:8083")
|
||||
|
|
|
|||
|
|
@ -97,8 +97,8 @@ HTTP_GW_REBALANCE_TIMER=30s
|
|||
# The number of errors on connection after which node is considered as unhealthy
|
||||
HTTP_GW_POOL_ERROR_THRESHOLD=100
|
||||
|
||||
# Enable zip compression to download files by common prefix.
|
||||
HTTP_GW_ZIP_COMPRESSION=false
|
||||
# Enable archive compression to download files by common prefix.
|
||||
HTTP_GW_ARCHIVE_COMPRESSION=false
|
||||
|
||||
HTTP_GW_TRACING_ENABLED=true
|
||||
HTTP_GW_TRACING_ENDPOINT="localhost:4317"
|
||||
|
|
|
|||
|
|
@ -121,8 +121,8 @@ index_page:
|
|||
enabled: false
|
||||
template_path: internal/handler/templates/index.gotmpl
|
||||
|
||||
zip:
|
||||
compression: false # Enable zip compression to download files by common prefix.
|
||||
archive:
|
||||
compression: false # Enable archive compression to download files by common prefix.
|
||||
|
||||
runtime:
|
||||
soft_memory_limit: 1gb
|
||||
|
|
|
|||
18
docs/api.md
18
docs/api.md
|
|
@ -1,11 +1,11 @@
|
|||
# HTTP Gateway Specification
|
||||
|
||||
| Route | Description |
|
||||
|-------------------------------------------------|----------------------------------------------|
|
||||
|-------------------------------------------------|--------------------------------------------------|
|
||||
| `/upload/{cid}` | [Put object](#put-object) |
|
||||
| `/get/{cid}/{oid}` | [Get object](#get-object) |
|
||||
| `/get_by_attribute/{cid}/{attr_key}/{attr_val}` | [Search object](#search-object) |
|
||||
| `/zip/{cid}/{prefix}` | [Download objects in archive](#download-zip) |
|
||||
| `/zip/{cid}/{prefix}`, `/tar/{cid}/{prefix}` | [Download objects in archive](#download-archive) |
|
||||
|
||||
**Note:** `cid` parameter can be base58 encoded container ID or container name
|
||||
(the name must be registered in NNS, see appropriate section in [README](../README.md#nns)).
|
||||
|
|
@ -57,10 +57,11 @@ Upload file as object with attributes to FrostFS.
|
|||
###### Headers
|
||||
|
||||
| Header | Description |
|
||||
|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
|-------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| Common headers | See [bearer token](#bearer-token). |
|
||||
| `X-Attribute-System-*` | Used to set system FrostFS object attributes <br/> (e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute). |
|
||||
| `X-Attribute-*` | Used to set regular object attributes <br/> (e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute). |
|
||||
| `X-Attribute-Explode-Archive` | If this header is set, gate reads files from uploaded `tar.gz` archive and creates object for each file in it. Set FilePath attribute as relative path from archive root. |
|
||||
| `Date` | This header is used to calculate the right `__SYSTEM__EXPIRATION` attribute for object. If the header is missing, the current server time is used. |
|
||||
|
||||
There are some reserved headers type of `X-Attribute-FROSTFS-*` (headers are arranged in descending order of priority):
|
||||
|
|
@ -269,9 +270,9 @@ If more than one object is found, an arbitrary one will be used to get attribute
|
|||
| 400 | Some error occurred during operation. |
|
||||
| 404 | Container or object not found. |
|
||||
|
||||
## Download zip
|
||||
## Download archive
|
||||
|
||||
Route: `/zip/{cid}/{prefix}`
|
||||
Route: `/zip/{cid}/{prefix}`, `/tar/{cid}/{prefix}`
|
||||
|
||||
| Route parameter | Type | Description |
|
||||
|-----------------|-----------|---------------------------------------------------------|
|
||||
|
|
@ -282,12 +283,13 @@ Route: `/zip/{cid}/{prefix}`
|
|||
|
||||
#### GET
|
||||
|
||||
Find objects by prefix for `FilePath` attributes. Return found objects in zip archive.
|
||||
Find objects by prefix for `FilePath` attributes. Return found objects in zip or tar archive.
|
||||
Name of files in archive sets to `FilePath` attribute of objects.
|
||||
Time of files sets to time when object has started downloading.
|
||||
You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` route.
|
||||
You can download all files in container that have `FilePath` attribute by `/zip/{cid}/` or
|
||||
`/tar/{cid}/` route.
|
||||
|
||||
Archive can be compressed (see http-gw [configuration](gate-configuration.md#zip-section)).
|
||||
Archive can be compressed (see http-gw [configuration](gate-configuration.md#archive-section)).
|
||||
|
||||
##### Request
|
||||
|
||||
|
|
|
|||
|
|
@ -218,17 +218,16 @@ upload_header:
|
|||
|-------------------------|--------|---------------|---------------|-------------------------------------------------------------|
|
||||
| `use_default_timestamp` | `bool` | yes | `false` | Create timestamp for object if it isn't provided by header. |
|
||||
|
||||
|
||||
# `zip` section
|
||||
# `archive` section
|
||||
|
||||
```yaml
|
||||
zip:
|
||||
archive:
|
||||
compression: false
|
||||
```
|
||||
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|---------------|--------|---------------|---------------|--------------------------------------------------------------|
|
||||
| `compression` | `bool` | yes | `false` | Enable zip compression when download files by common prefix. |
|
||||
|---------------|--------|---------------|---------------|------------------------------------------------------------------|
|
||||
| `compression` | `bool` | yes | `false` | Enable archive compression when download files by common prefix. |
|
||||
|
||||
|
||||
# `pprof` section
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, res
|
|||
|
||||
func (h *Handler) createZipFile(zw *zip.Writer, obj *object.Object) (io.Writer, error) {
|
||||
method := zip.Store
|
||||
if h.config.ZipCompression() {
|
||||
if h.config.ArchiveCompression() {
|
||||
method = zip.Deflate
|
||||
}
|
||||
|
||||
|
|
@ -148,7 +148,7 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res
|
|||
defer resSearch.Close()
|
||||
|
||||
var gzipWriter *gzip.Writer
|
||||
if h.config.ZipCompression() {
|
||||
if h.config.ArchiveCompression() {
|
||||
gzipWriter, _ = gzip.NewWriterLevel(w, gzip.DefaultCompression)
|
||||
} else {
|
||||
gzipWriter, _ = gzip.NewWriterLevel(w, gzip.NoCompression)
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ import (
|
|||
|
||||
type Config interface {
|
||||
DefaultTimestamp() bool
|
||||
ZipCompression() bool
|
||||
ArchiveCompression() bool
|
||||
ClientCut() bool
|
||||
IndexPageEnabled() bool
|
||||
IndexPageTemplate() string
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ func (c *configMock) DefaultTimestamp() bool {
|
|||
return false
|
||||
}
|
||||
|
||||
func (c *configMock) ZipCompression() bool {
|
||||
func (c *configMock) ArchiveCompression() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -197,7 +197,7 @@ func (h *Handler) uploadObject(c *fasthttp.RequestCtx, log *zap.Logger, bktInfo
|
|||
}
|
||||
|
||||
// explodeGzip read files from tar.gz archive and creates objects for each of them.
|
||||
// Sets FilePath attribute with name from tar.Header
|
||||
// Sets FilePath attribute with Name from tar.Header.
|
||||
func (h *Handler) explodeGzip(c *fasthttp.RequestCtx, log *zap.Logger, bktInfo *data.BucketInfo, file io.Reader) {
|
||||
gzipReader, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
|
|
|
|||
Loading…
Reference in a new issue