diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go
index fe75a6f6f3..c54b331f33 100644
--- a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go
+++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go
@@ -13,7 +13,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
- "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
@@ -135,11 +134,7 @@ func getMetabaseOpts(sh *shardconfig.Config) []meta.Option {
func getBlobstorOpts(ctx context.Context, sh *shardconfig.Config) []blobstor.Option {
result := []blobstor.Option{
- blobstor.WithCompressObjects(sh.Compress()),
- blobstor.WithCompressionLevel(compression.Level(sh.CompressionLevel())),
- blobstor.WithUncompressableContentTypes(sh.UncompressableContentTypes()),
- blobstor.WithCompressibilityEstimate(sh.EstimateCompressibility()),
- blobstor.WithCompressibilityEstimateThreshold(sh.EstimateCompressibilityThreshold()),
+ blobstor.WithCompression(sh.Compression()),
blobstor.WithStorages(getSubStorages(ctx, sh)),
blobstor.WithLogger(logger.NewLoggerWrapper(zap.NewNop())),
}
diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go
index f32953c584..f80401b5ba 100644
--- a/cmd/frostfs-node/config.go
+++ b/cmd/frostfs-node/config.go
@@ -129,13 +129,9 @@ type applicationConfiguration struct {
}
type shardCfg struct {
- compress bool
- compressionLevel compression.Level
- estimateCompressibility bool
- estimateCompressibilityThreshold float64
+ compression compression.Config
smallSizeObjectLimit uint64
- uncompressableContentType []string
refillMetabase bool
refillMetabaseWorkersCount int
mode shardmode.Mode
@@ -274,11 +270,7 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, source *s
target.refillMetabase = source.RefillMetabase()
target.refillMetabaseWorkersCount = source.RefillMetabaseWorkersCount()
target.mode = source.Mode()
- target.compress = source.Compress()
- target.compressionLevel = compression.Level(source.CompressionLevel())
- target.estimateCompressibility = source.EstimateCompressibility()
- target.estimateCompressibilityThreshold = source.EstimateCompressibilityThreshold()
- target.uncompressableContentType = source.UncompressableContentTypes()
+ target.compression = source.Compression()
target.smallSizeObjectLimit = source.SmallSizeLimit()
a.setShardWriteCacheConfig(&target, source)
@@ -1029,11 +1021,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID
ss := c.getSubstorageOpts(ctx, shCfg)
blobstoreOpts := []blobstor.Option{
- blobstor.WithCompressObjects(shCfg.compress),
- blobstor.WithCompressionLevel(shCfg.compressionLevel),
- blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType),
- blobstor.WithCompressibilityEstimate(shCfg.estimateCompressibility),
- blobstor.WithCompressibilityEstimateThreshold(shCfg.estimateCompressibilityThreshold),
+ blobstor.WithCompression(shCfg.compression),
blobstor.WithStorages(ss),
blobstor.WithLogger(c.log),
}
diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go
index afadb4c995..401c54edcd 100644
--- a/cmd/frostfs-node/config/engine/config_test.go
+++ b/cmd/frostfs-node/config/engine/config_test.go
@@ -15,6 +15,7 @@ import (
writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache"
configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos"
+ "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"github.com/stretchr/testify/require"
)
@@ -100,11 +101,11 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, 100, meta.BoltDB().MaxBatchSize())
require.Equal(t, 10*time.Millisecond, meta.BoltDB().MaxBatchDelay())
- require.Equal(t, true, sc.Compress())
- require.Equal(t, "fastest", sc.CompressionLevel())
- require.Equal(t, []string{"audio/*", "video/*"}, sc.UncompressableContentTypes())
- require.Equal(t, true, sc.EstimateCompressibility())
- require.Equal(t, float64(0.7), sc.EstimateCompressibilityThreshold())
+ require.Equal(t, true, sc.Compression().Enabled)
+ require.Equal(t, compression.LevelFastest, sc.Compression().Level)
+ require.Equal(t, []string{"audio/*", "video/*"}, sc.Compression().UncompressableContentTypes)
+ require.Equal(t, true, sc.Compression().EstimateCompressibility)
+ require.Equal(t, float64(0.7), sc.Compression().EstimateCompressibilityThreshold)
require.EqualValues(t, 102400, sc.SmallSizeLimit())
require.Equal(t, 2, len(ss))
@@ -237,9 +238,9 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, 200, meta.BoltDB().MaxBatchSize())
require.Equal(t, 20*time.Millisecond, meta.BoltDB().MaxBatchDelay())
- require.Equal(t, false, sc.Compress())
- require.Equal(t, "", sc.CompressionLevel())
- require.Equal(t, []string(nil), sc.UncompressableContentTypes())
+ require.Equal(t, false, sc.Compression().Enabled)
+ require.Equal(t, compression.LevelDefault, sc.Compression().Level)
+ require.Equal(t, []string(nil), sc.Compression().UncompressableContentTypes)
require.EqualValues(t, 102400, sc.SmallSizeLimit())
require.Equal(t, 2, len(ss))
diff --git a/cmd/frostfs-node/config/engine/shard/config.go b/cmd/frostfs-node/config/engine/shard/config.go
index 14e91f01ff..d42646da7a 100644
--- a/cmd/frostfs-node/config/engine/shard/config.go
+++ b/cmd/frostfs-node/config/engine/shard/config.go
@@ -8,6 +8,7 @@ import (
metabaseconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/metabase"
piloramaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/pilorama"
writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache"
+ "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
)
@@ -27,52 +28,27 @@ func From(c *config.Config) *Config {
return (*Config)(c)
}
-// Compress returns the value of "compress" config parameter.
-//
-// Returns false if the value is not a valid bool.
-func (x *Config) Compress() bool {
- return config.BoolSafe(
- (*config.Config)(x),
- "compress",
- )
-}
-
-// CompressionLevel returns the value of "compression_level" config parameter.
-//
-// Returns empty string if the value is not a valid string.
-func (x *Config) CompressionLevel() string {
- return config.StringSafe(
- (*config.Config)(x),
- "compression_level",
- )
-}
-
-// UncompressableContentTypes returns the value of "compress_skip_content_types" config parameter.
-//
-// Returns nil if a the value is missing or is invalid.
-func (x *Config) UncompressableContentTypes() []string {
- return config.StringSliceSafe(
- (*config.Config)(x),
- "compression_exclude_content_types")
-}
-
-// EstimateCompressibility returns the value of "estimate_compressibility" config parameter.
-//
-// Returns false if the value is not a valid bool.
-func (x *Config) EstimateCompressibility() bool {
- return config.BoolSafe(
- (*config.Config)(x),
- "compression_estimate_compressibility",
- )
+func (x *Config) Compression() compression.Config {
+ cc := (*config.Config)(x).Sub("compression")
+ if cc == nil {
+ return compression.Config{}
+ }
+ return compression.Config{
+ Enabled: config.BoolSafe(cc, "enabled"),
+ UncompressableContentTypes: config.StringSliceSafe(cc, "exclude_content_types"),
+ Level: compression.Level(config.StringSafe(cc, "level")),
+ EstimateCompressibility: config.BoolSafe(cc, "estimate_compressibility"),
+ EstimateCompressibilityThreshold: estimateCompressibilityThreshold(cc),
+ }
}
// EstimateCompressibilityThreshold returns the value of "estimate_compressibility_threshold" config parameter.
//
// Returns EstimateCompressibilityThresholdDefault if the value is not defined, not valid float or not in range [0.0; 1.0].
-func (x *Config) EstimateCompressibilityThreshold() float64 {
+func estimateCompressibilityThreshold(c *config.Config) float64 {
v := config.FloatOrDefault(
- (*config.Config)(x),
- "compression_estimate_compressibility_threshold",
+ c,
+ "estimate_compressibility_threshold",
EstimateCompressibilityThresholdDefault)
if v < 0.0 || v > 1.0 {
return EstimateCompressibilityThresholdDefault
diff --git a/config/example/node.env b/config/example/node.env
index fb4fc69dd7..9d054fe78f 100644
--- a/config/example/node.env
+++ b/config/example/node.env
@@ -122,7 +122,7 @@ FROSTFS_STORAGE_SHARD_0_METABASE_PERM=0644
FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_SIZE=100
FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_DELAY=10ms
### Blobstor config
-FROSTFS_STORAGE_SHARD_0_COMPRESS=true
+FROSTFS_STORAGE_SHARD_0_COMPRESSION_ENABLED=true
FROSTFS_STORAGE_SHARD_0_COMPRESSION_LEVEL=fastest
FROSTFS_STORAGE_SHARD_0_COMPRESSION_EXCLUDE_CONTENT_TYPES="audio/* video/*"
FROSTFS_STORAGE_SHARD_0_COMPRESSION_ESTIMATE_COMPRESSIBILITY=true
diff --git a/config/example/node.json b/config/example/node.json
index 1e9dd7835b..110e99ee88 100644
--- a/config/example/node.json
+++ b/config/example/node.json
@@ -183,13 +183,15 @@
"max_batch_size": 100,
"max_batch_delay": "10ms"
},
- "compress": true,
- "compression_level": "fastest",
- "compression_exclude_content_types": [
- "audio/*", "video/*"
- ],
- "compression_estimate_compressibility": true,
- "compression_estimate_compressibility_threshold": 0.7,
+ "compression": {
+ "enabled": true,
+ "level": "fastest",
+ "exclude_content_types": [
+ "audio/*", "video/*"
+ ],
+ "estimate_compressibility": true,
+ "estimate_compressibility_threshold": 0.7
+ },
"small_object_size": 102400,
"blobstor": [
{
@@ -323,7 +325,9 @@
"max_batch_size": 200,
"max_batch_delay": "20ms"
},
- "compress": false,
+ "compression": {
+ "enabled": false
+ },
"small_object_size": 102400,
"blobstor": [
{
diff --git a/config/example/node.yaml b/config/example/node.yaml
index 26c6e248c8..de5eaa1331 100644
--- a/config/example/node.yaml
+++ b/config/example/node.yaml
@@ -160,7 +160,8 @@ storage:
max_batch_delay: 5ms # maximum delay for a batch of operations to be executed
max_batch_size: 100 # maximum amount of operations in a single batch
- compress: false # turn on/off zstd compression of stored objects
+ compression:
+ enabled: false # turn on/off zstd compression of stored objects
small_object_size: 100 kb # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes
blobstor:
@@ -202,13 +203,14 @@ storage:
max_batch_size: 100
max_batch_delay: 10ms
- compress: true # turn on/off zstd compression of stored objects
- compression_level: fastest
- compression_exclude_content_types:
- - audio/*
- - video/*
- compression_estimate_compressibility: true
- compression_estimate_compressibility_threshold: 0.7
+ compression:
+ enabled: true # turn on/off zstd compression of stored objects
+ level: fastest
+ exclude_content_types:
+ - audio/*
+ - video/*
+ estimate_compressibility: true
+ estimate_compressibility_threshold: 0.7
blobstor:
- type: blobovnicza
diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md
index 19f6ee66d0..1494d2fca2 100644
--- a/docs/storage-node-configuration.md
+++ b/docs/storage-node-configuration.md
@@ -185,22 +185,41 @@ Contains configuration for each shard. Keys must be consecutive numbers starting
`default` subsection has the same format and specifies defaults for missing values.
The following table describes configuration for each shard.
-| Parameter | Type | Default value | Description |
-| ------------------------------------------------ | ------------------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `compress` | `bool` | `false` | Flag to enable compression. |
-| `compression_level` | `string` | `optimal` | Compression level. Available values are `optimal`, `fastest`, `smallest_size`. |
-| `compression_exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). |
-| `compression_estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. |
-| `compression_estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. |
-| `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` |
-| `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. |
-| `resync_metabase_worker_count` | `int` | `1000` | Count of concurrent workers to resync metabase. |
-| `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. |
-| `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. |
-| `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. |
-| `small_object_size` | `size` | `1M` | Maximum size of an object stored in blobovnicza tree. |
-| `gc` | [GC config](#gc-subsection) | | GC configuration. |
-| `limits` | [Shard limits config](#limits-subsection) | | Shard limits configuration. |
+| Parameter | Type | Default value | Description |
+| ------------------------------ | --------------------------------------------- | ------------- | --------------------------------------------------------------------------------------------------------- |
+| `compression` | [Compression config](#compression-subsection) | | Compression config. |
+| `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` |
+| `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. |
+| `resync_metabase_worker_count` | `int` | `1000` | Count of concurrent workers to resync metabase. |
+| `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. |
+| `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. |
+| `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. |
+| `small_object_size` | `size` | `1M` | Maximum size of an object stored in blobovnicza tree. |
+| `gc` | [GC config](#gc-subsection) | | GC configuration. |
+| `limits` | [Shard limits config](#limits-subsection) | | Shard limits configuration. |
+
+### `compression` subsection
+
+Contains compression config.
+
+```yaml
+compression:
+ enabled: true
+ level: smallest_size
+ exclude_content_types:
+ - audio/*
+ - video/*
+ estimate_compressibility: true
+ estimate_compressibility_threshold: 0.7
+```
+
+| Parameter | Type | Default value | Description |
+| ------------------------------------ | ---------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `enabled` | `bool` | `false` | Flag to enable compression. |
+| `level` | `string` | `optimal` | Compression level. Available values are `optimal`, `fastest`, `smallest_size`. |
+| `exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). |
+| `estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. |
+| `estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. |
### `blobstor` subsection
diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go
index cf67c6fe94..edb2c19461 100644
--- a/pkg/local_object_storage/blobstor/blobstor.go
+++ b/pkg/local_object_storage/blobstor/blobstor.go
@@ -95,52 +95,9 @@ func WithLogger(l *logger.Logger) Option {
}
}
-// WithCompressObjects returns option to toggle
-// compression of the stored objects.
-//
-// If true, Zstandard algorithm is used for data compression.
-//
-// If compressor (decompressor) creation failed,
-// the uncompressed option will be used, and the error
-// is recorded in the provided log.
-func WithCompressObjects(comp bool) Option {
+func WithCompression(comp compression.Config) Option {
return func(c *cfg) {
- c.compression.Enabled = comp
- }
-}
-
-func WithCompressionLevel(level compression.Level) Option {
- return func(c *cfg) {
- c.compression.Level = level
- }
-}
-
-// WithCompressibilityEstimate returns an option to use
-// normilized compressibility estimate to decide compress
-// data or not.
-//
-// See https://github.com/klauspost/compress/blob/v1.17.2/compressible.go#L5
-func WithCompressibilityEstimate(v bool) Option {
- return func(c *cfg) {
- c.compression.UseCompressEstimation = v
- }
-}
-
-// WithCompressibilityEstimateThreshold returns an option to set
-// normilized compressibility estimate threshold.
-//
-// See https://github.com/klauspost/compress/blob/v1.17.2/compressible.go#L5
-func WithCompressibilityEstimateThreshold(threshold float64) Option {
- return func(c *cfg) {
- c.compression.CompressEstimationThreshold = threshold
- }
-}
-
-// WithUncompressableContentTypes returns option to disable decompression
-// for specific content types as seen by object.AttributeContentType attribute.
-func WithUncompressableContentTypes(values []string) Option {
- return func(c *cfg) {
- c.compression.UncompressableContentTypes = values
+ c.compression.Config = comp
}
}
diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go
index 6cc56fa3b2..6ddeb6f008 100644
--- a/pkg/local_object_storage/blobstor/blobstor_test.go
+++ b/pkg/local_object_storage/blobstor/blobstor_test.go
@@ -9,6 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+ "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
@@ -51,7 +52,9 @@ func TestCompression(t *testing.T) {
newBlobStor := func(t *testing.T, compress bool) *BlobStor {
bs := New(
- WithCompressObjects(compress),
+ WithCompression(compression.Config{
+ Enabled: compress,
+ }),
WithStorages(defaultStorages(dir, smallSizeLimit)))
require.NoError(t, bs.Open(context.Background(), mode.ReadWrite))
require.NoError(t, bs.Init(context.Background()))
@@ -113,8 +116,10 @@ func TestBlobstor_needsCompression(t *testing.T) {
dir := t.TempDir()
bs := New(
- WithCompressObjects(compress),
- WithUncompressableContentTypes(ct),
+ WithCompression(compression.Config{
+ Enabled: compress,
+ UncompressableContentTypes: ct,
+ }),
WithStorages([]SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(
diff --git a/pkg/local_object_storage/blobstor/compression/compress.go b/pkg/local_object_storage/blobstor/compression/compress.go
index efe84ea2a3..c76cec9a10 100644
--- a/pkg/local_object_storage/blobstor/compression/compress.go
+++ b/pkg/local_object_storage/blobstor/compression/compress.go
@@ -32,8 +32,8 @@ type Config struct {
UncompressableContentTypes []string
Level Level
- UseCompressEstimation bool
- CompressEstimationThreshold float64
+ EstimateCompressibility bool
+ EstimateCompressibilityThreshold float64
}
// zstdFrameMagic contains first 4 bytes of any compressed object
@@ -101,9 +101,9 @@ func (c *Compressor) Compress(data []byte) []byte {
if c == nil || !c.Enabled {
return data
}
- if c.UseCompressEstimation {
+ if c.EstimateCompressibility {
estimated := compress.Estimate(data)
- if estimated >= c.CompressEstimationThreshold {
+ if estimated >= c.EstimateCompressibilityThreshold {
return c.compress(data)
}
return data
diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go
index ccfa510fe4..2786321a82 100644
--- a/pkg/local_object_storage/blobstor/iterate_test.go
+++ b/pkg/local_object_storage/blobstor/iterate_test.go
@@ -8,6 +8,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+ "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/memstore"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
@@ -24,7 +25,9 @@ func TestIterateObjects(t *testing.T) {
// create BlobStor instance
blobStor := New(
WithStorages(defaultStorages(p, smalSz)),
- WithCompressObjects(true),
+ WithCompression(compression.Config{
+ Enabled: true,
+ }),
)
defer os.RemoveAll(p)