george.bartolomey/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Pull requests Activity

[#1715] config: Add compression config section

Browse source 
To group all `compression_*` parameters together.

Change-Id: I11ad9600f731903753fef1adfbc0328ef75bbf87
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2025-04-14 17:36:51 +03:00
parent 8c746a914a
commit 0ee7467da5
12 changed files with 105 additions and 155 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go
View file

@ -13,7 +13,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine"
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
@ -135,11 +134,7 @@ func getMetabaseOpts(sh *shardconfig.Config) []meta.Option {
func getBlobstorOpts(ctx context.Context, sh *shardconfig.Config) []blobstor.Option {
result := []blobstor.Option{
blobstor.WithCompressObjects(sh.Compress()),
blobstor.WithCompressionLevel(compression.Level(sh.CompressionLevel())),
blobstor.WithUncompressableContentTypes(sh.UncompressableContentTypes()),
blobstor.WithCompressibilityEstimate(sh.EstimateCompressibility()),
blobstor.WithCompressibilityEstimateThreshold(sh.EstimateCompressibilityThreshold()),
blobstor.WithCompression(sh.Compression()),
blobstor.WithStorages(getSubStorages(ctx, sh)),
blobstor.WithLogger(logger.NewLoggerWrapper(zap.NewNop())),
}

18
cmd/frostfs-node/config.go
View file

@ -129,13 +129,9 @@ type applicationConfiguration struct {
}
type shardCfg struct {
compress bool
compressionLevel compression.Level
estimateCompressibility bool
estimateCompressibilityThreshold float64
compression compression.Config
smallSizeObjectLimit uint64
uncompressableContentType []string
refillMetabase bool
refillMetabaseWorkersCount int
mode shardmode.Mode
@ -274,11 +270,7 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, source *s
target.refillMetabase = source.RefillMetabase()
target.refillMetabaseWorkersCount = source.RefillMetabaseWorkersCount()
target.mode = source.Mode()
target.compress = source.Compress()
target.compressionLevel = compression.Level(source.CompressionLevel())
target.estimateCompressibility = source.EstimateCompressibility()
target.estimateCompressibilityThreshold = source.EstimateCompressibilityThreshold()
target.uncompressableContentType = source.UncompressableContentTypes()
target.compression = source.Compression()
target.smallSizeObjectLimit = source.SmallSizeLimit()
a.setShardWriteCacheConfig(&target, source)
@ -1029,11 +1021,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID
ss := c.getSubstorageOpts(ctx, shCfg)
blobstoreOpts := []blobstor.Option{
blobstor.WithCompressObjects(shCfg.compress),
blobstor.WithCompressionLevel(shCfg.compressionLevel),
blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType),
blobstor.WithCompressibilityEstimate(shCfg.estimateCompressibility),
blobstor.WithCompressibilityEstimateThreshold(shCfg.estimateCompressibilityThreshold),
blobstor.WithCompression(shCfg.compression),
blobstor.WithStorages(ss),
blobstor.WithLogger(c.log),
}

17
cmd/frostfs-node/config/engine/config_test.go
View file

@ -15,6 +15,7 @@ import (
writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache"
configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"github.com/stretchr/testify/require"
)
@ -100,11 +101,11 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, 100, meta.BoltDB().MaxBatchSize())
require.Equal(t, 10*time.Millisecond, meta.BoltDB().MaxBatchDelay())
require.Equal(t, true, sc.Compress())
require.Equal(t, "fastest", sc.CompressionLevel())
require.Equal(t, []string{"audio/*", "video/*"}, sc.UncompressableContentTypes())
require.Equal(t, true, sc.EstimateCompressibility())
require.Equal(t, float64(0.7), sc.EstimateCompressibilityThreshold())
require.Equal(t, true, sc.Compression().Enabled)
require.Equal(t, compression.LevelFastest, sc.Compression().Level)
require.Equal(t, []string{"audio/*", "video/*"}, sc.Compression().UncompressableContentTypes)
require.Equal(t, true, sc.Compression().EstimateCompressibility)
require.Equal(t, float64(0.7), sc.Compression().EstimateCompressibilityThreshold)
require.EqualValues(t, 102400, sc.SmallSizeLimit())
require.Equal(t, 2, len(ss))
@ -237,9 +238,9 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, 200, meta.BoltDB().MaxBatchSize())
require.Equal(t, 20*time.Millisecond, meta.BoltDB().MaxBatchDelay())
require.Equal(t, false, sc.Compress())
require.Equal(t, "", sc.CompressionLevel())
require.Equal(t, []string(nil), sc.UncompressableContentTypes())
require.Equal(t, false, sc.Compression().Enabled)
require.Equal(t, compression.LevelDefault, sc.Compression().Level)
require.Equal(t, []string(nil), sc.Compression().UncompressableContentTypes)
require.EqualValues(t, 102400, sc.SmallSizeLimit())
require.Equal(t, 2, len(ss))

56
cmd/frostfs-node/config/engine/shard/config.go
View file

@ -8,6 +8,7 @@ import (
metabaseconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/metabase"
piloramaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/pilorama"
writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
)
@ -27,52 +28,27 @@ func From(c *config.Config) *Config {
return (*Config)(c)
}
// Compress returns the value of "compress" config parameter.
//
// Returns false if the value is not a valid bool.
func (x *Config) Compress() bool {
return config.BoolSafe(
(*config.Config)(x),
"compress",
)
}
// CompressionLevel returns the value of "compression_level" config parameter.
//
// Returns empty string if the value is not a valid string.
func (x *Config) CompressionLevel() string {
return config.StringSafe(
(*config.Config)(x),
"compression_level",
)
}
// UncompressableContentTypes returns the value of "compress_skip_content_types" config parameter.
//
// Returns nil if a the value is missing or is invalid.
func (x *Config) UncompressableContentTypes() []string {
return config.StringSliceSafe(
(*config.Config)(x),
"compression_exclude_content_types")
}
// EstimateCompressibility returns the value of "estimate_compressibility" config parameter.
//
// Returns false if the value is not a valid bool.
func (x *Config) EstimateCompressibility() bool {
return config.BoolSafe(
(*config.Config)(x),
"compression_estimate_compressibility",
)
func (x *Config) Compression() compression.Config {
cc := (*config.Config)(x).Sub("compression")
if cc == nil {
return compression.Config{}
}
return compression.Config{
Enabled: config.BoolSafe(cc, "enabled"),
UncompressableContentTypes: config.StringSliceSafe(cc, "exclude_content_types"),
Level: compression.Level(config.StringSafe(cc, "level")),
EstimateCompressibility: config.BoolSafe(cc, "estimate_compressibility"),
EstimateCompressibilityThreshold: estimateCompressibilityThreshold(cc),
}
}
// EstimateCompressibilityThreshold returns the value of "estimate_compressibility_threshold" config parameter.
//
// Returns EstimateCompressibilityThresholdDefault if the value is not defined, not valid float or not in range [0.0; 1.0].
func (x *Config) EstimateCompressibilityThreshold() float64 {
func estimateCompressibilityThreshold(c *config.Config) float64 {
v := config.FloatOrDefault(
(*config.Config)(x),
"compression_estimate_compressibility_threshold",
c,
"estimate_compressibility_threshold",
EstimateCompressibilityThresholdDefault)
if v < 0.0 || v > 1.0 {
return EstimateCompressibilityThresholdDefault

2
config/example/node.env
View file

@ -122,7 +122,7 @@ FROSTFS_STORAGE_SHARD_0_METABASE_PERM=0644
FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_SIZE=100
FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_DELAY=10ms
### Blobstor config
FROSTFS_STORAGE_SHARD_0_COMPRESS=true
FROSTFS_STORAGE_SHARD_0_COMPRESSION_ENABLED=true
FROSTFS_STORAGE_SHARD_0_COMPRESSION_LEVEL=fastest
FROSTFS_STORAGE_SHARD_0_COMPRESSION_EXCLUDE_CONTENT_TYPES="audio/* video/*"
FROSTFS_STORAGE_SHARD_0_COMPRESSION_ESTIMATE_COMPRESSIBILITY=true

16
config/example/node.json
View file

@ -183,13 +183,15 @@
"max_batch_size": 100,
"max_batch_delay": "10ms"
},
"compress": true,
"compression_level": "fastest",
"compression_exclude_content_types": [
"compression": {
"enabled": true,
"level": "fastest",
"exclude_content_types": [
"audio/*", "video/*"
],
"compression_estimate_compressibility": true,
"compression_estimate_compressibility_threshold": 0.7,
"estimate_compressibility": true,
"estimate_compressibility_threshold": 0.7
},
"small_object_size": 102400,
"blobstor": [
{
@ -323,7 +325,9 @@
"max_batch_size": 200,
"max_batch_delay": "20ms"
},
"compress": false,
"compression": {
"enabled": false
},
"small_object_size": 102400,
"blobstor": [
{

14
config/example/node.yaml
View file

@ -160,7 +160,8 @@ storage:
max_batch_delay: 5ms # maximum delay for a batch of operations to be executed
max_batch_size: 100 # maximum amount of operations in a single batch
compress: false # turn on/off zstd compression of stored objects
compression:
enabled: false # turn on/off zstd compression of stored objects
small_object_size: 100 kb # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes
blobstor:
@ -202,13 +203,14 @@ storage:
max_batch_size: 100
max_batch_delay: 10ms
compress: true # turn on/off zstd compression of stored objects
compression_level: fastest
compression_exclude_content_types:
compression:
enabled: true # turn on/off zstd compression of stored objects
level: fastest
exclude_content_types:
- audio/*
- video/*
compression_estimate_compressibility: true
compression_estimate_compressibility_threshold: 0.7
estimate_compressibility: true
estimate_compressibility_threshold: 0.7
blobstor:
- type: blobovnicza

31
docs/storage-node-configuration.md
View file

@ -186,12 +186,8 @@ Contains configuration for each shard. Keys must be consecutive numbers starting
The following table describes configuration for each shard.
| Parameter | Type | Default value | Description |
| ------------------------------------------------ | ------------------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `compress` | `bool` | `false` | Flag to enable compression. |
| `compression_level` | `string` | `optimal` | Compression level. Available values are `optimal`, `fastest`, `smallest_size`. |
| `compression_exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). |
| `compression_estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. |
| `compression_estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. |
| ------------------------------ | --------------------------------------------- | ------------- | --------------------------------------------------------------------------------------------------------- |
| `compression` | [Compression config](#compression-subsection) | | Compression config. |
| `mode` | `string` | `read-write` | Shard Mode.<br/>Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` |
| `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. |
| `resync_metabase_worker_count` | `int` | `1000` | Count of concurrent workers to resync metabase. |
@ -202,6 +198,29 @@ The following table describes configuration for each shard.
| `gc` | [GC config](#gc-subsection) | | GC configuration. |
| `limits` | [Shard limits config](#limits-subsection) | | Shard limits configuration. |
### `compression` subsection
Contains compression config.
```yaml
compression:
enabled: true
level: smallest_size
exclude_content_types:
- audio/*
- video/*
estimate_compressibility: true
estimate_compressibility_threshold: 0.7
```
| Parameter | Type | Default value | Description |
| ------------------------------------ | ---------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `enabled` | `bool` | `false` | Flag to enable compression. |
| `level` | `string` | `optimal` | Compression level. Available values are `optimal`, `fastest`, `smallest_size`. |
| `exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). |
| `estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. |
| `estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. |
### `blobstor` subsection
Contains a list of substorages each with it's own type.

47
pkg/local_object_storage/blobstor/blobstor.go
View file

@ -95,52 +95,9 @@ func WithLogger(l *logger.Logger) Option {
}
}
// WithCompressObjects returns option to toggle
// compression of the stored objects.
//
// If true, Zstandard algorithm is used for data compression.
//
// If compressor (decompressor) creation failed,
// the uncompressed option will be used, and the error
// is recorded in the provided log.
func WithCompressObjects(comp bool) Option {
func WithCompression(comp compression.Config) Option {
return func(c *cfg) {
c.compression.Enabled = comp
}
}
func WithCompressionLevel(level compression.Level) Option {
return func(c *cfg) {
c.compression.Level = level
}
}
// WithCompressibilityEstimate returns an option to use
// normilized compressibility estimate to decide compress
// data or not.
//
// See https://github.com/klauspost/compress/blob/v1.17.2/compressible.go#L5
func WithCompressibilityEstimate(v bool) Option {
return func(c *cfg) {
c.compression.UseCompressEstimation = v
}
}
// WithCompressibilityEstimateThreshold returns an option to set
// normilized compressibility estimate threshold.
//
// See https://github.com/klauspost/compress/blob/v1.17.2/compressible.go#L5
func WithCompressibilityEstimateThreshold(threshold float64) Option {
return func(c *cfg) {
c.compression.CompressEstimationThreshold = threshold
}
}
// WithUncompressableContentTypes returns option to disable decompression
// for specific content types as seen by object.AttributeContentType attribute.
func WithUncompressableContentTypes(values []string) Option {
return func(c *cfg) {
c.compression.UncompressableContentTypes = values
c.compression.Config = comp
}
}

11
pkg/local_object_storage/blobstor/blobstor_test.go
View file

@ -9,6 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
@ -51,7 +52,9 @@ func TestCompression(t *testing.T) {
newBlobStor := func(t *testing.T, compress bool) *BlobStor {
bs := New(
WithCompressObjects(compress),
WithCompression(compression.Config{
Enabled: compress,
}),
WithStorages(defaultStorages(dir, smallSizeLimit)))
require.NoError(t, bs.Open(context.Background(), mode.ReadWrite))
require.NoError(t, bs.Init(context.Background()))
@ -113,8 +116,10 @@ func TestBlobstor_needsCompression(t *testing.T) {
dir := t.TempDir()
bs := New(
WithCompressObjects(compress),
WithUncompressableContentTypes(ct),
WithCompression(compression.Config{
Enabled: compress,
UncompressableContentTypes: ct,
}),
WithStorages([]SubStorage{
{
Storage: blobovniczatree.NewBlobovniczaTree(

8
pkg/local_object_storage/blobstor/compression/compress.go
View file

@ -32,8 +32,8 @@ type Config struct {
UncompressableContentTypes []string
Level Level
UseCompressEstimation bool
CompressEstimationThreshold float64
EstimateCompressibility bool
EstimateCompressibilityThreshold float64
}
// zstdFrameMagic contains first 4 bytes of any compressed object
@ -101,9 +101,9 @@ func (c *Compressor) Compress(data []byte) []byte {
if c == nil || !c.Enabled {
return data
}
if c.UseCompressEstimation {
if c.EstimateCompressibility {
estimated := compress.Estimate(data)
if estimated >= c.CompressEstimationThreshold {
if estimated >= c.EstimateCompressibilityThreshold {
return c.compress(data)
}
return data

5
pkg/local_object_storage/blobstor/iterate_test.go
View file

@ -8,6 +8,7 @@ import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/memstore"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
@ -24,7 +25,9 @@ func TestIterateObjects(t *testing.T) {
// create BlobStor instance
blobStor := New(
WithStorages(defaultStorages(p, smalSz)),
WithCompressObjects(true),
WithCompression(compression.Config{
Enabled: true,
}),
)
defer os.RemoveAll(p)