[#754] blobstor: Estimate compressability
All checks were successful
DCO action / DCO (pull_request) Successful in 1m59s
Vulncheck / Vulncheck (pull_request) Successful in 3m31s
Build / Build Components (1.20) (pull_request) Successful in 4m37s
Build / Build Components (1.21) (pull_request) Successful in 4m33s
Tests and linters / Tests (1.20) (pull_request) Successful in 4m54s
Tests and linters / Staticcheck (pull_request) Successful in 4m49s
Tests and linters / Tests with -race (pull_request) Successful in 5m9s
Tests and linters / Lint (pull_request) Successful in 6m4s
Tests and linters / Tests (1.21) (pull_request) Successful in 6m9s

Now it is possible to enable compressability estimation.
If data is likely uncompressable, it should reduce CPU time and memory.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2023-10-31 14:45:22 +03:00
parent 05b508f79a
commit c80b46fad3
11 changed files with 153 additions and 12 deletions

View file

@ -5,6 +5,7 @@ import (
"strings"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
"github.com/klauspost/compress"
"github.com/klauspost/compress/zstd"
)
@ -13,6 +14,9 @@ type Config struct {
Enabled bool
UncompressableContentTypes []string
UseCompressEstimation bool
CompressEstimationThreshold float64
encoder *zstd.Encoder
decoder *zstd.Decoder
}
@ -82,6 +86,17 @@ func (c *Config) Compress(data []byte) []byte {
if c == nil || !c.Enabled {
return data
}
if c.UseCompressEstimation {
estimated := compress.Estimate(data)
if estimated >= c.CompressEstimationThreshold {
return c.compress(data)
}
return data
}
return c.compress(data)
}
func (c *Config) compress(data []byte) []byte {
maxSize := c.encoder.MaxEncodedSize(len(data))
compressed := c.encoder.EncodeAll(data, make([]byte, 0, maxSize))
if len(data) < len(compressed) {