forked from TrueCloudLab/frostfs-node
[#754] blobstor: Estimate compressability
Now it is possible to enable compressability estimation. If data is likely uncompressable, it should reduce CPU time and memory. Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
parent
05b508f79a
commit
c80b46fad3
11 changed files with 153 additions and 12 deletions
|
@ -5,6 +5,7 @@ import (
|
|||
"strings"
|
||||
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"github.com/klauspost/compress"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
|
@ -13,6 +14,9 @@ type Config struct {
|
|||
Enabled bool
|
||||
UncompressableContentTypes []string
|
||||
|
||||
UseCompressEstimation bool
|
||||
CompressEstimationThreshold float64
|
||||
|
||||
encoder *zstd.Encoder
|
||||
decoder *zstd.Decoder
|
||||
}
|
||||
|
@ -82,6 +86,17 @@ func (c *Config) Compress(data []byte) []byte {
|
|||
if c == nil || !c.Enabled {
|
||||
return data
|
||||
}
|
||||
if c.UseCompressEstimation {
|
||||
estimated := compress.Estimate(data)
|
||||
if estimated >= c.CompressEstimationThreshold {
|
||||
return c.compress(data)
|
||||
}
|
||||
return data
|
||||
}
|
||||
return c.compress(data)
|
||||
}
|
||||
|
||||
func (c *Config) compress(data []byte) []byte {
|
||||
maxSize := c.encoder.MaxEncodedSize(len(data))
|
||||
compressed := c.encoder.EncodeAll(data, make([]byte, 0, maxSize))
|
||||
if len(data) < len(compressed) {
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue