All checks were successful
Vulncheck / Vulncheck (push) Successful in 1m10s
Pre-commit hooks / Pre-commit (push) Successful in 1m34s
Build / Build Components (push) Successful in 1m50s
Tests and linters / gopls check (push) Successful in 3m49s
Tests and linters / Run gofumpt (push) Successful in 3m56s
Tests and linters / Staticcheck (push) Successful in 4m33s
Tests and linters / Lint (push) Successful in 4m37s
OCI image / Build container images (push) Successful in 4m45s
Tests and linters / Tests (push) Successful in 5m1s
Tests and linters / Tests with -race (push) Successful in 6m6s
To group all `compression_*` parameters together. Change-Id: I11ad9600f731903753fef1adfbc0328ef75bbf87 Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
154 lines
3.8 KiB
Go
154 lines
3.8 KiB
Go
package compression
|
|
|
|
import (
|
|
"bytes"
|
|
"strings"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert"
|
|
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
"github.com/klauspost/compress"
|
|
"github.com/klauspost/compress/zstd"
|
|
)
|
|
|
|
type Level string
|
|
|
|
const (
|
|
LevelDefault Level = ""
|
|
LevelOptimal Level = "optimal"
|
|
LevelFastest Level = "fastest"
|
|
LevelSmallestSize Level = "smallest_size"
|
|
)
|
|
|
|
type Compressor struct {
|
|
Config
|
|
|
|
encoder *zstd.Encoder
|
|
decoder *zstd.Decoder
|
|
}
|
|
|
|
// Config represents common compression-related configuration.
|
|
type Config struct {
|
|
Enabled bool
|
|
UncompressableContentTypes []string
|
|
Level Level
|
|
|
|
EstimateCompressibility bool
|
|
EstimateCompressibilityThreshold float64
|
|
}
|
|
|
|
// zstdFrameMagic contains first 4 bytes of any compressed object
|
|
// https://github.com/klauspost/compress/blob/master/zstd/framedec.go#L58 .
|
|
var zstdFrameMagic = []byte{0x28, 0xb5, 0x2f, 0xfd}
|
|
|
|
// Init initializes compression routines.
|
|
func (c *Compressor) Init() error {
|
|
var err error
|
|
|
|
if c.Enabled {
|
|
c.encoder, err = zstd.NewWriter(nil, zstd.WithEncoderLevel(c.compressionLevel()))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
c.decoder, err = zstd.NewReader(nil)
|
|
return err
|
|
}
|
|
|
|
// NeedsCompression returns true if the object should be compressed.
|
|
// For an object to be compressed 2 conditions must hold:
|
|
// 1. Compression is enabled in settings.
|
|
// 2. Object MIME Content-Type is allowed for compression.
|
|
func (c *Config) NeedsCompression(obj *objectSDK.Object) bool {
|
|
if !c.Enabled || len(c.UncompressableContentTypes) == 0 {
|
|
return c.Enabled
|
|
}
|
|
|
|
for _, attr := range obj.Attributes() {
|
|
if attr.Key() == objectSDK.AttributeContentType {
|
|
for _, value := range c.UncompressableContentTypes {
|
|
match := false
|
|
switch {
|
|
case len(value) > 0 && value[len(value)-1] == '*':
|
|
match = strings.HasPrefix(attr.Value(), value[:len(value)-1])
|
|
case len(value) > 0 && value[0] == '*':
|
|
match = strings.HasSuffix(attr.Value(), value[1:])
|
|
default:
|
|
match = attr.Value() == value
|
|
}
|
|
if match {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return c.Enabled
|
|
}
|
|
|
|
// Decompress decompresses data if it starts with the magic
|
|
// and returns data untouched otherwise.
|
|
func (c *Compressor) Decompress(data []byte) ([]byte, error) {
|
|
if len(data) < 4 || !bytes.Equal(data[:4], zstdFrameMagic) {
|
|
return data, nil
|
|
}
|
|
return c.decoder.DecodeAll(data, nil)
|
|
}
|
|
|
|
// Compress compresses data if compression is enabled
|
|
// and returns data untouched otherwise.
|
|
func (c *Compressor) Compress(data []byte) []byte {
|
|
if c == nil || !c.Enabled {
|
|
return data
|
|
}
|
|
if c.EstimateCompressibility {
|
|
estimated := compress.Estimate(data)
|
|
if estimated >= c.EstimateCompressibilityThreshold {
|
|
return c.compress(data)
|
|
}
|
|
return data
|
|
}
|
|
return c.compress(data)
|
|
}
|
|
|
|
func (c *Compressor) compress(data []byte) []byte {
|
|
maxSize := c.encoder.MaxEncodedSize(len(data))
|
|
compressed := c.encoder.EncodeAll(data, make([]byte, 0, maxSize))
|
|
if len(data) < len(compressed) {
|
|
return data
|
|
}
|
|
return compressed
|
|
}
|
|
|
|
// Close closes encoder and decoder, returns any error occurred.
|
|
func (c *Compressor) Close() error {
|
|
var err error
|
|
if c.encoder != nil {
|
|
err = c.encoder.Close()
|
|
}
|
|
if c.decoder != nil {
|
|
c.decoder.Close()
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (c *Config) HasValidCompressionLevel() bool {
|
|
return c.Level == LevelDefault ||
|
|
c.Level == LevelOptimal ||
|
|
c.Level == LevelFastest ||
|
|
c.Level == LevelSmallestSize
|
|
}
|
|
|
|
func (c *Compressor) compressionLevel() zstd.EncoderLevel {
|
|
switch c.Level {
|
|
case LevelDefault, LevelOptimal:
|
|
return zstd.SpeedDefault
|
|
case LevelFastest:
|
|
return zstd.SpeedFastest
|
|
case LevelSmallestSize:
|
|
return zstd.SpeedBestCompression
|
|
default:
|
|
assert.Fail("unknown compression level", string(c.Level))
|
|
return zstd.SpeedDefault
|
|
}
|
|
}
|