diff --git a/backend/press/alg_xz.go b/backend/press/alg_xz.go new file mode 100644 index 000000000..415bbb117 --- /dev/null +++ b/backend/press/alg_xz.go @@ -0,0 +1,75 @@ +package press + +import ( + "bufio" + "io" + + "github.com/ulikunitz/xz" +) + +// AlgXZ represents the XZ compression algorithm +type AlgXZ struct { + blockSize uint32 + config xz.WriterConfig +} + +// InitializeXZ creates an Lz4 compression algorithm +func InitializeXZ(bs uint32) Algorithm { + a := new(AlgXZ) + a.blockSize = bs + a.config = xz.WriterConfig{} + return a +} + +// GetFileExtension returns file extension +func (a *AlgXZ) GetFileExtension() string { + return ".xz" +} + +// GetHeader returns the Lz4 compression header +func (a *AlgXZ) GetHeader() []byte { + return []byte{} +} + +// GetFooter returns +func (a *AlgXZ) GetFooter() []byte { + return []byte{} +} + +// CompressBlock that compresses a block using lz4 +func (a *AlgXZ) CompressBlock(in []byte, out io.Writer) (compressedSize uint32, uncompressedSize uint64, err error) { + // Initialize buffer + bufw := bufio.NewWriterSize(out, int(a.blockSize+(a.blockSize)>>4)) + + // Initialize block writer + outw, err := a.config.NewWriter(bufw) + if err != nil { + return 0, 0, err + } + + // Compress block + _, err = outw.Write(in) + if err != nil { + return 0, 0, err + } + + // Finalize gzip file, flush buffer and return + err = outw.Close() + if err != nil { + return 0, 0, err + } + blockSize := uint32(bufw.Buffered()) + err = bufw.Flush() + + return blockSize, uint64(len(in)), err +} + +// DecompressBlock decompresses Lz4 compressed block +func (a *AlgXZ) DecompressBlock(in io.Reader, out io.Writer, BlockSize uint32) (n int, err error) { + xzReader, err := xz.NewReader(in) + if err != nil { + return 0, err + } + written, err := io.Copy(out, xzReader) + return int(written), err +} diff --git a/backend/press/compression.go b/backend/press/compression.go index fa3bbc9d8..b60e1e4b1 100644 --- a/backend/press/compression.go +++ b/backend/press/compression.go @@ -62,6 +62,9 @@ func NewCompressionPreset(preset string) (*Compression, error) { case "gzip": alg := InitializeGzip(131072, 6) return NewCompression(Gzip, alg, 131070) // GZIP-default compression (medium)*/ + case "xz": + alg := InitializeXZ(1048576) + return NewCompression(XZ, alg, 1048576) // XZ compression (strong compression)*/ } return nil, errors.New("Compression mode doesn't exist") } @@ -75,6 +78,9 @@ func NewCompressionPresetNumber(preset int) (*Compression, error) { case Gzip: alg := InitializeGzip(131072, 6) return NewCompression(Gzip, alg, 131070) // GZIP-default compression (medium)*/ + case XZ: + alg := InitializeXZ(1048576) + return NewCompression(XZ, alg, 1048576) // XZ compression (strong compression)*/ } return nil, errors.New("Compression mode doesn't exist") } diff --git a/backend/press/compression_test.go b/backend/press/compression_test.go index d8a2390c1..ff625985c 100644 --- a/backend/press/compression_test.go +++ b/backend/press/compression_test.go @@ -122,7 +122,7 @@ func getCompressibleString(size int) string { } func TestCompression(t *testing.T) { - testCases := []string{"lz4", "gzip"} + testCases := []string{"lz4", "gzip", "xz"} for _, tc := range testCases { t.Run(tc, func(t *testing.T) { testSmallLarge(t, tc) diff --git a/backend/press/press.go b/backend/press/press.go index 9f3fd2825..abfc5c85e 100644 --- a/backend/press/press.go +++ b/backend/press/press.go @@ -40,6 +40,9 @@ func init() { }, { Value: "gzip", Help: "Standard gzip compression with fastest parameters.", + }, { + Value: "xz", + Help: "Standard xz compression with fastest parameters.", }, } diff --git a/backend/press/press_test.go b/backend/press/press_test.go index 0ebddb515..1f40d0498 100644 --- a/backend/press/press_test.go +++ b/backend/press/press_test.go @@ -96,3 +96,34 @@ func TestRemoteGzip(t *testing.T) { }, }) } + +// TestRemoteXz tests XZ compression +func TestRemoteXz(t *testing.T) { + if *fstest.RemoteName != "" { + t.Skip("Skipping as -remote set") + } + tempdir := filepath.Join(os.TempDir(), "rclone-press-test-xz") + name := "TestPressXz" + fstests.Run(t, &fstests.Opt{ + RemoteName: name + ":", + NilObject: (*Object)(nil), + UnimplementableFsMethods: []string{ + "OpenWriterAt", + "MergeDirs", + "DirCacheFlush", + "PutUnchecked", + "PutStream", + "UserInfo", + "Disconnect", + }, + UnimplementableObjectMethods: []string{ + "GetTier", + "SetTier", + }, + ExtraConfig: []fstests.ExtraConfigItem{ + {Name: name, Key: "type", Value: "press"}, + {Name: name, Key: "remote", Value: tempdir}, + {Name: name, Key: "compression_mode", Value: "xz"}, + }, + }) +}