From be2ce68b5cef040491b20ba0c7abe59d55ae5d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julio=20L=C3=B3pez?= <1953782+julio-lopez@users.noreply.github.com> Date: Thu, 16 Apr 2026 15:13:17 -0700 Subject: [PATCH] breaking(general): remove support for LZ4 (#5296) - It has been deprecated in Kopia since July 2022 (0985b80). - Dependency appears unmaintained. --- go.mod | 2 - go.sum | 14 ---- repo/compression/compression_ids.go | 2 +- repo/compression/compressor.go | 19 +++++ repo/compression/compressor_lz4.go | 80 +++---------------- repo/compression/compressor_test.go | 4 + repo/object/object_manager_test.go | 4 + .../docs/Advanced/Compression/_index.md | 41 +++++----- 8 files changed, 61 insertions(+), 105 deletions(-) diff --git a/go.mod b/go.mod index dce4ed109..2cad5d5f8 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,6 @@ require ( github.com/mxk/go-vss v1.2.1 github.com/natefinch/atomic v1.0.1 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 - github.com/pierrec/lz4 v2.6.1+incompatible github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.7.0 github.com/pkg/sftp v1.13.10 @@ -99,7 +98,6 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/frankban/quicktest v1.13.1 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect diff --git a/go.sum b/go.sum index d77a7baae..f7884be50 100644 --- a/go.sum +++ b/go.sum @@ -72,7 +72,6 @@ github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/T github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ= github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -102,8 +101,6 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/foomo/htpasswd v0.0.0-20200116085101-e3a90e78da9c h1:DBGU7zCwrrPPDsD6+gqKG8UfMxenWg9BOJE/Nmfph+4= github.com/foomo/htpasswd v0.0.0-20200116085101-e3a90e78da9c/go.mod h1:SHawtolbB0ZOFoRWgDwakX5WpwuIWAK88bUXVZqK0Ss= -github.com/frankban/quicktest v1.13.1 h1:xVm/f9seEhZFL9+n5kv5XLrGwy6elc4V9v/XFY2vmd8= -github.com/frankban/quicktest v1.13.1/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= @@ -137,7 +134,6 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/fswalker v0.3.3 h1:K2+d6cb3vNFjquVPRObIY+QaXJ6cbleVV6yZWLzkkQ8= github.com/google/fswalker v0.3.3/go.mod h1:9upMSscEE8oRi0WJ0rXZZYya1DmgUtJFhXAw7KNS3c4= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= @@ -181,12 +177,8 @@ github.com/kopia/htmluibuild v0.0.1-0.20260414002305-a859d43ee893 h1:J9g3qLDJAIe github.com/kopia/htmluibuild v0.0.1-0.20260414002305-a859d43ee893/go.mod h1:h53A5JM3t2qiwxqxusBe+PFgGcgZdS+DWCQvG5PTlto= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -219,8 +211,6 @@ github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+ github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= -github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -245,7 +235,6 @@ github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTU github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= @@ -349,7 +338,6 @@ golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.275.0 h1:vfY5d9vFVJeWEZT65QDd9hbndr7FyZ2+6mIzGAh71NI= @@ -365,10 +353,8 @@ google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216 h1:2TSTkQ8PMvGOD5eeqqRVv6Z9+BYI+bowK97RCr3W+9M= gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/repo/compression/compression_ids.go b/repo/compression/compression_ids.go index 0142f1577..5fd0dd2ef 100644 --- a/repo/compression/compression_ids.go +++ b/repo/compression/compression_ids.go @@ -23,7 +23,7 @@ headerPgzipBestSpeed HeaderID = 0x1301 headerPgzipBestCompression HeaderID = 0x1302 - headerLZ4Default HeaderID = 0x1400 + headerLZ4Removed HeaderID = 0x1400 // historically used for LZ4 and must not be reused. headerDeflateDefault HeaderID = 0x1500 headerDeflateBestSpeed HeaderID = 0x1501 diff --git a/repo/compression/compressor.go b/repo/compression/compressor.go index 52e10856f..ac10807b6 100644 --- a/repo/compression/compressor.go +++ b/repo/compression/compressor.go @@ -32,6 +32,7 @@ type Compressor interface { ByName = map[Name]Compressor{} HeaderIDToName = map[HeaderID]Name{} IsDeprecated = map[Name]bool{} + isUnsupported = map[HeaderID]bool{} ) // RegisterCompressor registers the provided compressor implementation. @@ -56,6 +57,13 @@ func RegisterDeprecatedCompressor(name Name, c Compressor) { IsDeprecated[name] = true } +func registerUnsupportedCompressor(name Name, c Compressor) { + RegisterCompressor(name, c) + + IsDeprecated[name] = true + isUnsupported[c.HeaderID()] = true +} + func compressionHeader(id HeaderID) []byte { b := make([]byte, compressionHeaderSize) binary.BigEndian.PutUint32(b, uint32(id)) @@ -81,6 +89,17 @@ func DecompressByHeader(output io.Writer, input io.Reader) error { return errors.Wrap(compressor.Decompress(output, input, false), "error decompressing") } +// IsSupported returns whether a named compression scheme is supported. +func IsSupported(name Name) bool { + c := ByName[name] + + if c == nil { + return false + } + + return !isUnsupported[c.HeaderID()] +} + func mustSucceed(err error) { impossible.PanicOnError(err) } diff --git a/repo/compression/compressor_lz4.go b/repo/compression/compressor_lz4.go index 2136a908b..d735b88cc 100644 --- a/repo/compression/compressor_lz4.go +++ b/repo/compression/compressor_lz4.go @@ -1,82 +1,26 @@ package compression import ( + "errors" "io" - "sync" - - "github.com/pierrec/lz4" - "github.com/pkg/errors" - - "github.com/kopia/kopia/internal/freepool" - "github.com/kopia/kopia/internal/iocopy" ) func init() { - RegisterDeprecatedCompressor("lz4", newLZ4Compressor(headerLZ4Default)) + registerUnsupportedCompressor("lz4", lz4Compressor{}) } -func newLZ4Compressor(id HeaderID) Compressor { - return &lz4Compressor{id, compressionHeader(id), sync.Pool{ - New: func() any { - return lz4.NewWriter(io.Discard) - }, - }} +var errLZ4NotSupported = errors.New("LZ4 compressor is not supported in recent versions of kopia, version v0.22.3 or older is needed to read legacy repositories that use the LZ4 compressor") + +type lz4Compressor struct{} + +func (c lz4Compressor) HeaderID() HeaderID { + return headerLZ4Removed } -type lz4Compressor struct { - id HeaderID - header []byte - pool sync.Pool +func (c lz4Compressor) Compress(_ io.Writer, _ io.Reader) error { + return errLZ4NotSupported } -func (c *lz4Compressor) HeaderID() HeaderID { - return c.id -} - -func (c *lz4Compressor) Compress(output io.Writer, input io.Reader) error { - if _, err := output.Write(c.header); err != nil { - return errors.Wrap(err, "unable to write header") - } - - //nolint:forcetypeassert - w := c.pool.Get().(*lz4.Writer) - defer c.pool.Put(w) - - w.Reset(output) - - if err := iocopy.JustCopy(w, input); err != nil { - return errors.Wrap(err, "compression error") - } - - if err := w.Close(); err != nil { - return errors.Wrap(err, "compression close error") - } - - return nil -} - -//nolint:gochecknoglobals -var lz4DecoderPool = freepool.New(func() *lz4.Reader { - return lz4.NewReader(nil) -}, func(v *lz4.Reader) { - v.Reset(nil) -}) - -func (c *lz4Compressor) Decompress(output io.Writer, input io.Reader, withHeader bool) error { - if withHeader { - if err := verifyCompressionHeader(input, c.header); err != nil { - return err - } - } - - dec := lz4DecoderPool.Take() - defer lz4DecoderPool.Return(dec) - - dec.Reset(input) - - if err := iocopy.JustCopy(output, dec); err != nil { - return errors.Wrap(err, "decompression error") - } - - return nil +func (c lz4Compressor) Decompress(_ io.Writer, _ io.Reader, _ bool) error { + return errLZ4NotSupported } diff --git a/repo/compression/compressor_test.go b/repo/compression/compressor_test.go index 024bebc82..a0d75c8f0 100644 --- a/repo/compression/compressor_test.go +++ b/repo/compression/compressor_test.go @@ -14,6 +14,10 @@ func TestMain(m *testing.M) { testutil.MyTestMain(m) } func TestCompressor(t *testing.T) { for id, comp := range ByHeaderID { + if isUnsupported[id] { + continue + } + t.Run(fmt.Sprintf("compressible-data-%x", id), func(t *testing.T) { // make sure all-zero data is compressed data := make([]byte, 10000) diff --git a/repo/object/object_manager_test.go b/repo/object/object_manager_test.go index 4913b315d..61479ed88 100644 --- a/repo/object/object_manager_test.go +++ b/repo/object/object_manager_test.go @@ -802,6 +802,10 @@ func TestEndToEndReadAndSeekWithCompression(t *testing.T) { for _, compressible := range []bool{false, true} { for compressorName := range compression.ByName { + if !compression.IsSupported(compressorName) { + continue + } + t.Run(string(compressorName), func(t *testing.T) { ctx := testlogging.Context(t) diff --git a/site/content/docs/Advanced/Compression/_index.md b/site/content/docs/Advanced/Compression/_index.md index 68910f7ca..a5e4dd14b 100644 --- a/site/content/docs/Advanced/Compression/_index.md +++ b/site/content/docs/Advanced/Compression/_index.md @@ -35,18 +35,17 @@ Repeating 1 times per compression method (total 466.7 MiB). 3. s2-parallel-4 127.1 MiB 2.3 GiB/s 2951 344.1 MiB 4. pgzip-best-speed 96.7 MiB 2.1 GiB/s 4127 324.1 MiB 5. pgzip 86.3 MiB 1.2 GiB/s 4132 298.7 MiB - 6. lz4 131.8 MiB 458.9 MiB/s 17 321.7 MiB - 7. zstd-fastest 79.8 MiB 356.2 MiB/s 22503 246 MiB - 8. zstd 76.8 MiB 323.7 MiB/s 22605 237.8 MiB - 9. deflate-best-speed 96.7 MiB 220.8 MiB/s 45 310.8 MiB - 10. gzip-best-speed 94.9 MiB 165 MiB/s 40 305.2 MiB - 11. deflate-default 86.3 MiB 143.1 MiB/s 34 311 MiB - 12. zstd-better-compression 74.2 MiB 104 MiB/s 22496 251.4 MiB - 13. pgzip-best-compression 83 MiB 55.9 MiB/s 4359 299.1 MiB - 14. gzip 83.6 MiB 40.5 MiB/s 69 304.8 MiB - 15. zstd-best-compression 68.9 MiB 19.2 MiB/s 22669 303.4 MiB - 16. deflate-best-compression 83 MiB 5.6 MiB/s 134 311 MiB - 17. gzip-best-compression 83 MiB 5.1 MiB/s 137 304.8 MiB + 6. zstd-fastest 79.8 MiB 356.2 MiB/s 22503 246 MiB + 7. zstd 76.8 MiB 323.7 MiB/s 22605 237.8 MiB + 8. deflate-best-speed 96.7 MiB 220.8 MiB/s 45 310.8 MiB + 9. gzip-best-speed 94.9 MiB 165 MiB/s 40 305.2 MiB + 10. deflate-default 86.3 MiB 143.1 MiB/s 34 311 MiB + 11. zstd-better-compression 74.2 MiB 104 MiB/s 22496 251.4 MiB + 12. pgzip-best-compression 83 MiB 55.9 MiB/s 4359 299.1 MiB + 13. gzip 83.6 MiB 40.5 MiB/s 69 304.8 MiB + 14. zstd-best-compression 68.9 MiB 19.2 MiB/s 22669 303.4 MiB + 15. deflate-best-compression 83 MiB 5.6 MiB/s 134 311 MiB + 16. gzip-best-compression 83 MiB 5.1 MiB/s 137 304.8 MiB ``` As you can see, s2 compression clearly favors performance over compression ratio. zstd on the other hand results almost half the size as s2. pgzip arguably offers the best balance on two worlds. @@ -73,21 +72,23 @@ Repeating 100 times per compression method (total 12.5 MiB). 8. gzip-best-speed 33.7 KiB 150.6 MiB/s 28 1.2 MiB 9. pgzip-best-speed 34.3 KiB 143.7 MiB/s 1649 220.2 MiB 10. deflate-default 31.2 KiB 126.3 MiB/s 22 1.1 MiB - 11. lz4 44.7 KiB 112.6 MiB/s 435 816.7 MiB - 12. pgzip 31.2 KiB 94.6 MiB/s 2634 277.5 MiB - 13. gzip 30.4 KiB 39.5 MiB/s 26 874.7 KiB - 14. deflate-best-compression 30.4 KiB 25.4 MiB/s 21 1 MiB - 15. gzip-best-compression 30.4 KiB 24.5 MiB/s 27 874.9 KiB - 16. pgzip-best-compression 30.4 KiB 23.1 MiB/s 2646 281.8 MiB - 17. zstd-best-compression 25.1 KiB 19.3 MiB/s 882 99.3 MiB + 11. pgzip 31.2 KiB 94.6 MiB/s 2634 277.5 MiB + 12. gzip 30.4 KiB 39.5 MiB/s 26 874.7 KiB + 13. deflate-best-compression 30.4 KiB 25.4 MiB/s 21 1 MiB + 14. gzip-best-compression 30.4 KiB 24.5 MiB/s 27 874.9 KiB + 15. pgzip-best-compression 30.4 KiB 23.1 MiB/s 2646 281.8 MiB + 16. zstd-best-compression 25.1 KiB 19.3 MiB/s 882 99.3 MiB ``` -While s2 significantly uses way less memory in this case, pgzip's numbers seem indifferent to the input size. [Turns out](https://github.com/klauspost/pgzip/issues/44), pgzip has different memory usage logic. It would quickly allocate necessary memory and plateau, which s2 is more on a linear fashion. Here is rough graph is demonstrate the difference: +While s2 uses significantly less memory in this case, pgzip's numbers seem indifferent to the input size. [Turns out](https://github.com/klauspost/pgzip/issues/44), pgzip has different memory usage logic. It would quickly allocate necessary memory and plateau, which s2 is more on a linear fashion. Here is rough graph is demonstrate the difference: ![s2 vs pgzip](s2_vs_pgzip.svg) Therefore, if your backup target is small, and memory is extremely restricted, s2 might be necessary. Otherwise, all algorithms are valid candidates. + Note: Newer Kopia versions no longer support reading contents that were compressed with the deprecated LZ4 algorithm. If your repository contains data written with LZ4, you must migrate it first using a Kopia version that still supports LZ4—for example by restoring the affected snapshots and/or repacking the repository with one of the currently supported compression algorithms—before upgrading. + + ### Minimum file size and extensions to compress As discussed above, some compression algorithms make sense only if the payload is large enough. So it might be beneficial to set a minimum file size when using these algorithms.