breaking(general): remove support for LZ4 (#5296)

- It has been deprecated in Kopia since July 2022 (0985b80).
- Dependency appears unmaintained.
This commit is contained in:
Julio López
2026-04-16 15:13:17 -07:00
committed by GitHub
parent c9334c9bc5
commit be2ce68b5c
8 changed files with 61 additions and 105 deletions

2
go.mod
View File

@@ -38,7 +38,6 @@ require (
github.com/mxk/go-vss v1.2.1
github.com/natefinch/atomic v1.0.1
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9
github.com/pierrec/lz4 v2.6.1+incompatible
github.com/pkg/errors v0.9.1
github.com/pkg/profile v1.7.0
github.com/pkg/sftp v1.13.10
@@ -99,7 +98,6 @@ require (
github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
github.com/felixge/fgprof v0.9.3 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/frankban/quicktest v1.13.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect

14
go.sum
View File

@@ -72,7 +72,6 @@ github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/T
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI=
github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ=
github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs=
github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -102,8 +101,6 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/foomo/htpasswd v0.0.0-20200116085101-e3a90e78da9c h1:DBGU7zCwrrPPDsD6+gqKG8UfMxenWg9BOJE/Nmfph+4=
github.com/foomo/htpasswd v0.0.0-20200116085101-e3a90e78da9c/go.mod h1:SHawtolbB0ZOFoRWgDwakX5WpwuIWAK88bUXVZqK0Ss=
github.com/frankban/quicktest v1.13.1 h1:xVm/f9seEhZFL9+n5kv5XLrGwy6elc4V9v/XFY2vmd8=
github.com/frankban/quicktest v1.13.1/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
@@ -137,7 +134,6 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/fswalker v0.3.3 h1:K2+d6cb3vNFjquVPRObIY+QaXJ6cbleVV6yZWLzkkQ8=
github.com/google/fswalker v0.3.3/go.mod h1:9upMSscEE8oRi0WJ0rXZZYya1DmgUtJFhXAw7KNS3c4=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
@@ -181,12 +177,8 @@ github.com/kopia/htmluibuild v0.0.1-0.20260414002305-a859d43ee893 h1:J9g3qLDJAIe
github.com/kopia/htmluibuild v0.0.1-0.20260414002305-a859d43ee893/go.mod h1:h53A5JM3t2qiwxqxusBe+PFgGcgZdS+DWCQvG5PTlto=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
@@ -219,8 +211,6 @@ github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -245,7 +235,6 @@ github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTU
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
@@ -349,7 +338,6 @@ golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/api v0.275.0 h1:vfY5d9vFVJeWEZT65QDd9hbndr7FyZ2+6mIzGAh71NI=
@@ -365,10 +353,8 @@ google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216 h1:2TSTkQ8PMvGOD5eeqqRVv6Z9+BYI+bowK97RCr3W+9M=
gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -23,7 +23,7 @@
headerPgzipBestSpeed HeaderID = 0x1301
headerPgzipBestCompression HeaderID = 0x1302
headerLZ4Default HeaderID = 0x1400
headerLZ4Removed HeaderID = 0x1400 // historically used for LZ4 and must not be reused.
headerDeflateDefault HeaderID = 0x1500
headerDeflateBestSpeed HeaderID = 0x1501

View File

@@ -32,6 +32,7 @@ type Compressor interface {
ByName = map[Name]Compressor{}
HeaderIDToName = map[HeaderID]Name{}
IsDeprecated = map[Name]bool{}
isUnsupported = map[HeaderID]bool{}
)
// RegisterCompressor registers the provided compressor implementation.
@@ -56,6 +57,13 @@ func RegisterDeprecatedCompressor(name Name, c Compressor) {
IsDeprecated[name] = true
}
func registerUnsupportedCompressor(name Name, c Compressor) {
RegisterCompressor(name, c)
IsDeprecated[name] = true
isUnsupported[c.HeaderID()] = true
}
func compressionHeader(id HeaderID) []byte {
b := make([]byte, compressionHeaderSize)
binary.BigEndian.PutUint32(b, uint32(id))
@@ -81,6 +89,17 @@ func DecompressByHeader(output io.Writer, input io.Reader) error {
return errors.Wrap(compressor.Decompress(output, input, false), "error decompressing")
}
// IsSupported returns whether a named compression scheme is supported.
func IsSupported(name Name) bool {
c := ByName[name]
if c == nil {
return false
}
return !isUnsupported[c.HeaderID()]
}
func mustSucceed(err error) {
impossible.PanicOnError(err)
}

View File

@@ -1,82 +1,26 @@
package compression
import (
"errors"
"io"
"sync"
"github.com/pierrec/lz4"
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/freepool"
"github.com/kopia/kopia/internal/iocopy"
)
func init() {
RegisterDeprecatedCompressor("lz4", newLZ4Compressor(headerLZ4Default))
registerUnsupportedCompressor("lz4", lz4Compressor{})
}
func newLZ4Compressor(id HeaderID) Compressor {
return &lz4Compressor{id, compressionHeader(id), sync.Pool{
New: func() any {
return lz4.NewWriter(io.Discard)
},
}}
var errLZ4NotSupported = errors.New("LZ4 compressor is not supported in recent versions of kopia, version v0.22.3 or older is needed to read legacy repositories that use the LZ4 compressor")
type lz4Compressor struct{}
func (c lz4Compressor) HeaderID() HeaderID {
return headerLZ4Removed
}
type lz4Compressor struct {
id HeaderID
header []byte
pool sync.Pool
func (c lz4Compressor) Compress(_ io.Writer, _ io.Reader) error {
return errLZ4NotSupported
}
func (c *lz4Compressor) HeaderID() HeaderID {
return c.id
}
func (c *lz4Compressor) Compress(output io.Writer, input io.Reader) error {
if _, err := output.Write(c.header); err != nil {
return errors.Wrap(err, "unable to write header")
}
//nolint:forcetypeassert
w := c.pool.Get().(*lz4.Writer)
defer c.pool.Put(w)
w.Reset(output)
if err := iocopy.JustCopy(w, input); err != nil {
return errors.Wrap(err, "compression error")
}
if err := w.Close(); err != nil {
return errors.Wrap(err, "compression close error")
}
return nil
}
//nolint:gochecknoglobals
var lz4DecoderPool = freepool.New(func() *lz4.Reader {
return lz4.NewReader(nil)
}, func(v *lz4.Reader) {
v.Reset(nil)
})
func (c *lz4Compressor) Decompress(output io.Writer, input io.Reader, withHeader bool) error {
if withHeader {
if err := verifyCompressionHeader(input, c.header); err != nil {
return err
}
}
dec := lz4DecoderPool.Take()
defer lz4DecoderPool.Return(dec)
dec.Reset(input)
if err := iocopy.JustCopy(output, dec); err != nil {
return errors.Wrap(err, "decompression error")
}
return nil
func (c lz4Compressor) Decompress(_ io.Writer, _ io.Reader, _ bool) error {
return errLZ4NotSupported
}

View File

@@ -14,6 +14,10 @@ func TestMain(m *testing.M) { testutil.MyTestMain(m) }
func TestCompressor(t *testing.T) {
for id, comp := range ByHeaderID {
if isUnsupported[id] {
continue
}
t.Run(fmt.Sprintf("compressible-data-%x", id), func(t *testing.T) {
// make sure all-zero data is compressed
data := make([]byte, 10000)

View File

@@ -802,6 +802,10 @@ func TestEndToEndReadAndSeekWithCompression(t *testing.T) {
for _, compressible := range []bool{false, true} {
for compressorName := range compression.ByName {
if !compression.IsSupported(compressorName) {
continue
}
t.Run(string(compressorName), func(t *testing.T) {
ctx := testlogging.Context(t)

View File

@@ -35,18 +35,17 @@ Repeating 1 times per compression method (total 466.7 MiB).
3. s2-parallel-4 127.1 MiB 2.3 GiB/s 2951 344.1 MiB
4. pgzip-best-speed 96.7 MiB 2.1 GiB/s 4127 324.1 MiB
5. pgzip 86.3 MiB 1.2 GiB/s 4132 298.7 MiB
6. lz4 131.8 MiB 458.9 MiB/s 17 321.7 MiB
7. zstd-fastest 79.8 MiB 356.2 MiB/s 22503 246 MiB
8. zstd 76.8 MiB 323.7 MiB/s 22605 237.8 MiB
9. deflate-best-speed 96.7 MiB 220.8 MiB/s 45 310.8 MiB
10. gzip-best-speed 94.9 MiB 165 MiB/s 40 305.2 MiB
11. deflate-default 86.3 MiB 143.1 MiB/s 34 311 MiB
12. zstd-better-compression 74.2 MiB 104 MiB/s 22496 251.4 MiB
13. pgzip-best-compression 83 MiB 55.9 MiB/s 4359 299.1 MiB
14. gzip 83.6 MiB 40.5 MiB/s 69 304.8 MiB
15. zstd-best-compression 68.9 MiB 19.2 MiB/s 22669 303.4 MiB
16. deflate-best-compression 83 MiB 5.6 MiB/s 134 311 MiB
17. gzip-best-compression 83 MiB 5.1 MiB/s 137 304.8 MiB
6. zstd-fastest 79.8 MiB 356.2 MiB/s 22503 246 MiB
7. zstd 76.8 MiB 323.7 MiB/s 22605 237.8 MiB
8. deflate-best-speed 96.7 MiB 220.8 MiB/s 45 310.8 MiB
9. gzip-best-speed 94.9 MiB 165 MiB/s 40 305.2 MiB
10. deflate-default 86.3 MiB 143.1 MiB/s 34 311 MiB
11. zstd-better-compression 74.2 MiB 104 MiB/s 22496 251.4 MiB
12. pgzip-best-compression 83 MiB 55.9 MiB/s 4359 299.1 MiB
13. gzip 83.6 MiB 40.5 MiB/s 69 304.8 MiB
14. zstd-best-compression 68.9 MiB 19.2 MiB/s 22669 303.4 MiB
15. deflate-best-compression 83 MiB 5.6 MiB/s 134 311 MiB
16. gzip-best-compression 83 MiB 5.1 MiB/s 137 304.8 MiB
```
As you can see, s2 compression clearly favors performance over compression ratio. zstd on the other hand results almost half the size as s2. pgzip arguably offers the best balance on two worlds.
@@ -73,21 +72,23 @@ Repeating 100 times per compression method (total 12.5 MiB).
8. gzip-best-speed 33.7 KiB 150.6 MiB/s 28 1.2 MiB
9. pgzip-best-speed 34.3 KiB 143.7 MiB/s 1649 220.2 MiB
10. deflate-default 31.2 KiB 126.3 MiB/s 22 1.1 MiB
11. lz4 44.7 KiB 112.6 MiB/s 435 816.7 MiB
12. pgzip 31.2 KiB 94.6 MiB/s 2634 277.5 MiB
13. gzip 30.4 KiB 39.5 MiB/s 26 874.7 KiB
14. deflate-best-compression 30.4 KiB 25.4 MiB/s 21 1 MiB
15. gzip-best-compression 30.4 KiB 24.5 MiB/s 27 874.9 KiB
16. pgzip-best-compression 30.4 KiB 23.1 MiB/s 2646 281.8 MiB
17. zstd-best-compression 25.1 KiB 19.3 MiB/s 882 99.3 MiB
11. pgzip 31.2 KiB 94.6 MiB/s 2634 277.5 MiB
12. gzip 30.4 KiB 39.5 MiB/s 26 874.7 KiB
13. deflate-best-compression 30.4 KiB 25.4 MiB/s 21 1 MiB
14. gzip-best-compression 30.4 KiB 24.5 MiB/s 27 874.9 KiB
15. pgzip-best-compression 30.4 KiB 23.1 MiB/s 2646 281.8 MiB
16. zstd-best-compression 25.1 KiB 19.3 MiB/s 882 99.3 MiB
```
While s2 significantly uses way less memory in this case, pgzip's numbers seem indifferent to the input size. [Turns out](https://github.com/klauspost/pgzip/issues/44), pgzip has different memory usage logic. It would quickly allocate necessary memory and plateau, which s2 is more on a linear fashion. Here is rough graph is demonstrate the difference:
While s2 uses significantly less memory in this case, pgzip's numbers seem indifferent to the input size. [Turns out](https://github.com/klauspost/pgzip/issues/44), pgzip has different memory usage logic. It would quickly allocate necessary memory and plateau, which s2 is more on a linear fashion. Here is rough graph is demonstrate the difference:
![s2 vs pgzip](s2_vs_pgzip.svg)
Therefore, if your backup target is small, and memory is extremely restricted, s2 might be necessary. Otherwise, all algorithms are valid candidates.
Note: Newer Kopia versions no longer support reading contents that were compressed with the deprecated LZ4 algorithm. If your repository contains data written with LZ4, you must migrate it first using a Kopia version that still supports LZ4—for example by restoring the affected snapshots and/or repacking the repository with one of the currently supported compression algorithms—before upgrading.
### Minimum file size and extensions to compress
As discussed above, some compression algorithms make sense only if the payload is large enough. So it might be beneficial to set a minimum file size when using these algorithms.