mirror of
https://github.com/kopia/kopia.git
synced 2026-05-09 15:23:02 -04:00
breaking(general): remove support for LZ4 (#5296)
- It has been deprecated in Kopia since July 2022 (0985b80).
- Dependency appears unmaintained.
This commit is contained in:
2
go.mod
2
go.mod
@@ -38,7 +38,6 @@ require (
|
||||
github.com/mxk/go-vss v1.2.1
|
||||
github.com/natefinch/atomic v1.0.1
|
||||
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9
|
||||
github.com/pierrec/lz4 v2.6.1+incompatible
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/pkg/profile v1.7.0
|
||||
github.com/pkg/sftp v1.13.10
|
||||
@@ -99,7 +98,6 @@ require (
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
|
||||
github.com/felixge/fgprof v0.9.3 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/frankban/quicktest v1.13.1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
|
||||
|
||||
14
go.sum
14
go.sum
@@ -72,7 +72,6 @@ github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/T
|
||||
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI=
|
||||
github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
|
||||
github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ=
|
||||
github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs=
|
||||
github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -102,8 +101,6 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/foomo/htpasswd v0.0.0-20200116085101-e3a90e78da9c h1:DBGU7zCwrrPPDsD6+gqKG8UfMxenWg9BOJE/Nmfph+4=
|
||||
github.com/foomo/htpasswd v0.0.0-20200116085101-e3a90e78da9c/go.mod h1:SHawtolbB0ZOFoRWgDwakX5WpwuIWAK88bUXVZqK0Ss=
|
||||
github.com/frankban/quicktest v1.13.1 h1:xVm/f9seEhZFL9+n5kv5XLrGwy6elc4V9v/XFY2vmd8=
|
||||
github.com/frankban/quicktest v1.13.1/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
@@ -137,7 +134,6 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/fswalker v0.3.3 h1:K2+d6cb3vNFjquVPRObIY+QaXJ6cbleVV6yZWLzkkQ8=
|
||||
github.com/google/fswalker v0.3.3/go.mod h1:9upMSscEE8oRi0WJ0rXZZYya1DmgUtJFhXAw7KNS3c4=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
|
||||
@@ -181,12 +177,8 @@ github.com/kopia/htmluibuild v0.0.1-0.20260414002305-a859d43ee893 h1:J9g3qLDJAIe
|
||||
github.com/kopia/htmluibuild v0.0.1-0.20260414002305-a859d43ee893/go.mod h1:h53A5JM3t2qiwxqxusBe+PFgGcgZdS+DWCQvG5PTlto=
|
||||
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
@@ -219,8 +211,6 @@ github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+
|
||||
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
|
||||
github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
@@ -245,7 +235,6 @@ github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTU
|
||||
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
@@ -349,7 +338,6 @@ golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/api v0.275.0 h1:vfY5d9vFVJeWEZT65QDd9hbndr7FyZ2+6mIzGAh71NI=
|
||||
@@ -365,10 +353,8 @@ google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216 h1:2TSTkQ8PMvGOD5eeqqRVv6Z9+BYI+bowK97RCr3W+9M=
|
||||
gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
headerPgzipBestSpeed HeaderID = 0x1301
|
||||
headerPgzipBestCompression HeaderID = 0x1302
|
||||
|
||||
headerLZ4Default HeaderID = 0x1400
|
||||
headerLZ4Removed HeaderID = 0x1400 // historically used for LZ4 and must not be reused.
|
||||
|
||||
headerDeflateDefault HeaderID = 0x1500
|
||||
headerDeflateBestSpeed HeaderID = 0x1501
|
||||
|
||||
@@ -32,6 +32,7 @@ type Compressor interface {
|
||||
ByName = map[Name]Compressor{}
|
||||
HeaderIDToName = map[HeaderID]Name{}
|
||||
IsDeprecated = map[Name]bool{}
|
||||
isUnsupported = map[HeaderID]bool{}
|
||||
)
|
||||
|
||||
// RegisterCompressor registers the provided compressor implementation.
|
||||
@@ -56,6 +57,13 @@ func RegisterDeprecatedCompressor(name Name, c Compressor) {
|
||||
IsDeprecated[name] = true
|
||||
}
|
||||
|
||||
func registerUnsupportedCompressor(name Name, c Compressor) {
|
||||
RegisterCompressor(name, c)
|
||||
|
||||
IsDeprecated[name] = true
|
||||
isUnsupported[c.HeaderID()] = true
|
||||
}
|
||||
|
||||
func compressionHeader(id HeaderID) []byte {
|
||||
b := make([]byte, compressionHeaderSize)
|
||||
binary.BigEndian.PutUint32(b, uint32(id))
|
||||
@@ -81,6 +89,17 @@ func DecompressByHeader(output io.Writer, input io.Reader) error {
|
||||
return errors.Wrap(compressor.Decompress(output, input, false), "error decompressing")
|
||||
}
|
||||
|
||||
// IsSupported returns whether a named compression scheme is supported.
|
||||
func IsSupported(name Name) bool {
|
||||
c := ByName[name]
|
||||
|
||||
if c == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return !isUnsupported[c.HeaderID()]
|
||||
}
|
||||
|
||||
func mustSucceed(err error) {
|
||||
impossible.PanicOnError(err)
|
||||
}
|
||||
|
||||
@@ -1,82 +1,26 @@
|
||||
package compression
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/pierrec/lz4"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/kopia/kopia/internal/freepool"
|
||||
"github.com/kopia/kopia/internal/iocopy"
|
||||
)
|
||||
|
||||
func init() {
|
||||
RegisterDeprecatedCompressor("lz4", newLZ4Compressor(headerLZ4Default))
|
||||
registerUnsupportedCompressor("lz4", lz4Compressor{})
|
||||
}
|
||||
|
||||
func newLZ4Compressor(id HeaderID) Compressor {
|
||||
return &lz4Compressor{id, compressionHeader(id), sync.Pool{
|
||||
New: func() any {
|
||||
return lz4.NewWriter(io.Discard)
|
||||
},
|
||||
}}
|
||||
var errLZ4NotSupported = errors.New("LZ4 compressor is not supported in recent versions of kopia, version v0.22.3 or older is needed to read legacy repositories that use the LZ4 compressor")
|
||||
|
||||
type lz4Compressor struct{}
|
||||
|
||||
func (c lz4Compressor) HeaderID() HeaderID {
|
||||
return headerLZ4Removed
|
||||
}
|
||||
|
||||
type lz4Compressor struct {
|
||||
id HeaderID
|
||||
header []byte
|
||||
pool sync.Pool
|
||||
func (c lz4Compressor) Compress(_ io.Writer, _ io.Reader) error {
|
||||
return errLZ4NotSupported
|
||||
}
|
||||
|
||||
func (c *lz4Compressor) HeaderID() HeaderID {
|
||||
return c.id
|
||||
}
|
||||
|
||||
func (c *lz4Compressor) Compress(output io.Writer, input io.Reader) error {
|
||||
if _, err := output.Write(c.header); err != nil {
|
||||
return errors.Wrap(err, "unable to write header")
|
||||
}
|
||||
|
||||
//nolint:forcetypeassert
|
||||
w := c.pool.Get().(*lz4.Writer)
|
||||
defer c.pool.Put(w)
|
||||
|
||||
w.Reset(output)
|
||||
|
||||
if err := iocopy.JustCopy(w, input); err != nil {
|
||||
return errors.Wrap(err, "compression error")
|
||||
}
|
||||
|
||||
if err := w.Close(); err != nil {
|
||||
return errors.Wrap(err, "compression close error")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
//nolint:gochecknoglobals
|
||||
var lz4DecoderPool = freepool.New(func() *lz4.Reader {
|
||||
return lz4.NewReader(nil)
|
||||
}, func(v *lz4.Reader) {
|
||||
v.Reset(nil)
|
||||
})
|
||||
|
||||
func (c *lz4Compressor) Decompress(output io.Writer, input io.Reader, withHeader bool) error {
|
||||
if withHeader {
|
||||
if err := verifyCompressionHeader(input, c.header); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
dec := lz4DecoderPool.Take()
|
||||
defer lz4DecoderPool.Return(dec)
|
||||
|
||||
dec.Reset(input)
|
||||
|
||||
if err := iocopy.JustCopy(output, dec); err != nil {
|
||||
return errors.Wrap(err, "decompression error")
|
||||
}
|
||||
|
||||
return nil
|
||||
func (c lz4Compressor) Decompress(_ io.Writer, _ io.Reader, _ bool) error {
|
||||
return errLZ4NotSupported
|
||||
}
|
||||
|
||||
@@ -14,6 +14,10 @@ func TestMain(m *testing.M) { testutil.MyTestMain(m) }
|
||||
|
||||
func TestCompressor(t *testing.T) {
|
||||
for id, comp := range ByHeaderID {
|
||||
if isUnsupported[id] {
|
||||
continue
|
||||
}
|
||||
|
||||
t.Run(fmt.Sprintf("compressible-data-%x", id), func(t *testing.T) {
|
||||
// make sure all-zero data is compressed
|
||||
data := make([]byte, 10000)
|
||||
|
||||
@@ -802,6 +802,10 @@ func TestEndToEndReadAndSeekWithCompression(t *testing.T) {
|
||||
|
||||
for _, compressible := range []bool{false, true} {
|
||||
for compressorName := range compression.ByName {
|
||||
if !compression.IsSupported(compressorName) {
|
||||
continue
|
||||
}
|
||||
|
||||
t.Run(string(compressorName), func(t *testing.T) {
|
||||
ctx := testlogging.Context(t)
|
||||
|
||||
|
||||
@@ -35,18 +35,17 @@ Repeating 1 times per compression method (total 466.7 MiB).
|
||||
3. s2-parallel-4 127.1 MiB 2.3 GiB/s 2951 344.1 MiB
|
||||
4. pgzip-best-speed 96.7 MiB 2.1 GiB/s 4127 324.1 MiB
|
||||
5. pgzip 86.3 MiB 1.2 GiB/s 4132 298.7 MiB
|
||||
6. lz4 131.8 MiB 458.9 MiB/s 17 321.7 MiB
|
||||
7. zstd-fastest 79.8 MiB 356.2 MiB/s 22503 246 MiB
|
||||
8. zstd 76.8 MiB 323.7 MiB/s 22605 237.8 MiB
|
||||
9. deflate-best-speed 96.7 MiB 220.8 MiB/s 45 310.8 MiB
|
||||
10. gzip-best-speed 94.9 MiB 165 MiB/s 40 305.2 MiB
|
||||
11. deflate-default 86.3 MiB 143.1 MiB/s 34 311 MiB
|
||||
12. zstd-better-compression 74.2 MiB 104 MiB/s 22496 251.4 MiB
|
||||
13. pgzip-best-compression 83 MiB 55.9 MiB/s 4359 299.1 MiB
|
||||
14. gzip 83.6 MiB 40.5 MiB/s 69 304.8 MiB
|
||||
15. zstd-best-compression 68.9 MiB 19.2 MiB/s 22669 303.4 MiB
|
||||
16. deflate-best-compression 83 MiB 5.6 MiB/s 134 311 MiB
|
||||
17. gzip-best-compression 83 MiB 5.1 MiB/s 137 304.8 MiB
|
||||
6. zstd-fastest 79.8 MiB 356.2 MiB/s 22503 246 MiB
|
||||
7. zstd 76.8 MiB 323.7 MiB/s 22605 237.8 MiB
|
||||
8. deflate-best-speed 96.7 MiB 220.8 MiB/s 45 310.8 MiB
|
||||
9. gzip-best-speed 94.9 MiB 165 MiB/s 40 305.2 MiB
|
||||
10. deflate-default 86.3 MiB 143.1 MiB/s 34 311 MiB
|
||||
11. zstd-better-compression 74.2 MiB 104 MiB/s 22496 251.4 MiB
|
||||
12. pgzip-best-compression 83 MiB 55.9 MiB/s 4359 299.1 MiB
|
||||
13. gzip 83.6 MiB 40.5 MiB/s 69 304.8 MiB
|
||||
14. zstd-best-compression 68.9 MiB 19.2 MiB/s 22669 303.4 MiB
|
||||
15. deflate-best-compression 83 MiB 5.6 MiB/s 134 311 MiB
|
||||
16. gzip-best-compression 83 MiB 5.1 MiB/s 137 304.8 MiB
|
||||
```
|
||||
|
||||
As you can see, s2 compression clearly favors performance over compression ratio. zstd on the other hand results almost half the size as s2. pgzip arguably offers the best balance on two worlds.
|
||||
@@ -73,21 +72,23 @@ Repeating 100 times per compression method (total 12.5 MiB).
|
||||
8. gzip-best-speed 33.7 KiB 150.6 MiB/s 28 1.2 MiB
|
||||
9. pgzip-best-speed 34.3 KiB 143.7 MiB/s 1649 220.2 MiB
|
||||
10. deflate-default 31.2 KiB 126.3 MiB/s 22 1.1 MiB
|
||||
11. lz4 44.7 KiB 112.6 MiB/s 435 816.7 MiB
|
||||
12. pgzip 31.2 KiB 94.6 MiB/s 2634 277.5 MiB
|
||||
13. gzip 30.4 KiB 39.5 MiB/s 26 874.7 KiB
|
||||
14. deflate-best-compression 30.4 KiB 25.4 MiB/s 21 1 MiB
|
||||
15. gzip-best-compression 30.4 KiB 24.5 MiB/s 27 874.9 KiB
|
||||
16. pgzip-best-compression 30.4 KiB 23.1 MiB/s 2646 281.8 MiB
|
||||
17. zstd-best-compression 25.1 KiB 19.3 MiB/s 882 99.3 MiB
|
||||
11. pgzip 31.2 KiB 94.6 MiB/s 2634 277.5 MiB
|
||||
12. gzip 30.4 KiB 39.5 MiB/s 26 874.7 KiB
|
||||
13. deflate-best-compression 30.4 KiB 25.4 MiB/s 21 1 MiB
|
||||
14. gzip-best-compression 30.4 KiB 24.5 MiB/s 27 874.9 KiB
|
||||
15. pgzip-best-compression 30.4 KiB 23.1 MiB/s 2646 281.8 MiB
|
||||
16. zstd-best-compression 25.1 KiB 19.3 MiB/s 882 99.3 MiB
|
||||
```
|
||||
|
||||
While s2 significantly uses way less memory in this case, pgzip's numbers seem indifferent to the input size. [Turns out](https://github.com/klauspost/pgzip/issues/44), pgzip has different memory usage logic. It would quickly allocate necessary memory and plateau, which s2 is more on a linear fashion. Here is rough graph is demonstrate the difference:
|
||||
While s2 uses significantly less memory in this case, pgzip's numbers seem indifferent to the input size. [Turns out](https://github.com/klauspost/pgzip/issues/44), pgzip has different memory usage logic. It would quickly allocate necessary memory and plateau, which s2 is more on a linear fashion. Here is rough graph is demonstrate the difference:
|
||||
|
||||

|
||||
|
||||
Therefore, if your backup target is small, and memory is extremely restricted, s2 might be necessary. Otherwise, all algorithms are valid candidates.
|
||||
|
||||
Note: Newer Kopia versions no longer support reading contents that were compressed with the deprecated LZ4 algorithm. If your repository contains data written with LZ4, you must migrate it first using a Kopia version that still supports LZ4—for example by restoring the affected snapshots and/or repacking the repository with one of the currently supported compression algorithms—before upgrading.
|
||||
|
||||
|
||||
### Minimum file size and extensions to compress
|
||||
|
||||
As discussed above, some compression algorithms make sense only if the payload is large enough. So it might be beneficial to set a minimum file size when using these algorithms.
|
||||
|
||||
Reference in New Issue
Block a user