mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-05-09 15:23:02 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(ui): support for deprecation of certain algorithms (#2122)

Browse Source 
Some compression algorithms are not recommended because they
allocate disproportionate amounts of memory. They are still
possible to use, just marked as NOT RECOMMENDED in the UI.
This commit is contained in:
Jarek Kowalski
2022-07-03 12:06:14 -07:00
committed by GitHub
parent 085ec8fc57
commit 0985b80488
8 changed files with 82 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
cli/command_benchmark_compression.go
View File

@@ -26,6 +26,7 @@ type commandBenchmarkCompression struct {
verifyStable bool
optionPrint bool
parallel int
deprecated bool
out textOutput
}
@@ -39,6 +40,7 @@ func (c *commandBenchmarkCompression) setup(svc appServices, parent commandParen
cmd.Flag("parallel", "Number of parallel goroutines").Default("1").IntVar(&c.parallel)
cmd.Flag("verify-stable", "Verify that compression is stable").BoolVar(&c.verifyStable)
cmd.Flag("print-options", "Print out options usable for repository creation").BoolVar(&c.optionPrint)
cmd.Flag("deprecated", "Included deprecated compression algorithms").BoolVar(&c.deprecated)
cmd.Action(svc.noRepositoryAction(c.run))
c.out.setup(svc)
}
@@ -107,6 +109,10 @@ func (c *commandBenchmarkCompression) run(ctx context.Context) error {
log(ctx).Infof("Repeating %v times per compression method (total %v). Override with --repeat=N.", repeatCount, units.BytesStringBase2(int64(repeatCount*len(data))))
for name, comp := range compression.ByName {
if compression.IsDeprecated[name] && !c.deprecated {
continue
}
log(ctx).Infof("Benchmarking compressor '%v'...", name)
cnt := repeatCount
@@ -197,13 +203,19 @@ func (c *commandBenchmarkCompression) printResults(results []compressionBechmark
c.out.printStdout("------------------------------------------------------------------------------------------------\n")
for ndx, r := range results {
c.out.printStdout("%3d. %-26v %-12v %-12v %-8v %v",
maybeDeprecated := ""
if compression.IsDeprecated[r.compression] {
maybeDeprecated = " (deprecated)"
}
c.out.printStdout("%3d. %-26v %-12v %-12v %-8v %v%v",
ndx,
r.compression,
units.BytesStringBase2(r.compressedSize),
units.BytesStringBase2(int64(r.throughput))+"/s",
r.allocations,
units.BytesStringBase2(r.allocBytes),
maybeDeprecated,
)
if c.optionPrint {

2
go.mod
View File

@@ -83,7 +83,7 @@ require (
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kopia/htmluibuild v0.0.0-20220702012237-9d334f452b6b
github.com/kopia/htmluibuild v0.0.0-20220703174648-d70429a8dad7
github.com/kr/fs v0.1.0 // indirect
github.com/mattn/go-ieproxy v0.0.1 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect

2
go.sum
View File

@@ -349,6 +349,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kopia/htmluibuild v0.0.0-20220702012237-9d334f452b6b h1:ox9MBvtiRvcnVjYQd5XYmk9IhRGMo7lK/pbJA1Ud2EY=
github.com/kopia/htmluibuild v0.0.0-20220702012237-9d334f452b6b/go.mod h1:eWer4rx9P8lJo2eKc+Q7AZ1dE1x1hJNdkbDFPzMu1Hw=
github.com/kopia/htmluibuild v0.0.0-20220703174648-d70429a8dad7 h1:OdKsyukYW8aww2naIL2HECJbcuicvJrOb884tI+2FWM=
github.com/kopia/htmluibuild v0.0.0-20220703174648-d70429a8dad7/go.mod h1:eWer4rx9P8lJo2eKc+Q7AZ1dE1x1hJNdkbDFPzMu1Hw=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=

50
internal/server/api_repo.go
View File

@@ -251,25 +251,59 @@ func handleRepoSetDescription(ctx context.Context, rc requestContext) (interface
func handleRepoSupportedAlgorithms(ctx context.Context, rc requestContext) (interface{}, *apiError) {
res := &serverapi.SupportedAlgorithmsResponse{
DefaultHashAlgorithm: hashing.DefaultAlgorithm,
HashAlgorithms: hashing.SupportedAlgorithms(),
DefaultHashAlgorithm: hashing.DefaultAlgorithm,
SupportedHashAlgorithms: toAlgorithmInfo(hashing.SupportedAlgorithms(), neverDeprecated),
DefaultEncryptionAlgorithm: encryption.DefaultAlgorithm,
EncryptionAlgorithms: encryption.SupportedAlgorithms(false),
DefaultEncryptionAlgorithm: encryption.DefaultAlgorithm,
SupportedEncryptionAlgorithms: toAlgorithmInfo(encryption.SupportedAlgorithms(false), neverDeprecated),
DefaultSplitterAlgorithm: splitter.DefaultAlgorithm,
SplitterAlgorithms: splitter.SupportedAlgorithms(),
DefaultSplitterAlgorithm: splitter.DefaultAlgorithm,
SupportedSplitterAlgorithms: toAlgorithmInfo(splitter.SupportedAlgorithms(), neverDeprecated),
}
for k := range compression.ByName {
res.CompressionAlgorithms = append(res.CompressionAlgorithms, string(k))
res.SupportedCompressionAlgorithms = append(res.SupportedCompressionAlgorithms, serverapi.AlgorithmInfo{
ID: string(k),
Deprecated: compression.IsDeprecated[k],
})
}
sort.Strings(res.CompressionAlgorithms)
sortAlgorithms(res.SupportedHashAlgorithms)
sortAlgorithms(res.SupportedEncryptionAlgorithms)
sortAlgorithms(res.SupportedCompressionAlgorithms)
sortAlgorithms(res.SupportedSplitterAlgorithms)
return res, nil
}
func neverDeprecated(n string) bool {
return false
}
func toAlgorithmInfo(names []string, isDeprecated func(id string) bool) []serverapi.AlgorithmInfo {
var result []serverapi.AlgorithmInfo
for _, n := range names {
result = append(result, serverapi.AlgorithmInfo{
ID: n,
Deprecated: isDeprecated(n),
})
}
return result
}
func sortAlgorithms(a []serverapi.AlgorithmInfo) {
sort.Slice(a, func(i, j int) bool {
if l, r := a[i].Deprecated, a[j].Deprecated; l != r {
// non-deprecated first
return !l
}
return a[i].ID < a[j].ID
})
}
func handleRepoGetThrottle(ctx context.Context, rc requestContext) (interface{}, *apiError) {
dr, ok := rc.rep.(repo.DirectRepository)
if !ok {

21
internal/serverapi/serverapi.go
View File

@@ -128,15 +128,22 @@ type ConnectRepositoryRequest struct {
SyncWaitTimeSeconds int `json:"syncWaitTime"` // if non-zero, force particular wait time, negative == forever
}
// AlgorithmInfo is an information about a single algorithm.
type AlgorithmInfo struct {
ID string `json:"id"`
Deprecated bool `json:"deprecated"`
}
// SupportedAlgorithmsResponse returns the list of supported algorithms for repository creation.
type SupportedAlgorithmsResponse struct {
DefaultHashAlgorithm string `json:"defaultHash"`
DefaultEncryptionAlgorithm string `json:"defaultEncryption"`
DefaultSplitterAlgorithm string `json:"defaultSplitter"`
HashAlgorithms []string `json:"hash"`
EncryptionAlgorithms []string `json:"encryption"`
SplitterAlgorithms []string `json:"splitter"`
CompressionAlgorithms []string `json:"compression"`
DefaultHashAlgorithm string `json:"defaultHash"`
DefaultEncryptionAlgorithm string `json:"defaultEncryption"`
DefaultSplitterAlgorithm string `json:"defaultSplitter"`
SupportedHashAlgorithms []AlgorithmInfo `json:"hash"`
SupportedEncryptionAlgorithms []AlgorithmInfo `json:"encryption"`
SupportedSplitterAlgorithms []AlgorithmInfo `json:"splitter"`
SupportedCompressionAlgorithms []AlgorithmInfo `json:"compression"`
}
// CreateSnapshotSourceRequest contains request to create snapshot source and optionally create first snapshot.

8
repo/compression/compressor.go
View File

@@ -28,6 +28,7 @@ type Compressor interface {
ByHeaderID = map[HeaderID]Compressor{}
ByName = map[Name]Compressor{}
HeaderIDToName = map[HeaderID]Name{}
IsDeprecated = map[Name]bool{}
)
// RegisterCompressor registers the provided compressor implementation.
@@ -45,6 +46,13 @@ func RegisterCompressor(name Name, c Compressor) {
HeaderIDToName[c.HeaderID()] = name
}
// RegisterDeprecatedCompressor registers the provided compressor implementation.
func RegisterDeprecatedCompressor(name Name, c Compressor) {
RegisterCompressor(name, c)
IsDeprecated[name] = true
}
func compressionHeader(id HeaderID) []byte {
b := make([]byte, compressionHeaderSize)
binary.BigEndian.PutUint32(b, uint32(id))

2
repo/compression/compressor_lz4.go
View File

@@ -11,7 +11,7 @@
)
func init() {
RegisterCompressor("lz4", newLZ4Compressor(headerLZ4Default))
RegisterDeprecatedCompressor("lz4", newLZ4Compressor(headerLZ4Default))
}
func newLZ4Compressor(id HeaderID) Compressor {

2
repo/compression/compressor_zstd.go
View File

@@ -14,7 +14,7 @@ func init() {
RegisterCompressor("zstd", newZstdCompressor(HeaderZstdDefault, zstd.SpeedDefault))
RegisterCompressor("zstd-fastest", newZstdCompressor(HeaderZstdFastest, zstd.SpeedFastest))
RegisterCompressor("zstd-better-compression", newZstdCompressor(HeaderZstdBetterCompression, zstd.SpeedBetterCompression))
RegisterCompressor("zstd-best-compression", newZstdCompressor(HeaderZstdBestCompression, zstd.SpeedBestCompression))
RegisterDeprecatedCompressor("zstd-best-compression", newZstdCompressor(HeaderZstdBestCompression, zstd.SpeedBestCompression))
}
func newZstdCompressor(id HeaderID, level zstd.EncoderLevel) Compressor {