mirror of
https://github.com/kopia/kopia.git
synced 2026-01-06 05:27:59 -05:00
* refactor(repository): added fs.DirectoryIterator This significantly reduces number of small allocations while taking snapshots of lots of files, which leads to faster snapshots. ``` $ runbench --kopia-exe ~/go/bin/kopia \ --compare-to-exe ~/go/bin/kopia-baseline --min-duration 30s \ ./snapshot-linux-parallel-4.sh DIFF duration: current:5.1 baseline:5.8 change:-13.0 % DIFF repo_size: current:1081614127.6 baseline:1081615302.8 change:-0.0 % DIFF num_files: current:60.0 baseline:60.0 change:0% DIFF avg_heap_objects: current:4802666.0 baseline:4905741.8 change:-2.1 % DIFF avg_heap_bytes: current:737397275.2 baseline:715263289.6 change:+3.1 % DIFF avg_ram: current:215.0 baseline:211.5 change:+1.6 % DIFF max_ram: current:294.8 baseline:311.4 change:-5.3 % DIFF avg_cpu: current:167.3 baseline:145.3 change:+15.1 % DIFF max_cpu: current:227.2 baseline:251.0 change:-9.5 % ``` * changed `Next()` API * mechanical move of the iterator to its own file * clarified comment * pr feedback * mechanical move of all localfs dependencies on os.FileInfo to a separate file * Update fs/entry.go Co-authored-by: ashmrtn <3891298+ashmrtn@users.noreply.github.com> * Update fs/entry_dir_iterator.go Co-authored-by: Julio Lopez <1953782+julio-lopez@users.noreply.github.com> * doc: clarified valid results from Next() --------- Co-authored-by: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Co-authored-by: Julio Lopez <1953782+julio-lopez@users.noreply.github.com>
154 lines
3.6 KiB
Go
154 lines
3.6 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
"github.com/kopia/kopia/fs"
|
|
"github.com/kopia/kopia/repo"
|
|
"github.com/kopia/kopia/repo/object"
|
|
"github.com/kopia/kopia/snapshot/snapshotfs"
|
|
)
|
|
|
|
type commandList struct {
|
|
long bool
|
|
recursive bool
|
|
showOID bool
|
|
errorSummary bool
|
|
path string
|
|
|
|
out textOutput
|
|
}
|
|
|
|
func (c *commandList) setup(svc appServices, parent commandParent) {
|
|
cmd := parent.Command("list", "List a directory stored in repository object.").Alias("ls")
|
|
|
|
cmd.Flag("long", "Long output").Short('l').BoolVar(&c.long)
|
|
cmd.Flag("recursive", "Recursive output").Short('r').BoolVar(&c.recursive)
|
|
cmd.Flag("show-object-id", "Show object IDs").Short('o').BoolVar(&c.showOID)
|
|
cmd.Flag("error-summary", "Emit error summary").Default("true").BoolVar(&c.errorSummary)
|
|
cmd.Arg("object-path", "Path").Required().StringVar(&c.path)
|
|
cmd.Action(svc.repositoryReaderAction(c.run))
|
|
|
|
c.out.setup(svc)
|
|
}
|
|
|
|
func (c *commandList) run(ctx context.Context, rep repo.Repository) error {
|
|
dir, err := snapshotfs.FilesystemDirectoryFromIDWithPath(ctx, rep, c.path, false)
|
|
if err != nil {
|
|
return errors.Wrap(err, "unable to get filesystem directory entry")
|
|
}
|
|
|
|
var prefix string
|
|
if !c.long {
|
|
prefix = c.path
|
|
if !strings.HasSuffix(prefix, "/") {
|
|
prefix += "/"
|
|
}
|
|
}
|
|
|
|
return c.listDirectory(ctx, dir, prefix, "")
|
|
}
|
|
|
|
func (c *commandList) listDirectory(ctx context.Context, d fs.Directory, prefix, indent string) error {
|
|
iter, err := d.Iterate(ctx)
|
|
if err != nil {
|
|
return err //nolint:wrapcheck
|
|
}
|
|
defer iter.Close()
|
|
|
|
e, err := iter.Next(ctx)
|
|
for e != nil {
|
|
if err2 := c.printDirectoryEntry(ctx, e, prefix, indent); err2 != nil {
|
|
return err2
|
|
}
|
|
|
|
e, err = iter.Next(ctx)
|
|
}
|
|
|
|
if err != nil {
|
|
return err //nolint:wrapcheck
|
|
}
|
|
|
|
if dws, ok := d.(fs.DirectoryWithSummary); ok && c.errorSummary {
|
|
if ds, _ := dws.Summary(ctx); ds != nil && ds.FatalErrorCount > 0 {
|
|
errorColor.Fprintf(c.out.stderr(), "\nNOTE: Encountered %v errors while snapshotting this directory:\n\n", ds.FatalErrorCount) //nolint:errcheck
|
|
|
|
for _, e := range ds.FailedEntries {
|
|
errorColor.Fprintf(c.out.stderr(), "- Error in \"%v\": %v\n", e.EntryPath, e.Error) //nolint:errcheck
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *commandList) printDirectoryEntry(ctx context.Context, e fs.Entry, prefix, indent string) error {
|
|
hoid, ok := e.(object.HasObjectID)
|
|
if !ok {
|
|
return errors.Errorf("entry without object ID")
|
|
}
|
|
|
|
objectID := hoid.ObjectID()
|
|
oid := objectID.String()
|
|
col := defaultColor
|
|
|
|
var (
|
|
errorSummary string
|
|
info string
|
|
)
|
|
|
|
if dws, ok := e.(fs.DirectoryWithSummary); ok && c.errorSummary {
|
|
if ds, _ := dws.Summary(ctx); ds != nil && ds.FatalErrorCount > 0 {
|
|
errorSummary = fmt.Sprintf(" (%v errors)", ds.FatalErrorCount)
|
|
col = errorColor
|
|
}
|
|
}
|
|
|
|
switch {
|
|
case c.long:
|
|
info = fmt.Sprintf(
|
|
"%v %12d %v %-34v %v%v",
|
|
e.Mode(),
|
|
e.Size(),
|
|
formatTimestamp(e.ModTime().Local()),
|
|
oid,
|
|
c.nameToDisplay(prefix, e),
|
|
errorSummary,
|
|
)
|
|
case c.showOID:
|
|
info = fmt.Sprintf("%-34v %v%v", oid, c.nameToDisplay(prefix, e), errorSummary)
|
|
|
|
default:
|
|
info = fmt.Sprintf("%v%v", c.nameToDisplay(prefix, e), errorSummary)
|
|
}
|
|
|
|
col.Fprintln(c.out.stdout(), info) //nolint:errcheck
|
|
|
|
if c.recursive {
|
|
if subdir, ok := e.(fs.Directory); ok {
|
|
if listerr := c.listDirectory(ctx, subdir, prefix+e.Name()+"/", indent+" "); listerr != nil {
|
|
return listerr
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *commandList) nameToDisplay(prefix string, e fs.Entry) string {
|
|
suffix := ""
|
|
if e.IsDir() {
|
|
suffix = "/"
|
|
}
|
|
|
|
if c.long || c.recursive {
|
|
return prefix + e.Name() + suffix
|
|
}
|
|
|
|
return e.Name()
|
|
}
|