cli: added standard --json flags to several commands (#910)

* cli: added standard --json flags to several commands

Fixes #272

* Update flag description

Co-authored-by: Julio López <julio+gh@kasten.io>
This commit is contained in:
Jarek Kowalski
2021-03-25 17:55:18 -07:00
committed by GitHub
parent dfe2e9c65e
commit 74833cefcb
19 changed files with 321 additions and 36 deletions

View File

@@ -7,23 +7,39 @@
"github.com/kopia/kopia/internal/acl"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/manifest"
)
var aclListCommand = aclCommands.Command("list", "List ACL entries").Alias("ls")
func runACLList(ctx context.Context, rep repo.Repository) error {
var jl jsonList
jl.begin()
defer jl.end()
entries, err := acl.LoadEntries(ctx, rep, nil)
if err != nil {
return errors.Wrap(err, "error loading ACL entries")
}
for _, e := range entries {
printStdout("id:%v user:%v access:%v target:%v\n", e.ManifestID, e.User, e.Access, e.Target)
if jsonOutput {
jl.emit(aclListItem{e.ManifestID, e})
} else {
printStdout("id:%v user:%v access:%v target:%v\n", e.ManifestID, e.User, e.Access, e.Target)
}
}
return nil
}
type aclListItem struct {
ID manifest.ID `json:"id"`
*acl.Entry
}
func init() {
registerJSONOutputFlags(aclListCommand)
aclListCommand.Action(repositoryReaderAction(runACLList))
}

View File

@@ -16,6 +16,11 @@
)
func runBlobList(ctx context.Context, rep repo.DirectRepository) error {
var jl jsonList
jl.begin()
defer jl.end()
return rep.BlobReader().ListBlobs(ctx, blob.ID(*blobListPrefix), func(b blob.Metadata) error {
if *blobListMaxSize != 0 && b.Length > *blobListMaxSize {
return nil
@@ -25,11 +30,16 @@ func runBlobList(ctx context.Context, rep repo.DirectRepository) error {
return nil
}
fmt.Printf("%-70v %10v %v\n", b.BlobID, b.Length, formatTimestamp(b.Timestamp))
if jsonOutput {
jl.emit(b)
} else {
fmt.Printf("%-70v %10v %v\n", b.BlobID, b.Length, formatTimestamp(b.Timestamp))
}
return nil
})
}
func init() {
registerJSONOutputFlags(blobListCommand)
blobListCommand.Action(directRepositoryReadAction(runBlobList))
}

View File

@@ -21,6 +21,11 @@
)
func runContentListCommand(ctx context.Context, rep repo.DirectRepository) error {
var jl jsonList
jl.begin()
defer jl.end()
var totalSize stats.CountSum
err := rep.ContentReader().IterateContents(
@@ -36,6 +41,11 @@ func(b content.Info) error {
totalSize.Add(int64(b.Length))
if jsonOutput {
jl.emit(b)
return nil
}
if *contentListLong {
optionalDeleted := ""
if b.Deleted {
@@ -69,6 +79,7 @@ func(b content.Info) error {
}
func init() {
registerJSONOutputFlags(contentListCommand)
contentListCommand.Action(directRepositoryReadAction(runContentListCommand))
setupContentIDRangeFlags(contentListCommand)
}

View File

@@ -18,6 +18,11 @@
)
func runListBlockIndexesAction(ctx context.Context, rep repo.DirectRepository) error {
var jl jsonList
jl.begin()
defer jl.end()
blks, err := rep.IndexBlobReader().IndexBlobs(ctx, *blockIndexListIncludeSuperseded)
if err != nil {
return errors.Wrap(err, "error listing index blobs")
@@ -39,10 +44,14 @@ func runListBlockIndexesAction(ctx context.Context, rep repo.DirectRepository) e
}
for _, b := range blks {
fmt.Printf("%-40v %10v %v %v\n", b.BlobID, b.Length, formatTimestampPrecise(b.Timestamp), b.Superseded)
if jsonOutput {
jl.emit(b)
} else {
fmt.Printf("%-40v %10v %v %v\n", b.BlobID, b.Length, formatTimestampPrecise(b.Timestamp), b.Superseded)
}
}
if *blockIndexListSummary {
if *blockIndexListSummary && !jsonOutput {
fmt.Printf("total %v indexes\n", len(blks))
}
@@ -50,5 +59,6 @@ func runListBlockIndexesAction(ctx context.Context, rep repo.DirectRepository) e
}
func init() {
registerJSONOutputFlags(blockIndexListCommand)
blockIndexListCommand.Action(directRepositoryReadAction(runListBlockIndexesAction))
}

View File

@@ -2,8 +2,6 @@
import (
"context"
"encoding/json"
"os"
"time"
"github.com/pkg/errors"
@@ -13,10 +11,7 @@
"github.com/kopia/kopia/repo/maintenance"
)
var (
maintenanceInfoCommand = maintenanceCommands.Command("info", "Display maintenance information").Alias("status")
maintenanceInfoJSON = maintenanceInfoCommand.Flag("json", "Show raw JSON data").Short('j').Bool()
)
var maintenanceInfoCommand = maintenanceCommands.Command("info", "Display maintenance information").Alias("status")
func runMaintenanceInfoCommand(ctx context.Context, rep repo.DirectRepository) error {
p, err := maintenance.GetParams(ctx, rep)
@@ -29,11 +24,8 @@ func runMaintenanceInfoCommand(ctx context.Context, rep repo.DirectRepository) e
return errors.Wrap(err, "unable to get maintenance schedule")
}
if *maintenanceInfoJSON {
e := json.NewEncoder(os.Stdout)
e.SetIndent("", " ")
e.Encode(s) //nolint:errcheck
if jsonOutput {
printStdout("%s\n", jsonBytes(s))
return nil
}
@@ -83,5 +75,6 @@ func displayCycleInfo(c *maintenance.CycleParams, t time.Time, rep repo.DirectRe
}
func init() {
registerJSONOutputFlags(maintenanceInfoCommand)
maintenanceInfoCommand.Action(directRepositoryReadAction(runMaintenanceInfoCommand))
}

View File

@@ -18,10 +18,16 @@
)
func init() {
registerJSONOutputFlags(manifestListCommand)
manifestListCommand.Action(repositoryReaderAction(listManifestItems))
}
func listManifestItems(ctx context.Context, rep repo.Repository) error {
var jl jsonList
jl.begin()
defer jl.end()
filter := map[string]string{}
for _, kv := range *manifestListFilter {
@@ -49,8 +55,12 @@ func listManifestItems(ctx context.Context, rep repo.Repository) error {
})
for _, it := range items {
t := it.Labels["type"]
fmt.Printf("%v %10v %v type:%v %v\n", it.ID, it.Length, formatTimestamp(it.ModTime.Local()), t, sortedMapValues(it.Labels))
if jsonOutput {
jl.emit(it)
} else {
t := it.Labels["type"]
fmt.Printf("%v %10v %v type:%v %v\n", it.ID, it.Length, formatTimestamp(it.ModTime.Local()), t, sortedMapValues(it.Labels))
}
}
return nil

View File

@@ -14,10 +14,16 @@
var policyListCommand = policyCommands.Command("list", "List policies.").Alias("ls")
func init() {
registerJSONOutputFlags(policyListCommand)
policyListCommand.Action(repositoryReaderAction(listPolicies))
}
func listPolicies(ctx context.Context, rep repo.Repository) error {
var jl jsonList
jl.begin()
defer jl.end()
policies, err := policy.ListPolicies(ctx, rep)
if err != nil {
return errors.Wrap(err, "error listing policies")
@@ -28,7 +34,11 @@ func listPolicies(ctx context.Context, rep repo.Repository) error {
})
for _, pol := range policies {
fmt.Println(pol.ID(), pol.Target())
if jsonOutput {
jl.emit(policy.TargetWithPolicy{ID: pol.ID(), Target: pol.Target(), Policy: pol})
} else {
fmt.Println(pol.ID(), pol.Target())
}
}
return nil

View File

@@ -17,10 +17,10 @@
policyShowCommand = policyCommands.Command("show", "Show snapshot policy.").Alias("get")
policyShowGlobal = policyShowCommand.Flag("global", "Get global policy").Bool()
policyShowTargets = policyShowCommand.Arg("target", "Target to show the policy for").Strings()
policyShowJSON = policyShowCommand.Flag("json", "Show JSON").Short('j').Bool()
)
func init() {
registerJSONOutputFlags(policyShowCommand)
policyShowCommand.Action(repositoryReaderAction(showPolicy))
}
@@ -36,8 +36,8 @@ func showPolicy(ctx context.Context, rep repo.Repository) error {
return errors.Wrapf(err, "can't get effective policy for %q", target)
}
if *policyShowJSON {
fmt.Println(effective)
if jsonOutput {
printStdout("%s\n", jsonBytes(effective))
} else {
printPolicy(effective, policies)
}

View File

@@ -262,6 +262,11 @@ func reportSnapshotStatus(ctx context.Context, manifest *snapshot.Manifest) erro
snapID := manifest.ID
if jsonOutput {
printStdout("%s\n", jsonIndentedBytes(manifest, " "))
return nil
}
log(ctx).Infof("Created%v snapshot with root %v and ID %v in %v", maybePartial, manifest.RootObjectID(), snapID, manifest.EndTime.Sub(manifest.StartTime).Truncate(time.Second))
if ds := manifest.RootEntry.DirSummary; ds != nil {
@@ -359,5 +364,6 @@ func shouldSnapshotSource(ctx context.Context, src snapshot.SourceInfo, rep repo
}
func init() {
registerJSONOutputFlags(snapshotCreateCommand)
snapshotCreateCommand.Action(repositoryWriterAction(runSnapshotCommand))
}

View File

@@ -85,6 +85,11 @@ func findManifestIDs(ctx context.Context, rep repo.Repository, source string) ([
}
func runSnapshotsCommand(ctx context.Context, rep repo.Repository) error {
var jl jsonList
jl.begin()
defer jl.end()
manifestIDs, relPath, err := findManifestIDs(ctx, rep, *snapshotListPath)
if err != nil {
return err
@@ -95,6 +100,16 @@ func runSnapshotsCommand(ctx context.Context, rep repo.Repository) error {
return errors.Wrap(err, "unable to load snapshots")
}
if jsonOutput {
for _, snapshotGroup := range snapshot.GroupBySource(manifests) {
for _, m := range snapshotGroup {
jl.emit(m)
}
}
return nil
}
return outputManifestGroups(ctx, rep, manifests, strings.Split(relPath, "/"))
}
@@ -282,5 +297,6 @@ func deltaBytes(b int64) string {
}
func init() {
registerJSONOutputFlags(snapshotListCommand)
snapshotListCommand.Action(repositoryReaderAction(runSnapshotsCommand))
}

View File

@@ -12,18 +12,28 @@
var userListCommand = userCommands.Command("list", "List users").Alias("ls")
func runUserList(ctx context.Context, rep repo.Repository) error {
var jl jsonList
jl.begin()
defer jl.end()
profiles, err := user.ListUserProfiles(ctx, rep)
if err != nil {
return errors.Wrap(err, "error listing user profiles")
}
for _, p := range profiles {
printStdout("%v\n", p.Username)
if jsonOutput {
jl.emit(p)
} else {
printStdout("%v\n", p.Username)
}
}
return nil
}
func init() {
registerJSONOutputFlags(userListCommand)
userListCommand.Action(repositoryReaderAction(runUserList))
}

117
cli/json_output.go Normal file
View File

@@ -0,0 +1,117 @@
package cli
import (
"encoding/json"
"github.com/alecthomas/kingpin"
"github.com/kopia/kopia/snapshot"
)
var (
jsonOutput = false
jsonIndent = false
jsonVerbose = false // output addnon-essential stats as part of JSON
)
func registerJSONOutputFlags(cmd *kingpin.CmdClause) {
cmd.Flag("json", "Output result in JSON format to stdout").BoolVar(&jsonOutput)
cmd.Flag("json-indent", "Output result in indented JSON format to stdout").Hidden().BoolVar(&jsonIndent)
cmd.Flag("json-verbose", "Output non-essential data (e.g. statistics) in JSON format").Hidden().BoolVar(&jsonVerbose)
}
func cleanupSnapshotManifestForJSON(v *snapshot.Manifest) interface{} {
m := *v
if !jsonVerbose {
return struct {
*snapshot.Manifest
// trick to remove 'stats' completely.
Stats string `json:"stats,omitempty"`
}{Manifest: v}
}
return &m
}
func cleanupSnapshotManifestListForJSON(manifests []*snapshot.Manifest) interface{} {
var res []interface{}
for _, m := range manifests {
res = append(res, cleanupSnapshotManifestForJSON(m))
}
return res
}
func cleanupForJSON(v interface{}) interface{} {
switch v := v.(type) {
case *snapshot.Manifest:
return cleanupSnapshotManifestForJSON(v)
case []*snapshot.Manifest:
return cleanupSnapshotManifestListForJSON(v)
default:
return v
}
}
func jsonBytes(v interface{}) []byte {
return jsonIndentedBytes(v, "")
}
func jsonIndentedBytes(v interface{}, indent string) []byte {
v = cleanupForJSON(v)
var (
b []byte
err error
)
if jsonIndent {
b, err = json.MarshalIndent(v, indent+"", indent+" ")
} else {
b, err = json.Marshal(v)
}
if err != nil {
panic("error serializing JSON, that should not happen: " + err.Error())
}
return b
}
type jsonList struct {
separator string
}
func (l *jsonList) begin() {
if jsonOutput {
printStdout("[")
if !jsonIndent {
l.separator = "\n "
}
}
}
func (l *jsonList) end() {
if jsonOutput {
if !jsonIndent {
printStdout("\n")
}
printStdout("]")
}
}
func (l *jsonList) emit(v interface{}) {
printStdout(l.separator)
printStdout("%s", jsonBytes(v))
if jsonIndent {
l.separator = ","
} else {
l.separator = ",\n "
}
}

View File

@@ -23,14 +23,14 @@ type Entry interface {
// OwnerInfo describes owner of a filesystem entry.
type OwnerInfo struct {
UserID uint32
GroupID uint32
UserID uint32 `json:"uid"`
GroupID uint32 `json:"gid"`
}
// DeviceInfo describes the device this filesystem entry is on.
type DeviceInfo struct {
Dev uint64
Rdev uint64
Dev uint64 `json:"dev"`
Rdev uint64 `json:"rdev"`
}
// Entries is a list of entries sorted by name.

View File

@@ -22,7 +22,7 @@ type Manifest struct {
StartTime time.Time `json:"startTime"`
EndTime time.Time `json:"endTime"`
Stats Stats `json:"stats"`
Stats Stats `json:"stats,omitempty"`
IncompleteReason string `json:"incomplete,omitempty"`
RootEntry *DirEntry `json:"rootEntry"`

View File

@@ -11,6 +11,13 @@
// ErrPolicyNotFound is returned when the policy is not found.
var ErrPolicyNotFound = errors.New("policy not found")
// TargetWithPolicy wraps a policy with its target and ID.
type TargetWithPolicy struct {
ID string `json:"id"`
Target snapshot.SourceInfo `json:"target"`
*Policy
}
// Policy describes snapshot policy for a single source.
type Policy struct {
Labels map[string]string `json:"-"`

View File

@@ -12,6 +12,7 @@
"github.com/kopia/kopia/internal/serverapi"
"github.com/kopia/kopia/internal/testlogging"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/tests/testenv"
)
@@ -65,8 +66,13 @@ func testAPIServerRepository(t *testing.T, serverStartArgs []string, useGRPC, al
e1.RunAndExpectSuccess(t, "repo", "connect", "filesystem", "--path", e.RepoDir, "--override-username", "not-foo", "--override-hostname", "bar")
e1.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
originalPBlobCount := len(e1.RunAndExpectSuccess(t, "blob", "list", "--prefix=p"))
originalQBlobCount := len(e1.RunAndExpectSuccess(t, "blob", "list", "--prefix=q"))
var pBlobsBefore, qBlobsBefore []blob.Metadata
mustParseJSONLines(t, e1.RunAndExpectSuccess(t, "blob", "list", "--prefix=p", "--json"), &pBlobsBefore)
mustParseJSONLines(t, e1.RunAndExpectSuccess(t, "blob", "list", "--prefix=q", "--json"), &qBlobsBefore)
originalPBlobCount := len(pBlobsBefore)
originalQBlobCount := len(qBlobsBefore)
tlsCert := filepath.Join(e.ConfigDir, "tls.cert")
tlsKey := filepath.Join(e.ConfigDir, "tls.key")

View File

@@ -3,6 +3,9 @@
import (
"testing"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/repo/manifest"
"github.com/kopia/kopia/snapshot/policy"
"github.com/kopia/kopia/tests/testenv"
)
@@ -17,12 +20,33 @@ func TestDefaultGlobalPolicy(t *testing.T) {
e.RunAndExpectSuccess(t, "policy", "show", "--global")
// verify we created global policy entry
globalPolicyBlockID := e.RunAndVerifyOutputLineCount(t, 1, "content", "ls")[0]
e.RunAndExpectSuccess(t, "content", "show", "-jz", globalPolicyBlockID)
var contents []content.Info
mustParseJSONLines(t, e.RunAndExpectSuccess(t, "content", "ls", "--json"), &contents)
if got, want := len(contents), 1; got != want {
t.Fatalf("unexpected number of contents %v, want %v", got, want)
}
globalPolicyContentID := contents[0].ID
e.RunAndExpectSuccess(t, "content", "show", "-jz", string(globalPolicyContentID))
// make sure the policy is visible in the manifest list
e.RunAndVerifyOutputLineCount(t, 1, "manifest", "list", "--filter=type:policy", "--filter=policyType:global")
var manifests []manifest.EntryMetadata
mustParseJSONLines(t, e.RunAndExpectSuccess(t, "manifest", "list", "--filter=type:policy", "--filter=policyType:global", "--json"), &manifests)
if got, want := len(manifests), 1; got != want {
t.Fatalf("unexpected number of manifests %v, want %v", got, want)
}
// make sure the policy is visible in the policy list
e.RunAndVerifyOutputLineCount(t, 1, "policy", "list")
var plist []policy.TargetWithPolicy
mustParseJSONLines(t, e.RunAndExpectSuccess(t, "policy", "list", "--json"), &plist)
if got, want := len(plist), 1; got != want {
t.Fatalf("unexpected number of policies %v, want %v", got, want)
}
}

View File

@@ -1,6 +1,7 @@
package endtoend_test
import (
"encoding/json"
"os"
"path"
"path/filepath"
@@ -15,6 +16,7 @@
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/tests/testenv"
)
@@ -39,8 +41,26 @@ func TestSnapshotCreate(t *testing.T) {
e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir2)
e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir2)
var man1, man2 snapshot.Manifest
mustParseJSONLines(t, e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir2, "--json"), &man1)
mustParseJSONLines(t, e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir2, "--json"), &man2)
if man1.RootEntry.ObjectID == "" {
t.Fatalf("missing root id")
}
if man1.RootEntry.ObjectID != man2.RootEntry.ObjectID {
t.Fatalf("unexpected difference in root objects %v vs %v", man1.RootEntry.ObjectID, man2.RootEntry.ObjectID)
}
var manifests []snapshot.Manifest
mustParseJSONLines(t, e.RunAndExpectSuccess(t, "snapshot", "list", "-a", "--json"), &manifests)
if got, want := len(manifests), 6; got != want {
t.Fatalf("unexpected number of snapshots %v want %v", got, want)
}
sources := e.ListSnapshotsAndExpectSuccess(t)
// will only list snapshots we created, not foo@foo
@@ -590,3 +610,15 @@ func createFileStructure(baseDir string, files []testFileEntry) error {
return nil
}
func mustParseJSONLines(t *testing.T, lines []string, v interface{}) {
t.Helper()
allJSON := strings.Join(lines, "\n")
dec := json.NewDecoder(strings.NewReader(allJSON))
dec.DisallowUnknownFields()
if err := dec.Decode(v); err != nil {
t.Fatalf("failed to parse JSON %v: %v", allJSON, err)
}
}

View File

@@ -9,6 +9,7 @@
"time"
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/repo/content"
"github.com/kopia/kopia/tests/testenv"
)
@@ -55,7 +56,13 @@ func TestSnapshotGC(t *testing.T) {
e.RunAndExpectSuccess(t, "snapshot", "gc")
// data block + directory block + manifest block + manifest block from manifest deletion
e.RunAndVerifyOutputLineCount(t, expectedContentCount, "content", "list")
var contentInfo []content.Info
mustParseJSONLines(t, e.RunAndExpectSuccess(t, "content", "list", "--json"), &contentInfo)
if got, want := len(contentInfo), expectedContentCount; got != want {
t.Fatalf("unexpected number of contents: %v, want %v", got, want)
}
// garbage-collect for real, but contents are too recent so won't be deleted
e.RunAndExpectSuccess(t, "snapshot", "gc", "--delete")