diff --git a/cli/command_repository_connect.go b/cli/command_repository_connect.go index c0f2f4324..7e03fbde5 100644 --- a/cli/command_repository_connect.go +++ b/cli/command_repository_connect.go @@ -22,6 +22,7 @@ connectMaxListCacheDuration time.Duration connectHostname string connectUsername string + connectCheckForUpdates bool ) func setupConnectOptions(cmd *kingpin.CmdClause) { @@ -34,6 +35,7 @@ func setupConnectOptions(cmd *kingpin.CmdClause) { cmd.Flag("max-list-cache-duration", "Duration of index cache").Default("600s").Hidden().DurationVar(&connectMaxListCacheDuration) cmd.Flag("override-hostname", "Override hostname used by this repository connection").Hidden().StringVar(&connectHostname) cmd.Flag("override-username", "Override username used by this repository connection").Hidden().StringVar(&connectUsername) + cmd.Flag("check-for-updates", "Periodically check for Kopia updates on GitHub").Default("true").Envar(checkForUpdatesEnvar).BoolVar(&connectCheckForUpdates) } func connectOptions() *repo.ConnectOptions { @@ -69,6 +71,7 @@ func runConnectCommandWithStorageAndPassword(ctx context.Context, st blob.Storag } printStderr("Connected to repository.\n") + maybeInitializeUpdateCheck(ctx) return nil } diff --git a/cli/command_repository_disconnect.go b/cli/command_repository_disconnect.go index b0e316418..13a4ab8d9 100644 --- a/cli/command_repository_disconnect.go +++ b/cli/command_repository_disconnect.go @@ -15,5 +15,7 @@ func init() { } func runDisconnectCommand(ctx context.Context) error { + removeUpdateState() + return repo.Disconnect(ctx, repositoryConfigFileName()) } diff --git a/cli/config.go b/cli/config.go index 59bcc9dca..8313c3cf8 100644 --- a/cli/config.go +++ b/cli/config.go @@ -63,6 +63,8 @@ func openRepository(ctx context.Context, opts *repo.Options, required bool) (*re return nil, nil } + maybePrintUpdateNotification(ctx) + pass, err := getPasswordFromFlags(ctx, false, true) if err != nil { return nil, errors.Wrap(err, "get password") diff --git a/cli/update_check.go b/cli/update_check.go new file mode 100644 index 000000000..6e5674cad --- /dev/null +++ b/cli/update_check.go @@ -0,0 +1,225 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/fatih/color" + "github.com/natefinch/atomic" + "github.com/pkg/errors" + "golang.org/x/mod/semver" + + "github.com/kopia/kopia/repo" +) + +const checkForUpdatesEnvar = "KOPIA_CHECK_FOR_UPDATES" + +// hidden flags to control auto-update behavior +var ( + initialUpdateCheckDelay = app.Flag("initial-update-check-delay", "Initial delay before first time update check").Default("24h").Hidden().Envar("KOPIA_INITIAL_UPDATE_CHECK_DELAY").Duration() + updateCheckInterval = app.Flag("update-check-interval", "Interval between update checks").Default("168h").Hidden().Envar("KOPIA_UPDATE_CHECK_INTERVAL").Duration() + updateAvailableNotifyInterval = app.Flag("update-available-notify-interval", "Interval between update notifications").Default("1h").Hidden().Envar("KOPIA_UPDATE_NOTIFY_INTERVAL").Duration() +) + +const ( + latestReleaseGitHubURL = "https://api.github.com/repos/kopia/kopia/releases/latest" + checksumsURL = "https://github.com/kopia/kopia/releases/download/%v/checksums.txt.sig" + autoUpdateNotice = ` +NOTICE: Kopia will check for updates on GitHub every 7 days, starting 24 hours after first use. +To disable this behavior, set environment variable ` + checkForUpdatesEnvar + `=false +Alternatively you can remove the file "%v". + +` + updateAvailableNotice = ` +Upgrade of Kopia from %v to %v is available. +Visit https://github.com/kopia/kopia/releases/latest to download it. + +` +) + +var noticeColor = color.New(color.FgCyan) + +// updateState is persisted in a JSON file and used to determine when to check for updates +// and whether to notify user about updates. +type updateState struct { + NextCheckTime time.Time `json:"nextCheckTimestamp"` + NextNotifyTime time.Time `json:"nextNotifyTimestamp"` + AvailableVersion string `json:"availableVersion"` +} + +// updateStateFilename returns the name of the update state. +func updateStateFilename() string { + return filepath.Join(repositoryConfigFileName() + ".update-info.json") +} + +// writeUpdateState writes update state file. +func writeUpdateState(us *updateState) error { + var buf bytes.Buffer + + if err := json.NewEncoder(&buf).Encode(us); err != nil { + return errors.Wrap(err, "unable to marshal JSON") + } + + return atomic.WriteFile(updateStateFilename(), &buf) +} + +func removeUpdateState() { + os.Remove(updateStateFilename()) // nolint:errcheck +} + +// getUpdateState reads the update state file if available. +func getUpdateState() (*updateState, error) { + f, err := os.Open(updateStateFilename()) + if err != nil { + return nil, errors.Wrap(err, "unable to open update state file") + } + defer f.Close() //nolint:errcheck + + us := &updateState{} + if err := json.NewDecoder(f).Decode(us); err != nil { + return nil, errors.Wrap(err, "unable to parse update state") + } + + return us, nil +} + +// maybeInitializeUpdateCheck optionally writes update state file with initial update +// set 24 hours from now. +func maybeInitializeUpdateCheck(ctx context.Context) { + if connectCheckForUpdates { + us := &updateState{ + NextCheckTime: time.Now().Add(*initialUpdateCheckDelay), + NextNotifyTime: time.Now().Add(*initialUpdateCheckDelay), + } + if err := writeUpdateState(us); err != nil { + log(ctx).Debugf("error initializing update state") + return + } + + noticeColor.Fprintf(os.Stderr, autoUpdateNotice, updateStateFilename()) //nolint:errcheck + } else { + removeUpdateState() + } +} + +// getLatestReleaseNameFromGitHub gets the name of the release marked 'latest' on GitHub. +func getLatestReleaseNameFromGitHub() (string, error) { + resp, err := http.DefaultClient.Get(latestReleaseGitHubURL) + if err != nil { + return "", errors.Wrap(err, "unable to get latest release from github") + } + defer resp.Body.Close() //nolint:errcheck + + if resp.StatusCode != http.StatusOK { + return "", errors.Errorf("invalid status code from GitHub: %v", resp.StatusCode) + } + + var responseObject struct { + Name string `json:"name"` + } + + if err := json.NewDecoder(resp.Body).Decode(&responseObject); err != nil { + return "", errors.Wrap(err, "invalid Github API response") + } + + return responseObject.Name, nil +} + +// verifyGitHubReleaseIsComplete downloads checksum file to verify that the release is complete. +func verifyGitHubReleaseIsComplete(releaseName string) error { + u := fmt.Sprintf(checksumsURL, releaseName) + + resp, err := http.DefaultClient.Get(u) + if err != nil { + return errors.Wrap(err, "unable to download releases checksum") + } + + defer resp.Body.Close() //nolint:errcheck + + if resp.StatusCode != http.StatusOK { + return errors.Errorf("invalid status code from GitHub: %v", resp.StatusCode) + } + + return nil +} + +func maybeCheckForUpdates() (string, error) { + if v := os.Getenv(checkForUpdatesEnvar); v != "" { + // see if environment variable is set to false. + if b, err := strconv.ParseBool(v); err == nil && !b { + return "", errors.Errorf("update check disabled") + } + } + + us, err := getUpdateState() + if err != nil { + return "", err + } + + if time.Now().After(us.NextCheckTime) { + // before we check for update, write update state file again, so if this fails + // we won't bother GitHub for a while + us.NextCheckTime = time.Now().Add(*updateCheckInterval) + if err = writeUpdateState(us); err != nil { + return "", errors.Wrap(err, "unable to write update state") + } + + newAvailableVersion, err := getLatestReleaseNameFromGitHub() + if err != nil { + return "", errors.Wrap(err, "update to get latest release from GitHub") + } + + // we got updated version from GitHub, write it in a state file again + if newAvailableVersion != us.AvailableVersion { + if err = verifyGitHubReleaseIsComplete(newAvailableVersion); err != nil { + return "", errors.Wrap(err, "unable to validate GitHub release") + } + + us.AvailableVersion = newAvailableVersion + + if err := writeUpdateState(us); err != nil { + return "", errors.Wrap(err, "unable to write update state") + } + } + } + + if us.AvailableVersion == "" || semver.Compare(repo.BuildVersion, us.AvailableVersion) >= 0 { + // no new version available + return "", nil + } + + if time.Now().After(us.NextNotifyTime) { + us.NextNotifyTime = time.Now().Add(*updateAvailableNotifyInterval) + if err := writeUpdateState(us); err != nil { + return "", errors.Wrap(err, "unable to write update state") + } + + return us.AvailableVersion, nil + } + + // no time to notify yet + return "", nil +} + +// maybePrintUpdateNotification prints notification about available version. +func maybePrintUpdateNotification(ctx context.Context) { + updatedVersion, err := maybeCheckForUpdates() + if err != nil { + log(ctx).Debugf("unable to check for updates: %v", err) + return + } + + if updatedVersion == "" { + log(ctx).Debugf("no updated version available") + return + } + + noticeColor.Fprintf(os.Stderr, updateAvailableNotice, repo.BuildVersion, updatedVersion) //nolint:errcheck +} diff --git a/go.mod b/go.mod index c1bc16d15..602a1e1c3 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/chmduquesne/rollinghash v4.0.0+incompatible github.com/danieljoos/wincred v1.0.2 // indirect github.com/efarrer/iothrottler v0.0.1 + github.com/fatih/color v1.7.0 github.com/godbus/dbus v4.1.0+incompatible // indirect github.com/golang/protobuf v1.3.2 github.com/google/fswalker v0.2.0 @@ -34,6 +35,7 @@ require ( gocloud.dev v0.18.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d golang.org/x/exp v0.0.0-20190829153037-c13cbed26979 + golang.org/x/mod v0.1.0 golang.org/x/net v0.0.0-20190923162816-aa69164e4478 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sync v0.0.0-20190423024810-112230192c58 diff --git a/go.sum b/go.sum index 7d529e009..3f0c84f47 100644 --- a/go.sum +++ b/go.sum @@ -95,6 +95,7 @@ github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7j github.com/efarrer/iothrottler v0.0.1 h1:N5uXoCpk8T1nfl8z7l4YIJUI/2/mL5pQNsOkeMuVnH8= github.com/efarrer/iothrottler v0.0.1/go.mod h1:zGWF5N0NKSCskcPFytDAFwI121DdU/NfW4XOjpTR+ys= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= @@ -240,6 +241,7 @@ github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb h1:hXqqXzQtJbENrsb+rsIqkVqcg4FUJL0SQFGw08Dgivw= @@ -247,6 +249,7 @@ github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb/go.mod h1:31jz6HN github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -412,6 +415,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.1.0 h1:sfUMP1Gu8qASkorDVjnMuvgJzwFbTZSeXFiGBYAVdl4= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/tests/end_to_end_test/auto_update_test.go b/tests/end_to_end_test/auto_update_test.go new file mode 100644 index 000000000..96206b89a --- /dev/null +++ b/tests/end_to_end_test/auto_update_test.go @@ -0,0 +1,100 @@ +package endtoend_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kopia/kopia/tests/testenv" +) + +func TestAutoUpdateEnableTest(t *testing.T) { + cases := []struct { + desc string + extraArgs []string + extraEnv []string + wantEnabled bool + wantInitialDelay time.Duration + }{ + {desc: "Default", wantEnabled: true, wantInitialDelay: 24 * time.Hour}, + {desc: "DisabledByFlag", extraArgs: []string{"--no-check-for-updates"}, wantEnabled: false}, + {desc: "DisabledByEnvar-false", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=false"}, wantEnabled: false}, + {desc: "DisabledByEnvar-0", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=0"}, wantEnabled: false}, + {desc: "DisabledByEnvar-f", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=f"}, wantEnabled: false}, + {desc: "DisabledByEnvar-False", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=False"}, wantEnabled: false}, + {desc: "DisabledByEnvar-FALSE", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=FALSE"}, wantEnabled: false}, + {desc: "DisabledByEnvarOverriddenByFlag", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=false"}, extraArgs: []string{"--check-for-updates"}, wantEnabled: true, wantInitialDelay: 24 * time.Hour}, + {desc: "EnabledByEnvarOverriddenByFlag", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=true"}, extraArgs: []string{"--no-check-for-updates"}, wantEnabled: false, wantInitialDelay: 24 * time.Hour}, + + {desc: "InitialUpdateCheckIntervalFlag", extraEnv: []string{"KOPIA_INITIAL_UPDATE_CHECK_DELAY=1h"}, wantEnabled: true, wantInitialDelay: 1 * time.Hour}, + {desc: "InitialUpdateCheckIntervalEnvar", extraArgs: []string{"--initial-update-check-delay=3h"}, wantEnabled: true, wantInitialDelay: 3 * time.Hour}, + } + + os.Unsetenv("KOPIA_CHECK_FOR_UPDATES") + + for _, tc := range cases { + tc := tc + + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + e := testenv.NewCLITest(t) + defer e.Cleanup(t) + + // create repo + args := append([]string{ + "repo", "create", "filesystem", "--path", e.RepoDir, + }, tc.extraArgs...) + + e.Environment = append(e.Environment, tc.extraEnv...) + e.RunAndExpectSuccess(t, args...) + + updateInfoFile := filepath.Join(e.ConfigDir, ".kopia.config.update-info.json") + _, err := os.Stat(updateInfoFile) + if got, want := err == nil, tc.wantEnabled; got != want { + t.Errorf("update check enabled: %v, wanted %v", got, want) + } + + e.RunAndExpectSuccess(t, "repo", "disconnect") + if _, err = os.Stat(updateInfoFile); !os.IsNotExist(err) { + t.Errorf("update info file was not removed.") + } + + args = append([]string{ + "repo", "connect", "filesystem", "--path", e.RepoDir, + }, tc.extraArgs...) + e.RunAndExpectSuccess(t, args...) + + // make sure connect behaves the same way as create + f, err := os.Open(updateInfoFile) + if got, want := err == nil, tc.wantEnabled; got != want { + t.Fatalf("update check enabled: %v, wanted %v", got, want) + } + if err == nil { + defer f.Close() + + var state struct { + NextCheckTime time.Time `json:"nextCheckTimestamp"` + } + + if err := json.NewDecoder(f).Decode(&state); err != nil { + t.Fatalf("invalid state JSON: %v", err) + } + + // verify that initial delay is approximately wantInitialDelay from now +/- 1 minute + if got, want := time.Now().Add(tc.wantInitialDelay), state.NextCheckTime; absDuration(got.Sub(want)) > 1*time.Minute { + t.Errorf("unexpected NextCheckTime: %v, want approx %v", got, want) + } + } + }) + } +} + +func absDuration(d time.Duration) time.Duration { + if d >= 0 { + return d + } + + return -d +} diff --git a/tests/testenv/cli_test_env.go b/tests/testenv/cli_test_env.go index 233f4d5b3..f0f17b61d 100644 --- a/tests/testenv/cli_test_env.go +++ b/tests/testenv/cli_test_env.go @@ -1,4 +1,4 @@ -// Package testenv contains environment for use in testing. +// Package testenv contains Environment for use in testing. package testenv import ( @@ -35,7 +35,7 @@ type CLITest struct { Exe string fixedArgs []string - environment []string + Environment []string } // SourceInfo reprents a single source (user@host:/path) with its snapshots. @@ -92,11 +92,11 @@ func NewCLITest(t *testing.T) *CLITest { ConfigDir: ConfigDir, Exe: filepath.FromSlash(exe), fixedArgs: fixedArgs, - environment: []string{"KOPIA_PASSWORD=" + repoPassword}, + Environment: []string{"KOPIA_PASSWORD=" + repoPassword}, } } -// Cleanup cleans up the test environment unless the test has failed. +// Cleanup cleans up the test Environment unless the test has failed. func (e *CLITest) Cleanup(t *testing.T) { if t.Failed() { t.Logf("skipped cleanup for failed test, examine repository: %v", e.RepoDir) @@ -133,7 +133,7 @@ func (e *CLITest) RunAndProcessStderr(t *testing.T, callback func(line string) b // nolint:gosec c := exec.Command(e.Exe, cmdArgs...) - c.Env = append(os.Environ(), e.environment...) + c.Env = append(os.Environ(), e.Environment...) stderrPipe, err := c.StderrPipe() if err != nil { @@ -205,7 +205,7 @@ func (e *CLITest) Run(t *testing.T, args ...string) (stdout, stderr []string, er // nolint:gosec c := exec.Command(e.Exe, cmdArgs...) - c.Env = append(os.Environ(), e.environment...) + c.Env = append(os.Environ(), e.Environment...) errOut := &bytes.Buffer{} c.Stderr = errOut