cli: implemented update check, fixes #119

2026-02-18 15:05:46 -05:00 · 2020-02-28 21:20:43 -08:00
parent 4b1e01b238
commit fb181257bf
8 changed files with 344 additions and 6 deletions
--- a/cli/command_repository_connect.go
+++ b/cli/command_repository_connect.go
@@ -22,6 +22,7 @@
 	connectMaxListCacheDuration   time.Duration
 	connectHostname               string
 	connectUsername               string
+	connectCheckForUpdates        bool
 )

 func setupConnectOptions(cmd *kingpin.CmdClause) {
@@ -34,6 +35,7 @@ func setupConnectOptions(cmd *kingpin.CmdClause) {
 	cmd.Flag("max-list-cache-duration", "Duration of index cache").Default("600s").Hidden().DurationVar(&connectMaxListCacheDuration)
 	cmd.Flag("override-hostname", "Override hostname used by this repository connection").Hidden().StringVar(&connectHostname)
 	cmd.Flag("override-username", "Override username used by this repository connection").Hidden().StringVar(&connectUsername)
+	cmd.Flag("check-for-updates", "Periodically check for Kopia updates on GitHub").Default("true").Envar(checkForUpdatesEnvar).BoolVar(&connectCheckForUpdates)
 }

 func connectOptions() *repo.ConnectOptions {
@@ -69,6 +71,7 @@ func runConnectCommandWithStorageAndPassword(ctx context.Context, st blob.Storag
 	}

 	printStderr("Connected to repository.\n")
+	maybeInitializeUpdateCheck(ctx)

 	return nil
 }
--- a/cli/command_repository_disconnect.go
+++ b/cli/command_repository_disconnect.go
@@ -15,5 +15,7 @@ func init() {
 }

 func runDisconnectCommand(ctx context.Context) error {
+	removeUpdateState()
+
 	return repo.Disconnect(ctx, repositoryConfigFileName())
 }
--- a/cli/config.go
+++ b/cli/config.go
@@ -63,6 +63,8 @@ func openRepository(ctx context.Context, opts *repo.Options, required bool) (*re
 		return nil, nil
 	}

+	maybePrintUpdateNotification(ctx)
+
 	pass, err := getPasswordFromFlags(ctx, false, true)
 	if err != nil {
 		return nil, errors.Wrap(err, "get password")
--- a/cli/update_check.go
+++ b/cli/update_check.go
@@ -0,0 +1,225 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/natefinch/atomic"
+	"github.com/pkg/errors"
+	"golang.org/x/mod/semver"
+
+	"github.com/kopia/kopia/repo"
+)
+
+const checkForUpdatesEnvar = "KOPIA_CHECK_FOR_UPDATES"
+
+// hidden flags to control auto-update behavior
+var (
+	initialUpdateCheckDelay       = app.Flag("initial-update-check-delay", "Initial delay before first time update check").Default("24h").Hidden().Envar("KOPIA_INITIAL_UPDATE_CHECK_DELAY").Duration()
+	updateCheckInterval           = app.Flag("update-check-interval", "Interval between update checks").Default("168h").Hidden().Envar("KOPIA_UPDATE_CHECK_INTERVAL").Duration()
+	updateAvailableNotifyInterval = app.Flag("update-available-notify-interval", "Interval between update notifications").Default("1h").Hidden().Envar("KOPIA_UPDATE_NOTIFY_INTERVAL").Duration()
+)
+
+const (
+	latestReleaseGitHubURL = "https://api.github.com/repos/kopia/kopia/releases/latest"
+	checksumsURL           = "https://github.com/kopia/kopia/releases/download/%v/checksums.txt.sig"
+	autoUpdateNotice       = `
+NOTICE: Kopia will check for updates on GitHub every 7 days, starting 24 hours after first use.
+To disable this behavior, set environment variable ` + checkForUpdatesEnvar + `=false
+Alternatively you can remove the file "%v".
+
+`
+	updateAvailableNotice = `
+Upgrade of Kopia from %v to %v is available.
+Visit https://github.com/kopia/kopia/releases/latest to download it.
+
+`
+)
+
+var noticeColor = color.New(color.FgCyan)
+
+// updateState is persisted in a JSON file and used to determine when to check for updates
+// and whether to notify user about updates.
+type updateState struct {
+	NextCheckTime    time.Time `json:"nextCheckTimestamp"`
+	NextNotifyTime   time.Time `json:"nextNotifyTimestamp"`
+	AvailableVersion string    `json:"availableVersion"`
+}
+
+// updateStateFilename returns the name of the update state.
+func updateStateFilename() string {
+	return filepath.Join(repositoryConfigFileName() + ".update-info.json")
+}
+
+// writeUpdateState writes update state file.
+func writeUpdateState(us *updateState) error {
+	var buf bytes.Buffer
+
+	if err := json.NewEncoder(&buf).Encode(us); err != nil {
+		return errors.Wrap(err, "unable to marshal JSON")
+	}
+
+	return atomic.WriteFile(updateStateFilename(), &buf)
+}
+
+func removeUpdateState() {
+	os.Remove(updateStateFilename()) // nolint:errcheck
+}
+
+// getUpdateState reads the update state file if available.
+func getUpdateState() (*updateState, error) {
+	f, err := os.Open(updateStateFilename())
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to open update state file")
+	}
+	defer f.Close() //nolint:errcheck
+
+	us := &updateState{}
+	if err := json.NewDecoder(f).Decode(us); err != nil {
+		return nil, errors.Wrap(err, "unable to parse update state")
+	}
+
+	return us, nil
+}
+
+// maybeInitializeUpdateCheck optionally writes update state file with initial update
+// set 24 hours from now.
+func maybeInitializeUpdateCheck(ctx context.Context) {
+	if connectCheckForUpdates {
+		us := &updateState{
+			NextCheckTime:  time.Now().Add(*initialUpdateCheckDelay),
+			NextNotifyTime: time.Now().Add(*initialUpdateCheckDelay),
+		}
+		if err := writeUpdateState(us); err != nil {
+			log(ctx).Debugf("error initializing update state")
+			return
+		}
+
+		noticeColor.Fprintf(os.Stderr, autoUpdateNotice, updateStateFilename()) //nolint:errcheck
+	} else {
+		removeUpdateState()
+	}
+}
+
+// getLatestReleaseNameFromGitHub gets the name of the release marked 'latest' on GitHub.
+func getLatestReleaseNameFromGitHub() (string, error) {
+	resp, err := http.DefaultClient.Get(latestReleaseGitHubURL)
+	if err != nil {
+		return "", errors.Wrap(err, "unable to get latest release from github")
+	}
+	defer resp.Body.Close() //nolint:errcheck
+
+	if resp.StatusCode != http.StatusOK {
+		return "", errors.Errorf("invalid status code from GitHub: %v", resp.StatusCode)
+	}
+
+	var responseObject struct {
+		Name string `json:"name"`
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(&responseObject); err != nil {
+		return "", errors.Wrap(err, "invalid Github API response")
+	}
+
+	return responseObject.Name, nil
+}
+
+// verifyGitHubReleaseIsComplete downloads checksum file to verify that the release is complete.
+func verifyGitHubReleaseIsComplete(releaseName string) error {
+	u := fmt.Sprintf(checksumsURL, releaseName)
+
+	resp, err := http.DefaultClient.Get(u)
+	if err != nil {
+		return errors.Wrap(err, "unable to download releases checksum")
+	}
+
+	defer resp.Body.Close() //nolint:errcheck
+
+	if resp.StatusCode != http.StatusOK {
+		return errors.Errorf("invalid status code from GitHub: %v", resp.StatusCode)
+	}
+
+	return nil
+}
+
+func maybeCheckForUpdates() (string, error) {
+	if v := os.Getenv(checkForUpdatesEnvar); v != "" {
+		// see if environment variable is set to false.
+		if b, err := strconv.ParseBool(v); err == nil && !b {
+			return "", errors.Errorf("update check disabled")
+		}
+	}
+
+	us, err := getUpdateState()
+	if err != nil {
+		return "", err
+	}
+
+	if time.Now().After(us.NextCheckTime) {
+		// before we check for update, write update state file again, so if this fails
+		// we won't bother GitHub for a while
+		us.NextCheckTime = time.Now().Add(*updateCheckInterval)
+		if err = writeUpdateState(us); err != nil {
+			return "", errors.Wrap(err, "unable to write update state")
+		}
+
+		newAvailableVersion, err := getLatestReleaseNameFromGitHub()
+		if err != nil {
+			return "", errors.Wrap(err, "update to get latest release from GitHub")
+		}
+
+		// we got updated version from GitHub, write it in a state file again
+		if newAvailableVersion != us.AvailableVersion {
+			if err = verifyGitHubReleaseIsComplete(newAvailableVersion); err != nil {
+				return "", errors.Wrap(err, "unable to validate GitHub release")
+			}
+
+			us.AvailableVersion = newAvailableVersion
+
+			if err := writeUpdateState(us); err != nil {
+				return "", errors.Wrap(err, "unable to write update state")
+			}
+		}
+	}
+
+	if us.AvailableVersion == "" || semver.Compare(repo.BuildVersion, us.AvailableVersion) >= 0 {
+		// no new version available
+		return "", nil
+	}
+
+	if time.Now().After(us.NextNotifyTime) {
+		us.NextNotifyTime = time.Now().Add(*updateAvailableNotifyInterval)
+		if err := writeUpdateState(us); err != nil {
+			return "", errors.Wrap(err, "unable to write update state")
+		}
+
+		return us.AvailableVersion, nil
+	}
+
+	// no time to notify yet
+	return "", nil
+}
+
+// maybePrintUpdateNotification prints notification about available version.
+func maybePrintUpdateNotification(ctx context.Context) {
+	updatedVersion, err := maybeCheckForUpdates()
+	if err != nil {
+		log(ctx).Debugf("unable to check for updates: %v", err)
+		return
+	}
+
+	if updatedVersion == "" {
+		log(ctx).Debugf("no updated version available")
+		return
+	}
+
+	noticeColor.Fprintf(os.Stderr, updateAvailableNotice, repo.BuildVersion, updatedVersion) //nolint:errcheck
+}
--- a/go.mod
+++ b/go.mod
@@ -13,6 +13,7 @@ require (
 	github.com/chmduquesne/rollinghash v4.0.0+incompatible
 	github.com/danieljoos/wincred v1.0.2 // indirect
 	github.com/efarrer/iothrottler v0.0.1
+	github.com/fatih/color v1.7.0
 	github.com/godbus/dbus v4.1.0+incompatible // indirect
 	github.com/golang/protobuf v1.3.2
 	github.com/google/fswalker v0.2.0
@@ -34,6 +35,7 @@ require (
 	gocloud.dev v0.18.0
 	golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
 	golang.org/x/exp v0.0.0-20190829153037-c13cbed26979
+	golang.org/x/mod v0.1.0
 	golang.org/x/net v0.0.0-20190923162816-aa69164e4478
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
--- a/go.sum
+++ b/go.sum
@@ -95,6 +95,7 @@ github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7j
 github.com/efarrer/iothrottler v0.0.1 h1:N5uXoCpk8T1nfl8z7l4YIJUI/2/mL5pQNsOkeMuVnH8=
 github.com/efarrer/iothrottler v0.0.1/go.mod h1:zGWF5N0NKSCskcPFytDAFwI121DdU/NfW4XOjpTR+ys=
 github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
@@ -240,6 +241,7 @@ github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb h1:hXqqXzQtJbENrsb+rsIqkVqcg4FUJL0SQFGw08Dgivw=
@@ -247,6 +249,7 @@ github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb/go.mod h1:31jz6HN
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -412,6 +415,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.1.0 h1:sfUMP1Gu8qASkorDVjnMuvgJzwFbTZSeXFiGBYAVdl4=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
--- a/tests/end_to_end_test/auto_update_test.go
+++ b/tests/end_to_end_test/auto_update_test.go
@@ -0,0 +1,100 @@
+package endtoend_test
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/kopia/kopia/tests/testenv"
+)
+
+func TestAutoUpdateEnableTest(t *testing.T) {
+	cases := []struct {
+		desc             string
+		extraArgs        []string
+		extraEnv         []string
+		wantEnabled      bool
+		wantInitialDelay time.Duration
+	}{
+		{desc: "Default", wantEnabled: true, wantInitialDelay: 24 * time.Hour},
+		{desc: "DisabledByFlag", extraArgs: []string{"--no-check-for-updates"}, wantEnabled: false},
+		{desc: "DisabledByEnvar-false", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=false"}, wantEnabled: false},
+		{desc: "DisabledByEnvar-0", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=0"}, wantEnabled: false},
+		{desc: "DisabledByEnvar-f", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=f"}, wantEnabled: false},
+		{desc: "DisabledByEnvar-False", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=False"}, wantEnabled: false},
+		{desc: "DisabledByEnvar-FALSE", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=FALSE"}, wantEnabled: false},
+		{desc: "DisabledByEnvarOverriddenByFlag", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=false"}, extraArgs: []string{"--check-for-updates"}, wantEnabled: true, wantInitialDelay: 24 * time.Hour},
+		{desc: "EnabledByEnvarOverriddenByFlag", extraEnv: []string{"KOPIA_CHECK_FOR_UPDATES=true"}, extraArgs: []string{"--no-check-for-updates"}, wantEnabled: false, wantInitialDelay: 24 * time.Hour},
+
+		{desc: "InitialUpdateCheckIntervalFlag", extraEnv: []string{"KOPIA_INITIAL_UPDATE_CHECK_DELAY=1h"}, wantEnabled: true, wantInitialDelay: 1 * time.Hour},
+		{desc: "InitialUpdateCheckIntervalEnvar", extraArgs: []string{"--initial-update-check-delay=3h"}, wantEnabled: true, wantInitialDelay: 3 * time.Hour},
+	}
+
+	os.Unsetenv("KOPIA_CHECK_FOR_UPDATES")
+
+	for _, tc := range cases {
+		tc := tc
+
+		t.Run(tc.desc, func(t *testing.T) {
+			t.Parallel()
+			e := testenv.NewCLITest(t)
+			defer e.Cleanup(t)
+
+			// create repo
+			args := append([]string{
+				"repo", "create", "filesystem", "--path", e.RepoDir,
+			}, tc.extraArgs...)
+
+			e.Environment = append(e.Environment, tc.extraEnv...)
+			e.RunAndExpectSuccess(t, args...)
+
+			updateInfoFile := filepath.Join(e.ConfigDir, ".kopia.config.update-info.json")
+			_, err := os.Stat(updateInfoFile)
+			if got, want := err == nil, tc.wantEnabled; got != want {
+				t.Errorf("update check enabled: %v, wanted %v", got, want)
+			}
+
+			e.RunAndExpectSuccess(t, "repo", "disconnect")
+			if _, err = os.Stat(updateInfoFile); !os.IsNotExist(err) {
+				t.Errorf("update info file was not removed.")
+			}
+
+			args = append([]string{
+				"repo", "connect", "filesystem", "--path", e.RepoDir,
+			}, tc.extraArgs...)
+			e.RunAndExpectSuccess(t, args...)
+
+			// make sure connect behaves the same way as create
+			f, err := os.Open(updateInfoFile)
+			if got, want := err == nil, tc.wantEnabled; got != want {
+				t.Fatalf("update check enabled: %v, wanted %v", got, want)
+			}
+			if err == nil {
+				defer f.Close()
+
+				var state struct {
+					NextCheckTime time.Time `json:"nextCheckTimestamp"`
+				}
+
+				if err := json.NewDecoder(f).Decode(&state); err != nil {
+					t.Fatalf("invalid state JSON: %v", err)
+				}
+
+				// verify that initial delay is approximately wantInitialDelay from now +/- 1 minute
+				if got, want := time.Now().Add(tc.wantInitialDelay), state.NextCheckTime; absDuration(got.Sub(want)) > 1*time.Minute {
+					t.Errorf("unexpected NextCheckTime: %v, want approx %v", got, want)
+				}
+			}
+		})
+	}
+}
+
+func absDuration(d time.Duration) time.Duration {
+	if d >= 0 {
+		return d
+	}
+
+	return -d
+}
--- a/tests/testenv/cli_test_env.go
+++ b/tests/testenv/cli_test_env.go
@@ -1,4 +1,4 @@
-// Package testenv contains environment for use in testing.
+// Package testenv contains Environment for use in testing.
 package testenv

 import (
@@ -35,7 +35,7 @@ type CLITest struct {
 	Exe       string

 	fixedArgs   []string
-	environment []string
+	Environment []string
 }

 // SourceInfo reprents a single source (user@host:/path) with its snapshots.
@@ -92,11 +92,11 @@ func NewCLITest(t *testing.T) *CLITest {
 		ConfigDir:   ConfigDir,
 		Exe:         filepath.FromSlash(exe),
 		fixedArgs:   fixedArgs,
-		environment: []string{"KOPIA_PASSWORD=" + repoPassword},
+		Environment: []string{"KOPIA_PASSWORD=" + repoPassword},
 	}
 }

-// Cleanup cleans up the test environment unless the test has failed.
+// Cleanup cleans up the test Environment unless the test has failed.
 func (e *CLITest) Cleanup(t *testing.T) {
 	if t.Failed() {
 		t.Logf("skipped cleanup for failed test, examine repository: %v", e.RepoDir)
@@ -133,7 +133,7 @@ func (e *CLITest) RunAndProcessStderr(t *testing.T, callback func(line string) b

 	// nolint:gosec
 	c := exec.Command(e.Exe, cmdArgs...)
-	c.Env = append(os.Environ(), e.environment...)
+	c.Env = append(os.Environ(), e.Environment...)

 	stderrPipe, err := c.StderrPipe()
 	if err != nil {
@@ -205,7 +205,7 @@ func (e *CLITest) Run(t *testing.T, args ...string) (stdout, stderr []string, er

 	// nolint:gosec
 	c := exec.Command(e.Exe, cmdArgs...)
-	c.Env = append(os.Environ(), e.environment...)
+	c.Env = append(os.Environ(), e.Environment...)

 	errOut := &bytes.Buffer{}
 	c.Stderr = errOut