rclone: provider improvements (#795)

switched local passwd hash to APR1 to support older rclones
install well-known version of rclone for testing

Makefile

@@ -208,13 +208,16 @@ dev-deps:
 	GO111MODULE=off go get -u github.com/newhook/go-symbols
 	GO111MODULE=off go get -u github.com/sqs/goreturns
 
-test-with-coverage:
+
+test-with-coverage: export RCLONE_EXE=$(rclone)
+test-with-coverage: $(rclone) 
 	$(GO_TEST) -count=1 -coverprofile=tmp.cov --coverpkg $(COVERAGE_PACKAGES) -timeout 300s $(shell go list ./...)
 
 test-with-coverage-pkgonly:
 	$(GO_TEST) -count=1 -coverprofile=tmp.cov -timeout 300s github.com/kopia/kopia/...
 
-test: $(gotestsum)
+test: export RCLONE_EXE=$(rclone)
+test: $(gotestsum) $(rclone)
 	$(GO_TEST) -count=1 -timeout $(UNIT_TESTS_TIMEOUT) ./...
 
 vtest: $(gotestsum)

repo/blob/rclone/rclone_storage.go

@@ -97,8 +97,13 @@ func runRCloneAndWaitForServerAddress(c *exec.Cmd, startupTimeout time.Duration)
 			s := bufio.NewScanner(stderr)
 			for s.Scan() {
 				l := s.Text()
+
 				if p := strings.Index(l, "https://"); p >= 0 {
 					rcloneAddressChan <- l[p:]
+
+					// consume the remainder of stderr to avoid blocking rclone
+					go ioutil.ReadAll(stderr) //nolint:errcheck
+
 					return
 				}
 			}
@@ -165,7 +170,7 @@ func New(ctx context.Context, opt *Options) (blob.Storage, error) {
 	webdavUsername := "u" + uuid.New().String()
 	webdavPassword := "p" + uuid.New().String()
 
-	if err = htpasswd.SetPassword(temporaryHtpassword, webdavUsername, webdavPassword, htpasswd.HashBCrypt); err != nil {
+	if err = htpasswd.SetPassword(temporaryHtpassword, webdavUsername, webdavPassword, htpasswd.HashAPR1); err != nil {
 		return nil, errors.Wrap(err, "unable to write htpasswd file")
 	}
 
@@ -175,6 +180,7 @@ func New(ctx context.Context, opt *Options) (blob.Storage, error) {
 	}
 
 	arguments := append([]string{
+		"-vv",
 		"serve", "webdav", opt.RemotePath,
 		"--addr", "127.0.0.1:0", // allocate random port,
 		"--cert", temporaryCertPath,

repo/blob/rclone/rclone_storage_test.go

@@ -19,10 +19,22 @@ import (
 func TestRCloneStorage(t *testing.T) {
 	ctx := testlogging.Context(t)
 
-	if err := exec.Command("rclone", "version").Run(); err != nil {
-		t.Skip("rclone not installed")
+	rcloneExe := os.Getenv("RCLONE_EXE")
+	if rcloneExe == "" {
+		rcloneExe = "rclone"
 	}
 
+	if err := exec.Command(rcloneExe, "version").Run(); err != nil {
+		if os.Getenv("CI") == "" {
+			t.Skip("rclone not installed")
+		} else {
+			// on CI fail hard
+			t.Fatal("rclone not installed")
+		}
+	}
+
+	t.Logf("using rclone exe: %v", rcloneExe)
+
 	// directory where rclone will store files
 	dataDir, err := ioutil.TempDir("", "rclonetest")
 	if err != nil {
@@ -34,6 +46,7 @@ func TestRCloneStorage(t *testing.T) {
 	st, err := rclone.New(ctx, &rclone.Options{
 		// pass local file as remote path.
 		RemotePath: dataDir,
+		RCloneExe:  rcloneExe,
 	})
 	if err != nil {
 		t.Fatalf("unable to connect to rclone backend: %v", err)

tools/tools.mk

@@ -18,6 +18,7 @@ kopia_arch_name=amd64
 node_arch_name=x64
 goreleaser_arch_name=x86_64
 linter_arch_name=amd64
+rclone_arch_name=amd64
 
 raw_arch:=$(shell uname -m)
 ifeq ($(raw_arch),aarch64)
@@ -25,6 +26,7 @@ ifeq ($(raw_arch),aarch64)
 	node_arch_name=arm64
 	goreleaser_arch_name=arm64
 	linter_arch_name=arm64
+	rclone_arch_name=arm64
 endif
 
 ifeq ($(raw_arch),armv7l)
@@ -32,6 +34,7 @@ ifeq ($(raw_arch),armv7l)
 	node_arch_name=armv7l
 	goreleaser_arch_name=armv6
 	linter_arch_name=armv7
+	rclone_arch_name=arm
 endif
 
 # uname will be Windows, Darwin, Linux
@@ -43,6 +46,7 @@ ifeq ($(OS),Windows_NT)
 	path_separator=;
 	date_ymd := $(shell powershell -noprofile -executionpolicy bypass -Command "(Get-Date).ToString('yyyyMMdd')")
 	date_full := $(shell powershell -noprofile -executionpolicy bypass -Command "(Get-Date).datetime")
+
 ifeq ($(UNIX_SHELL_ON_WINDOWS),true)
 	mkdir=mkdir -p
 	move=mv
@@ -81,6 +85,8 @@ ifneq ($(gh_tag_tmp),$(GITHUB_REF))
 export TRAVIS_TAG=$(gh_tag_tmp)
 endif
 
+endif
+
 ifeq ($(uname),Windows)
 export TRAVIS_OS_NAME=windows
 endif
@@ -93,8 +99,6 @@ ifeq ($(uname),Darwin)
 export TRAVIS_OS_NAME=osx
 endif
 
-endif
-
 # detect REPO_OWNER
 export REPO_OWNER=unknown-repo-owner
 
@@ -139,6 +143,7 @@ NODE_VERSION=14.15.4
 HUGO_VERSION=0.74.3
 GOTESTSUM_VERSION=0.5.3
 GORELEASER_VERSION=v0.140.1
+RCLONE_VERSION=1.53.4
 
 # goveralls
 GOVERALLS_TOOL=$(TOOLS_DIR)/bin/goveralls
@@ -233,6 +238,18 @@ endif
 
 endif
 
+
+# rclone
+rclone_dir=$(TOOLS_DIR)$(slash)rclone-$(RCLONE_VERSION)
+rclone=$(rclone_dir)$(slash)rclone$(exe_suffix)
+
+$(rclone):
+	@echo Downloading RCLONE_VERSION v$(RCLONE_VERSION) to $(rclone)
+	-$(mkdir) $(TOOLS_DIR)$(slash)rclone-$(RCLONE_VERSION)
+
+	curl -LsS -o $(rclone_dir).zip https://github.com/rclone/rclone/releases/download/v$(RCLONE_VERSION)/rclone-v$(RCLONE_VERSION)-$(TRAVIS_OS_NAME)-$(kopia_arch_name).zip
+	unzip -j -q $(rclone_dir).zip -d $(rclone_dir)
+
 gotestsum=$(TOOLS_DIR)/bin/gotestsum
 
 $(gotestsum): export GO111MODULE=off