diff --git a/Makefile b/Makefile index 3066fd569..ac2204356 100644 --- a/Makefile +++ b/Makefile @@ -208,13 +208,16 @@ dev-deps: GO111MODULE=off go get -u github.com/newhook/go-symbols GO111MODULE=off go get -u github.com/sqs/goreturns -test-with-coverage: + +test-with-coverage: export RCLONE_EXE=$(rclone) +test-with-coverage: $(rclone) $(GO_TEST) -count=1 -coverprofile=tmp.cov --coverpkg $(COVERAGE_PACKAGES) -timeout 300s $(shell go list ./...) test-with-coverage-pkgonly: $(GO_TEST) -count=1 -coverprofile=tmp.cov -timeout 300s github.com/kopia/kopia/... -test: $(gotestsum) +test: export RCLONE_EXE=$(rclone) +test: $(gotestsum) $(rclone) $(GO_TEST) -count=1 -timeout $(UNIT_TESTS_TIMEOUT) ./... vtest: $(gotestsum) diff --git a/repo/blob/rclone/rclone_storage.go b/repo/blob/rclone/rclone_storage.go index d9b61ec78..41ab5fe5a 100644 --- a/repo/blob/rclone/rclone_storage.go +++ b/repo/blob/rclone/rclone_storage.go @@ -97,8 +97,13 @@ func runRCloneAndWaitForServerAddress(c *exec.Cmd, startupTimeout time.Duration) s := bufio.NewScanner(stderr) for s.Scan() { l := s.Text() + if p := strings.Index(l, "https://"); p >= 0 { rcloneAddressChan <- l[p:] + + // consume the remainder of stderr to avoid blocking rclone + go ioutil.ReadAll(stderr) //nolint:errcheck + return } } @@ -165,7 +170,7 @@ func New(ctx context.Context, opt *Options) (blob.Storage, error) { webdavUsername := "u" + uuid.New().String() webdavPassword := "p" + uuid.New().String() - if err = htpasswd.SetPassword(temporaryHtpassword, webdavUsername, webdavPassword, htpasswd.HashBCrypt); err != nil { + if err = htpasswd.SetPassword(temporaryHtpassword, webdavUsername, webdavPassword, htpasswd.HashAPR1); err != nil { return nil, errors.Wrap(err, "unable to write htpasswd file") } @@ -175,6 +180,7 @@ func New(ctx context.Context, opt *Options) (blob.Storage, error) { } arguments := append([]string{ + "-vv", "serve", "webdav", opt.RemotePath, "--addr", "127.0.0.1:0", // allocate random port, "--cert", temporaryCertPath, diff --git a/repo/blob/rclone/rclone_storage_test.go b/repo/blob/rclone/rclone_storage_test.go index f0d316f84..c2683634c 100644 --- a/repo/blob/rclone/rclone_storage_test.go +++ b/repo/blob/rclone/rclone_storage_test.go @@ -19,10 +19,22 @@ import ( func TestRCloneStorage(t *testing.T) { ctx := testlogging.Context(t) - if err := exec.Command("rclone", "version").Run(); err != nil { - t.Skip("rclone not installed") + rcloneExe := os.Getenv("RCLONE_EXE") + if rcloneExe == "" { + rcloneExe = "rclone" } + if err := exec.Command(rcloneExe, "version").Run(); err != nil { + if os.Getenv("CI") == "" { + t.Skip("rclone not installed") + } else { + // on CI fail hard + t.Fatal("rclone not installed") + } + } + + t.Logf("using rclone exe: %v", rcloneExe) + // directory where rclone will store files dataDir, err := ioutil.TempDir("", "rclonetest") if err != nil { @@ -34,6 +46,7 @@ func TestRCloneStorage(t *testing.T) { st, err := rclone.New(ctx, &rclone.Options{ // pass local file as remote path. RemotePath: dataDir, + RCloneExe: rcloneExe, }) if err != nil { t.Fatalf("unable to connect to rclone backend: %v", err) diff --git a/tools/tools.mk b/tools/tools.mk index eda538331..521c8ca6b 100644 --- a/tools/tools.mk +++ b/tools/tools.mk @@ -18,6 +18,7 @@ kopia_arch_name=amd64 node_arch_name=x64 goreleaser_arch_name=x86_64 linter_arch_name=amd64 +rclone_arch_name=amd64 raw_arch:=$(shell uname -m) ifeq ($(raw_arch),aarch64) @@ -25,6 +26,7 @@ ifeq ($(raw_arch),aarch64) node_arch_name=arm64 goreleaser_arch_name=arm64 linter_arch_name=arm64 + rclone_arch_name=arm64 endif ifeq ($(raw_arch),armv7l) @@ -32,6 +34,7 @@ ifeq ($(raw_arch),armv7l) node_arch_name=armv7l goreleaser_arch_name=armv6 linter_arch_name=armv7 + rclone_arch_name=arm endif # uname will be Windows, Darwin, Linux @@ -43,6 +46,7 @@ ifeq ($(OS),Windows_NT) path_separator=; date_ymd := $(shell powershell -noprofile -executionpolicy bypass -Command "(Get-Date).ToString('yyyyMMdd')") date_full := $(shell powershell -noprofile -executionpolicy bypass -Command "(Get-Date).datetime") + ifeq ($(UNIX_SHELL_ON_WINDOWS),true) mkdir=mkdir -p move=mv @@ -81,6 +85,8 @@ ifneq ($(gh_tag_tmp),$(GITHUB_REF)) export TRAVIS_TAG=$(gh_tag_tmp) endif +endif + ifeq ($(uname),Windows) export TRAVIS_OS_NAME=windows endif @@ -93,8 +99,6 @@ ifeq ($(uname),Darwin) export TRAVIS_OS_NAME=osx endif -endif - # detect REPO_OWNER export REPO_OWNER=unknown-repo-owner @@ -139,6 +143,7 @@ NODE_VERSION=14.15.4 HUGO_VERSION=0.74.3 GOTESTSUM_VERSION=0.5.3 GORELEASER_VERSION=v0.140.1 +RCLONE_VERSION=1.53.4 # goveralls GOVERALLS_TOOL=$(TOOLS_DIR)/bin/goveralls @@ -233,6 +238,18 @@ endif endif + +# rclone +rclone_dir=$(TOOLS_DIR)$(slash)rclone-$(RCLONE_VERSION) +rclone=$(rclone_dir)$(slash)rclone$(exe_suffix) + +$(rclone): + @echo Downloading RCLONE_VERSION v$(RCLONE_VERSION) to $(rclone) + -$(mkdir) $(TOOLS_DIR)$(slash)rclone-$(RCLONE_VERSION) + + curl -LsS -o $(rclone_dir).zip https://github.com/rclone/rclone/releases/download/v$(RCLONE_VERSION)/rclone-v$(RCLONE_VERSION)-$(TRAVIS_OS_NAME)-$(kopia_arch_name).zip + unzip -j -q $(rclone_dir).zip -d $(rclone_dir) + gotestsum=$(TOOLS_DIR)/bin/gotestsum $(gotestsum): export GO111MODULE=off