Makefile: refactored tools to be installable on Windows with only minimal deps (make/curl/unzip)

This allows full kopia and kopia-ui to be built on Windows along with
running lint and integration tests.

.travis.yml

@@ -14,7 +14,7 @@ before_install:
 - |-
     case $TRAVIS_OS_NAME in
       windows)
-        choco install --no-progress -y make
+        choco install --no-progress -y make unzip curl
         ;;
     esac
 install: make travis-setup

Makefile

@@ -31,18 +31,19 @@ clean:
 play:
 	go run cmd/playground/main.go
 
-lint: $(LINTER_TOOL)
-	# try running linter a couple of times since it is pretty flaky, especially on cold start
-	for retry in 1 2 3; do echo "Running linter, attempt #$$retry"; $(LINTER_TOOL) --deadline 180s run && s=0 && break || s=$$? && sleep 1; done; (exit $$s)
+lint: $(linter)
+	$(linter) --deadline 180s run $(linter_flags)
 
-lint-and-log: $(LINTER_TOOL)
-	$(LINTER_TOOL) --deadline 180s run | tee .linterr.txt
+lint-and-log: $(linter)
+	$(linter) --deadline 180s run $(linter_flags) | tee .linterr.txt
 
 vet:
 	go vet -all .
 
-travis-setup: travis-install-gpg-key travis-install-test-credentials
+travis-setup: travis-install-gpg-key travis-install-test-credentials all-tools
 	go mod download
+	make -C htmlui node_modules
+	make -C app node_modules
 
 website:
 	$(MAKE) -C site build
@@ -50,34 +51,33 @@ website:
 html-ui:
 	$(MAKE) -C htmlui build-html
 
-html-ui-bindata: html-ui $(BINDATA_TOOL)
-	(cd htmlui/build && $(BINDATA_TOOL) -fs -tags embedhtml -o "$(CURDIR)/internal/server/htmlui_bindata.go" -pkg server -ignore '.map' . static/css static/js static/media)
+html-ui-bindata: html-ui $(go_bindata)
+	(cd htmlui/build && $(go_bindata) -fs -tags embedhtml -o "$(CURDIR)/internal/server/htmlui_bindata.go" -pkg server -ignore '.map' . static/css static/js static/media)
 
-html-ui-bindata-fallback: $(BINDATA_TOOL)
-	(cd internal/server && $(BINDATA_TOOL) -fs -tags !embedhtml -o "$(CURDIR)/internal/server/htmlui_fallback.go" -pkg server index.html)
+html-ui-bindata-fallback: $(go_bindata)
+	(cd internal/server && $(go_bindata) -fs -tags !embedhtml -o "$(CURDIR)/internal/server/htmlui_fallback.go" -pkg server index.html)
 
 # by default build unpacked Kopia UI for current OS only
 KOPIA_UI_BUILD_TARGET=build-electron-dir
 
-ifeq ($(TRAVIS_OS_NAME),osx)
-# build and package Mac app
-KOPIA_UI_BUILD_TARGET=build-all-mac
-endif
-ifeq ($(TRAVIS_OS_NAME),linux)
-# build and package Windows and Linux app
-KOPIA_UI_BUILD_TARGET=build-all-win-linux-docker
+ifneq ($(TRAVIS_OS_NAME),)
+# on Travis, also build the setup package.
+KOPIA_UI_BUILD_TARGET=build-electron
 endif
 
 kopia-ui: goreleaser
 	$(MAKE) -C app $(KOPIA_UI_BUILD_TARGET)
 
 ifeq ($(TRAVIS_OS_NAME),windows)
-travis-release: install-noui test
+travis-release: install kopia-ui
+	$(MAKE) lint test
 	$(MAKE) integration-tests
 endif
 
 ifeq ($(TRAVIS_OS_NAME),osx)
-travis-release: lint kopia-ui test
+travis-release: install kopia-ui
+	$(MAKE) lint test
+	$(MAKE) integration-tests
 endif
 
 ifeq ($(TRAVIS_OS_NAME),linux)
@@ -105,8 +105,8 @@ ifeq ($(TRAVIS_TAG),)
 	GORELEASER_OPTIONS+=--snapshot
 endif
 
-goreleaser: $(GORELEASER_TOOL)
-	$(GORELEASER_TOOL) $(GORELEASER_OPTIONS)
+goreleaser: $(goreleaser)
+	$(goreleaser) $(GORELEASER_OPTIONS)
 
 ifeq ($(TRAVIS_PULL_REQUEST),false)
 

app/Makefile

@@ -1,33 +1,21 @@
 SHELL:=/bin/bash
 
 dev: node_modules
-	PATH=$(TOOL_PATH) $(NPM_TOOL) run dev
+	$(npm) $(npm_flags) run dev
 
 run: build-html
-	PATH=$(TOOL_PATH) $(NPM_TOOL) run start-electron-prebuilt
+	$(npm) $(npm_flags) run start-electron-prebuilt
 
 build-html: node_modules
-	PATH=$(TOOL_PATH) $(NPM_TOOL) run build-html
+	$(npm) $(npm_flags) run build-html
 
 build-electron-dir: node_modules build-html
-	PATH=$(TOOL_PATH) $(NPM_TOOL) run build-electron-dir
+	$(npm) $(npm_flags) run build-electron-dir
 
-build-all-mac: node_modules build-html
-	PATH=$(TOOL_PATH) $(NPM_TOOL) run build-electron-mac
-
-build-all-win-linux-docker:
-	docker run --rm -ti \
-        --env-file <(env | grep -E "^DEBUG|^NODE_|^ELECTRON_|^YARN_|^NPM_|^CI|^CIRCLE|^TRAVIS_TAG|^TRAVIS|^TRAVIS_REPO_|^TRAVIS_BUILD_|^TRAVIS_BRANCH|^TRAVIS_PULL_REQUEST_|^APPVEYOR_|^CSC_|^GH_|^GITHUB_|^BT_|^AWS_|^STRIP|^BUILD_") \
-		--env ELECTRON_CACHE="/root/.cache/electron" \
-		--env ELECTRON_BUILDER_CACHE="/root/.cache/electron-builder" \
-		-v $(CURDIR):/project \
-		-v $(CURDIR)/../dist:/dist \
-		-v ~/.cache/electron-docker:/root/.cache/electron \
-		-v ~/.cache/electron-builder-docker:/root/.cache/electron-builder \
-		electronuserland/builder:wine \
-		/bin/bash -c "npm install && npm run build-html && npm run build-electron-win-linux"
+build-electron: node_modules build-html
+	$(npm) $(npm_flags) run build-electron
 
 include ../tools/tools.mk
 
-node_modules: $(NPM_TOOL)
-	PATH=$(TOOL_PATH) $(NPM_TOOL) install
+node_modules: $(npm)
+	$(npm) $(npm_flags) install

app/package.json

@@ -38,6 +38,10 @@
       "buildResources": "assets",
       "output": "../dist/kopia-ui"
     },
+    "nsis": {
+      "oneClick": false,
+      "perMachine": false
+    },
     "win": {
       "extraResources": [
         {
@@ -87,11 +91,6 @@
     "eject": "react-scripts eject",
     "start-electron": "electron .",
     "build-electron": "electron-builder",
-    "build-electron-all": "electron-builder -wlm",
-    "build-electron-win": "electron-builder -w --dir",
-    "build-electron-linux": "electron-builder -l",
-    "build-electron-win-linux": "electron-builder -wl",
-    "build-electron-mac": "electron-builder -m",
     "build-electron-dir": "electron-builder --dir",
     "start-electron-prebuilt": "ELECTRON_IS_DEV=0 electron .",
     "dev": "concurrently \"react-scripts start\" \"sleep 3s; electron .\""

htmlui/Makefile

@@ -3,13 +3,13 @@ all:
 include ../tools/tools.mk
 
 dev: node_modules
-	PATH=$(TOOL_PATH) $(NPM_TOOL) run start
+	$(npm) $(npm_flags) run start
 
 build-html: node_modules
-	PATH=$(TOOL_PATH) $(NPM_TOOL) run build
+	$(npm) $(npm_flags) run build
 
-node_modules: $(NPM_TOOL)
-	PATH=$(TOOL_PATH) $(NPM_TOOL) install
+node_modules: $(npm)
+	$(npm) $(npm_flags) install
 
 clean:
 	rm -rf node_modules build

site/Makefile

@@ -6,19 +6,24 @@ all: build
 
 include ../tools/tools.mk
 
-build: gen-cli-reference-pages $(HUGO_TOOL) node_modules
-	PATH=$(TOOL_PATH) $(HUGO_TOOL)
+build: gen-cli-reference-pages $(hugo) node_modules
+	$(hugo)
 
-server: $(HUGO_TOOL)
-	PATH=$(TOOL_PATH) $(HUGO_TOOL) server --watch=$(WATCH)
+server: $(hugo)
+	$(hugo) server --watch=$(WATCH)
 
-node_modules: $(NPM_TOOL)
-	PATH=$(TOOL_PATH) $(NPM_TOOL) install
+node_modules: $(npm)
+	$(npm) $(npm_flags) install
 
 clean:
 	rm -rf public/ resources/ node_modules/ $(TOOLS_DIR)/
 
 gen-cli-reference-pages:
+ifeq ($(OS),Windows_NT)
+	-rd /q /s content\\docs\\Reference\\Command-Line\\Advanced
+	-rd /q /s content\\docs\\Reference\\Command-Line\\Common
+else
 	rm -rf content/docs/Reference/Command-Line/Advanced
 	rm -rf content/docs/Reference/Command-Line/Common
+endif
 	go run github.com/kopia/kopia/site/cli2md

tools/tools.mk

@@ -1,9 +1,39 @@
-SELF_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST))))
-TOOLS_DIR:=$(SELF_DIR)/.tools
-uname := $(shell uname -s)
+# to build on Windows you need the following basic Unix tools in your PATH installed using Chocolatey
+#
+# make
+# unzip
+# curl
+
+# uname will be Windows, Darwin, Linux
+ifeq ($(OS),Windows_NT)
+	exe_suffix := .exe
+	cmd_suffix := .cmd
+	uname := Windows
+	slash=\\
+	path_separator=;
+ifeq ($(TRAVIS_OS_NAME),windows)
+	mkdir=mkdir -p
+	move=mv
+	slash=/
+else
+	mkdir=md
+	move=move
+endif
+
+else
+	uname := $(shell uname -s)
+	exe_suffix :=
+	cmd_suffix :=
+	slash=/
+	path_separator=:
+	mkdir=mkdir -p
+endif
+
+SELF_DIR := $(subst /,$(slash),$(realpath $(dir $(lastword $(MAKEFILE_LIST)))))
+TOOLS_DIR:=$(SELF_DIR)$(slash).tools
 
 # tool versions
-GOLANGCI_LINT_VERSION=v1.23.3
+GOLANGCI_LINT_VERSION=1.23.6
 NODE_VERSION=12.13.0
 HUGO_VERSION=0.59.1
 GORELEASER_VERSION=v0.125.0
@@ -12,55 +42,103 @@ GORELEASER_VERSION=v0.125.0
 GOVERALLS_TOOL=$(TOOLS_DIR)/bin/goveralls
 
 $(GOVERALLS_TOOL):
-	mkdir -p $(TOOLS_DIR)
+	$(mkdir) $(TOOLS_DIR)
 	GO111MODULE=off GOPATH=$(TOOLS_DIR) go get github.com/mattn/goveralls
 
 # nodejs / npm
-NPM_TOOL=$(TOOLS_DIR)/nodejs/node/bin/npm
-TOOL_PATH="$(PATH):$(TOOLS_DIR)/nodejs/node/bin"
+node_base_dir=$(TOOLS_DIR)$(slash)node-$(NODE_VERSION)
+node_dir=$(node_base_dir)$(slash)node$(slash)bin
+npm=$(node_dir)$(slash)npm$(cmd_suffix)
+npm_flags=--scripts-prepend-node-path=auto
 
-$(NPM_TOOL):
-	echo SELF_DIR: $(SELF_DIR)
-	mkdir -p $(TOOLS_DIR)/nodejs
+ifneq ($(uname),Windows)
+PATH:=$(node_dir)$(path_separator)$(PATH)
+endif
+
+ifeq ($(TRAVIS_OS_NAME),windows)
+PATH:=$(node_dir)$(path_separator)$(PATH)
+endif
+
+$(npm):
+	@echo Downloading Node v$(NODE_VERSION) with NPM path $(npm)
+	$(mkdir) $(node_base_dir)$(slash)node
+
+ifeq ($(uname),Windows)
+	curl -Ls -o $(node_base_dir).zip https://nodejs.org/dist/v$(NODE_VERSION)/node-v$(NODE_VERSION)-win-x64.zip
+	unzip -q $(node_base_dir).zip -d $(node_base_dir)
+	$(move) $(node_base_dir)\\node-v$(NODE_VERSION)-win-x64 $(node_base_dir)\\node\\bin
+else
 
 ifeq ($(uname),Linux)
-	curl -LsS https://nodejs.org/dist/v$(NODE_VERSION)/node-v$(NODE_VERSION)-linux-x64.tar.gz | tar zx -C $(TOOLS_DIR)/nodejs
+	curl -LsS https://nodejs.org/dist/v$(NODE_VERSION)/node-v$(NODE_VERSION)-linux-x64.tar.gz | tar zx -C $(node_base_dir)
 else
-	curl -LsS https://nodejs.org/dist/v$(NODE_VERSION)/node-v$(NODE_VERSION)-darwin-x64.tar.gz | tar zx -C $(TOOLS_DIR)/nodejs
+	curl -LsS https://nodejs.org/dist/v$(NODE_VERSION)/node-v$(NODE_VERSION)-darwin-x64.tar.gz | tar zx -C $(node_base_dir)
+endif
+	mv $(node_base_dir)/node-v$(NODE_VERSION)*/* $(node_base_dir)/node
 endif
-	mv $(TOOLS_DIR)/nodejs/node-v$(NODE_VERSION)* $(TOOLS_DIR)/nodejs/node/
 
-BINDATA_TOOL=$(TOOLS_DIR)/bin/go-bindata
+go_bindata=$(TOOLS_DIR)$(slash)bin$(slash)go-bindata$(exe_suffix)
 
-$(BINDATA_TOOL):
-	go build -o $(BINDATA_TOOL) github.com/go-bindata/go-bindata/go-bindata
+$(go_bindata):
+	go build -o $(go_bindata) github.com/go-bindata/go-bindata/go-bindata
 
 # linter
-LINTER_TOOL=$(TOOLS_DIR)/bin/golangci-lint-$(GOLANGCI_LINT_VERSION)
+linter_dir=$(TOOLS_DIR)$(slash)golangci-lint-$(GOLANGCI_LINT_VERSION)
+linter=$(linter_dir)$(slash)golangci-lint$(exe_suffix)
+linter_flags=
 
-$(LINTER_TOOL):
-	mkdir -p $(TOOLS_DIR)
-	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(TOOLS_DIR)/bin/ $(GOLANGCI_LINT_VERSION)
-	ln -sf $(TOOLS_DIR)/bin/golangci-lint $(LINTER_TOOL)
+ifeq ($(uname),Windows)
+linter_flags=-D gofmt -D goimports
+endif
+
+$(linter):
+	@echo Downloading GolangCI-lint v$(GOLANGCI_LINT_VERSION) to $(linter)
+ifeq ($(uname),Windows)
+	-$(mkdir) $(linter_dir)
+	curl -LsS -o $(linter_dir).zip https://github.com/golangci/golangci-lint/releases/download/v$(GOLANGCI_LINT_VERSION)/golangci-lint-$(GOLANGCI_LINT_VERSION)-windows-amd64.zip
+	unzip -q $(linter_dir).zip -d $(linter_dir)
+	$(move) $(linter_dir)\golangci-lint-$(GOLANGCI_LINT_VERSION)-windows-amd64\golangci-lint.exe $(linter)
+else
+	mkdir -p $(linter_dir)
+	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(linter_dir) v$(GOLANGCI_LINT_VERSION)
+endif
 
 # hugo
-HUGO_TOOL=$(TOOLS_DIR)/hugo/hugo
+hugo_dir=$(TOOLS_DIR)$(slash)hugo-$(HUGO_VERSION)
+hugo=$(hugo_dir)$(slash)hugo$(exe_suffix)
 
-$(HUGO_TOOL):
-	mkdir -p $(TOOLS_DIR)/hugo
+$(hugo):
+	@echo Downloading Hugo v$(HUGO_VERSION) to $(hugo)
+	$(mkdir) $(TOOLS_DIR)$(slash)hugo-$(HUGO_VERSION)
+
+ifeq ($(uname),Windows)
+	curl -LsS -o $(hugo_dir).zip https://github.com/gohugoio/hugo/releases/download/v$(HUGO_VERSION)/hugo_extended_$(HUGO_VERSION)_Windows-64bit.zip
+	unzip -q $(hugo_dir).zip -d $(hugo_dir)
+else
 
 ifeq ($(uname),Linux)
-	curl -LsS https://github.com/gohugoio/hugo/releases/download/v$(HUGO_VERSION)/hugo_extended_$(HUGO_VERSION)_Linux-64bit.tar.gz | tar zxv -C $(TOOLS_DIR)/hugo
+	curl -LsS https://github.com/gohugoio/hugo/releases/download/v$(HUGO_VERSION)/hugo_extended_$(HUGO_VERSION)_Linux-64bit.tar.gz | tar zxv -C $(hugo_dir)
 else
-	curl -LsS https://github.com/gohugoio/hugo/releases/download/v$(HUGO_VERSION)/hugo_extended_$(HUGO_VERSION)_macOS-64bit.tar.gz | tar zxv -C $(TOOLS_DIR)/hugo
+	curl -LsS https://github.com/gohugoio/hugo/releases/download/v$(HUGO_VERSION)/hugo_extended_$(HUGO_VERSION)_macOS-64bit.tar.gz | tar zxv -C $(hugo_dir)
 endif
 
-# linter
-GORELEASER_TOOL=$(TOOLS_DIR)/goreleaser-$(GORELEASER_VERSION)/goreleaser
+endif
 
-$(GORELEASER_TOOL):
-	mkdir -p $(TOOLS_DIR)/goreleaser-$(GORELEASER_VERSION)
+# goreleaser
+goreleaser_dir=$(TOOLS_DIR)$(slash)goreleaser-$(GORELEASER_VERSION)
+goreleaser=$(goreleaser_dir)$(slash)goreleaser$(exe_suffix)
+
+$(goreleaser):
+	@echo Downloading GoReleaser $(GORELEASER_VERSION) to $(goreleaser)
+	-$(mkdir) $(goreleaser_dir)
+ifeq ($(uname),Windows)
+	curl -LsS -o $(goreleaser_dir).zip https://github.com/goreleaser/goreleaser/releases/download/$(GORELEASER_VERSION)/goreleaser_Windows_x86_64.zip
+	unzip -q $(goreleaser_dir).zip -d $(goreleaser_dir)
+else
 	curl -LsS https://github.com/goreleaser/goreleaser/releases/download/$(GORELEASER_VERSION)/goreleaser_$$(uname -s)_$$(uname -m).tar.gz | tar zx -C $(TOOLS_DIR)/goreleaser-$(GORELEASER_VERSION)
+endif
 
 clean-tools:
-	rm -rf $(TOOLS_DIR)
+	rm -rf $(TOOLS_DIR)
+
+all-tools: $(npm) $(goreleaser) $(linter) $(hugo) $(go_bindata)