diff --git a/.travis.yml b/.travis.yml index 357b7242b..ece97d73e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ before_install: - |- case $TRAVIS_OS_NAME in windows) - choco install --no-progress -y make + choco install --no-progress -y make unzip curl ;; esac install: make travis-setup diff --git a/Makefile b/Makefile index 77e5910d9..cc48a54ea 100644 --- a/Makefile +++ b/Makefile @@ -31,18 +31,19 @@ clean: play: go run cmd/playground/main.go -lint: $(LINTER_TOOL) - # try running linter a couple of times since it is pretty flaky, especially on cold start - for retry in 1 2 3; do echo "Running linter, attempt #$$retry"; $(LINTER_TOOL) --deadline 180s run && s=0 && break || s=$$? && sleep 1; done; (exit $$s) +lint: $(linter) + $(linter) --deadline 180s run $(linter_flags) -lint-and-log: $(LINTER_TOOL) - $(LINTER_TOOL) --deadline 180s run | tee .linterr.txt +lint-and-log: $(linter) + $(linter) --deadline 180s run $(linter_flags) | tee .linterr.txt vet: go vet -all . -travis-setup: travis-install-gpg-key travis-install-test-credentials +travis-setup: travis-install-gpg-key travis-install-test-credentials all-tools go mod download + make -C htmlui node_modules + make -C app node_modules website: $(MAKE) -C site build @@ -50,34 +51,33 @@ website: html-ui: $(MAKE) -C htmlui build-html -html-ui-bindata: html-ui $(BINDATA_TOOL) - (cd htmlui/build && $(BINDATA_TOOL) -fs -tags embedhtml -o "$(CURDIR)/internal/server/htmlui_bindata.go" -pkg server -ignore '.map' . static/css static/js static/media) +html-ui-bindata: html-ui $(go_bindata) + (cd htmlui/build && $(go_bindata) -fs -tags embedhtml -o "$(CURDIR)/internal/server/htmlui_bindata.go" -pkg server -ignore '.map' . static/css static/js static/media) -html-ui-bindata-fallback: $(BINDATA_TOOL) - (cd internal/server && $(BINDATA_TOOL) -fs -tags !embedhtml -o "$(CURDIR)/internal/server/htmlui_fallback.go" -pkg server index.html) +html-ui-bindata-fallback: $(go_bindata) + (cd internal/server && $(go_bindata) -fs -tags !embedhtml -o "$(CURDIR)/internal/server/htmlui_fallback.go" -pkg server index.html) # by default build unpacked Kopia UI for current OS only KOPIA_UI_BUILD_TARGET=build-electron-dir -ifeq ($(TRAVIS_OS_NAME),osx) -# build and package Mac app -KOPIA_UI_BUILD_TARGET=build-all-mac -endif -ifeq ($(TRAVIS_OS_NAME),linux) -# build and package Windows and Linux app -KOPIA_UI_BUILD_TARGET=build-all-win-linux-docker +ifneq ($(TRAVIS_OS_NAME),) +# on Travis, also build the setup package. +KOPIA_UI_BUILD_TARGET=build-electron endif kopia-ui: goreleaser $(MAKE) -C app $(KOPIA_UI_BUILD_TARGET) ifeq ($(TRAVIS_OS_NAME),windows) -travis-release: install-noui test +travis-release: install kopia-ui + $(MAKE) lint test $(MAKE) integration-tests endif ifeq ($(TRAVIS_OS_NAME),osx) -travis-release: lint kopia-ui test +travis-release: install kopia-ui + $(MAKE) lint test + $(MAKE) integration-tests endif ifeq ($(TRAVIS_OS_NAME),linux) @@ -105,8 +105,8 @@ ifeq ($(TRAVIS_TAG),) GORELEASER_OPTIONS+=--snapshot endif -goreleaser: $(GORELEASER_TOOL) - $(GORELEASER_TOOL) $(GORELEASER_OPTIONS) +goreleaser: $(goreleaser) + $(goreleaser) $(GORELEASER_OPTIONS) ifeq ($(TRAVIS_PULL_REQUEST),false) diff --git a/app/Makefile b/app/Makefile index 3512316e8..afc096cb6 100644 --- a/app/Makefile +++ b/app/Makefile @@ -1,33 +1,21 @@ SHELL:=/bin/bash dev: node_modules - PATH=$(TOOL_PATH) $(NPM_TOOL) run dev + $(npm) $(npm_flags) run dev run: build-html - PATH=$(TOOL_PATH) $(NPM_TOOL) run start-electron-prebuilt + $(npm) $(npm_flags) run start-electron-prebuilt build-html: node_modules - PATH=$(TOOL_PATH) $(NPM_TOOL) run build-html + $(npm) $(npm_flags) run build-html build-electron-dir: node_modules build-html - PATH=$(TOOL_PATH) $(NPM_TOOL) run build-electron-dir + $(npm) $(npm_flags) run build-electron-dir -build-all-mac: node_modules build-html - PATH=$(TOOL_PATH) $(NPM_TOOL) run build-electron-mac - -build-all-win-linux-docker: - docker run --rm -ti \ - --env-file <(env | grep -E "^DEBUG|^NODE_|^ELECTRON_|^YARN_|^NPM_|^CI|^CIRCLE|^TRAVIS_TAG|^TRAVIS|^TRAVIS_REPO_|^TRAVIS_BUILD_|^TRAVIS_BRANCH|^TRAVIS_PULL_REQUEST_|^APPVEYOR_|^CSC_|^GH_|^GITHUB_|^BT_|^AWS_|^STRIP|^BUILD_") \ - --env ELECTRON_CACHE="/root/.cache/electron" \ - --env ELECTRON_BUILDER_CACHE="/root/.cache/electron-builder" \ - -v $(CURDIR):/project \ - -v $(CURDIR)/../dist:/dist \ - -v ~/.cache/electron-docker:/root/.cache/electron \ - -v ~/.cache/electron-builder-docker:/root/.cache/electron-builder \ - electronuserland/builder:wine \ - /bin/bash -c "npm install && npm run build-html && npm run build-electron-win-linux" +build-electron: node_modules build-html + $(npm) $(npm_flags) run build-electron include ../tools/tools.mk -node_modules: $(NPM_TOOL) - PATH=$(TOOL_PATH) $(NPM_TOOL) install +node_modules: $(npm) + $(npm) $(npm_flags) install diff --git a/app/package.json b/app/package.json index e42adb779..50f814cc3 100644 --- a/app/package.json +++ b/app/package.json @@ -38,6 +38,10 @@ "buildResources": "assets", "output": "../dist/kopia-ui" }, + "nsis": { + "oneClick": false, + "perMachine": false + }, "win": { "extraResources": [ { @@ -87,11 +91,6 @@ "eject": "react-scripts eject", "start-electron": "electron .", "build-electron": "electron-builder", - "build-electron-all": "electron-builder -wlm", - "build-electron-win": "electron-builder -w --dir", - "build-electron-linux": "electron-builder -l", - "build-electron-win-linux": "electron-builder -wl", - "build-electron-mac": "electron-builder -m", "build-electron-dir": "electron-builder --dir", "start-electron-prebuilt": "ELECTRON_IS_DEV=0 electron .", "dev": "concurrently \"react-scripts start\" \"sleep 3s; electron .\"" diff --git a/htmlui/Makefile b/htmlui/Makefile index 0d2e0b9f5..ed205a7ba 100644 --- a/htmlui/Makefile +++ b/htmlui/Makefile @@ -3,13 +3,13 @@ all: include ../tools/tools.mk dev: node_modules - PATH=$(TOOL_PATH) $(NPM_TOOL) run start + $(npm) $(npm_flags) run start build-html: node_modules - PATH=$(TOOL_PATH) $(NPM_TOOL) run build + $(npm) $(npm_flags) run build -node_modules: $(NPM_TOOL) - PATH=$(TOOL_PATH) $(NPM_TOOL) install +node_modules: $(npm) + $(npm) $(npm_flags) install clean: rm -rf node_modules build \ No newline at end of file diff --git a/site/Makefile b/site/Makefile index b76dfab1c..514ab006e 100644 --- a/site/Makefile +++ b/site/Makefile @@ -6,19 +6,24 @@ all: build include ../tools/tools.mk -build: gen-cli-reference-pages $(HUGO_TOOL) node_modules - PATH=$(TOOL_PATH) $(HUGO_TOOL) +build: gen-cli-reference-pages $(hugo) node_modules + $(hugo) -server: $(HUGO_TOOL) - PATH=$(TOOL_PATH) $(HUGO_TOOL) server --watch=$(WATCH) +server: $(hugo) + $(hugo) server --watch=$(WATCH) -node_modules: $(NPM_TOOL) - PATH=$(TOOL_PATH) $(NPM_TOOL) install +node_modules: $(npm) + $(npm) $(npm_flags) install clean: rm -rf public/ resources/ node_modules/ $(TOOLS_DIR)/ gen-cli-reference-pages: +ifeq ($(OS),Windows_NT) + -rd /q /s content\\docs\\Reference\\Command-Line\\Advanced + -rd /q /s content\\docs\\Reference\\Command-Line\\Common +else rm -rf content/docs/Reference/Command-Line/Advanced rm -rf content/docs/Reference/Command-Line/Common +endif go run github.com/kopia/kopia/site/cli2md diff --git a/tools/tools.mk b/tools/tools.mk index abc68766a..d1dbee17b 100644 --- a/tools/tools.mk +++ b/tools/tools.mk @@ -1,9 +1,39 @@ -SELF_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) -TOOLS_DIR:=$(SELF_DIR)/.tools -uname := $(shell uname -s) +# to build on Windows you need the following basic Unix tools in your PATH installed using Chocolatey +# +# make +# unzip +# curl + +# uname will be Windows, Darwin, Linux +ifeq ($(OS),Windows_NT) + exe_suffix := .exe + cmd_suffix := .cmd + uname := Windows + slash=\\ + path_separator=; +ifeq ($(TRAVIS_OS_NAME),windows) + mkdir=mkdir -p + move=mv + slash=/ +else + mkdir=md + move=move +endif + +else + uname := $(shell uname -s) + exe_suffix := + cmd_suffix := + slash=/ + path_separator=: + mkdir=mkdir -p +endif + +SELF_DIR := $(subst /,$(slash),$(realpath $(dir $(lastword $(MAKEFILE_LIST))))) +TOOLS_DIR:=$(SELF_DIR)$(slash).tools # tool versions -GOLANGCI_LINT_VERSION=v1.23.3 +GOLANGCI_LINT_VERSION=1.23.6 NODE_VERSION=12.13.0 HUGO_VERSION=0.59.1 GORELEASER_VERSION=v0.125.0 @@ -12,55 +42,103 @@ GORELEASER_VERSION=v0.125.0 GOVERALLS_TOOL=$(TOOLS_DIR)/bin/goveralls $(GOVERALLS_TOOL): - mkdir -p $(TOOLS_DIR) + $(mkdir) $(TOOLS_DIR) GO111MODULE=off GOPATH=$(TOOLS_DIR) go get github.com/mattn/goveralls # nodejs / npm -NPM_TOOL=$(TOOLS_DIR)/nodejs/node/bin/npm -TOOL_PATH="$(PATH):$(TOOLS_DIR)/nodejs/node/bin" +node_base_dir=$(TOOLS_DIR)$(slash)node-$(NODE_VERSION) +node_dir=$(node_base_dir)$(slash)node$(slash)bin +npm=$(node_dir)$(slash)npm$(cmd_suffix) +npm_flags=--scripts-prepend-node-path=auto -$(NPM_TOOL): - echo SELF_DIR: $(SELF_DIR) - mkdir -p $(TOOLS_DIR)/nodejs +ifneq ($(uname),Windows) +PATH:=$(node_dir)$(path_separator)$(PATH) +endif + +ifeq ($(TRAVIS_OS_NAME),windows) +PATH:=$(node_dir)$(path_separator)$(PATH) +endif + +$(npm): + @echo Downloading Node v$(NODE_VERSION) with NPM path $(npm) + $(mkdir) $(node_base_dir)$(slash)node + +ifeq ($(uname),Windows) + curl -Ls -o $(node_base_dir).zip https://nodejs.org/dist/v$(NODE_VERSION)/node-v$(NODE_VERSION)-win-x64.zip + unzip -q $(node_base_dir).zip -d $(node_base_dir) + $(move) $(node_base_dir)\\node-v$(NODE_VERSION)-win-x64 $(node_base_dir)\\node\\bin +else ifeq ($(uname),Linux) - curl -LsS https://nodejs.org/dist/v$(NODE_VERSION)/node-v$(NODE_VERSION)-linux-x64.tar.gz | tar zx -C $(TOOLS_DIR)/nodejs + curl -LsS https://nodejs.org/dist/v$(NODE_VERSION)/node-v$(NODE_VERSION)-linux-x64.tar.gz | tar zx -C $(node_base_dir) else - curl -LsS https://nodejs.org/dist/v$(NODE_VERSION)/node-v$(NODE_VERSION)-darwin-x64.tar.gz | tar zx -C $(TOOLS_DIR)/nodejs + curl -LsS https://nodejs.org/dist/v$(NODE_VERSION)/node-v$(NODE_VERSION)-darwin-x64.tar.gz | tar zx -C $(node_base_dir) +endif + mv $(node_base_dir)/node-v$(NODE_VERSION)*/* $(node_base_dir)/node endif - mv $(TOOLS_DIR)/nodejs/node-v$(NODE_VERSION)* $(TOOLS_DIR)/nodejs/node/ -BINDATA_TOOL=$(TOOLS_DIR)/bin/go-bindata +go_bindata=$(TOOLS_DIR)$(slash)bin$(slash)go-bindata$(exe_suffix) -$(BINDATA_TOOL): - go build -o $(BINDATA_TOOL) github.com/go-bindata/go-bindata/go-bindata +$(go_bindata): + go build -o $(go_bindata) github.com/go-bindata/go-bindata/go-bindata # linter -LINTER_TOOL=$(TOOLS_DIR)/bin/golangci-lint-$(GOLANGCI_LINT_VERSION) +linter_dir=$(TOOLS_DIR)$(slash)golangci-lint-$(GOLANGCI_LINT_VERSION) +linter=$(linter_dir)$(slash)golangci-lint$(exe_suffix) +linter_flags= -$(LINTER_TOOL): - mkdir -p $(TOOLS_DIR) - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(TOOLS_DIR)/bin/ $(GOLANGCI_LINT_VERSION) - ln -sf $(TOOLS_DIR)/bin/golangci-lint $(LINTER_TOOL) +ifeq ($(uname),Windows) +linter_flags=-D gofmt -D goimports +endif + +$(linter): + @echo Downloading GolangCI-lint v$(GOLANGCI_LINT_VERSION) to $(linter) +ifeq ($(uname),Windows) + -$(mkdir) $(linter_dir) + curl -LsS -o $(linter_dir).zip https://github.com/golangci/golangci-lint/releases/download/v$(GOLANGCI_LINT_VERSION)/golangci-lint-$(GOLANGCI_LINT_VERSION)-windows-amd64.zip + unzip -q $(linter_dir).zip -d $(linter_dir) + $(move) $(linter_dir)\golangci-lint-$(GOLANGCI_LINT_VERSION)-windows-amd64\golangci-lint.exe $(linter) +else + mkdir -p $(linter_dir) + curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(linter_dir) v$(GOLANGCI_LINT_VERSION) +endif # hugo -HUGO_TOOL=$(TOOLS_DIR)/hugo/hugo +hugo_dir=$(TOOLS_DIR)$(slash)hugo-$(HUGO_VERSION) +hugo=$(hugo_dir)$(slash)hugo$(exe_suffix) -$(HUGO_TOOL): - mkdir -p $(TOOLS_DIR)/hugo +$(hugo): + @echo Downloading Hugo v$(HUGO_VERSION) to $(hugo) + $(mkdir) $(TOOLS_DIR)$(slash)hugo-$(HUGO_VERSION) + +ifeq ($(uname),Windows) + curl -LsS -o $(hugo_dir).zip https://github.com/gohugoio/hugo/releases/download/v$(HUGO_VERSION)/hugo_extended_$(HUGO_VERSION)_Windows-64bit.zip + unzip -q $(hugo_dir).zip -d $(hugo_dir) +else ifeq ($(uname),Linux) - curl -LsS https://github.com/gohugoio/hugo/releases/download/v$(HUGO_VERSION)/hugo_extended_$(HUGO_VERSION)_Linux-64bit.tar.gz | tar zxv -C $(TOOLS_DIR)/hugo + curl -LsS https://github.com/gohugoio/hugo/releases/download/v$(HUGO_VERSION)/hugo_extended_$(HUGO_VERSION)_Linux-64bit.tar.gz | tar zxv -C $(hugo_dir) else - curl -LsS https://github.com/gohugoio/hugo/releases/download/v$(HUGO_VERSION)/hugo_extended_$(HUGO_VERSION)_macOS-64bit.tar.gz | tar zxv -C $(TOOLS_DIR)/hugo + curl -LsS https://github.com/gohugoio/hugo/releases/download/v$(HUGO_VERSION)/hugo_extended_$(HUGO_VERSION)_macOS-64bit.tar.gz | tar zxv -C $(hugo_dir) endif -# linter -GORELEASER_TOOL=$(TOOLS_DIR)/goreleaser-$(GORELEASER_VERSION)/goreleaser +endif -$(GORELEASER_TOOL): - mkdir -p $(TOOLS_DIR)/goreleaser-$(GORELEASER_VERSION) +# goreleaser +goreleaser_dir=$(TOOLS_DIR)$(slash)goreleaser-$(GORELEASER_VERSION) +goreleaser=$(goreleaser_dir)$(slash)goreleaser$(exe_suffix) + +$(goreleaser): + @echo Downloading GoReleaser $(GORELEASER_VERSION) to $(goreleaser) + -$(mkdir) $(goreleaser_dir) +ifeq ($(uname),Windows) + curl -LsS -o $(goreleaser_dir).zip https://github.com/goreleaser/goreleaser/releases/download/$(GORELEASER_VERSION)/goreleaser_Windows_x86_64.zip + unzip -q $(goreleaser_dir).zip -d $(goreleaser_dir) +else curl -LsS https://github.com/goreleaser/goreleaser/releases/download/$(GORELEASER_VERSION)/goreleaser_$$(uname -s)_$$(uname -m).tar.gz | tar zx -C $(TOOLS_DIR)/goreleaser-$(GORELEASER_VERSION) +endif clean-tools: - rm -rf $(TOOLS_DIR) \ No newline at end of file + rm -rf $(TOOLS_DIR) + +all-tools: $(npm) $(goreleaser) $(linter) $(hugo) $(go_bindata)