diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml index ed68e800d..d409378e7 100644 --- a/.github/workflows/make.yml +++ b/.github/workflows/make.yml @@ -11,86 +11,18 @@ on: # run on Mondays at 8AM - cron: '0 8 * * 1' env: + # environment variables shared between build steps + # do not include sensitive credentials and tokens here, instead pass them + # directly to tools that need them to limit the blast radius in case one of them + # becomes compromised and leaks credentials to external sites. # required by Makefile UNIX_SHELL_ON_WINDOWS: true - - # PUBLISH_BINARIES=true publishes the binaries to github - PUBLISH_BINARIES: ${{ secrets.PUBLISH_BINARIES }} - + # set to true if Publish Artifacts should run + PUBLISH_ARTIFACTS: ${{ secrets.PUBLISH_ARTIFACTS }} # where to publish releases for non-tagged commits NON_TAG_RELEASE_REPO: ${{ secrets.NON_TAG_RELEASE_REPO }} - - # encrypt various secrets stored as files - CREDENTIAL_ENCRYPTION_KEY: ${{ secrets.CREDENTIAL_ENCRYPTION_KEY }} - CREDENTIAL_ENCRYPTION_IV: ${{ secrets.CREDENTIAL_ENCRYPTION_IV }} - - # Apple ID and app-specific password for notarizaton - APPLEID: ${{ secrets.APPLEID }} - APPLEIDPASS: ${{ secrets.APPLEIDPASS }} - KOPIA_UI_NOTARIZE: ${{ secrets.KOPIA_UI_NOTARIZE }} - - # tool to install Windows signing certificate - WINDOWS_SIGNING_TOOLS_URL: ${{ secrets.WINDOWS_SIGNING_TOOLS_URL }} - WINDOWS_SIGN_USER: ${{ secrets.WINDOWS_SIGN_USER }} - WINDOWS_SIGN_AUTH: ${{ secrets.WINDOWS_SIGN_AUTH }} - WINDOWS_CERT_SHA1: ${{ secrets.WINDOWS_CERT_SHA1 }} - WINDOWS_SIGN_TOOL: ${{ secrets.WINDOWS_SIGN_TOOL }} - - # macOS signing certificate (base64-encoded), used by Electron Builder - CSC_LINK: ${{ secrets.CSC_LINK }} - CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }} - MACOS_SIGNING_IDENTITY: ${{ secrets.MACOS_SIGNING_IDENTITY }} - - # used to publish docker images - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - - # used in Azure tests - KOPIA_AZURE_TEST_CONTAINER: ${{ secrets.KOPIA_AZURE_TEST_CONTAINER }} - KOPIA_AZURE_TEST_STORAGE_ACCOUNT: ${{ secrets.KOPIA_AZURE_TEST_STORAGE_ACCOUNT }} - KOPIA_AZURE_TEST_STORAGE_KEY: ${{ secrets.KOPIA_AZURE_TEST_STORAGE_KEY }} - - # used in B2 tests - KOPIA_B2_TEST_BUCKET: ${{ secrets.KOPIA_B2_TEST_BUCKET }} - KOPIA_B2_TEST_KEY: ${{ secrets.KOPIA_B2_TEST_KEY }} - KOPIA_B2_TEST_KEY_ID: ${{ secrets.KOPIA_B2_TEST_KEY_ID }} - - # used in GCS tests - KOPIA_GCS_CREDENTIALS_FILE: ${{ secrets.KOPIA_GCS_CREDENTIALS_FILE }} - KOPIA_GCS_TEST_BUCKET: ${{ secrets.KOPIA_GCS_TEST_BUCKET }} - - # used in S3 tests - KOPIA_S3_TEST_ENDPOINT: ${{ secrets.KOPIA_S3_TEST_ENDPOINT }} - KOPIA_S3_TEST_ACCESS_KEY_ID: ${{ secrets.KOPIA_S3_TEST_ACCESS_KEY_ID }} - KOPIA_S3_TEST_SECRET_ACCESS_KEY: ${{ secrets.KOPIA_S3_TEST_SECRET_ACCESS_KEY }} - KOPIA_S3_TEST_BUCKET: ${{ secrets.KOPIA_S3_TEST_BUCKET }} - KOPIA_S3_TEST_REGION: ${{ secrets.KOPIA_S3_TEST_REGION }} - KOPIA_S3_TEST_STS_ACCESS_KEY_ID: ${{ secrets.KOPIA_S3_TEST_STS_ACCESS_KEY_ID }} - KOPIA_S3_TEST_STS_SECRET_ACCESS_KEY: ${{ secrets.KOPIA_S3_TEST_STS_SECRET_ACCESS_KEY }} - KOPIA_S3_TEST_SESSION_TOKEN: ${{ secrets.KOPIA_S3_TEST_SESSION_TOKEN }} - - KOPIA_S3_CREDS: ${{ secrets.KOPIA_S3_CREDS }} - KOPIA_S3_WASABI_CREDS: ${{ secrets.KOPIA_S3_WASABI_CREDS }} - KOPIA_S3_WASABI_VERSIONED_CREDS: ${{ secrets.KOPIA_S3_WASABI_VERSIONED_CREDS }} - - # used in rclone tests - KOPIA_RCLONE_EMBEDDED_CONFIG_B64: ${{ secrets.KOPIA_RCLONE_EMBEDDED_CONFIG_B64 }} - - # used in SFTP tests - KOPIA_SFTP_TEST_HOST: ${{ secrets.KOPIA_SFTP_TEST_HOST }} - KOPIA_SFTP_TEST_PORT: ${{ secrets.KOPIA_SFTP_TEST_PORT }} - KOPIA_SFTP_TEST_USER: ${{ secrets.KOPIA_SFTP_TEST_USER }} - KOPIA_SFTP_TEST_PATH: ${{ secrets.KOPIA_SFTP_TEST_PATH }} - KOPIA_SFTP_KEYFILE: ${{ secrets.KOPIA_SFTP_KEYFILE }} - KOPIA_SFTP_KNOWN_HOSTS_FILE: ${{ secrets.KOPIA_SFTP_KNOWN_HOSTS_FILE }} - - # used in WebDAV tests - KOPIA_WEBDAV_TEST_URL: ${{ secrets.KOPIA_WEBDAV_TEST_URL }} - KOPIA_WEBDAV_TEST_USERNAME: ${{ secrets.KOPIA_WEBDAV_TEST_USERNAME }} - KOPIA_WEBDAV_TEST_PASSWORD: ${{ secrets.KOPIA_WEBDAV_TEST_PASSWORD }} - - # Code Coverage token - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + # RPM and APT packages GCS bucket/hostname. + PACKAGES_HOST: ${{ secrets.PACKAGES_HOST }} jobs: build: strategy: @@ -125,16 +57,54 @@ jobs: fetch-depth: 0 - name: Setup run: make -j4 ci-setup + - name: Build HTML + # build HTML separately without passing any sensitive credentials to the build + # since it involves a bunch of NPM scripts. + run: make html-ui + - name: Install macOS certificates + # install signing tools and credentials for macOS and Windows outside of main + # build process. + run: make macos-certificates + env: + # macOS signing certificate (base64-encoded), used by Electron Builder + CSC_LINK: ${{ secrets.CSC_LINK }} + CSC_KEYCHAIN: ${{ secrets.CSC_KEYCHAIN }} + CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }} + MACOS_SIGNING_IDENTITY: ${{ secrets.MACOS_SIGNING_IDENTITY }} + if: ${{ contains(matrix.os, 'macos') }} + - name: Install Windows signing tools + # install signing tools and credentials for macOS and Windows outside of main + # build process. + run: make windows-signing-tools + env: + # tool to install Windows signing certificate + WINDOWS_SIGNING_TOOLS_URL: ${{ secrets.WINDOWS_SIGNING_TOOLS_URL }} + WINDOWS_SIGN_TOOL: ${{ secrets.WINDOWS_SIGN_TOOL }} + if: ${{ contains(matrix.os, 'windows') }} - name: Build run: make ci-build + env: + # Apple ID and app-specific password for notarizaton, used by Electron Builder + APPLEID: ${{ secrets.APPLEID }} + APPLEIDPASS: ${{ secrets.APPLEIDPASS }} + KOPIA_UI_NOTARIZE: ${{ secrets.KOPIA_UI_NOTARIZE }} + + # tool to install Windows signing certificate + WINDOWS_SIGN_USER: ${{ secrets.WINDOWS_SIGN_USER }} + WINDOWS_SIGN_AUTH: ${{ secrets.WINDOWS_SIGN_AUTH }} + WINDOWS_CERT_SHA1: ${{ secrets.WINDOWS_CERT_SHA1 }} + WINDOWS_SIGN_TOOL: ${{ secrets.WINDOWS_SIGN_TOOL }} + + # macOS signing certificate (base64-encoded), used by Electron Builder + MACOS_SIGNING_IDENTITY: ${{ secrets.MACOS_SIGNING_IDENTITY }} - name: Tests run: make ci-tests continue-on-error: ${{ github.event_name != 'pull_request' }} - name: Integration Tests run: make -j2 ci-integration-tests continue-on-error: ${{ github.event_name != 'pull_request' }} - - name: Publish - run: make ci-publish + - name: Publish Coverage Results + run: make ci-publish-coverage - name: Upload Kopia Artifacts uses: actions/upload-artifact@v2 with: @@ -168,7 +138,7 @@ jobs: name: Stage And Publish Artifacts runs-on: ubuntu-latest needs: build - if: ${{ github.event_name != 'pull_request' }} + if: github.event_name != 'pull_request' steps: - uses: actions/checkout@v2 - name: Set up QEMU @@ -187,15 +157,38 @@ jobs: path: dist_binaries - name: Display structure of downloaded files run: ls -lR dist/ dist_binaries/ - - name: Install CI Credentials - run: make -j4 ci-credentials + - name: Install GPG Key + run: make ci-gpg-key + env: + GPG_KEYRING: ${{secrets.GPG_KEYRING}} - name: Stage Release run: make stage-release - name: Push Github Release run: make push-github-release env: GITHUB_TOKEN: ${{secrets.GH_TOKEN}} - - name: Publish Other Packages - run: make publish-packages + - name: Install GCS Credentials + run: make ci-gcs-creds + env: + GCS_CREDENTIALS: ${{secrets.GCS_CREDENTIALS}} + - name: Publish APT + # this needs GCS credentials and GPG keys installed before. + run: make publish-apt + - name: Publish RPM + # this needs GCS credentials and GPG keys installed before. + run: make publish-rpm + - name: Publish Homebrew + # this only pushes to a GitHub repository. + run: make publish-homebrew env: GITHUB_TOKEN: ${{secrets.GH_TOKEN}} + - name: Publish Scoop + # this only pushes to a GitHub repository. + run: make publish-scoop + env: + GITHUB_TOKEN: ${{secrets.GH_TOKEN}} + - name: Publish Docker + run: make publish-docker + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/provider-tests.yml b/.github/workflows/provider-tests.yml index 7299ae511..22c0cd35e 100644 --- a/.github/workflows/provider-tests.yml +++ b/.github/workflows/provider-tests.yml @@ -16,3 +16,48 @@ jobs: fetch-depth: 0 - name: Provider Tests run: make provider-tests + env: + # used in Azure tests + KOPIA_AZURE_TEST_CONTAINER: ${{ secrets.KOPIA_AZURE_TEST_CONTAINER }} + KOPIA_AZURE_TEST_STORAGE_ACCOUNT: ${{ secrets.KOPIA_AZURE_TEST_STORAGE_ACCOUNT }} + KOPIA_AZURE_TEST_STORAGE_KEY: ${{ secrets.KOPIA_AZURE_TEST_STORAGE_KEY }} + + # used in B2 tests + KOPIA_B2_TEST_BUCKET: ${{ secrets.KOPIA_B2_TEST_BUCKET }} + KOPIA_B2_TEST_KEY: ${{ secrets.KOPIA_B2_TEST_KEY }} + KOPIA_B2_TEST_KEY_ID: ${{ secrets.KOPIA_B2_TEST_KEY_ID }} + + # used in GCS tests + KOPIA_GCS_CREDENTIALS_FILE: ${{ secrets.KOPIA_GCS_CREDENTIALS_FILE }} + KOPIA_GCS_TEST_BUCKET: ${{ secrets.KOPIA_GCS_TEST_BUCKET }} + + # used in S3 tests + KOPIA_S3_TEST_ENDPOINT: ${{ secrets.KOPIA_S3_TEST_ENDPOINT }} + KOPIA_S3_TEST_ACCESS_KEY_ID: ${{ secrets.KOPIA_S3_TEST_ACCESS_KEY_ID }} + KOPIA_S3_TEST_SECRET_ACCESS_KEY: ${{ secrets.KOPIA_S3_TEST_SECRET_ACCESS_KEY }} + KOPIA_S3_TEST_BUCKET: ${{ secrets.KOPIA_S3_TEST_BUCKET }} + KOPIA_S3_TEST_REGION: ${{ secrets.KOPIA_S3_TEST_REGION }} + KOPIA_S3_TEST_STS_ACCESS_KEY_ID: ${{ secrets.KOPIA_S3_TEST_STS_ACCESS_KEY_ID }} + KOPIA_S3_TEST_STS_SECRET_ACCESS_KEY: ${{ secrets.KOPIA_S3_TEST_STS_SECRET_ACCESS_KEY }} + KOPIA_S3_TEST_SESSION_TOKEN: ${{ secrets.KOPIA_S3_TEST_SESSION_TOKEN }} + + KOPIA_S3_CREDS: ${{ secrets.KOPIA_S3_CREDS }} + KOPIA_S3_WASABI_CREDS: ${{ secrets.KOPIA_S3_WASABI_CREDS }} + KOPIA_S3_WASABI_VERSIONED_CREDS: ${{ secrets.KOPIA_S3_WASABI_VERSIONED_CREDS }} + + # used in rclone tests + KOPIA_RCLONE_EMBEDDED_CONFIG_B64: ${{ secrets.KOPIA_RCLONE_EMBEDDED_CONFIG_B64 }} + + # used in SFTP tests + KOPIA_SFTP_TEST_HOST: ${{ secrets.KOPIA_SFTP_TEST_HOST }} + KOPIA_SFTP_TEST_PORT: ${{ secrets.KOPIA_SFTP_TEST_PORT }} + KOPIA_SFTP_TEST_USER: ${{ secrets.KOPIA_SFTP_TEST_USER }} + KOPIA_SFTP_TEST_PATH: ${{ secrets.KOPIA_SFTP_TEST_PATH }} + KOPIA_SFTP_KEYFILE: ${{ secrets.KOPIA_SFTP_KEYFILE }} + KOPIA_SFTP_KNOWN_HOSTS_FILE: ${{ secrets.KOPIA_SFTP_KNOWN_HOSTS_FILE }} + + # used in WebDAV tests + KOPIA_WEBDAV_TEST_URL: ${{ secrets.KOPIA_WEBDAV_TEST_URL }} + KOPIA_WEBDAV_TEST_USERNAME: ${{ secrets.KOPIA_WEBDAV_TEST_USERNAME }} + KOPIA_WEBDAV_TEST_PASSWORD: ${{ secrets.KOPIA_WEBDAV_TEST_PASSWORD }} + diff --git a/Makefile b/Makefile index 88fb3c926..6ed86c0e7 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ endif htmlui-node-modules: $(npm) make -C htmlui deps -ci-setup: ci-credentials go-modules all-tools htmlui-node-modules app-node-modules +ci-setup: go-modules all-tools htmlui-node-modules app-node-modules ifeq ($(CI),true) -git checkout go.mod go.sum endif @@ -102,7 +102,6 @@ htmlui/build/index.html: html-ui # on macOS build and sign AMD64, ARM64 and Universal binary and *.tar.gz files for them dist/kopia_darwin_universal/kopia dist/kopia_darwin_amd64/kopia dist/kopia_darwin_arm6/kopia: htmlui/build/index.html $(all_go_sources) - $(MAKE) signing-tools GOARCH=arm64 go build $(KOPIA_BUILD_FLAGS) -o dist/kopia_darwin_arm64/kopia -tags embedhtml GOARCH=amd64 go build $(KOPIA_BUILD_FLAGS) -o dist/kopia_darwin_amd64/kopia -tags embedhtml mkdir -p dist/kopia_darwin_universal @@ -118,7 +117,6 @@ endif # on Windows build and sign AMD64 and *.zip file dist/kopia_windows_amd64/kopia.exe: htmlui/build/index.html $(all_go_sources) - $(MAKE) signing-tools GOOS=windows GOARCH=amd64 go build $(KOPIA_BUILD_FLAGS) -o dist/kopia_windows_amd64/kopia.exe -tags embedhtml ifneq ($(WINDOWS_SIGN_TOOL),) tools/.tools/signtool.exe sign //sha1 $(WINDOWS_CERT_SHA1) //fd sha256 //tr "http://timestamp.digicert.com" //v dist/kopia_windows_amd64/kopia.exe @@ -154,22 +152,14 @@ ci-tests: lint vet test-with-coverage ci-integration-tests: integration-tests robustness-tool-tests $(MAKE) stress-test -ci-publish: -ifeq ($(GOOS)/$(GOARCH),linux/amd64) - $(MAKE) create-long-term-repository - $(MAKE) publish-coverage-results -endif - -publish-coverage-results: +ci-publish-coverage: +ifeq ($(GOOS)/$(GOARCH)/$(IS_PULL_REQUEST),linux/amd64/false) -bash -c "bash <(curl -s https://codecov.io/bash) -f coverage.txt" +endif # goreleaser - builds packages for all platforms when on linux/amd64, # but don't publish here, we'll upload to GitHub separately. -GORELEASER_OPTIONS=--rm-dist --parallelism=6 --skip-publish - -ifneq ($(PUBLISH_BINARIES)/$(IS_PULL_REQUEST)/$(GOOS)/$(GOARCH),true/false/linux/amd64) - GORELEASER_OPTIONS+=--skip-sign -endif +GORELEASER_OPTIONS=--rm-dist --parallelism=6 --skip-publish --skip-sign ifeq ($(CI_TAG),) GORELEASER_OPTIONS+=--snapshot @@ -274,62 +264,36 @@ official-release: goreturns: find . -name '*.go' | xargs goreturns -w --local github.com/kopia/kopia -# see if we have access to credentials encryption key -ifeq ($(CREDENTIAL_ENCRYPTION_KEY),) - -ci-credentials: - @echo CI credentials not available. - ci-gpg-key: - @echo Not installing GPG keys. - +ifneq ($(GPG_KEYRING),) + @echo "$(GPG_KEYRING)" | base64 -d | gpg --import else - -ci-gpg-key: -ifneq ($(GOOS),windows) - openssl aes-256-cbc -K "$(CREDENTIAL_ENCRYPTION_KEY)" -iv "$(CREDENTIAL_ENCRYPTION_IV)" -in kopia.gpg.enc -out /tmp/kopia.gpg -d - gpg --import /tmp/kopia.gpg + @echo No GPG keyring endif -ci-credentials: ci-gpg-key - -ifneq ($(GOOS),windows) - @echo Installing GPG key... - openssl aes-256-cbc -K "$(CREDENTIAL_ENCRYPTION_KEY)" -iv "$(CREDENTIAL_ENCRYPTION_IV)" -in kopia.gpg.enc -out /tmp/kopia.gpg -d - gpg --import /tmp/kopia.gpg - openssl aes-256-cbc -K "$(CREDENTIAL_ENCRYPTION_KEY)" -iv "$(CREDENTIAL_ENCRYPTION_IV)" -in tests/credentials/gcs/test_service_account.json.enc -out repo/blob/gcs/test_service_account.json -d - openssl aes-256-cbc -K "$(CREDENTIAL_ENCRYPTION_KEY)" -iv "$(CREDENTIAL_ENCRYPTION_IV)" -in tests/credentials/sftp/id_kopia.enc -out repo/blob/sftp/id_kopia -d - openssl aes-256-cbc -K "$(CREDENTIAL_ENCRYPTION_KEY)" -iv "$(CREDENTIAL_ENCRYPTION_IV)" -in tests/credentials/sftp/known_hosts.enc -out repo/blob/sftp/known_hosts -d - openssl aes-256-cbc -K "$(CREDENTIAL_ENCRYPTION_KEY)" -iv "$(CREDENTIAL_ENCRYPTION_IV)" -in tools/boto.enc -out tools/.boto -d - -ifeq ($(GOARCH),amd64) - $(MAKE) install-google-cloud-sdk-if-not-present - $(HOME)/google-cloud-sdk/bin/gcloud auth activate-service-account --key-file repo/blob/gcs/test_service_account.json +ci-gcs-creds: +ifneq ($(GCS_CREDENTIALS),) + @echo $(GCS_CREDENTIALS) | base64 -d | gzip -d | gcloud auth activate-service-account --key-file=/dev/stdin +else + @echo No GPG credentials. endif -endif - -endif - -install-google-cloud-sdk-if-not-present: - if [ ! -d $(HOME)/google-cloud-sdk ]; then $(retry) $(MAKE) install-google-cloud-sdk; fi - -install-google-cloud-sdk: - -rm -rf $(HOME)/google-cloud-sdk - echo Installing Google Cloud SDK... - curl -s https://sdk.cloud.google.com | CLOUDSDK_CORE_DISABLE_PROMPTS=1 bash 2>/dev/null >/dev/null - echo Finished Installing Google Cloud SDK. RELEASE_STAGING_DIR=$(CURDIR)/.release stage-release: rm -rf $(RELEASE_STAGING_DIR) mkdir -p $(RELEASE_STAGING_DIR) + + # copy all dist files to a staging directory find dist -type f -exec cp -v {} $(RELEASE_STAGING_DIR) \; + + # sign RPMs + find $(RELEASE_STAGING_DIR) -type f -name '*.rpm' -exec rpm --define "%_gpg_name Kopia Builder" --addsign {} \; + + # regenerate checksums file and sign it (cd $(RELEASE_STAGING_DIR) && sha256sum * > checksums.txt) cat $(RELEASE_STAGING_DIR)/checksums.txt -ifneq ($(CREDENTIAL_ENCRYPTION_KEY),) gpg --output $(RELEASE_STAGING_DIR)/checksums.txt.sig --detach-sig $(RELEASE_STAGING_DIR)/checksums.txt -endif ifeq ($(IS_PULL_REQUEST),false) ifneq ($(CI_TAG),) @@ -347,7 +311,7 @@ endif endif endif -push-github-release: $(github_release) +push-github-release: ifneq ($(GH_RELEASE_REPO),) @echo Creating Github Release $(GH_RELEASE_NAME) in $(GH_RELEASE_REPO) with flags $(GH_RELEASE_FLAGS) gh --repo $(GH_RELEASE_REPO) release view $(GH_RELEASE_NAME) || gh --repo $(GH_RELEASE_REPO) release create $(GH_RELEASE_FLAGS) $(GH_RELEASE_NAME) @@ -374,16 +338,24 @@ create-long-term-repository: endif -publish-packages: -ifeq ($(REPO_OWNER)/$(GOOS)/$(GOARCH)/$(IS_PULL_REQUEST),kopia/linux/amd64/false) - $(CURDIR)/tools/apt-publish.sh $(CURDIR)/dist - $(CURDIR)/tools/rpm-publish.sh $(CURDIR)/dist - $(CURDIR)/tools/homebrew-publish.sh $(CURDIR)/dist $(KOPIA_VERSION_NO_PREFIX) - $(CURDIR)/tools/scoop-publish.sh $(CURDIR)/dist $(KOPIA_VERSION_NO_PREFIX) +publish-apt: + $(CURDIR)/tools/apt-publish.sh $(RELEASE_STAGING_DIR) + +publish-rpm: + $(CURDIR)/tools/rpm-publish.sh $(RELEASE_STAGING_DIR) + +publish-homebrew: + $(CURDIR)/tools/homebrew-publish.sh $(RELEASE_STAGING_DIR) $(KOPIA_VERSION_NO_PREFIX) + +publish-scoop: + $(CURDIR)/tools/scoop-publish.sh $(RELEASE_STAGING_DIR) $(KOPIA_VERSION_NO_PREFIX) + +publish-docker: +ifneq ($(DOCKERHUB_TOKEN),) @echo $(DOCKERHUB_TOKEN) | docker login --username $(DOCKERHUB_USERNAME) --password-stdin $(CURDIR)/tools/docker-publish.sh $(CURDIR)/dist_binaries else - @echo Not pushing packages on pull request builds. + @echo DOCKERHUB_TOKEN is not set. endif PERF_BENCHMARK_INSTANCE=kopia-perf diff --git a/kopia.gpg.enc b/kopia.gpg.enc deleted file mode 100644 index 34e83c14f..000000000 Binary files a/kopia.gpg.enc and /dev/null differ diff --git a/site/content/docs/Installation/_index.md b/site/content/docs/Installation/_index.md index 37ab33d77..5c042dddc 100644 --- a/site/content/docs/Installation/_index.md +++ b/site/content/docs/Installation/_index.md @@ -196,8 +196,8 @@ $ sha256sum --check checksums.txt # Verify signature file $ gpg --verify checksums.txt.sig gpg: assuming signed data in 'checksums.txt' -gpg: Signature made Wed May 15 20:41:41 2019 PDT -gpg: using RSA key A3B5843ED70529C23162E3687713E6D88ED70D9D +gpg: Signature made Thu Apr 15 22:02:31 2021 PDT +gpg: using RSA key 7FB99DFD47809F0D5339D7D92273699AFD56A556 gpg: Good signature from "Kopia Builder " [ultimate] ``` diff --git a/site/content/signing-key b/site/content/signing-key index 198010dc8..b547ab968 100644 --- a/site/content/signing-key +++ b/site/content/signing-key @@ -1,30 +1,29 @@ -----BEGIN PGP PUBLIC KEY BLOCK----- -mQENBFzc1PMBCADUxStBWF421+r7zcE4gInfXNIPMt/xl5ZbcWGNtZLf2R3nEcGf -VpQdxMarooZCDh9EXv0S1A0LzaYBsYE6VFS1GKcuUwrRhSbZvzPYks3K0Cvs0bGW -88lYIDaWH3VsJztapWSwA9nSY+XNgpInq+HXseJfy1omOQ5IXF7yW12t/PXfiQSR -jOc9c+00xrwW7nwmNLyLGRjFP1U0hkZczUdu+yxmPr2a/AhfMSL7rq+Y0MDQL/dt -s08fGuXVec9T+uU/60LF/+j2yWcgaCTZkU+XiBCvx5s8lW/ucWK/8wPw8m+GuX49 -T3ky5A5Q5XdFPt6O16YL3zv78pLeiT32CJ7vABEBAAG0IEtvcGlhIEJ1aWxkZXIg -PGJ1aWxkZXJAa29waWEuaW8+iQFUBBMBCAA+FiEEo7WEPtcFKcIxYuNodxPm2I7X -DZ0FAlzc1PMCGwMFCQPCZwAFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQdxPm -2I7XDZ1ASggArGHKQ/h5jIfMi6nKKe/VZ2F20HwlwABnBZum6rHjH8+puLUUMSY/ -qZg+0AtyDz3jZsNNqlkiZANQRnpoV34mn/pO8ARuOC8ChJHy6fPvLAezgJrUBVHq -zfiVeOIEDmV09DMjKputDAezIjKP96XKaBGlRMWrb2hVAEwXmBidcfG58YEQ8bt5 -twqkyhhDvyaakIM8MZ9YFI+QRqU5NcstF/Bb7JsUhoVcqGRR+HM1flu6Tq+N19ZZ -u78GNJbv7i1Pg3PgILaxZOyfLO7JfyBGIYkGxyi9I2UF76xsETA+nRSAg3NkbXEw -Vw35ZGTWlFIOXYF7KWLuIg/Rz3kGKG5Pg7kBDQRc3NTzAQgAp6o6BxEyxSVb/ATe -pnnfsrSA0xLiKZLObd+kkF9xuSKvYy9jXtv+1haWM928Fs5aNTcnfvJj4b09MX0/ -Az5+bgCL621kqh/y9g/F6IoCU3l/UP6udJYP182yV4L0fvYDCtExwhUH1wTNQPXR -s7sVFWZSN8ukndLbFBIUJaLcNn9P//QVs/aK6lvFJZQXxaT2LiMGXxU4XM6RQfg6 -IkNyMhcEpJ6lMULd62QJBKu4PppauUCtoYn60leIbCUefBhTQsiU2YH0mNvZCJtv -A4/HmBQvdfIrsR2YYq6ddQmL52ZCprO+np27K3qS6zErFpfVYjir83PeEuOKfTJs -lhKWLQARAQABiQE8BBgBCAAmFiEEo7WEPtcFKcIxYuNodxPm2I7XDZ0FAlzc1PMC -GwwFCQPCZwAACgkQdxPm2I7XDZ0oEggAl8rpECpMt3bHWWvKSu3SGwR7o60hycBv -Z78ylPCSwSmAfKJGZdkDwm96Snr/ogkb1d6KJnmVqr4LQrjkk/YQ6iGKym95QoK4 -YWn2CucZT6xj2U8h3VT3+HbsA4/pdpxfbHq2iVFjWxj9BfQP2pp6gYMiX0uQtaj1 -czl+9wQhXX5atqQfCa442zPrc9tzNlGOgkSXoeHYgMiBw2c2Oy1QOMZhL3ZR8WUO -79Zx8A0IiU80KLUnyv6BSIZcchwnIlJbZHpCo1Xp0gpxkg9PcC8dhF2lt013gHi2 -P9+AhrmfJ9hJ2VI5kX2ApzdHPGVozEo/hm/IFJ6a1dLpN8lfdZyiqw== -=o6o/ +mQINBGB47AYBEADZyGFu5RB5c8rX/goflaTL6Z7FcYs0oLGw5DS4g+YCqWV5PPor +OuI9BsqH0fIcUeHmWl2DNohNx13K78H6LM5BvutCf2yOc0ktx6jv9uUXBKjEgRHH +hoNvNRVXJMH8wBCH7yU5JgA60x/mZw5pUsB1VGIhM3T9gvEz3Or7OshitG+3txGU +DBgCERclskZ+tTPxW6oQn96ZiInItOlkGmjv4bbpCavlE684OE89KBh/TM81xBXa +kd3aX9E35lpfwMjrnkNSiGRoy2Z0Dx8Ox2wbtfnTz4jVzgqkMmSMYWXrvRSCiisq +rEnEJZ3Y7DFmrj4dVESVVMPVQZMcim/NLpS/4cxFYpma7oj6EQ0FAFxar1E59drK +CNSKN3pj72MzQGFE53T2q7IJ/H7ICZcvuZUhfkbmKTjNZOJlealfmlrftcbiDZbY +9ge2chnNtT5WAY/junAGE7bqZlvInp2IzR1lJkxRhK1Dbg0mIBHY0h7PNm7BvNbD +RguMmEvDQUMCbzjRPyXs/2q28uNqnwDYGzOh5wSTyUks1cGR9JhkAO/n7EHsJDyW +dRQmXfAl/f/9Tbt/D31N+T7JmWsBVvhxJQoKUGWnKuelpUr8zegTy29z2Xii68tW +s6jMGCbmKn6JvVHjunBemEAWlT0ZI/+ETER+krHZQ9Z9TFkcl9m2Yq63gwARAQAB +tCBLb3BpYSBCdWlsZGVyIDxidWlsZGVyQGtvcGlhLmlvPokCVAQTAQgAPhYhBH+5 +nf1HgJ8NUznX2SJzaZr9VqVWBQJgeOwGAhsDBQkDwmcABQsJCAcCBhUKCQgLAgQW +AgMBAh4BAheAAAoJECJzaZr9VqVW2A4QALsCaXZCWUlr6sg5RgM5TScOYNxt+qqc +ViIY/EkEhQ9tb77d6BW9JqduXEEgfUtBZ96fQpXEqcf49Cyiqezf9Bq6OKLNS15x +mBae78kZVMER2pGgvFM5ZrNURZO3mTjcdMx941GdR1rdXIKspkNapkGXhIBArYHt +2OQkM4XAblU/ai2EXFHaRiN4H7Id536iqpt8HBH/kpXMbOgxuFrhVn8Ze89UpI+W +WcoXZ4VaYzs5rBop5aM4YncshBodH0UlUK9/mhu0kioPiJA75DYg1MK/TeKL6yMr +T9MvU7aFZkm0G/4O68xfWWqbB4xlnUBU9PwqF0Pkg3fpVKQifvhaeJz+KrxAyt6V +ShHnnw2wh9S3wEr6SuaA2ivGIfjDEd9dVSVbxnQD0p+/NKqcSFr7/RB7+1n4l8j0 +UFa0mJTSB4xJvDhWflmYqRox/x/4LjpwRE1U5PX7gwJ3yELwy6ybJN4826nN5a6Y +XU+OVMR7pL4UuC+8MACKRnVq7Tw92E0ttYDhYAZvjGHmOjBtdQcj9eqJ8K/vf/DU ++MD+vCFNUkhq3V4LoNE6K5Uz6ESwDbCNSiyxO3Xd+c0yElDozjXioMLVAuKE/STX +6Do6WJUVDbP0ygbR6a1AGJU2/mVICfob6ai3FvjazWRxPjTtZlpHuOBu4JP2e0iT +iJYG/llgQmKv +=VNmX -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/credentials/gcs/test_service_account.json.enc b/tests/credentials/gcs/test_service_account.json.enc deleted file mode 100644 index 74bb6a04d..000000000 Binary files a/tests/credentials/gcs/test_service_account.json.enc and /dev/null differ diff --git a/tests/credentials/sftp/id_kopia.enc b/tests/credentials/sftp/id_kopia.enc deleted file mode 100644 index dd27c6294..000000000 Binary files a/tests/credentials/sftp/id_kopia.enc and /dev/null differ diff --git a/tests/credentials/sftp/known_hosts.enc b/tests/credentials/sftp/known_hosts.enc deleted file mode 100644 index 24b6c8d32..000000000 Binary files a/tests/credentials/sftp/known_hosts.enc and /dev/null differ diff --git a/tools/apt-publish.sh b/tools/apt-publish.sh index 88e647aab..3fde05995 100755 --- a/tools/apt-publish.sh +++ b/tools/apt-publish.sh @@ -1,12 +1,12 @@ #!/bin/bash set -e -GS_PREFIX=gs://packages.kopia.io/apt -GPG_KEY_ID=A3B5843ED70529C23162E3687713E6D88ED70D9D +GS_PREFIX=gs://$PACKAGES_HOST/apt +GPG_KEY_ID=7FB99DFD47809F0D5339D7D92273699AFD56A556 PKGDIR=$1 RETAIN_UNSTABLE_DEB_COUNT=2 -if [ "$REPO_OWNER" != "kopia" ]; then - echo Not publishing APT package because current repo owner is $REPO_OWNER +if [ -z "$PACKAGES_HOST" ]; then + echo Not publishing APT package because PACKAGES_HOST is not set. exit 0 fi @@ -47,7 +47,7 @@ for d in $distributions; do gsutil -m rsync -r -d $GS_PREFIX/dists/$d $WORK_DIR/dists/$d for a in $architectures; do if [ "$d" == "unstable" ]; then - delete_old_deb $WORK_DIR/dists/$d/main/binary-$a + delete_old_deb $WORK_DIR/dists/$d/main/binary-$a || echo Unable to delete old deb fi done done @@ -89,6 +89,7 @@ for f in $deb_files; do if grep $bn\$ $packages_dir/Packages > /dev/null; then echo $bn already in $packages_dir/Packages else + mkdir -p $packages_dir cp -av $f $packages_dir fi done diff --git a/tools/boto.enc b/tools/boto.enc deleted file mode 100644 index f0fafa8fe..000000000 --- a/tools/boto.enc +++ /dev/null @@ -1,2 +0,0 @@ -Z0ܤI?$L="_Lw\/ -TΆC[Ԁ?؍˜sL1hݤaI}9n~w%U+ n]Ç_]C0Lkcţ dfwaGP%MF_2!;# \ No newline at end of file diff --git a/tools/homebrew-publish.sh b/tools/homebrew-publish.sh index c2c284b52..29c40702d 100755 --- a/tools/homebrew-publish.sh +++ b/tools/homebrew-publish.sh @@ -3,12 +3,12 @@ set -e dist_dir=$1 ver=$2 -target_repo=kopia/homebrew-kopia -source_repo=kopia/kopia +target_repo=$REPO_OWNER/homebrew-kopia +source_repo=$REPO_OWNER/kopia if [ "$CI_TAG" == "" ]; then - target_repo=kopia/homebrew-test-builds - source_repo=kopia/kopia-test-builds + target_repo=$REPO_OWNER/homebrew-test-builds + source_repo=$REPO_OWNER/kopia-test-builds fi if [ "$GITHUB_TOKEN" == "" ]; then diff --git a/tools/rpm-publish.sh b/tools/rpm-publish.sh index a9c2a7ac2..b08313c25 100755 --- a/tools/rpm-publish.sh +++ b/tools/rpm-publish.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -GS_PREFIX=gs://packages.kopia.io/rpm +GS_PREFIX=gs://$PACKAGES_HOST/rpm PKGDIR=$1 RETAIN_UNSTABLE_RPM_COUNT=2 @@ -13,8 +13,8 @@ if [ -z "$PKGDIR" ]; then exit 1 fi -if [ "$REPO_OWNER" != "kopia" ]; then - echo Not publishing RPM package because current repo owner is $REPO_OWNER +if [ -z "$PACKAGES_HOST" ]; then + echo Not publishing APT package because PACKAGES_HOST is not set. exit 0 fi diff --git a/tools/scoop-publish.sh b/tools/scoop-publish.sh index b82074236..dffb91a0b 100755 --- a/tools/scoop-publish.sh +++ b/tools/scoop-publish.sh @@ -3,12 +3,12 @@ set -e dist_dir=$1 ver=$2 -target_repo=kopia/scoop-bucket -source_repo=kopia/kopia +target_repo=$REPO_OWNER/scoop-bucket +source_repo=$REPO_OWNER/kopia if [ "$CI_TAG" == "" ]; then - target_repo=kopia/scoop-test-builds - source_repo=kopia/kopia-test-builds + target_repo=$REPO_OWNER/scoop-test-builds + source_repo=$REPO_OWNER/kopia-test-builds fi if [ "$GITHUB_TOKEN" == "" ]; then diff --git a/tools/tools.mk b/tools/tools.mk index d83d6af0e..9f8cade60 100644 --- a/tools/tools.mk +++ b/tools/tools.mk @@ -275,9 +275,10 @@ windows_signing_dir=$(TOOLS_DIR)$(slash)win_signing # name of the temporary keychain to import signing keys into (will be deleted and re-created by 'signing-tools' target) MACOS_KEYCHAIN=kopia-build.keychain +export CSC_KEYCHAIN:=$(MACOS_KEYCHAIN) +export CSC_NAME:=$(MACOS_SIGNING_IDENTITY) -signing-tools: - +windows-signing-tools: ifeq ($(GOOS)/$(CI),windows/true) ifneq ($(WINDOWS_SIGNING_TOOLS_URL),) echo Installing Windows signing tools to $(windows_signing_dir)... @@ -286,15 +287,16 @@ ifneq ($(WINDOWS_SIGNING_TOOLS_URL),) unzip -a -q $(windows_signing_dir).zip -d $(windows_signing_dir) pwsh -noprofile -executionpolicy bypass $(windows_signing_dir)\\setup.ps1 else - echo Not installing Windows signing tools because WINDOWS_SIGNING_TOOLS_URL is not set + @echo Not installing Windows signing tools because WINDOWS_SIGNING_TOOLS_URL is not set endif endif -ifeq ($(GOOS)/$(CI),darwin/true) -ifneq ($(CSC_LINK),) # create and unlock a keychain with random strong password and import macOS signing certificate from .p12. -signing-tools: KEYCHAIN_PASSWORD:=$(shell uuidgen) -signing-tools: +ifeq ($(GOOS)/$(CI),darwin/true) +macos-certificates: KEYCHAIN_PASSWORD:=$(shell uuidgen) +endif +macos-certificates: +ifneq ($(CSC_LINK),) @rm -fv $(HOME)/Library/Keychains/$(MACOS_KEYCHAIN)-db @echo "$(CSC_LINK)" | base64 -d > /tmp/certs.p12 @security create-keychain -p $(KEYCHAIN_PASSWORD) $(MACOS_KEYCHAIN) @@ -304,7 +306,8 @@ signing-tools: @security set-keychain-settings -u $(MACOS_KEYCHAIN) @rm -f /tmp/certs.p12 @security set-key-partition-list -S apple: -s -k $(KEYCHAIN_PASSWORD) $(MACOS_KEYCHAIN) > /dev/null -endif +else + @echo Not installing macOS certificates because CSC_LINK is not set. endif # disable some tools on non-default architectures