diff --git a/.gitignore b/.gitignore index 64e9d2007..a0c6b34a6 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ known_hosts node_modules/ *_bindata.go .tmp.* +.boto \ No newline at end of file diff --git a/Makefile b/Makefile index 6de40678e..bfeadb87f 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ GO_TEST=go test TEST_FLAGS?= KOPIA_INTEGRATION_EXE=$(CURDIR)/dist/integration/kopia.exe FIO_DOCKER_TAG=ljishen/fio +BOTO_PATH=/Users/jarek/Projects/Kopia/tools/.boto all: test lint vet integration-tests @@ -107,6 +108,7 @@ travis-release: $(retry) $(MAKE) layering-test $(retry) $(MAKE) integration-tests ifeq ($(TRAVIS_OS_NAME),linux) + $(MAKE) apt-publish $(MAKE) robustness-tool-tests $(MAKE) website $(MAKE) stress-test @@ -251,6 +253,7 @@ ifneq ($(TRAVIS_OS_NAME),windows) openssl aes-256-cbc -K "$(encrypted_fa1db4b894bb_key)" -iv "$(encrypted_fa1db4b894bb_iv)" -in tests/credentials/gcs/test_service_account.json.enc -out repo/blob/gcs/test_service_account.json -d openssl aes-256-cbc -K "$(encrypted_fa1db4b894bb_key)" -iv "$(encrypted_fa1db4b894bb_iv)" -in tests/credentials/sftp/id_kopia.enc -out repo/blob/sftp/id_kopia -d openssl aes-256-cbc -K "$(encrypted_fa1db4b894bb_key)" -iv "$(encrypted_fa1db4b894bb_iv)" -in tests/credentials/sftp/known_hosts.enc -out repo/blob/sftp/known_hosts -d + openssl aes-256-cbc -K "$(encrypted_fa1db4b894bb_key)" -iv "$(encrypted_fa1db4b894bb_iv)" -in tools/boto.enc -out tools/.boto -d endif travis-install-cloud-sdk: travis-install-test-credentials @@ -282,3 +285,11 @@ travis-create-long-term-repository: echo Not creating long-term repository. endif + +ifeq ($(TRAVIS_PULL_REQUEST),false) +apt-publish: + $(CURDIR)/tools/apt-publish.sh $(CURDIR)/dist +else +apt-publish: + @echo Not pushing to APT repository on pull request builds. +endif diff --git a/tools/apt-publish.sh b/tools/apt-publish.sh new file mode 100755 index 000000000..11cd0c0af --- /dev/null +++ b/tools/apt-publish.sh @@ -0,0 +1,99 @@ +#!/bin/bash +GS_PREFIX=gs://packages.kopia.io/apt +GPG_KEY_ID=A3B5843ED70529C23162E3687713E6D88ED70D9D +PKGDIR=$1 + +if [ -z "$PKGDIR" ]; then + echo usage $0: /path/to/dist + exit 1 +fi + +if [ ! -d "$PKGDIR" ]; then + echo $PKGDIR must be a directory containing '*.deb' files + exit 1 +fi + +distributions="stable testing unstable" +architectures="amd64 arm64 armhf" + +WORK_DIR=/tmp/apt-publish +rm -rf "$WORK_DIR" +mkdir -p "$WORK_DIR" + +echo Downloading Package lists... + +deb_files=$(find $1 -name '*.deb') + +# download 'Packages' files from GCS for all supported distributions and architectures +for d in $distributions; do + for a in $architectures; do + mkdir -pv $WORK_DIR/dists/$d/main/binary-$a + touch $WORK_DIR/dists/$d/main/binary-$a/Packages + gsutil cp -av $GS_PREFIX/dists/$d/main/binary-$a/Packages $WORK_DIR/dists/$d/main/binary-$a/Packages + done +done + +# sort all files into appropriate binary directories +for f in $deb_files; do + if [[ "$f" =~ _([^_]*)_(linux_)?([^_]+).deb$ ]]; then + ver=${BASH_REMATCH[1]#v} + arch=${BASH_REMATCH[3]} + dists="" + + if [[ $ver =~ "next" ]]; then + # ignore -next versions which are from goreleaser snapshots + continue + fi + + # x.y.z + if [[ $ver =~ [0-9]+\.[0-9]+\.[0-9]+$ ]]; then + dists="stable testing" + fi + + # x.y.z-prerelease + if [[ $ver =~ [0-9]+\.[0-9]+\.[0-9]+\-.*$ ]]; then + dists="testing" + fi + + # 20x.y.z-hash + if [[ $ver =~ 20[0-9]+\.[0-9]+\.[0-9]+-.*$ ]]; then + dists="unstable" + fi + + echo "$f $arch $dists" + + bn=$(basename $f) + for d in $dists; do + packages_dir=$WORK_DIR/dists/$d/main/binary-$arch + if grep $bn\$ $packages_dir/Packages > /dev/null; then + echo $bn already in $packages_dir/Packages + else + cp -av $f $packages_dir + fi + done + fi +done + +# append to 'Packages' file for all files placed in the work directory +for d in $distributions; do + for a in $architectures; do + mkdir -pv $WORK_DIR/dists/$d/main/binary-$a + (cd $WORK_DIR && dpkg-scanpackages --multiversion -a $a dists/$d/main/binary-$a >> dists/$d/main/binary-$a/Packages) + gzip -k $WORK_DIR/dists/$d/main/binary-$a/Packages + done +done + +# generate Release/InRelease/Release.gpg files for all distributions +for d in $distributions; do + docker run -it --rm -v $WORK_DIR/dists/$d:/root marshallofsound/apt-ftparchive release \ + -o APT::FTPArchive::Release::Architectures="$architectures" \ + -o APT::FTPArchive::Release::Codename="$d" \ + -o APT::FTPArchive::Release::Suite="$d" \ + . > $WORK_DIR/dists/$d/Release + + gpg --default-key $GPG_KEY_ID -abs -o - $WORK_DIR/dists/$d/Release > $WORK_DIR/dists/$d/Release.gpg + gpg --default-key $GPG_KEY_ID --clearsign -o - $WORK_DIR/dists/$d/Release > $WORK_DIR/dists/$d/InRelease +done + +# sync back to GCS +gsutil -m rsync -r $WORK_DIR/ $GS_PREFIX/ \ No newline at end of file diff --git a/tools/boto.enc b/tools/boto.enc new file mode 100644 index 000000000..f0fafa8fe --- /dev/null +++ b/tools/boto.enc @@ -0,0 +1,2 @@ +·Z0ܤI?¯$Lþ«�="_øŠL�wó\ÀÖ/ +½Á·T¯ΆÂ��C[Ô€ÿ?¸ôØ�²Â˜sL·–1£Òhž¹Ý¤aIƒ�È}’9n~w¦%ëмUå+ nó•]ßþÅðÇ�ë_ò¿å]¤¬ÐìàC0L—kcÅ£ Ôdfwa–GPä%‘ÆM”ÉFÑÜ_„2”!;Ú#œý \ No newline at end of file