tools: added apt-publish tool that push to APT repository

The repository is in GCS and the script will automatically copy
DEB files from dist/ to the proper locations and will regenerate
APT package index.

.gitignore

@@ -20,3 +20,4 @@ known_hosts
 node_modules/
 *_bindata.go
 .tmp.*
+.boto

Makefile

@@ -3,6 +3,7 @@ GO_TEST=go test
 TEST_FLAGS?=
 KOPIA_INTEGRATION_EXE=$(CURDIR)/dist/integration/kopia.exe
 FIO_DOCKER_TAG=ljishen/fio
+BOTO_PATH=/Users/jarek/Projects/Kopia/tools/.boto
 
 all: test lint vet integration-tests
 
@@ -107,6 +108,7 @@ travis-release:
 	$(retry) $(MAKE) layering-test
 	$(retry) $(MAKE) integration-tests
 ifeq ($(TRAVIS_OS_NAME),linux)
+	$(MAKE) apt-publish
 	$(MAKE) robustness-tool-tests
 	$(MAKE) website
 	$(MAKE) stress-test
@@ -251,6 +253,7 @@ ifneq ($(TRAVIS_OS_NAME),windows)
 	openssl aes-256-cbc -K "$(encrypted_fa1db4b894bb_key)" -iv "$(encrypted_fa1db4b894bb_iv)" -in tests/credentials/gcs/test_service_account.json.enc -out repo/blob/gcs/test_service_account.json -d
 	openssl aes-256-cbc -K "$(encrypted_fa1db4b894bb_key)" -iv "$(encrypted_fa1db4b894bb_iv)" -in tests/credentials/sftp/id_kopia.enc -out repo/blob/sftp/id_kopia -d
 	openssl aes-256-cbc -K "$(encrypted_fa1db4b894bb_key)" -iv "$(encrypted_fa1db4b894bb_iv)" -in tests/credentials/sftp/known_hosts.enc -out repo/blob/sftp/known_hosts -d
+	openssl aes-256-cbc -K "$(encrypted_fa1db4b894bb_key)" -iv "$(encrypted_fa1db4b894bb_iv)" -in tools/boto.enc -out tools/.boto -d
 endif
 
 travis-install-cloud-sdk: travis-install-test-credentials
@@ -282,3 +285,11 @@ travis-create-long-term-repository:
 	echo Not creating long-term repository.
 
 endif
+
+ifeq ($(TRAVIS_PULL_REQUEST),false)
+apt-publish:
+	$(CURDIR)/tools/apt-publish.sh $(CURDIR)/dist
+else
+apt-publish:
+	@echo Not pushing to APT repository on pull request builds.
+endif

tools/apt-publish.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+GS_PREFIX=gs://packages.kopia.io/apt
+GPG_KEY_ID=A3B5843ED70529C23162E3687713E6D88ED70D9D
+PKGDIR=$1
+
+if [ -z "$PKGDIR" ]; then
+  echo usage $0: /path/to/dist
+  exit 1
+fi
+
+if [ ! -d "$PKGDIR" ]; then
+  echo $PKGDIR must be a directory containing '*.deb' files
+  exit 1
+fi
+
+distributions="stable testing unstable"
+architectures="amd64 arm64 armhf"
+
+WORK_DIR=/tmp/apt-publish
+rm -rf "$WORK_DIR"
+mkdir -p "$WORK_DIR"
+
+echo Downloading Package lists...
+
+deb_files=$(find $1 -name '*.deb')
+
+# download 'Packages' files from GCS for all supported distributions and architectures
+for d in $distributions; do
+  for a in $architectures; do
+    mkdir -pv $WORK_DIR/dists/$d/main/binary-$a
+    touch $WORK_DIR/dists/$d/main/binary-$a/Packages
+    gsutil cp -av $GS_PREFIX/dists/$d/main/binary-$a/Packages $WORK_DIR/dists/$d/main/binary-$a/Packages
+  done
+done
+
+# sort all files into appropriate binary directories
+for f in $deb_files; do
+  if [[ "$f" =~ _([^_]*)_(linux_)?([^_]+).deb$ ]]; then
+    ver=${BASH_REMATCH[1]#v}
+    arch=${BASH_REMATCH[3]}
+    dists=""
+
+    if [[ $ver =~ "next" ]]; then
+      # ignore -next versions which are from goreleaser snapshots
+      continue
+    fi
+
+    # x.y.z
+    if [[ $ver =~ [0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+      dists="stable testing"
+    fi
+
+    # x.y.z-prerelease
+    if [[ $ver =~ [0-9]+\.[0-9]+\.[0-9]+\-.*$ ]]; then
+      dists="testing"
+    fi
+
+    # 20x.y.z-hash
+    if [[ $ver =~ 20[0-9]+\.[0-9]+\.[0-9]+-.*$ ]]; then
+      dists="unstable"
+    fi
+
+    echo "$f $arch $dists"
+
+    bn=$(basename $f)
+    for d in $dists; do
+      packages_dir=$WORK_DIR/dists/$d/main/binary-$arch
+      if grep $bn\$ $packages_dir/Packages > /dev/null; then
+        echo $bn already in $packages_dir/Packages
+      else
+        cp -av $f $packages_dir
+      fi
+    done
+  fi
+done
+
+# append to 'Packages' file for all files placed in the work directory
+for d in $distributions; do
+  for a in $architectures; do
+    mkdir -pv $WORK_DIR/dists/$d/main/binary-$a
+    (cd $WORK_DIR && dpkg-scanpackages --multiversion -a $a dists/$d/main/binary-$a >> dists/$d/main/binary-$a/Packages)
+    gzip -k $WORK_DIR/dists/$d/main/binary-$a/Packages
+  done
+done
+
+# generate Release/InRelease/Release.gpg files for all distributions
+for d in $distributions; do
+  docker run -it --rm -v $WORK_DIR/dists/$d:/root marshallofsound/apt-ftparchive release \
+    -o APT::FTPArchive::Release::Architectures="$architectures" \
+    -o APT::FTPArchive::Release::Codename="$d" \
+    -o APT::FTPArchive::Release::Suite="$d" \
+    . > $WORK_DIR/dists/$d/Release
+
+  gpg --default-key $GPG_KEY_ID -abs -o - $WORK_DIR/dists/$d/Release > $WORK_DIR/dists/$d/Release.gpg
+  gpg --default-key $GPG_KEY_ID --clearsign -o - $WORK_DIR/dists/$d/Release > $WORK_DIR/dists/$d/InRelease
+done
+
+# sync back to GCS
+gsutil -m rsync -r $WORK_DIR/ $GS_PREFIX/

tools/boto.enc

@@ -0,0 +1,2 @@
+·Z0�₪I?¯$L‏«�="_ר�L�wף\ְײ/
+½ֱ·T¯־†ֲ��C[װ€�?¸פ״�²ֲ˜sL·–1£ׂh�¹�₪aIƒ�ָ}’9n~w¦%כ�¼Uו+ nף•]�‏ֵנֳ‡�כ_ע¿ו]₪¬׀לאC0L—kcֵ£ װdfwa–GPה%‘ֶM”ֹFׁ�_„2”!;�#�‎