From 00cb2e40e304100da7817325eb0a81c6b8e33ecd Mon Sep 17 00:00:00 2001 From: Max Weber Date: Tue, 2 Jun 2020 21:36:51 -0600 Subject: [PATCH 1/2] package: retry failed uploads --- build_manifest.sh | 2 +- build_plugin.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build_manifest.sh b/build_manifest.sh index 3a19417a9..3e0989e26 100755 --- a/build_manifest.sh +++ b/build_manifest.sh @@ -70,7 +70,7 @@ perl -e "print pack('N', -s \"$MANIFEST.sig\")" > "$MANIFEST.out" cat "$MANIFEST.sig" >> "$MANIFEST.out" cat "$MANIFEST" >> "$MANIFEST.out" -curl --fail \ +curl --fail --retry 5 --retry-connrefused \ --user "$REPO_CREDS" \ --upload-file "$MANIFEST.out" "$REPO_ROOT/$RUNELITE_VERSION/manifest.js" diff --git a/build_plugin.sh b/build_plugin.sh index 9ba646eb3..3902cf464 100755 --- a/build_plugin.sh +++ b/build_plugin.sh @@ -98,7 +98,7 @@ if [ -e "icon.png" ]; then ICON_UPLOAD=("--upload-file" "icon.png" "$LOCATION.png") fi -curl --fail \ +curl --fail --retry 5 --retry-connrefused \ --user "$REPO_CREDS" \ --upload-file "$BUILDDIR/plugin.manifest" "$LOCATION.manifest" \ --upload-file "$BUILDDIR/plugin.jar" "$LOCATION.jar" \ From 36bf656da418b1c94161b0eb861ad929b82e1a72 Mon Sep 17 00:00:00 2001 From: Max Weber Date: Tue, 2 Jun 2020 21:47:19 -0600 Subject: [PATCH 2/2] package: use a single curl for downloading existing manifest chunks This makes the build_manifest step much faster --- build_manifest.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/build_manifest.sh b/build_manifest.sh index 3e0989e26..f1a3d6471 100755 --- a/build_manifest.sh +++ b/build_manifest.sh @@ -32,11 +32,12 @@ set -e -x RUNELITE_VERSION="$(cat "runelite.version")" MANIFEST="$(mktemp /tmp/manifest.XXXXXXXX)" -trap "rm -f ""$MANIFEST*""" EXIT +trap "rm -rf ""$MANIFEST*""" EXIT +MANIFEST_DIR="$MANIFEST.sub/" +mkdir "$MANIFEST_DIR" -echo "[" > "$MANIFEST" +MANIFEST_CHUNK_DOWNLOAD=() -IS_FIRST=true for PLUGINFILE in plugins/*; do # read in the plugin descriptor disabled= @@ -49,18 +50,22 @@ for PLUGINFILE in plugins/*; do PLUGIN_ID=$(basename "$PLUGINFILE") LOCATION="$REPO_ROOT/$RUNELITE_VERSION/$PLUGIN_ID/$commit" + MANIFEST_CHUNK_DOWNLOAD+=('--output' "$MANIFEST_DIR/$PLUGIN_ID" "$LOCATION.manifest") +done - RET=0 - curl --fail "$LOCATION.manifest" > "$MANIFEST.sub" || RET=$? - [ $RET -ne 0 ] && continue +curl --fail --retry 5 --retry-connrefused \ + --parallel \ + "${MANIFEST_CHUNK_DOWNLOAD[@]}" +IS_FIRST=true +echo "[" > "$MANIFEST" +for MANIFEST_CHUNK in "$MANIFEST_DIR"/*; do if [[ "$IS_FIRST" != true ]]; then echo "," >> "$MANIFEST" fi IS_FIRST= - cat "$MANIFEST.sub" >> "$MANIFEST" + cat "$MANIFEST_CHUNK" >> "$MANIFEST" done - echo "]" >> "$MANIFEST" # shellcheck disable=SC2059