diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e17f3b464..033944f42 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,12 +5,12 @@ on: description: "List of plugins to build, or 'ALL'" required: false COMMIT_RANGE: - description: "Commit range to build 1234abc...5689def" + description: "Commit range to build 1234abc..5689def" required: false push: pull_request: jobs: - execute: + build: # any forks that predate this repo having an action will have actions # enabled by default, which will fail in a lot of cases because the branch # is new, which makes the differential build fail @@ -28,7 +28,7 @@ jobs: path: | ~/.gradle/caches/ ~/.gradle/wrapper/ - key: package-2.0.1 + key: package-2.0.2 - name: prepare run: | pushd package @@ -38,7 +38,6 @@ jobs: env: REPO_CREDS: ${{ secrets.REPO_CREDS }} REPO_ROOT: ${{ secrets.REPO_ROOT }} - SIGNING_KEY: ${{ secrets.SIGNING_KEY }} # workflow_dispatch FORCE_BUILD: ${{ github.event.inputs.FORCE_BUILD }} COMMIT_RANGE: ${{ github.event.inputs.COMMIT_RANGE }} @@ -55,4 +54,39 @@ jobs: else export PACKAGE_COMMIT_RANGE="${COMMIT_RANGE:-${COMMIT_BEFORE:+$COMMIT_BEFORE...$COMMIT_AFTER}}" fi - java -XX:+UseParallelGC -jar package/package/build/libs/package.jar \ No newline at end of file + exec java -XX:+UseParallelGC -cp package/package/build/libs/package.jar net.runelite.pluginhub.packager.Packager + - uses: actions/upload-artifact@v2 + with: + name: manifest_diff + path: /tmp/manifest_diff + retention-days: 1 + upload: + if: (github.event_name != 'push' || github.repository_owner == 'runelite') && github.event_name != 'pull_request' + needs: build + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-java@v1 + with: + java-version: 11 + - uses: actions/cache@v2 + with: + path: | + ~/.gradle/caches/ + ~/.gradle/wrapper/ + key: upload-2.0.2 + - uses: actions/download-artifact@v2 + with: + name: manifest_diff + path: /tmp + - name: upload + env: + REPO_CREDS: ${{ secrets.REPO_CREDS }} + REPO_ROOT: ${{ secrets.REPO_ROOT }} + SIGNING_KEY: ${{ secrets.SIGNING_KEY }} + run: | + pushd package + ./gradlew :upload:run + popd \ No newline at end of file diff --git a/package/package/build.gradle b/package/package/build.gradle index 5a37ad911..fca319c10 100644 --- a/package/package/build.gradle +++ b/package/package/build.gradle @@ -37,6 +37,7 @@ dependencies { implementation "org.ow2.asm:asm:7.0" implementation "com.squareup.okhttp3:okhttp:3.14.9" implementation "com.google.code.gson:gson:2.8.5" + implementation project(":upload") def lombok = "org.projectlombok:lombok:1.18.4"; compileOnly lombok @@ -55,5 +56,6 @@ jar { } test { + dependsOn ":initLib:shadowJar" workingDir new File(project.rootDir, "../") } \ No newline at end of file diff --git a/package/package/src/main/java/net/runelite/pluginhub/packager/Packager.java b/package/package/src/main/java/net/runelite/pluginhub/packager/Packager.java index 8c37703c4..fcc775372 100644 --- a/package/package/src/main/java/net/runelite/pluginhub/packager/Packager.java +++ b/package/package/src/main/java/net/runelite/pluginhub/packager/Packager.java @@ -28,29 +28,20 @@ import com.google.common.base.Splitter; import com.google.common.base.Stopwatch; import com.google.common.base.Strings; import com.google.common.collect.Queues; -import com.google.common.collect.Sets; import com.google.common.io.Files; -import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; import java.io.Closeable; -import java.io.DataOutputStream; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; -import java.security.Signature; -import java.security.SignatureException; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.List; import java.util.Queue; -import java.util.Set; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -61,14 +52,12 @@ import java.util.stream.StreamSupport; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import okhttp3.HttpUrl; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import okio.BufferedSource; +import net.runelite.pluginhub.uploader.ManifestDiff; +import net.runelite.pluginhub.uploader.UploadConfiguration; +import net.runelite.pluginhub.uploader.Util; @Slf4j -public class Packager +public class Packager implements Closeable { private static final File PLUGIN_ROOT = new File("./plugins"); public static final File PACKAGE_ROOT = new File("./package/").getAbsoluteFile(); @@ -88,27 +77,22 @@ public class Packager private final AtomicInteger numDone = new AtomicInteger(0); private final int numTotal; - @Setter - private boolean ignoreOldManifest; - @Setter private boolean alwaysPrintLog; @Getter private boolean failed; + private ManifestDiff diff = new ManifestDiff(); + public Packager(List buildList) throws IOException { this.buildList = buildList; this.numTotal = buildList.size(); - this.runeliteVersion = readRLVersion(); + this.runeliteVersion = Util.readRLVersion(); } - Set newManifests = Sets.newConcurrentHashSet(); - Set remove = Sets.newConcurrentHashSet(); - - public void buildPlugins() - throws IOException, InvalidKeyException, NoSuchAlgorithmException, SignatureException + public void buildPlugins() throws IOException { Queue buildQueue = Queues.synchronizedQueue(new ArrayDeque<>(buildList)); List buildThreads = IntStream.range(0, 8) @@ -137,84 +121,20 @@ public class Packager } } - if (uploadConfig.isComplete()) + Gson gson = new Gson(); + String diffJSON = gson.toJson(diff); + log.info("manifest change: {}", diffJSON); + + try (FileOutputStream fos = new FileOutputStream("/tmp/manifest_diff")) { - Gson gson = new Gson(); - HttpUrl manifestURL = uploadConfig.getUploadRepoRoot().newBuilder() - .addPathSegment("manifest.js") - .build(); - - List manifests = new ArrayList<>(); - if (!ignoreOldManifest) - { - try (Response res = uploadConfig.getClient().newCall(new Request.Builder() - .url(manifestURL) - .get() - .build()) - .execute()) - { - if (res.code() != 404) - { - Util.check(res); - - BufferedSource src = res.body().source(); - - byte[] signature = new byte[src.readInt()]; - src.readFully(signature); - - byte[] data = src.readByteArray(); - Signature s = Signature.getInstance("SHA256withRSA"); - s.initVerify(uploadConfig.getCert()); - s.update(data); - - if (!s.verify(signature)) - { - throw new RuntimeException("Unable to verify external plugin manifest"); - } - - manifests = gson.fromJson(new String(data, StandardCharsets.UTF_8), - new TypeToken>() - { - }.getType()); - } - } - } - - manifests.removeIf(m -> remove.contains(m.getInternalName())); - manifests.addAll(newManifests); - manifests.sort(Comparator.comparing(ExternalPluginManifest::getInternalName)); - - { - byte[] data = gson.toJson(manifests).getBytes(StandardCharsets.UTF_8); - Signature s = Signature.getInstance("SHA256withRSA"); - s.initSign(uploadConfig.getKey()); - s.update(data); - byte[] sig = s.sign(); - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new DataOutputStream(out).writeInt(sig.length); - out.write(sig); - out.write(data); - byte[] manifest = out.toByteArray(); - - try (Response res = uploadConfig.getClient().newCall(new Request.Builder() - .url(manifestURL) - .put(RequestBody.create(null, manifest)) - .build()) - .execute()) - { - Util.check(res); - } - } - - uploadConfig.getClient().connectionPool().evictAll(); + fos.write(diffJSON.getBytes(StandardCharsets.UTF_8)); } } private void buildPlugin(File plugin) { - remove.add(plugin.getName()); + diff.getRemove().add(plugin.getName()); if (!plugin.exists()) { @@ -245,7 +165,7 @@ public class Packager p.uploadLog(uploadConfig); } - newManifests.add(p.getManifest()); + diff.getAdd().add(p.getManifest()); log.info("{}: done in {}ms [{}/{}]", p.getInternalName(), p.getBuildTimeMS(), numDone.get() + 1, numTotal); } catch (PluginBuildException e) @@ -327,11 +247,16 @@ public class Packager }; } - public static String readRLVersion() throws IOException + public void setIgnoreOldManifest(boolean ignore) { - return Files.asCharSource(new File("./runelite.version"), StandardCharsets.UTF_8).read().trim(); + diff.setIgnoreOldManifest(ignore); } + @Override + public void close() + { + uploadConfig.close(); + } public static void main(String... args) throws Exception { @@ -347,6 +272,7 @@ public class Packager else if ("ALL".equals(System.getenv("FORCE_BUILD"))) { buildList = listAllPlugins(); + isBuildingAll = true; } else if (!Strings.isNullOrEmpty(System.getenv("FORCE_BUILD"))) { @@ -413,13 +339,17 @@ public class Packager throw new RuntimeException("missing env vars"); } - Packager pkg = new Packager(buildList); - pkg.getUploadConfig().fromEnvironment(pkg.getRuneliteVersion()); - pkg.setAlwaysPrintLog(!pkg.getUploadConfig().isComplete()); - pkg.setIgnoreOldManifest(isBuildingAll); - pkg.buildPlugins(); + boolean failed; + try (Packager pkg = new Packager(buildList)) + { + pkg.getUploadConfig().fromEnvironment(pkg.getRuneliteVersion()); + pkg.setAlwaysPrintLog(!pkg.getUploadConfig().isComplete()); + pkg.setIgnoreOldManifest(isBuildingAll); + pkg.buildPlugins(); + failed = pkg.isFailed(); + } - if (testFailure || pkg.isFailed() && !isBuildingAll) + if (testFailure || (failed && !isBuildingAll)) { System.exit(1); } diff --git a/package/package/src/main/java/net/runelite/pluginhub/packager/Plugin.java b/package/package/src/main/java/net/runelite/pluginhub/packager/Plugin.java index 03b1407dc..ae21fa090 100644 --- a/package/package/src/main/java/net/runelite/pluginhub/packager/Plugin.java +++ b/package/package/src/main/java/net/runelite/pluginhub/packager/Plugin.java @@ -63,6 +63,8 @@ import javax.imageio.ImageIO; import lombok.Getter; import lombok.Setter; import lombok.SneakyThrows; +import net.runelite.pluginhub.uploader.ExternalPluginManifest; +import net.runelite.pluginhub.uploader.UploadConfiguration; import okhttp3.HttpUrl; import org.gradle.tooling.CancellationTokenSource; import org.gradle.tooling.GradleConnectionException; @@ -211,13 +213,34 @@ public class Plugin implements Closeable iconFile = new File(repositoryDirectory, "icon.png"); } + private void waitAndCheck(Process process, String name, long timeout, TimeUnit timeoutUnit) throws PluginBuildException + { + try + { + if (!process.waitFor(timeout, timeoutUnit)) + { + process.destroy(); + throw PluginBuildException.of(this, name + " failed to complete in a reasonable time"); + } + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + + if (process.exitValue() != 0) + { + throw PluginBuildException.of(this, name + " exited with " + process.exitValue()); + } + } + public void download() throws IOException, PluginBuildException { Process gitclone = new ProcessBuilder("git", "clone", "--config", "advice.detachedHead=false", this.repositoryURL, repositoryDirectory.getAbsolutePath()) .redirectOutput(ProcessBuilder.Redirect.appendTo(logFile)) .redirectError(ProcessBuilder.Redirect.appendTo(logFile)) .start(); - Util.waitAndCheck(this, gitclone, "git clone", 2, TimeUnit.MINUTES); + waitAndCheck(gitclone, "git clone", 2, TimeUnit.MINUTES); Process gitcheckout = new ProcessBuilder("git", "checkout", commit + "^{commit}") @@ -225,7 +248,7 @@ public class Plugin implements Closeable .redirectError(ProcessBuilder.Redirect.appendTo(logFile)) .directory(repositoryDirectory) .start(); - Util.waitAndCheck(this, gitcheckout, "git checkout", 2, TimeUnit.MINUTES); + waitAndCheck(gitcheckout, "git checkout", 2, TimeUnit.MINUTES); } public void build(String runeliteVersion) throws IOException, PluginBuildException diff --git a/package/package/src/test/java/net/runelite/pluginhub/packager/PluginTest.java b/package/package/src/test/java/net/runelite/pluginhub/packager/PluginTest.java index e42e4003d..0ba278b4e 100644 --- a/package/package/src/test/java/net/runelite/pluginhub/packager/PluginTest.java +++ b/package/package/src/test/java/net/runelite/pluginhub/packager/PluginTest.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Properties; import lombok.extern.slf4j.Slf4j; +import net.runelite.pluginhub.uploader.Util; import org.junit.Assert; import org.junit.Test; @@ -74,7 +75,7 @@ public class PluginTest { try (Plugin p = createExamplePlugin("example")) { - p.build(Packager.readRLVersion()); + p.build(Util.readRLVersion()); p.assembleManifest(); } } @@ -88,7 +89,7 @@ public class PluginTest Properties props = Plugin.loadProperties(propFile); props.setProperty("plugins", "com.nonexistent"); writeProperties(props, propFile); - p.build(Packager.readRLVersion()); + p.build(Util.readRLVersion()); p.assembleManifest(); Assert.fail(); } diff --git a/package/settings.gradle b/package/settings.gradle index 6b436e59a..216699349 100644 --- a/package/settings.gradle +++ b/package/settings.gradle @@ -1,4 +1,5 @@ rootProject.name = "package-root" include "initLib" +include "upload" include "package" \ No newline at end of file diff --git a/package/travis.sh b/package/travis.sh index 85764fcc4..6bdac43c1 100755 --- a/package/travis.sh +++ b/package/travis.sh @@ -35,4 +35,9 @@ popd PACKAGE_IS_PR="$TRAVIS_PULL_REQUEST" \ PACKAGE_COMMIT_RANGE="$TRAVIS_COMMIT_RANGE" \ - java -XX:+UseParallelGC -jar package/package/build/libs/package.jar \ No newline at end of file +SIGNING_KEY="" \ + java -XX:+UseParallelGC -cp package/package/build/libs/package.jar net.runelite.pluginhub.packager.Package + +if [[ "${TRAVIS_PULL_REQUEST:-false}" != "false" ]]; then + java -XX:+UseParallelGC -cp package/package/build/libs/package.jar net.runelite.pluginhub.packager.Uploader +fi \ No newline at end of file diff --git a/package/upload/build.gradle b/package/upload/build.gradle new file mode 100644 index 000000000..426e87560 --- /dev/null +++ b/package/upload/build.gradle @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2020 Abex + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +repositories { + mavenCentral() +} + +dependencies { + implementation "com.google.code.findbugs:jsr305:3.0.2" + implementation "com.google.guava:guava:23.2-jre" + implementation "com.squareup.okhttp3:okhttp:3.14.9" + implementation "com.google.code.gson:gson:2.8.5" + + def lombok = "org.projectlombok:lombok:1.18.4"; + compileOnly lombok + annotationProcessor lombok + testCompileOnly lombok + testAnnotationProcessor lombok + + testImplementation "junit:junit:4.12" + testImplementation "com.squareup.okhttp3:mockwebserver:3.14.9" +} + +task run(type: JavaExec) { + main = "net.runelite.pluginhub.uploader.Uploader" + classpath = sourceSets.main.runtimeClasspath + workingDir = new File(project.rootDir, "../") +} + +test { + workingDir new File(project.rootDir, "../") +} \ No newline at end of file diff --git a/package/package/src/main/java/net/runelite/pluginhub/packager/ExternalPluginManifest.java b/package/upload/src/main/java/net/runelite/pluginhub/uploader/ExternalPluginManifest.java similarity index 97% rename from package/package/src/main/java/net/runelite/pluginhub/packager/ExternalPluginManifest.java rename to package/upload/src/main/java/net/runelite/pluginhub/uploader/ExternalPluginManifest.java index 959721d19..e78b354c5 100644 --- a/package/package/src/main/java/net/runelite/pluginhub/packager/ExternalPluginManifest.java +++ b/package/upload/src/main/java/net/runelite/pluginhub/uploader/ExternalPluginManifest.java @@ -22,7 +22,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package net.runelite.pluginhub.packager; +package net.runelite.pluginhub.uploader; import java.net.URL; import javax.annotation.Nullable; diff --git a/package/upload/src/main/java/net/runelite/pluginhub/uploader/ManifestDiff.java b/package/upload/src/main/java/net/runelite/pluginhub/uploader/ManifestDiff.java new file mode 100644 index 000000000..f0645f405 --- /dev/null +++ b/package/upload/src/main/java/net/runelite/pluginhub/uploader/ManifestDiff.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 Abex + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package net.runelite.pluginhub.uploader; + +import com.google.common.collect.Sets; +import java.util.Set; +import lombok.Getter; +import lombok.Setter; + +public class ManifestDiff +{ + @Getter + private Set remove = Sets.newConcurrentHashSet(); + + @Getter + private Set add = Sets.newConcurrentHashSet(); + + @Getter + @Setter + private boolean ignoreOldManifest; +} diff --git a/package/upload/src/main/java/net/runelite/pluginhub/uploader/SigningConfiguration.java b/package/upload/src/main/java/net/runelite/pluginhub/uploader/SigningConfiguration.java new file mode 100644 index 000000000..65a1f6ebc --- /dev/null +++ b/package/upload/src/main/java/net/runelite/pluginhub/uploader/SigningConfiguration.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2020 Abex + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package net.runelite.pluginhub.uploader; + +import java.security.InvalidKeyException; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PublicKey; +import java.security.Signature; +import java.security.SignatureException; +import java.security.interfaces.RSAPrivateCrtKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.RSAPublicKeySpec; +import java.util.Base64; + +public class SigningConfiguration +{ + private final RSAPrivateCrtKey key; + private final PublicKey cert; + + public static SigningConfiguration fromEnvironment() + { + return new SigningConfiguration(System.getenv("SIGNING_KEY")); + } + + public SigningConfiguration(String keyStr) + { + try + { + KeyFactory kf = KeyFactory.getInstance("RSA"); + + byte[] pkcs8 = Base64.getMimeDecoder().decode(keyStr + .replace("\\n", "\n") + .replaceAll(" |-----(BEGIN|END) PRIVATE KEY-----(\n?)", "")); + key = (RSAPrivateCrtKey) kf.generatePrivate(new PKCS8EncodedKeySpec(pkcs8)); + cert = kf.generatePublic(new RSAPublicKeySpec(key.getModulus(), key.getPublicExponent())); + } + catch (NoSuchAlgorithmException | InvalidKeySpecException e) + { + throw new RuntimeException(e); + } + } + + public byte[] sign(byte[] data) throws NoSuchAlgorithmException, InvalidKeyException, SignatureException + { + Signature s = Signature.getInstance("SHA256withRSA"); + s.initSign(key); + s.update(data); + return s.sign(); + } + + public boolean verify(byte[] sig, byte[] data) throws NoSuchAlgorithmException, InvalidKeyException, SignatureException + { + Signature s = Signature.getInstance("SHA256withRSA"); + s.initVerify(cert); + s.update(data); + return s.verify(sig); + } +} diff --git a/package/package/src/main/java/net/runelite/pluginhub/packager/UploadConfiguration.java b/package/upload/src/main/java/net/runelite/pluginhub/uploader/UploadConfiguration.java similarity index 73% rename from package/package/src/main/java/net/runelite/pluginhub/packager/UploadConfiguration.java rename to package/upload/src/main/java/net/runelite/pluginhub/uploader/UploadConfiguration.java index 1bad84e1b..253bc257d 100644 --- a/package/package/src/main/java/net/runelite/pluginhub/packager/UploadConfiguration.java +++ b/package/upload/src/main/java/net/runelite/pluginhub/uploader/UploadConfiguration.java @@ -22,19 +22,13 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package net.runelite.pluginhub.packager; +package net.runelite.pluginhub.uploader; import com.google.common.base.Strings; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.security.KeyFactory; -import java.security.NoSuchAlgorithmException; -import java.security.PublicKey; -import java.security.interfaces.RSAPrivateCrtKey; -import java.security.spec.InvalidKeySpecException; -import java.security.spec.PKCS8EncodedKeySpec; -import java.security.spec.RSAPublicKeySpec; import java.util.Base64; import lombok.Getter; import lombok.Setter; @@ -47,10 +41,8 @@ import okhttp3.Response; @Getter @Accessors(chain = true) -public class UploadConfiguration +public class UploadConfiguration implements Closeable { - private RSAPrivateCrtKey key; - private PublicKey cert; private OkHttpClient client; @Setter @@ -64,7 +56,6 @@ public class UploadConfiguration return this; } - setKey(System.getenv("SIGNING_KEY")); setClient(System.getenv("REPO_CREDS")); String uploadRepoRootStr = System.getenv("REPO_ROOT"); @@ -81,34 +72,7 @@ public class UploadConfiguration public boolean isComplete() { - return key != null && cert != null && client != null && uploadRepoRoot != null; - } - - public UploadConfiguration setKey(String keyStr) - { - if (keyStr == null) - { - key = null; - cert = null; - return this; - } - - try - { - KeyFactory kf = KeyFactory.getInstance("RSA"); - - byte[] pkcs8 = Base64.getMimeDecoder().decode(keyStr - .replace("\\n", "\n") - .replaceAll(" |-----(BEGIN|END) PRIVATE KEY-----(\n?)", "")); - key = (RSAPrivateCrtKey) kf.generatePrivate(new PKCS8EncodedKeySpec(pkcs8)); - cert = kf.generatePublic(new RSAPublicKeySpec(key.getModulus(), key.getPublicExponent())); - } - catch (NoSuchAlgorithmException | InvalidKeySpecException e) - { - throw new RuntimeException(e); - } - - return this; + return client != null && uploadRepoRoot != null; } public UploadConfiguration setClient(String credentials) @@ -154,4 +118,13 @@ public class UploadConfiguration Util.check(res); } } + + @Override + public void close() + { + if (client != null) + { + client.connectionPool().evictAll(); + } + } } diff --git a/package/upload/src/main/java/net/runelite/pluginhub/uploader/Uploader.java b/package/upload/src/main/java/net/runelite/pluginhub/uploader/Uploader.java new file mode 100644 index 000000000..2832049b1 --- /dev/null +++ b/package/upload/src/main/java/net/runelite/pluginhub/uploader/Uploader.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2020 Abex + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package net.runelite.pluginhub.uploader; + +import com.google.common.io.Files; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.SignatureException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import okhttp3.HttpUrl; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okio.BufferedSource; + +public class Uploader +{ + public static void main(String... args) throws IOException, NoSuchAlgorithmException, SignatureException, InvalidKeyException + { + Gson gson = new Gson(); + + String diffJSON = Files.asCharSource(new File("/tmp/manifest_diff"), StandardCharsets.UTF_8) + .read(); + ManifestDiff diff = gson.fromJson(diffJSON, ManifestDiff.class); + + try (UploadConfiguration uploadConfig = new UploadConfiguration().fromEnvironment(Util.readRLVersion())) + { + SigningConfiguration signingConfig = SigningConfiguration.fromEnvironment(); + + HttpUrl manifestURL = uploadConfig.getUploadRepoRoot().newBuilder() + .addPathSegment("manifest.js") + .build(); + + List manifests = new ArrayList<>(); + if (!diff.isIgnoreOldManifest()) + { + try (Response res = uploadConfig.getClient().newCall(new Request.Builder() + .url(manifestURL) + .get() + .build()) + .execute()) + { + if (res.code() != 404) + { + Util.check(res); + + BufferedSource src = res.body().source(); + + byte[] signature = new byte[src.readInt()]; + src.readFully(signature); + + byte[] data = src.readByteArray(); + if (!signingConfig.verify(signature, data)) + { + throw new RuntimeException("Unable to verify external plugin manifest"); + } + + manifests = gson.fromJson(new String(data, StandardCharsets.UTF_8), + new TypeToken>() + { + }.getType()); + } + } + } + + manifests.removeIf(m -> diff.getRemove().contains(m.getInternalName())); + manifests.addAll(diff.getAdd()); + manifests.sort(Comparator.comparing(ExternalPluginManifest::getInternalName)); + + { + byte[] data = gson.toJson(manifests).getBytes(StandardCharsets.UTF_8); + byte[] sig = signingConfig.sign(data); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new DataOutputStream(out).writeInt(sig.length); + out.write(sig); + out.write(data); + byte[] manifest = out.toByteArray(); + + try (Response res = uploadConfig.getClient().newCall(new Request.Builder() + .url(manifestURL) + .put(RequestBody.create(null, manifest)) + .build()) + .execute()) + { + Util.check(res); + } + } + } + } +} diff --git a/package/package/src/main/java/net/runelite/pluginhub/packager/Util.java b/package/upload/src/main/java/net/runelite/pluginhub/uploader/Util.java similarity index 73% rename from package/package/src/main/java/net/runelite/pluginhub/packager/Util.java rename to package/upload/src/main/java/net/runelite/pluginhub/uploader/Util.java index a81ad2894..b5c36689b 100644 --- a/package/package/src/main/java/net/runelite/pluginhub/packager/Util.java +++ b/package/upload/src/main/java/net/runelite/pluginhub/uploader/Util.java @@ -22,10 +22,12 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package net.runelite.pluginhub.packager; +package net.runelite.pluginhub.uploader; +import com.google.common.io.Files; +import java.io.File; import java.io.IOException; -import java.util.concurrent.TimeUnit; +import java.nio.charset.StandardCharsets; import okhttp3.Response; public class Util @@ -34,27 +36,6 @@ public class Util { } - public static void waitAndCheck(Plugin plugin, Process process, String name, long timeout, TimeUnit timeoutUnit) throws PluginBuildException - { - try - { - if (!process.waitFor(timeout, timeoutUnit)) - { - process.destroy(); - throw PluginBuildException.of(plugin, name + " failed to complete in a reasonable time"); - } - } - catch (InterruptedException e) - { - throw new RuntimeException(e); - } - - if (process.exitValue() != 0) - { - throw PluginBuildException.of(plugin, name + " exited with " + process.exitValue()); - } - } - public static void check(Response res) throws IOException { if ((res.code() / 100) != 2) @@ -62,4 +43,9 @@ public class Util throw new IOException(res.request().url() + ": " + res.code() + " " + res.message()); } } + + public static String readRLVersion() throws IOException + { + return Files.asCharSource(new File("./runelite.version"), StandardCharsets.UTF_8).read().trim(); + } } diff --git a/package/package/src/test/java/net/runelite/pluginhub/packager/UploadConfigurationTest.java b/package/upload/src/test/java/net/runelite/pluginhub/uploader/SigningConfigurationTest.java similarity index 68% rename from package/package/src/test/java/net/runelite/pluginhub/packager/UploadConfigurationTest.java rename to package/upload/src/test/java/net/runelite/pluginhub/uploader/SigningConfigurationTest.java index 6021cfee1..7d0a3e3a3 100644 --- a/package/package/src/test/java/net/runelite/pluginhub/packager/UploadConfigurationTest.java +++ b/package/upload/src/test/java/net/runelite/pluginhub/uploader/SigningConfigurationTest.java @@ -22,25 +22,16 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package net.runelite.pluginhub.packager; +package net.runelite.pluginhub.uploader; -import java.io.IOException; import java.nio.charset.StandardCharsets; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; -import java.security.Signature; import java.security.SignatureException; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import okhttp3.mockwebserver.RecordedRequest; import org.junit.Assert; import org.junit.Test; -public class UploadConfigurationTest +public class SigningConfigurationTest { public static final String TEST_SIGNING_KEY = "-----BEGIN PRIVATE KEY-----\n" + "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDw78Jgex/z/Wqp\n" + @@ -71,59 +62,15 @@ public class UploadConfigurationTest "PTrpFFdp8oDxvgezLBFzqd8=\n" + "-----END PRIVATE KEY-----"; - @Test - public void createClientWithCredentials() throws IOException, InterruptedException - { - MockWebServer server = new MockWebServer(); - - server.enqueue(new MockResponse().setResponseCode(520).setBody("some cloudflare html")); - server.enqueue(new MockResponse().setResponseCode(200).setBody("ok")); - - OkHttpClient client = new UploadConfiguration() - .setClient("Aladdin:open sesame") - .getClient(); - - try (Response res = client.newCall(new Request.Builder() - .put(RequestBody.create(null, "foo")) - .url(server.url("/")) - .build()) - .execute()) - { - Assert.assertEquals(res.code(), 200); - Assert.assertEquals(res.body().string(), "ok"); - } - - RecordedRequest r2 = server.takeRequest(); - Assert.assertEquals(r2.getHeader("Authorization"), "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="); - } @Test public void testSigning() throws NoSuchAlgorithmException, InvalidKeyException, SignatureException { - UploadConfiguration cfg = new UploadConfiguration() - .setKey(TEST_SIGNING_KEY); + SigningConfiguration cfg = new SigningConfiguration(TEST_SIGNING_KEY); byte[] data = "Hello World".getBytes(StandardCharsets.UTF_8); - byte[] sig; - { - Signature s = Signature.getInstance("SHA256withRSA"); - s.initSign(cfg.getKey()); - s.update(data); - sig = s.sign(); - } - - { - Signature s = Signature.getInstance("SHA256withRSA"); - s.initVerify(cfg.getCert()); - s.update(data); - Assert.assertTrue(s.verify(sig)); - } - - { - Signature s = Signature.getInstance("SHA256withRSA"); - s.initVerify(cfg.getCert()); - s.update("moo".getBytes(StandardCharsets.UTF_8)); - Assert.assertFalse(s.verify(sig)); - } + byte[] sig = cfg.sign(data); + Assert.assertTrue(cfg.verify(sig, data)); + Assert.assertFalse(cfg.verify(sig, "moo".getBytes(StandardCharsets.UTF_8))); } } \ No newline at end of file diff --git a/package/upload/src/test/java/net/runelite/pluginhub/uploader/UploadConfigurationTest.java b/package/upload/src/test/java/net/runelite/pluginhub/uploader/UploadConfigurationTest.java new file mode 100644 index 000000000..8282298e4 --- /dev/null +++ b/package/upload/src/test/java/net/runelite/pluginhub/uploader/UploadConfigurationTest.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2020 Abex + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package net.runelite.pluginhub.uploader; + +import java.io.IOException; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.Assert; +import org.junit.Test; + +public class UploadConfigurationTest +{ + @Test + public void createClientWithCredentials() throws IOException, InterruptedException + { + MockWebServer server = new MockWebServer(); + + server.enqueue(new MockResponse().setResponseCode(520).setBody("some cloudflare html")); + server.enqueue(new MockResponse().setResponseCode(200).setBody("ok")); + + OkHttpClient client = new UploadConfiguration() + .setClient("Aladdin:open sesame") + .getClient(); + + try (Response res = client.newCall(new Request.Builder() + .put(RequestBody.create(null, "foo")) + .url(server.url("/")) + .build()) + .execute()) + { + Assert.assertEquals(res.code(), 200); + Assert.assertEquals(res.body().string(), "ok"); + } + + RecordedRequest r2 = server.takeRequest(); + Assert.assertEquals(r2.getHeader("Authorization"), "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="); + } +} \ No newline at end of file