mirror/plugin-hub
1
0
Fork 0
mirror of https://github.com/runelite/plugin-hub.git synced 2025-12-23 22:48:49 -05:00
Code Issues Packages Projects Releases Wiki Activity

package: split manifest uploading into it's own job

Browse Source
This commit is contained in:
Max Weber
2020-10-29 14:09:22 -06:00
parent da8574e3ed
commit 66edf61da6
16 changed files with 505 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
.github/workflows/build.yml vendored
View File

@@ -5,12 +5,12 @@ on:
description: "List of plugins to build, or 'ALL'"
required: false
COMMIT_RANGE:
description: "Commit range to build 1234abc...5689def"
description: "Commit range to build 1234abc..5689def"
required: false
push:
pull_request:
jobs:
execute:
build:
# any forks that predate this repo having an action will have actions
# enabled by default, which will fail in a lot of cases because the branch
# is new, which makes the differential build fail
@@ -28,7 +28,7 @@ jobs:
path: |
~/.gradle/caches/
~/.gradle/wrapper/
key: package-2.0.1
key: package-2.0.2
- name: prepare
run: |
pushd package
@@ -38,7 +38,6 @@ jobs:
env:
REPO_CREDS: ${{ secrets.REPO_CREDS }}
REPO_ROOT: ${{ secrets.REPO_ROOT }}
SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
# workflow_dispatch
FORCE_BUILD: ${{ github.event.inputs.FORCE_BUILD }}
COMMIT_RANGE: ${{ github.event.inputs.COMMIT_RANGE }}
@@ -55,4 +54,39 @@ jobs:
else
export PACKAGE_COMMIT_RANGE="${COMMIT_RANGE:-${COMMIT_BEFORE:+$COMMIT_BEFORE...$COMMIT_AFTER}}"
fi
java -XX:+UseParallelGC -jar package/package/build/libs/package.jar
exec java -XX:+UseParallelGC -cp package/package/build/libs/package.jar net.runelite.pluginhub.packager.Packager
- uses: actions/upload-artifact@v2
with:
name: manifest_diff
path: /tmp/manifest_diff
retention-days: 1
upload:
if: (github.event_name != 'push' || github.repository_owner == 'runelite') && github.event_name != 'pull_request'
needs: build
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- uses: actions/setup-java@v1
with:
java-version: 11
- uses: actions/cache@v2
with:
path: |
~/.gradle/caches/
~/.gradle/wrapper/
key: upload-2.0.2
- uses: actions/download-artifact@v2
with:
name: manifest_diff
path: /tmp
- name: upload
env:
REPO_CREDS: ${{ secrets.REPO_CREDS }}
REPO_ROOT: ${{ secrets.REPO_ROOT }}
SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
run: |
pushd package
./gradlew :upload:run
popd

2
package/package/build.gradle
View File

@@ -37,6 +37,7 @@ dependencies {
implementation "org.ow2.asm:asm:7.0"
implementation "com.squareup.okhttp3:okhttp:3.14.9"
implementation "com.google.code.gson:gson:2.8.5"
implementation project(":upload")
def lombok = "org.projectlombok:lombok:1.18.4";
compileOnly lombok
@@ -55,5 +56,6 @@ jar {
}
test {
dependsOn ":initLib:shadowJar"
workingDir new File(project.rootDir, "../")
}

140
package/package/src/main/java/net/runelite/pluginhub/packager/Packager.java
View File

@@ -28,29 +28,20 @@ import com.google.common.base.Splitter;
import com.google.common.base.Stopwatch;
import com.google.common.base.Strings;
import com.google.common.collect.Queues;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.Signature;
import java.security.SignatureException;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -61,14 +52,12 @@ import java.util.stream.StreamSupport;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import okhttp3.HttpUrl;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okio.BufferedSource;
import net.runelite.pluginhub.uploader.ManifestDiff;
import net.runelite.pluginhub.uploader.UploadConfiguration;
import net.runelite.pluginhub.uploader.Util;
@Slf4j
public class Packager
public class Packager implements Closeable
{
private static final File PLUGIN_ROOT = new File("./plugins");
public static final File PACKAGE_ROOT = new File("./package/").getAbsoluteFile();
@@ -88,27 +77,22 @@ public class Packager
private final AtomicInteger numDone = new AtomicInteger(0);
private final int numTotal;
@Setter
private boolean ignoreOldManifest;
@Setter
private boolean alwaysPrintLog;
@Getter
private boolean failed;
private ManifestDiff diff = new ManifestDiff();
public Packager(List<File> buildList) throws IOException
{
this.buildList = buildList;
this.numTotal = buildList.size();
this.runeliteVersion = readRLVersion();
this.runeliteVersion = Util.readRLVersion();
}
Set<ExternalPluginManifest> newManifests = Sets.newConcurrentHashSet();
Set<String> remove = Sets.newConcurrentHashSet();
public void buildPlugins()
throws IOException, InvalidKeyException, NoSuchAlgorithmException, SignatureException
public void buildPlugins() throws IOException
{
Queue<File> buildQueue = Queues.synchronizedQueue(new ArrayDeque<>(buildList));
List<Thread> buildThreads = IntStream.range(0, 8)
@@ -137,84 +121,20 @@ public class Packager
}
}
if (uploadConfig.isComplete())
Gson gson = new Gson();
String diffJSON = gson.toJson(diff);
log.info("manifest change: {}", diffJSON);
try (FileOutputStream fos = new FileOutputStream("/tmp/manifest_diff"))
{
Gson gson = new Gson();
HttpUrl manifestURL = uploadConfig.getUploadRepoRoot().newBuilder()
.addPathSegment("manifest.js")
.build();
List<ExternalPluginManifest> manifests = new ArrayList<>();
if (!ignoreOldManifest)
{
try (Response res = uploadConfig.getClient().newCall(new Request.Builder()
.url(manifestURL)
.get()
.build())
.execute())
{
if (res.code() != 404)
{
Util.check(res);
BufferedSource src = res.body().source();
byte[] signature = new byte[src.readInt()];
src.readFully(signature);
byte[] data = src.readByteArray();
Signature s = Signature.getInstance("SHA256withRSA");
s.initVerify(uploadConfig.getCert());
s.update(data);
if (!s.verify(signature))
{
throw new RuntimeException("Unable to verify external plugin manifest");
}
manifests = gson.fromJson(new String(data, StandardCharsets.UTF_8),
new TypeToken<List<ExternalPluginManifest>>()
{
}.getType());
}
}
}
manifests.removeIf(m -> remove.contains(m.getInternalName()));
manifests.addAll(newManifests);
manifests.sort(Comparator.comparing(ExternalPluginManifest::getInternalName));
{
byte[] data = gson.toJson(manifests).getBytes(StandardCharsets.UTF_8);
Signature s = Signature.getInstance("SHA256withRSA");
s.initSign(uploadConfig.getKey());
s.update(data);
byte[] sig = s.sign();
ByteArrayOutputStream out = new ByteArrayOutputStream();
new DataOutputStream(out).writeInt(sig.length);
out.write(sig);
out.write(data);
byte[] manifest = out.toByteArray();
try (Response res = uploadConfig.getClient().newCall(new Request.Builder()
.url(manifestURL)
.put(RequestBody.create(null, manifest))
.build())
.execute())
{
Util.check(res);
}
}
uploadConfig.getClient().connectionPool().evictAll();
fos.write(diffJSON.getBytes(StandardCharsets.UTF_8));
}
}
private void buildPlugin(File plugin)
{
remove.add(plugin.getName());
diff.getRemove().add(plugin.getName());
if (!plugin.exists())
{
@@ -245,7 +165,7 @@ public class Packager
p.uploadLog(uploadConfig);
}
newManifests.add(p.getManifest());
diff.getAdd().add(p.getManifest());
log.info("{}: done in {}ms [{}/{}]", p.getInternalName(), p.getBuildTimeMS(), numDone.get() + 1, numTotal);
}
catch (PluginBuildException e)
@@ -327,11 +247,16 @@ public class Packager
};
}
public static String readRLVersion() throws IOException
public void setIgnoreOldManifest(boolean ignore)
{
return Files.asCharSource(new File("./runelite.version"), StandardCharsets.UTF_8).read().trim();
diff.setIgnoreOldManifest(ignore);
}
@Override
public void close()
{
uploadConfig.close();
}
public static void main(String... args) throws Exception
{
@@ -347,6 +272,7 @@ public class Packager
else if ("ALL".equals(System.getenv("FORCE_BUILD")))
{
buildList = listAllPlugins();
isBuildingAll = true;
}
else if (!Strings.isNullOrEmpty(System.getenv("FORCE_BUILD")))
{
@@ -413,13 +339,17 @@ public class Packager
throw new RuntimeException("missing env vars");
}
Packager pkg = new Packager(buildList);
pkg.getUploadConfig().fromEnvironment(pkg.getRuneliteVersion());
pkg.setAlwaysPrintLog(!pkg.getUploadConfig().isComplete());
pkg.setIgnoreOldManifest(isBuildingAll);
pkg.buildPlugins();
boolean failed;
try (Packager pkg = new Packager(buildList))
{
pkg.getUploadConfig().fromEnvironment(pkg.getRuneliteVersion());
pkg.setAlwaysPrintLog(!pkg.getUploadConfig().isComplete());
pkg.setIgnoreOldManifest(isBuildingAll);
pkg.buildPlugins();
failed = pkg.isFailed();
}
if (testFailure || pkg.isFailed() && !isBuildingAll)
if (testFailure || (failed && !isBuildingAll))
{
System.exit(1);
}

27
package/package/src/main/java/net/runelite/pluginhub/packager/Plugin.java
View File

@@ -63,6 +63,8 @@ import javax.imageio.ImageIO;
import lombok.Getter;
import lombok.Setter;
import lombok.SneakyThrows;
import net.runelite.pluginhub.uploader.ExternalPluginManifest;
import net.runelite.pluginhub.uploader.UploadConfiguration;
import okhttp3.HttpUrl;
import org.gradle.tooling.CancellationTokenSource;
import org.gradle.tooling.GradleConnectionException;
@@ -211,13 +213,34 @@ public class Plugin implements Closeable
iconFile = new File(repositoryDirectory, "icon.png");
}
private void waitAndCheck(Process process, String name, long timeout, TimeUnit timeoutUnit) throws PluginBuildException
{
try
{
if (!process.waitFor(timeout, timeoutUnit))
{
process.destroy();
throw PluginBuildException.of(this, name + " failed to complete in a reasonable time");
}
}
catch (InterruptedException e)
{
throw new RuntimeException(e);
}
if (process.exitValue() != 0)
{
throw PluginBuildException.of(this, name + " exited with " + process.exitValue());
}
}
public void download() throws IOException, PluginBuildException
{
Process gitclone = new ProcessBuilder("git", "clone", "--config", "advice.detachedHead=false", this.repositoryURL, repositoryDirectory.getAbsolutePath())
.redirectOutput(ProcessBuilder.Redirect.appendTo(logFile))
.redirectError(ProcessBuilder.Redirect.appendTo(logFile))
.start();
Util.waitAndCheck(this, gitclone, "git clone", 2, TimeUnit.MINUTES);
waitAndCheck(gitclone, "git clone", 2, TimeUnit.MINUTES);
Process gitcheckout = new ProcessBuilder("git", "checkout", commit + "^{commit}")
@@ -225,7 +248,7 @@ public class Plugin implements Closeable
.redirectError(ProcessBuilder.Redirect.appendTo(logFile))
.directory(repositoryDirectory)
.start();
Util.waitAndCheck(this, gitcheckout, "git checkout", 2, TimeUnit.MINUTES);
waitAndCheck(gitcheckout, "git checkout", 2, TimeUnit.MINUTES);
}
public void build(String runeliteVersion) throws IOException, PluginBuildException

5
package/package/src/test/java/net/runelite/pluginhub/packager/PluginTest.java
View File

@@ -31,6 +31,7 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Properties;
import lombok.extern.slf4j.Slf4j;
import net.runelite.pluginhub.uploader.Util;
import org.junit.Assert;
import org.junit.Test;
@@ -74,7 +75,7 @@ public class PluginTest
{
try (Plugin p = createExamplePlugin("example"))
{
p.build(Packager.readRLVersion());
p.build(Util.readRLVersion());
p.assembleManifest();
}
}
@@ -88,7 +89,7 @@ public class PluginTest
Properties props = Plugin.loadProperties(propFile);
props.setProperty("plugins", "com.nonexistent");
writeProperties(props, propFile);
p.build(Packager.readRLVersion());
p.build(Util.readRLVersion());
p.assembleManifest();
Assert.fail();
}

1
package/settings.gradle
View File

@@ -1,4 +1,5 @@
rootProject.name = "package-root"
include "initLib"
include "upload"
include "package"

7
package/travis.sh
View File

@@ -35,4 +35,9 @@ popd
PACKAGE_IS_PR="$TRAVIS_PULL_REQUEST" \
PACKAGE_COMMIT_RANGE="$TRAVIS_COMMIT_RANGE" \
java -XX:+UseParallelGC -jar package/package/build/libs/package.jar
SIGNING_KEY="" \
java -XX:+UseParallelGC -cp package/package/build/libs/package.jar net.runelite.pluginhub.packager.Package
if [[ "${TRAVIS_PULL_REQUEST:-false}" != "false" ]]; then
java -XX:+UseParallelGC -cp package/package/build/libs/package.jar net.runelite.pluginhub.packager.Uploader
fi

53
package/upload/build.gradle Normal file
View File

@@ -0,0 +1,53 @@
/*
* Copyright (c) 2020 Abex
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
repositories {
mavenCentral()
}
dependencies {
implementation "com.google.code.findbugs:jsr305:3.0.2"
implementation "com.google.guava:guava:23.2-jre"
implementation "com.squareup.okhttp3:okhttp:3.14.9"
implementation "com.google.code.gson:gson:2.8.5"
def lombok = "org.projectlombok:lombok:1.18.4";
compileOnly lombok
annotationProcessor lombok
testCompileOnly lombok
testAnnotationProcessor lombok
testImplementation "junit:junit:4.12"
testImplementation "com.squareup.okhttp3:mockwebserver:3.14.9"
}
task run(type: JavaExec) {
main = "net.runelite.pluginhub.uploader.Uploader"
classpath = sourceSets.main.runtimeClasspath
workingDir = new File(project.rootDir, "../")
}
test {
workingDir new File(project.rootDir, "../")
}

2
package/package/src/main/java/net/runelite/pluginhub/packager/ExternalPluginManifest.java → package/upload/src/main/java/net/runelite/pluginhub/uploader/ExternalPluginManifest.java
View File

@@ -22,7 +22,7 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.pluginhub.packager;
package net.runelite.pluginhub.uploader;
import java.net.URL;
import javax.annotation.Nullable;

43
package/upload/src/main/java/net/runelite/pluginhub/uploader/ManifestDiff.java Normal file
View File

@@ -0,0 +1,43 @@
/*
* Copyright (c) 2020 Abex
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.pluginhub.uploader;
import com.google.common.collect.Sets;
import java.util.Set;
import lombok.Getter;
import lombok.Setter;
public class ManifestDiff
{
@Getter
private Set<String> remove = Sets.newConcurrentHashSet();
@Getter
private Set<ExternalPluginManifest> add = Sets.newConcurrentHashSet();
@Getter
@Setter
private boolean ignoreOldManifest;
}

82
package/upload/src/main/java/net/runelite/pluginhub/uploader/SigningConfiguration.java Normal file
View File

@@ -0,0 +1,82 @@
/*
* Copyright (c) 2020 Abex
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.pluginhub.uploader;
import java.security.InvalidKeyException;
import java.security.KeyFactory;
import java.security.NoSuchAlgorithmException;
import java.security.PublicKey;
import java.security.Signature;
import java.security.SignatureException;
import java.security.interfaces.RSAPrivateCrtKey;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.PKCS8EncodedKeySpec;
import java.security.spec.RSAPublicKeySpec;
import java.util.Base64;
public class SigningConfiguration
{
private final RSAPrivateCrtKey key;
private final PublicKey cert;
public static SigningConfiguration fromEnvironment()
{
return new SigningConfiguration(System.getenv("SIGNING_KEY"));
}
public SigningConfiguration(String keyStr)
{
try
{
KeyFactory kf = KeyFactory.getInstance("RSA");
byte[] pkcs8 = Base64.getMimeDecoder().decode(keyStr
.replace("\\n", "\n")
.replaceAll(" |-----(BEGIN|END) PRIVATE KEY-----(\n?)", ""));
key = (RSAPrivateCrtKey) kf.generatePrivate(new PKCS8EncodedKeySpec(pkcs8));
cert = kf.generatePublic(new RSAPublicKeySpec(key.getModulus(), key.getPublicExponent()));
}
catch (NoSuchAlgorithmException | InvalidKeySpecException e)
{
throw new RuntimeException(e);
}
}
public byte[] sign(byte[] data) throws NoSuchAlgorithmException, InvalidKeyException, SignatureException
{
Signature s = Signature.getInstance("SHA256withRSA");
s.initSign(key);
s.update(data);
return s.sign();
}
public boolean verify(byte[] sig, byte[] data) throws NoSuchAlgorithmException, InvalidKeyException, SignatureException
{
Signature s = Signature.getInstance("SHA256withRSA");
s.initVerify(cert);
s.update(data);
return s.verify(sig);
}
}

53
package/package/src/main/java/net/runelite/pluginhub/packager/UploadConfiguration.java → package/upload/src/main/java/net/runelite/pluginhub/uploader/UploadConfiguration.java
View File

@@ -22,19 +22,13 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.pluginhub.packager;
package net.runelite.pluginhub.uploader;
import com.google.common.base.Strings;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.security.KeyFactory;
import java.security.NoSuchAlgorithmException;
import java.security.PublicKey;
import java.security.interfaces.RSAPrivateCrtKey;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.PKCS8EncodedKeySpec;
import java.security.spec.RSAPublicKeySpec;
import java.util.Base64;
import lombok.Getter;
import lombok.Setter;
@@ -47,10 +41,8 @@ import okhttp3.Response;
@Getter
@Accessors(chain = true)
public class UploadConfiguration
public class UploadConfiguration implements Closeable
{
private RSAPrivateCrtKey key;
private PublicKey cert;
private OkHttpClient client;
@Setter
@@ -64,7 +56,6 @@ public class UploadConfiguration
return this;
}
setKey(System.getenv("SIGNING_KEY"));
setClient(System.getenv("REPO_CREDS"));
String uploadRepoRootStr = System.getenv("REPO_ROOT");
@@ -81,34 +72,7 @@ public class UploadConfiguration
public boolean isComplete()
{
return key != null && cert != null && client != null && uploadRepoRoot != null;
}
public UploadConfiguration setKey(String keyStr)
{
if (keyStr == null)
{
key = null;
cert = null;
return this;
}
try
{
KeyFactory kf = KeyFactory.getInstance("RSA");
byte[] pkcs8 = Base64.getMimeDecoder().decode(keyStr
.replace("\\n", "\n")
.replaceAll(" |-----(BEGIN|END) PRIVATE KEY-----(\n?)", ""));
key = (RSAPrivateCrtKey) kf.generatePrivate(new PKCS8EncodedKeySpec(pkcs8));
cert = kf.generatePublic(new RSAPublicKeySpec(key.getModulus(), key.getPublicExponent()));
}
catch (NoSuchAlgorithmException | InvalidKeySpecException e)
{
throw new RuntimeException(e);
}
return this;
return client != null && uploadRepoRoot != null;
}
public UploadConfiguration setClient(String credentials)
@@ -154,4 +118,13 @@ public class UploadConfiguration
Util.check(res);
}
}
@Override
public void close()
{
if (client != null)
{
client.connectionPool().evictAll();
}
}
}

122
package/upload/src/main/java/net/runelite/pluginhub/uploader/Uploader.java Normal file
View File

@@ -0,0 +1,122 @@
/*
* Copyright (c) 2020 Abex
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.pluginhub.uploader;
import com.google.common.io.Files;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import okhttp3.HttpUrl;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okio.BufferedSource;
public class Uploader
{
public static void main(String... args) throws IOException, NoSuchAlgorithmException, SignatureException, InvalidKeyException
{
Gson gson = new Gson();
String diffJSON = Files.asCharSource(new File("/tmp/manifest_diff"), StandardCharsets.UTF_8)
.read();
ManifestDiff diff = gson.fromJson(diffJSON, ManifestDiff.class);
try (UploadConfiguration uploadConfig = new UploadConfiguration().fromEnvironment(Util.readRLVersion()))
{
SigningConfiguration signingConfig = SigningConfiguration.fromEnvironment();
HttpUrl manifestURL = uploadConfig.getUploadRepoRoot().newBuilder()
.addPathSegment("manifest.js")
.build();
List<ExternalPluginManifest> manifests = new ArrayList<>();
if (!diff.isIgnoreOldManifest())
{
try (Response res = uploadConfig.getClient().newCall(new Request.Builder()
.url(manifestURL)
.get()
.build())
.execute())
{
if (res.code() != 404)
{
Util.check(res);
BufferedSource src = res.body().source();
byte[] signature = new byte[src.readInt()];
src.readFully(signature);
byte[] data = src.readByteArray();
if (!signingConfig.verify(signature, data))
{
throw new RuntimeException("Unable to verify external plugin manifest");
}
manifests = gson.fromJson(new String(data, StandardCharsets.UTF_8),
new TypeToken<List<ExternalPluginManifest>>()
{
}.getType());
}
}
}
manifests.removeIf(m -> diff.getRemove().contains(m.getInternalName()));
manifests.addAll(diff.getAdd());
manifests.sort(Comparator.comparing(ExternalPluginManifest::getInternalName));
{
byte[] data = gson.toJson(manifests).getBytes(StandardCharsets.UTF_8);
byte[] sig = signingConfig.sign(data);
ByteArrayOutputStream out = new ByteArrayOutputStream();
new DataOutputStream(out).writeInt(sig.length);
out.write(sig);
out.write(data);
byte[] manifest = out.toByteArray();
try (Response res = uploadConfig.getClient().newCall(new Request.Builder()
.url(manifestURL)
.put(RequestBody.create(null, manifest))
.build())
.execute())
{
Util.check(res);
}
}
}
}
}

32
package/package/src/main/java/net/runelite/pluginhub/packager/Util.java → package/upload/src/main/java/net/runelite/pluginhub/uploader/Util.java
View File

@@ -22,10 +22,12 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.pluginhub.packager;
package net.runelite.pluginhub.uploader;
import com.google.common.io.Files;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import java.nio.charset.StandardCharsets;
import okhttp3.Response;
public class Util
@@ -34,27 +36,6 @@ public class Util
{
}
public static void waitAndCheck(Plugin plugin, Process process, String name, long timeout, TimeUnit timeoutUnit) throws PluginBuildException
{
try
{
if (!process.waitFor(timeout, timeoutUnit))
{
process.destroy();
throw PluginBuildException.of(plugin, name + " failed to complete in a reasonable time");
}
}
catch (InterruptedException e)
{
throw new RuntimeException(e);
}
if (process.exitValue() != 0)
{
throw PluginBuildException.of(plugin, name + " exited with " + process.exitValue());
}
}
public static void check(Response res) throws IOException
{
if ((res.code() / 100) != 2)
@@ -62,4 +43,9 @@ public class Util
throw new IOException(res.request().url() + ": " + res.code() + " " + res.message());
}
}
public static String readRLVersion() throws IOException
{
return Files.asCharSource(new File("./runelite.version"), StandardCharsets.UTF_8).read().trim();
}
}

65
package/package/src/test/java/net/runelite/pluginhub/packager/UploadConfigurationTest.java → package/upload/src/test/java/net/runelite/pluginhub/uploader/SigningConfigurationTest.java
View File

@@ -22,25 +22,16 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.pluginhub.packager;
package net.runelite.pluginhub.uploader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.Signature;
import java.security.SignatureException;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.mockwebserver.MockResponse;
import okhttp3.mockwebserver.MockWebServer;
import okhttp3.mockwebserver.RecordedRequest;
import org.junit.Assert;
import org.junit.Test;
public class UploadConfigurationTest
public class SigningConfigurationTest
{
public static final String TEST_SIGNING_KEY = "-----BEGIN PRIVATE KEY-----\n" +
"MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDw78Jgex/z/Wqp\n" +
@@ -71,59 +62,15 @@ public class UploadConfigurationTest
"PTrpFFdp8oDxvgezLBFzqd8=\n" +
"-----END PRIVATE KEY-----";
@Test
public void createClientWithCredentials() throws IOException, InterruptedException
{
MockWebServer server = new MockWebServer();
server.enqueue(new MockResponse().setResponseCode(520).setBody("some cloudflare html"));
server.enqueue(new MockResponse().setResponseCode(200).setBody("ok"));
OkHttpClient client = new UploadConfiguration()
.setClient("Aladdin:open sesame")
.getClient();
try (Response res = client.newCall(new Request.Builder()
.put(RequestBody.create(null, "foo"))
.url(server.url("/"))
.build())
.execute())
{
Assert.assertEquals(res.code(), 200);
Assert.assertEquals(res.body().string(), "ok");
}
RecordedRequest r2 = server.takeRequest();
Assert.assertEquals(r2.getHeader("Authorization"), "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==");
}
@Test
public void testSigning() throws NoSuchAlgorithmException, InvalidKeyException, SignatureException
{
UploadConfiguration cfg = new UploadConfiguration()
.setKey(TEST_SIGNING_KEY);
SigningConfiguration cfg = new SigningConfiguration(TEST_SIGNING_KEY);
byte[] data = "Hello World".getBytes(StandardCharsets.UTF_8);
byte[] sig;
{
Signature s = Signature.getInstance("SHA256withRSA");
s.initSign(cfg.getKey());
s.update(data);
sig = s.sign();
}
{
Signature s = Signature.getInstance("SHA256withRSA");
s.initVerify(cfg.getCert());
s.update(data);
Assert.assertTrue(s.verify(sig));
}
{
Signature s = Signature.getInstance("SHA256withRSA");
s.initVerify(cfg.getCert());
s.update("moo".getBytes(StandardCharsets.UTF_8));
Assert.assertFalse(s.verify(sig));
}
byte[] sig = cfg.sign(data);
Assert.assertTrue(cfg.verify(sig, data));
Assert.assertFalse(cfg.verify(sig, "moo".getBytes(StandardCharsets.UTF_8)));
}
}

65
package/upload/src/test/java/net/runelite/pluginhub/uploader/UploadConfigurationTest.java Normal file
View File

@@ -0,0 +1,65 @@
/*
* Copyright (c) 2020 Abex
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package net.runelite.pluginhub.uploader;
import java.io.IOException;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.mockwebserver.MockResponse;
import okhttp3.mockwebserver.MockWebServer;
import okhttp3.mockwebserver.RecordedRequest;
import org.junit.Assert;
import org.junit.Test;
public class UploadConfigurationTest
{
@Test
public void createClientWithCredentials() throws IOException, InterruptedException
{
MockWebServer server = new MockWebServer();
server.enqueue(new MockResponse().setResponseCode(520).setBody("some cloudflare html"));
server.enqueue(new MockResponse().setResponseCode(200).setBody("ok"));
OkHttpClient client = new UploadConfiguration()
.setClient("Aladdin:open sesame")
.getClient();
try (Response res = client.newCall(new Request.Builder()
.put(RequestBody.create(null, "foo"))
.url(server.url("/"))
.build())
.execute())
{
Assert.assertEquals(res.code(), 200);
Assert.assertEquals(res.body().string(), "ok");
}
RecordedRequest r2 = server.takeRequest();
Assert.assertEquals(r2.getHeader("Authorization"), "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==");
}
}