mirror/fdroidclient
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidclient.git synced 2026-04-22 16:01:37 -04:00
Code Issues Packages Projects Releases Wiki Activity

[app] kill old IndexUpdaters and related code as this is now in libraries via RepoUpdater

Browse Source
This commit is contained in:
Torsten Grote
2022-06-15 10:19:32 -03:00
parent 8ace3e1129
commit 6ffb0dc8d7
20 changed files with 119 additions and 1858 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
app/src/androidTest/java/org/fdroid/fdroid/updater/SwapRepoEmulatorTest.java
View File

@@ -1,6 +1,5 @@
package org.fdroid.fdroid.updater;
import android.content.ContentValues;
import android.content.Context;
import android.content.Intent;
import android.content.pm.ApplicationInfo;
@@ -10,19 +9,11 @@ import androidx.test.platform.app.InstrumentationRegistry;
import androidx.test.filters.LargeTest;
import android.text.TextUtils;
import android.util.Log;
import org.fdroid.fdroid.BuildConfig;
import org.fdroid.fdroid.FDroidApp;
import org.fdroid.fdroid.Hasher;
import org.fdroid.fdroid.IndexUpdater;
import org.fdroid.fdroid.Preferences;
import org.fdroid.fdroid.Utils;
import org.fdroid.fdroid.data.Apk;
import org.fdroid.fdroid.data.ApkProvider;
import org.fdroid.fdroid.data.App;
import org.fdroid.fdroid.data.AppProvider;
import org.fdroid.fdroid.data.Repo;
import org.fdroid.fdroid.data.RepoProvider;
import org.fdroid.fdroid.data.Schema;
import org.fdroid.fdroid.nearby.LocalHTTPD;
import org.fdroid.fdroid.nearby.LocalRepoKeyStore;
import org.fdroid.fdroid.nearby.LocalRepoManager;
@@ -34,16 +25,12 @@ import java.io.File;
import java.io.IOException;
import java.net.Socket;
import java.security.cert.Certificate;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -58,7 +45,7 @@ public class SwapRepoEmulatorTest {
@Ignore
@Test
public void testSwap()
throws IOException, LocalRepoKeyStore.InitException, IndexUpdater.UpdateException, InterruptedException {
throws IOException, LocalRepoKeyStore.InitException, InterruptedException {
Looper.prepare();
LocalHTTPD localHttpd = null;
try {
@@ -108,50 +95,50 @@ public class SwapRepoEmulatorTest {
assertFalse(TextUtils.isEmpty(signingCert));
assertFalse(TextUtils.isEmpty(Utils.calcFingerprint(localCert)));
// Repo repoToDelete = RepoProvider.Helper.findByAddress(context, FDroidApp.repo.getAddress());
// while (repoToDelete != null) {
// Log.d(TAG, "Removing old test swap repo matching this one: " + repoToDelete.address);
// RepoProvider.Helper.remove(context, repoToDelete.getId());
// repoToDelete = RepoProvider.Helper.findByAddress(context, FDroidApp.repo.getAddress());
// }
//
// ContentValues values = new ContentValues(4);
// values.put(Schema.RepoTable.Cols.SIGNING_CERT, signingCert);
// values.put(Schema.RepoTable.Cols.ADDRESS, FDroidApp.repo.getAddress());
// values.put(Schema.RepoTable.Cols.NAME, "");
// values.put(Schema.RepoTable.Cols.IS_SWAP, true);
// final String lastEtag = UUID.randomUUID().toString();
// values.put(Schema.RepoTable.Cols.LAST_ETAG, lastEtag);
// RepoProvider.Helper.insert(context, values);
// Repo repo = RepoProvider.Helper.findByAddress(context, FDroidApp.repo.getAddress());
// assertTrue(repo.isSwap);
// assertNotEquals(-1, repo.getId());
// assertEquals(lastEtag, repo.lastetag);
// assertNull(repo.lastUpdated);
//
// assertTrue(isPortInUse(FDroidApp.ipAddressString, FDroidApp.port));
// Thread.sleep(100);
// IndexUpdater updater = new IndexUpdater(context, repo);
// updater.update();
// assertTrue(updater.hasChanged());
//
// repo = RepoProvider.Helper.findByAddress(context, FDroidApp.repo.getAddress());
// final Date lastUpdated = repo.lastUpdated;
// assertTrue("repo lastUpdated should be updated", new Date(2019, 5, 13).compareTo(repo.lastUpdated) > 0);
// App app = AppProvider.Helper.findSpecificApp(context.getContentResolver(),
// context.getPackageName(), repo.getId());
// assertEquals(context.getPackageName(), app.packageName);
// List<Apk> apks = ApkProvider.Helper.findByRepo(context, repo, Schema.ApkTable.Cols.ALL);
// assertEquals(1, apks.size());
// for (Apk apk : apks) {
// Log.i(TAG, "Apk: " + apk);
// assertEquals(context.getPackageName(), apk.packageName);
// assertEquals(BuildConfig.VERSION_NAME, apk.versionName);
// assertEquals(BuildConfig.VERSION_CODE, apk.versionCode);
// assertEquals(app.repoId, apk.repoId);
// }
//Repo repoToDelete = RepoProvider.Helper.findByAddress(context, FDroidApp.repo.getAddress());
//while (repoToDelete != null) {
// Log.d(TAG, "Removing old test swap repo matching this one: " + repoToDelete.address);
// RepoProvider.Helper.remove(context, repoToDelete.getId());
// repoToDelete = RepoProvider.Helper.findByAddress(context, FDroidApp.repo.getAddress());
//}
//
//ContentValues values = new ContentValues(4);
//values.put(Schema.RepoTable.Cols.SIGNING_CERT, signingCert);
//values.put(Schema.RepoTable.Cols.ADDRESS, FDroidApp.repo.getAddress());
//values.put(Schema.RepoTable.Cols.NAME, "");
//values.put(Schema.RepoTable.Cols.IS_SWAP, true);
//final String lastEtag = UUID.randomUUID().toString();
//values.put(Schema.RepoTable.Cols.LAST_ETAG, lastEtag);
//RepoProvider.Helper.insert(context, values);
//Repo repo = RepoProvider.Helper.findByAddress(context, FDroidApp.repo.getAddress());
//assertTrue(repo.isSwap);
//assertNotEquals(-1, repo.getId());
//assertEquals(lastEtag, repo.lastetag);
//assertNull(repo.lastUpdated);
//
assertTrue(isPortInUse(FDroidApp.ipAddressString, FDroidApp.port));
//Thread.sleep(100);
//IndexUpdater updater = new IndexUpdater(context, repo);
//updater.update();
//assertTrue(updater.hasChanged());
//
//repo = RepoProvider.Helper.findByAddress(context, FDroidApp.repo.getAddress());
//final Date lastUpdated = repo.lastUpdated;
//assertTrue("repo lastUpdated should be updated", new Date(2019, 5, 13).compareTo(repo.lastUpdated) > 0);
//
//App app = AppProvider.Helper.findSpecificApp(context.getContentResolver(),
// context.getPackageName(), repo.getId());
//assertEquals(context.getPackageName(), app.packageName);
//
//List<Apk> apks = ApkProvider.Helper.findByRepo(context, repo, Schema.ApkTable.Cols.ALL);
//assertEquals(1, apks.size());
//for (Apk apk : apks) {
// Log.i(TAG, "Apk: " + apk);
// assertEquals(context.getPackageName(), apk.packageName);
// assertEquals(BuildConfig.VERSION_NAME, apk.versionName);
// assertEquals(BuildConfig.VERSION_CODE, apk.versionCode);
// assertEquals(app.repoId, apk.repoId);
//}
Intent mainIntent = new Intent(Intent.ACTION_MAIN, null);
mainIntent.addCategory(Intent.CATEGORY_LAUNCHER);
@@ -165,14 +152,14 @@ public class SwapRepoEmulatorTest {
}
LocalRepoService.runProcess(context, packageNames.toArray(new String[0]));
// updater = new IndexUpdater(context, repo);
// updater.update();
// assertTrue(updater.hasChanged());
// assertTrue("repo lastUpdated should be updated", lastUpdated.compareTo(repo.lastUpdated) < 0);
//
// for (String packageName : packageNames) {
// assertNotNull(ApkProvider.Helper.findByPackageName(context, packageName));
// }
//updater = new IndexUpdater(context, repo);
//updater.update();
//assertTrue(updater.hasChanged());
//assertTrue("repo lastUpdated should be updated", lastUpdated.compareTo(repo.lastUpdated) < 0);
//
//for (String packageName : packageNames) {
// assertNotNull(ApkProvider.Helper.findByPackageName(context, packageName));
//}
} finally {
if (localHttpd != null) {
localHttpd.stop();

7
app/src/full/java/org/fdroid/fdroid/nearby/SDCardScannerService.java
View File

@@ -30,9 +30,8 @@ import android.os.Environment;
import android.os.Process;
import android.util.Log;
import org.fdroid.fdroid.IndexUpdater;
import org.fdroid.fdroid.IndexV1Updater;
import org.fdroid.fdroid.Utils;
import org.fdroid.index.SigningException;
import java.io.File;
import java.io.FileInputStream;
@@ -153,7 +152,7 @@ public class SDCardScannerService extends IntentService {
if (file.isDirectory()) {
searchDirectory(file);
} else {
if (IndexV1Updater.SIGNED_FILE_NAME.equals(file.getName())) {
if (TreeUriUtils.SIGNED_FILE_NAME.equals(file.getName())) {
registerRepo(file);
}
}
@@ -165,7 +164,7 @@ public class SDCardScannerService extends IntentService {
try {
inputStream = new FileInputStream(file);
TreeUriScannerIntentService.registerRepo(this, inputStream, Uri.fromFile(file.getParentFile()));
} catch (IOException | IndexUpdater.SigningException e) {
} catch (IOException | SigningException e) {
e.printStackTrace();
} finally {
Utils.closeQuietly(inputStream);

43
app/src/full/java/org/fdroid/fdroid/nearby/TreeUriScannerIntentService.java
View File

@@ -35,16 +35,18 @@ import org.apache.commons.io.IOUtils;
import org.fdroid.database.Repository;
import org.fdroid.fdroid.AddRepoIntentService;
import org.fdroid.fdroid.FDroidApp;
import org.fdroid.fdroid.IndexUpdater;
import org.fdroid.fdroid.IndexV1Updater;
import org.fdroid.fdroid.R;
import org.fdroid.fdroid.Utils;
import org.fdroid.index.SigningException;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.security.CodeSigner;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.jar.JarInputStream;
@@ -130,7 +132,7 @@ public class TreeUriScannerIntentService extends IntentService {
}
/**
* Recursively search for {@link IndexV1Updater#SIGNED_FILE_NAME} starting
* Recursively search for {@link TreeUriUtils#SIGNED_FILE_NAME} starting
* from the given directory, looking at files first before recursing into
* directories. This is "depth last" since the index file is much more
* likely to be shallow than deep, and there can be a lot of files to
@@ -148,7 +150,7 @@ public class TreeUriScannerIntentService extends IntentService {
if (documentFile.isDirectory()) {
dirs.add(documentFile);
} else if (!foundIndex) {
if (IndexV1Updater.SIGNED_FILE_NAME.equals(documentFile.getName())) {
if (TreeUriUtils.SIGNED_FILE_NAME.equals(documentFile.getName())) {
registerRepo(documentFile);
foundIndex = true;
}
@@ -160,7 +162,7 @@ public class TreeUriScannerIntentService extends IntentService {
}
/**
* For all files called {@link IndexV1Updater#SIGNED_FILE_NAME} found, check
* For all files called {@link TreeUriUtils#SIGNED_FILE_NAME} found, check
* the JAR signature and read the fingerprint of the signing certificate.
* The fingerprint is then used to find whether this local repo is a mirror
* of an existing repo, or a totally new repo. In order to verify the
@@ -173,7 +175,7 @@ public class TreeUriScannerIntentService extends IntentService {
try {
inputStream = getContentResolver().openInputStream(index.getUri());
registerRepo(this, inputStream, index.getParentFile().getUri());
} catch (IOException | IndexUpdater.SigningException e) {
} catch (IOException | SigningException e) {
e.printStackTrace();
} finally {
Utils.closeQuietly(inputStream);
@@ -181,16 +183,16 @@ public class TreeUriScannerIntentService extends IntentService {
}
public static void registerRepo(Context context, InputStream inputStream, Uri repoUri)
throws IOException, IndexUpdater.SigningException {
throws IOException, SigningException {
if (inputStream == null) {
return;
}
File destFile = File.createTempFile("dl-", IndexV1Updater.SIGNED_FILE_NAME, context.getCacheDir());
File destFile = File.createTempFile("dl-", TreeUriUtils.SIGNED_FILE_NAME, context.getCacheDir());
FileUtils.copyInputStreamToFile(inputStream, destFile);
JarFile jarFile = new JarFile(destFile, true);
JarEntry indexEntry = (JarEntry) jarFile.getEntry(IndexV1Updater.DATA_FILE_NAME);
JarEntry indexEntry = (JarEntry) jarFile.getEntry(TreeUriUtils.DATA_FILE_NAME);
IOUtils.readLines(jarFile.getInputStream(indexEntry));
Certificate certificate = IndexUpdater.getSigningCertFromJar(indexEntry);
Certificate certificate = getSigningCertFromJar(indexEntry);
String fingerprint = Utils.calcFingerprint(certificate);
Log.i(TAG, "Got fingerprint: " + fingerprint);
destFile.delete();
@@ -207,4 +209,25 @@ public class TreeUriScannerIntentService extends IntentService {
AddRepoIntentService.addRepo(context, repoUri, fingerprint);
// TODO rework IndexUpdater.getSigningCertFromJar to work for here
}
/**
* FDroid's index.jar is signed using a particular format and does not allow lots of
* signing setups that would be valid for a regular jar. This validates those
* restrictions.
*/
static X509Certificate getSigningCertFromJar(JarEntry jarEntry) throws SigningException {
final CodeSigner[] codeSigners = jarEntry.getCodeSigners();
if (codeSigners == null || codeSigners.length == 0) {
throw new SigningException("No signature found in index");
}
/* we could in theory support more than 1, but as of now we do not */
if (codeSigners.length > 1) {
throw new SigningException("index.jar must be signed by a single code signer!");
}
List<? extends Certificate> certs = codeSigners[0].getSignerCertPath().getCertificates();
if (certs.size() != 1) {
throw new SigningException("index.jar code signers must only have a single certificate!");
}
return (X509Certificate) certs.get(0);
}
}

2
app/src/full/java/org/fdroid/fdroid/nearby/TreeUriUtils.java
View File

@@ -23,6 +23,8 @@ public final class TreeUriUtils {
public static final String TAG = "TreeUriUtils";
private static final String PRIMARY_VOLUME_NAME = "primary";
public static final String SIGNED_FILE_NAME = "index-v1.jar";
public static final String DATA_FILE_NAME = "index-v1.json";
@Nullable
public static String getFullPathFromTreeUri(Context context, @Nullable final Uri treeUri) {

10
app/src/main/AndroidManifest.xml
View File

@@ -444,16 +444,6 @@
android:name="org.fdroid.fdroid.data.ApkProvider"
android:authorities="${applicationId}.data.ApkProvider"
android:exported="false" />
<!-- Note: AppThemeTransparent, this activity shows dialogs only -->
<provider
android:name="org.fdroid.fdroid.data.TempApkProvider"
android:authorities="${applicationId}.data.TempApkProvider"
android:exported="false" />
<!-- Note: AppThemeTransparent, this activity shows dialogs only -->
<provider
android:name="org.fdroid.fdroid.data.TempAppProvider"
android:authorities="${applicationId}.data.TempAppProvider"
android:exported="false" />
<provider
android:name="org.fdroid.fdroid.data.AppPrefsProvider"

516
app/src/main/java/org/fdroid/fdroid/IndexUpdater.java
View File

@@ -1,516 +0,0 @@
/*
* Copyright (C) 2018 Senecto Limited
* Copyright (C) 2016 Blue Jay Wireless
* Copyright (C) 2015-2016 Daniel Martí <mvdan@mvdan.cc>
* Copyright (C) 2014-2018 Hans-Christoph Steiner <hans@eds.org>
* Copyright (C) 2014-2016 Peter Serwylo <peter@serwylo.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 3
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
package org.fdroid.fdroid;
import android.content.ContentResolver;
import android.content.ContentValues;
import android.content.Context;
import android.content.pm.PackageInfo;
import android.content.res.Resources.NotFoundException;
import android.text.TextUtils;
import android.util.Log;
import android.util.Pair;
import org.fdroid.download.Downloader;
import org.fdroid.fdroid.data.Apk;
import org.fdroid.fdroid.data.ApkProvider;
import org.fdroid.fdroid.data.App;
import org.fdroid.fdroid.data.AppProvider;
import org.fdroid.fdroid.data.Repo;
import org.fdroid.fdroid.data.RepoPersister;
import org.fdroid.fdroid.data.RepoProvider;
import org.fdroid.fdroid.data.RepoPushRequest;
import org.fdroid.fdroid.data.RepoXMLHandler;
import org.fdroid.fdroid.data.Schema.RepoTable;
import org.fdroid.fdroid.installer.InstallManagerService;
import org.fdroid.fdroid.installer.InstallerService;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.security.CodeSigner;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import androidx.annotation.NonNull;
// TODO move to org.fdroid.fdroid.updater
// TODO reduce visibility of methods once in .updater package (.e.g tests need it public now)
/**
* Updates the local database with a repository's app/apk metadata and verifying
* the JAR signature on the file received from the repository. As an overview:
* <ul>
* <li>Download the {@code index.jar}
* <li>Verify that it is signed correctly and by the correct certificate
* <li>Parse the {@code index.xml} that is in {@code index.jar}
* <li>Save the resulting repo, apps, and apks to the database.
* <li>Process any push install/uninstall requests included in the repository
* </ul>
* <b>WARNING</b>: this class is the central piece of the entire security model of
* FDroid! Avoid modifying it when possible, if you absolutely must, be very,
* very careful with the changes that you are making!
*/
public class IndexUpdater {
private static final String TAG = "IndexUpdater";
public static final String SIGNED_FILE_NAME = "index.jar";
public static final String DATA_FILE_NAME = "index.xml";
final String indexUrl;
@NonNull
final Context context;
@NonNull
final Repo repo;
boolean hasChanged;
private String cacheTag;
private X509Certificate signingCertFromJar;
@NonNull
private final RepoPersister persister;
private final List<RepoPushRequest> repoPushRequestList = new ArrayList<>();
/**
* Updates an app repo as read out of the database into a {@link Repo} instance.
*
* @param repo A {@link Repo} read out of the local database
*/
public IndexUpdater(@NonNull Context context, @NonNull Repo repo) {
this.context = context;
this.repo = repo;
this.persister = new RepoPersister(context, repo);
this.indexUrl = getIndexUrl(repo);
}
protected String getIndexUrl(@NonNull Repo repo) {
return repo.getFileUrl(SIGNED_FILE_NAME);
}
public boolean hasChanged() {
return hasChanged;
}
private Pair<Downloader, File> downloadIndex() throws UpdateException {
File destFile = null;
Downloader downloader = null;
try {
destFile = File.createTempFile("dl-", "", context.getCacheDir());
destFile.deleteOnExit(); // this probably does nothing, but maybe...
// TODO we don't use this anymore
// downloader = DownloaderFactory.createWithTryFirstMirror(repo, Uri.parse(indexUrl), destFile);
// downloader.setCacheTag(repo.lastetag);
// downloader.setListener(downloadListener);
// downloader.download();
} catch (IOException e) {
if (destFile != null) {
if (!destFile.delete()) {
Log.w(TAG, "Couldn't delete file: " + destFile.getAbsolutePath());
}
}
throw new UpdateException(repo, "Error getting F-Droid index file", e);
} // TODO is it safe to delete destFile in finally block?
return new Pair<>(downloader, destFile);
}
/**
* All repos are represented by a signed jar file, {@code index.jar}, which contains
* a single file, {@code index.xml}. This takes the {@code index.jar}, verifies the
* signature, then returns the unzipped {@code index.xml}.
*
* @return whether this version of the repo index was found and processed
* @throws UpdateException All error states will come from here.
*/
public boolean update() throws UpdateException {
final Pair<Downloader, File> pair = downloadIndex();
final Downloader downloader = pair.first;
final File destFile = pair.second;
hasChanged = downloader.hasChanged();
if (hasChanged) {
// Don't worry about checking the status code for 200. If it was a
// successful download, then we will have a file ready to use:
cacheTag = downloader.getCacheTag();
processDownloadedFile(destFile);
processRepoPushRequests(repoPushRequestList);
}
return true;
}
private ContentValues repoDetailsToSave;
private String signingCertFromIndexXml;
private RepoXMLHandler.IndexReceiver createIndexReceiver() {
return new RepoXMLHandler.IndexReceiver() {
@Override
public void receiveRepo(String name, String description, String signingCert, int maxAge,
int version, long timestamp, String icon, String[] mirrors) {
signingCertFromIndexXml = signingCert;
repoDetailsToSave = prepareRepoDetailsForSaving(name, description, maxAge, version,
timestamp, icon, mirrors, cacheTag);
}
@Override
public void receiveApp(App app, List<Apk> packages) {
try {
persister.saveToDb(app, packages);
} catch (UpdateException e) {
throw new RuntimeException("Error while saving repo details to database.", e);
}
}
@Override
public void receiveRepoPushRequest(RepoPushRequest repoPushRequest) {
repoPushRequestList.add(repoPushRequest);
}
};
}
public void processDownloadedFile(File downloadedFile) throws UpdateException {
InputStream indexInputStream = null;
try {
if (downloadedFile == null || !downloadedFile.exists()) {
throw new UpdateException(repo, downloadedFile + " does not exist!");
}
JarFile jarFile = new JarFile(downloadedFile, true);
JarEntry indexEntry = (JarEntry) jarFile.getEntry(IndexUpdater.DATA_FILE_NAME);
indexInputStream = new ProgressBufferedInputStream(jarFile.getInputStream(indexEntry),
processIndexListener, (int) indexEntry.getSize());
// Process the index...
SAXParserFactory factory = SAXParserFactory.newInstance();
factory.setNamespaceAware(true);
final SAXParser parser = factory.newSAXParser();
final XMLReader reader = parser.getXMLReader();
final RepoXMLHandler repoXMLHandler = new RepoXMLHandler(repo, createIndexReceiver());
reader.setContentHandler(repoXMLHandler);
reader.parse(new InputSource(indexInputStream));
long timestamp = repoDetailsToSave.getAsLong(RepoTable.Cols.TIMESTAMP);
if (timestamp < repo.timestamp) {
throw new UpdateException(repo, "index.jar is older that current index! "
+ timestamp + " < " + repo.timestamp);
}
signingCertFromJar = getSigningCertFromJar(indexEntry);
// JarEntry can only read certificates after the file represented by that JarEntry
// has been read completely, so verification cannot run until now...
assertSigningCertFromXmlCorrect();
commitToDb();
} catch (SAXException | ParserConfigurationException | IOException e) {
throw new UpdateException(repo, "Error parsing index", e);
} finally {
Utils.closeQuietly(indexInputStream);
if (downloadedFile != null) {
if (!downloadedFile.delete()) {
Log.w(TAG, "Couldn't delete file: " + downloadedFile.getAbsolutePath());
}
}
}
}
protected final ProgressListener downloadListener = new ProgressListener() {
@Override
public void onProgress(long bytesRead, long totalBytes) {
UpdateService.reportDownloadProgress(context, indexUrl, bytesRead, totalBytes);
}
};
protected final ProgressListener processIndexListener = new ProgressListener() {
@Override
public void onProgress(long bytesRead, long totalBytes) {
UpdateService.reportProcessIndexProgress(context, indexUrl, bytesRead, totalBytes);
}
};
protected void notifyProcessingApps(int appsSaved, int totalApps) {
UpdateService.reportProcessingAppsProgress(context, indexUrl, appsSaved, totalApps);
}
protected void notifyCommittingToDb() {
notifyProcessingApps(0, -1);
}
private void commitToDb() throws UpdateException {
Log.i(TAG, "Repo signature verified, saving app metadata to database.");
notifyCommittingToDb();
persister.commit(repoDetailsToSave, repo.getId());
}
private void assertSigningCertFromXmlCorrect() throws SigningException {
// no signing cert read from database, this is the first use
if (repo.signingCertificate == null) {
verifyAndStoreTOFUCerts(signingCertFromIndexXml, signingCertFromJar);
}
verifyCerts(signingCertFromIndexXml, signingCertFromJar);
}
/**
* Update tracking data for the repo represented by this instance (index version, etag,
* description, human-readable name, etc. This is not reused in {@link IndexV1Updater}
* because its too tied up into the old parsing flow in this class.
*/
private ContentValues prepareRepoDetailsForSaving(String name, String description, int maxAge,
int version, long timestamp, String icon,
String[] mirrors, String cacheTag) {
ContentValues values = new ContentValues();
values.put(RepoTable.Cols.LAST_UPDATED, Utils.formatTime(new Date(), ""));
if (repo.lastetag == null || !repo.lastetag.equals(cacheTag)) {
values.put(RepoTable.Cols.LAST_ETAG, cacheTag);
}
if (version != Repo.INT_UNSET_VALUE && version != repo.version) {
Utils.debugLog(TAG, "Repo specified a new version: from " + repo.version + " to " + version);
values.put(RepoTable.Cols.VERSION, version);
}
if (maxAge != Repo.INT_UNSET_VALUE && maxAge != repo.maxage) {
Utils.debugLog(TAG, "Repo specified a new maximum age - updated");
values.put(RepoTable.Cols.MAX_AGE, maxAge);
}
if (description != null && !description.equals(repo.description)) {
values.put(RepoTable.Cols.DESCRIPTION, description);
}
if (name != null && !name.equals(repo.name)) {
values.put(RepoTable.Cols.NAME, name);
}
// Always put a timestamp here, even if it is the same. This is because we are dependent
// on it later on in the process. Specifically, when updating from a HTTP server that
// doesn't send out etags with its responses, it will trigger a full blown repo update
// every time, even if all the values in the index are the same (name, description, etc).
// In such a case, the remainder of the update process will proceed, and ask for this
// timestamp.
values.put(RepoTable.Cols.TIMESTAMP, timestamp);
if (icon != null && !icon.equals(repo.icon)) {
values.put(RepoTable.Cols.ICON, icon);
}
if (mirrors != null && mirrors.length > 0 && !Arrays.equals(mirrors, repo.mirrors)) {
values.put(RepoTable.Cols.MIRRORS, Utils.serializeCommaSeparatedString(mirrors));
}
return values;
}
public static class UpdateException extends Exception {
private static final long serialVersionUID = -4492452418826132803L;
public UpdateException(Repo repo, String message) {
super((repo != null ? repo.name + ": " : "") + message);
}
public UpdateException(Repo repo, String message, Exception cause) {
super((repo != null ? repo.name + ": " : "") + message, cause);
}
}
public static class SigningException extends UpdateException {
public SigningException(String message) {
super(null, "Repository was not signed correctly: " + message);
}
public SigningException(Repo repo, String message) {
super(repo, "Repository was not signed correctly: " + message);
}
}
/**
* FDroid's index.jar is signed using a particular format and does not allow lots of
* signing setups that would be valid for a regular jar. This validates those
* restrictions.
*/
public static X509Certificate getSigningCertFromJar(JarEntry jarEntry) throws SigningException {
final CodeSigner[] codeSigners = jarEntry.getCodeSigners();
if (codeSigners == null || codeSigners.length == 0) {
throw new SigningException("No signature found in index");
}
/* we could in theory support more than 1, but as of now we do not */
if (codeSigners.length > 1) {
throw new SigningException("index.jar must be signed by a single code signer!");
}
List<? extends Certificate> certs = codeSigners[0].getSignerCertPath().getCertificates();
if (certs.size() != 1) {
throw new SigningException("index.jar code signers must only have a single certificate!");
}
return (X509Certificate) certs.get(0);
}
/**
* A new repo can be added with or without the fingerprint of the signing
* certificate. If no fingerprint is supplied, then do a pure TOFU and just
* store the certificate as valid. If there is a fingerprint, then first
* check that the signing certificate in the jar matches that fingerprint.
*/
private void verifyAndStoreTOFUCerts(String certFromIndexXml, X509Certificate rawCertFromJar)
throws SigningException {
if (repo.signingCertificate != null) {
return; // there is a repo.signingCertificate already, nothing to TOFU
}
/* The first time a repo is added, it can be added with the signing certificate's
* fingerprint. In that case, check that fingerprint against what is
* actually in the index.jar itself. If no fingerprint, just store the
* signing certificate */
if (repo.fingerprint != null) {
String fingerprintFromIndexXml = Utils.calcFingerprint(certFromIndexXml);
String fingerprintFromJar = Utils.calcFingerprint(rawCertFromJar);
if (!repo.fingerprint.equalsIgnoreCase(fingerprintFromIndexXml)
|| !repo.fingerprint.equalsIgnoreCase(fingerprintFromJar)) {
throw new SigningException(repo, "Supplied certificate fingerprint does not match!");
}
} // else - no info to check things are valid, so just Trust On First Use
Utils.debugLog(TAG, "Saving new signing certificate in the database for " + repo.address);
ContentValues values = new ContentValues(2);
values.put(RepoTable.Cols.LAST_UPDATED, Utils.formatTime(new Date(), ""));
values.put(RepoTable.Cols.SIGNING_CERT, Hasher.hex(rawCertFromJar));
RepoProvider.Helper.update(context, repo, values);
}
/**
* FDroid works with three copies of the signing certificate:
* <li>in the downloaded jar</li>
* <li>in the index XML</li>
* <li>stored in the local database</li>
* It would work better removing the copy from the index XML, but it needs to stay
* there for backwards compatibility since the old TOFU process requires it. Therefore,
* since all three have to be present, all three are compared.
*
* @param certFromIndexXml the cert written into the header of the index XML
* @param rawCertFromJar the {@link X509Certificate} embedded in the downloaded jar
*/
private void verifyCerts(String certFromIndexXml, X509Certificate rawCertFromJar) throws SigningException {
// convert binary data to string version that is used in FDroid's database
String certFromJar = Hasher.hex(rawCertFromJar);
// repo and repo.signingCertificate must be pre-loaded from the database
if (TextUtils.isEmpty(repo.signingCertificate)
|| TextUtils.isEmpty(certFromJar)
|| TextUtils.isEmpty(certFromIndexXml)) {
throw new SigningException(repo, "A empty repo or signing certificate is invalid!");
}
// though its called repo.signingCertificate, its actually a X509 certificate
if (repo.signingCertificate.equals(certFromJar)
&& repo.signingCertificate.equals(certFromIndexXml)
&& certFromIndexXml.equals(certFromJar)) {
return; // we have a match!
}
throw new SigningException(repo, "Signing certificate does not match!");
}
/**
* Server index XML can include optional {@code install} and {@code uninstall}
* requests. This processes those requests, figuring out whether the client
* should always accept, prompt the user, or ignore those requests on a
* per repo basis. There is also a compile-time option as a failsafe.
*
*/
void processRepoPushRequests(List<RepoPushRequest> requestEntries) {
try {
if (!context.getResources().getBoolean(R.bool.config_allowPushRequests)) {
return;
}
} catch (NotFoundException e) {
Utils.debugLog(TAG, "allowPushRequests configuration not found, defaulting to false");
return;
}
for (RepoPushRequest repoPushRequest : requestEntries) {
String packageName = repoPushRequest.packageName;
PackageInfo packageInfo = Utils.getPackageInfo(context, packageName);
if (RepoPushRequest.INSTALL.equals(repoPushRequest.request)) {
ContentResolver cr = context.getContentResolver();
// TODO: In the future, this needs to be able to specify which repository to get
// the package from. Better yet, we should be able to specify the hash of a package
// to install (especially when we move to using hashes more as identifiers than we
// do right now).
App app = AppProvider.Helper.findHighestPriorityMetadata(cr, packageName);
if (app == null) {
Utils.debugLog(TAG, packageName + " not in local database, ignoring request to"
+ repoPushRequest.request);
continue;
}
int versionCode;
if (repoPushRequest.versionCode == null) {
versionCode = app.autoInstallVersionCode;
} else {
versionCode = repoPushRequest.versionCode;
}
if (packageInfo != null && versionCode == packageInfo.versionCode) {
Utils.debugLog(TAG, repoPushRequest + " already installed, ignoring");
} else {
Apk apk = ApkProvider.Helper.findApkFromAnyRepo(context, packageName, versionCode);
InstallManagerService.queue(context, app, apk);
}
} else if (RepoPushRequest.UNINSTALL.equals(repoPushRequest.request)) {
if (packageInfo == null) {
Utils.debugLog(TAG, "ignoring request, not installed: " + repoPushRequest);
continue;
}
if (repoPushRequest.versionCode == null
|| repoPushRequest.versionCode == packageInfo.versionCode) {
Apk apk = ApkProvider.Helper.findApkFromAnyRepo(context, repoPushRequest.packageName,
packageInfo.versionCode);
if (apk == null) {
Log.i(TAG, "Push " + repoPushRequest.packageName + " request not found in any repo!");
} else {
InstallerService.uninstall(context, apk);
}
} else {
Utils.debugLog(TAG, "ignoring request based on versionCode:" + repoPushRequest);
}
} else {
Utils.debugLog(TAG, "Unknown Repo Push Request: " + repoPushRequest.request);
}
}
}
}

449
app/src/main/java/org/fdroid/fdroid/IndexV1Updater.java
View File

@@ -1,449 +0,0 @@
/*
* Copyright (C) 2017-2018 Hans-Christoph Steiner <hans@eds.org>
* Copyright (C) 2017 Peter Serwylo <peter@serwylo.com>
* Copyright (C) 2017 Chirayu Desai
* Copyright (C) 2018 Senecto Limited
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 3
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
package org.fdroid.fdroid;
import android.content.ContentValues;
import android.content.Context;
import android.content.pm.PackageInfo;
import android.text.TextUtils;
import android.util.Log;
import androidx.annotation.NonNull;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.FileUtils;
import org.fdroid.fdroid.data.Apk;
import org.fdroid.fdroid.data.App;
import org.fdroid.fdroid.data.Repo;
import org.fdroid.fdroid.data.RepoPersister;
import org.fdroid.fdroid.data.RepoProvider;
import org.fdroid.fdroid.data.RepoPushRequest;
import org.fdroid.fdroid.data.Schema;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.jar.JarEntry;
/**
* Receives the index data about all available apps and packages via the V1
* JSON data {@link #DATA_FILE_NAME}, embedded in a signed jar
* {@link #SIGNED_FILE_NAME}. This uses the Jackson library to parse the JSON,
* with {@link App} and {@link Apk} being instantiated directly from the JSON
* by Jackson. This is possible but not wise to do with {@link Repo} since that
* class has many fields that are related to security components of the
* implementation internal to this app.
* <p>
* All non-{@code public} fields and fields tagged with {@code @JsonIgnore} are
* ignored. All methods are ignored unless they are tagged with {@code @JsonProperty}.
* This setup prevents the situation where future developers add variables to the
* App/Apk classes, resulting in malicious servers being able to populate those
* variables.
*/
public class IndexV1Updater extends IndexUpdater {
public static final String TAG = "IndexV1Updater";
public static final String SIGNED_FILE_NAME = "index-v1.jar";
public static final String DATA_FILE_NAME = "index-v1.json";
private static String platformSigCache;
public IndexV1Updater(@NonNull Context context, @NonNull Repo repo) {
super(context, repo);
}
@Override
protected String getIndexUrl(@NonNull Repo repo) {
return repo.getFileUrl(SIGNED_FILE_NAME);
}
/**
* @return whether this successfully found an index of this version
* @throws IndexUpdater.UpdateException
* @see org.fdroid.fdroid.net.DownloaderService#handleIntent(android.content.Intent)
*/
@Override
public boolean update() throws IndexUpdater.UpdateException {
File destFile = null;
// Downloader downloader;
try {
destFile = File.createTempFile("dl-", "", context.getCacheDir());
destFile.deleteOnExit(); // this probably does nothing, but maybe...
// TODO we don't use that anymore
// read file name from file
// downloader = DownloaderFactory.createWithTryFirstMirror(repo, Uri.parse(indexUrl), destFile);
// downloader.setCacheTag(repo.lastetag);
// downloader.setListener(downloadListener);
// downloader.download();
// hasChanged = downloader.hasChanged();
if (!hasChanged) {
return true;
}
// processDownloadedIndex(destFile, downloader.getCacheTag());
} catch (IOException e) {
if (destFile != null) {
FileUtils.deleteQuietly(destFile);
}
throw new IndexUpdater.UpdateException(repo, "Error getting F-Droid index file", e);
} // TODO is it safe to delete destFile in finally block?
return true;
}
//private void processDownloadedIndex(File outputFile, String cacheTag)
// throws IOException, IndexUpdater.UpdateException {
// JarFile jarFile = new JarFile(outputFile, true);
// JarEntry indexEntry = (JarEntry) jarFile.getEntry(DATA_FILE_NAME);
// InputStream indexInputStream = new ProgressBufferedInputStream(jarFile.getInputStream(indexEntry),
// processIndexListener, (int) indexEntry.getSize());
// processIndexV1(indexInputStream, indexEntry, cacheTag);
// jarFile.close();
//}
/**
* Get the standard {@link ObjectMapper} instance used for parsing {@code index-v1.json}.
* This ignores unknown properties so that old releases won't crash when new things are
* added to {@code index-v1.json}. This is required for both forward compatibility,
* but also because ignoring such properties when coming from a malicious server seems
* reasonable anyway.
*/
public static ObjectMapper getObjectMapperInstance(long repoId) {
ObjectMapper mapper = new ObjectMapper();
mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
mapper.setInjectableValues(new InjectableValues.Std().addValue("repoId", repoId));
mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.PUBLIC_ONLY);
return mapper;
}
/**
* Parses the index and feeds it to the database via {@link Repo}, {@link App},
* and {@link Apk} instances. This uses {@link RepoPersister} to add the apps
* and packages to the database in {@link RepoPersister#saveToDb(App, List)}
* to write the {@link Repo}, and commit the whole thing in
* {@link RepoPersister#commit(ContentValues, long)}. One confusing thing about this
* whole process is that {@link RepoPersister} needs to first create and entry
* in the database, then fetch the ID from the database to populate
* {@link Repo#id}. That has to happen first, then the rest of the {@code Repo}
* data must be added later.
*
* @param indexInputStream {@link InputStream} to {@code index-v1.json}
* @param etag the {@code etag} value from HTTP headers
* @throws IOException
* @throws UpdateException
*/
public void processIndexV1(InputStream indexInputStream, JarEntry indexEntry, String etag)
throws IOException, UpdateException {
Utils.Profiler profiler = new Utils.Profiler(TAG);
profiler.log("Starting to process index-v1.json");
ObjectMapper mapper = getObjectMapperInstance(repo.getId());
JsonFactory f = mapper.getFactory();
JsonParser parser = f.createParser(indexInputStream);
HashMap<String, Object> repoMap = null;
App[] apps = null;
Map<String, String[]> requests = null;
Map<String, List<Apk>> packages = null;
parser.nextToken(); // go into the main object block
while (true) {
String fieldName = parser.nextFieldName();
if (fieldName == null) {
break;
}
switch (fieldName) {
case "repo":
repoMap = parseRepo(mapper, parser);
break;
case "requests":
requests = parseRequests(mapper, parser);
break;
case "apps":
apps = parseApps(mapper, parser);
break;
case "packages":
packages = parsePackages(mapper, parser);
break;
}
}
parser.close(); // ensure resources get cleaned up timely and properly
profiler.log("Finished processing index-v1.json. Now verifying certificate...");
if (repoMap == null) {
return;
}
long timestamp = (Long) repoMap.get("timestamp") / 1000;
if (repo.timestamp > timestamp) {
throw new IndexUpdater.UpdateException(repo, "index.jar is older that current index! "
+ timestamp + " < " + repo.timestamp);
}
X509Certificate certificate = getSigningCertFromJar(indexEntry);
verifySigningCertificate(certificate);
profiler.log("Certificate verified. Now saving to database...");
// timestamp is absolutely required
repo.timestamp = timestamp;
// below are optional, can be null
repo.lastetag = etag;
repo.name = getStringRepoValue(repoMap, "name");
repo.icon = getStringRepoValue(repoMap, "icon");
repo.description = getStringRepoValue(repoMap, "description");
// ensure the canonical URL is included in the "mirrors" list as the first entry
LinkedHashSet<String> mirrors = new LinkedHashSet<>();
mirrors.add(repo.address);
mirrors.addAll(getStringListRepoValue(repoMap, "mirrors"));
repo.mirrors = mirrors.toArray(new String[mirrors.size()]);
// below are optional, can be default value
repo.maxage = getIntRepoValue(repoMap, "maxage");
repo.version = getIntRepoValue(repoMap, "version");
if (TextUtils.isEmpty(platformSigCache)) {
PackageInfo androidPackageInfo = Utils.getPackageInfoWithSignatures(context, "android");
platformSigCache = Utils.getPackageSigner(androidPackageInfo);
}
RepoPersister repoPersister = new RepoPersister(context, repo);
if (apps != null && apps.length > 0) {
int appCount = 0;
for (App app : apps) {
appCount++;
List<Apk> apks = null;
if (packages != null) {
apks = packages.get(app.packageName);
}
if (apks == null) {
Log.i(TAG, "processIndexV1 empty packages");
apks = new ArrayList<>(0);
}
if (apks.size() > 0) {
app.preferredSigner = apks.get(0).sig;
app.isApk = true;
for (Apk apk : apks) {
if (!apk.isApk()) {
app.isApk = false;
} else if (apk.sig.equals(platformSigCache)) {
app.preferredSigner = platformSigCache;
}
}
}
if (appCount % 50 == 0) {
notifyProcessingApps(appCount, apps.length);
}
repoPersister.saveToDb(app, apks);
}
}
profiler.log("Saved to database, but only a temporary table. Now persisting to database...");
notifyCommittingToDb();
ContentValues contentValues = new ContentValues();
contentValues.put(Schema.RepoTable.Cols.LAST_UPDATED, Utils.formatTime(new Date(), ""));
contentValues.put(Schema.RepoTable.Cols.TIMESTAMP, repo.timestamp);
contentValues.put(Schema.RepoTable.Cols.LAST_ETAG, repo.lastetag);
if (repo.version != Repo.INT_UNSET_VALUE) {
contentValues.put(Schema.RepoTable.Cols.VERSION, repo.version);
}
if (repo.maxage != Repo.INT_UNSET_VALUE) {
contentValues.put(Schema.RepoTable.Cols.MAX_AGE, repo.maxage);
}
if (repo.description != null) {
contentValues.put(Schema.RepoTable.Cols.DESCRIPTION, repo.description);
}
if (repo.name != null) {
contentValues.put(Schema.RepoTable.Cols.NAME, repo.name);
}
if (repo.icon != null) {
contentValues.put(Schema.RepoTable.Cols.ICON, repo.icon);
}
if (repo.mirrors != null && repo.mirrors.length > 0) {
contentValues.put(Schema.RepoTable.Cols.MIRRORS, Utils.serializeCommaSeparatedString(repo.mirrors));
}
repoPersister.commit(contentValues, repo.getId());
profiler.log("Persisted to database.");
if (repo.pushRequests == Repo.PUSH_REQUEST_ACCEPT_ALWAYS) {
processRepoPushRequests(requests);
Utils.debugLog(TAG, "Completed Repo Push Requests: " + requests);
}
}
private int getIntRepoValue(Map<String, Object> repoMap, String key) {
Object value = repoMap.get(key);
if (value != null && value instanceof Integer) {
return (Integer) value;
}
return Repo.INT_UNSET_VALUE;
}
private String getStringRepoValue(Map<String, Object> repoMap, String key) {
Object value = repoMap.get(key);
if (value != null && value instanceof String) {
return (String) value;
}
return null;
}
@SuppressWarnings("unchecked")
private List<String> getStringListRepoValue(Map<String, Object> repoMap, String key) {
Object value = repoMap.get(key);
if (value != null && value instanceof ArrayList) {
return (List<String>) value;
}
return Collections.emptyList();
}
private HashMap<String, Object> parseRepo(ObjectMapper mapper, JsonParser parser) throws IOException {
TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() {
};
parser.nextToken();
parser.nextToken();
return mapper.readValue(parser, typeRef);
}
private Map<String, String[]> parseRequests(ObjectMapper mapper, JsonParser parser) throws IOException {
TypeReference<HashMap<String, String[]>> typeRef = new TypeReference<HashMap<String, String[]>>() {
};
parser.nextToken(); // START_OBJECT
return mapper.readValue(parser, typeRef);
}
private App[] parseApps(ObjectMapper mapper, JsonParser parser) throws IOException {
TypeReference<App[]> typeRef = new TypeReference<App[]>() {
};
parser.nextToken(); // START_ARRAY
return mapper.readValue(parser, typeRef);
}
private Map<String, List<Apk>> parsePackages(ObjectMapper mapper, JsonParser parser) throws IOException {
TypeReference<HashMap<String, List<Apk>>> typeRef = new TypeReference<HashMap<String, List<Apk>>>() {
};
parser.nextToken(); // START_OBJECT
return mapper.readValue(parser, typeRef);
}
/**
* Verify that the signing certificate used to sign {@link #SIGNED_FILE_NAME}
* matches the signing stored in the database for this repo. {@link #repo} and
* {@code repo.signingCertificate} must be pre-loaded from the database before
* running this, if this is an existing repo. If the repo does not exist,
* this will run the TOFU process.
* <p>
* Index V1 works with two copies of the signing certificate:
* <li>in the downloaded jar</li>
* <li>stored in the local database</li>
* <p>
* A new repo can be added with or without the fingerprint of the signing
* certificate. If no fingerprint is supplied, then do a pure TOFU and just
* store the certificate as valid. If there is a fingerprint, then first
* check that the signing certificate in the jar matches that fingerprint.
* <p>
* This is also responsible for adding the {@link Repo} instance to the
* database for the first time.
* <p>
* This is the same as {@link IndexUpdater#verifyCerts(String, X509Certificate)},
* {@link IndexUpdater#verifyAndStoreTOFUCerts(String, X509Certificate)}, and
* {@link IndexUpdater#assertSigningCertFromXmlCorrect()} except there is no
* embedded copy of the signing certificate in the index data.
*
* @param rawCertFromJar the {@link X509Certificate} embedded in the downloaded jar
* @see IndexUpdater#verifyAndStoreTOFUCerts(String, X509Certificate)
* @see IndexUpdater#verifyCerts(String, X509Certificate)
* @see IndexUpdater#assertSigningCertFromXmlCorrect()
*/
private void verifySigningCertificate(X509Certificate rawCertFromJar) throws SigningException {
String certFromJar = Hasher.hex(rawCertFromJar);
if (TextUtils.isEmpty(certFromJar)) {
throw new SigningException(repo, SIGNED_FILE_NAME + " must have an included signing certificate!");
}
if (repo.signingCertificate == null) {
if (repo.fingerprint != null) {
String fingerprintFromJar = Utils.calcFingerprint(rawCertFromJar);
if (!repo.fingerprint.equalsIgnoreCase(fingerprintFromJar)) {
throw new SigningException(repo, "Supplied certificate fingerprint does not match!");
}
}
Utils.debugLog(TAG, "Saving new signing certificate to database for " + repo.address);
ContentValues values = new ContentValues(2);
values.put(Schema.RepoTable.Cols.LAST_UPDATED, Utils.formatTime(new Date(), ""));
values.put(Schema.RepoTable.Cols.SIGNING_CERT, Hasher.hex(rawCertFromJar));
RepoProvider.Helper.update(context, repo, values);
repo.signingCertificate = certFromJar;
}
if (TextUtils.isEmpty(repo.signingCertificate)) {
throw new SigningException(repo, "A empty repo signing certificate is invalid!");
}
if (repo.signingCertificate.equals(certFromJar)) {
return; // we have a match!
}
throw new SigningException(repo, "Signing certificate does not match!");
}
/**
* The {@code index-v1} version of {@link IndexUpdater#processRepoPushRequests(List)}
*/
private void processRepoPushRequests(Map<String, String[]> requests) {
if (requests == null) {
Utils.debugLog(TAG, "RepoPushRequests are null");
} else {
List<RepoPushRequest> repoPushRequestList = new ArrayList<>();
for (Map.Entry<String, String[]> requestEntry : requests.entrySet()) {
String request = requestEntry.getKey();
for (String packageName : requestEntry.getValue()) {
repoPushRequestList.add(new RepoPushRequest(request, packageName, null));
}
}
processRepoPushRequests(repoPushRequestList);
}
}
}

5
app/src/main/java/org/fdroid/fdroid/data/Apk.java
View File

@@ -54,11 +54,6 @@ import androidx.annotation.Nullable;
* They are mapped to JSON field names, the {@code fdroidserver} internal variable
* names, and the {@code fdroiddata} YAML field names. Only the instance variables
* decorated with {@code @JsonIgnore} are not directly mapped.
* <p>
* <b>NOTE:</b>If an instance variable is only meant for internal state, and not for
* representing data coming from the server, then it must also be decorated with
* {@code @JsonIgnore} to prevent abuse! The tests for
* {@link org.fdroid.fdroid.IndexV1Updater} will also have to be updated.
*
* @see <a href="https://gitlab.com/fdroid/fdroiddata">fdroiddata</a>
* @see <a href="https://gitlab.com/fdroid/fdroidserver">fdroidserver</a>

140
app/src/main/java/org/fdroid/fdroid/data/App.java
View File

@@ -3,7 +3,6 @@ package org.fdroid.fdroid.data;
import android.content.ContentValues;
import android.content.Context;
import android.content.pm.ApplicationInfo;
import android.content.pm.FeatureInfo;
import android.content.pm.PackageInfo;
import android.content.pm.PackageManager;
import android.content.res.AssetManager;
@@ -45,24 +44,16 @@ import org.xmlpull.v1.XmlPullParserException;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;
import java.security.cert.Certificate;
import java.security.cert.CertificateEncodingException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import info.guardianproject.netcipher.NetCipher;
@@ -80,11 +71,6 @@ import androidx.core.os.LocaleListCompat;
* They are mapped to JSON field names, the {@code fdroidserver} internal variable
* names, and the {@code fdroiddata} YAML field names. Only the instance variables
* decorated with {@code @JsonIgnore} are not directly mapped.
* <p>
* <b>NOTE:</b>If an instance variable is only meant for internal state, and not for
* representing data coming from the server, then it must also be decorated with
* {@code @JsonIgnore} to prevent abuse! The tests for
* {@link org.fdroid.fdroid.IndexV1Updater} will also have to be updated.
*
* @see <a href="https://gitlab.com/fdroid/fdroiddata">fdroiddata</a>
* @see <a href="https://gitlab.com/fdroid/fdroidserver">fdroidserver</a>
@@ -968,50 +954,6 @@ public class App extends ValueObject implements Comparable<App>, Parcelable {
+ "/Android/obb/" + packageName);
}
private void setFromPackageInfo(PackageManager pm, PackageInfo packageInfo) {
this.packageName = packageInfo.packageName;
final String installerPackageName = pm.getInstallerPackageName(packageName);
CharSequence installerPackageLabel = null;
if (!TextUtils.isEmpty(installerPackageName)) {
try {
ApplicationInfo installerAppInfo = pm.getApplicationInfo(installerPackageName,
PackageManager.GET_META_DATA);
installerPackageLabel = installerAppInfo.loadLabel(pm);
} catch (PackageManager.NameNotFoundException e) {
Log.w(TAG, "Could not get app info: " + installerPackageName, e);
}
}
if (TextUtils.isEmpty(installerPackageLabel)) {
installerPackageLabel = installerPackageName;
}
ApplicationInfo appInfo = packageInfo.applicationInfo;
final CharSequence appDescription = appInfo.loadDescription(pm);
if (TextUtils.isEmpty(appDescription)) {
this.summary = "(installed by " + installerPackageLabel + ")";
} else if (appDescription.length() > 40) {
this.summary = (String) appDescription.subSequence(0, 40);
} else {
this.summary = (String) appDescription;
}
this.added = new Date(packageInfo.firstInstallTime);
this.lastUpdated = new Date(packageInfo.lastUpdateTime);
this.description = "<p>";
if (!TextUtils.isEmpty(appDescription)) {
this.description += appDescription + "\n";
}
this.description += "(installed by " + installerPackageLabel
+ ", first installed on " + this.added
+ ", last updated on " + this.lastUpdated + ")</p>";
this.name = (String) appInfo.loadLabel(pm);
this.iconFromApk = getIconName(packageName, packageInfo.versionCode);
this.installedVersionName = packageInfo.versionName;
this.installedVersionCode = packageInfo.versionCode;
this.compatible = true;
}
public static void initInstalledObbFiles(Apk apk) {
File obbdir = getObbDir(apk.packageName);
FileFilter filter = new RegexFileFilter("(main|patch)\\.[0-9-][0-9]*\\." + apk.packageName + "\\.obb");
@@ -1035,79 +977,6 @@ public class App extends ValueObject implements Comparable<App>, Parcelable {
}
}
@SuppressWarnings("EmptyForIteratorPad")
private void initInstalledApk(Context context, Apk apk, PackageInfo packageInfo, SanitizedFile apkFile)
throws IOException, CertificateEncodingException {
apk.compatible = true;
apk.versionName = packageInfo.versionName;
apk.versionCode = packageInfo.versionCode;
apk.added = this.added;
int[] minTargetMax = getMinTargetMaxSdkVersions(context, packageName);
apk.minSdkVersion = minTargetMax[0];
apk.targetSdkVersion = minTargetMax[1];
apk.maxSdkVersion = minTargetMax[2];
apk.packageName = this.packageName;
apk.requestedPermissions = packageInfo.requestedPermissions;
apk.apkName = apk.packageName + "_" + apk.versionCode + ".apk";
initInstalledObbFiles(apk);
final FeatureInfo[] features = packageInfo.reqFeatures;
if (features != null && features.length > 0) {
apk.features = new String[features.length];
for (int i = 0; i < features.length; i++) {
apk.features[i] = features[i].name;
}
}
if (!apkFile.canRead()) {
return;
}
apk.installedFile = apkFile;
JarFile apkJar = new JarFile(apkFile);
HashSet<String> abis = new HashSet<>(3);
Pattern pattern = Pattern.compile("^lib/([a-z0-9-]+)/.*");
for (Enumeration<JarEntry> jarEntries = apkJar.entries(); jarEntries.hasMoreElements(); ) {
JarEntry jarEntry = jarEntries.nextElement();
Matcher matcher = pattern.matcher(jarEntry.getName());
if (matcher.matches()) {
abis.add(matcher.group(1));
}
}
apk.nativecode = abis.toArray(new String[abis.size()]);
final JarEntry aSignedEntry = (JarEntry) apkJar.getEntry("AndroidManifest.xml");
if (aSignedEntry == null) {
apkJar.close();
throw new CertificateEncodingException("null signed entry!");
}
final InputStream tmpIn = apkJar.getInputStream(aSignedEntry);
byte[] buff = new byte[2048];
//noinspection StatementWithEmptyBody
while (tmpIn.read(buff, 0, buff.length) != -1) {
/*
* NOP - apparently have to READ from the JarEntry before you can
* call getCerficates() and have it return != null. Yay Java.
*/
}
tmpIn.close();
if (aSignedEntry.getCertificates() == null
|| aSignedEntry.getCertificates().length == 0) {
apkJar.close();
throw new CertificateEncodingException("No Certificates found!");
}
final Certificate signer = aSignedEntry.getCertificates()[0];
byte[] rawCertBytes = signer.getEncoded();
apkJar.close();
apk.sig = Utils.getsig(rawCertBytes);
}
/**
* Attempts to find the installed {@link Apk} in the given list of APKs. If not found, will lookup the
* the details of the installed app and use that to instantiate an {@link Apk} to be returned.
@@ -1381,14 +1250,6 @@ public class App extends ValueObject implements Comparable<App>, Parcelable {
return TextUtils.isEmpty(liberapay) ? null : "https://liberapay.com/" + liberapay;
}
/**
* @see App#autoInstallVersionName for why this uses a getter while other member variables are
* publicly accessible.
*/
public String getAutoInstallVersionName() {
return autoInstallVersionName;
}
/**
* {@link PackageManager} doesn't give us {@code minSdkVersion}, {@code targetSdkVersion},
* and {@code maxSdkVersion}, so we have to parse it straight from {@code <uses-sdk>} in
@@ -1397,6 +1258,7 @@ public class App extends ValueObject implements Comparable<App>, Parcelable {
*
* @see <a href="https://developer.android.com/guide/topics/manifest/uses-sdk-element.html">&lt;uses-sdk&gt;</a>
*/
@SuppressWarnings("unused") // TODO port to lib
private static int[] getMinTargetMaxSdkVersions(Context context, String packageName) {
int minSdkVersion = Apk.SDK_VERSION_MIN_VALUE;
int targetSdkVersion = Apk.SDK_VERSION_MIN_VALUE;

3
app/src/main/java/org/fdroid/fdroid/data/AppProvider.java
View File

@@ -890,8 +890,7 @@ public class AppProvider extends FDroidProvider {
}
/**
* Helper method used by both the genuine {@link AppProvider} and the temporary version used
* by the repo updater ({@link TempAppProvider}).
* Helper method used by the genuine {@link AppProvider}.
* <p>
* Query the database table specified by {@code uri}, which is usually (always?)
* {@link AppMetadataTable} with specified {@code selection} and {@code sortOrder}.

196
app/src/main/java/org/fdroid/fdroid/data/RepoPersister.java
View File

@@ -1,196 +0,0 @@
package org.fdroid.fdroid.data;
import android.content.ContentProviderOperation;
import android.content.ContentValues;
import android.content.Context;
import android.content.OperationApplicationException;
import android.net.Uri;
import android.os.RemoteException;
import org.fdroid.fdroid.CompatibilityChecker;
import org.fdroid.fdroid.IndexUpdater;
import org.fdroid.fdroid.Utils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import androidx.annotation.NonNull;
public class RepoPersister {
private static final String TAG = "RepoPersister";
/**
* Crappy benchmark with a Nexus 4, Android 5.0 on a fairly crappy internet connection I get:
* * 25 = 37 seconds
* * 50 = 33 seconds
* * 100 = 30 seconds
* * 200 = 32 seconds
* Raising this means more memory consumption, so we'd like it to be low, but not
* so low that it takes too long.
*/
private static final int MAX_APP_BUFFER = 50;
@NonNull
private final Repo repo;
private boolean hasBeenInitialized;
@NonNull
private final Context context;
@NonNull
private final List<App> appsToSave = new ArrayList<>();
@NonNull
private final Map<String, List<Apk>> apksToSave = new HashMap<>();
@NonNull
private final CompatibilityChecker checker;
public RepoPersister(@NonNull Context context, @NonNull Repo repo) {
this.repo = repo;
this.context = context;
checker = new CompatibilityChecker(context);
}
public void saveToDb(App app, List<Apk> packages) throws IndexUpdater.UpdateException {
appsToSave.add(app);
apksToSave.put(app.packageName, packages);
if (appsToSave.size() >= MAX_APP_BUFFER) {
flushBufferToDb();
}
}
public void commit(ContentValues repoDetailsToSave, long repoIdToCommit) throws IndexUpdater.UpdateException {
flushBufferToDb();
TempAppProvider.Helper.commitAppsAndApks(context, repoIdToCommit);
RepoProvider.Helper.update(context, repo, repoDetailsToSave);
}
private void flushBufferToDb() throws IndexUpdater.UpdateException {
if (!hasBeenInitialized) {
// This is where we will store all of the metadata before committing at the
// end of the process. This is due to the fact that we can't verify the cert
// the index was signed with until we've finished reading it - and we don't
// want to put stuff in the real database until we are sure it is from a
// trusted source. It also helps performance as it is done via an in-memory database.
TempAppProvider.Helper.init(context, repo.getId());
hasBeenInitialized = true;
}
if (apksToSave.size() > 0 || appsToSave.size() > 0) {
Utils.debugLog(TAG, "Flushing details of up to " + MAX_APP_BUFFER + " apps/packages to the database.");
Map<String, Long> appIds = flushAppsToDbInBatch();
flushApksToDbInBatch(appIds);
apksToSave.clear();
appsToSave.clear();
}
}
private void flushApksToDbInBatch(Map<String, Long> appIds) throws IndexUpdater.UpdateException {
List<Apk> apksToSaveList = new ArrayList<>();
for (Map.Entry<String, List<Apk>> entries : apksToSave.entrySet()) {
for (Apk apk : entries.getValue()) {
apk.appId = appIds.get(apk.packageName);
}
apksToSaveList.addAll(entries.getValue());
}
calcApkCompatibilityFlags(apksToSaveList);
ArrayList<ContentProviderOperation> apkOperations = insertApks(apksToSaveList);
try {
context.getContentResolver().applyBatch(TempApkProvider.getAuthority(), apkOperations);
} catch (RemoteException | OperationApplicationException e) {
throw new IndexUpdater.UpdateException(repo, "An internal error occurred while updating the database", e);
}
}
/**
* Will first insert new or update existing rows in the database for each {@link RepoPersister#appsToSave}.
* Then, will query the database for the ID + packageName for each of these apps, so that they
* can be returned and the relevant apks can be joined to the app table correctly.
*/
private Map<String, Long> flushAppsToDbInBatch() throws IndexUpdater.UpdateException {
ArrayList<ContentProviderOperation> appOperations = insertApps(appsToSave);
try {
context.getContentResolver().applyBatch(TempAppProvider.getAuthority(), appOperations);
return getIdsForPackages(appsToSave);
} catch (RemoteException | OperationApplicationException e) {
throw new IndexUpdater.UpdateException(repo, "An internal error occurred while updating the database", e);
}
}
/**
* Although this might seem counter intuitive - receiving a list of apps, then querying the
* database again for info about these apps, it is required because the apps came from the
* repo metadata, but we are really interested in their IDs from the database. These IDs only
* exist in SQLite and not the repo metadata.
*/
private Map<String, Long> getIdsForPackages(List<App> apps) {
List<String> packageNames = new ArrayList<>(appsToSave.size());
for (App app : apps) {
packageNames.add(app.packageName);
}
String[] projection = {
Schema.AppMetadataTable.Cols.ROW_ID,
Schema.AppMetadataTable.Cols.Package.PACKAGE_NAME,
};
List<App> fromDb = TempAppProvider.Helper.findByPackageNames(context, packageNames, repo.id, projection);
Map<String, Long> ids = new HashMap<>(fromDb.size());
for (App app : fromDb) {
ids.put(app.packageName, app.getId());
}
return ids;
}
private ArrayList<ContentProviderOperation> insertApps(List<App> apps) {
ArrayList<ContentProviderOperation> operations = new ArrayList<>(apps.size());
for (App app : apps) {
ContentValues values = app.toContentValues();
Uri uri = TempAppProvider.getContentUri();
operations.add(ContentProviderOperation.newInsert(uri).withValues(values).build());
}
return operations;
}
private ArrayList<ContentProviderOperation> insertApks(List<Apk> packages) {
ArrayList<ContentProviderOperation> operations = new ArrayList<>(packages.size());
for (Apk apk : packages) {
ContentValues values = apk.toContentValues();
Uri uri = TempApkProvider.getContentUri();
operations.add(ContentProviderOperation.newInsert(uri).withValues(values).build());
}
return operations;
}
/**
* This cannot be offloaded to the database (as we did with the query which
* updates apps, depending on whether their apks are compatible or not).
* The reason is that we need to interact with the CompatibilityChecker
* in order to see if, and why an apk is not compatible.
*/
private void calcApkCompatibilityFlags(List<Apk> apks) {
for (final Apk apk : apks) {
final List<String> reasons = checker.getIncompatibleReasons(apk);
if (reasons.isEmpty()) {
apk.compatible = true;
apk.incompatibleReasons = null;
} else {
apk.compatible = false;
apk.incompatibleReasons = reasons.toArray(new String[reasons.size()]);
}
}
}
}

3
app/src/main/java/org/fdroid/fdroid/data/RepoPushRequest.java
View File

@@ -48,9 +48,6 @@ public class RepoPushRequest {
* Create a new instance. {@code request} is validated against the list of
* valid install requests. {@code packageName} has a safety validation to
* make sure that only valid Android/Java Package Name characters are included.
* If validation fails, the the values are set to {@code null}, which are
* handled in {@link org.fdroid.fdroid.IndexV1Updater#processRepoPushRequests(List)}
* or {@link org.fdroid.fdroid.IndexUpdater#processRepoPushRequests(List)}
*/
public RepoPushRequest(String request, String packageName, @Nullable String versionCode) {
if (VALID_REQUESTS.contains(request)) {

124
app/src/main/java/org/fdroid/fdroid/data/TempApkProvider.java
View File

@@ -1,124 +0,0 @@
package org.fdroid.fdroid.data;
import android.content.ContentValues;
import android.content.Context;
import android.content.UriMatcher;
import android.database.sqlite.SQLiteDatabase;
import android.net.Uri;
import org.fdroid.fdroid.data.Schema.ApkTable;
import org.fdroid.fdroid.data.Schema.ApkTable.Cols;
import androidx.annotation.NonNull;
/**
* This class does all of its operations in a temporary sqlite table.
*/
@SuppressWarnings("LineLength")
public class TempApkProvider extends ApkProvider {
private static final String PROVIDER_NAME = "TempApkProvider";
static final String TABLE_TEMP_APK = "temp_" + ApkTable.NAME;
private static final String PATH_INIT = "init";
private static final int CODE_INIT = 10000;
private static final UriMatcher MATCHER = new UriMatcher(-1);
static {
MATCHER.addURI(getAuthority(), PATH_INIT + "/#", CODE_INIT);
MATCHER.addURI(getAuthority(), PATH_APK_FROM_ANY_REPO + "/#/*", CODE_APK_FROM_ANY_REPO);
MATCHER.addURI(getAuthority(), PATH_APK_FROM_REPO + "/#/#", CODE_APK_FROM_REPO);
}
@Override
protected String getTableName() {
return TABLE_TEMP_APK;
}
@Override
protected String getApkAntiFeatureJoinTableName() {
return TempAppProvider.TABLE_TEMP_APK_ANTI_FEATURE_JOIN;
}
@Override
protected String getAppTableName() {
return TempAppProvider.TABLE_TEMP_APP;
}
public static String getAuthority() {
return AUTHORITY + "." + PROVIDER_NAME;
}
public static Uri getContentUri() {
return Uri.parse("content://" + getAuthority());
}
public static class Helper {
/**
* Deletes the old temporary table (if it exists). Then creates a new temporary apk provider
* table and populates it with all the data from the real apk provider table.
*
* This is package local because it must be invoked after
* {@link org.fdroid.fdroid.data.TempAppProvider.Helper#init(Context, long)}. Due to this
* dependence, that method invokes this one itself, rather than leaving it to the
* {@link RepoPersister}.
*/
static void init(Context context, long repoIdToUpdate) {
Uri uri = getContentUri().buildUpon()
.appendPath(PATH_INIT)
.appendPath(Long.toString(repoIdToUpdate))
.build();
context.getContentResolver().insert(uri, new ContentValues());
}
}
@Override
public Uri insert(@NonNull Uri uri, ContentValues values) {
if (MATCHER.match(uri) == CODE_INIT) {
initTable(Long.parseLong(uri.getLastPathSegment()));
return null;
}
return super.insert(uri, values);
}
@Override
public int update(@NonNull Uri uri, ContentValues values, String where, String[] whereArgs) {
throw new UnsupportedOperationException("Invalid URI for apk content provider: " + uri);
}
@Override
public int delete(@NonNull Uri uri, String where, String[] whereArgs) {
throw new UnsupportedOperationException("Invalid URI for apk content provider: " + uri);
}
private void initTable(long repoIdBeingUpdated) {
final SQLiteDatabase db = db();
final String memoryDbName = TempAppProvider.DB;
db.execSQL(DBHelper.CREATE_TABLE_APK.replaceFirst(ApkTable.NAME, memoryDbName + "." + getTableName()));
db.execSQL(DBHelper.CREATE_TABLE_APK_ANTI_FEATURE_JOIN.replaceFirst(Schema.ApkAntiFeatureJoinTable.NAME, memoryDbName + "." + getApkAntiFeatureJoinTableName()));
String where = ApkTable.NAME + "." + Cols.REPO_ID + " != ?";
String[] whereArgs = new String[]{Long.toString(repoIdBeingUpdated)};
db.execSQL(TempAppProvider.copyData(Cols.ALL_COLS, ApkTable.NAME, memoryDbName + "." + getTableName(), where), whereArgs);
String antiFeaturesWhere =
Schema.ApkAntiFeatureJoinTable.NAME + "." + Schema.ApkAntiFeatureJoinTable.Cols.APK_ID + " IN " +
"(SELECT innerApk." + Cols.ROW_ID + " FROM " + ApkTable.NAME + " AS innerApk " +
"WHERE innerApk." + Cols.REPO_ID + " != ?)";
db.execSQL(TempAppProvider.copyData(
Schema.ApkAntiFeatureJoinTable.Cols.ALL_COLS,
Schema.ApkAntiFeatureJoinTable.NAME,
memoryDbName + "." + getApkAntiFeatureJoinTableName(),
antiFeaturesWhere), whereArgs);
db.execSQL("CREATE INDEX IF NOT EXISTS " + memoryDbName + ".apk_appId on " + getTableName() + " (" + Cols.APP_ID + ");");
db.execSQL("CREATE INDEX IF NOT EXISTS " + memoryDbName + ".apk_compatible ON " + getTableName() + " (" + Cols.IS_COMPATIBLE + ");");
}
}

281
app/src/main/java/org/fdroid/fdroid/data/TempAppProvider.java
View File

@@ -1,281 +0,0 @@
package org.fdroid.fdroid.data;
import android.content.ContentValues;
import android.content.Context;
import android.content.UriMatcher;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteException;
import android.net.Uri;
import android.text.TextUtils;
import org.fdroid.fdroid.data.Schema.ApkTable;
import org.fdroid.fdroid.data.Schema.AppMetadataTable;
import org.fdroid.fdroid.data.Schema.AppMetadataTable.Cols;
import org.fdroid.fdroid.data.Schema.CatJoinTable;
import org.fdroid.fdroid.data.Schema.PackageTable;
import java.util.List;
import androidx.annotation.NonNull;
/**
* This class does all of its operations in a temporary sqlite table.
*/
@SuppressWarnings("LineLength")
public class TempAppProvider extends AppProvider {
/**
* The name of the in memory database used for updating.
*/
static final String DB = "temp_update_db";
private static final String PROVIDER_NAME = "TempAppProvider";
static final String TABLE_TEMP_APP = "temp_" + AppMetadataTable.NAME;
static final String TABLE_TEMP_APK_ANTI_FEATURE_JOIN = "temp_" + Schema.ApkAntiFeatureJoinTable.NAME;
static final String TABLE_TEMP_CAT_JOIN = "temp_" + CatJoinTable.NAME;
private static final String PATH_INIT = "init";
private static final String PATH_COMMIT = "commit";
private static final int CODE_INIT = 10000;
private static final int CODE_COMMIT = CODE_INIT + 1;
private static final int APPS = CODE_COMMIT + 1;
private static final UriMatcher MATCHER = new UriMatcher(-1);
static {
MATCHER.addURI(getAuthority(), PATH_INIT + "/#", CODE_INIT);
MATCHER.addURI(getAuthority(), PATH_COMMIT + "/#", CODE_COMMIT);
MATCHER.addURI(getAuthority(), PATH_APPS + "/#/*", APPS);
MATCHER.addURI(getAuthority(), PATH_SPECIFIC_APP + "/#/*", CODE_SINGLE);
}
@Override
protected String getTableName() {
return TABLE_TEMP_APP;
}
@Override
protected String getCatJoinTableName() {
return TABLE_TEMP_CAT_JOIN;
}
public static String getAuthority() {
return AUTHORITY + "." + PROVIDER_NAME;
}
public static Uri getContentUri() {
return Uri.parse("content://" + getAuthority());
}
public static Uri getAppsUri(List<String> apps, long repoId) {
return getContentUri().buildUpon()
.appendPath(PATH_APPS)
.appendPath(Long.toString(repoId))
.appendPath(TextUtils.join(",", apps))
.build();
}
private AppQuerySelection queryRepoApps(long repoId, String packageNames) {
return queryPackageNames(packageNames, PackageTable.NAME + "." + PackageTable.Cols.PACKAGE_NAME)
.add(queryRepo(repoId));
}
private AppQuerySelection queryRepo(long repoId) {
String[] args = new String[]{Long.toString(repoId)};
String selection = getTableName() + "." + Cols.REPO_ID + " = ? ";
return new AppQuerySelection(selection, args);
}
public static class Helper {
/**
* Deletes the old temporary table (if it exists). Then creates a new temporary apk provider
* table and populates it with all the data from the real apk provider table.
*/
public static void init(Context context, long repoIdToUpdate) {
Uri uri = getContentUri().buildUpon()
.appendPath(PATH_INIT)
.appendPath(Long.toString(repoIdToUpdate))
.build();
context.getContentResolver().insert(uri, new ContentValues());
TempApkProvider.Helper.init(context, repoIdToUpdate);
}
public static List<App> findByPackageNames(Context context,
List<String> packageNames, long repoId, String[] projection) {
Uri uri = getAppsUri(packageNames, repoId);
Cursor cursor = context.getContentResolver().query(uri, projection, null, null, null);
return AppProvider.Helper.cursorToList(cursor);
}
/**
* Saves data from the temp table to the apk table, by removing _EVERYTHING_ from the real
* apk table and inserting all of the records from here. The temporary table is then removed.
*/
public static void commitAppsAndApks(Context context, long repoIdToCommit) {
Uri uri = getContentUri().buildUpon()
.appendPath(PATH_COMMIT)
.appendPath(Long.toString(repoIdToCommit))
.build();
context.getContentResolver().insert(uri, new ContentValues());
}
}
@Override
protected String getApkTableName() {
return TempApkProvider.TABLE_TEMP_APK;
}
protected String getApkAntiFeatureJoinTableName() {
return TempApkProvider.TABLE_TEMP_APK;
}
@Override
public Uri insert(@NonNull Uri uri, ContentValues values) {
switch (MATCHER.match(uri)) {
case CODE_INIT:
initTable(Long.parseLong(uri.getLastPathSegment()));
return null;
case CODE_COMMIT:
updateAllAppDetails();
commitTable(Long.parseLong(uri.getLastPathSegment()));
return null;
default:
return super.insert(uri, values);
}
}
@Override
public int update(@NonNull Uri uri, ContentValues values, String where, String[] whereArgs) {
throw new UnsupportedOperationException("Update not supported for " + uri + ".");
}
@Override
public Cursor query(@NonNull Uri uri, String[] projection,
String customSelection, String[] selectionArgs, String sortOrder) {
AppQuerySelection selection = new AppQuerySelection(customSelection, selectionArgs);
switch (MATCHER.match(uri)) {
case APPS:
List<String> segments = uri.getPathSegments();
selection = selection.add(queryRepoApps(Long.parseLong(segments.get(1)), segments.get(2)));
break;
}
return super.runQuery(uri, selection, projection, true, sortOrder, 0);
}
private void ensureTempTableDetached(SQLiteDatabase db) {
try {
// Ideally we'd ask SQLite if the temp table is attached, but that is not possible.
// Instead, we resort to hackery:
// If the first statement does not throw an exception, then the temp db is attached and the second
// statement will detach the database.
db.rawQuery("SELECT * FROM " + DB + "." + getTableName() + " WHERE 0", null);
db.execSQL("DETACH DATABASE " + DB);
} catch (SQLiteException ignored) {
}
}
private void initTable(long repoIdBeingUpdated) {
final SQLiteDatabase db = db();
String mainApp = AppMetadataTable.NAME;
String tempApp = DB + "." + getTableName();
String mainCat = CatJoinTable.NAME;
String tempCat = DB + "." + getCatJoinTableName();
ensureTempTableDetached(db);
db.execSQL("ATTACH DATABASE ':memory:' AS " + DB);
db.execSQL(DBHelper.CREATE_TABLE_APP_METADATA.replaceFirst(AppMetadataTable.NAME, tempApp));
db.execSQL(DBHelper.CREATE_TABLE_CAT_JOIN.replaceFirst(CatJoinTable.NAME, tempCat));
String appWhere = mainApp + "." + Cols.REPO_ID + " != ?";
String[] repoArgs = new String[]{Long.toString(repoIdBeingUpdated)};
db.execSQL(copyData(Cols.ALL_COLS, mainApp, tempApp, appWhere), repoArgs);
// TODO: String catWhere = mainCat + "." + CatJoinTable.Cols..Cols.REPO_ID + " != ?";
db.execSQL(copyData(CatJoinTable.Cols.ALL_COLS, mainCat, tempCat, null));
db.execSQL("CREATE INDEX IF NOT EXISTS " + DB + ".app_id ON " + getTableName() + " (" + Cols.PACKAGE_ID + ");");
db.execSQL("CREATE INDEX IF NOT EXISTS " + DB + ".app_upstreamVercode ON " + getTableName() + " (" + Cols.SUGGESTED_VERSION_CODE + ");");
db.execSQL("CREATE INDEX IF NOT EXISTS " + DB + ".app_compatible ON " + getTableName() + " (" + Cols.IS_COMPATIBLE + ");");
}
/**
* Constructs an INSERT INTO ... SELECT statement as a means from getting data from one table
* into another. The list of columns to copy are explicitly specified using colsToCopy.
*/
static String copyData(String[] colsToCopy, String fromTable, String toTable, String where) {
String cols = TextUtils.join(", ", colsToCopy);
String sql = "INSERT INTO " + toTable + " (" + cols + ") SELECT " + cols + " FROM " + fromTable;
if (!TextUtils.isEmpty(where)) {
sql += " WHERE " + where;
}
return sql;
}
private void commitTable(long repoIdToCommit) {
final SQLiteDatabase db = db();
try {
db.beginTransaction();
final String tempApp = DB + "." + TABLE_TEMP_APP;
final String tempApk = DB + "." + TempApkProvider.TABLE_TEMP_APK;
final String tempCatJoin = DB + "." + TABLE_TEMP_CAT_JOIN;
final String tempAntiFeatureJoin = DB + "." + TABLE_TEMP_APK_ANTI_FEATURE_JOIN;
final String[] repoArgs = new String[]{Long.toString(repoIdToCommit)};
db.execSQL("DELETE FROM " + AppMetadataTable.NAME + " WHERE " + Cols.REPO_ID + " = ?", repoArgs);
db.execSQL(copyData(Cols.ALL_COLS, tempApp, AppMetadataTable.NAME, Cols.REPO_ID + " = ?"), repoArgs);
db.execSQL("DELETE FROM " + ApkTable.NAME + " WHERE " + ApkTable.Cols.REPO_ID + " = ?", repoArgs);
db.execSQL(copyData(ApkTable.Cols.ALL_COLS, tempApk, ApkTable.NAME, ApkTable.Cols.REPO_ID + " = ?"), repoArgs);
db.execSQL("DELETE FROM " + CatJoinTable.NAME + " WHERE " + getCatRepoWhere(CatJoinTable.NAME), repoArgs);
db.execSQL(copyData(CatJoinTable.Cols.ALL_COLS, tempCatJoin, CatJoinTable.NAME, getCatRepoWhere(tempCatJoin)), repoArgs);
db.execSQL(
"DELETE FROM " + Schema.ApkAntiFeatureJoinTable.NAME + " " +
"WHERE " + getAntiFeatureRepoWhere(Schema.ApkAntiFeatureJoinTable.NAME), repoArgs);
db.execSQL(copyData(
Schema.ApkAntiFeatureJoinTable.Cols.ALL_COLS,
tempAntiFeatureJoin,
Schema.ApkAntiFeatureJoinTable.NAME,
getAntiFeatureRepoWhere(tempAntiFeatureJoin)), repoArgs);
db.setTransactionSuccessful();
getContext().getContentResolver().notifyChange(AppProvider.getContentUri(), null);
getContext().getContentResolver().notifyChange(ApkProvider.getContentUri(), null);
getContext().getContentResolver().notifyChange(CategoryProvider.getContentUri(), null);
} finally {
db.endTransaction();
db.execSQL("DETACH DATABASE " + DB); // Can't be done in a transaction.
}
}
private String getCatRepoWhere(String categoryTable) {
String catRepoSubquery =
"SELECT DISTINCT innerCatJoin." + CatJoinTable.Cols.ROW_ID + " " +
"FROM " + categoryTable + " AS innerCatJoin " +
"JOIN " + getTableName() + " AS app ON (app." + Cols.ROW_ID + " = innerCatJoin." + CatJoinTable.Cols.APP_METADATA_ID + ") " +
"WHERE app." + Cols.REPO_ID + " = ?";
return CatJoinTable.Cols.ROW_ID + " IN (" + catRepoSubquery + ")";
}
private String getAntiFeatureRepoWhere(String antiFeatureTable) {
String subquery =
"SELECT innerApk." + ApkTable.Cols.ROW_ID + " " +
"FROM " + ApkTable.NAME + " AS innerApk " +
"WHERE innerApk." + ApkTable.Cols.REPO_ID + " = ?";
return antiFeatureTable + "." + Schema.ApkAntiFeatureJoinTable.Cols.APK_ID + " IN (" + subquery + ")";
}
}

1
app/src/main/java/org/fdroid/fdroid/net/DownloaderService.java
View File

@@ -239,7 +239,6 @@ public class DownloaderService extends Service {
*
* @param intent The {@link Intent} passed via {@link
* android.content.Context#startService(Intent)}.
* @see org.fdroid.fdroid.IndexV1Updater#update()
*/
private void handleIntent(Intent intent) {
final Uri uri = intent.getData();

4
app/src/main/java/org/fdroid/fdroid/views/ManageReposActivity.java
View File

@@ -56,7 +56,6 @@ import org.fdroid.download.Mirror;
import org.fdroid.fdroid.AddRepoIntentService;
import org.fdroid.fdroid.AppUpdateStatusManager;
import org.fdroid.fdroid.FDroidApp;
import org.fdroid.fdroid.IndexUpdater;
import org.fdroid.fdroid.Preferences;
import org.fdroid.fdroid.R;
import org.fdroid.fdroid.UpdateService;
@@ -621,7 +620,8 @@ public class ManageReposActivity extends AppCompatActivity implements RepoAdapte
return Pair.create(statusCode, addressWithoutIndex);
}
final Uri uri = builder.appendPath(IndexUpdater.SIGNED_FILE_NAME).build();
// check for v1 index as this is the last one we can still handle
final Uri uri = builder.appendPath("index-v1.jar").build();
try {
final URL url = new URL(uri.toString());

19
app/src/main/res/values/config.xml
View File

@@ -1,19 +0,0 @@
<!--
~ Copyright (C) 2021 The Calyx Institute
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<resources>
<bool name="config_allowPushRequests">false</bool>
</resources>

10
app/src/test/java/org/fdroid/fdroid/RepoUrlsTest.java
View File

@@ -119,21 +119,19 @@ public class RepoUrlsTest {
@Test
public void testIndexUrls() {
testReposWithFile(IndexUpdater.SIGNED_FILE_NAME, tr -> {
testReposWithFile("index.jar", tr -> {
Repo repo = new Repo();
repo.address = tr.repoUrl;
IndexUpdater updater = new IndexUpdater(context, repo);
return updater.getIndexUrl(repo);
return repo.getFileUrl("index.jar");
});
}
@Test
public void testIndexV1Urls() {
testReposWithFile(IndexV1Updater.SIGNED_FILE_NAME, tr -> {
testReposWithFile("index-v1.jar", tr -> {
Repo repo = new Repo();
repo.address = tr.repoUrl;
IndexV1Updater updater = new IndexV1Updater(context, repo);
return updater.getIndexUrl(repo);
return repo.getFileUrl("index-v1.jar");
});
}

15
app/src/testFull/java/org/fdroid/fdroid/nearby/LocalRepoKeyStoreTest.java
View File

@@ -4,8 +4,8 @@ import android.content.Context;
import android.text.TextUtils;
import org.apache.commons.io.IOUtils;
import org.fdroid.fdroid.IndexUpdater;
import org.fdroid.fdroid.Utils;
import org.fdroid.index.SigningException;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.RobolectricTestRunner;
@@ -29,14 +29,17 @@ import static org.junit.Assert.assertNotNull;
@RunWith(RobolectricTestRunner.class)
public class LocalRepoKeyStoreTest {
private static final String SIGNED_FILE_NAME = "index.jar";
private static final String DATA_FILE_NAME = "index.xml";
@Test
public void testSignZip() throws IOException, LocalRepoKeyStore.InitException, IndexUpdater.SigningException {
public void testSignZip() throws IOException, LocalRepoKeyStore.InitException, SigningException {
Context context = ApplicationProvider.getApplicationContext();
File xmlIndexJarUnsigned = File.createTempFile(getClass().getName(), "unsigned.jar");
BufferedOutputStream bo = new BufferedOutputStream(new FileOutputStream(xmlIndexJarUnsigned));
JarOutputStream jo = new JarOutputStream(bo);
JarEntry je = new JarEntry(IndexUpdater.DATA_FILE_NAME);
JarEntry je = new JarEntry(DATA_FILE_NAME);
jo.putNextEntry(je);
InputStream inputStream =
getClass().getClassLoader().getResourceAsStream("all_fields_index-v1.json");
@@ -48,13 +51,13 @@ public class LocalRepoKeyStoreTest {
Certificate localCert = localRepoKeyStore.getCertificate();
assertFalse(TextUtils.isEmpty(Utils.calcFingerprint(localCert)));
File xmlIndexJar = File.createTempFile(getClass().getName(), IndexUpdater.SIGNED_FILE_NAME);
File xmlIndexJar = File.createTempFile(getClass().getName(), SIGNED_FILE_NAME);
localRepoKeyStore.signZip(xmlIndexJarUnsigned, xmlIndexJar);
JarFile jarFile = new JarFile(xmlIndexJar, true);
JarEntry indexEntry = (JarEntry) jarFile.getEntry(IndexUpdater.DATA_FILE_NAME);
JarEntry indexEntry = (JarEntry) jarFile.getEntry(DATA_FILE_NAME);
byte[] data = IOUtils.toByteArray(jarFile.getInputStream(indexEntry));
assertEquals(6431, data.length);
assertNotNull(IndexUpdater.getSigningCertFromJar(indexEntry));
assertNotNull(TreeUriScannerIntentService.getSigningCertFromJar(indexEntry));
}
}

28
app/src/testFull/java/org/fdroid/fdroid/updater/SwapRepoTest.java
View File

@@ -4,7 +4,6 @@ import android.content.ContentResolver;
import android.content.ContentValues;
import android.content.Context;
import android.content.ContextWrapper;
import android.content.Intent;
import android.content.pm.ApplicationInfo;
import android.content.pm.PackageInfo;
import android.content.pm.PackageManager;
@@ -13,19 +12,13 @@ import android.text.TextUtils;
import org.apache.commons.net.util.SubnetUtils;
import org.fdroid.fdroid.FDroidApp;
import org.fdroid.fdroid.Hasher;
import org.fdroid.fdroid.IndexUpdater;
import org.fdroid.fdroid.Preferences;
import org.fdroid.fdroid.TestUtils;
import org.fdroid.fdroid.Utils;
import org.fdroid.fdroid.data.Apk;
import org.fdroid.fdroid.data.ApkProvider;
import org.fdroid.fdroid.data.AppProvider;
import org.fdroid.fdroid.data.DBHelper;
import org.fdroid.fdroid.data.Repo;
import org.fdroid.fdroid.data.RepoProvider;
import org.fdroid.fdroid.data.Schema;
import org.fdroid.fdroid.data.ShadowApp;
import org.fdroid.fdroid.data.TempAppProvider;
import org.fdroid.fdroid.nearby.LocalHTTPD;
import org.fdroid.fdroid.nearby.LocalRepoKeyStore;
import org.fdroid.fdroid.nearby.LocalRepoManager;
@@ -43,7 +36,6 @@ import org.robolectric.shadows.ShadowLog;
import java.io.File;
import java.io.IOException;
import java.security.cert.Certificate;
import java.util.List;
import androidx.test.core.app.ApplicationProvider;
@@ -95,7 +87,7 @@ public class SwapRepoTest {
*/
@Test
public void testSwap()
throws IOException, LocalRepoKeyStore.InitException, IndexUpdater.UpdateException, InterruptedException {
throws IOException, LocalRepoKeyStore.InitException, InterruptedException {
PackageManager packageManager = context.getPackageManager();
@@ -145,10 +137,10 @@ public class SwapRepoTest {
assertFalse(TextUtils.isEmpty(Utils.calcFingerprint(localCert)));
Repo repo = createRepo("", FDroidApp.repo.getAddress(), context, signingCert);
IndexUpdater updater = new IndexUpdater(context, repo);
updater.update();
assertTrue(updater.hasChanged());
updater.processDownloadedFile(indexJarFile);
//IndexUpdater updater = new IndexUpdater(context, repo);
//updater.update();
//assertTrue(updater.hasChanged());
//updater.processDownloadedFile(indexJarFile);
boolean foundRepo = false;
for (Repo repoFromDb : RepoProvider.Helper.all(context)) {
@@ -160,11 +152,11 @@ public class SwapRepoTest {
assertTrue(foundRepo);
assertNotEquals(-1, repo.getId());
// List<Apk> apks = ApkProvider.Helper.findByRepo(context, repo, Schema.ApkTable.Cols.ALL);
// assertEquals(1, apks.size());
// for (Apk apk : apks) {
// System.out.println(apk);
// }
//List<Apk> apks = ApkProvider.Helper.findByRepo(context, repo, Schema.ApkTable.Cols.ALL);
//assertEquals(1, apks.size());
//for (Apk apk : apks) {
// System.out.println(apk);
//}
//MultiIndexUpdaterTest.assertApksExist(apks, context.getPackageName(), new int[]{BuildConfig.VERSION_CODE});
Thread.sleep(10000);
} finally {