From 73c761dea90c1afda1799a02e1e372480c4a9751 Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Tue, 22 Oct 2024 11:59:24 -0300 Subject: [PATCH 1/2] [index] rename location to countryCode in MirrorV2 --- libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2.kt b/libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2.kt index e925cc7bc..732e6c4da 100644 --- a/libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2.kt +++ b/libs/index/src/commonMain/kotlin/org/fdroid/index/v2/IndexV2.kt @@ -108,7 +108,7 @@ public typealias LocalizedFileListV2 = Map> @Serializable public data class MirrorV2( val url: String, - val location: String? = null, + val countryCode: String? = null, ) @Serializable From 8a676010baf0a128617f310c5648ae94bdc7f155 Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Tue, 22 Oct 2024 12:04:02 -0300 Subject: [PATCH 2/2] [db] rename location to countryCode in Mirror --- .../8.json | 1098 +++++++++++++++++ .../database/CountryCodeMigrationTest.kt | 99 ++ .../org/fdroid/database/RepositoryDiffTest.kt | 20 +- .../org/fdroid/database/FDroidDatabase.kt | 5 +- .../java/org/fdroid/database/Migrations.kt | 25 + .../java/org/fdroid/database/Repository.kt | 6 +- .../kotlin/org/fdroid/download/Mirror.kt | 2 +- .../src/main/assets/diff-empty-max/1337.json | 4 +- .../src/main/assets/diff-empty-max/23.json | 4 +- .../src/main/assets/diff-empty-max/42.json | 4 +- .../src/main/assets/index-max-v2.json | 4 +- .../kotlin/org/fdroid/test/TestRepoUtils.kt | 2 +- 12 files changed, 1257 insertions(+), 16 deletions(-) create mode 100644 libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/8.json create mode 100644 libs/database/src/dbTest/java/org/fdroid/database/CountryCodeMigrationTest.kt diff --git a/libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/8.json b/libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/8.json new file mode 100644 index 000000000..adfbc5572 --- /dev/null +++ b/libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/8.json @@ -0,0 +1,1098 @@ +{ + "formatVersion": 1, + "database": { + "version": 8, + "identityHash": "99cea240ee67939715bf099681b6b4d9", + "entities": [ + { + "tableName": "CoreRepository", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `icon` TEXT, `address` TEXT NOT NULL, `webBaseUrl` TEXT, `timestamp` INTEGER NOT NULL, `version` INTEGER, `formatVersion` TEXT, `maxAge` INTEGER, `description` TEXT NOT NULL, `certificate` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon", + "columnName": "icon", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "address", + "columnName": "address", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "webBaseUrl", + "columnName": "webBaseUrl", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "timestamp", + "columnName": "timestamp", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "version", + "columnName": "version", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "formatVersion", + "columnName": "formatVersion", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "maxAge", + "columnName": "maxAge", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "certificate", + "columnName": "certificate", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "repoId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "Mirror", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `url` TEXT NOT NULL, `countryCode` TEXT, PRIMARY KEY(`repoId`, `url`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "countryCode", + "columnName": "countryCode", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "url" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "tableName": "AntiFeature", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `id` TEXT NOT NULL, `icon` TEXT NOT NULL, `name` TEXT NOT NULL, `description` TEXT NOT NULL, PRIMARY KEY(`repoId`, `id`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon", + "columnName": "icon", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "id" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "tableName": "Category", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `id` TEXT NOT NULL, `icon` TEXT NOT NULL, `name` TEXT NOT NULL, `description` TEXT NOT NULL, PRIMARY KEY(`repoId`, `id`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon", + "columnName": "icon", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "id" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "tableName": "ReleaseChannel", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `id` TEXT NOT NULL, `icon` TEXT NOT NULL, `name` TEXT NOT NULL, `description` TEXT NOT NULL, PRIMARY KEY(`repoId`, `id`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon", + "columnName": "icon", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "id" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "tableName": "RepositoryPreferences", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `weight` INTEGER NOT NULL, `enabled` INTEGER NOT NULL, `lastUpdated` INTEGER, `lastETag` TEXT, `userMirrors` TEXT, `disabledMirrors` TEXT, `username` TEXT, `password` TEXT, PRIMARY KEY(`repoId`))", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "weight", + "columnName": "weight", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "enabled", + "columnName": "enabled", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastUpdated", + "columnName": "lastUpdated", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "lastETag", + "columnName": "lastETag", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "userMirrors", + "columnName": "userMirrors", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "disabledMirrors", + "columnName": "disabledMirrors", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "username", + "columnName": "username", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "password", + "columnName": "password", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "AppMetadata", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `added` INTEGER NOT NULL, `lastUpdated` INTEGER NOT NULL, `name` TEXT, `summary` TEXT, `description` TEXT, `localizedName` TEXT, `localizedSummary` TEXT, `webSite` TEXT, `changelog` TEXT, `license` TEXT, `sourceCode` TEXT, `issueTracker` TEXT, `translation` TEXT, `preferredSigner` TEXT, `video` TEXT, `authorName` TEXT, `authorEmail` TEXT, `authorWebSite` TEXT, `authorPhone` TEXT, `donate` TEXT, `liberapayID` TEXT, `liberapay` TEXT, `openCollective` TEXT, `bitcoin` TEXT, `litecoin` TEXT, `flattrID` TEXT, `categories` TEXT, `isCompatible` INTEGER NOT NULL, PRIMARY KEY(`repoId`, `packageName`), FOREIGN KEY(`repoId`) REFERENCES `CoreRepository`(`repoId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "added", + "columnName": "added", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastUpdated", + "columnName": "lastUpdated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "summary", + "columnName": "summary", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "localizedName", + "columnName": "localizedName", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "localizedSummary", + "columnName": "localizedSummary", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "webSite", + "columnName": "webSite", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "changelog", + "columnName": "changelog", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "license", + "columnName": "license", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "sourceCode", + "columnName": "sourceCode", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "issueTracker", + "columnName": "issueTracker", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "translation", + "columnName": "translation", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "preferredSigner", + "columnName": "preferredSigner", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "video", + "columnName": "video", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "authorName", + "columnName": "authorName", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "authorEmail", + "columnName": "authorEmail", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "authorWebSite", + "columnName": "authorWebSite", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "authorPhone", + "columnName": "authorPhone", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "donate", + "columnName": "donate", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "liberapayID", + "columnName": "liberapayID", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "liberapay", + "columnName": "liberapay", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "openCollective", + "columnName": "openCollective", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "bitcoin", + "columnName": "bitcoin", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "litecoin", + "columnName": "litecoin", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "flattrID", + "columnName": "flattrID", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "categories", + "columnName": "categories", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "isCompatible", + "columnName": "isCompatible", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "packageName" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "CoreRepository", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId" + ], + "referencedColumns": [ + "repoId" + ] + } + ] + }, + { + "ftsVersion": "FTS4", + "ftsOptions": { + "tokenizer": "unicode61 \"remove_diacritics=0\"", + "tokenizerArgs": [], + "contentTable": "AppMetadata", + "languageIdColumnName": "", + "matchInfo": "FTS4", + "notIndexedColumns": [], + "prefixSizes": [], + "preferredOrder": "ASC" + }, + "contentSyncTriggers": [ + "CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_BEFORE_UPDATE BEFORE UPDATE ON `AppMetadata` BEGIN DELETE FROM `AppMetadataFts` WHERE `docid`=OLD.`rowid`; END", + "CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_BEFORE_DELETE BEFORE DELETE ON `AppMetadata` BEGIN DELETE FROM `AppMetadataFts` WHERE `docid`=OLD.`rowid`; END", + "CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_AFTER_UPDATE AFTER UPDATE ON `AppMetadata` BEGIN INSERT INTO `AppMetadataFts`(`docid`, `repoId`, `packageName`, `localizedName`, `localizedSummary`) VALUES (NEW.`rowid`, NEW.`repoId`, NEW.`packageName`, NEW.`localizedName`, NEW.`localizedSummary`); END", + "CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_AppMetadataFts_AFTER_INSERT AFTER INSERT ON `AppMetadata` BEGIN INSERT INTO `AppMetadataFts`(`docid`, `repoId`, `packageName`, `localizedName`, `localizedSummary`) VALUES (NEW.`rowid`, NEW.`repoId`, NEW.`packageName`, NEW.`localizedName`, NEW.`localizedSummary`); END" + ], + "tableName": "AppMetadataFts", + "createSql": "CREATE VIRTUAL TABLE IF NOT EXISTS `${TABLE_NAME}` USING FTS4(`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `localizedName` TEXT, `localizedSummary` TEXT, tokenize=unicode61 \"remove_diacritics=0\", content=`AppMetadata`)", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "localizedName", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "summary", + "columnName": "localizedSummary", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "LocalizedFile", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `type` TEXT NOT NULL, `locale` TEXT NOT NULL, `name` TEXT NOT NULL, `sha256` TEXT, `size` INTEGER, `ipfsCidV1` TEXT, PRIMARY KEY(`repoId`, `packageName`, `type`, `locale`), FOREIGN KEY(`repoId`, `packageName`) REFERENCES `AppMetadata`(`repoId`, `packageName`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "locale", + "columnName": "locale", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "sha256", + "columnName": "sha256", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "size", + "columnName": "size", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "ipfsCidV1", + "columnName": "ipfsCidV1", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "packageName", + "type", + "locale" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "AppMetadata", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId", + "packageName" + ], + "referencedColumns": [ + "repoId", + "packageName" + ] + } + ] + }, + { + "tableName": "LocalizedFileList", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `type` TEXT NOT NULL, `locale` TEXT NOT NULL, `name` TEXT NOT NULL, `sha256` TEXT, `size` INTEGER, `ipfsCidV1` TEXT, PRIMARY KEY(`repoId`, `packageName`, `type`, `locale`, `name`), FOREIGN KEY(`repoId`, `packageName`) REFERENCES `AppMetadata`(`repoId`, `packageName`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "locale", + "columnName": "locale", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "sha256", + "columnName": "sha256", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "size", + "columnName": "size", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "ipfsCidV1", + "columnName": "ipfsCidV1", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "packageName", + "type", + "locale", + "name" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "AppMetadata", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId", + "packageName" + ], + "referencedColumns": [ + "repoId", + "packageName" + ] + } + ] + }, + { + "tableName": "Version", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `versionId` TEXT NOT NULL, `added` INTEGER NOT NULL, `releaseChannels` TEXT, `antiFeatures` TEXT, `whatsNew` TEXT, `isCompatible` INTEGER NOT NULL, `file_name` TEXT NOT NULL, `file_sha256` TEXT NOT NULL, `file_size` INTEGER, `file_ipfsCidV1` TEXT, `src_name` TEXT, `src_sha256` TEXT, `src_size` INTEGER, `src_ipfsCidV1` TEXT, `manifest_versionName` TEXT NOT NULL, `manifest_versionCode` INTEGER NOT NULL, `manifest_maxSdkVersion` INTEGER, `manifest_nativecode` TEXT, `manifest_features` TEXT, `manifest_usesSdk_minSdkVersion` INTEGER, `manifest_usesSdk_targetSdkVersion` INTEGER, `manifest_signer_sha256` TEXT, `manifest_signer_hasMultipleSigners` INTEGER, PRIMARY KEY(`repoId`, `packageName`, `versionId`), FOREIGN KEY(`repoId`, `packageName`) REFERENCES `AppMetadata`(`repoId`, `packageName`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "versionId", + "columnName": "versionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "added", + "columnName": "added", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "releaseChannels", + "columnName": "releaseChannels", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "antiFeatures", + "columnName": "antiFeatures", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "whatsNew", + "columnName": "whatsNew", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "isCompatible", + "columnName": "isCompatible", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "file.name", + "columnName": "file_name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "file.sha256", + "columnName": "file_sha256", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "file.size", + "columnName": "file_size", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "file.ipfsCidV1", + "columnName": "file_ipfsCidV1", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "src.name", + "columnName": "src_name", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "src.sha256", + "columnName": "src_sha256", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "src.size", + "columnName": "src_size", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "src.ipfsCidV1", + "columnName": "src_ipfsCidV1", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "manifest.versionName", + "columnName": "manifest_versionName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "manifest.versionCode", + "columnName": "manifest_versionCode", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "manifest.maxSdkVersion", + "columnName": "manifest_maxSdkVersion", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "manifest.nativecode", + "columnName": "manifest_nativecode", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "manifest.features", + "columnName": "manifest_features", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "manifest.usesSdk.minSdkVersion", + "columnName": "manifest_usesSdk_minSdkVersion", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "manifest.usesSdk.targetSdkVersion", + "columnName": "manifest_usesSdk_targetSdkVersion", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "manifest.signer.sha256", + "columnName": "manifest_signer_sha256", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "manifest.signer.hasMultipleSigners", + "columnName": "manifest_signer_hasMultipleSigners", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "packageName", + "versionId" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "AppMetadata", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId", + "packageName" + ], + "referencedColumns": [ + "repoId", + "packageName" + ] + } + ] + }, + { + "tableName": "VersionedString", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`repoId` INTEGER NOT NULL, `packageName` TEXT NOT NULL, `versionId` TEXT NOT NULL, `type` TEXT NOT NULL, `name` TEXT NOT NULL, `version` INTEGER, PRIMARY KEY(`repoId`, `packageName`, `versionId`, `type`, `name`), FOREIGN KEY(`repoId`, `packageName`, `versionId`) REFERENCES `Version`(`repoId`, `packageName`, `versionId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "repoId", + "columnName": "repoId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "versionId", + "columnName": "versionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "version", + "columnName": "version", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "repoId", + "packageName", + "versionId", + "type", + "name" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "Version", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "repoId", + "packageName", + "versionId" + ], + "referencedColumns": [ + "repoId", + "packageName", + "versionId" + ] + } + ] + }, + { + "tableName": "AppPrefs", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`packageName` TEXT NOT NULL, `ignoreVersionCodeUpdate` INTEGER NOT NULL, `preferredRepoId` INTEGER, `appPrefReleaseChannels` TEXT, PRIMARY KEY(`packageName`))", + "fields": [ + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "ignoreVersionCodeUpdate", + "columnName": "ignoreVersionCodeUpdate", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "preferredRepoId", + "columnName": "preferredRepoId", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "appPrefReleaseChannels", + "columnName": "appPrefReleaseChannels", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "packageName" + ] + }, + "indices": [], + "foreignKeys": [] + } + ], + "views": [ + { + "viewName": "LocalizedIcon", + "createSql": "CREATE VIEW `${VIEW_NAME}` AS SELECT * FROM LocalizedFile WHERE type='icon'" + }, + { + "viewName": "HighestVersion", + "createSql": "CREATE VIEW `${VIEW_NAME}` AS SELECT repoId, packageName, antiFeatures FROM Version\n GROUP BY repoId, packageName HAVING MAX(manifest_versionCode)" + }, + { + "viewName": "PreferredRepo", + "createSql": "CREATE VIEW `${VIEW_NAME}` AS SELECT packageName, repoId AS preferredRepoId FROM AppMetadata\n JOIN RepositoryPreferences AS pref USING (repoId)\n LEFT JOIN AppPrefs USING (packageName)\n WHERE pref.enabled = 1 AND (repoId = COALESCE(preferredRepoId, repoId) OR\n NOT EXISTS (SELECT 1 FROM AppMetadata WHERE repoId=AppPrefs.preferredRepoId AND packageName=AppPrefs.packageName)\n )\n GROUP BY packageName HAVING MAX(pref.weight)" + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '99cea240ee67939715bf099681b6b4d9')" + ] + } +} \ No newline at end of file diff --git a/libs/database/src/dbTest/java/org/fdroid/database/CountryCodeMigrationTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/CountryCodeMigrationTest.kt new file mode 100644 index 000000000..0a20fbc2d --- /dev/null +++ b/libs/database/src/dbTest/java/org/fdroid/database/CountryCodeMigrationTest.kt @@ -0,0 +1,99 @@ +package org.fdroid.database + +import android.content.ContentValues +import android.database.sqlite.SQLiteDatabase +import androidx.room.Room +import androidx.room.testing.MigrationTestHelper +import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory +import androidx.test.core.app.ApplicationProvider +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import org.junit.Assert.assertEquals +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith + +private const val TEST_DB = "migration-test" + +@RunWith(AndroidJUnit4::class) +internal class CountryCodeMigrationTest { + + @get:Rule + val helper: MigrationTestHelper = MigrationTestHelper( + InstrumentationRegistry.getInstrumentation(), + FDroidDatabaseInt::class.java, + listOf(CountryCodeMigration()), + FrameworkSQLiteOpenHelperFactory(), + ) + + private val repo = InitialRepository( + name = "F-Droid", + address = "https://f-droid.org/repo", + description = "The official F-Droid Free Software repository. " + + "Everything in this repository is always built from the source code.", + certificate = "3082035e30820246a00302010202044c49cd00300d06092a864886f70d010105", + version = 13L, + enabled = true, + weight = 1, + ) + + @Test + fun migrateCountryCode() { + helper.createDatabase(TEST_DB, 7).use { db -> + // Database has schema version 7. Insert some data using SQL queries. + // We can't use DAO classes because they expect the latest schema. + val repoId = db.insert( + CoreRepository.TABLE, + SQLiteDatabase.CONFLICT_FAIL, ContentValues().apply { + put("name", Converters.localizedTextV2toString(mapOf("en-US" to repo.name))) + put( + "description", + Converters.localizedTextV2toString(mapOf("en-US" to repo.description)) + ) + put("address", repo.address) + put("timestamp", 42) + put("certificate", repo.certificate) + }) + db.insert( + RepositoryPreferences.TABLE, + SQLiteDatabase.CONFLICT_FAIL, ContentValues().apply { + put("repoId", repoId) + put("enabled", repo.enabled) + put("lastETag", "foo") + put("weight", repo.weight) + }) + db.insert( + Mirror.TABLE, + SQLiteDatabase.CONFLICT_FAIL, ContentValues().apply { + put("repoId", repoId) + put("url", "foo") + put("location", "bar") + }) + } + + // Re-open the database with version 8, auto-migrations are applied automatically + helper.runMigrationsAndValidate(TEST_DB, 8, true).close() + + // now get the Room DB, so we can use our DAOs for verifying the migration + Room.databaseBuilder( + ApplicationProvider.getApplicationContext(), + FDroidDatabaseInt::class.java, + TEST_DB + ) + .addMigrations(MIGRATION_2_3, MIGRATION_5_6) + .allowMainThreadQueries() + .build().use { db -> + // check repo got timestamp and etag reset + val repos = db.getRepositoryDao().getRepositories() + assertEquals(1, repos.size) + assertEquals(null, repos[0].lastETag) + assertEquals(-1, repos[0].timestamp) + // check mirror + assertEquals(1, repos[0].mirrors.size) + val mirror = repos[0].mirrors[0] + assertEquals(repos[0].repoId, mirror.repoId) + assertEquals("foo", mirror.url) + assertEquals("bar", mirror.countryCode) + } + } +} diff --git a/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDiffTest.kt b/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDiffTest.kt index 0c733255d..a9a1c0fbb 100644 --- a/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDiffTest.kt +++ b/libs/database/src/dbTest/java/org/fdroid/database/RepositoryDiffTest.kt @@ -7,6 +7,7 @@ import kotlinx.serialization.json.jsonObject import org.fdroid.database.TestUtils.assertRepoEquals import org.fdroid.index.v2.AntiFeatureV2 import org.fdroid.index.v2.CategoryV2 +import org.fdroid.index.v2.MirrorV2 import org.fdroid.index.v2.ReleaseChannelV2 import org.fdroid.index.v2.RepoV2 import org.fdroid.test.DiffUtils.applyDiff @@ -61,7 +62,8 @@ internal class RepositoryDiffTest : DbTest() { val updateTimestamp = Random.nextLong() val json = """ { - "timestamp": $updateTimestamp + "timestamp": $updateTimestamp, + "unknown": "field" }""".trimIndent() // decode diff from JSON and update DB with it @@ -97,6 +99,22 @@ internal class RepositoryDiffTest : DbTest() { } } + @Test + fun mirrorUnknownKeyDiff() { + val repo = getRandomRepo() + val json = """ + { + "mirrors": [ + { "url": "foo", "countryCode": "bar", "unknown": "doesntexist" } + ] + }""".trimIndent() + testDiff(repo, json) { repos -> + val expectedMirrors = setOf(Mirror(repos[0].repoId, "foo", "bar")) + assertEquals(expectedMirrors, repos[0].mirrors.toSet()) + assertRepoEquals(repo.copy(mirrors = listOf(MirrorV2("foo", "bar"))), repos[0]) + } + } + @Test fun descriptionDiff() { val repo = getRandomRepo().copy(description = mapOf("de" to "foo", "en" to "bar")) diff --git a/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt b/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt index 61719d52f..d842845ac 100644 --- a/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt +++ b/libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt @@ -16,7 +16,7 @@ import java.util.concurrent.Callable // When bumping this version, please make sure to add one (or more) migration(s) below! // Consider also providing tests for that migration. // Don't forget to commit the new schema to the git repo as well. - version = 7, + version = 8, entities = [ // repo CoreRepository::class, @@ -49,7 +49,8 @@ import java.util.concurrent.Callable AutoMigration(4, 5), // 5 to 6 is a manual migration AutoMigration(6, 7), - // add future migrations here (if they are easy enough to be done automatically) + AutoMigration(7, 8, CountryCodeMigration::class), + // add future migrations above! ], ) @TypeConverters(Converters::class) diff --git a/libs/database/src/main/java/org/fdroid/database/Migrations.kt b/libs/database/src/main/java/org/fdroid/database/Migrations.kt index 530d8f7c7..3afd5fabe 100644 --- a/libs/database/src/main/java/org/fdroid/database/Migrations.kt +++ b/libs/database/src/main/java/org/fdroid/database/Migrations.kt @@ -3,6 +3,7 @@ package org.fdroid.database import android.content.ContentValues import android.database.Cursor import android.database.sqlite.SQLiteDatabase.CONFLICT_FAIL +import androidx.room.RenameColumn import androidx.room.migration.AutoMigrationSpec import androidx.room.migration.Migration import androidx.sqlite.db.SupportSQLiteDatabase @@ -134,3 +135,27 @@ internal val MIGRATION_5_6 = object : Migration(5, 6) { db.execSQL("INSERT INTO AppMetadataFts(AppMetadataFts) VALUES('rebuild')") } } + +/** + * Somebody changed the initial IndexV2 definition of MirrorV2.location to MirrorV2.countryCode + * in fdroidserver and doesn't want to undo this rename. + * So now we need to handle this in the client to be in line with the index format produced. + */ +@RenameColumn( + tableName = Mirror.TABLE, + fromColumnName = "location", + toColumnName = "countryCode", +) +internal class CountryCodeMigration : AutoMigrationSpec { + override fun onPostMigrate(db: SupportSQLiteDatabase) { + // reset timestamps and etags so next repo updates pull full index, refresh all data + db.beginTransaction() + try { + db.execSQL("UPDATE ${CoreRepository.TABLE} SET timestamp = -1") + db.execSQL("UPDATE ${RepositoryPreferences.TABLE} SET lastETag = NULL") + db.setTransactionSuccessful() + } finally { + db.endTransaction() + } + } +} diff --git a/libs/database/src/main/java/org/fdroid/database/Repository.kt b/libs/database/src/main/java/org/fdroid/database/Repository.kt index 293bfb1e0..df13de916 100644 --- a/libs/database/src/main/java/org/fdroid/database/Repository.kt +++ b/libs/database/src/main/java/org/fdroid/database/Repository.kt @@ -232,7 +232,7 @@ public data class Repository internal constructor( internal data class Mirror( val repoId: Long, val url: String, - val location: String? = null, + val countryCode: String? = null, ) { internal companion object { const val TABLE = "Mirror" @@ -240,14 +240,14 @@ internal data class Mirror( fun toDownloadMirror(): org.fdroid.download.Mirror = org.fdroid.download.Mirror( baseUrl = url, - location = location, + countryCode = countryCode, ) } internal fun MirrorV2.toMirror(repoId: Long) = Mirror( repoId = repoId, url = url, - location = location, + countryCode = countryCode, ) /** diff --git a/libs/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt b/libs/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt index 587bba8b7..18a1efa1f 100644 --- a/libs/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt +++ b/libs/download/src/commonMain/kotlin/org/fdroid/download/Mirror.kt @@ -8,7 +8,7 @@ import mu.KotlinLogging public data class Mirror @JvmOverloads constructor( val baseUrl: String, - val location: String? = null, + val countryCode: String? = null, /** * If this is true, this as an IPFS HTTP gateway that only accepts CIDv1 and not regular paths. * So use this mirror only, if you have a CIDv1 available for supplying it to [getUrl]. diff --git a/libs/sharedTest/src/main/assets/diff-empty-max/1337.json b/libs/sharedTest/src/main/assets/diff-empty-max/1337.json index cba38a830..aedb20f1e 100644 --- a/libs/sharedTest/src/main/assets/diff-empty-max/1337.json +++ b/libs/sharedTest/src/main/assets/diff-empty-max/1337.json @@ -27,11 +27,11 @@ "mirrors": [ { "url": "https://max-v1.com", - "location": "us" + "countryCode": "us" }, { "url": "https://max-v1.org", - "location": "nl" + "countryCode": "nl" } ], "timestamp": 9223372036854775807, diff --git a/libs/sharedTest/src/main/assets/diff-empty-max/23.json b/libs/sharedTest/src/main/assets/diff-empty-max/23.json index 6f5fde040..32b7a3d8a 100644 --- a/libs/sharedTest/src/main/assets/diff-empty-max/23.json +++ b/libs/sharedTest/src/main/assets/diff-empty-max/23.json @@ -25,11 +25,11 @@ "mirrors": [ { "url": "https://max-v1.com", - "location": "us" + "countryCode": "us" }, { "url": "https://max-v1.org", - "location": "nl" + "countryCode": "nl" } ], "timestamp": 9223372036854775807, diff --git a/libs/sharedTest/src/main/assets/diff-empty-max/42.json b/libs/sharedTest/src/main/assets/diff-empty-max/42.json index 74e78f84a..72269cfcb 100644 --- a/libs/sharedTest/src/main/assets/diff-empty-max/42.json +++ b/libs/sharedTest/src/main/assets/diff-empty-max/42.json @@ -25,11 +25,11 @@ "mirrors": [ { "url": "https://max-v1.com", - "location": "us" + "countryCode": "us" }, { "url": "https://max-v1.org", - "location": "nl" + "countryCode": "nl" } ], "timestamp": 9223372036854775807, diff --git a/libs/sharedTest/src/main/assets/index-max-v2.json b/libs/sharedTest/src/main/assets/index-max-v2.json index 696390e52..3635c6731 100644 --- a/libs/sharedTest/src/main/assets/index-max-v2.json +++ b/libs/sharedTest/src/main/assets/index-max-v2.json @@ -26,11 +26,11 @@ "mirrors": [ { "url": "https://max-v1.com", - "location": "us" + "countryCode": "us" }, { "url": "https://max-v1.org", - "location": "nl" + "countryCode": "nl" } ], "timestamp": 9223372036854775807, diff --git a/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestRepoUtils.kt b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestRepoUtils.kt index e3e4b6bd3..8b49a9216 100644 --- a/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestRepoUtils.kt +++ b/libs/sharedTest/src/main/kotlin/org/fdroid/test/TestRepoUtils.kt @@ -16,7 +16,7 @@ object TestRepoUtils { fun getRandomMirror(): MirrorV2 = MirrorV2( url = getRandomString(), - location = getRandomString().orNull() + countryCode = getRandomString().orNull() ) fun getRandomLocalizedTextV2(size: Int = Random.nextInt(0, 23)): LocalizedTextV2 =