From bf051039550cd1140513ce7f38fe7341ae92b9a8 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 00:45:18 +0200 Subject: [PATCH 01/19] import: copy ZIP, use File instead of InputStream --- .../main/java/protect/card_locker/Utils.java | 17 +++++++++++++++++ .../importexport/CatimaImporter.java | 5 ++++- .../card_locker/importexport/FidmeImporter.java | 6 +++++- .../card_locker/importexport/Importer.java | 4 ++-- .../importexport/MultiFormatImporter.java | 8 +++++++- .../importexport/StocardImporter.java | 6 +++++- .../importexport/VoucherVaultImporter.java | 6 +++++- 7 files changed, 45 insertions(+), 7 deletions(-) diff --git a/app/src/main/java/protect/card_locker/Utils.java b/app/src/main/java/protect/card_locker/Utils.java index d98439e66..5bbaeeb08 100644 --- a/app/src/main/java/protect/card_locker/Utils.java +++ b/app/src/main/java/protect/card_locker/Utils.java @@ -481,6 +481,23 @@ public class Utils { return new File(context.getCacheDir() + "/" + name); } + public static File copyToTempFile(Context context, InputStream input, String name) { + File file = createTempFile(context, name); + try (FileOutputStream out = new FileOutputStream(file)) { + byte[] buf = new byte[4096]; + int len; + while ((len = input.read(buf)) != -1) { + out.write(buf, 0, len); + } + out.close(); + input.close(); + return file; + } catch (IOException e) { + Log.d("store temp file", "failed writing temp file, name: " + name); + return null; + } + } + public static String saveTempImage(Context context, Bitmap in, String name, Bitmap.CompressFormat format) { File image = createTempFile(context, name); try (FileOutputStream out = new FileOutputStream(image)) { diff --git a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java index 52d521781..8a7761179 100644 --- a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java @@ -13,6 +13,8 @@ import org.apache.commons.csv.CSVRecord; import java.io.BufferedInputStream; import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -39,7 +41,8 @@ import protect.card_locker.ZipUtils; * A header is expected for the each table showing the names of the columns. */ public class CatimaImporter implements Importer { - public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password) throws IOException, FormatException, InterruptedException { + public void importData(Context context, SQLiteDatabase database, File inputFile, char[] password) throws IOException, FormatException, InterruptedException { + InputStream input = new FileInputStream(inputFile); InputStream bufferedInputStream = new BufferedInputStream(input); bufferedInputStream.mark(100); diff --git a/app/src/main/java/protect/card_locker/importexport/FidmeImporter.java b/app/src/main/java/protect/card_locker/importexport/FidmeImporter.java index b1e411a0f..b7b64560b 100644 --- a/app/src/main/java/protect/card_locker/importexport/FidmeImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/FidmeImporter.java @@ -11,6 +11,8 @@ import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; import org.json.JSONException; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.StringReader; @@ -31,7 +33,8 @@ import protect.card_locker.Utils; * A header is expected for the each table showing the names of the columns. */ public class FidmeImporter implements Importer { - public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password) throws IOException, FormatException, JSONException, ParseException { + public void importData(Context context, SQLiteDatabase database, File inputFile, char[] password) throws IOException, FormatException, JSONException, ParseException { + InputStream input = new FileInputStream(inputFile); // We actually retrieve a .zip file ZipInputStream zipInputStream = new ZipInputStream(input, password); @@ -70,6 +73,7 @@ public class FidmeImporter implements Importer { } zipInputStream.close(); + input.close(); } /** diff --git a/app/src/main/java/protect/card_locker/importexport/Importer.java b/app/src/main/java/protect/card_locker/importexport/Importer.java index 41f73df26..72c32d377 100644 --- a/app/src/main/java/protect/card_locker/importexport/Importer.java +++ b/app/src/main/java/protect/card_locker/importexport/Importer.java @@ -5,8 +5,8 @@ import android.database.sqlite.SQLiteDatabase; import org.json.JSONException; +import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.text.ParseException; import protect.card_locker.FormatException; @@ -23,5 +23,5 @@ public interface Importer { * @throws IOException * @throws FormatException */ - void importData(Context context, SQLiteDatabase database, InputStream input, char[] password) throws IOException, FormatException, InterruptedException, JSONException, ParseException; + void importData(Context context, SQLiteDatabase database, File inputFile, char[] password) throws IOException, FormatException, InterruptedException, JSONException, ParseException; } diff --git a/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java b/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java index a5306dd33..78ee564b0 100644 --- a/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java @@ -6,10 +6,14 @@ import android.util.Log; import net.lingala.zip4j.exception.ZipException; +import java.io.File; import java.io.InputStream; +import protect.card_locker.Utils; + public class MultiFormatImporter { private static final String TAG = "Catima"; + private static final String TEMP_ZIP_NAME = MultiFormatImporter.class.getSimpleName() + ".zip"; /** * Attempts to import data from the input stream of the @@ -42,9 +46,10 @@ public class MultiFormatImporter { String error = null; if (importer != null) { + File inputFile = Utils.copyToTempFile(context, input, TEMP_ZIP_NAME); database.beginTransaction(); try { - importer.importData(context, database, input, password); + importer.importData(context, database, inputFile, password); database.setTransactionSuccessful(); return new ImportExportResult(ImportExportResultType.Success); } catch (ZipException e) { @@ -59,6 +64,7 @@ public class MultiFormatImporter { error = e.toString(); } finally { database.endTransaction(); + inputFile.delete(); } } else { error = "Unsupported data format imported: " + format.name(); diff --git a/app/src/main/java/protect/card_locker/importexport/StocardImporter.java b/app/src/main/java/protect/card_locker/importexport/StocardImporter.java index 1aa90e3e0..df0d79451 100644 --- a/app/src/main/java/protect/card_locker/importexport/StocardImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/StocardImporter.java @@ -16,6 +16,8 @@ import org.apache.commons.csv.CSVRecord; import org.json.JSONException; import org.json.JSONObject; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -42,7 +44,7 @@ import protect.card_locker.ZipUtils; public class StocardImporter implements Importer { private static final String TAG = "Catima"; - public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password) throws IOException, FormatException, JSONException, ParseException { + public void importData(Context context, SQLiteDatabase database, File inputFile, char[] password) throws IOException, FormatException, JSONException, ParseException { HashMap> loyaltyCardHashMap = new HashMap<>(); HashMap> providers = new HashMap<>(); @@ -62,6 +64,7 @@ public class StocardImporter implements Importer { throw new FormatException("Issue parsing CSV data", e); } + InputStream input = new FileInputStream(inputFile); ZipInputStream zipInputStream = new ZipInputStream(input, password); String[] providersFileName = null; @@ -245,6 +248,7 @@ public class StocardImporter implements Importer { } zipInputStream.close(); + input.close(); } private boolean startsWith(String[] full, String[] start, int minExtraLength) { diff --git a/app/src/main/java/protect/card_locker/importexport/VoucherVaultImporter.java b/app/src/main/java/protect/card_locker/importexport/VoucherVaultImporter.java index b0d103366..08454626b 100644 --- a/app/src/main/java/protect/card_locker/importexport/VoucherVaultImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/VoucherVaultImporter.java @@ -12,6 +12,8 @@ import org.json.JSONException; import org.json.JSONObject; import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -36,7 +38,8 @@ import protect.card_locker.Utils; * A header is expected for the each table showing the names of the columns. */ public class VoucherVaultImporter implements Importer { - public void importData(Context context, SQLiteDatabase database, InputStream input, char[] password) throws IOException, FormatException, JSONException, ParseException { + public void importData(Context context, SQLiteDatabase database, File inputFile, char[] password) throws IOException, FormatException, JSONException, ParseException { + InputStream input = new FileInputStream(inputFile); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8)); StringBuilder sb = new StringBuilder(); @@ -130,5 +133,6 @@ public class VoucherVaultImporter implements Importer { } bufferedReader.close(); + input.close(); } } \ No newline at end of file From 5f99f2b17eb5b1434e6eafe8af15f8facfa94586 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 00:57:00 +0200 Subject: [PATCH 02/19] Utils: add imageFiles() --- .../main/java/protect/card_locker/DBHelper.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/app/src/main/java/protect/card_locker/DBHelper.java b/app/src/main/java/protect/card_locker/DBHelper.java index 53ec2ea5c..8bdd5fbc3 100644 --- a/app/src/main/java/protect/card_locker/DBHelper.java +++ b/app/src/main/java/protect/card_locker/DBHelper.java @@ -16,7 +16,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Currency; import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Set; public class DBHelper extends SQLiteOpenHelper { public static final String DATABASE_NAME = "Catima.db"; @@ -323,6 +325,21 @@ public class DBHelper extends SQLiteOpenHelper { } } + public static Set imageFiles(Context context, final SQLiteDatabase database) { + Set files = new HashSet<>(); + Cursor cardCursor = getLoyaltyCardCursor(database); + while (cardCursor.moveToNext()) { + LoyaltyCard card = LoyaltyCard.toLoyaltyCard(cardCursor); + for (ImageLocationType imageLocationType : ImageLocationType.values()) { + String name = Utils.getCardImageFileName(card.id, imageLocationType); + if (Utils.retrieveCardImageAsFile(context, name).exists()) { + files.add(name); + } + } + } + return files; + } + private static ContentValues generateFTSContentValues(final int id, final String store, final String note) { // FTS on Android is severely limited and can only search for word starting with a certain string // So for each word, we grab every single substring From 3a5973a04dab4ac5532cdbd595d2645503db9968 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 01:22:07 +0200 Subject: [PATCH 03/19] Utils: add checksum() --- .../main/java/protect/card_locker/Utils.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/app/src/main/java/protect/card_locker/Utils.java b/app/src/main/java/protect/card_locker/Utils.java index 5bbaeeb08..bae7e5df9 100644 --- a/app/src/main/java/protect/card_locker/Utils.java +++ b/app/src/main/java/protect/card_locker/Utils.java @@ -50,6 +50,8 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.text.NumberFormat; import java.text.ParseException; import java.util.Calendar; @@ -647,4 +649,22 @@ public class Utils { public static int getHeaderColor(Context context, LoyaltyCard loyaltyCard) { return loyaltyCard.headerColor != null ? loyaltyCard.headerColor : LetterBitmap.getDefaultColor(context, loyaltyCard.store); } + + public static String checksum(InputStream input) throws IOException { + try { + MessageDigest md = MessageDigest.getInstance("SHA-1"); + byte[] buf = new byte[4096]; + int len; + while ((len = input.read(buf)) != -1) { + md.update(buf, 0, len); + } + StringBuilder sb = new StringBuilder(); + for (byte b : md.digest()) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (NoSuchAlgorithmException _e) { + return null; + } + } } From da9e3bb6b2aba44ec95c8e3f7061eb0d3c77260e Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 02:22:23 +0200 Subject: [PATCH 04/19] CatimaImporter: read ZIP twice, get checksums --- .../importexport/CatimaImporter.java | 43 +++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java index 8a7761179..99029236b 100644 --- a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java @@ -24,7 +24,9 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Currency; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import protect.card_locker.CatimaBarcode; import protect.card_locker.DBHelper; @@ -42,24 +44,25 @@ import protect.card_locker.ZipUtils; */ public class CatimaImporter implements Importer { public void importData(Context context, SQLiteDatabase database, File inputFile, char[] password) throws IOException, FormatException, InterruptedException { - InputStream input = new FileInputStream(inputFile); - InputStream bufferedInputStream = new BufferedInputStream(input); - bufferedInputStream.mark(100); + // Pass #1: get hashes and parse CSV + InputStream input1 = new FileInputStream(inputFile); + InputStream bufferedInputStream1 = new BufferedInputStream(input1); + bufferedInputStream1.mark(100); + ZipInputStream zipInputStream1 = new ZipInputStream(bufferedInputStream1, password); // First, check if this is a zip file - ZipInputStream zipInputStream = new ZipInputStream(bufferedInputStream, password); - boolean isZipFile = false; - LocalFileHeader localFileHeader; - while ((localFileHeader = zipInputStream.getNextEntry()) != null) { + Map imageChecksums = new HashMap<>(); + + while ((localFileHeader = zipInputStream1.getNextEntry()) != null) { isZipFile = true; String fileName = Uri.parse(localFileHeader.getFileName()).getLastPathSegment(); if (fileName.equals("catima.csv")) { - importCSV(context, database, zipInputStream); + importCSV(context, database, zipInputStream1); } else if (fileName.endsWith(".png")) { - Utils.saveCardImage(context, ZipUtils.readImage(zipInputStream), fileName); + imageChecksums.put(fileName, Utils.checksum(zipInputStream1)); } else { throw new FormatException("Unexpected file in import: " + fileName); } @@ -67,11 +70,27 @@ public class CatimaImporter implements Importer { if (!isZipFile) { // This is not a zip file, try importing as bare CSV - bufferedInputStream.reset(); - importCSV(context, database, bufferedInputStream); + bufferedInputStream1.reset(); + importCSV(context, database, bufferedInputStream1); + input1.close(); + return; } - input.close(); + input1.close(); + + // Pass #2: save images + InputStream input2 = new FileInputStream(inputFile); + InputStream bufferedInputStream2 = new BufferedInputStream(input2); + ZipInputStream zipInputStream2 = new ZipInputStream(bufferedInputStream2, password); + + while ((localFileHeader = zipInputStream2.getNextEntry()) != null) { + String fileName = Uri.parse(localFileHeader.getFileName()).getLastPathSegment(); + if (fileName.endsWith(".png")) { + Utils.saveCardImage(context, ZipUtils.readImage(zipInputStream2), fileName); + } + } + + input2.close(); } public void importCSV(Context context, SQLiteDatabase database, InputStream input) throws IOException, FormatException, InterruptedException { From b8fa4d706059dfadabed311917a3fee063da9a99 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 03:12:29 +0200 Subject: [PATCH 05/19] DBHelper: add DEFAULT_ZOOM_LEVEL --- app/src/main/java/protect/card_locker/DBHelper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/protect/card_locker/DBHelper.java b/app/src/main/java/protect/card_locker/DBHelper.java index 8bdd5fbc3..356ffdbae 100644 --- a/app/src/main/java/protect/card_locker/DBHelper.java +++ b/app/src/main/java/protect/card_locker/DBHelper.java @@ -24,6 +24,7 @@ public class DBHelper extends SQLiteOpenHelper { public static final String DATABASE_NAME = "Catima.db"; public static final int ORIGINAL_DATABASE_VERSION = 1; public static final int DATABASE_VERSION = 16; + public static final int DEFAULT_ZOOM_LEVEL = 100; public static class LoyaltyCardDbGroups { public static final String TABLE = "groups"; @@ -108,7 +109,7 @@ public class DBHelper extends SQLiteOpenHelper { LoyaltyCardDbIds.BARCODE_TYPE + " TEXT," + LoyaltyCardDbIds.STAR_STATUS + " INTEGER DEFAULT '0'," + LoyaltyCardDbIds.LAST_USED + " INTEGER DEFAULT '0', " + - LoyaltyCardDbIds.ZOOM_LEVEL + " INTEGER DEFAULT '100', " + + LoyaltyCardDbIds.ZOOM_LEVEL + " INTEGER DEFAULT '" + DEFAULT_ZOOM_LEVEL + "', " + LoyaltyCardDbIds.ARCHIVE_STATUS + " INTEGER DEFAULT '0' )"); // create associative table for cards in groups @@ -312,7 +313,7 @@ public class DBHelper extends SQLiteOpenHelper { if (oldVersion < 14 && newVersion >= 14) { db.execSQL("ALTER TABLE " + LoyaltyCardDbIds.TABLE - + " ADD COLUMN " + LoyaltyCardDbIds.ZOOM_LEVEL + " INTEGER DEFAULT '100' "); + + " ADD COLUMN " + LoyaltyCardDbIds.ZOOM_LEVEL + " INTEGER DEFAULT '" + DEFAULT_ZOOM_LEVEL + "' "); } if (oldVersion < 15 && newVersion >= 15) { db.execSQL("ALTER TABLE " + LoyaltyCardDbIds.TABLE From 84d7e15b5ca4c07b9217eabef2bb67e1eff2176b Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 03:29:31 +0200 Subject: [PATCH 06/19] LoyaltyCard: add isDuplicate(); Utils: add equals() --- .../java/protect/card_locker/LoyaltyCard.java | 23 ++++++++++++------- .../main/java/protect/card_locker/Utils.java | 9 ++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/app/src/main/java/protect/card_locker/LoyaltyCard.java b/app/src/main/java/protect/card_locker/LoyaltyCard.java index d8cbdea9b..cd4f0e26a 100644 --- a/app/src/main/java/protect/card_locker/LoyaltyCard.java +++ b/app/src/main/java/protect/card_locker/LoyaltyCard.java @@ -14,30 +14,28 @@ public class LoyaltyCard implements Parcelable { public final int id; public final String store; public final String note; + @Nullable public final Date validFrom; + @Nullable public final Date expiry; public final BigDecimal balance; + @Nullable public final Currency balanceType; public final String cardId; - @Nullable public final String barcodeId; - @Nullable public final CatimaBarcode barcodeType; - @Nullable public final Integer headerColor; - public final int starStatus; public final int archiveStatus; public final long lastUsed; public int zoomLevel; - public LoyaltyCard(final int id, final String store, final String note, final Date validFrom, - final Date expiry, final BigDecimal balance, final Currency balanceType, - final String cardId, @Nullable final String barcodeId, - @Nullable final CatimaBarcode barcodeType, + public LoyaltyCard(final int id, final String store, final String note, @Nullable final Date validFrom, + @Nullable final Date expiry, final BigDecimal balance, @Nullable final Currency balanceType, + final String cardId, @Nullable final String barcodeId, @Nullable final CatimaBarcode barcodeType, @Nullable final Integer headerColor, final int starStatus, final long lastUsed, final int zoomLevel, final int archiveStatus) { this.id = id; @@ -145,6 +143,15 @@ public class LoyaltyCard implements Parcelable { return new LoyaltyCard(id, store, note, validFrom, expiry, balance, balanceType, cardId, barcodeId, barcodeType, headerColor, starred, lastUsed, zoomLevel, archived); } + public static boolean isDuplicate(final LoyaltyCard a, final LoyaltyCard b) { + // Skip lastUsed + return a.id == b.id && a.store.equals(b.store) && a.note.equals(b.note) && Utils.equals(a.validFrom, b.validFrom) + && Utils.equals(a.expiry, b.expiry) && a.balance.equals(b.balance) && Utils.equals(a.balanceType, b.balanceType) + && a.cardId.equals(b.cardId) && Utils.equals(a.barcodeId, b.barcodeId) && Utils.equals(a.barcodeType, b.barcodeType) + && Utils.equals(a.headerColor, b.headerColor) && a.starStatus == b.starStatus && a.zoomLevel == b.zoomLevel + && a.archiveStatus == b.archiveStatus; + } + @Override public int describeContents() { return id; diff --git a/app/src/main/java/protect/card_locker/Utils.java b/app/src/main/java/protect/card_locker/Utils.java index bae7e5df9..010323835 100644 --- a/app/src/main/java/protect/card_locker/Utils.java +++ b/app/src/main/java/protect/card_locker/Utils.java @@ -667,4 +667,13 @@ public class Utils { return null; } } + + public static boolean equals(final Object a, final Object b) { + if (a == null && b == null) { + return true; + } else if (a == null || b == null) { + return false; + } + return a.equals(b); + } } From 901c2d815451d6b08c96350efa6fccf570ea203e Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 04:43:27 +0200 Subject: [PATCH 07/19] LoyaltyCard: fix isDuplicate() --- app/src/main/java/protect/card_locker/LoyaltyCard.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/src/main/java/protect/card_locker/LoyaltyCard.java b/app/src/main/java/protect/card_locker/LoyaltyCard.java index cd4f0e26a..6b7075069 100644 --- a/app/src/main/java/protect/card_locker/LoyaltyCard.java +++ b/app/src/main/java/protect/card_locker/LoyaltyCard.java @@ -144,12 +144,12 @@ public class LoyaltyCard implements Parcelable { } public static boolean isDuplicate(final LoyaltyCard a, final LoyaltyCard b) { - // Skip lastUsed + // Skip lastUsed & zoomLevel return a.id == b.id && a.store.equals(b.store) && a.note.equals(b.note) && Utils.equals(a.validFrom, b.validFrom) && Utils.equals(a.expiry, b.expiry) && a.balance.equals(b.balance) && Utils.equals(a.balanceType, b.balanceType) - && a.cardId.equals(b.cardId) && Utils.equals(a.barcodeId, b.barcodeId) && Utils.equals(a.barcodeType, b.barcodeType) - && Utils.equals(a.headerColor, b.headerColor) && a.starStatus == b.starStatus && a.zoomLevel == b.zoomLevel - && a.archiveStatus == b.archiveStatus; + && a.cardId.equals(b.cardId) && Utils.equals(a.barcodeId, b.barcodeId) + && Utils.equals(a.barcodeType == null ? null : a.barcodeType.format(), b.barcodeType == null ? null : b.barcodeType.format()) + && Utils.equals(a.headerColor, b.headerColor) && a.starStatus == b.starStatus && a.archiveStatus == b.archiveStatus; } @Override From d5d53b241ac8d441dc7806a317789f61891d52f8 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 04:44:24 +0200 Subject: [PATCH 08/19] Utils: add getRenamedCardImageFileName() --- .../main/java/protect/card_locker/Utils.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/app/src/main/java/protect/card_locker/Utils.java b/app/src/main/java/protect/card_locker/Utils.java index 010323835..92daa8326 100644 --- a/app/src/main/java/protect/card_locker/Utils.java +++ b/app/src/main/java/protect/card_locker/Utils.java @@ -60,6 +60,8 @@ import java.util.Date; import java.util.GregorianCalendar; import java.util.Locale; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import protect.card_locker.preferences.Settings; @@ -382,6 +384,24 @@ public class Utils { return cardImageFileNameBuilder.toString(); } + static public String getRenamedCardImageFileName(final String fileName, final Map idMap) { + Pattern pattern = Pattern.compile("^(card_)(\\d+)(_(?:front|back|icon)\\.png)$"); + Matcher matcher = pattern.matcher(fileName); + if (matcher.matches()) { + StringBuilder cardImageFileNameBuilder = new StringBuilder(); + cardImageFileNameBuilder.append(matcher.group(1)); + try { + int id = Integer.parseInt(matcher.group(2)); + cardImageFileNameBuilder.append(idMap.getOrDefault(id, id)); + } catch (NumberFormatException _e) { + return null; + } + cardImageFileNameBuilder.append(matcher.group(3)); + return cardImageFileNameBuilder.toString(); + } + return null; + } + static public void saveCardImage(Context context, Bitmap bitmap, String fileName) throws FileNotFoundException { if (bitmap == null) { context.deleteFile(fileName); From 48510494eb61287d989acb182d38da710d2a94a1 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 05:05:44 +0200 Subject: [PATCH 09/19] Utils: split off CARD_IMAGE_FILENAME_REGEX --- app/src/main/java/protect/card_locker/Utils.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/src/main/java/protect/card_locker/Utils.java b/app/src/main/java/protect/card_locker/Utils.java index 92daa8326..946cf278e 100644 --- a/app/src/main/java/protect/card_locker/Utils.java +++ b/app/src/main/java/protect/card_locker/Utils.java @@ -80,6 +80,8 @@ public class Utils { public static final int CARD_IMAGE_FROM_FILE_BACK = 9; public static final int CARD_IMAGE_FROM_FILE_ICON = 10; + public static final String CARD_IMAGE_FILENAME_REGEX = "^(card_)(\\d+)(_(?:front|back|icon)\\.png)$"; + static final double LUMINANCE_MIDPOINT = 0.5; static final int BITMAP_SIZE_SMALL = 512; @@ -385,7 +387,7 @@ public class Utils { } static public String getRenamedCardImageFileName(final String fileName, final Map idMap) { - Pattern pattern = Pattern.compile("^(card_)(\\d+)(_(?:front|back|icon)\\.png)$"); + Pattern pattern = Pattern.compile(CARD_IMAGE_FILENAME_REGEX); Matcher matcher = pattern.matcher(fileName); if (matcher.matches()) { StringBuilder cardImageFileNameBuilder = new StringBuilder(); From 1425d4af586211f232f8b48cc7d43941cc3d0376 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 05:07:48 +0200 Subject: [PATCH 10/19] CatimaImporter: add saveAndDeduplicate() & refactor --- .../importexport/CatimaImporter.java | 167 +++++++++++++----- 1 file changed, 127 insertions(+), 40 deletions(-) diff --git a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java index 99029236b..1ab38c714 100644 --- a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java @@ -27,11 +27,14 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import protect.card_locker.CatimaBarcode; import protect.card_locker.DBHelper; import protect.card_locker.FormatException; import protect.card_locker.Group; +import protect.card_locker.ImageLocationType; +import protect.card_locker.LoyaltyCard; import protect.card_locker.Utils; import protect.card_locker.ZipUtils; @@ -43,6 +46,18 @@ import protect.card_locker.ZipUtils; * A header is expected for the each table showing the names of the columns. */ public class CatimaImporter implements Importer { + public static class ImportedData { + public final List cards; + public final List groups; + public final List> cardGroups; + + ImportedData(final List cards, final List groups, final List> cardGroups) { + this.cards = cards; + this.groups = groups; + this.cardGroups = cardGroups; + } + } + public void importData(Context context, SQLiteDatabase database, File inputFile, char[] password) throws IOException, FormatException, InterruptedException { // Pass #1: get hashes and parse CSV InputStream input1 = new FileInputStream(inputFile); @@ -54,14 +69,18 @@ public class CatimaImporter implements Importer { boolean isZipFile = false; LocalFileHeader localFileHeader; Map imageChecksums = new HashMap<>(); + ImportedData importedData = null; while ((localFileHeader = zipInputStream1.getNextEntry()) != null) { isZipFile = true; String fileName = Uri.parse(localFileHeader.getFileName()).getLastPathSegment(); if (fileName.equals("catima.csv")) { - importCSV(context, database, zipInputStream1); + importedData = importCSV(zipInputStream1); } else if (fileName.endsWith(".png")) { + if (!fileName.matches(Utils.CARD_IMAGE_FILENAME_REGEX)) { + throw new FormatException("Unexpected PNG file in import: " + fileName); + } imageChecksums.put(fileName, Utils.checksum(zipInputStream1)); } else { throw new FormatException("Unexpected file in import: " + fileName); @@ -71,50 +90,104 @@ public class CatimaImporter implements Importer { if (!isZipFile) { // This is not a zip file, try importing as bare CSV bufferedInputStream1.reset(); - importCSV(context, database, bufferedInputStream1); - input1.close(); - return; + importedData = importCSV(bufferedInputStream1); } - input1.close(); + if (importedData == null) { + throw new FormatException("No imported data"); + } - // Pass #2: save images - InputStream input2 = new FileInputStream(inputFile); - InputStream bufferedInputStream2 = new BufferedInputStream(input2); - ZipInputStream zipInputStream2 = new ZipInputStream(bufferedInputStream2, password); + Map idMap = saveAndDeduplicate(context, database, importedData, imageChecksums); - while ((localFileHeader = zipInputStream2.getNextEntry()) != null) { - String fileName = Uri.parse(localFileHeader.getFileName()).getLastPathSegment(); - if (fileName.endsWith(".png")) { - Utils.saveCardImage(context, ZipUtils.readImage(zipInputStream2), fileName); + if (isZipFile) { + // Pass #2: save images + InputStream input2 = new FileInputStream(inputFile); + InputStream bufferedInputStream2 = new BufferedInputStream(input2); + ZipInputStream zipInputStream2 = new ZipInputStream(bufferedInputStream2, password); + + while ((localFileHeader = zipInputStream2.getNextEntry()) != null) { + String fileName = Uri.parse(localFileHeader.getFileName()).getLastPathSegment(); + if (fileName.endsWith(".png")) { + String newFileName = Utils.getRenamedCardImageFileName(fileName, idMap); + Utils.saveCardImage(context, ZipUtils.readImage(zipInputStream2), newFileName); + } + } + } + } + + public Map saveAndDeduplicate(Context context, SQLiteDatabase database, final ImportedData data, final Map imageChecksums) throws IOException { + Map idMap = new HashMap<>(); + Set existingImages = DBHelper.imageFiles(context, database); + + for (LoyaltyCard card : data.cards) { + LoyaltyCard existing = DBHelper.getLoyaltyCard(database, card.id); + if (existing == null) { + DBHelper.insertLoyaltyCard(database, card.id, card.store, card.note, card.validFrom, card.expiry, card.balance, card.balanceType, + card.cardId, card.barcodeId, card.barcodeType, card.headerColor, card.starStatus, card.lastUsed, card.archiveStatus); + } else if (!isDuplicate(context, existing, card, existingImages, imageChecksums)) { + long newId = DBHelper.insertLoyaltyCard(database, card.store, card.note, card.validFrom, card.expiry, card.balance, card.balanceType, + card.cardId, card.barcodeId, card.barcodeType, card.headerColor, card.starStatus, card.lastUsed, card.archiveStatus); + idMap.put(card.id, (int) newId); } } - input2.close(); + for (String group : data.groups) { + DBHelper.insertGroup(database, group); + } + + for (Map.Entry entry : data.cardGroups) { + int cardId = idMap.getOrDefault(entry.getKey(), entry.getKey()); + String groupId = entry.getValue(); + List cardGroups = DBHelper.getLoyaltyCardGroups(database, cardId); + cardGroups.add(DBHelper.getGroup(database, groupId)); + DBHelper.setLoyaltyCardGroups(database, cardId, cardGroups); + } + + return idMap; } - public void importCSV(Context context, SQLiteDatabase database, InputStream input) throws IOException, FormatException, InterruptedException { + public boolean isDuplicate(Context context, final LoyaltyCard existing, final LoyaltyCard card, final Set existingImages, final Map imageChecksums) throws IOException { + if (!LoyaltyCard.isDuplicate(existing, card)) { + return false; + } + for (ImageLocationType imageLocationType : ImageLocationType.values()) { + String name = Utils.getCardImageFileName(existing.id, imageLocationType); + boolean exists = existingImages.contains(name); + if (exists != imageChecksums.containsKey(name)) { + return false; + } + if (exists) { + File file = Utils.retrieveCardImageAsFile(context, name); + if (!imageChecksums.get(name).equals(Utils.checksum(new FileInputStream(file)))) { + return false; + } + } + } + return true; + } + + public ImportedData importCSV(InputStream input) throws IOException, FormatException, InterruptedException { BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8)); int version = parseVersion(bufferedReader); switch (version) { case 1: - parseV1(database, bufferedReader); - break; + return parseV1(bufferedReader); case 2: - parseV2(context, database, bufferedReader); - break; + return parseV2(bufferedReader); default: throw new FormatException(String.format("No code to parse version %s", version)); } } - public void parseV1(SQLiteDatabase database, BufferedReader input) throws IOException, FormatException, InterruptedException { + public ImportedData parseV1(BufferedReader input) throws IOException, FormatException, InterruptedException { + ImportedData data = new ImportedData(new ArrayList<>(), new ArrayList<>(), new ArrayList<>()); final CSVParser parser = new CSVParser(input, CSVFormat.RFC4180.builder().setHeader().build()); try { for (CSVRecord record : parser) { - importLoyaltyCard(database, record); + LoyaltyCard card = importLoyaltyCard(record); + data.cards.add(card); if (Thread.currentThread().isInterrupted()) { throw new InterruptedException(); @@ -125,9 +198,15 @@ public class CatimaImporter implements Importer { } catch (IllegalArgumentException | IllegalStateException e) { throw new FormatException("Issue parsing CSV data", e); } + + return data; } - public void parseV2(Context context, SQLiteDatabase database, BufferedReader input) throws IOException, FormatException, InterruptedException { + public ImportedData parseV2(BufferedReader input) throws IOException, FormatException, InterruptedException { + List cards = new ArrayList<>(); + List groups = new ArrayList<>(); + List> cardGroups = new ArrayList<>(); + int part = 0; StringBuilder stringPart = new StringBuilder(); @@ -145,7 +224,7 @@ public class CatimaImporter implements Importer { break; case 1: try { - parseV2Groups(database, stringPart.toString()); + groups = parseV2Groups(stringPart.toString()); sectionParsed = true; } catch (FormatException e) { // We may have a multiline field, try again @@ -153,7 +232,7 @@ public class CatimaImporter implements Importer { break; case 2: try { - parseV2Cards(context, database, stringPart.toString()); + cards = parseV2Cards(stringPart.toString()); sectionParsed = true; } catch (FormatException e) { // We may have a multiline field, try again @@ -161,7 +240,7 @@ public class CatimaImporter implements Importer { break; case 3: try { - parseV2CardGroups(database, stringPart.toString()); + cardGroups = parseV2CardGroups(stringPart.toString()); sectionParsed = true; } catch (FormatException e) { // We may have a multiline field, try again @@ -188,9 +267,11 @@ public class CatimaImporter implements Importer { } catch (FormatException e) { throw new FormatException("Issue parsing CSV data", e); } + + return new ImportedData(cards, groups, cardGroups); } - public void parseV2Groups(SQLiteDatabase database, String data) throws IOException, FormatException, InterruptedException { + public List parseV2Groups(String data) throws IOException, FormatException, InterruptedException { // Parse groups final CSVParser groupParser = new CSVParser(new StringReader(data), CSVFormat.RFC4180.builder().setHeader().build()); @@ -210,12 +291,15 @@ public class CatimaImporter implements Importer { groupParser.close(); } + List groups = new ArrayList<>(); for (CSVRecord record : records) { - importGroup(database, record); + String group = importGroup(record); + groups.add(group); } + return groups; } - public void parseV2Cards(Context context, SQLiteDatabase database, String data) throws IOException, FormatException, InterruptedException { + public List parseV2Cards(String data) throws IOException, FormatException, InterruptedException { // Parse cards final CSVParser cardParser = new CSVParser(new StringReader(data), CSVFormat.RFC4180.builder().setHeader().build()); @@ -235,12 +319,15 @@ public class CatimaImporter implements Importer { cardParser.close(); } + List cards = new ArrayList<>(); for (CSVRecord record : records) { - importLoyaltyCard(database, record); + LoyaltyCard card = importLoyaltyCard(record); + cards.add(card); } + return cards; } - public void parseV2CardGroups(SQLiteDatabase database, String data) throws IOException, FormatException, InterruptedException { + public List> parseV2CardGroups(String data) throws IOException, FormatException, InterruptedException { // Parse card group mappings final CSVParser cardGroupParser = new CSVParser(new StringReader(data), CSVFormat.RFC4180.builder().setHeader().build()); @@ -260,9 +347,12 @@ public class CatimaImporter implements Importer { cardGroupParser.close(); } + List> cardGroups = new ArrayList<>(); for (CSVRecord record : records) { - importCardGroupMapping(database, record); + Map.Entry entry = importCardGroupMapping(record); + cardGroups.add(entry); } + return cardGroups; } /** @@ -298,8 +388,7 @@ public class CatimaImporter implements Importer { * Import a single loyalty card into the database using the given * session. */ - private void importLoyaltyCard(SQLiteDatabase database, CSVRecord record) - throws FormatException { + private LoyaltyCard importLoyaltyCard(CSVRecord record) throws FormatException { int id = CSVHelpers.extractInt(DBHelper.LoyaltyCardDbIds.ID, record); String store = CSVHelpers.extractString(DBHelper.LoyaltyCardDbIds.STORE, record, ""); @@ -396,28 +485,28 @@ public class CatimaImporter implements Importer { // We catch this exception so we can still import old backups } - DBHelper.insertLoyaltyCard(database, id, store, note, validFrom, expiry, balance, balanceType, cardId, barcodeId, barcodeType, headerColor, starStatus, lastUsed, archiveStatus); + return new LoyaltyCard(id, store, note, validFrom, expiry, balance, balanceType, cardId, barcodeId, barcodeType, headerColor, starStatus, lastUsed, DBHelper.DEFAULT_ZOOM_LEVEL, archiveStatus); } /** * Import a single group into the database using the given * session. */ - private void importGroup(SQLiteDatabase database, CSVRecord record) throws FormatException { + private String importGroup(CSVRecord record) throws FormatException { String id = CSVHelpers.extractString(DBHelper.LoyaltyCardDbGroups.ID, record, null); if (id == null) { throw new FormatException("Group has no ID: " + record); } - DBHelper.insertGroup(database, id); + return id; } /** * Import a single card to group mapping into the database using the given * session. */ - private void importCardGroupMapping(SQLiteDatabase database, CSVRecord record) throws FormatException { + private Map.Entry importCardGroupMapping(CSVRecord record) throws FormatException { int cardId = CSVHelpers.extractInt(DBHelper.LoyaltyCardDbIdsGroups.cardID, record); String groupId = CSVHelpers.extractString(DBHelper.LoyaltyCardDbIdsGroups.groupID, record, null); @@ -425,8 +514,6 @@ public class CatimaImporter implements Importer { throw new FormatException("Group has no ID: " + record); } - List cardGroups = DBHelper.getLoyaltyCardGroups(database, cardId); - cardGroups.add(DBHelper.getGroup(database, groupId)); - DBHelper.setLoyaltyCardGroups(database, cardId, cardGroups); + return Map.entry(cardId, groupId); } } From ba896fc1db71c6e0f86220735ed6ed88f53fe157 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 18:02:52 +0200 Subject: [PATCH 11/19] DBHelper: don't use DEFAULT_ZOOM_LEVEL in migration --- app/src/main/java/protect/card_locker/DBHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/protect/card_locker/DBHelper.java b/app/src/main/java/protect/card_locker/DBHelper.java index 356ffdbae..fa670894e 100644 --- a/app/src/main/java/protect/card_locker/DBHelper.java +++ b/app/src/main/java/protect/card_locker/DBHelper.java @@ -313,7 +313,7 @@ public class DBHelper extends SQLiteOpenHelper { if (oldVersion < 14 && newVersion >= 14) { db.execSQL("ALTER TABLE " + LoyaltyCardDbIds.TABLE - + " ADD COLUMN " + LoyaltyCardDbIds.ZOOM_LEVEL + " INTEGER DEFAULT '" + DEFAULT_ZOOM_LEVEL + "' "); + + " ADD COLUMN " + LoyaltyCardDbIds.ZOOM_LEVEL + " INTEGER DEFAULT '100' "); } if (oldVersion < 15 && newVersion >= 15) { db.execSQL("ALTER TABLE " + LoyaltyCardDbIds.TABLE From 9ee96b88e8f61570bb301efca09c2455877df649 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 18:04:26 +0200 Subject: [PATCH 12/19] CatimaImporter: add .close() --- .../java/protect/card_locker/importexport/CatimaImporter.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java index 1ab38c714..b5f6c63ae 100644 --- a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java @@ -93,6 +93,8 @@ public class CatimaImporter implements Importer { importedData = importCSV(bufferedInputStream1); } + input1.close(); + if (importedData == null) { throw new FormatException("No imported data"); } @@ -112,6 +114,8 @@ public class CatimaImporter implements Importer { Utils.saveCardImage(context, ZipUtils.readImage(zipInputStream2), newFileName); } } + + input2.close(); } } From f783be7a4f06c7eca0222e16b12536b616232948 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 18:13:49 +0200 Subject: [PATCH 13/19] importer: handle inputFile errors better --- .../main/java/protect/card_locker/Utils.java | 22 +++++------ .../importexport/MultiFormatImporter.java | 37 ++++++++++++------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/app/src/main/java/protect/card_locker/Utils.java b/app/src/main/java/protect/card_locker/Utils.java index 946cf278e..b3ee9f9db 100644 --- a/app/src/main/java/protect/card_locker/Utils.java +++ b/app/src/main/java/protect/card_locker/Utils.java @@ -505,21 +505,17 @@ public class Utils { return new File(context.getCacheDir() + "/" + name); } - public static File copyToTempFile(Context context, InputStream input, String name) { + public static File copyToTempFile(Context context, InputStream input, String name) throws IOException { File file = createTempFile(context, name); - try (FileOutputStream out = new FileOutputStream(file)) { - byte[] buf = new byte[4096]; - int len; - while ((len = input.read(buf)) != -1) { - out.write(buf, 0, len); - } - out.close(); - input.close(); - return file; - } catch (IOException e) { - Log.d("store temp file", "failed writing temp file, name: " + name); - return null; + FileOutputStream out = new FileOutputStream(file); + byte[] buf = new byte[4096]; + int len; + while ((len = input.read(buf)) != -1) { + out.write(buf, 0, len); } + out.close(); + input.close(); + return file; } public static String saveTempImage(Context context, Bitmap in, String name, Bitmap.CompressFormat format) { diff --git a/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java b/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java index 78ee564b0..99eb3ef46 100644 --- a/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/MultiFormatImporter.java @@ -7,6 +7,7 @@ import android.util.Log; import net.lingala.zip4j.exception.ZipException; import java.io.File; +import java.io.IOException; import java.io.InputStream; import protect.card_locker.Utils; @@ -46,25 +47,33 @@ public class MultiFormatImporter { String error = null; if (importer != null) { - File inputFile = Utils.copyToTempFile(context, input, TEMP_ZIP_NAME); - database.beginTransaction(); + File inputFile; try { - importer.importData(context, database, inputFile, password); - database.setTransactionSuccessful(); - return new ImportExportResult(ImportExportResultType.Success); - } catch (ZipException e) { - if (e.getType().equals(ZipException.Type.WRONG_PASSWORD)) { - return new ImportExportResult(ImportExportResultType.BadPassword); - } else { + inputFile = Utils.copyToTempFile(context, input, TEMP_ZIP_NAME); + database.beginTransaction(); + try { + importer.importData(context, database, inputFile, password); + database.setTransactionSuccessful(); + return new ImportExportResult(ImportExportResultType.Success); + } catch (ZipException e) { + if (e.getType().equals(ZipException.Type.WRONG_PASSWORD)) { + return new ImportExportResult(ImportExportResultType.BadPassword); + } else { + Log.e(TAG, "Failed to import data", e); + error = e.toString(); + } + } catch (Exception e) { Log.e(TAG, "Failed to import data", e); error = e.toString(); + } finally { + database.endTransaction(); + if (!inputFile.delete()) { + Log.w(TAG, "Failed to delete temporary ZIP file (should not be a problem) " + inputFile); + } } - } catch (Exception e) { - Log.e(TAG, "Failed to import data", e); + } catch (IOException e) { + Log.e(TAG, "Failed to copy ZIP file", e); error = e.toString(); - } finally { - database.endTransaction(); - inputFile.delete(); } } else { error = "Unsupported data format imported: " + format.name(); From d11e2c166b0e6fd0409f8b56a52cb13dbefba989 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 18:29:23 +0200 Subject: [PATCH 14/19] Utils.copyToTempFile(): use try for resource management --- app/src/main/java/protect/card_locker/Utils.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/app/src/main/java/protect/card_locker/Utils.java b/app/src/main/java/protect/card_locker/Utils.java index b3ee9f9db..dd634489a 100644 --- a/app/src/main/java/protect/card_locker/Utils.java +++ b/app/src/main/java/protect/card_locker/Utils.java @@ -507,15 +507,14 @@ public class Utils { public static File copyToTempFile(Context context, InputStream input, String name) throws IOException { File file = createTempFile(context, name); - FileOutputStream out = new FileOutputStream(file); - byte[] buf = new byte[4096]; - int len; - while ((len = input.read(buf)) != -1) { - out.write(buf, 0, len); + try (input; FileOutputStream out = new FileOutputStream(file)) { + byte[] buf = new byte[4096]; + int len; + while ((len = input.read(buf)) != -1) { + out.write(buf, 0, len); + } + return file; } - out.close(); - input.close(); - return file; } public static String saveTempImage(Context context, Bitmap in, String name, Bitmap.CompressFormat format) { From 9cf9959b6be17c40c79000eb5e3d8339292760d0 Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 19:51:54 +0200 Subject: [PATCH 15/19] add importExistingCardsAfterModification test --- .../protect/card_locker/ImportExportTest.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/app/src/test/java/protect/card_locker/ImportExportTest.java b/app/src/test/java/protect/card_locker/ImportExportTest.java index 4c74b67ce..90358a58d 100644 --- a/app/src/test/java/protect/card_locker/ImportExportTest.java +++ b/app/src/test/java/protect/card_locker/ImportExportTest.java @@ -196,6 +196,35 @@ public class ImportExportTest { cursor.close(); } + private void checkLoyaltyCardsAndDuplicates(int numCards) { + Cursor cursor = DBHelper.getLoyaltyCardCursor(mDatabase); + + while (cursor.moveToNext()) { + LoyaltyCard card = LoyaltyCard.toLoyaltyCard(cursor); + + // ID goes up for duplicates (b/c the cursor orders by store), down for originals + int index = card.id > numCards ? card.id - numCards : numCards - card.id + 1; + // balance is doubled for modified originals + int balance = card.id > numCards ? index : index * 2; + + String expectedStore = String.format("store, \"%4d", index); + String expectedNote = String.format("note, \"%4d", index); + + assertEquals(expectedStore, card.store); + assertEquals(expectedNote, card.note); + assertEquals(null, card.validFrom); + assertEquals(null, card.expiry); + assertEquals(new BigDecimal(String.valueOf(balance)), card.balance); + assertEquals(null, card.balanceType); + assertEquals(BARCODE_DATA, card.cardId); + assertEquals(null, card.barcodeId); + assertEquals(BARCODE_TYPE.format(), card.barcodeType.format()); + assertEquals(Integer.valueOf(index), card.headerColor); + assertEquals(0, card.starStatus); + } + cursor.close(); + } + /** * Check that all of the cards follow the pattern * specified in addLoyaltyCardsSomeStarred(), and are in sequential order @@ -477,6 +506,40 @@ public class ImportExportTest { TestHelpers.getEmptyDb(activity); } + @Test + public void importExistingCardsAfterModification() throws IOException { + final int NUM_CARDS = 10; + + TestHelpers.addLoyaltyCards(mDatabase, NUM_CARDS); + + ByteArrayOutputStream outData = new ByteArrayOutputStream(); + OutputStreamWriter outStream = new OutputStreamWriter(outData); + + // Export into CSV data + ImportExportResult result = MultiFormatExporter.exportData(activity.getApplicationContext(), mDatabase, outData, DataFormat.Catima, null); + assertEquals(ImportExportResultType.Success, result.resultType()); + outStream.close(); + + // Modify existing cards + for (int index = 1; index <= NUM_CARDS; index++) { + int id = NUM_CARDS - index + 1; + DBHelper.updateLoyaltyCardBalance(mDatabase, id, new BigDecimal(String.valueOf(index * 2))); + } + + ByteArrayInputStream inData = new ByteArrayInputStream(outData.toByteArray()); + + // Import the CSV data on top of the existing database + result = MultiFormatImporter.importData(activity.getApplicationContext(), mDatabase, inData, DataFormat.Catima, null); + assertEquals(ImportExportResultType.Success, result.resultType()); + + assertEquals(NUM_CARDS * 2, DBHelper.getLoyaltyCardCount(mDatabase)); + + checkLoyaltyCardsAndDuplicates(NUM_CARDS); + + // Clear the database for the next format under test + TestHelpers.getEmptyDb(activity); + } + @Test public void corruptedImportNothingSaved() { final int NUM_CARDS = 10; From 3ae665b70ffbb9d42cc1891d9d385249a9107f6f Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Sun, 16 Jul 2023 20:16:01 +0200 Subject: [PATCH 16/19] DBHelper: add note to DEFAULT_ZOOM_LEVEL --- app/src/main/java/protect/card_locker/DBHelper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/src/main/java/protect/card_locker/DBHelper.java b/app/src/main/java/protect/card_locker/DBHelper.java index fa670894e..56b2c4a71 100644 --- a/app/src/main/java/protect/card_locker/DBHelper.java +++ b/app/src/main/java/protect/card_locker/DBHelper.java @@ -24,6 +24,8 @@ public class DBHelper extends SQLiteOpenHelper { public static final String DATABASE_NAME = "Catima.db"; public static final int ORIGINAL_DATABASE_VERSION = 1; public static final int DATABASE_VERSION = 16; + + // NB: changing this value requires a migration public static final int DEFAULT_ZOOM_LEVEL = 100; public static class LoyaltyCardDbGroups { From ab030ba002c171cfede564859e2f143589290a8a Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Mon, 17 Jul 2023 20:23:52 +0200 Subject: [PATCH 17/19] LocaltyCard.isDuplicate(): reformat & add comments Co-authored-by: Sylvia van Os --- .../java/protect/card_locker/LoyaltyCard.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/protect/card_locker/LoyaltyCard.java b/app/src/main/java/protect/card_locker/LoyaltyCard.java index 6b7075069..100e17a94 100644 --- a/app/src/main/java/protect/card_locker/LoyaltyCard.java +++ b/app/src/main/java/protect/card_locker/LoyaltyCard.java @@ -145,11 +145,20 @@ public class LoyaltyCard implements Parcelable { public static boolean isDuplicate(final LoyaltyCard a, final LoyaltyCard b) { // Skip lastUsed & zoomLevel - return a.id == b.id && a.store.equals(b.store) && a.note.equals(b.note) && Utils.equals(a.validFrom, b.validFrom) - && Utils.equals(a.expiry, b.expiry) && a.balance.equals(b.balance) && Utils.equals(a.balanceType, b.balanceType) - && a.cardId.equals(b.cardId) && Utils.equals(a.barcodeId, b.barcodeId) - && Utils.equals(a.barcodeType == null ? null : a.barcodeType.format(), b.barcodeType == null ? null : b.barcodeType.format()) - && Utils.equals(a.headerColor, b.headerColor) && a.starStatus == b.starStatus && a.archiveStatus == b.archiveStatus; + return a.id == b.id && // non-nullable int + a.store.equals(b.store) && // non-nullable String + a.note.equals(b.note) && // non-nullable String + Utils.equals(a.validFrom, b.validFrom) && // nullable Date + Utils.equals(a.expiry, b.expiry) && // nullable Date + a.balance.equals(b.balance) && // non-nullable BigDecimal + Utils.equals(a.balanceType, b.balanceType) && // nullable Currency + a.cardId.equals(b.cardId) && // non-nullable String + Utils.equals(a.barcodeId, b.barcodeId) && // nullable String + Utils.equals(a.barcodeType == null ? null : a.barcodeType.format(), + b.barcodeType == null ? null : b.barcodeType.format()) && // nullable CatimaBarcode with no overridden .equals(), so we need to check .format() + Utils.equals(a.headerColor, b.headerColor) && // nullable Integer + a.starStatus == b.starStatus && // non-nullable int + a.archiveStatus == b.archiveStatus; // non-nullable int } @Override From ac0f6f6f3ea735eaf1b58b1ecb4974056a82642f Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Mon, 17 Jul 2023 20:35:02 +0200 Subject: [PATCH 18/19] Utils.getRenamedCardImageFileName(): add javadoc Co-authored-by: Sylvia van Os --- app/src/main/java/protect/card_locker/Utils.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/src/main/java/protect/card_locker/Utils.java b/app/src/main/java/protect/card_locker/Utils.java index dd634489a..f5d28455b 100644 --- a/app/src/main/java/protect/card_locker/Utils.java +++ b/app/src/main/java/protect/card_locker/Utils.java @@ -386,6 +386,13 @@ public class Utils { return cardImageFileNameBuilder.toString(); } + /** + * Returns a card image filename (string) with the ID replaced according to the map if the input is a valid card image filename (string), otherwise null. + * + * @param fileName e.g. "card_1_front.png" + * @param idMap e.g. Map.of(1, 2) + * @return String e.g. "card_2_front.png" + */ static public String getRenamedCardImageFileName(final String fileName, final Map idMap) { Pattern pattern = Pattern.compile(CARD_IMAGE_FILENAME_REGEX); Matcher matcher = pattern.matcher(fileName); From d350d0b2c797be623c4d3f5e49e8eddf911c8aac Mon Sep 17 00:00:00 2001 From: FC Stegerman Date: Mon, 17 Jul 2023 20:38:27 +0200 Subject: [PATCH 19/19] CatimaImporter: add comment about card group import Co-authored-by: Sylvia van Os --- .../java/protect/card_locker/importexport/CatimaImporter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java index b5f6c63ae..cca491f2d 100644 --- a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java @@ -142,6 +142,7 @@ public class CatimaImporter implements Importer { for (Map.Entry entry : data.cardGroups) { int cardId = idMap.getOrDefault(entry.getKey(), entry.getKey()); String groupId = entry.getValue(); + // For existing & newly imported cards, add the groups from the import to the internal state List cardGroups = DBHelper.getLoyaltyCardGroups(database, cardId); cardGroups.add(DBHelper.getGroup(database, groupId)); DBHelper.setLoyaltyCardGroups(database, cardId, cardGroups);