diff --git a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java index 99029236b..1ab38c714 100644 --- a/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java +++ b/app/src/main/java/protect/card_locker/importexport/CatimaImporter.java @@ -27,11 +27,14 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import protect.card_locker.CatimaBarcode; import protect.card_locker.DBHelper; import protect.card_locker.FormatException; import protect.card_locker.Group; +import protect.card_locker.ImageLocationType; +import protect.card_locker.LoyaltyCard; import protect.card_locker.Utils; import protect.card_locker.ZipUtils; @@ -43,6 +46,18 @@ import protect.card_locker.ZipUtils; * A header is expected for the each table showing the names of the columns. */ public class CatimaImporter implements Importer { + public static class ImportedData { + public final List cards; + public final List groups; + public final List> cardGroups; + + ImportedData(final List cards, final List groups, final List> cardGroups) { + this.cards = cards; + this.groups = groups; + this.cardGroups = cardGroups; + } + } + public void importData(Context context, SQLiteDatabase database, File inputFile, char[] password) throws IOException, FormatException, InterruptedException { // Pass #1: get hashes and parse CSV InputStream input1 = new FileInputStream(inputFile); @@ -54,14 +69,18 @@ public class CatimaImporter implements Importer { boolean isZipFile = false; LocalFileHeader localFileHeader; Map imageChecksums = new HashMap<>(); + ImportedData importedData = null; while ((localFileHeader = zipInputStream1.getNextEntry()) != null) { isZipFile = true; String fileName = Uri.parse(localFileHeader.getFileName()).getLastPathSegment(); if (fileName.equals("catima.csv")) { - importCSV(context, database, zipInputStream1); + importedData = importCSV(zipInputStream1); } else if (fileName.endsWith(".png")) { + if (!fileName.matches(Utils.CARD_IMAGE_FILENAME_REGEX)) { + throw new FormatException("Unexpected PNG file in import: " + fileName); + } imageChecksums.put(fileName, Utils.checksum(zipInputStream1)); } else { throw new FormatException("Unexpected file in import: " + fileName); @@ -71,50 +90,104 @@ public class CatimaImporter implements Importer { if (!isZipFile) { // This is not a zip file, try importing as bare CSV bufferedInputStream1.reset(); - importCSV(context, database, bufferedInputStream1); - input1.close(); - return; + importedData = importCSV(bufferedInputStream1); } - input1.close(); + if (importedData == null) { + throw new FormatException("No imported data"); + } - // Pass #2: save images - InputStream input2 = new FileInputStream(inputFile); - InputStream bufferedInputStream2 = new BufferedInputStream(input2); - ZipInputStream zipInputStream2 = new ZipInputStream(bufferedInputStream2, password); + Map idMap = saveAndDeduplicate(context, database, importedData, imageChecksums); - while ((localFileHeader = zipInputStream2.getNextEntry()) != null) { - String fileName = Uri.parse(localFileHeader.getFileName()).getLastPathSegment(); - if (fileName.endsWith(".png")) { - Utils.saveCardImage(context, ZipUtils.readImage(zipInputStream2), fileName); + if (isZipFile) { + // Pass #2: save images + InputStream input2 = new FileInputStream(inputFile); + InputStream bufferedInputStream2 = new BufferedInputStream(input2); + ZipInputStream zipInputStream2 = new ZipInputStream(bufferedInputStream2, password); + + while ((localFileHeader = zipInputStream2.getNextEntry()) != null) { + String fileName = Uri.parse(localFileHeader.getFileName()).getLastPathSegment(); + if (fileName.endsWith(".png")) { + String newFileName = Utils.getRenamedCardImageFileName(fileName, idMap); + Utils.saveCardImage(context, ZipUtils.readImage(zipInputStream2), newFileName); + } + } + } + } + + public Map saveAndDeduplicate(Context context, SQLiteDatabase database, final ImportedData data, final Map imageChecksums) throws IOException { + Map idMap = new HashMap<>(); + Set existingImages = DBHelper.imageFiles(context, database); + + for (LoyaltyCard card : data.cards) { + LoyaltyCard existing = DBHelper.getLoyaltyCard(database, card.id); + if (existing == null) { + DBHelper.insertLoyaltyCard(database, card.id, card.store, card.note, card.validFrom, card.expiry, card.balance, card.balanceType, + card.cardId, card.barcodeId, card.barcodeType, card.headerColor, card.starStatus, card.lastUsed, card.archiveStatus); + } else if (!isDuplicate(context, existing, card, existingImages, imageChecksums)) { + long newId = DBHelper.insertLoyaltyCard(database, card.store, card.note, card.validFrom, card.expiry, card.balance, card.balanceType, + card.cardId, card.barcodeId, card.barcodeType, card.headerColor, card.starStatus, card.lastUsed, card.archiveStatus); + idMap.put(card.id, (int) newId); } } - input2.close(); + for (String group : data.groups) { + DBHelper.insertGroup(database, group); + } + + for (Map.Entry entry : data.cardGroups) { + int cardId = idMap.getOrDefault(entry.getKey(), entry.getKey()); + String groupId = entry.getValue(); + List cardGroups = DBHelper.getLoyaltyCardGroups(database, cardId); + cardGroups.add(DBHelper.getGroup(database, groupId)); + DBHelper.setLoyaltyCardGroups(database, cardId, cardGroups); + } + + return idMap; } - public void importCSV(Context context, SQLiteDatabase database, InputStream input) throws IOException, FormatException, InterruptedException { + public boolean isDuplicate(Context context, final LoyaltyCard existing, final LoyaltyCard card, final Set existingImages, final Map imageChecksums) throws IOException { + if (!LoyaltyCard.isDuplicate(existing, card)) { + return false; + } + for (ImageLocationType imageLocationType : ImageLocationType.values()) { + String name = Utils.getCardImageFileName(existing.id, imageLocationType); + boolean exists = existingImages.contains(name); + if (exists != imageChecksums.containsKey(name)) { + return false; + } + if (exists) { + File file = Utils.retrieveCardImageAsFile(context, name); + if (!imageChecksums.get(name).equals(Utils.checksum(new FileInputStream(file)))) { + return false; + } + } + } + return true; + } + + public ImportedData importCSV(InputStream input) throws IOException, FormatException, InterruptedException { BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8)); int version = parseVersion(bufferedReader); switch (version) { case 1: - parseV1(database, bufferedReader); - break; + return parseV1(bufferedReader); case 2: - parseV2(context, database, bufferedReader); - break; + return parseV2(bufferedReader); default: throw new FormatException(String.format("No code to parse version %s", version)); } } - public void parseV1(SQLiteDatabase database, BufferedReader input) throws IOException, FormatException, InterruptedException { + public ImportedData parseV1(BufferedReader input) throws IOException, FormatException, InterruptedException { + ImportedData data = new ImportedData(new ArrayList<>(), new ArrayList<>(), new ArrayList<>()); final CSVParser parser = new CSVParser(input, CSVFormat.RFC4180.builder().setHeader().build()); try { for (CSVRecord record : parser) { - importLoyaltyCard(database, record); + LoyaltyCard card = importLoyaltyCard(record); + data.cards.add(card); if (Thread.currentThread().isInterrupted()) { throw new InterruptedException(); @@ -125,9 +198,15 @@ public class CatimaImporter implements Importer { } catch (IllegalArgumentException | IllegalStateException e) { throw new FormatException("Issue parsing CSV data", e); } + + return data; } - public void parseV2(Context context, SQLiteDatabase database, BufferedReader input) throws IOException, FormatException, InterruptedException { + public ImportedData parseV2(BufferedReader input) throws IOException, FormatException, InterruptedException { + List cards = new ArrayList<>(); + List groups = new ArrayList<>(); + List> cardGroups = new ArrayList<>(); + int part = 0; StringBuilder stringPart = new StringBuilder(); @@ -145,7 +224,7 @@ public class CatimaImporter implements Importer { break; case 1: try { - parseV2Groups(database, stringPart.toString()); + groups = parseV2Groups(stringPart.toString()); sectionParsed = true; } catch (FormatException e) { // We may have a multiline field, try again @@ -153,7 +232,7 @@ public class CatimaImporter implements Importer { break; case 2: try { - parseV2Cards(context, database, stringPart.toString()); + cards = parseV2Cards(stringPart.toString()); sectionParsed = true; } catch (FormatException e) { // We may have a multiline field, try again @@ -161,7 +240,7 @@ public class CatimaImporter implements Importer { break; case 3: try { - parseV2CardGroups(database, stringPart.toString()); + cardGroups = parseV2CardGroups(stringPart.toString()); sectionParsed = true; } catch (FormatException e) { // We may have a multiline field, try again @@ -188,9 +267,11 @@ public class CatimaImporter implements Importer { } catch (FormatException e) { throw new FormatException("Issue parsing CSV data", e); } + + return new ImportedData(cards, groups, cardGroups); } - public void parseV2Groups(SQLiteDatabase database, String data) throws IOException, FormatException, InterruptedException { + public List parseV2Groups(String data) throws IOException, FormatException, InterruptedException { // Parse groups final CSVParser groupParser = new CSVParser(new StringReader(data), CSVFormat.RFC4180.builder().setHeader().build()); @@ -210,12 +291,15 @@ public class CatimaImporter implements Importer { groupParser.close(); } + List groups = new ArrayList<>(); for (CSVRecord record : records) { - importGroup(database, record); + String group = importGroup(record); + groups.add(group); } + return groups; } - public void parseV2Cards(Context context, SQLiteDatabase database, String data) throws IOException, FormatException, InterruptedException { + public List parseV2Cards(String data) throws IOException, FormatException, InterruptedException { // Parse cards final CSVParser cardParser = new CSVParser(new StringReader(data), CSVFormat.RFC4180.builder().setHeader().build()); @@ -235,12 +319,15 @@ public class CatimaImporter implements Importer { cardParser.close(); } + List cards = new ArrayList<>(); for (CSVRecord record : records) { - importLoyaltyCard(database, record); + LoyaltyCard card = importLoyaltyCard(record); + cards.add(card); } + return cards; } - public void parseV2CardGroups(SQLiteDatabase database, String data) throws IOException, FormatException, InterruptedException { + public List> parseV2CardGroups(String data) throws IOException, FormatException, InterruptedException { // Parse card group mappings final CSVParser cardGroupParser = new CSVParser(new StringReader(data), CSVFormat.RFC4180.builder().setHeader().build()); @@ -260,9 +347,12 @@ public class CatimaImporter implements Importer { cardGroupParser.close(); } + List> cardGroups = new ArrayList<>(); for (CSVRecord record : records) { - importCardGroupMapping(database, record); + Map.Entry entry = importCardGroupMapping(record); + cardGroups.add(entry); } + return cardGroups; } /** @@ -298,8 +388,7 @@ public class CatimaImporter implements Importer { * Import a single loyalty card into the database using the given * session. */ - private void importLoyaltyCard(SQLiteDatabase database, CSVRecord record) - throws FormatException { + private LoyaltyCard importLoyaltyCard(CSVRecord record) throws FormatException { int id = CSVHelpers.extractInt(DBHelper.LoyaltyCardDbIds.ID, record); String store = CSVHelpers.extractString(DBHelper.LoyaltyCardDbIds.STORE, record, ""); @@ -396,28 +485,28 @@ public class CatimaImporter implements Importer { // We catch this exception so we can still import old backups } - DBHelper.insertLoyaltyCard(database, id, store, note, validFrom, expiry, balance, balanceType, cardId, barcodeId, barcodeType, headerColor, starStatus, lastUsed, archiveStatus); + return new LoyaltyCard(id, store, note, validFrom, expiry, balance, balanceType, cardId, barcodeId, barcodeType, headerColor, starStatus, lastUsed, DBHelper.DEFAULT_ZOOM_LEVEL, archiveStatus); } /** * Import a single group into the database using the given * session. */ - private void importGroup(SQLiteDatabase database, CSVRecord record) throws FormatException { + private String importGroup(CSVRecord record) throws FormatException { String id = CSVHelpers.extractString(DBHelper.LoyaltyCardDbGroups.ID, record, null); if (id == null) { throw new FormatException("Group has no ID: " + record); } - DBHelper.insertGroup(database, id); + return id; } /** * Import a single card to group mapping into the database using the given * session. */ - private void importCardGroupMapping(SQLiteDatabase database, CSVRecord record) throws FormatException { + private Map.Entry importCardGroupMapping(CSVRecord record) throws FormatException { int cardId = CSVHelpers.extractInt(DBHelper.LoyaltyCardDbIdsGroups.cardID, record); String groupId = CSVHelpers.extractString(DBHelper.LoyaltyCardDbIdsGroups.groupID, record, null); @@ -425,8 +514,6 @@ public class CatimaImporter implements Importer { throw new FormatException("Group has no ID: " + record); } - List cardGroups = DBHelper.getLoyaltyCardGroups(database, cardId); - cardGroups.add(DBHelper.getGroup(database, groupId)); - DBHelper.setLoyaltyCardGroups(database, cardId, cardGroups); + return Map.entry(cardId, groupId); } }