Improve Stocard importer

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Unreleased
+
+Changes:
+
+- Improve Stocard importer
+
 ## v2.2.0 (2021-08-02)
 
 Changes:

app/src/main/java/protect/card_locker/importexport/StocardImporter.java

@@ -9,20 +9,22 @@ import com.google.zxing.BarcodeFormat;
 import net.lingala.zip4j.io.inputstream.ZipInputStream;
 import net.lingala.zip4j.model.LocalFileHeader;
 
-import org.json.JSONArray;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
 import org.json.JSONException;
 import org.json.JSONObject;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.math.BigDecimal;
 import java.text.ParseException;
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.List;
 
 import protect.card_locker.DBHelper;
 import protect.card_locker.FormatException;
+import protect.card_locker.R;
 import protect.card_locker.Utils;
 import protect.card_locker.ZipUtils;
 
@@ -37,7 +39,25 @@ public class StocardImporter implements Importer
 {
     public void importData(Context context, DBHelper db, InputStream input, char[] password) throws IOException, FormatException, JSONException, ParseException {
         HashMap<String, HashMap<String, Object>> loyaltyCardHashMap = new HashMap<>();
-        HashMap<String, String> providers = new HashMap<>();
+        HashMap<String, HashMap<String, String>> providers = new HashMap<>();
+
+        final CSVParser parser = new CSVParser(new InputStreamReader(context.getResources().openRawResource(R.raw.stocard_stores)), CSVFormat.RFC4180.withHeader());
+
+        try
+        {
+            for (CSVRecord record : parser)
+            {
+                HashMap<String, String> recordData = new HashMap<>();
+                recordData.put("name", record.get("name"));
+                recordData.put("barcodeFormat", record.get("barcodeFormat"));
+
+                providers.put(record.get("_id"), recordData);
+            }
+
+            parser.close();
+        } catch(IllegalArgumentException|IllegalStateException e) {
+            throw new FormatException("Issue parsing CSV data", e);
+        }
 
         ZipInputStream zipInputStream = new ZipInputStream(input, password);
 
@@ -68,9 +88,7 @@ public class StocardImporter implements Importer
                 };
             }
 
-            if (startsWith(nameParts, providersFileName, 0) && !localFileHeader.isDirectory()) {
-                providers = parseProviders(zipInputStream);
-            } else if (startsWith(nameParts, cardBaseName, 1)) {
+            if (startsWith(nameParts, cardBaseName, 1)) {
                 // Extract cardName
                 cardName = nameParts[cardBaseName.length].split("\\.", 2)[0];
 
@@ -139,10 +157,13 @@ public class StocardImporter implements Importer
         database.beginTransaction();
 
         for (HashMap<String, Object> loyaltyCardData : loyaltyCardHashMap.values()) {
-            String store = providers.get(loyaltyCardData.get("_providerId").toString());
+            String providerId = (String) loyaltyCardData.get("_providerId");
+            HashMap<String, String> providerData = providers.get(providerId);
+
+            String store = providerData != null ? providerData.get("name") : providerId;
             String note = (String) Utils.hashmapGetOrDefault(loyaltyCardData, "note", "");
             String cardId = (String) loyaltyCardData.get("cardId");
-            String barcodeTypeString = (String) Utils.hashmapGetOrDefault(loyaltyCardData, "barcodeType", null);
+            String barcodeTypeString = (String) Utils.hashmapGetOrDefault(loyaltyCardData, "barcodeType", providerData != null ? providerData.get("barcodeFormat") : null);
             BarcodeFormat barcodeType = null;
             if (barcodeTypeString != null) {
                 if (barcodeTypeString.equals("RSS_DATABAR_EXPANDED")) {
@@ -194,33 +215,4 @@ public class StocardImporter implements Importer
 
         return loyaltyCardHashMap;
     }
-
-    private HashMap<String, String> parseProviders(ZipInputStream zipInputStream) throws IOException, JSONException {
-        // FIXME: This is probably completely wrong, but it works for the one and only test file I have
-        JSONObject jsonObject = ZipUtils.readJSON(zipInputStream);
-
-        JSONArray providerIdList = jsonObject.getJSONArray("provider_id_list");
-        JSONArray providerList = jsonObject.getJSONArray("provider_list");
-
-        // Resort, put IDs with - in them after IDs without any -
-        List<String> providerIds = new ArrayList<>();
-        List<String> customProviderIds = new ArrayList<>();
-
-        for (int i = 0; i < providerIdList.length(); i++) {
-            String providerId = providerIdList.get(i).toString();
-            if (providerId.contains("-")) {
-                customProviderIds.add(providerId);
-            } else {
-                providerIds.add(providerId);
-            }
-        }
-        providerIds.addAll(customProviderIds);
-
-        HashMap<String, String> providers = new HashMap<>();
-        for (int i = 0; i < jsonObject.getInt("number_of_cards"); i++) {
-            providers.put(providerIds.get(i), providerList.get(i).toString());
-        }
-
-        return providers;
-    }
 }

app/src/main/res/raw/readme.md

@@ -0,0 +1,7 @@
+# stocard_stores.csv
+
+stocard_stores.csv was created by extracting /data/data/de.stocard/de.stocard.stocard/databases/stores on a rooted devices and running the following command over it:
+
+    sqlite3 -header -csv stores "select _id,name,barcodeFormat from stores" > stocard_stores.csv
+
+Only used for data portability reasons (ensuring importing works). Do NOT copy this anywhere else or use it for any purpose other than ensuring we can import a GDPR-provided export. We want to make sure this stays under fair use.

app/src/main/res/values/strings.xml

@@ -188,7 +188,7 @@
     <string name="importLoyaltyCardKeychain">Import from Loyalty Card Keychain</string>
     <string name="importLoyaltyCardKeychainMessage">Select your <i>LoyaltyCardKeychain.csv</i> export from Loyalty Card Keychain to import.\nOr create it from the "Import/Export" menu in Loyalty Card Keychain by pressing "Export" there first.</string>
     <string name="importStocard">Import from Stocard</string>
-    <string name="importStocardMessage">Select your <i>***-sync.zip</i> export from Stocard to import, and select the barcode types manually afterwards.\nOr get it by e-mailing support@stocardapp.com asking for an export of your data.</string>
+    <string name="importStocardMessage">Select your <i>***-sync.zip</i> export from Stocard to import.\nOr get it by e-mailing support@stocardapp.com asking for an export of your data.</string>
     <string name="importVoucherVault">Import from Voucher Vault</string>
     <string name="importVoucherVaultMessage">Select your <i>vouchervault.json</i> export from Voucher Vault to import.\nOr create it by pressing "Export" in Voucher Vault first.</string>
     <string name="barcodeId">Barcode value</string>

app/src/test/java/protect/card_locker/ImportExportTest.java

@@ -1197,7 +1197,7 @@ public class ImportExportTest
         assertEquals(null, card.balanceType);
         assertEquals("55555", card.cardId);
         assertEquals(null, card.barcodeId);
-        assertEquals(null, card.barcodeType);
+        assertEquals(BarcodeFormat.EAN_13, card.barcodeType);
         assertEquals(0, card.starStatus);
 
         assertNull(Utils.retrieveCardImage(activity.getApplicationContext(), 1, true));
@@ -1212,7 +1212,7 @@ public class ImportExportTest
         assertEquals(null, card.balanceType);
         assertEquals("7649484", card.cardId);
         assertEquals(null, card.barcodeId);
-        assertEquals(null, card.barcodeType);
+        assertEquals(BarcodeFormat.EAN_13, card.barcodeType);
         assertEquals(0, card.starStatus);
 
         assertTrue(BitmapFactory.decodeStream(getClass().getResourceAsStream("stocard-front.jpg")).sameAs(Utils.retrieveCardImage(activity.getApplicationContext(), 2, true)));
@@ -1220,7 +1220,8 @@ public class ImportExportTest
 
         card = db.getLoyaltyCard(3);
 
-        assertEquals("jö", card.store);
+        // I don't think we can know this one, but falling back to an unique store name is at least something
+        assertEquals("63536738-d64b-48ae-aeb8-82761523fa67", card.store);
         assertEquals("", card.note);
         assertEquals(null, card.expiry);
         assertEquals(new BigDecimal("0"), card.balance);