dir: Add new version of DeployData with parsed appstream data

We load the per-app appstream xml.gz during deploy and inject name,
summary and version from it into the deploy data. For older deploys
that don't have this data we re-create it when needed (with the hope
that updates to app eventually makes this unnecessary due to new deploys).

Additionally this removes the support for generating deploy files where
they don't exist, as support for this predates xdg-app 0.6, so should not
exist in real life.

Closes: #2409
Approved by: alexlarsson

common/flatpak-dir-private.h

@@ -319,6 +319,7 @@ gboolean       flatpak_remove_override_keyfile (const char  *app_id,
                                                 gboolean     user,
                                                 GError     **error);
 
+int                 flatpak_deploy_data_get_version (GVariant *deploy_data);
 const char *        flatpak_deploy_data_get_origin (GVariant *deploy_data);
 const char *        flatpak_deploy_data_get_commit (GVariant *deploy_data);
 const char **       flatpak_deploy_data_get_subpaths (GVariant *deploy_data);
@@ -327,6 +328,9 @@ const char *        flatpak_deploy_data_get_alt_id (GVariant *deploy_data);
 const char *        flatpak_deploy_data_get_eol (GVariant *deploy_data);
 const char *        flatpak_deploy_data_get_eol_rebase (GVariant *deploy_data);
 const char *        flatpak_deploy_data_get_runtime (GVariant *deploy_data);
+const char *        flatpak_deploy_data_get_appdata_name (GVariant *deploy_data);
+const char *        flatpak_deploy_data_get_appdata_summary (GVariant *deploy_data);
+const char *        flatpak_deploy_data_get_appdata_version (GVariant *deploy_data);
 
 GFile *        flatpak_deploy_get_dir (FlatpakDeploy *deploy);
 GVariant *     flatpak_load_deploy_data (GFile        *deploy_dir,

common/flatpak-dir.c

@@ -97,10 +97,6 @@ static gboolean flatpak_dir_remote_fetch_summary (FlatpakDir   *self,
                                                   GCancellable *cancellable,
                                                   GError      **error);
 
-static GVariant * flatpak_create_deploy_data_from_old (GFile        *deploy_dir,
-                                                       GCancellable *cancellable,
-                                                       GError      **error);
-
 static gboolean flatpak_dir_cleanup_remote_for_url_change (FlatpakDir   *self,
                                                            const char   *remote_name,
                                                            const char   *url,
@@ -132,6 +128,7 @@ static void flatpak_dir_log (FlatpakDir *self,
    (flatpak_dir_log) (self,__FILE__, __LINE__, __FUNCTION__, \
                       NULL, change, remote, ref, commit, old_commit, url, format, __VA_ARGS__)
 
+static GVariant *upgrade_deploy_data (GVariant *deploy_data, GFile *deploy_dir, const char *ref);
 
 typedef struct
 {
@@ -579,23 +576,21 @@ flatpak_load_deploy_data (GFile        *deploy_dir,
   g_autoptr(GError) my_error = NULL;
   char *data = NULL;
   gsize data_size;
+  g_autoptr(GVariant) deploy_data = NULL;
 
   data_file = g_file_get_child (deploy_dir, "deploy");
   if (!g_file_load_contents (data_file, cancellable, &data, &data_size, NULL, &my_error))
-    {
-      if (!g_error_matches (my_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
-        {
-          g_propagate_error (error, g_steal_pointer (&my_error));
-          return NULL;
-        }
+    return NULL;
 
-      return flatpak_create_deploy_data_from_old (deploy_dir,
-                                                  cancellable, error);
-    }
+  deploy_data = g_variant_ref_sink (g_variant_new_from_data (FLATPAK_DEPLOY_DATA_GVARIANT_FORMAT,
+                                                             data, data_size,
+                                                             FALSE, g_free, data));
 
-  return g_variant_ref_sink (g_variant_new_from_data (FLATPAK_DEPLOY_DATA_GVARIANT_FORMAT,
-                                                      data, data_size,
-                                                      FALSE, g_free, data));
+
+  if (flatpak_deploy_data_get_version (deploy_data) < required_version)
+    return upgrade_deploy_data (deploy_data, deploy_dir, ref);
+
+  return g_steal_pointer (&deploy_data);
 }
 
 
@@ -2080,48 +2075,91 @@ flatpak_deploy_data_get_commit (GVariant *deploy_data)
   return commit;
 }
 
+int
+flatpak_deploy_data_get_version (GVariant *deploy_data)
+{
+  g_autoptr(GVariant) metadata = g_variant_get_child_value (deploy_data, 4);
+  int version = 0;
+
+  g_variant_lookup (metadata, "deploy-version", "i", &version);
+
+  return version;
+}
+
+static const char *
+flatpak_deploy_data_get_string (GVariant *deploy_data, const char *key)
+{
+  g_autoptr(GVariant) metadata = g_variant_get_child_value (deploy_data, 4);
+  const char *value = NULL;
+
+  g_variant_lookup (metadata, key, "&s", &value);
+
+  return value;
+}
+
+static const char *
+flatpak_deploy_data_get_localed_string (GVariant *deploy_data, const char *key)
+{
+  g_autoptr(GVariant) metadata = g_variant_get_child_value (deploy_data, 4);
+  const char * const * languages = g_get_language_names ();
+  const char *value = NULL;
+  int i;
+
+  for (i = 0; languages[i]; ++i)
+    {
+      g_autofree char *localed_key = NULL;
+      if (strcmp (languages[i], "C") == 0)
+        localed_key = g_strdup (key);
+      else
+        localed_key = g_strdup_printf ("%s@%s", key, languages[i]);
+
+      if (g_variant_lookup (metadata, localed_key, "&s", &value))
+        return value;
+    }
+
+  return NULL;
+}
+
 const char *
 flatpak_deploy_data_get_alt_id (GVariant *deploy_data)
 {
-  g_autoptr(GVariant) metadata = g_variant_get_child_value (deploy_data, 4);
-  const char *alt_id = NULL;
-
-  g_variant_lookup (metadata, "alt-id", "&s", &alt_id);
-
-  return alt_id;
+  return flatpak_deploy_data_get_string (deploy_data, "alt-id");
 }
 
 const char *
 flatpak_deploy_data_get_eol (GVariant *deploy_data)
 {
-  g_autoptr(GVariant) metadata = g_variant_get_child_value (deploy_data, 4);
-  const char *eol = NULL;
-
-  g_variant_lookup (metadata, "eol", "&s", &eol);
-
-  return eol;
+  return flatpak_deploy_data_get_string (deploy_data, "eol");
 }
 
 const char *
 flatpak_deploy_data_get_eol_rebase (GVariant *deploy_data)
 {
-  g_autoptr(GVariant) metadata = g_variant_get_child_value (deploy_data, 4);
-  const char *eol = NULL;
-
-  g_variant_lookup (metadata, "eolr", "&s", &eol);
-
-  return eol;
+  return flatpak_deploy_data_get_string (deploy_data, "eolr");
 }
 
 const char *
 flatpak_deploy_data_get_runtime (GVariant *deploy_data)
 {
-  g_autoptr(GVariant) metadata = g_variant_get_child_value (deploy_data, 4);
-  const char *runtime = NULL;
+  return flatpak_deploy_data_get_string (deploy_data, "runtime");
+}
 
-  g_variant_lookup (metadata, "runtime", "&s", &runtime);
+const char *
+flatpak_deploy_data_get_appdata_name (GVariant *deploy_data)
+{
+  return flatpak_deploy_data_get_localed_string (deploy_data, "appdata-name");
+}
 
-  return runtime;
+const char *
+flatpak_deploy_data_get_appdata_summary (GVariant *deploy_data)
+{
+  return flatpak_deploy_data_get_localed_string (deploy_data, "appdata-summary");
+}
+
+const char *
+flatpak_deploy_data_get_appdata_version (GVariant *deploy_data)
+{
+    return flatpak_deploy_data_get_string (deploy_data, "appdata-version");
 }
 
 /*<private>
@@ -2147,98 +2185,178 @@ flatpak_deploy_data_get_installed_size (GVariant *deploy_data)
   return GUINT64_FROM_BE (size);
 }
 
+static char *
+read_appdata_xml_from_deploy_dir (GFile *deploy_dir, const char *id)
+{
+  g_autoptr(GFile) appdata_file = NULL;
+  g_autofree char *appdata_name = NULL;
+  g_autoptr(GFileInputStream) appdata_in = NULL;
+  gsize size;
+
+  appdata_name = g_strconcat (id, ".xml.gz", NULL);
+  appdata_file  = flatpak_build_file (deploy_dir, "files/share/app-info/xmls", appdata_name, NULL);
+
+  appdata_in = g_file_read (appdata_file, NULL, NULL);
+  if (appdata_in)
+    {
+      g_autoptr(GZlibDecompressor) decompressor = g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP);
+      g_autoptr(GInputStream) converter = g_converter_input_stream_new (G_INPUT_STREAM (appdata_in), G_CONVERTER (decompressor));
+      g_autoptr(GBytes) appdata_xml = NULL;
+
+      appdata_xml = flatpak_read_stream (converter, TRUE, NULL);
+      if (appdata_xml)
+        return g_bytes_unref_to_data (g_steal_pointer (&appdata_xml), &size);
+    }
+
+  return NULL;
+}
+
+static void
+add_locale_metadata_string (GVariantBuilder *metadata_builder,
+                            const char *keyname,
+                            GHashTable *values)
+{
+  if (values == NULL)
+    return;
+
+  GLNX_HASH_TABLE_FOREACH_KV (values, const char *, locale, const char *, value)
+    {
+      const char *key;
+      g_autofree char *key_free = NULL;
+      if (strcmp (locale, "C") == 0)
+        key = keyname;
+      else
+        {
+          key_free = g_strdup_printf ("%s@%s", keyname, locale);
+          key = key_free;
+        }
+
+      g_variant_builder_add (metadata_builder, "{s@v}", key,
+                             g_variant_new_variant (g_variant_new_string (value)));
+    }
+}
+
+static void
+add_appdata_to_deploy_data (GVariantBuilder *metadata_builder,
+                            GFile      *deploy_dir,
+                            const char *id)
+{
+  g_autoptr(AsStore) store = as_store_new ();
+  g_autofree char *appdata_xml = NULL;
+  GPtrArray *apps;
+
+  appdata_xml = read_appdata_xml_from_deploy_dir (deploy_dir, id);
+  if (appdata_xml == NULL)
+    return;
+
+  as_store_from_xml (store, appdata_xml, NULL, NULL);
+  apps = as_store_get_apps (store);
+  if (apps->len > 0)
+    {
+      AsApp *app = g_ptr_array_index (apps, 0);
+      AsRelease *release = as_app_get_release_default (app);
+
+      add_locale_metadata_string (metadata_builder, "appdata-name", as_app_get_names (app));
+      add_locale_metadata_string (metadata_builder, "appdata-summary", as_app_get_comments (app));
+      if (release)
+        g_variant_builder_add (metadata_builder, "{s@v}", "appdata-version",
+                               g_variant_new_variant (g_variant_new_string (as_release_get_version (release))));
+    }
+}
+
 static GVariant *
-flatpak_dir_new_deploy_data (const char *origin,
+flatpak_dir_new_deploy_data (FlatpakDir *self,
+                             GFile      *deploy_dir,
+                             GVariant   *commit_metadata,
+                             GKeyFile   *metadata,
+                             const char *id,
+                             const char *origin,
                              const char *commit,
                              char      **subpaths,
-                             guint64     installed_size,
-                             GVariant   *metadata)
+                             guint64     installed_size)
 {
   char *empty_subpaths[] = {NULL};
-  GVariantBuilder builder;
+  GVariantBuilder metadata_builder;
+  g_autofree char *application_runtime = NULL;
+  const char *alt_id = NULL;
+  const char *eol = NULL;
+  const char *eol_rebase = NULL;
 
-  if (metadata == NULL)
-    {
-      g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}"));
-      metadata = g_variant_builder_end (&builder);
-    }
+  g_variant_lookup (commit_metadata, "xa.alt-id", "&s", &alt_id);
+  g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_ENDOFLIFE, "&s", &eol);
+  g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_ENDOFLIFE_REBASE, "&s", &eol_rebase);
+
+  application_runtime = g_key_file_get_string (metadata,
+                                               FLATPAK_METADATA_GROUP_APPLICATION,
+                                               FLATPAK_METADATA_KEY_RUNTIME, NULL);
+
+  g_variant_builder_init (&metadata_builder, G_VARIANT_TYPE ("a{sv}"));
+  g_variant_builder_add (&metadata_builder, "{s@v}", "deploy-version",
+                         g_variant_new_variant (g_variant_new_int32 (FLATPAK_DEPLOY_VERSION_CURRENT)));
+  if (alt_id)
+    g_variant_builder_add (&metadata_builder, "{s@v}", "alt-id",
+                           g_variant_new_variant (g_variant_new_string (alt_id)));
+  if (eol)
+    g_variant_builder_add (&metadata_builder, "{s@v}", "eol",
+                           g_variant_new_variant (g_variant_new_string (eol)));
+  if (eol_rebase)
+    g_variant_builder_add (&metadata_builder, "{s@v}", "eolr",
+                           g_variant_new_variant (g_variant_new_string (eol_rebase)));
+  if (application_runtime)
+    g_variant_builder_add (&metadata_builder, "{s@v}", "runtime",
+                           g_variant_new_variant (g_variant_new_string (application_runtime)));
+
+  add_appdata_to_deploy_data (&metadata_builder, deploy_dir, id);
 
   return g_variant_ref_sink (g_variant_new ("(ss^ast@a{sv})",
                                             origin,
                                             commit,
                                             subpaths ? subpaths : empty_subpaths,
                                             GUINT64_TO_BE (installed_size),
-                                            metadata));
-}
-
-static char **
-get_old_subpaths (GFile        *deploy_base,
-                  GCancellable *cancellable,
-                  GError      **error)
-{
-  g_autoptr(GFile) file = NULL;
-  g_autofree char *data = NULL;
-  g_autoptr(GError) my_error = NULL;
-  g_autoptr(GPtrArray) subpaths = NULL;
-  g_auto(GStrv) lines = NULL;
-  int i;
-
-  file = g_file_get_child (deploy_base, "subpaths");
-  if (!g_file_load_contents (file, cancellable, &data, NULL, NULL, &my_error))
-    {
-      if (g_error_matches (my_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
-        {
-          data = g_strdup ("");
-        }
-      else
-        {
-          g_propagate_error (error, g_steal_pointer (&my_error));
-          return NULL;
-        }
-    }
-
-  lines = g_strsplit (data, "\n", 0);
-
-  subpaths = g_ptr_array_new ();
-  for (i = 0; lines[i] != NULL; i++)
-    {
-      lines[i] = g_strstrip (lines[i]);
-      if (lines[i][0] == '/')
-        g_ptr_array_add (subpaths, g_strdup (lines[i]));
-    }
-
-  g_ptr_array_add (subpaths, NULL);
-  return (char **) g_ptr_array_free (subpaths, FALSE);
+                                            g_variant_builder_end (&metadata_builder)));
 }
 
 static GVariant *
-flatpak_create_deploy_data_from_old (GFile        *deploy_dir,
-                                     GCancellable *cancellable,
-                                     GError      **error)
+upgrade_deploy_data (GVariant *deploy_data, GFile *deploy_dir, const char *ref)
 {
-  g_autoptr(GFile) deploy_base = NULL;
-  g_autofree char *old_origin = NULL;
-  g_autofree char *commit = NULL;
-  g_auto(GStrv) old_subpaths = NULL;
-  g_autoptr(GFile) origin = NULL;
-  guint64 installed_size;
+  g_autoptr(GVariant) metadata = g_variant_get_child_value (deploy_data, 4);
+  GVariantBuilder metadata_builder;
+  g_autofree const char **subpaths = NULL;
+  int i, n, old_version;
 
-  deploy_base = g_file_get_parent (deploy_dir);
-  commit = g_file_get_basename (deploy_dir);
+  g_variant_builder_init (&metadata_builder, G_VARIANT_TYPE ("a{sv}"));
 
-  origin = g_file_get_child (deploy_base, "origin");
-  if (!g_file_load_contents (origin, cancellable, &old_origin, NULL, NULL, error))
-    return NULL;
+  g_variant_builder_add (&metadata_builder, "{s@v}", "deploy-version",
+                         g_variant_new_variant (g_variant_new_int32 (FLATPAK_DEPLOY_VERSION_CURRENT)));
 
-  old_subpaths = get_old_subpaths (deploy_base, cancellable, error);
-  if (old_subpaths == NULL)
-    return NULL;
+  /* Copy all metadata except version from old */
+  n = g_variant_n_children (metadata);
+  for (i = 0; i < n; i++)
+    {
+      g_autoptr(GVariant) child = g_variant_get_child_value (metadata, i);
+      const char *key;
+      g_variant_get_child (child, 0, "&s", &key);
+      if (strcmp (key, "deploy-version") == 0)
+        continue;
+      g_variant_builder_add_value (&metadata_builder, child);
+    }
 
-  /* For backwards compat we return a 0 installed size, its to slow to regenerate */
-  installed_size = 0;
 
-  return flatpak_dir_new_deploy_data (old_origin, commit, old_subpaths,
-                                      installed_size, NULL);
+  old_version = flatpak_deploy_data_get_version (deploy_data);
+  if (old_version < 1)
+    {
+      g_auto(GStrv) ref_parts = NULL;
+      ref_parts = g_strsplit (ref, "/", -1);
+      add_appdata_to_deploy_data (&metadata_builder, deploy_dir, ref_parts[1]);
+    }
+
+  subpaths = flatpak_deploy_data_get_subpaths (deploy_data);
+  return g_variant_ref_sink (g_variant_new ("(ss^ast@a{sv})",
+                                            flatpak_deploy_data_get_origin (deploy_data),
+                                            flatpak_deploy_data_get_commit (deploy_data),
+                                            subpaths,
+                                            GUINT64_TO_BE (flatpak_deploy_data_get_installed_size (deploy_data)),
+                                            g_variant_builder_end (&metadata_builder)));
 }
 
 GVariant *
@@ -6766,8 +6884,8 @@ flatpak_dir_deploy (FlatpakDir          *self,
   g_autoptr(GFile) real_checkoutdir = NULL;
   g_autoptr(GFile) dotref = NULL;
   g_autoptr(GFile) files_etc = NULL;
-  g_autoptr(GFile) metadata = NULL;
   g_autoptr(GFile) deploy_data_file = NULL;
+  g_autoptr(GVariant) commit_data = NULL;
   g_autoptr(GVariant) deploy_data = NULL;
   g_autoptr(GFile) export = NULL;
   g_autoptr(GFile) extradir = NULL;
@@ -6777,26 +6895,25 @@ flatpak_dir_deploy (FlatpakDir          *self,
   const char *checksum;
   glnx_autofd int checkoutdir_dfd = -1;
   g_autoptr(GFile) tmp_dir_template = NULL;
-  g_autoptr(GVariant) commit_data = NULL;
   g_autofree char *tmp_dir_path = NULL;
-  const char *alt_id = NULL;
-  const char *eol = NULL;
-  const char *eol_rebase = NULL;
   const char *xa_metadata = NULL;
   const char *xa_ref = NULL;
   g_autofree char *checkout_basename = NULL;
   gboolean created_extra_data = FALSE;
   g_autoptr(GVariant) commit_metadata = NULL;
-  GVariantBuilder metadata_builder;
   g_auto(GLnxLockFile) lock = { 0, };
   g_autoptr(GFile) metadata_file = NULL;
   g_autofree char *metadata_contents = NULL;
-  g_autofree char *application_runtime = NULL;
+  g_auto(GStrv) ref_parts = NULL;
   gboolean is_app;
 
   if (!flatpak_dir_ensure_repo (self, cancellable, error))
     return FALSE;
 
+  ref_parts = flatpak_decompose_ref (ref, error);
+  if (ref_parts == NULL)
+    return FALSE;
+
   /* Keep a shared repo lock to avoid prunes removing objects we're relying on
    * while we do the checkout. This could happen if the ref changes after we
    * read its current value for the checkout. */
@@ -6834,10 +6951,6 @@ flatpak_dir_deploy (FlatpakDir          *self,
     return FALSE;
 
   commit_metadata = g_variant_get_child_value (commit_data, 0);
-  g_variant_lookup (commit_metadata, "xa.alt-id", "&s", &alt_id);
-  g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_ENDOFLIFE, "&s", &eol);
-  g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_ENDOFLIFE_REBASE, "&s", &eol_rebase);
-
   checkout_basename = flatpak_dir_get_deploy_subdir (self, checksum, subpaths);
 
   real_checkoutdir = g_file_get_child (deploy_base, checkout_basename);
@@ -7032,21 +7145,17 @@ flatpak_dir_deploy (FlatpakDir          *self,
         }
     }
 
+  keyfile = g_key_file_new ();
   metadata_file = g_file_resolve_relative_path (checkoutdir, "metadata");
   if (g_file_load_contents (metadata_file, NULL,
                             &metadata_contents, NULL, NULL, NULL))
     {
-      g_autoptr(GKeyFile) keyfile = g_key_file_new ();
       if (!g_key_file_load_from_data (keyfile,
                                       metadata_contents,
                                       -1,
                                       0, error))
         return FALSE;
 
-      application_runtime = g_key_file_get_string (keyfile,
-                                                   FLATPAK_METADATA_GROUP_APPLICATION,
-                                                   FLATPAK_METADATA_KEY_RUNTIME, NULL);
-
       if (!flatpak_check_required_version (ref, keyfile, error))
         return FALSE;
     }
@@ -7071,16 +7180,6 @@ flatpak_dir_deploy (FlatpakDir          *self,
     return TRUE;
 
 
-  keyfile = g_key_file_new ();
-  metadata = g_file_get_child (checkoutdir, "metadata");
-  if (g_file_query_exists (metadata, cancellable))
-    {
-      g_autofree char *path = g_file_get_path (metadata);
-
-      if (!g_key_file_load_from_file (keyfile, path, G_KEY_FILE_NONE, error))
-        return FALSE;
-    }
-
   export = g_file_get_child (checkoutdir, "export");
 
   /* Never export any binaries bundled with the app */
@@ -7139,7 +7238,6 @@ flatpak_dir_deploy (FlatpakDir          *self,
     }
   else /* is app */
     {
-      g_auto(GStrv) ref_parts = g_strsplit (ref, "/", -1);
       g_autoptr(GFile) wrapper = g_file_get_child (bindir, ref_parts[1]);
       g_autofree char *escaped_app = maybe_quote (ref_parts[1]);
       g_autofree char *escaped_branch = maybe_quote (ref_parts[3]);
@@ -7170,25 +7268,15 @@ flatpak_dir_deploy (FlatpakDir          *self,
 
     }
 
-  g_variant_builder_init (&metadata_builder, G_VARIANT_TYPE ("a{sv}"));
-  if (alt_id)
-    g_variant_builder_add (&metadata_builder, "{s@v}", "alt-id",
-                           g_variant_new_variant (g_variant_new_string (alt_id)));
-  if (eol)
-    g_variant_builder_add (&metadata_builder, "{s@v}", "eol",
-                           g_variant_new_variant (g_variant_new_string (eol)));
-  if (eol_rebase)
-    g_variant_builder_add (&metadata_builder, "{s@v}", "eolr",
-                           g_variant_new_variant (g_variant_new_string (eol_rebase)));
-  if (application_runtime)
-    g_variant_builder_add (&metadata_builder, "{s@v}", "runtime",
-                           g_variant_new_variant (g_variant_new_string (application_runtime)));
-
-  deploy_data = flatpak_dir_new_deploy_data (origin,
+  deploy_data = flatpak_dir_new_deploy_data (self,
+                                             checkoutdir,
+                                             commit_metadata,
+                                             keyfile,
+                                             ref_parts[1],
+                                             origin,
                                              checksum,
                                              (char **) subpaths,
-                                             installed_size,
-                                             g_variant_builder_end (&metadata_builder));
+                                             installed_size);
 
   deploy_data_file = g_file_get_child (checkoutdir, "deploy");
   if (!flatpak_variant_save (deploy_data_file, deploy_data, cancellable, error))