image-source: Support oci-archive: image sources

Add support for `oci-archive:` image sources by temporarily
unpacking the archive using libarchive.

Co-authored-by: Sebastian Wick <sebastian.wick@redhat.com>

app/flatpak-builtins-install.c

@@ -352,6 +352,7 @@ flatpak_builtin_install (int argc, char **argv, GCancellable *cancellable, GErro
   if (!opt_bundle && !opt_from && !opt_image && argc >= 2)
     {
       if (g_str_has_prefix (argv[1], "oci:") ||
+          g_str_has_prefix (argv[1], "oci-archive:") ||
           g_str_has_prefix (argv[1], "docker:"))
         opt_image = TRUE;
       else if (flatpak_file_arg_has_suffix (argv[1], ".flatpakref"))

common/flatpak-image-source.c

@@ -119,24 +119,15 @@ flatpak_image_source_new (FlatpakOciRegistry *registry,
   return g_steal_pointer (&self);
 }
 
-FlatpakImageSource *
-flatpak_image_source_new_local (GFile        *file,
-                                const char   *reference,
-                                GCancellable *cancellable,
-                                GError      **error)
+static FlatpakImageSource *
+flatpak_image_source_new_local_for_registry (FlatpakOciRegistry *registry,
+                                             const char         *reference,
+                                             GCancellable       *cancellable,
+                                             GError            **error)
 {
-  g_autofree char *dir_uri = NULL;
-  g_autofree char *target_ref = NULL;
-  g_autoptr(FlatpakImageSource) image_source = NULL;
-  g_autoptr(FlatpakOciRegistry) registry = NULL;
   g_autoptr(FlatpakOciIndex) index = NULL;
   const FlatpakOciManifestDescriptor *desc;
 
-  dir_uri = g_file_get_uri (file);
-  registry = flatpak_oci_registry_new (dir_uri, FALSE, -1, cancellable, error);
-  if (registry == NULL)
-    return NULL;
-
   index = flatpak_oci_registry_load_index (registry, cancellable, error);
   if (index == NULL)
     return NULL;
@@ -163,6 +154,23 @@ flatpak_image_source_new_local (GFile        *file,
   return flatpak_image_source_new (registry, NULL, desc->parent.digest, cancellable, error);
 }
 
+FlatpakImageSource *
+flatpak_image_source_new_local (GFile        *file,
+                                const char   *reference,
+                                GCancellable *cancellable,
+                                GError      **error)
+{
+  g_autofree char *dir_uri = NULL;
+  g_autoptr(FlatpakOciRegistry) registry = NULL;
+
+  dir_uri = g_file_get_uri (file);
+  registry = flatpak_oci_registry_new (dir_uri, FALSE, -1, cancellable, error);
+  if (registry == NULL)
+    return NULL;
+
+  return flatpak_image_source_new_local_for_registry (registry, reference, cancellable, error);
+}
+
 FlatpakImageSource *
 flatpak_image_source_new_remote (const char   *uri,
                                  const char   *oci_repository,
@@ -225,6 +233,20 @@ flatpak_image_source_new_for_location (const char   *location,
 
       return flatpak_image_source_new_local (path, reference, cancellable, error);
     }
+  else if (g_str_has_prefix (location, "oci-archive:"))
+    {
+      g_autoptr(FlatpakOciRegistry) registry = NULL;
+      g_autoptr(GFile) path = NULL;
+      g_autofree char *reference = NULL;
+
+      get_path_and_reference (location, &path, &reference);
+
+      registry = flatpak_oci_registry_new_for_archive (path, cancellable, error);
+      if (registry == NULL)
+        return NULL;
+
+      return flatpak_image_source_new_local_for_registry (registry, reference, cancellable, error);
+    }
   else if (g_str_has_prefix (location, "docker:"))
     {
       g_autoptr(FlatpakOciRegistry) registry = NULL;

common/flatpak-oci-registry-private.h

@@ -50,11 +50,15 @@ typedef struct FlatpakOciLayerWriter FlatpakOciLayerWriter;
 
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakOciLayerWriter, g_object_unref)
 
-FlatpakOciRegistry  *  flatpak_oci_registry_new (const char           *uri,
-                                                 gboolean for_write,
-                                                 int tmp_dfd,
-                                                 GCancellable         * cancellable,
-                                                 GError              **error);
+
+FlatpakOciRegistry  *  flatpak_oci_registry_new (const char    *uri,
+                                                 gboolean       for_write,
+                                                 int            tmp_dfd,
+                                                 GCancellable  *cancellable,
+                                                 GError       **error);
+FlatpakOciRegistry *   flatpak_oci_registry_new_for_archive (GFile        *archive,
+                                                             GCancellable *cancellable,
+                                                             GError      **error);
 void                   flatpak_oci_registry_set_token (FlatpakOciRegistry *self,
                                                        const char *token);
 gboolean               flatpak_oci_registry_is_local (FlatpakOciRegistry *self);

common/flatpak-oci-registry.c

@@ -27,6 +27,7 @@
 #include "libglnx.h"
 
 #include <archive.h>
+#include <archive_entry.h>
 #include <gpgme.h>
 #include "flatpak-image-source-private.h"
 #include "flatpak-oci-registry-private.h"
@@ -69,11 +70,13 @@ struct FlatpakOciRegistry
   gboolean valid;
   gboolean is_docker;
   char    *uri;
+  GFile   *archive;
   int      tmp_dfd;
   char    *token;
 
   /* Local repos */
   int dfd;
+  GLnxTmpDir *tmp_dir;
 
   /* Remote repos */
   FlatpakHttpSession *http_session;
@@ -90,6 +93,7 @@ enum {
   PROP_0,
 
   PROP_URI,
+  PROP_ARCHIVE,
   PROP_FOR_WRITE,
   PROP_TMP_DFD,
 };
@@ -98,6 +102,14 @@ G_DEFINE_TYPE_WITH_CODE (FlatpakOciRegistry, flatpak_oci_registry, G_TYPE_OBJECT
                          G_IMPLEMENT_INTERFACE (G_TYPE_INITABLE,
                                                 flatpak_oci_registry_initable_iface_init))
 
+static void
+glnx_tmpdir_free (GLnxTmpDir *tmpf)
+{
+  (void)glnx_tmpdir_delete (tmpf, NULL, NULL);
+  g_free (tmpf);
+}
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(GLnxTmpDir, glnx_tmpdir_free)
+
 static gchar *
 parse_relative_uri (GUri *base_uri,
                     const char *subpath,
@@ -124,6 +136,8 @@ flatpak_oci_registry_finalize (GObject *object)
   g_clear_pointer (&self->base_uri, g_uri_unref);
   g_free (self->uri);
   g_free (self->token);
+  g_clear_object (&self->archive);
+  g_clear_pointer (&self->tmp_dir, glnx_tmpdir_free);
 
   G_OBJECT_CLASS (flatpak_oci_registry_parent_class)->finalize (object);
 }
@@ -151,6 +165,10 @@ flatpak_oci_registry_set_property (GObject      *object,
         }
       break;
 
+    case PROP_ARCHIVE:
+      self->archive = g_value_dup_object (value);
+      break;
+
     case PROP_FOR_WRITE:
       self->for_write = g_value_get_boolean (value);
       break;
@@ -179,6 +197,10 @@ flatpak_oci_registry_get_property (GObject    *object,
       g_value_set_string (value, self->uri);
       break;
 
+    case PROP_ARCHIVE:
+      g_value_set_object (value, self->archive);
+      break;
+
     case PROP_FOR_WRITE:
       g_value_set_boolean (value, self->for_write);
       break;
@@ -209,6 +231,13 @@ flatpak_oci_registry_class_init (FlatpakOciRegistryClass *klass)
                                                         "",
                                                         NULL,
                                                         G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
+  g_object_class_install_property (object_class,
+                                   PROP_ARCHIVE,
+                                   g_param_spec_object ("archive",
+                                                        "",
+                                                        "",
+                                                        G_TYPE_FILE,
+                                                        G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
   g_object_class_install_property (object_class,
                                    PROP_TMP_DFD,
                                    g_param_spec_int ("tmp-dfd",
@@ -277,6 +306,21 @@ flatpak_oci_registry_new (const char   *uri,
   return oci_registry;
 }
 
+FlatpakOciRegistry *
+flatpak_oci_registry_new_for_archive (GFile        *archive,
+                                      GCancellable *cancellable,
+                                      GError      **error)
+{
+  FlatpakOciRegistry *oci_registry;
+
+  oci_registry = g_initable_new (FLATPAK_TYPE_OCI_REGISTRY,
+                                 cancellable, error,
+                                 "archive", archive,
+                                 NULL);
+
+  return oci_registry;
+}
+
 static int
 local_open_file (int           dfd,
                  const char   *subpath,
@@ -457,13 +501,164 @@ verify_oci_version (GBytes *oci_layout_bytes, gboolean *not_json, GCancellable *
   return TRUE;
 }
 
+/*
+ * Code to extract an archive such as a tarfile into a temporary directory
+ *
+ * Based on: https://github.com/libarchive/libarchive/wiki/Examples#A_Complete_Extractor
+ *
+ * We treat ARCHIVE_WARNING as fatal - while this might be too strict, it
+ * will avoid surprises.
+ */
+
+static gboolean
+propagate_libarchive_error (GError         **error,
+                            struct archive  *a)
+{
+  g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+               "%s", archive_error_string (a));
+  return FALSE;
+}
+
+static gboolean
+copy_data (struct archive  *ar,
+           struct archive  *aw,
+           GError         **error)
+{
+  int r;
+  const void *buff;
+  size_t size;
+  gint64 offset;
+
+  while (TRUE)
+    {
+      r = archive_read_data_block (ar, &buff, &size, &offset);
+
+      if (r == ARCHIVE_EOF)
+        return TRUE;
+
+      if (r == ARCHIVE_RETRY)
+        continue;
+
+      if (r != ARCHIVE_OK)
+        return propagate_libarchive_error (error, ar);
+
+      while (TRUE)
+        {
+          r = archive_write_data_block (aw, buff, size, offset);
+
+          if (r == ARCHIVE_RETRY)
+            continue;
+
+          if (r == ARCHIVE_OK)
+            break;
+
+          return propagate_libarchive_error (error, aw);
+        }
+    }
+}
+
+static gboolean
+unpack_archive (GFile   *archive,
+                char    *destination,
+                GError **error)
+{
+  g_autoptr(FlatpakAutoArchiveRead) a = NULL;
+  g_autoptr(FlatpakAutoArchiveWrite) ext = NULL;
+  int flags;
+  int r;
+
+  flags = 0;
+  flags |= ARCHIVE_EXTRACT_SECURE_NODOTDOT;
+  flags |= ARCHIVE_EXTRACT_SECURE_SYMLINKS;
+
+  a = archive_read_new ();
+  archive_read_support_format_all (a);
+  archive_read_support_filter_all (a);
+
+  ext = archive_write_disk_new ();
+  archive_write_disk_set_options (ext, flags);
+  archive_write_disk_set_standard_lookup (ext);
+
+  r = archive_read_open_filename (a, g_file_get_path(archive), 10240);
+  if (r != ARCHIVE_OK)
+    return propagate_libarchive_error (error, a);
+
+  while (TRUE)
+    {
+      g_autofree char *target_path = NULL;
+      struct archive_entry *entry;
+
+      r = archive_read_next_header (a, &entry);
+      if (r == ARCHIVE_EOF)
+        break;
+
+      if (r != ARCHIVE_OK)
+        return propagate_libarchive_error (error, a);
+
+      target_path = g_build_filename (destination, archive_entry_pathname (entry), NULL);
+      archive_entry_set_pathname (entry, target_path);
+
+      r = archive_write_header (ext, entry);
+      if (r != ARCHIVE_OK)
+        return propagate_libarchive_error (error, ext);
+
+      if (archive_entry_size (entry) > 0)
+        {
+          if (!copy_data (a, ext, error))
+            return FALSE;
+        }
+
+      r = archive_write_finish_entry (ext);
+      if (r != ARCHIVE_OK)
+        return propagate_libarchive_error (error, ext);
+    }
+
+  r = archive_read_close (a);
+  if (r != ARCHIVE_OK)
+    return propagate_libarchive_error (error, a);
+
+  r = archive_write_close (ext);
+  if (r != ARCHIVE_OK)
+    return propagate_libarchive_error (error, ext);
+
+  return TRUE;
+}
+
+static const char *
+get_download_tmpdir (void)
+{
+  /* We don't use TMPDIR because the downloaded artifacts can be
+   * very big, and we want to prefer /var/tmp to /tmp.
+   */
+  const char *tmpdir = g_getenv ("FLATPAK_DOWNLOAD_TMPDIR");
+  if (tmpdir)
+    return tmpdir;
+
+  return "/var/tmp";
+}
+
+static GLnxTmpDir *
+download_tmpdir_new (GError **error)
+{
+  g_autoptr(GLnxTmpDir) tmp_dir = g_new0 (GLnxTmpDir, 1);
+  glnx_autofd int base_dfd = -1;
+
+  if (!glnx_opendirat (AT_FDCWD, get_download_tmpdir (), TRUE, &base_dfd, error))
+    return NULL;
+
+  if (!glnx_mkdtempat (base_dfd, "oci-XXXXXX", 0700, tmp_dir, error))
+    return NULL;
+
+  return g_steal_pointer (&tmp_dir);
+}
+
 static gboolean
 flatpak_oci_registry_ensure_local (FlatpakOciRegistry *self,
                                    gboolean            for_write,
                                    GCancellable       *cancellable,
                                    GError            **error)
 {
-  g_autoptr(GFile) dir = g_file_new_for_uri (self->uri);
+  g_autoptr(GLnxTmpDir) local_tmp_dir = NULL;
   glnx_autofd int local_dfd = -1;
   int dfd;
   g_autoptr(GError) local_error = NULL;
@@ -472,9 +667,28 @@ flatpak_oci_registry_ensure_local (FlatpakOciRegistry *self,
   gboolean not_json;
 
   if (self->dfd != -1)
-    dfd = self->dfd;
+    {
+      dfd = self->dfd;
+    }
+  else if (self->archive)
+    {
+      local_tmp_dir = download_tmpdir_new (error);
+      if (!local_tmp_dir)
+        return FALSE;
+
+      if (!unpack_archive (self->archive, local_tmp_dir->path, error))
+        return FALSE;
+
+      if (!glnx_opendirat (AT_FDCWD, local_tmp_dir->path,
+                           TRUE, &local_dfd, error))
+        return FALSE;
+
+      dfd = local_dfd;
+    }
   else
     {
+      g_autoptr(GFile) dir = g_file_new_for_uri (self->uri);
+
       if (!glnx_opendirat (AT_FDCWD, flatpak_file_get_path_cached (dir),
                            TRUE, &local_dfd, &local_error))
         {
@@ -537,8 +751,11 @@ flatpak_oci_registry_ensure_local (FlatpakOciRegistry *self,
         self->token = g_strndup (g_bytes_get_data (token_bytes, NULL), g_bytes_get_size (token_bytes));
     }
 
-  if (self->dfd == -1 && local_dfd != -1)
-    self->dfd = g_steal_fd (&local_dfd);
+  if (self->dfd == -1)
+    {
+      self->dfd = g_steal_fd (&local_dfd);
+      self->tmp_dir = g_steal_pointer (&local_tmp_dir);
+    }
 
   return TRUE;
 }
@@ -589,20 +806,15 @@ flatpak_oci_registry_initable_init (GInitable    *initable,
   FlatpakOciRegistry *self = FLATPAK_OCI_REGISTRY (initable);
   gboolean res;
 
+  g_warn_if_fail (self->archive || self->uri);
+
   if (self->tmp_dfd == -1)
     {
-      /* We don't use TMPDIR because the downloaded artifacts can be
-       * very big, and we want to prefer /var/tmp to /tmp.
-       */
-      const char *tmpdir = g_getenv ("FLATPAK_DOWNLOAD_TMPDIR");
-      if (tmpdir == NULL)
-        tmpdir = "/var/tmp";
-
-      if (!glnx_opendirat (AT_FDCWD, tmpdir, TRUE, &self->tmp_dfd, error))
+      if (!glnx_opendirat (AT_FDCWD, get_download_tmpdir (), TRUE, &self->tmp_dfd, error))
         return FALSE;
     }
 
-  if (g_str_has_prefix (self->uri, "file:/"))
+  if (self->archive || g_str_has_prefix (self->uri, "file:/"))
     res = flatpak_oci_registry_ensure_local (self, self->for_write, cancellable, error);
   else
     res = flatpak_oci_registry_ensure_remote (self, self->for_write, cancellable, error);
@@ -1332,15 +1544,6 @@ typedef struct
 
 G_DEFINE_TYPE (FlatpakOciLayerWriter, flatpak_oci_layer_writer, G_TYPE_OBJECT)
 
-static gboolean
-propagate_libarchive_error (GError        **error,
-                            struct archive *a)
-{
-  g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-               "%s", archive_error_string (a));
-  return FALSE;
-}
-
 static void
 flatpak_oci_layer_writer_reset (FlatpakOciLayerWriter *self)
 {

common/flatpak-transaction.c

@@ -3028,7 +3028,7 @@ flatpak_transaction_add_install_bundle (FlatpakTransaction *self,
  * reference.
  *
  * @image_location is specified in containers-transports(5) form. Only a subset
- * of transports are supported: oci: and docker:.
+ * of transports are supported: oci:, oci-archive:, and docker:.
  *
  * Returns: %TRUE on success; %FALSE with @error set on failure.
  */

doc/flatpak-install.xml

@@ -145,8 +145,8 @@
                   Treat <arg choice="plain">LOCATION</arg> as the location of a Flatpak in
                   OCI image format. <arg choice="plan>">LOCATION</arg> is in the format of
                   <citerefentry><refentrytitle>containers-transports</refentrytitle><manvolnum>5</manvolnum></citerefentry>.
-                  Supported schemes are <literal>docker://</literal> and <literal>oci:</literal>.
-                  This is assumed if the argument starts
+                  Supported schemes are <literal>docker://</literal>, <literal>oci:</literal>,
+                  and <literal>oci-archive:</literal>. This is assumed if the argument starts
                   with one of these schemes.
                 </para></listitem>
             </varlistentry>

tests/test-oci.sh

@@ -23,7 +23,7 @@ set -euo pipefail
 
 skip_without_bwrap
 
-echo "1..3"
+echo "1..4"
 
 setup_repo_no_add oci
 
@@ -85,4 +85,26 @@ assert_file_has_content flatpak-list '^org.test.Platform	*platform-origin$'
 ${FLATPAK} --user remotes --show-disabled > remotes-list
 assert_file_has_content remotes-list '^platform-origin'
 
+${FLATPAK} ${U} -y uninstall org.test.Platform >&2
+
 ok "install oci"
+
+# Trying installing an OCI archive bundle
+
+(cd oci/platform-image && tar cf - .) > oci/platform-image.tar
+
+${FLATPAK} --user list --columns=application,origin > flatpak-list
+assert_not_file_has_content flatpak-list 'org.test.Platform'
+
+${FLATPAK} --user remotes --show-disabled > remotes-list
+assert_not_file_has_content remotes-list '^platform-origin'
+
+$FLATPAK --user -y install oci-archive:oci/platform-image.tar >&2
+
+${FLATPAK} --user list --columns=application,origin > flatpak-list
+assert_file_has_content flatpak-list '^org.test.Platform	*platform-origin$'
+
+${FLATPAK} --user remotes --show-disabled > remotes-list
+assert_file_has_content remotes-list '^platform-origin'
+
+ok "install oci archive"