From 1cbae3d1af69285b99465e85e2ab6d6b9b9bda35 Mon Sep 17 00:00:00 2001
From: "Owen W. Taylor" <otaylor@fishsoup.net>
Date: Thu, 12 Jul 2018 18:13:11 +0200
Subject: [PATCH] Cache the downloaded registry index in compressed form

The OCI index information should be highly compressable (especially if
icons are remote URI's rather than data URI's) so downloading it and
storing it compressed will provide sigificant efficiency gains.

Closes: #1910
Approved by: alexlarsson
---
 common/flatpak-dir.c          |  4 ++--
 common/flatpak-json-private.h |  4 ++++
 common/flatpak-json.c         | 21 +++++++++++++++++++++
 common/flatpak-oci-registry.c | 22 +++++++++++++++-------
 4 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c
index 245d16cc..673fa999 100644
--- a/common/flatpak-dir.c
+++ b/common/flatpak-dir.c
@@ -1026,7 +1026,7 @@ flatpak_dir_get_oci_index_location (FlatpakDir *self,
                                     const char *remote,
                                     GError    **error)
 {
-  return flatpak_dir_get_oci_cache_file (self, remote, ".index", error);
+  return flatpak_dir_get_oci_cache_file (self, remote, ".index.gz", error);
 }
 
 static GFile *
@@ -1060,7 +1060,7 @@ flatpak_dir_remove_oci_files (FlatpakDir   *self,
                               GCancellable *cancellable,
                               GError      **error)
 {
-  if (!flatpak_dir_remove_oci_file (self, remote, ".index", cancellable, error) ||
+  if (!flatpak_dir_remove_oci_file (self, remote, ".index.gz", cancellable, error) ||
       !flatpak_dir_remove_oci_file (self, remote, ".summary", cancellable, error))
     return FALSE;
 
diff --git a/common/flatpak-json-private.h b/common/flatpak-json-private.h
index e9e9b5f5..20b7b8e5 100644
--- a/common/flatpak-json-private.h
+++ b/common/flatpak-json-private.h
@@ -104,6 +104,10 @@ JsonNode   *flatpak_json_to_node (FlatpakJson *self);
 FlatpakJson *flatpak_json_from_bytes (GBytes  *bytes,
                                       GType    type,
                                       GError **error);
+FlatpakJson *flatpak_json_from_stream (GInputStream *stream,
+                                       GType         type,
+                                       GCancellable *cancellable,
+                                       GError      **error);
 GBytes     *flatpak_json_to_bytes (FlatpakJson *self);
 
 G_END_DECLS
diff --git a/common/flatpak-json.c b/common/flatpak-json.c
index 9ef32db3..c700982b 100644
--- a/common/flatpak-json.c
+++ b/common/flatpak-json.c
@@ -371,6 +371,27 @@ flatpak_json_from_bytes (GBytes  *bytes,
   return flatpak_json_from_node (root, type, error);
 }
 
+FlatpakJson *
+flatpak_json_from_stream (GInputStream *stream,
+                          GType         type,
+                          GCancellable *cancellable,
+                          GError      **error)
+{
+  g_autoptr(JsonParser) parser = NULL;
+  JsonNode *root = NULL;
+
+  parser = json_parser_new ();
+  if (!json_parser_load_from_stream (parser,
+                                     stream,
+                                     cancellable,
+                                     error))
+    return NULL;
+
+  root = json_parser_get_root (parser);
+
+  return flatpak_json_from_node (root, type, error);
+}
+
 static JsonNode *
 marshal (JsonObject          *parent,
          const char          *name,
diff --git a/common/flatpak-oci-registry.c b/common/flatpak-oci-registry.c
index 9bb15b37..4fa158c9 100644
--- a/common/flatpak-oci-registry.c
+++ b/common/flatpak-oci-registry.c
@@ -1955,7 +1955,7 @@ flatpak_oci_index_ensure_cached (SoupSession  *soup_session,
 
   if (!flatpak_cache_http_uri (soup_session,
                                query_uri,
-                               0,
+                               FLATPAK_HTTP_FLAGS_STORE_COMPRESSED,
                                AT_FDCWD, index_path,
                                NULL, NULL,
                                cancellable, error))
@@ -1969,19 +1969,27 @@ load_oci_index (GFile        *index,
                 GCancellable *cancellable,
                 GError      **error)
 {
-  g_autoptr(GMappedFile) mfile = NULL;
-  g_autoptr(GBytes) index_bytes = NULL;
+  g_autoptr(GFileInputStream) in = NULL;
+  g_autoptr(GZlibDecompressor) decompressor = NULL;
+  g_autoptr(GInputStream) converter = NULL;
+  g_autoptr(GError) local_error = NULL;
   g_autoptr(FlatpakJson) json = NULL;
 
-  mfile = g_mapped_file_new (flatpak_file_get_path_cached (index), FALSE, error);
-  if (!mfile)
+  in = g_file_read (index, cancellable, error);
+  if (in == NULL)
     return FALSE;
 
-  index_bytes = g_mapped_file_get_bytes (mfile);
-  json = flatpak_json_from_bytes (index_bytes, FLATPAK_TYPE_OCI_INDEX_RESPONSE, error);
+  decompressor = g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP);
+  converter = g_converter_input_stream_new (G_INPUT_STREAM (in), G_CONVERTER (decompressor));
+
+  json = flatpak_json_from_stream (G_INPUT_STREAM (converter), FLATPAK_TYPE_OCI_INDEX_RESPONSE,
+                                   cancellable, error);
   if (json == NULL)
     return NULL;
 
+  if (!g_input_stream_close (G_INPUT_STREAM (in), cancellable, error))
+    g_warning ("Error closing http stream: %s", local_error->message);
+
   return (FlatpakOciIndexResponse *) g_steal_pointer (&json);
 }