Allow using a compressed cache for HTTP downloads

Add a new flag for flatpak_cache_http_uri() that adds Accept-Encoding: gzip
to the request, and if the result is returned compressed, stores the data
compressed. If the data result is return uncompressed, it's compressed.

Closes: #1910
Approved by: alexlarsson

common/flatpak-utils-http-private.h

@@ -30,6 +30,7 @@ SoupSession * flatpak_create_soup_session (const char *user_agent);
 typedef enum {
   FLATPAK_HTTP_FLAGS_NONE = 0,
   FLATPAK_HTTP_FLAGS_ACCEPT_OCI = 1 << 0,
+  FLATPAK_HTTP_FLAGS_STORE_COMPRESSED = 2 << 0,
 } FlatpakHTTPFlags;
 
 typedef void (*FlatpakLoadUriProgress) (guint64  downloaded_bytes,

common/flatpak-utils-http.c

@@ -21,6 +21,7 @@
 #include "flatpak-utils-http-private.h"
 #include "flatpak-oci-registry-private.h"
 
+#include <gio/gunixoutputstream.h>
 #include <libsoup/soup.h>
 #include "libglnx/libglnx.h"
 
@@ -39,6 +40,7 @@ typedef struct
 {
   GMainLoop             *loop;
   GError                *error;
+  gboolean               store_compressed;
 
   GOutputStream         *out; /*or */
   GString               *content; /* or */
@@ -333,6 +335,17 @@ stream_closed (GObject *source, GAsyncResult *res, gpointer user_data)
   if (!g_input_stream_close_finish (stream, res, &error))
     g_warning ("Error closing http stream: %s", error->message);
 
+  if (data->out_tmpfile)
+    {
+      if (!g_output_stream_close (data->out, data->cancellable, &error))
+        {
+          if (data->error == NULL)
+            g_propagate_error (&data->error, g_steal_pointer (&error));
+        }
+
+      g_clear_pointer (&data->out, g_object_unref);
+    }
+
   g_main_loop_quit (data->loop);
 }
 
@@ -351,6 +364,7 @@ load_uri_read_cb (GObject *source, GAsyncResult *res, gpointer user_data)
       g_input_stream_close_async (stream,
                                   G_PRIORITY_DEFAULT, NULL,
                                   stream_closed, data);
+
       return;
     }
 
@@ -370,15 +384,6 @@ load_uri_read_cb (GObject *source, GAsyncResult *res, gpointer user_data)
 
       data->downloaded_bytes += n_written;
     }
-  else if (data->out_tmpfile != NULL)
-    {
-      if (glnx_loop_write (data->out_tmpfile->fd,
-                           data->buffer, nread) < 0)
-        {
-          glnx_throw_errno_prefix (&data->error, "write");
-          return;
-        }
-    }
   else
     {
       data->downloaded_bytes += nread;
@@ -452,10 +457,26 @@ load_uri_callback (GObject      *source_object,
 
   if (data->out_tmpfile)
     {
+      g_autoptr(GOutputStream) out = NULL;
+
       if (!glnx_open_tmpfile_linkable_at (data->out_tmpfile_parent_dfd, ".",
                                           O_WRONLY, data->out_tmpfile,
                                           &data->error))
         return;
+
+      g_assert (data->out == NULL);
+
+      out = g_unix_output_stream_new (data->out_tmpfile->fd, FALSE);
+      if (data->store_compressed &&
+          g_strcmp0 (soup_message_headers_get_one (msg->response_headers, "Content-Encoding"), "gzip") != 0)
+        {
+          g_autoptr(GZlibCompressor) compressor = g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, -1);
+          data->out = g_converter_output_stream_new (out, G_CONVERTER (compressor));
+        }
+      else
+        {
+          data->out = g_steal_pointer (&out);
+        }
     }
 
   g_input_stream_read_async (in, data->buffer, sizeof (data->buffer),
@@ -723,6 +744,13 @@ flatpak_cache_http_uri (SoupSession           *soup_session,
     soup_message_headers_replace (m->request_headers, "Accept",
                                   "application/vnd.oci.image.manifest.v1+json");
 
+  if (flags & FLATPAK_HTTP_FLAGS_STORE_COMPRESSED)
+    {
+      soup_message_headers_replace (m->request_headers, "Accept-Encoding",
+                                    "gzip");
+      data.store_compressed = TRUE;
+    }
+
   soup_request_send_async (SOUP_REQUEST (request),
                            cancellable,
                            load_uri_callback, &data);

tests/http-utils-test-server.py

@@ -3,9 +3,12 @@
 from wsgiref.handlers import format_date_time
 from email.utils import parsedate
 from calendar import timegm
+import gzip
 from urlparse import parse_qs
 import BaseHTTPServer
 import time
+import zlib
+from StringIO import StringIO
 
 server_start_time = int(time.time())
 
@@ -59,10 +62,23 @@ class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         if response == 200:
             self.send_header("Content-Type", "text/plain; charset=UTF-8")
 
+        contents = "path=" + self.path + "\n"
+
+        if not 'ignore-accept-encoding' in query:
+            accept_encoding = self.headers.get("Accept-Encoding")
+            if accept_encoding and accept_encoding == 'gzip':
+                self.send_header("Content-Encoding", "gzip")
+
+                buf = StringIO()
+                gzfile = gzip.GzipFile(mode='w', fileobj=buf)
+                gzfile.write(contents)
+                gzfile.close()
+                contents = buf.getvalue()
+
         self.end_headers()
 
         if response == 200:
-            self.wfile.write("path=" + self.path + "\n");
+            self.wfile.write(contents)
 
 def test():
     BaseHTTPServer.test(RequestHandler)

tests/httpcache.c

@@ -4,19 +4,32 @@ int
 main (int argc, char *argv[])
 {
   SoupSession *session = flatpak_create_soup_session (PACKAGE_STRING);
-  g_autoptr(GFile) dest = NULL;
   GError *error = NULL;
+  const char *url, *dest;
+  int flags = 0;
 
-  if (argc != 3)
+  if (argc == 3)
     {
-      g_printerr("Usage testhttp URL DEST\n");
+      url = argv[1];
+      dest = argv[2];
+    }
+  else if (argc == 4 && g_strcmp0 (argv[1], "--compressed") == 0)
+    {
+      url = argv[2];
+      dest = argv[3];
+      flags |= FLATPAK_HTTP_FLAGS_STORE_COMPRESSED;
+    }
+  else
+    {
+      g_printerr("Usage httpcache [--compressed] URL DEST\n");
       return 1;
     }
 
+
   if (!flatpak_cache_http_uri (session,
-			       argv[1],
-			       0,
-			       AT_FDCWD, argv[2],
+			       url,
+			       flags,
+			       AT_FDCWD, dest,
 			       NULL, NULL, NULL, &error))
     {
       g_print ("%s\n", error->message);

tests/test-http-utils.sh

@@ -28,10 +28,15 @@ port=$(cat httpd-port-main)
 
 assert_result() {
     test_string=$1
+    compressed=
+    if [ "$2" = "--compressed" ] ; then
+	compressed="--compressed"
+	shift
+    fi
     remote=$2
     local=$3
 
-    out=`httpcache "http://localhost:$port$remote" $local || :`
+    out=`httpcache $compressed "http://localhost:$port$remote" $local || :`
 
     case "$out" in
 	$test_string*)
@@ -62,7 +67,7 @@ have_xattrs() {
     setfattr -n user.testvalue -v somevalue $1/test-xattrs > /dev/null 2>&1
 }
 
-echo "1..4"
+echo "1..6"
 
 # Without anything else, cached for 30 minutes
 assert_ok "/" $test_tmpdir/output
@@ -107,12 +112,26 @@ rm -f $test_tmpdir/output*
 
 echo 'ok http revalidation'
 
+# Test compressd downloading and storage
+assert_ok --compressed "/compress" $test_tmpdir/output
+contents=$(gunzip -c < $test_tmpdir/output)
+assert_streq $contents path=/compress
+rm -f $test_tmpdir/output*
+
+echo 'ok compressed download'
+
+# Test uncompressed downloading with compressed storage
+assert_ok --compressed "/compress?ignore-accept-encoding" $test_tmpdir/output
+contents=$(gunzip -c < $test_tmpdir/output)
+assert_streq $contents path=/compress?ignore-accept-encoding
+rm -f $test_tmpdir/output*
+
+echo 'ok compress after download'
+
 # Testing that things work with without xattr support
 
 if have_xattrs $test_tmpdir ; then
-    ls $test_tmpdir 1>&2
     assert_ok "/?etag&no-cache" $test_tmpdir/output
-    ls $test_tmpdir 1>&2
     assert_not_has_file $test_tmpdir/output.flatpak.http
     assert_304 "/?etag&no-cache" $test_tmpdir/output
     rm -f $test_tmpdir/output*