diff --git a/common/flatpak-utils-http-private.h b/common/flatpak-utils-http-private.h index c8fa690f..4738f309 100644 --- a/common/flatpak-utils-http-private.h +++ b/common/flatpak-utils-http-private.h @@ -30,6 +30,7 @@ SoupSession * flatpak_create_soup_session (const char *user_agent); typedef enum { FLATPAK_HTTP_FLAGS_NONE = 0, FLATPAK_HTTP_FLAGS_ACCEPT_OCI = 1 << 0, + FLATPAK_HTTP_FLAGS_STORE_COMPRESSED = 2 << 0, } FlatpakHTTPFlags; typedef void (*FlatpakLoadUriProgress) (guint64 downloaded_bytes, diff --git a/common/flatpak-utils-http.c b/common/flatpak-utils-http.c index 30cba214..03b2215d 100644 --- a/common/flatpak-utils-http.c +++ b/common/flatpak-utils-http.c @@ -21,6 +21,7 @@ #include "flatpak-utils-http-private.h" #include "flatpak-oci-registry-private.h" +#include #include #include "libglnx/libglnx.h" @@ -39,6 +40,7 @@ typedef struct { GMainLoop *loop; GError *error; + gboolean store_compressed; GOutputStream *out; /*or */ GString *content; /* or */ @@ -333,6 +335,17 @@ stream_closed (GObject *source, GAsyncResult *res, gpointer user_data) if (!g_input_stream_close_finish (stream, res, &error)) g_warning ("Error closing http stream: %s", error->message); + if (data->out_tmpfile) + { + if (!g_output_stream_close (data->out, data->cancellable, &error)) + { + if (data->error == NULL) + g_propagate_error (&data->error, g_steal_pointer (&error)); + } + + g_clear_pointer (&data->out, g_object_unref); + } + g_main_loop_quit (data->loop); } @@ -351,6 +364,7 @@ load_uri_read_cb (GObject *source, GAsyncResult *res, gpointer user_data) g_input_stream_close_async (stream, G_PRIORITY_DEFAULT, NULL, stream_closed, data); + return; } @@ -370,15 +384,6 @@ load_uri_read_cb (GObject *source, GAsyncResult *res, gpointer user_data) data->downloaded_bytes += n_written; } - else if (data->out_tmpfile != NULL) - { - if (glnx_loop_write (data->out_tmpfile->fd, - data->buffer, nread) < 0) - { - glnx_throw_errno_prefix (&data->error, "write"); - return; - } - } else { data->downloaded_bytes += nread; @@ -452,10 +457,26 @@ load_uri_callback (GObject *source_object, if (data->out_tmpfile) { + g_autoptr(GOutputStream) out = NULL; + if (!glnx_open_tmpfile_linkable_at (data->out_tmpfile_parent_dfd, ".", O_WRONLY, data->out_tmpfile, &data->error)) return; + + g_assert (data->out == NULL); + + out = g_unix_output_stream_new (data->out_tmpfile->fd, FALSE); + if (data->store_compressed && + g_strcmp0 (soup_message_headers_get_one (msg->response_headers, "Content-Encoding"), "gzip") != 0) + { + g_autoptr(GZlibCompressor) compressor = g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, -1); + data->out = g_converter_output_stream_new (out, G_CONVERTER (compressor)); + } + else + { + data->out = g_steal_pointer (&out); + } } g_input_stream_read_async (in, data->buffer, sizeof (data->buffer), @@ -723,6 +744,13 @@ flatpak_cache_http_uri (SoupSession *soup_session, soup_message_headers_replace (m->request_headers, "Accept", "application/vnd.oci.image.manifest.v1+json"); + if (flags & FLATPAK_HTTP_FLAGS_STORE_COMPRESSED) + { + soup_message_headers_replace (m->request_headers, "Accept-Encoding", + "gzip"); + data.store_compressed = TRUE; + } + soup_request_send_async (SOUP_REQUEST (request), cancellable, load_uri_callback, &data); diff --git a/tests/http-utils-test-server.py b/tests/http-utils-test-server.py index d0f4ef85..9451e6ce 100644 --- a/tests/http-utils-test-server.py +++ b/tests/http-utils-test-server.py @@ -3,9 +3,12 @@ from wsgiref.handlers import format_date_time from email.utils import parsedate from calendar import timegm +import gzip from urlparse import parse_qs import BaseHTTPServer import time +import zlib +from StringIO import StringIO server_start_time = int(time.time()) @@ -59,10 +62,23 @@ class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): if response == 200: self.send_header("Content-Type", "text/plain; charset=UTF-8") + contents = "path=" + self.path + "\n" + + if not 'ignore-accept-encoding' in query: + accept_encoding = self.headers.get("Accept-Encoding") + if accept_encoding and accept_encoding == 'gzip': + self.send_header("Content-Encoding", "gzip") + + buf = StringIO() + gzfile = gzip.GzipFile(mode='w', fileobj=buf) + gzfile.write(contents) + gzfile.close() + contents = buf.getvalue() + self.end_headers() if response == 200: - self.wfile.write("path=" + self.path + "\n"); + self.wfile.write(contents) def test(): BaseHTTPServer.test(RequestHandler) diff --git a/tests/httpcache.c b/tests/httpcache.c index 321c99f8..3c4e9473 100644 --- a/tests/httpcache.c +++ b/tests/httpcache.c @@ -4,19 +4,32 @@ int main (int argc, char *argv[]) { SoupSession *session = flatpak_create_soup_session (PACKAGE_STRING); - g_autoptr(GFile) dest = NULL; GError *error = NULL; + const char *url, *dest; + int flags = 0; - if (argc != 3) + if (argc == 3) { - g_printerr("Usage testhttp URL DEST\n"); + url = argv[1]; + dest = argv[2]; + } + else if (argc == 4 && g_strcmp0 (argv[1], "--compressed") == 0) + { + url = argv[2]; + dest = argv[3]; + flags |= FLATPAK_HTTP_FLAGS_STORE_COMPRESSED; + } + else + { + g_printerr("Usage httpcache [--compressed] URL DEST\n"); return 1; } + if (!flatpak_cache_http_uri (session, - argv[1], - 0, - AT_FDCWD, argv[2], + url, + flags, + AT_FDCWD, dest, NULL, NULL, NULL, &error)) { g_print ("%s\n", error->message); diff --git a/tests/test-http-utils.sh b/tests/test-http-utils.sh index 6a37c742..ad7599d3 100755 --- a/tests/test-http-utils.sh +++ b/tests/test-http-utils.sh @@ -28,10 +28,15 @@ port=$(cat httpd-port-main) assert_result() { test_string=$1 + compressed= + if [ "$2" = "--compressed" ] ; then + compressed="--compressed" + shift + fi remote=$2 local=$3 - out=`httpcache "http://localhost:$port$remote" $local || :` + out=`httpcache $compressed "http://localhost:$port$remote" $local || :` case "$out" in $test_string*) @@ -62,7 +67,7 @@ have_xattrs() { setfattr -n user.testvalue -v somevalue $1/test-xattrs > /dev/null 2>&1 } -echo "1..4" +echo "1..6" # Without anything else, cached for 30 minutes assert_ok "/" $test_tmpdir/output @@ -107,12 +112,26 @@ rm -f $test_tmpdir/output* echo 'ok http revalidation' +# Test compressd downloading and storage +assert_ok --compressed "/compress" $test_tmpdir/output +contents=$(gunzip -c < $test_tmpdir/output) +assert_streq $contents path=/compress +rm -f $test_tmpdir/output* + +echo 'ok compressed download' + +# Test uncompressed downloading with compressed storage +assert_ok --compressed "/compress?ignore-accept-encoding" $test_tmpdir/output +contents=$(gunzip -c < $test_tmpdir/output) +assert_streq $contents path=/compress?ignore-accept-encoding +rm -f $test_tmpdir/output* + +echo 'ok compress after download' + # Testing that things work with without xattr support if have_xattrs $test_tmpdir ; then - ls $test_tmpdir 1>&2 assert_ok "/?etag&no-cache" $test_tmpdir/output - ls $test_tmpdir 1>&2 assert_not_has_file $test_tmpdir/output.flatpak.http assert_304 "/?etag&no-cache" $test_tmpdir/output rm -f $test_tmpdir/output*