From 951aed561ab08d9dd94919cbd6db5a7a0cb5e209 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Mon, 2 Jul 2018 13:00:48 -0400 Subject: [PATCH] Add flatpak_cache_http_uri: cache downloads based on HTTP headers Add a new function, flatpak_cache_http_uri() that when passed an URL and a local destination location, either a) downloads the content and stores it at the destination location, storing HTTP cache header information like Last-Modified, Etag into user xattrs (if available) or a separate file or b) if the downloaded content is already present, checks the header information to decide whether the downloaded content can be used or needs to be revalidated witha conditional request. Tests are added that use a special case test server that adds HTTP caching headers and reacts to them based on query parameters. A small test binary 'httpcache' is added for the tests to use. Closes: #1910 Approved by: alexlarsson --- .gitignore | 1 + common/flatpak-utils-http-private.h | 9 + common/flatpak-utils-http.c | 502 +++++++++++++++++++++++++++- tests/Makefile.am.inc | 10 + tests/http-utils-test-server.py | 71 ++++ tests/httpcache.c | 30 ++ tests/test-http-utils.sh | 141 ++++++++ tests/test-webserver.sh | 5 +- 8 files changed, 764 insertions(+), 5 deletions(-) create mode 100644 tests/http-utils-test-server.py create mode 100644 tests/httpcache.c create mode 100755 tests/test-http-utils.sh diff --git a/.gitignore b/.gitignore index 762e5048..d1f57cd3 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ flatpak.pc common/flatpak-enum-types.c common/flatpak-enum-types.h test-libflatpak +httpcache Flatpak-1.0.* /app/parse-datetime.c /doc/reference/gtkdoc-check.log diff --git a/common/flatpak-utils-http-private.h b/common/flatpak-utils-http-private.h index 6b0e3bf2..84af1d49 100644 --- a/common/flatpak-utils-http-private.h +++ b/common/flatpak-utils-http-private.h @@ -52,5 +52,14 @@ gboolean flatpak_download_http_uri (SoupSession *soup_session, gpointer user_data, GCancellable *cancellable, GError **error); +gboolean flatpak_cache_http_uri (SoupSession *soup_session, + const char *uri, + FlatpakHTTPFlags flags, + int dest_dfd, + const char *dest_subpath, + FlatpakLoadUriProgress progress, + gpointer user_data, + GCancellable *cancellable, + GError **error); #endif /* __FLATPAK_UTILS_HTTP_H__ */ diff --git a/common/flatpak-utils-http.c b/common/flatpak-utils-http.c index 63c1c6ec..d89de299 100644 --- a/common/flatpak-utils-http.c +++ b/common/flatpak-utils-http.c @@ -1,5 +1,5 @@ /* - * Copyright © 2014 Red Hat, Inc + * Copyright © 2018 Red Hat, Inc * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -22,22 +22,307 @@ #include "flatpak-oci-registry-private.h" #include +#include "libglnx/libglnx.h" + +#include +#include + +typedef struct +{ + char *uri; + char *etag; + gint64 last_modified; + gint64 expires; +} CacheHttpData; typedef struct { GMainLoop *loop; GError *error; - GOutputStream *out; + + GOutputStream *out; /*or */ + GString *content; /* or */ + GLnxTmpfile *out_tmpfile; + int out_tmpfile_parent_dfd; + guint64 downloaded_bytes; - GString *content; char buffer[16 * 1024]; FlatpakLoadUriProgress progress; GCancellable *cancellable; gpointer user_data; guint64 last_progress_time; + CacheHttpData *cache_data; char *etag; } LoadUriData; +#define CACHE_HTTP_XATTR "user.flatpak.http" +#define CACHE_HTTP_SUFFIX ".flatpak.http" +#define CACHE_HTTP_TYPE "(sstt)" + +static void +clear_cache_http_data (CacheHttpData *data, + gboolean clear_uri) +{ + if (clear_uri) + g_clear_pointer (&data->uri, g_free); + g_clear_pointer (&data->etag, g_free); + data->last_modified = 0; + data->expires = 0; +} + +static void +free_cache_http_data (CacheHttpData *data) +{ + clear_cache_http_data (data, TRUE); + g_free (data); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (CacheHttpData, free_cache_http_data) + +static GBytes * +serialize_cache_http_data (CacheHttpData * data) +{ + g_autoptr(GVariant) cache_variant = NULL; + + cache_variant = g_variant_ref_sink (g_variant_new (CACHE_HTTP_TYPE, + data->uri, + data->etag ? data->etag : "", + data->last_modified, + data->expires)); + if (G_BYTE_ORDER != G_BIG_ENDIAN) + { + g_autoptr(GVariant) tmp_variant = cache_variant; + cache_variant = g_variant_byteswap (tmp_variant); + } + + return g_variant_get_data_as_bytes (cache_variant); +} + +static void +deserialize_cache_http_data (CacheHttpData *data, + GBytes *bytes) +{ + g_autoptr(GVariant) cache_variant = NULL; + + cache_variant = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE (CACHE_HTTP_TYPE), + bytes, + FALSE)); + if (G_BYTE_ORDER != G_BIG_ENDIAN) + { + g_autoptr(GVariant) tmp_variant = cache_variant; + cache_variant = g_variant_byteswap (tmp_variant); + } + + g_variant_get (cache_variant, + CACHE_HTTP_TYPE, + &data->uri, + &data->etag, + &data->last_modified, + &data->expires); +} + +static CacheHttpData * +load_cache_http_data (int dfd, + char *name, + gboolean *no_xattr, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(CacheHttpData) data = NULL; + + g_autoptr(GBytes) cache_bytes = glnx_lgetxattrat (dfd, name, + CACHE_HTTP_XATTR, + error); + if (cache_bytes == NULL) + { + if (errno == ENOTSUP) + { + g_autofree char *cache_file = NULL; + glnx_autofd int fd = -1; + + g_clear_error (error); + *no_xattr = TRUE; + + cache_file = g_strconcat (name, CACHE_HTTP_SUFFIX, NULL); + + if (!glnx_openat_rdonly (dfd, cache_file, FALSE, + &fd, error)) + return FALSE; + + cache_bytes = glnx_fd_readall_bytes (fd, cancellable, error); + if (!cache_bytes) + return NULL; + } + else if (errno == ENOENT || errno == ENODATA) + { + g_clear_error (error); + return g_new0 (CacheHttpData, 1); + } + else + { + return NULL; + } + } + + + data = g_new0 (CacheHttpData, 1); + deserialize_cache_http_data (data, cache_bytes); + return g_steal_pointer (&data); +} + +static void +set_cache_http_data_from_headers (CacheHttpData *data, + SoupMessage *msg) +{ + const char *etag = soup_message_headers_get_one (msg->response_headers, "ETag"); + const char *last_modified = soup_message_headers_get_one (msg->response_headers, "Last-Modified"); + const char *cache_control = soup_message_headers_get_list (msg->response_headers, "Cache-Control"); + const char *expires = soup_message_headers_get_list (msg->response_headers, "Expires"); + gboolean expires_computed = FALSE; + + /* The original HTTP 1/1 specification only required sending the ETag header in a 304 + * response, and implied that a cache might need to save the old Cache-Control + * values. The updated RFC 7232 from 2014 requires sending Cache-Control, ETags, and + * Expire if they would have been sent in the original 200 response, and recommends + * sending Last-Modified for requests without an etag. Since sending these headers was + * apparently normal previously, for simplicity we assume the RFC 7232 behavior and start + * from scratch for a 304 response. + */ + clear_cache_http_data (data, FALSE); + + if (etag && *etag) + { + data->etag = g_strdup (etag); + } + else if (last_modified && *last_modified) + { + SoupDate *date = soup_date_new_from_string (last_modified); + if (date) + { + data->last_modified = soup_date_to_time_t (date); + soup_date_free (date); + } + } + + if (cache_control && *cache_control) + { + GHashTable *params = soup_header_parse_param_list (cache_control); + GHashTableIter iter; + gpointer key, value; + + g_hash_table_iter_init (&iter, params); + while (g_hash_table_iter_next (&iter, &key, &value)) + { + if (g_strcmp0 (key, "max-age") == 0) + { + char *end; + + char *max_age = value; + int max_age_sec = g_ascii_strtoll (max_age, &end, 10); + if (*max_age != '\0' && *end == '\0') + { + GTimeVal now; + g_get_current_time (&now); + data->expires = now.tv_sec + max_age_sec; + expires_computed = TRUE; + } + } + else if (g_strcmp0 (key, "no-cache") == 0) + { + data->expires = 0; + expires_computed = TRUE; + } + } + } + + if (!expires_computed && expires && *expires) + { + SoupDate *date = soup_date_new_from_string (expires); + if (date) + { + data->expires = soup_date_to_time_t (date); + soup_date_free (date); + expires_computed = TRUE; + } + } + + if (!expires_computed) + { + /* If nothing implies an expires time, use 30 minutes. Browsers use + * 0.1 * (Date - Last-Modified), but it's clearly appropriate here, and + * better if server's send a value. + */ + GTimeVal now; + g_get_current_time (&now); + data->expires = now.tv_sec + 1800; + } +} + +static gboolean +save_cache_http_data_xattr (int fd, + GBytes *bytes, + GError **error) +{ + if (TEMP_FAILURE_RETRY (fsetxattr (fd, (char *) CACHE_HTTP_XATTR, + g_bytes_get_data (bytes, NULL), + g_bytes_get_size (bytes), + 0)) < 0) + return glnx_throw_errno_prefix (error, "fsetxattr"); + + return TRUE; +} + +static gboolean +save_cache_http_data_fallback (int fd, + GBytes *bytes, + GError **error) +{ + if (glnx_loop_write (fd, + g_bytes_get_data (bytes, NULL), + g_bytes_get_size (bytes)) < 0) + return glnx_throw_errno_prefix (error, "write"); + + return TRUE; +} + +static gboolean +save_cache_http_data_to_file (int dfd, + char *name, + GBytes *bytes, + gboolean no_xattr, + GCancellable *cancellable, + GError **error) +{ + glnx_autofd int fd = -1; + g_autofree char *fallback_name = NULL; + + if (!no_xattr) + { + if (!glnx_openat_rdonly (dfd, name, FALSE, + &fd, error)) + return FALSE; + + if (save_cache_http_data_xattr (fd, bytes, error)) + return TRUE; + + if (errno == ENOTSUP) + g_clear_error (error); + else + return FALSE; + } + + fallback_name = g_strconcat (name, CACHE_HTTP_SUFFIX, NULL); + if (!glnx_file_replace_contents_at (dfd, fallback_name, + g_bytes_get_data (bytes, NULL), + g_bytes_get_size (bytes), + 0, + cancellable, + error)) + return FALSE; + + return TRUE; +} + static void stream_closed (GObject *source, GAsyncResult *res, gpointer user_data) { @@ -86,6 +371,15 @@ load_uri_read_cb (GObject *source, GAsyncResult *res, gpointer user_data) data->downloaded_bytes += n_written; } + else if (data->out_tmpfile != NULL) + { + if (glnx_loop_write (data->out_tmpfile->fd, + data->buffer, nread) < 0) + { + glnx_throw_errno_prefix (&data->error, "write"); + return; + } + } else { data->downloaded_bytes += nread; @@ -130,6 +424,9 @@ load_uri_callback (GObject *source_object, switch (msg->status_code) { case 304: + if (data->cache_data) + set_cache_http_data_from_headers (data->cache_data, msg); + domain = FLATPAK_OCI_ERROR; code = FLATPAK_OCI_ERROR_NOT_CHANGED; break; @@ -151,8 +448,19 @@ load_uri_callback (GObject *source_object, return; } + if (data->cache_data) + set_cache_http_data_from_headers (data->cache_data, msg); + data->etag = g_strdup (soup_message_headers_get_one (msg->response_headers, "ETag")); + if (data->out_tmpfile) + { + if (!glnx_open_tmpfile_linkable_at (data->out_tmpfile_parent_dfd, ".", + O_WRONLY, data->out_tmpfile, + &data->error)) + return; + } + g_input_stream_read_async (in, data->buffer, sizeof (data->buffer), G_PRIORITY_DEFAULT, data->cancellable, load_uri_read_cb, data); @@ -286,6 +594,7 @@ flatpak_download_http_uri (SoupSession *soup_session, if (request == NULL) return FALSE; + m = soup_request_http_get_message (request); if (flags & FLATPAK_HTTP_FLAGS_ACCEPT_OCI) soup_message_headers_replace (m->request_headers, "Accept", "application/vnd.oci.image.manifest.v1+json"); @@ -306,3 +615,190 @@ flatpak_download_http_uri (SoupSession *soup_session, return TRUE; } + +static gboolean +sync_and_rename_tmpfile (GLnxTmpfile *tmpfile, + const char *dest_name, + GError **error) +{ + /* Filesystem paranoia: If we end up with the new metadata but not + * the new data, then because the cache headers are in the metadata, + * we'll never re-download. (If we just want to avoid losing both + * the old and new data, skipping fdatasync when the destination is + * missing works, but it won't here.) + * + * This will cause a bunch of fdatasyncs when downloading the icons for + * a large appstream the first time, would mostly be a problem with a + * very fast internet connection and a slow spinning drive. + * Possible solution: update in new directory without fdatasync + * (copying in any existing cached icons to revalidate), syncfs(), then + * atomic symlink. + */ + if (fdatasync (tmpfile->fd) != 0) + return glnx_throw_errno_prefix (error, "fdatasync"); + + if (!glnx_link_tmpfile_at (tmpfile, + GLNX_LINK_TMPFILE_REPLACE, + tmpfile->src_dfd, dest_name, error)) + return FALSE; + + return TRUE; +} + +gboolean +flatpak_cache_http_uri (SoupSession *soup_session, + const char *uri, + FlatpakHTTPFlags flags, + int dest_dfd, + const char *dest_subpath, + FlatpakLoadUriProgress progress, + gpointer user_data, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(SoupRequestHTTP) request = NULL; + g_autoptr(GMainLoop) loop = NULL; + g_autoptr(GMainContext) context = NULL; + g_autoptr(CacheHttpData) cache_data = NULL; + g_autofree char *parent_path = g_path_get_dirname (dest_subpath); + g_autofree char *name = g_path_get_basename (dest_subpath); + glnx_autofd int dfd = -1; + gboolean no_xattr = FALSE; + LoadUriData data = { NULL }; + g_auto(GLnxTmpfile) out_tmpfile = { 0 }; + g_auto(GLnxTmpfile) cache_tmpfile = { 0 }; + g_autoptr(GBytes) cache_bytes = NULL; + SoupMessage *m; + + if (!glnx_opendirat (dest_dfd, parent_path, TRUE, &dfd, error)) + return FALSE; + + cache_data = load_cache_http_data (dfd, name, &no_xattr, + cancellable, error); + if (!cache_data) + return FALSE; + + if (g_strcmp0 (cache_data->uri, uri) != 0) + clear_cache_http_data (cache_data, TRUE); + + if (cache_data->uri) + { + GTimeVal now; + + g_get_current_time (&now); + if (cache_data->expires > now.tv_sec) + { + if (error) + *error = g_error_new (FLATPAK_OCI_ERROR, + FLATPAK_OCI_ERROR_NOT_CHANGED, + "Reusing cached value"); + return FALSE; + } + } + + if (cache_data->uri == NULL) + cache_data->uri = g_strdup (uri); + + /* Must revalidate */ + + g_debug ("Loading %s using libsoup", uri); + + context = g_main_context_ref_thread_default (); + + loop = g_main_loop_new (context, TRUE); + data.loop = loop; + data.cache_data = cache_data; + data.out_tmpfile = &out_tmpfile; + data.out_tmpfile_parent_dfd = dfd; + data.progress = progress; + data.cancellable = cancellable; + data.user_data = user_data; + data.last_progress_time = g_get_monotonic_time (); + + request = soup_session_request_http (soup_session, "GET", + uri, error); + if (request == NULL) + return FALSE; + + m = soup_request_http_get_message (request); + + if (cache_data->etag && cache_data->etag[0]) + soup_message_headers_replace (m->request_headers, "If-None-Match", cache_data->etag); + else if (cache_data->last_modified != 0) + { + SoupDate *date = soup_date_new_from_time_t (cache_data->last_modified); + g_autofree char *date_str = soup_date_to_string (date, SOUP_DATE_HTTP); + soup_message_headers_replace (m->request_headers, "If-Modified-Since", date_str); + soup_date_free (date); + } + + if (flags & FLATPAK_HTTP_FLAGS_ACCEPT_OCI) + soup_message_headers_replace (m->request_headers, "Accept", + "application/vnd.oci.image.manifest.v1+json"); + + soup_request_send_async (SOUP_REQUEST (request), + cancellable, + load_uri_callback, &data); + + g_main_loop_run (loop); + + if (data.error) + { + if (data.error->domain == FLATPAK_OCI_ERROR && + data.error->code == FLATPAK_OCI_ERROR_NOT_CHANGED) + { + GError *tmp_error = NULL; + + cache_bytes = serialize_cache_http_data (cache_data); + + if (!save_cache_http_data_to_file (dfd, name, cache_bytes, no_xattr, + cancellable, &tmp_error)) + { + g_clear_error (&data.error); + g_propagate_error (error, tmp_error); + + return FALSE; + } + } + + g_propagate_error (error, data.error); + return FALSE; + } + + cache_bytes = serialize_cache_http_data (cache_data); + if (!no_xattr) + { + if (!save_cache_http_data_xattr (out_tmpfile.fd, cache_bytes, error)) + { + if (errno != ENOTSUP) + return FALSE; + + g_clear_error (error); + no_xattr = TRUE; + } + } + + if (no_xattr) + { + if (!glnx_open_tmpfile_linkable_at (dfd, ".", O_WRONLY, &cache_tmpfile, error)) + return FALSE; + + if (!save_cache_http_data_fallback (cache_tmpfile.fd, cache_bytes, error)) + return FALSE; + } + + if (!sync_and_rename_tmpfile (&out_tmpfile, name, error)) + return FALSE; + + if (no_xattr) + { + g_autofree char *fallback_name = g_strconcat (name, CACHE_HTTP_SUFFIX, NULL); + + if (!sync_and_rename_tmpfile (&cache_tmpfile, fallback_name, error)) + return FALSE; + } + + g_debug ("Received %" G_GUINT64_FORMAT " bytes", data.downloaded_bytes); + + return TRUE; +} diff --git a/tests/Makefile.am.inc b/tests/Makefile.am.inc index ed9fb105..b378d7a0 100644 --- a/tests/Makefile.am.inc +++ b/tests/Makefile.am.inc @@ -22,6 +22,13 @@ testlibrary_LDADD = \ $(NULL) testlibrary_SOURCES = tests/testlibrary.c +httpcache_CFLAGS = $(AM_CFLAGS) $(BASE_CFLAGS) $(OSTREE_CFLAGS) $(SOUP_CFLAGS) $(JSON_CFLAGS) $(APPSTREAM_GLIB_CFLAGS) \ + -DFLATPAK_COMPILATION \ + -DLOCALEDIR=\"$(localedir)\" +httpcache_LDADD = $(AM_LDADD) $(BASE_LIBS) $(OSTREE_LIBS) $(SOUP_LIBS) $(JSON_LIBS) $(APPSTREAM_GLIB_LIBS) \ + libglnx.la libflatpak-common.la +httpcache_SOURCES = tests/httpcache.c + tests/services/org.freedesktop.Flatpak.service: session-helper/org.freedesktop.Flatpak.service.in mkdir -p tests/services $(AM_V_GEN) $(SED) -e "s|\@libexecdir\@|$(abs_top_builddir)|" $< > $@ @@ -46,6 +53,7 @@ tests/test-basic.sh: tests/package_version.txt dist_installed_test_extra_scripts += \ buildutil/tap-driver.sh \ + tests/http-utils-test-server.py \ tests/make-multi-collection-id-repo.sh \ tests/make-test-app.sh \ tests/make-test-runtime.sh \ @@ -79,6 +87,7 @@ endif dist_test_scripts = \ tests/test-basic.sh \ tests/test-build-update-repo.sh \ + tests/test-http-utils.sh \ tests/test-run.sh \ tests/test-run-system.sh \ tests/test-run-deltas.sh \ @@ -96,6 +105,7 @@ dist_test_scripts = \ $(NULL) test_programs = testlibrary +noinst_PROGRAMS += httpcache @VALGRIND_CHECK_RULES@ VALGRIND_SUPPRESSIONS_FILES=tests/flatpak.supp tests/glib.supp diff --git a/tests/http-utils-test-server.py b/tests/http-utils-test-server.py new file mode 100644 index 00000000..d0f4ef85 --- /dev/null +++ b/tests/http-utils-test-server.py @@ -0,0 +1,71 @@ +#!/usr/bin/python2 + +from wsgiref.handlers import format_date_time +from email.utils import parsedate +from calendar import timegm +from urlparse import parse_qs +import BaseHTTPServer +import time + +server_start_time = int(time.time()) + +def parse_http_date(date): + parsed = parsedate(date) + if parsed is not None: + return timegm(parsed) + else: + return None + +class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + def do_GET(self): + parts = self.path.split('?', 1) + path = parts[0] + if len(parts) == 1: + query = {} + else: + query = parse_qs(parts[1], keep_blank_values=True) + + response = 200 + add_headers = {} + + if 'modified-time' in query: + modified_since = self.headers.get("If-Modified-Since") + if modified_since: + modified_since_time = parse_http_date(modified_since) + if modified_since_time <= server_start_time: + response = 304 + add_headers["Last-Modified"] = format_date_time(server_start_time) + + if 'etag' in query: + etag = str(server_start_time) + + if self.headers.get("If-None-Match") == etag: + response = 304 + add_headers['Etag'] = etag + + self.send_response(response) + for k, v in add_headers.items(): + self.send_header(k, v) + + if 'max-age' in query: + self.send_header('Cache-Control', 'max-age=' + query['max-age'][0]) + if 'no-cache' in query: + self.send_header('Cache-Control', 'no-cache') + if 'expires-past' in query: + self.send_header('Expires', format_date_time(server_start_time - 3600)) + if 'expires-future' in query: + self.send_header('Expires', format_date_time(server_start_time + 3600)) + + if response == 200: + self.send_header("Content-Type", "text/plain; charset=UTF-8") + + self.end_headers() + + if response == 200: + self.wfile.write("path=" + self.path + "\n"); + +def test(): + BaseHTTPServer.test(RequestHandler) + +if __name__ == '__main__': + test() diff --git a/tests/httpcache.c b/tests/httpcache.c new file mode 100644 index 00000000..321c99f8 --- /dev/null +++ b/tests/httpcache.c @@ -0,0 +1,30 @@ +#include "common/flatpak-utils-private.h" + +int +main (int argc, char *argv[]) +{ + SoupSession *session = flatpak_create_soup_session (PACKAGE_STRING); + g_autoptr(GFile) dest = NULL; + GError *error = NULL; + + if (argc != 3) + { + g_printerr("Usage testhttp URL DEST\n"); + return 1; + } + + if (!flatpak_cache_http_uri (session, + argv[1], + 0, + AT_FDCWD, argv[2], + NULL, NULL, NULL, &error)) + { + g_print ("%s\n", error->message); + return 1; + } + else + { + g_print ("Server returned status 200: ok\n"); + return 0; + } +} diff --git a/tests/test-http-utils.sh b/tests/test-http-utils.sh new file mode 100755 index 00000000..6a37c742 --- /dev/null +++ b/tests/test-http-utils.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# +# Copyright (C) 2018 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +$(dirname $0)/test-webserver.sh "" "python2 $test_srcdir/http-utils-test-server.py 0" +FLATPAK_HTTP_PID=$(cat httpd-pid) +mv httpd-port httpd-port-main +port=$(cat httpd-port-main) + +assert_result() { + test_string=$1 + remote=$2 + local=$3 + + out=`httpcache "http://localhost:$port$remote" $local || :` + + case "$out" in + $test_string*) + return + ;; + *) + echo "For $remote => $local, expected '$test_string', got '$out'" + exit 1 + ;; + esac +} + +assert_cached() { + assert_result "Reusing cached value" $@ +} + +assert_304() { + assert_result "Server returned status 304:" $@ +} + +assert_ok() { + assert_result "Server returned status 200:" $@ +} + + +have_xattrs() { + touch $1/test-xattrs + setfattr -n user.testvalue -v somevalue $1/test-xattrs > /dev/null 2>&1 +} + +echo "1..4" + +# Without anything else, cached for 30 minutes +assert_ok "/" $test_tmpdir/output +assert_cached "/" $test_tmpdir/output +rm -f $test_tmpdir/output* + +# An explicit cache-lifetime +assert_ok "/?max-age=3600" $test_tmpdir/output +assert_cached "/?max-age=3600" $test_tmpdir/output +rm -f $test_tmpdir/output* + +# Turn off caching +assert_ok "/?max-age=0" $test_tmpdir/output +assert_ok "/?max-age=0" $test_tmpdir/output +rm -f $test_tmpdir/output* + +# Turn off caching a different way +assert_ok "/?no-cache" $test_tmpdir/output +assert_ok "/?no-cache" $test_tmpdir/output +rm -f $test_tmpdir/output* + +# Expires support +assert_ok "/?expires-future" $test_tmpdir/output +assert_cached "/?expires-future" $test_tmpdir/output +rm -f $test_tmpdir/output* + +assert_ok "/?expires-past" $test_tmpdir/output +assert_ok "/?expires-past" $test_tmpdir/output +rm -f $test_tmpdir/output* + +echo 'ok http cache lifetimes' + +# Revalation with an etag +assert_ok "/?etag&no-cache" $test_tmpdir/output +assert_304 "/?etag&no-cache" $test_tmpdir/output +rm -f $test_tmpdir/output* + +# Revalation with an modified time +assert_ok "/?modified-time&no-cache" $test_tmpdir/output +assert_304 "/?modified-time&no-cache" $test_tmpdir/output +rm -f $test_tmpdir/output* + +echo 'ok http revalidation' + +# Testing that things work with without xattr support + +if have_xattrs $test_tmpdir ; then + ls $test_tmpdir 1>&2 + assert_ok "/?etag&no-cache" $test_tmpdir/output + ls $test_tmpdir 1>&2 + assert_not_has_file $test_tmpdir/output.flatpak.http + assert_304 "/?etag&no-cache" $test_tmpdir/output + rm -f $test_tmpdir/output* + echo "ok with-xattrs" +else + echo "ok with-xattrs # skip /var/tmp doesn't have user xattr support" +fi + +# Testing fallback without xattr support + +no_xattrs_tempdir=`mktemp -d /tmp/test-flatpak-XXXXXX` +no_xattrs_cleanup () { + rm -rf test_tmpdir + cleanup +} +trap no_xattrs_cleanup EXIT + +if have_xattrs $no_xattrs_tempdir ; then + echo "ok no-xattrs # skip /tmp has user xattr support" +else + assert_ok "/?etag&no-cache" $no_xattrs_tempdir/output + assert_has_file $no_xattrs_tempdir/output.flatpak.http + assert_304 "/?etag&no-cache" $no_xattrs_tempdir/output + rm -f $no_xattrs_tempdir/output* + echo "ok no-xattrs" +fi diff --git a/tests/test-webserver.sh b/tests/test-webserver.sh index 48e545cb..7accd0b0 100755 --- a/tests/test-webserver.sh +++ b/tests/test-webserver.sh @@ -3,10 +3,11 @@ set -euo pipefail dir=$1 +cmd=${2:-python -m SimpleHTTPServer 0} test_tmpdir=$(pwd) -cd ${dir} -env PYTHONUNBUFFERED=1 setsid python -m SimpleHTTPServer 0 >${test_tmpdir}/httpd-output & +[ "$dir" != "" ] && cd ${dir} +env PYTHONUNBUFFERED=1 setsid $cmd >${test_tmpdir}/httpd-output & child_pid=$! echo "Web server pid: $child_pid" >&2