From 9c06cbb36083d8d89a669d67d1bad2bc82138de8 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Sat, 29 Jul 2017 17:43:10 +0200 Subject: [PATCH] Add support for --extra-sources-url=URL This makes flatpak-builder look in the specified URL for mirrored sources before downloading the original URL. --- builder/builder-context.c | 68 ++++++++++++++++++++++++++++++++ builder/builder-context.h | 8 ++++ builder/builder-main.c | 19 +++++++++ builder/builder-source-archive.c | 11 +++--- builder/builder-source-file.c | 10 ++--- builder/builder-utils.c | 14 ++++++- builder/builder-utils.h | 2 +- doc/flatpak-builder.xml | 18 +++++++++ 8 files changed, 136 insertions(+), 14 deletions(-) diff --git a/builder/builder-context.c b/builder/builder-context.c index 48d8ed95..d0337dc9 100644 --- a/builder/builder-context.c +++ b/builder/builder-context.c @@ -31,9 +31,11 @@ #include #include +#include #include "flatpak-utils.h" #include "builder-context.h" #include "builder-cache.h" +#include "builder-utils.h" struct BuilderContext { @@ -48,6 +50,7 @@ struct BuilderContext GFile *download_dir; GPtrArray *sources_dirs; + GPtrArray *sources_urls; GFile *state_dir; GFile *build_dir; GFile *cache_dir; @@ -112,6 +115,7 @@ builder_context_finalize (GObject *object) glnx_release_lock_file(&self->rofiles_file_lock); g_clear_pointer (&self->sources_dirs, g_ptr_array_unref); + g_clear_pointer (&self->sources_urls, g_ptr_array_unref); G_OBJECT_CLASS (builder_context_parent_class)->finalize (object); } @@ -310,6 +314,70 @@ builder_context_find_in_sources_dirs (BuilderContext *self, return res; } +GPtrArray * +builder_context_get_sources_urls (BuilderContext *self) +{ + return self->sources_urls; +} + +void +builder_context_set_sources_urls (BuilderContext *self, + GPtrArray *sources_urls) +{ + g_clear_pointer (&self->sources_urls, g_ptr_array_unref); + self->sources_urls = g_ptr_array_ref (sources_urls); +} + +gboolean +builder_context_download_uri (BuilderContext *self, + const char *url, + GFile *dest, + char *sha256, + GError **error) +{ + int i; + g_autoptr(SoupURI) original_uri = soup_uri_new (url); + + if (original_uri == NULL) + return flatpak_fail (error, _("Could not parse URI “%s”"), url); + + g_print ("Downloading %s\n", url); + + if (self->sources_urls != NULL) + { + g_autofree char *base_name = g_path_get_basename (soup_uri_get_path (original_uri)); + g_autofree char *rel = g_build_filename ("downloads", sha256, base_name, NULL); + + for (i = 0; i < self->sources_urls->len; i++) + { + SoupURI *base_uri = g_ptr_array_index (self->sources_urls, i); + g_autoptr(SoupURI) mirror_uri = soup_uri_new_with_base (base_uri, rel); + g_autofree char *mirror_uri_str = soup_uri_to_string (mirror_uri, FALSE); + g_print ("Trying mirror %s\n", mirror_uri_str); + g_autoptr(GError) my_error = NULL; + + if (builder_download_uri (mirror_uri, + dest, + sha256, + builder_context_get_soup_session (self), + &my_error)) + return TRUE; + + if (!g_error_matches (my_error, SOUP_HTTP_ERROR, SOUP_STATUS_NOT_FOUND)) + g_warning ("Error downloading from mirror: %s\n", my_error->message); + } + } + + if (!builder_download_uri (original_uri, + dest, + sha256, + builder_context_get_soup_session (self), + error)) + return FALSE; + + return TRUE; +} + GFile * builder_context_get_cache_dir (BuilderContext *self) { diff --git a/builder/builder-context.h b/builder/builder-context.h index e13e84c4..6872b416 100644 --- a/builder/builder-context.h +++ b/builder/builder-context.h @@ -56,6 +56,14 @@ GFile * builder_context_find_in_sources_dirs (BuilderContext *self, ...) G_GNUC_NULL_TERMINATED; GFile * builder_context_find_in_sources_dirs_va (BuilderContext *self, va_list args); +GPtrArray * builder_context_get_sources_urls (BuilderContext *self); +void builder_context_set_sources_urls (BuilderContext *self, + GPtrArray *sources_urls); +gboolean builder_context_download_uri (BuilderContext *self, + const char *url, + GFile *dest, + char *sha256, + GError **error); SoupSession * builder_context_get_soup_session (BuilderContext *self); const char * builder_context_get_arch (BuilderContext *self); void builder_context_set_arch (BuilderContext *self, diff --git a/builder/builder-main.c b/builder/builder-main.c index f3414134..15454260 100644 --- a/builder/builder-main.c +++ b/builder/builder-main.c @@ -65,6 +65,7 @@ static char *opt_body; static char *opt_gpg_homedir; static char **opt_key_ids; static char **opt_sources_dirs; +static char **opt_sources_urls; static int opt_jobs; static char *opt_mirror_screenshots_url; @@ -82,6 +83,7 @@ static GOptionEntry entries[] = { { "download-only", 0, 0, G_OPTION_ARG_NONE, &opt_download_only, "Only download sources, don't build", NULL }, { "bundle-sources", 0, 0, G_OPTION_ARG_NONE, &opt_bundle_sources, "Bundle module sources as runtime", NULL }, { "extra-sources", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_sources_dirs, "Add a directory of sources specified by SOURCE-DIR, multiple uses of this option possible", "SOURCE-DIR"}, + { "extra-sources-url", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_sources_urls, "Add a url of sources specified by SOURCE-URL multiple uses of this option possible", "SOURCE-URL"}, { "build-only", 0, 0, G_OPTION_ARG_NONE, &opt_build_only, "Stop after build, don't run clean and finish phases", NULL }, { "finish-only", 0, 0, G_OPTION_ARG_NONE, &opt_finish_only, "Only run clean and finish and export phases", NULL }, { "export-only", 0, 0, G_OPTION_ARG_NONE, &opt_export_only, "Only run export phase", NULL }, @@ -352,6 +354,23 @@ main (int argc, builder_context_set_sources_dirs (build_context, sources_dirs); } + if (opt_sources_urls) + { + g_autoptr(GPtrArray) sources_urls = NULL; + sources_urls = g_ptr_array_new_with_free_func ((GDestroyNotify)soup_uri_free); + for (i = 0; opt_sources_urls[i] != NULL; i++) + { + SoupURI *uri = soup_uri_new (opt_sources_urls[i]); + if (uri == NULL) + { + g_printerr ("Invalid URL '%s'", opt_sources_urls[i]); + return 1; + } + g_ptr_array_add (sources_urls, uri); + } + builder_context_set_sources_urls (build_context, sources_urls); + } + if (opt_arch) builder_context_set_arch (build_context, opt_arch); diff --git a/builder/builder-source-archive.c b/builder/builder-source-archive.c index 16ef1f97..ff9452b4 100644 --- a/builder/builder-source-archive.c +++ b/builder/builder-source-archive.c @@ -332,12 +332,11 @@ builder_source_archive_download (BuilderSource *source, return FALSE; } - g_print ("Downloading %s\n", self->url); - if (!builder_download_uri (self->url, - file, - self->sha256, - builder_context_get_soup_session (context), - error)) + if (!builder_context_download_uri (context, + self->url, + file, + self->sha256, + error)) return FALSE; return TRUE; diff --git a/builder/builder-source-file.c b/builder/builder-source-file.c index e22c59a0..d6792bd1 100644 --- a/builder/builder-source-file.c +++ b/builder/builder-source-file.c @@ -327,11 +327,11 @@ builder_source_file_download (BuilderSource *source, return FALSE; } - if (!builder_download_uri (self->url, - file, - self->sha256, - builder_context_get_soup_session (context), - error)) + if (!builder_context_download_uri (context, + self->url, + file, + self->sha256, + error)) return FALSE; return TRUE; diff --git a/builder/builder-utils.c b/builder/builder-utils.c index 91d8b79f..be09ee57 100644 --- a/builder/builder-utils.c +++ b/builder/builder-utils.c @@ -1628,7 +1628,7 @@ download_progress_cleanup (DownloadPromptData *progress_data) } gboolean -builder_download_uri (const char *url, +builder_download_uri (SoupURI *uri, GFile *dest, char *sha256, SoupSession *session, @@ -1656,7 +1656,7 @@ builder_download_uri (const char *url, if (out == NULL) return FALSE; - req = soup_session_request (session, url, error); + req = soup_session_request_uri (session, uri, error); if (req == NULL) return FALSE; @@ -1664,6 +1664,16 @@ builder_download_uri (const char *url, if (input == NULL) return FALSE; + if (SOUP_IS_REQUEST_HTTP (req)) + { + g_autoptr(SoupMessage) msg = soup_request_http_get_message (SOUP_REQUEST_HTTP(req)); + if (!SOUP_STATUS_IS_SUCCESSFUL(msg->status_code)) + { + g_set_error_literal (error, SOUP_HTTP_ERROR, msg->status_code, msg->reason_phrase); + return FALSE; + } + } + if (!flatpak_splice_update_checksum (G_OUTPUT_STREAM (out), input, checksum, download_progress, &progress_data, NULL, error)) { unlink (flatpak_file_get_path_cached (tmp)); diff --git a/builder/builder-utils.h b/builder/builder-utils.h index 0f7d742b..48120f3a 100644 --- a/builder/builder-utils.h +++ b/builder/builder-utils.h @@ -63,7 +63,7 @@ gboolean builder_maybe_host_spawnv (GFile *dir, GError **error, const gchar * const *argv); -gboolean builder_download_uri (const char *url, +gboolean builder_download_uri (SoupURI *uri, GFile *dest, char *sha256, SoupSession *soup_session, diff --git a/doc/flatpak-builder.xml b/doc/flatpak-builder.xml index c32a2379..5967fc06 100644 --- a/doc/flatpak-builder.xml +++ b/doc/flatpak-builder.xml @@ -404,6 +404,24 @@ + + + + + When downloading sources (archives, files, git, bzr), look in this + directory for pre-existing copies and use them instead of downloading. + + + + + + + + When downloading sources (archives, files, git, bzr), look at this url + for mirrored downloads before downloading from the original url. + + +