/* vi:set et sw=2 sts=2 cin cino=t0,f0,(0,{s,>2s,n-s,^-s,e-s:
 * Copyright © 2018 Red Hat, Inc
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library. If not, see <http://www.gnu.org/licenses/>.
 *
 * Authors:
 *       Alexander Larsson <alexl@redhat.com>
 */

#include "config.h"

#include <gio/gio.h>
#include <glib-unix.h>
#include "flatpak-utils-http-private.h"
#include "flatpak-uri-private.h"
#include "flatpak-oci-registry-private.h"

#include <gio/gunixoutputstream.h>
#include "libglnx.h"

#include <sys/types.h>
#include <sys/xattr.h>
#include <curl/curl.h>

/* These macros came from 7.43.0, but we want to check
 * for versions a bit earlier than that (to work on CentOS 7),
 * so define them here if we're using an older version.
 */
#ifndef CURL_VERSION_BITS
#define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|z)
#endif
#ifndef CURL_AT_LEAST_VERSION
#define CURL_AT_LEAST_VERSION(x,y,z) (LIBCURL_VERSION_NUM >= CURL_VERSION_BITS(x, y, z))
#endif

#define FLATPAK_HTTP_TIMEOUT_SECS 60

/* copied from libostree */
#define DEFAULT_N_NETWORK_RETRIES 5

G_DEFINE_QUARK (flatpak_http_error, flatpak_http_error)

/* Holds information about CA and client certificates found in
 * system-wide and per-user certificate directories as documented
 * in container-certs.d(5).
 */
struct FlatpakCertificates
{
  char *ca_cert_file;
  char *client_cert_file;
  char *client_key_file;
};

/* Information about the cache status of a file.
   Encoded in an xattr on the cached file, or a file on the side if xattrs don't work.
*/
typedef struct
{
  char  *uri;
  char  *etag;
  gint64 last_modified;
  gint64 expires;
} CacheHttpData;

typedef struct
{
  GMainContext          *context;
  gboolean               done;
  GError                *error;

  /* Input args */

  FlatpakHTTPFlags       flags;
  const char            *auth;
  const char            *token;
  FlatpakCertificates   *certificates;
  FlatpakLoadUriProgress progress;
  GCancellable          *cancellable;
  gpointer               user_data;
  CacheHttpData         *cache_data;

  /* Output from the request, set even on http server errors */

  guint64               downloaded_bytes;
  int                   status;
  char                  *hdr_content_type;
  char                  *hdr_www_authenticate;
  char                  *hdr_etag;
  char                  *hdr_last_modified;
  char                  *hdr_cache_control;
  char                  *hdr_expires;
  char                  *hdr_content_encoding;

  /* Data destination */

  GOutputStream         *out; /*or */
  GString               *content; /* or */
  GLnxTmpfile           *out_tmpfile;
  int                    out_tmpfile_parent_dfd;

  /* Used during operation */

  char                   buffer[16 * 1024];
  guint64                last_progress_time;
  gboolean               store_compressed;

} LoadUriData;

static void
clear_load_uri_data_headers (LoadUriData *data)
{
  g_clear_pointer (&data->hdr_content_type, g_free);
  g_clear_pointer (&data->hdr_www_authenticate, g_free);
  g_clear_pointer (&data->hdr_etag, g_free);
  g_clear_pointer (&data->hdr_last_modified, g_free);
  g_clear_pointer (&data->hdr_cache_control, g_free);
  g_clear_pointer (&data->hdr_expires, g_free);
  g_clear_pointer (&data->hdr_content_encoding, g_free);
}

/* Reset between requests retries */
static void
reset_load_uri_data (LoadUriData *data)
{
  g_clear_error (&data->error);
  data->status = 0;
  data->downloaded_bytes = 0;
  if (data->content)
    g_string_set_size (data->content, 0);

  clear_load_uri_data_headers (data);

  if (data->out_tmpfile)
    {
      glnx_tmpfile_clear (data->out_tmpfile);
      g_clear_pointer (&data->out, g_object_unref);
    }

  /* Reset the progress */
  if (data->progress)
    data->progress (0, data->user_data);
}

/* Free allocated data at end of full repeated download */
static void
clear_load_uri_data (LoadUriData *data)
{
  if (data->content)
    {
      g_string_free (data->content, TRUE);
      data->content = NULL;
    }

  g_clear_error (&data->error);

  clear_load_uri_data_headers (data);
}

G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(LoadUriData, clear_load_uri_data)

static gboolean
check_http_status (guint status_code,
                   GError **error)
{
  GQuark domain;
  int code;

  if (status_code >= 200 && status_code < 300)
    return TRUE;

  switch (status_code)
    {
    case 304: /* Not Modified */
      domain = FLATPAK_HTTP_ERROR;
      code = FLATPAK_HTTP_ERROR_NOT_CHANGED;
      break;

    case 401: /* Unauthorized */
      domain = FLATPAK_HTTP_ERROR;
      code = FLATPAK_HTTP_ERROR_UNAUTHORIZED;
      break;

    case 403: /* Forbidden */
    case 404: /* Not found */
    case 410: /* Gone */
      domain = G_IO_ERROR;
      code = G_IO_ERROR_NOT_FOUND;
      break;

    case 408: /* Request Timeout */
      domain = G_IO_ERROR;
      code = G_IO_ERROR_TIMED_OUT;
      break;

    case 500: /* Internal Server Error */
      /* The server did return something, but it was useless to us, so that’s basically equivalent to not returning */
      domain = G_IO_ERROR;
      code = G_IO_ERROR_HOST_UNREACHABLE;
      break;

    default:
      domain = G_IO_ERROR;
      code = G_IO_ERROR_FAILED;
    }

  g_set_error (error, domain, code,
               "Server returned status %u",
               status_code);
  return FALSE;
}

FlatpakCertificates*
flatpak_get_certificates_for_uri (const char  *uri,
                                  GError     **error)
{
  g_autoptr(FlatpakCertificates) certificates = NULL;
  g_autoptr(GUri) parsed_uri = NULL;
  g_autofree char *hostport = NULL;
  const char *system_certs_d = NULL;
  g_autofree char *certs_path_str = NULL;
  g_auto(GStrv) certs_path = NULL;

  certificates = g_new0 (FlatpakCertificates, 1);

  parsed_uri = g_uri_parse (uri, G_URI_FLAGS_PARSE_RELAXED, error);
  if (!parsed_uri)
    return NULL;

  if (!g_uri_get_host (parsed_uri))
    return NULL;

  if (g_uri_get_port (parsed_uri) != -1)
    hostport = g_strdup_printf ("%s:%d", g_uri_get_host (parsed_uri), g_uri_get_port (parsed_uri));
  else
    hostport = g_strdup (g_uri_get_host (parsed_uri));

  system_certs_d = g_getenv ("FLATPAK_SYSTEM_CERTS_D");
  if (system_certs_d == NULL || system_certs_d[0] == '\0')
    system_certs_d = "/etc/containers/certs.d:/etc/docker/certs.d";

  /* containers/image hardcodes ~/.config and doesn't honor XDG_CONFIG_HOME */
  certs_path_str = g_strconcat (g_get_user_config_dir(), "/containers/certs.d:",
                                system_certs_d, NULL);
  certs_path = g_strsplit (certs_path_str, ":", -1);

  for (int i = 0; certs_path[i]; i++)
    {
      g_autoptr(GFile) certs_dir = g_file_new_for_path (certs_path[i]);
      g_autoptr(GFile) host_dir = g_file_get_child (certs_dir, hostport);
      g_autoptr(GFileEnumerator) enumerator = NULL;
      g_autoptr(GError) local_error = NULL;

      enumerator = g_file_enumerate_children (host_dir, G_FILE_ATTRIBUTE_STANDARD_NAME,
                                              G_FILE_QUERY_INFO_NONE,
                                              NULL, &local_error);
      if (enumerator == NULL)
        {
          /* This matches libpod - missing certificate directory or a permission
           * error causes the directory to be skipped; any other error is fatal
           */
          if (g_error_matches(local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND) ||
              g_error_matches(local_error, G_IO_ERROR, G_IO_ERROR_PERMISSION_DENIED))
            {
              g_clear_error (&local_error);
              continue;
            }
          else
            {
              g_propagate_error (error, g_steal_pointer (&local_error));
              return NULL;
            }
        }

      while (TRUE)
        {
          GFile *child;
          g_autofree char *basename = NULL;

          if (!g_file_enumerator_iterate (enumerator, NULL, &child, NULL, error))
            return NULL;

          if (child == NULL)
            break;

          basename = g_file_get_basename (child);

          /* In libpod, all CA certificates are added to the CA certificate
           * database. We just use the first in readdir order.
           */
          if (g_str_has_suffix (basename, ".crt") && certificates->ca_cert_file == NULL)
            certificates->ca_cert_file = g_file_get_path (child);

          if (g_str_has_suffix (basename, ".cert"))
            {
              g_autofree char *nosuffix = g_strndup (basename, strlen (basename) - 5);
              g_autofree char *key_basename = g_strconcat (nosuffix, ".key", NULL);
              g_autoptr(GFile) key_file = g_file_get_child (host_dir, key_basename);

              if (!g_file_query_exists (key_file, NULL))
                {
                  g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
                               "missing key %s for client cert %s. "
                               "Note that CA certificates should use the extension .crt",
                               g_file_peek_path (key_file),
                               g_file_peek_path (child));
                  return NULL;
                }

              /* In libpod, all client certificates are added, and then the go TLS
               * code selects the best based on TLS negotiation. We just pick the first
               * in readdir order
               * */
              if (certificates->client_cert_file == NULL)
                {
                  certificates->client_cert_file = g_file_get_path (child);
                  certificates->client_key_file = g_file_get_path (key_file);
                }
            }

          if (g_str_has_suffix (basename, ".key"))
            {
              g_autofree char *nosuffix = g_strndup (basename, strlen (basename) - 4);
              g_autofree char *cert_basename = g_strconcat (nosuffix, ".cert", NULL);
              g_autoptr(GFile) cert_file = g_file_get_child (host_dir, cert_basename);

              if (!g_file_query_exists (cert_file, NULL))
                {
                  g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
                               "missing client certificate %s for key %s",
                               g_file_peek_path (cert_file),
                               g_file_peek_path (child));
                  return NULL;
                }
            }
        }
    }

  return g_steal_pointer (&certificates);
}

FlatpakCertificates *
flatpak_certificates_copy (FlatpakCertificates *other)
{
  FlatpakCertificates *certificates = g_new0 (FlatpakCertificates, 1);

  certificates->ca_cert_file = g_strdup (other->ca_cert_file);
  certificates->client_cert_file = g_strdup (other->client_cert_file);
  certificates->client_key_file = g_strdup (other->client_key_file);

  return certificates;
}

void
flatpak_certificates_free (FlatpakCertificates *certificates)
{
  g_clear_pointer (&certificates->ca_cert_file, g_free);
  g_clear_pointer (&certificates->client_cert_file, g_free);
  g_clear_pointer (&certificates->client_key_file, g_free);

  g_free (certificates);
}

/************************************************************************
 *                        Curl implementation                           *
 ************************************************************************/

typedef struct curl_slist auto_curl_slist;

G_DEFINE_AUTOPTR_CLEANUP_FUNC (auto_curl_slist, curl_slist_free_all)

struct FlatpakHttpSession {
  CURL *curl;
  GMutex lock;
};

static void
check_header(char **value_out,
             const char *header,
             char *buffer,
             size_t realsize)
{
  size_t hlen = strlen (header);

  if (realsize < hlen + 1)
    return;

  if (g_ascii_strncasecmp (buffer, header, hlen) != 0 ||
      buffer[hlen] != ':')
    return;

  buffer += hlen + 1;
  realsize -= hlen + 1;

  while (realsize > 0 && g_ascii_isspace (*buffer))
    {
      buffer++;
      realsize--;
    }

  while (realsize > 0 && g_ascii_isspace (buffer[realsize-1]))
    realsize--;

  g_free (*value_out); /* Use the last header */
  *value_out = g_strndup (buffer, realsize);
}

static size_t
_header_cb (char *buffer,
            size_t size,
            size_t nitems,
            void *userdata)
{
  size_t realsize = size * nitems;
  LoadUriData *data = (LoadUriData *)userdata;

  check_header(&data->hdr_content_type, "content-type", buffer, realsize);
  check_header(&data->hdr_www_authenticate, "WWW-Authenticate", buffer, realsize);

  check_header(&data->hdr_etag, "ETag", buffer, realsize);
  check_header(&data->hdr_last_modified, "Last-Modified", buffer, realsize);
  check_header(&data->hdr_cache_control, "Cache-Control", buffer, realsize);
  check_header(&data->hdr_expires, "Expires", buffer, realsize);
  check_header(&data->hdr_content_encoding, "Content-Encoding", buffer, realsize);

  return realsize;
}

static size_t
_write_cb (void *content_data,
           size_t size,
           size_t nmemb,
           void *userp)
{
  size_t realsize = size * nmemb;
  LoadUriData *data = (LoadUriData *)userp;
  gsize n_written = 0;

  /* If first write to tmpfile, initiate if needed */
  if (data->content == NULL && data->out == NULL &&
      data->out_tmpfile != NULL)
    {
      g_autoptr(GOutputStream) out = NULL;
      g_autoptr(GError) tmp_error = NULL;

      if (!glnx_open_tmpfile_linkable_at (data->out_tmpfile_parent_dfd, ".",
                                          O_WRONLY, data->out_tmpfile,
                                          &tmp_error))
        {
          g_warning ("Failed to open http tmpfile: %s\n", tmp_error->message);
          return 0; /* This short read will make curl report an error */
        }

      out = g_unix_output_stream_new (data->out_tmpfile->fd, FALSE);
      if (data->store_compressed &&
          g_strcmp0 (data->hdr_content_encoding, "gzip") != 0)
        {
          g_autoptr(GZlibCompressor) compressor = g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, -1);
          data->out = g_converter_output_stream_new (out, G_CONVERTER (compressor));
        }
      else
        {
          data->out = g_steal_pointer (&out);
        }
    }

  /* Check for cancellation */
  if (g_cancellable_is_cancelled (data->cancellable))
    return 0; /* Returning 0 (short read) makes curl abort the transfer */

  if (data->content)
    {
      g_string_append_len (data->content, content_data, realsize);
      n_written = realsize;
    }
  else if (data->out)
    {
      /* We ignore the error here, but reporting a short read will make curl report the error */
      g_output_stream_write_all (data->out, content_data, realsize,
                                 &n_written, NULL, NULL);
    }

  data->downloaded_bytes += realsize;

  if (g_get_monotonic_time () - data->last_progress_time > 1 * G_USEC_PER_SEC)
    {
      if (data->progress)
        data->progress (data->downloaded_bytes, data->user_data);
      data->last_progress_time = g_get_monotonic_time ();
    }

  return realsize;
}

FlatpakHttpSession *
flatpak_create_http_session (const char *user_agent)
{
  FlatpakHttpSession *session = g_new0 (FlatpakHttpSession, 1);
  CURLcode rc;
  CURL *curl;

  session->curl = curl = curl_easy_init();
  g_assert (session->curl != NULL);

  g_mutex_init (&session->lock);

  curl_easy_setopt (curl, CURLOPT_USERAGENT, user_agent);
#if CURL_AT_LEAST_VERSION(7, 85, 0)
  rc = curl_easy_setopt (curl, CURLOPT_PROTOCOLS_STR, "http,https");
#else
  rc = curl_easy_setopt (curl, CURLOPT_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS));
#endif
  g_assert (rc == CURLE_OK);

  curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);

  /* Note: curl automatically respects the http_proxy env var */

  if (g_getenv ("OSTREE_DEBUG_HTTP"))
    curl_easy_setopt (curl, CURLOPT_VERBOSE, 1L);

  /* Picked the current version in F25 as of 20170127, since
   * there are numerous HTTP/2 fixes since the original version in
   * libcurl 7.43.0.
   */
#if CURL_AT_LEAST_VERSION(7, 51, 0)
  if ((curl_version_info (CURLVERSION_NOW))->features & CURL_VERSION_HTTP2) {
    rc = curl_easy_setopt (curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
    g_assert (rc == CURLE_OK);
  }
#endif
  /* https://github.com/curl/curl/blob/curl-7_53_0/docs/examples/http2-download.c */
#if (CURLPIPE_MULTIPLEX > 0)
  /* wait for pipe connection to confirm */
  rc = curl_easy_setopt (curl, CURLOPT_PIPEWAIT, 1L);
  g_assert (rc == CURLE_OK);
#endif

  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, _write_cb);
  curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, _header_cb);

  /* Abort the connection if connecting to the server takes too long. This
   * timeout has no effect after a connection is established. */
  curl_easy_setopt (curl, CURLOPT_CONNECTTIMEOUT, (long)FLATPAK_HTTP_TIMEOUT_SECS);

  /* Abort the download if it’s slower than 10KB/sec for 60 seconds. An example
   * compressed summary file is 1.5MB in size, so anything slower than this rate
   * will mean it takes over 2.5 minutes to download just the summary file. */
  curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, (long)FLATPAK_HTTP_TIMEOUT_SECS);
  curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 10000L);

  return session;
}

void
flatpak_http_session_free (FlatpakHttpSession* session)
{
  g_mutex_lock (&session->lock);
  curl_easy_cleanup (session->curl);
  g_mutex_unlock (&session->lock);
  g_mutex_clear (&session->lock);
  g_free (session);
}

static void
set_error_from_curl (GError        **error,
                     const char     *uri,
                     CURLcode        res,
                     GCancellable   *cancellable)
{
  GQuark domain = G_IO_ERROR;
  int code;

  switch (res)
    {
    case CURLE_WRITE_ERROR:
      /* Check if this was due to cancellation */
      if (g_cancellable_is_cancelled (cancellable))
        code = G_IO_ERROR_CANCELLED;
      else
        code = G_IO_ERROR_FAILED;
      break;
    case CURLE_COULDNT_CONNECT:
    case CURLE_COULDNT_RESOLVE_HOST:
    case CURLE_COULDNT_RESOLVE_PROXY:
      code = G_IO_ERROR_HOST_NOT_FOUND;
      break;
    case CURLE_OPERATION_TIMEDOUT:
      code = G_IO_ERROR_TIMED_OUT;
      break;
    default:
      code =  G_IO_ERROR_FAILED;
    }

  g_set_error (error, domain, code,
               "While fetching %s: [%u] %s", uri, res,
               curl_easy_strerror (res));
}

static gboolean
flatpak_download_http_uri_once (FlatpakHttpSession    *session,
                                LoadUriData           *data,
                                const char            *uri,
                                GError               **error)
{
  CURLcode res;
  g_autofree char *auth_header = NULL;
  g_autofree char *cache_header = NULL;
  g_autoptr(auto_curl_slist) header_list = NULL;
  g_autoptr(GMutexLocker) curl_lock = g_mutex_locker_new (&session->lock);
  long response;
  CURL *curl = session->curl;

  g_info ("Loading %s using curl", uri);

  curl_easy_setopt (curl, CURLOPT_URL, uri);
  curl_easy_setopt (curl, CURLOPT_WRITEDATA, (void *)data);
  curl_easy_setopt (curl, CURLOPT_HEADERDATA, (void *)data);

  if (data->certificates)
    {
      if (data->certificates->ca_cert_file)
        curl_easy_setopt (curl, CURLOPT_CAINFO, data->certificates->ca_cert_file);

      if (data->certificates->client_cert_file)
        {
          curl_easy_setopt (curl, CURLOPT_SSLCERT, data->certificates->client_cert_file);
          curl_easy_setopt (curl, CURLOPT_SSLKEY, data->certificates->client_key_file);
        }
    }

  if (data->flags & FLATPAK_HTTP_FLAGS_HEAD)
    curl_easy_setopt (curl, CURLOPT_NOBODY, 1L);
  else
    curl_easy_setopt (curl, CURLOPT_HTTPGET, 1L);

  if (data->flags & FLATPAK_HTTP_FLAGS_ACCEPT_OCI)
    header_list = curl_slist_append (header_list,
                                     "Accept: " FLATPAK_OCI_MEDIA_TYPE_IMAGE_MANIFEST ", " FLATPAK_DOCKER_MEDIA_TYPE_IMAGE_MANIFEST2 ", " FLATPAK_OCI_MEDIA_TYPE_IMAGE_INDEX);

  if (data->auth)
    auth_header = g_strdup_printf ("Authorization: Basic %s", data->auth);
  else if (data->token)
    auth_header = g_strdup_printf ("Authorization: Bearer %s", data->token);
  if (auth_header)
    header_list = curl_slist_append (header_list, auth_header);

  if (data->cache_data)
    {
      CacheHttpData *cache_data = data->cache_data;

      if (cache_data->etag && cache_data->etag[0])
        cache_header = g_strdup_printf ("If-None-Match: %s", cache_data->etag);
      else if (cache_data->last_modified != 0)
        {
          g_autoptr(GDateTime) date = g_date_time_new_from_unix_utc (cache_data->last_modified);
          g_autofree char *date_str = flatpak_format_http_date (date);
          cache_header = g_strdup_printf ("If-Modified-Since: %s", date_str);
        }
      if (cache_header)
        header_list = curl_slist_append (header_list, cache_header);
    }

  curl_easy_setopt (curl, CURLOPT_HTTPHEADER, header_list);

  if (data->flags & FLATPAK_HTTP_FLAGS_STORE_COMPRESSED)
    {
      curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "gzip");
      curl_easy_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, 0L);
      data->store_compressed = TRUE;
    }
  else
    {
      /* enable all supported built-in compressions */
      curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "");
      curl_easy_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, 1L);
      data->store_compressed = FALSE;
    }

  res = curl_easy_perform (session->curl);

  curl_easy_setopt (session->curl, CURLOPT_HTTPHEADER, NULL); /* Don't point to freed list */

  if (res != CURLE_OK)
    {
      set_error_from_curl (error, uri, res, data->cancellable);

      /* Make sure we clear the tmpfile stream we possible created during the request */
      if (data->out_tmpfile && data->out)
        g_clear_pointer (&data->out, g_object_unref);

      return FALSE;
    }

  if (data->out_tmpfile && data->out)
    {
      /* Flush the writes */
      if (!g_output_stream_close (data->out, data->cancellable, error))
        return FALSE;

      g_clear_pointer (&data->out, g_object_unref);
    }

  if (data->progress)
    data->progress (data->downloaded_bytes, data->user_data);

  curl_easy_getinfo (session->curl, CURLINFO_RESPONSE_CODE, &response);

  data->status = response;

  if ((data->flags & FLATPAK_HTTP_FLAGS_NOCHECK_STATUS) == 0 &&
      !check_http_status (data->status, error))
    return FALSE;

  g_info ("Received %" G_GUINT64_FORMAT " bytes", data->downloaded_bytes);

  /* This is not really needed, but the auto-pointer confuses some compilers in the CI */
  g_clear_pointer (&curl_lock, g_mutex_locker_free);

  return TRUE;
}

/* Check whether a particular operation should be retried. This is entirely
 * based on how it failed (if at all) last time, and whether the operation has
 * some retries left. The retry count is set when the operation is first
 * created, and must be decremented by the caller. (@n_retries_remaining == 0)
 * will always return %FALSE from this function.
 *
 * This code is copied from libostree's _ostree_fetcher_should_retry_request()
 */
static gboolean
flatpak_http_should_retry_request (const GError *error,
                                   guint         n_retries_remaining)
{
  if (error == NULL || n_retries_remaining == 0)
    return FALSE;

  /* Return TRUE for transient errors. */
  if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_TIMED_OUT) ||
      g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_NOT_FOUND) ||
      g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_UNREACHABLE) ||
      g_error_matches (error, G_IO_ERROR, G_IO_ERROR_PARTIAL_INPUT) ||
      g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CONNECTION_CLOSED) ||
      g_error_matches (error, G_RESOLVER_ERROR, G_RESOLVER_ERROR_NOT_FOUND) ||
      g_error_matches (error, G_RESOLVER_ERROR, G_RESOLVER_ERROR_TEMPORARY_FAILURE))
    {
      g_info ("Should retry request (remaining: %u retries), due to transient error: %s",
              n_retries_remaining, error->message);
      return TRUE;
    }

  return FALSE;
}

GBytes *
flatpak_load_uri_full (FlatpakHttpSession    *http_session,
                       const char            *uri,
                       FlatpakCertificates   *certificates,
                       FlatpakHTTPFlags       flags,
                       const char            *auth,
                       const char            *token,
                       FlatpakLoadUriProgress progress,
                       gpointer               user_data,
                       int                   *out_status,
                       char                 **out_content_type,
                       char                 **out_www_authenticate,
                       GCancellable          *cancellable,
                       GError               **error)
{
  g_auto(LoadUriData) data = { NULL };
  g_autoptr(GError) local_error = NULL;
  guint n_retries_remaining = DEFAULT_N_NETWORK_RETRIES;
  g_autoptr(GMainContextPopDefault) main_context = NULL;
  gboolean success = FALSE;

  /* Ensure we handle file: uris the same independent of backend */
  if (g_ascii_strncasecmp (uri, "file:", 5) == 0)
    {
      g_autoptr(GFile) file = g_file_new_for_uri (uri);
      gchar *contents;
      gsize len;

      if (!g_file_load_contents (file, cancellable, &contents, &len, NULL, error))
        return NULL;

      return g_bytes_new_take (g_steal_pointer (&contents), len);
    }

  main_context = flatpak_main_context_new_default ();

  data.context = main_context;
  data.progress = progress;
  data.user_data = user_data;
  data.last_progress_time = g_get_monotonic_time ();
  data.cancellable = cancellable;
  data.flags = flags;
  data.certificates = certificates;
  data.auth = auth;
  data.token = token;

  data.content = g_string_new ("");

  do
    {
      if (n_retries_remaining < DEFAULT_N_NETWORK_RETRIES)
        {
          g_clear_error (&local_error);
          reset_load_uri_data (&data);
        }

      success = flatpak_download_http_uri_once (http_session, &data, uri, &local_error);

      if (success)
        break;

      g_assert (local_error != NULL);
    }
  while (flatpak_http_should_retry_request (local_error, n_retries_remaining--));

  if (success)
    {
      if (out_content_type)
        *out_content_type = g_steal_pointer (&data.hdr_content_type);

      if (out_www_authenticate)
        *out_www_authenticate = g_steal_pointer (&data.hdr_www_authenticate);

      if (out_status)
        *out_status = data.status;

      return g_string_free_to_bytes (g_steal_pointer (&data.content));
    }

  g_assert (local_error != NULL);
  g_propagate_error (error, g_steal_pointer (&local_error));
  return NULL;
}


GBytes *
flatpak_load_uri (FlatpakHttpSession    *http_session,
                  const char            *uri,
                  FlatpakHTTPFlags       flags,
                  const char            *token,
                  FlatpakLoadUriProgress progress,
                  gpointer               user_data,
                  char                 **out_content_type,
                  GCancellable          *cancellable,
                  GError               **error)
{
  return flatpak_load_uri_full (http_session, uri, NULL, flags, NULL, token,
                                progress, user_data, NULL, out_content_type, NULL,
                                cancellable, error);
}

gboolean
flatpak_download_http_uri (FlatpakHttpSession    *http_session,
                           const char            *uri,
                           FlatpakCertificates   *certificates,
                           FlatpakHTTPFlags       flags,
                           GOutputStream         *out,
                           const char            *token,
                           FlatpakLoadUriProgress progress,
                           gpointer               user_data,
                           GCancellable          *cancellable,
                           GError               **error)
{
  g_auto(LoadUriData) data = { NULL };
  g_autoptr(GError) local_error = NULL;
  guint n_retries_remaining = DEFAULT_N_NETWORK_RETRIES;
  g_autoptr(GMainContextPopDefault) main_context = NULL;
  gboolean success = FALSE;

  main_context = flatpak_main_context_new_default ();

  data.context = main_context;
  data.progress = progress;
  data.user_data = user_data;
  data.last_progress_time = g_get_monotonic_time ();
  data.cancellable = cancellable;
  data.certificates = certificates;
  data.flags = flags;
  data.token = token;

  data.out = out;

  do
    {
      if (n_retries_remaining < DEFAULT_N_NETWORK_RETRIES)
        {
          g_clear_error (&local_error);
          reset_load_uri_data (&data);
        }

      success =  flatpak_download_http_uri_once (http_session, &data, uri, &local_error);

      if (success)
        break;

      g_assert (local_error != NULL);

      /* If the output stream has already been written to we can't retry.
       * TODO: use a range request to resume the download */
      if (data.downloaded_bytes > 0)
        break;
    }
  while (flatpak_http_should_retry_request (local_error, n_retries_remaining--));

  if (success)
    return TRUE;

  g_assert (local_error != NULL);
  g_propagate_error (error, g_steal_pointer (&local_error));
  return FALSE;
}

/************************************************************************
 *                        Cached http support                           *
 ***********************************************************************/

#define CACHE_HTTP_XATTR "user.flatpak.http"
#define CACHE_HTTP_SUFFIX ".flatpak.http"
#define CACHE_HTTP_TYPE "(sstt)"

static void
clear_cache_http_data (CacheHttpData *data,
                       gboolean       clear_uri)
{
  if (clear_uri)
    g_clear_pointer (&data->uri, g_free);
  g_clear_pointer (&data->etag, g_free);
  data->last_modified = 0;
  data->expires = 0;
}

static void
free_cache_http_data (CacheHttpData *data)
{
  clear_cache_http_data (data, TRUE);
  g_free (data);
}

G_DEFINE_AUTOPTR_CLEANUP_FUNC (CacheHttpData, free_cache_http_data)

static GBytes *
serialize_cache_http_data (CacheHttpData * data)
{
  g_autoptr(GVariant) cache_variant = NULL;

  cache_variant = g_variant_ref_sink (g_variant_new (CACHE_HTTP_TYPE,
                                                     data->uri,
                                                     data->etag ? data->etag : "",
                                                     data->last_modified,
                                                     data->expires));
  if (G_BYTE_ORDER != G_BIG_ENDIAN)
    {
      g_autoptr(GVariant) tmp_variant = cache_variant;
      cache_variant = g_variant_byteswap (tmp_variant);
    }

  return g_variant_get_data_as_bytes (cache_variant);
}

static void
deserialize_cache_http_data (CacheHttpData *data,
                             GBytes        *bytes)
{
  g_autoptr(GVariant) cache_variant = NULL;

  cache_variant = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE (CACHE_HTTP_TYPE),
                                                                bytes,
                                                                FALSE));
  if (G_BYTE_ORDER != G_BIG_ENDIAN)
    {
      g_autoptr(GVariant) tmp_variant = cache_variant;
      cache_variant = g_variant_byteswap (tmp_variant);
    }

  g_variant_get (cache_variant,
                 CACHE_HTTP_TYPE,
                 &data->uri,
                 &data->etag,
                 &data->last_modified,
                 &data->expires);
}

static CacheHttpData *
load_cache_http_data (int           dfd,
                      char         *name,
                      gboolean     *no_xattr,
                      GCancellable *cancellable,
                      GError      **error)
{
  g_autoptr(CacheHttpData) data = NULL;
  g_autoptr(GBytes) cache_bytes = glnx_lgetxattrat (dfd, name,
                                                    CACHE_HTTP_XATTR,
                                                    error);
  if (cache_bytes == NULL)
    {
      if (errno == ENOTSUP)
        {
          g_autofree char *cache_file = NULL;
          glnx_autofd int fd = -1;

          g_clear_error (error);
          *no_xattr = TRUE;

          cache_file = g_strconcat (name, CACHE_HTTP_SUFFIX, NULL);

          if (!glnx_openat_rdonly (dfd, cache_file, FALSE,
                                   &fd, error))
            return FALSE;

          cache_bytes = glnx_fd_readall_bytes (fd, cancellable, error);
          if (!cache_bytes)
            return NULL;
        }
      else if (errno == ENOENT || errno == ENODATA)
        {
          g_clear_error (error);
          return g_new0 (CacheHttpData, 1);
        }
      else
        {
          return NULL;
        }
    }


  data = g_new0 (CacheHttpData, 1);
  deserialize_cache_http_data (data, cache_bytes);
  return g_steal_pointer (&data);
}

static gboolean
save_cache_http_data_xattr (int      fd,
                            GBytes  *bytes,
                            GError **error)
{
  if (TEMP_FAILURE_RETRY (fsetxattr (fd, (char *) CACHE_HTTP_XATTR,
                                     g_bytes_get_data (bytes, NULL),
                                     g_bytes_get_size (bytes),
                                     0)) < 0)
    return glnx_throw_errno_prefix (error, "fsetxattr");

  return TRUE;
}

static gboolean
save_cache_http_data_fallback (int      fd,
                               GBytes  *bytes,
                               GError **error)
{
  if (glnx_loop_write (fd,
                       g_bytes_get_data (bytes, NULL),
                       g_bytes_get_size (bytes)) < 0)
    return glnx_throw_errno_prefix (error, "write");

  return TRUE;
}

static gboolean
save_cache_http_data_to_file (int           dfd,
                              char         *name,
                              GBytes       *bytes,
                              gboolean      no_xattr,
                              GCancellable *cancellable,
                              GError      **error)
{
  glnx_autofd int fd = -1;
  g_autofree char *fallback_name = NULL;

  if (!no_xattr)
    {
      if (!glnx_openat_rdonly (dfd, name, FALSE,
                               &fd, error))
        return FALSE;

      if (save_cache_http_data_xattr (fd, bytes, error))
        return TRUE;

      if (errno == ENOTSUP)
        g_clear_error (error);
      else
        return FALSE;
    }

  fallback_name = g_strconcat (name, CACHE_HTTP_SUFFIX, NULL);
  if (!glnx_file_replace_contents_at (dfd, fallback_name,
                                      g_bytes_get_data (bytes, NULL),
                                      g_bytes_get_size (bytes),
                                      0,
                                      cancellable,
                                      error))
    return FALSE;

  return TRUE;
}

static gboolean
sync_and_rename_tmpfile (GLnxTmpfile *tmpfile,
                         const char  *dest_name,
                         GError     **error)
{
  /* Filesystem paranoia: If we end up with the new metadata but not
   * the new data, then because the cache headers are in the metadata,
   * we'll never re-download.  (If we just want to avoid losing both
   * the old and new data, skipping fdatasync when the destination is
   * missing works, but it won't here.)
   *
   * This will cause a bunch of fdatasyncs when downloading the icons for
   * a large appstream the first time, would mostly be a problem with a
   * very fast internet connection and a slow spinning drive.
   * Possible solution: update in new directory without fdatasync
   * (copying in any existing cached icons to revalidate), syncfs(), then
   * atomic symlink.
   */
  if (fdatasync (tmpfile->fd) != 0)
    return glnx_throw_errno_prefix (error, "fdatasync");

  if (fchmod (tmpfile->fd, 0644) != 0)
    return glnx_throw_errno_prefix (error, "fchmod");

  if (!glnx_link_tmpfile_at (tmpfile,
                             GLNX_LINK_TMPFILE_REPLACE,
                             tmpfile->src_dfd, dest_name, error))
    return FALSE;

  return TRUE;
}

static void
set_cache_http_data_from_headers (CacheHttpData *cache_data,
                                  LoadUriData *data)
{
  const char *etag = data->hdr_etag;
  const char *last_modified = data->hdr_last_modified;
  const char *cache_control = data->hdr_cache_control;
  const char *expires = data->hdr_expires;
  gboolean expires_computed = FALSE;

  /* The original HTTP 1/1 specification only required sending the ETag header in a 304
   * response, and implied that a cache might need to save the old Cache-Control
   * values. The updated RFC 7232 from 2014 requires sending Cache-Control, ETags, and
   * Expire if they would have been sent in the original 200 response, and recommends
   * sending Last-Modified for requests without an etag. Since sending these headers was
   * apparently normal previously, for simplicity we assume the RFC 7232 behavior and start
   * from scratch for a 304 response.
   */
  clear_cache_http_data (cache_data, FALSE);

  if (etag && *etag)
    {
      cache_data->etag = g_strdup (etag);
    }
  else if (last_modified && *last_modified)
    {
      g_autoptr(GDateTime) date = flatpak_parse_http_time (last_modified);
      if (date)
        cache_data->last_modified = g_date_time_to_unix (date);
    }

  if (cache_control && *cache_control)
    {
      g_autoptr(GHashTable) params = flatpak_parse_http_header_param_list (cache_control);
      GHashTableIter iter;
      gpointer key, value;

      g_hash_table_iter_init (&iter, params);
      while (g_hash_table_iter_next (&iter, &key, &value))
        {
          if (g_strcmp0 (key, "max-age") == 0)
            {
              char *end;

              char *max_age = value;
              int max_age_sec = g_ascii_strtoll (max_age,  &end, 10);
              if (*max_age != '\0' && *end == '\0')
                {
                  GTimeVal now;
                  g_get_current_time (&now);
                  cache_data->expires = now.tv_sec + max_age_sec;
                  expires_computed = TRUE;
                }
            }
          else if (g_strcmp0 (key, "no-cache") == 0)
            {
              cache_data->expires = 0;
              expires_computed = TRUE;
            }
        }
    }

  if (!expires_computed && expires && *expires)
    {
      g_autoptr(GDateTime) date = flatpak_parse_http_time (expires);
      if (date)
        {
          cache_data->expires = g_date_time_to_unix (date);
          expires_computed = TRUE;
        }
    }

  if (!expires_computed)
    {
      /* If nothing implies an expires time, use 30 minutes. Browsers use
       * 0.1 * (Date - Last-Modified), but it's clearly appropriate here, and
       * better if server's send a value.
       */
      GTimeVal now;
      g_get_current_time (&now);
      cache_data->expires = now.tv_sec + 1800;
    }
}

gboolean
flatpak_cache_http_uri (FlatpakHttpSession    *http_session,
                        const char            *uri,
                        FlatpakCertificates   *certificates,
                        FlatpakHTTPFlags       flags,
                        int                    dest_dfd,
                        const char            *dest_subpath,
                        FlatpakLoadUriProgress progress,
                        gpointer               user_data,
                        GCancellable          *cancellable,
                        GError               **error)
{
  g_auto(LoadUriData) data = { NULL };
  g_autoptr(GError) local_error = NULL;
  guint n_retries_remaining = DEFAULT_N_NETWORK_RETRIES;
  g_autoptr(CacheHttpData) cache_data = NULL;
  g_autoptr(GMainContextPopDefault) main_context = NULL;
  g_autofree char *parent_path = g_path_get_dirname (dest_subpath);
  g_autofree char *name = g_path_get_basename (dest_subpath);
  g_auto(GLnxTmpfile) out_tmpfile = { 0 };
  g_auto(GLnxTmpfile) cache_tmpfile = { 0 };
  g_autoptr(GBytes) cache_bytes = NULL;
  gboolean no_xattr = FALSE;
  glnx_autofd int cache_dfd = -1;
  gboolean success;

  if (!glnx_opendirat (dest_dfd, parent_path, TRUE, &cache_dfd, error))
    return FALSE;

  cache_data = load_cache_http_data (cache_dfd, name, &no_xattr,
                                     cancellable, error);
  if (!cache_data)
    return FALSE;

  if (g_strcmp0 (cache_data->uri, uri) != 0)
    clear_cache_http_data (cache_data, TRUE);

  if (cache_data->uri)
    {
      GTimeVal now;

      g_get_current_time (&now);
      if (cache_data->expires > now.tv_sec)
        {
          g_set_error (error, FLATPAK_HTTP_ERROR,
                       FLATPAK_HTTP_ERROR_NOT_CHANGED,
                       "Reusing cached value");
          return FALSE;
        }
    }

  if (cache_data->uri == NULL)
    cache_data->uri = g_strdup (uri);

  /* Missing from cache, or expired so must revalidate via etag/last-modified headers */

  main_context = flatpak_main_context_new_default ();

  data.context = main_context;
  data.progress = progress;
  data.user_data = user_data;
  data.last_progress_time = g_get_monotonic_time ();
  data.cancellable = cancellable;
  data.flags = flags;
  data.certificates = certificates;

  data.cache_data = cache_data;

  data.out_tmpfile = &out_tmpfile;
  data.out_tmpfile_parent_dfd = cache_dfd;

  do
    {
      if (n_retries_remaining < DEFAULT_N_NETWORK_RETRIES)
        {
          g_clear_error (&local_error);
          reset_load_uri_data (&data);
        }

      success = flatpak_download_http_uri_once (http_session, &data, uri, &local_error);

      if (success)
        break;

      g_assert (local_error != NULL);
    }
  while (flatpak_http_should_retry_request (local_error, n_retries_remaining--));

  /* Update the cache data on success or cache-valid */
  if (success || g_error_matches (local_error, FLATPAK_HTTP_ERROR, FLATPAK_HTTP_ERROR_NOT_CHANGED))
    {
      set_cache_http_data_from_headers (cache_data, &data);
      cache_bytes = serialize_cache_http_data (cache_data);
    }

  if (local_error)
    {
      if (cache_bytes)
        {
          GError *tmp_error = NULL;

          if (!save_cache_http_data_to_file (cache_dfd, name, cache_bytes, no_xattr,
                                             cancellable, &tmp_error))
            {
              g_clear_error (&local_error);
              g_propagate_error (error, tmp_error);

              return FALSE;
            }
        }

      g_propagate_error (error, g_steal_pointer (&local_error));
      return FALSE;
    }

  if (!no_xattr)
    {
      if (!save_cache_http_data_xattr (out_tmpfile.fd, cache_bytes, error))
        {
          if (errno != ENOTSUP)
            return FALSE;

          g_clear_error (error);
          no_xattr = TRUE;
        }
    }

  if (no_xattr)
    {
      if (!glnx_open_tmpfile_linkable_at (cache_dfd, ".", O_WRONLY, &cache_tmpfile, error))
        return FALSE;

      if (!save_cache_http_data_fallback (cache_tmpfile.fd, cache_bytes, error))
        return FALSE;
    }

  if (!sync_and_rename_tmpfile (&out_tmpfile, name, error))
    return FALSE;

  if (no_xattr)
    {
      g_autofree char *fallback_name = g_strconcat (name, CACHE_HTTP_SUFFIX, NULL);

      if (!sync_and_rename_tmpfile (&cache_tmpfile, fallback_name, error))
        return FALSE;
    }

  return TRUE;
}