From 06970e015f1abeeeecd065c8d36651d319beb5a9 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 2 May 2024 19:01:57 +0100 Subject: [PATCH] utils: Move more repository functionality into repo-utils Signed-off-by: Simon McVittie --- app/flatpak-builtins-build-bundle.c | 1 + app/flatpak-builtins-build-commit-from.c | 1 + app/flatpak-builtins-build-export.c | 1 + app/flatpak-builtins-build-import-bundle.c | 1 + app/flatpak-builtins-create-usb.c | 1 + common/flatpak-repo-utils-private.h | 44 + common/flatpak-repo-utils.c | 2293 ++++++++++++++++++++ common/flatpak-utils-private.h | 41 - common/flatpak-utils.c | 2290 ------------------- po/POTFILES.in | 1 + 10 files changed, 2343 insertions(+), 2331 deletions(-) diff --git a/app/flatpak-builtins-build-bundle.c b/app/flatpak-builtins-build-bundle.c index 53df7adc..ba9646d5 100644 --- a/app/flatpak-builtins-build-bundle.c +++ b/app/flatpak-builtins-build-bundle.c @@ -34,6 +34,7 @@ #include "libglnx.h" #include "flatpak-builtins.h" +#include "flatpak-repo-utils-private.h" #include "flatpak-utils-private.h" #include "flatpak-oci-registry-private.h" #include "flatpak-chain-input-stream-private.h" diff --git a/app/flatpak-builtins-build-commit-from.c b/app/flatpak-builtins-build-commit-from.c index 7b03aa6a..677f2f31 100644 --- a/app/flatpak-builtins-build-commit-from.c +++ b/app/flatpak-builtins-build-commit-from.c @@ -30,6 +30,7 @@ #include "libglnx.h" #include "flatpak-builtins.h" +#include "flatpak-repo-utils-private.h" #include "flatpak-utils-private.h" #include "parse-datetime.h" diff --git a/app/flatpak-builtins-build-export.c b/app/flatpak-builtins-build-export.c index e2d85dd8..d4cbc489 100644 --- a/app/flatpak-builtins-build-export.c +++ b/app/flatpak-builtins-build-export.c @@ -30,6 +30,7 @@ #include "libglnx.h" #include "flatpak-builtins.h" +#include "flatpak-repo-utils-private.h" #include "flatpak-utils-private.h" #include "parse-datetime.h" diff --git a/app/flatpak-builtins-build-import-bundle.c b/app/flatpak-builtins-build-import-bundle.c index 79ed93b8..d5d6fc26 100644 --- a/app/flatpak-builtins-build-import-bundle.c +++ b/app/flatpak-builtins-build-import-bundle.c @@ -30,6 +30,7 @@ #include "libglnx.h" #include "flatpak-builtins.h" +#include "flatpak-repo-utils-private.h" #include "flatpak-utils-private.h" #include "flatpak-oci-registry-private.h" diff --git a/app/flatpak-builtins-create-usb.c b/app/flatpak-builtins-create-usb.c index 08710535..fc593a29 100644 --- a/app/flatpak-builtins-create-usb.c +++ b/app/flatpak-builtins-create-usb.c @@ -30,6 +30,7 @@ #include "libglnx.h" #include "flatpak-builtins.h" +#include "flatpak-repo-utils-private.h" #include "flatpak-builtins-utils.h" #include "flatpak-utils-private.h" #include "flatpak-error.h" diff --git a/common/flatpak-repo-utils-private.h b/common/flatpak-repo-utils-private.h index 29eea488..a1fb3417 100644 --- a/common/flatpak-repo-utils-private.h +++ b/common/flatpak-repo-utils-private.h @@ -127,3 +127,47 @@ guint flatpak_repo_get_summary_history_length (OstreeRepo *repo); gboolean flatpak_repo_set_gpg_keys (OstreeRepo *repo, GBytes *bytes, GError **error); + +gboolean flatpak_repo_collect_sizes (OstreeRepo *repo, + GFile *root, + guint64 *installed_size, + guint64 *download_size, + GCancellable *cancellable, + GError **error); +GVariant *flatpak_commit_get_extra_data_sources (GVariant *commitv, + GError **error); +GVariant *flatpak_repo_get_extra_data_sources (OstreeRepo *repo, + const char *rev, + GCancellable *cancellable, + GError **error); +void flatpak_repo_parse_extra_data_sources (GVariant *extra_data_sources, + int index, + const char **name, + guint64 *download_size, + guint64 *installed_size, + const guchar **sha256, + const char **uri); +GVariant *flatpak_repo_load_summary (OstreeRepo *repo, + GError **error); +GVariant *flatpak_repo_load_summary_index (OstreeRepo *repo, + GError **error); +GVariant *flatpak_repo_load_digested_summary (OstreeRepo *repo, + const char *digest, + GError **error); + +GBytes *flatpak_summary_apply_diff (GBytes *old, + GBytes *diff, + GError **error); + +typedef enum +{ + FLATPAK_REPO_UPDATE_FLAG_NONE = 0, + FLATPAK_REPO_UPDATE_FLAG_DISABLE_INDEX = 1 << 0, +} FlatpakRepoUpdateFlags; + +gboolean flatpak_repo_update (OstreeRepo *repo, + FlatpakRepoUpdateFlags flags, + const char **gpg_key_ids, + const char *gpg_homedir, + GCancellable *cancellable, + GError **error); diff --git a/common/flatpak-repo-utils.c b/common/flatpak-repo-utils.c index 02e9afa8..29ddc4a2 100644 --- a/common/flatpak-repo-utils.c +++ b/common/flatpak-repo-utils.c @@ -22,6 +22,14 @@ #include "config.h" #include "flatpak-repo-utils-private.h" +#include + +#include + +#include "flatpak-utils-private.h" +#include "flatpak-variant-private.h" +#include "flatpak-variant-impl-private.h" + gboolean flatpak_repo_set_title (OstreeRepo *repo, const char *title, @@ -316,3 +324,2288 @@ flatpak_repo_get_summary_history_length (OstreeRepo *repo) return length; } + +GVariant * +flatpak_commit_get_extra_data_sources (GVariant *commitv, + GError **error) +{ + g_autoptr(GVariant) commit_metadata = NULL; + g_autoptr(GVariant) extra_data_sources = NULL; + + commit_metadata = g_variant_get_child_value (commitv, 0); + extra_data_sources = g_variant_lookup_value (commit_metadata, + "xa.extra-data-sources", + G_VARIANT_TYPE ("a(ayttays)")); + + if (extra_data_sources == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + _("No extra data sources")); + return NULL; + } + + return g_steal_pointer (&extra_data_sources); +} + +GVariant * +flatpak_repo_get_extra_data_sources (OstreeRepo *repo, + const char *rev, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GVariant) commitv = NULL; + + if (!ostree_repo_load_variant (repo, + OSTREE_OBJECT_TYPE_COMMIT, + rev, &commitv, error)) + return NULL; + + return flatpak_commit_get_extra_data_sources (commitv, error); +} + +void +flatpak_repo_parse_extra_data_sources (GVariant *extra_data_sources, + int index, + const char **name, + guint64 *download_size, + guint64 *installed_size, + const guchar **sha256, + const char **uri) +{ + g_autoptr(GVariant) sha256_v = NULL; + g_variant_get_child (extra_data_sources, index, "(^aytt@ay&s)", + name, + download_size, + installed_size, + &sha256_v, + uri); + + if (download_size) + *download_size = GUINT64_FROM_BE (*download_size); + + if (installed_size) + *installed_size = GUINT64_FROM_BE (*installed_size); + + if (sha256) + *sha256 = ostree_checksum_bytes_peek (sha256_v); +} + +#define OSTREE_GIO_FAST_QUERYINFO ("standard::name,standard::type,standard::size,standard::is-symlink,standard::symlink-target," \ + "unix::device,unix::inode,unix::mode,unix::uid,unix::gid,unix::rdev") + +static gboolean +_flatpak_repo_collect_sizes (OstreeRepo *repo, + GFile *file, + GFileInfo *file_info, + guint64 *installed_size, + guint64 *download_size, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GFileEnumerator) dir_enum = NULL; + GFileInfo *child_info_tmp; + g_autoptr(GError) temp_error = NULL; + + if (file_info != NULL && g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) + { + const char *checksum = ostree_repo_file_get_checksum (OSTREE_REPO_FILE (file)); + guint64 obj_size; + guint64 file_size = g_file_info_get_size (file_info); + + if (installed_size) + *installed_size += ((file_size + 511) / 512) * 512; + + if (download_size) + { + g_autoptr(GInputStream) input = NULL; + GInputStream *base_input; + g_autoptr(GError) local_error = NULL; + + if (!ostree_repo_query_object_storage_size (repo, + OSTREE_OBJECT_TYPE_FILE, checksum, + &obj_size, cancellable, &local_error)) + { + int fd; + struct stat stbuf; + + /* Ostree does not look at the staging directory when querying storage + size, so may return a NOT_FOUND error here. We work around this + by loading the object and walking back until we find the original + fd which we can fstat(). */ + if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + return FALSE; + + if (!ostree_repo_load_file (repo, checksum, &input, NULL, NULL, NULL, error)) + return FALSE; + + base_input = input; + while (G_IS_FILTER_INPUT_STREAM (base_input)) + base_input = g_filter_input_stream_get_base_stream (G_FILTER_INPUT_STREAM (base_input)); + + if (!G_IS_UNIX_INPUT_STREAM (base_input)) + return flatpak_fail (error, "Unable to find size of commit %s, not an unix stream", checksum); + + fd = g_unix_input_stream_get_fd (G_UNIX_INPUT_STREAM (base_input)); + + if (fstat (fd, &stbuf) != 0) + return glnx_throw_errno_prefix (error, "Can't find commit size: "); + + obj_size = stbuf.st_size; + } + + *download_size += obj_size; + } + } + + if (file_info == NULL || g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) + { + dir_enum = g_file_enumerate_children (file, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, error); + if (!dir_enum) + return FALSE; + + + while ((child_info_tmp = g_file_enumerator_next_file (dir_enum, cancellable, &temp_error))) + { + g_autoptr(GFileInfo) child_info = child_info_tmp; + const char *name = g_file_info_get_name (child_info); + g_autoptr(GFile) child = g_file_get_child (file, name); + + if (!_flatpak_repo_collect_sizes (repo, child, child_info, installed_size, download_size, cancellable, error)) + return FALSE; + } + } + + return TRUE; +} + +gboolean +flatpak_repo_collect_sizes (OstreeRepo *repo, + GFile *root, + guint64 *installed_size, + guint64 *download_size, + GCancellable *cancellable, + GError **error) +{ + /* Initialize the sums */ + if (installed_size) + *installed_size = 0; + if (download_size) + *download_size = 0; + return _flatpak_repo_collect_sizes (repo, root, NULL, installed_size, download_size, cancellable, error); +} + +static void +flatpak_repo_collect_extra_data_sizes (OstreeRepo *repo, + const char *rev, + guint64 *installed_size, + guint64 *download_size) +{ + g_autoptr(GVariant) extra_data_sources = NULL; + gsize n_extra_data; + int i; + + extra_data_sources = flatpak_repo_get_extra_data_sources (repo, rev, NULL, NULL); + if (extra_data_sources == NULL) + return; + + n_extra_data = g_variant_n_children (extra_data_sources); + if (n_extra_data == 0) + return; + + for (i = 0; i < n_extra_data; i++) + { + guint64 extra_download_size; + guint64 extra_installed_size; + + flatpak_repo_parse_extra_data_sources (extra_data_sources, i, + NULL, + &extra_download_size, + &extra_installed_size, + NULL, NULL); + if (installed_size) + *installed_size += extra_installed_size; + if (download_size) + *download_size += extra_download_size; + } +} + +/* Loads the old compat summary file from a local repo */ +GVariant * +flatpak_repo_load_summary (OstreeRepo *repo, + GError **error) +{ + glnx_autofd int fd = -1; + g_autoptr(GMappedFile) mfile = NULL; + g_autoptr(GBytes) bytes = NULL; + + fd = openat (ostree_repo_get_dfd (repo), "summary", O_RDONLY | O_CLOEXEC); + if (fd < 0) + { + glnx_set_error_from_errno (error); + return NULL; + } + + mfile = g_mapped_file_new_from_fd (fd, FALSE, error); + if (!mfile) + return NULL; + + bytes = g_mapped_file_get_bytes (mfile); + + return g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes, TRUE)); +} + +GVariant * +flatpak_repo_load_summary_index (OstreeRepo *repo, + GError **error) +{ + glnx_autofd int fd = -1; + g_autoptr(GMappedFile) mfile = NULL; + g_autoptr(GBytes) bytes = NULL; + + fd = openat (ostree_repo_get_dfd (repo), "summary.idx", O_RDONLY | O_CLOEXEC); + if (fd < 0) + { + glnx_set_error_from_errno (error); + return NULL; + } + + mfile = g_mapped_file_new_from_fd (fd, FALSE, error); + if (!mfile) + return NULL; + + bytes = g_mapped_file_get_bytes (mfile); + + return g_variant_ref_sink (g_variant_new_from_bytes (FLATPAK_SUMMARY_INDEX_GVARIANT_FORMAT, bytes, TRUE)); +} + +static gboolean +flatpak_repo_save_compat_summary (OstreeRepo *repo, + GVariant *summary, + time_t *out_old_sig_mtime, + GCancellable *cancellable, + GError **error) +{ + int repo_dfd = ostree_repo_get_dfd (repo); + struct stat stbuf; + time_t old_sig_mtime = 0; + GLnxFileReplaceFlags flags; + + flags = GLNX_FILE_REPLACE_INCREASING_MTIME; + if (ostree_repo_get_disable_fsync (repo)) + flags |= GLNX_FILE_REPLACE_NODATASYNC; + else + flags |= GLNX_FILE_REPLACE_DATASYNC_NEW; + + if (!glnx_file_replace_contents_at (repo_dfd, "summary", + g_variant_get_data (summary), + g_variant_get_size (summary), + flags, + cancellable, error)) + return FALSE; + + if (fstatat (repo_dfd, "summary.sig", &stbuf, AT_SYMLINK_NOFOLLOW) == 0) + old_sig_mtime = stbuf.st_mtime; + + if (unlinkat (repo_dfd, "summary.sig", 0) != 0 && + G_UNLIKELY (errno != ENOENT)) + { + glnx_set_error_from_errno (error); + return FALSE; + } + + *out_old_sig_mtime = old_sig_mtime; + return TRUE; +} + +static gboolean +flatpak_repo_save_summary_index (OstreeRepo *repo, + GVariant *index, + const char *index_digest, + GBytes *index_sig, + GCancellable *cancellable, + GError **error) +{ + int repo_dfd = ostree_repo_get_dfd (repo); + GLnxFileReplaceFlags flags; + + if (index == NULL) + { + if (unlinkat (repo_dfd, "summary.idx", 0) != 0 && + G_UNLIKELY (errno != ENOENT)) + { + glnx_set_error_from_errno (error); + return FALSE; + } + if (unlinkat (repo_dfd, "summary.idx.sig", 0) != 0 && + G_UNLIKELY (errno != ENOENT)) + { + glnx_set_error_from_errno (error); + return FALSE; + } + + return TRUE; + } + + flags = GLNX_FILE_REPLACE_INCREASING_MTIME; + if (ostree_repo_get_disable_fsync (repo)) + flags |= GLNX_FILE_REPLACE_NODATASYNC; + else + flags |= GLNX_FILE_REPLACE_DATASYNC_NEW; + + if (index_sig) + { + g_autofree char *path = g_strconcat ("summaries/", index_digest, ".idx.sig", NULL); + + if (!glnx_shutil_mkdir_p_at (repo_dfd, "summaries", + 0775, cancellable, error)) + return FALSE; + + if (!glnx_file_replace_contents_at (repo_dfd, path, + g_bytes_get_data (index_sig, NULL), + g_bytes_get_size (index_sig), + flags, + cancellable, error)) + return FALSE; + } + + if (!glnx_file_replace_contents_at (repo_dfd, "summary.idx", + g_variant_get_data (index), + g_variant_get_size (index), + flags, + cancellable, error)) + return FALSE; + + /* Update the non-indexed summary.idx.sig file that was introduced in 1.9.1 but + * was made unnecessary in 1.9.3. Lets keep it for a while until everyone updates + */ + if (index_sig) + { + if (!glnx_file_replace_contents_at (repo_dfd, "summary.idx.sig", + g_bytes_get_data (index_sig, NULL), + g_bytes_get_size (index_sig), + flags, + cancellable, error)) + return FALSE; + } + else + { + if (unlinkat (repo_dfd, "summary.idx.sig", 0) != 0 && + G_UNLIKELY (errno != ENOENT)) + { + glnx_set_error_from_errno (error); + return FALSE; + } + } + + return TRUE; +} + +GVariant * +flatpak_repo_load_digested_summary (OstreeRepo *repo, + const char *digest, + GError **error) +{ + glnx_autofd int fd = -1; + g_autoptr(GMappedFile) mfile = NULL; + g_autoptr(GBytes) bytes = NULL; + g_autoptr(GBytes) compressed_bytes = NULL; + g_autofree char *path = NULL; + g_autofree char *filename = NULL; + + filename = g_strconcat (digest, ".gz", NULL); + path = g_build_filename ("summaries", filename, NULL); + + fd = openat (ostree_repo_get_dfd (repo), path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + { + glnx_set_error_from_errno (error); + return NULL; + } + + mfile = g_mapped_file_new_from_fd (fd, FALSE, error); + if (!mfile) + return NULL; + + compressed_bytes = g_mapped_file_get_bytes (mfile); + bytes = flatpak_zlib_decompress_bytes (compressed_bytes, error); + if (bytes == NULL) + return NULL; + + return g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes, TRUE)); +} + +static char * +flatpak_repo_save_digested_summary (OstreeRepo *repo, + const char *name, + GVariant *summary, + GCancellable *cancellable, + GError **error) +{ + int repo_dfd = ostree_repo_get_dfd (repo); + g_autofree char *digest = NULL; + g_autofree char *filename = NULL; + g_autofree char *path = NULL; + g_autoptr(GBytes) data = NULL; + g_autoptr(GBytes) compressed_data = NULL; + struct stat stbuf; + + if (!glnx_shutil_mkdir_p_at (repo_dfd, "summaries", + 0775, + cancellable, + error)) + return NULL; + + digest = g_compute_checksum_for_data (G_CHECKSUM_SHA256, + g_variant_get_data (summary), + g_variant_get_size (summary)); + filename = g_strconcat (digest, ".gz", NULL); + + path = g_build_filename ("summaries", filename, NULL); + + /* Check for pre-existing (non-truncated) copy and avoid re-writing it */ + if (fstatat (repo_dfd, path, &stbuf, 0) == 0 && + stbuf.st_size != 0) + { + g_info ("Reusing digested summary at %s for %s", path, name); + return g_steal_pointer (&digest); + } + + data = g_variant_get_data_as_bytes (summary); + compressed_data = flatpak_zlib_compress_bytes (data, -1, error); + if (compressed_data == NULL) + return NULL; + + if (!glnx_file_replace_contents_at (repo_dfd, path, + g_bytes_get_data (compressed_data, NULL), + g_bytes_get_size (compressed_data), + ostree_repo_get_disable_fsync (repo) ? GLNX_FILE_REPLACE_NODATASYNC : GLNX_FILE_REPLACE_DATASYNC_NEW, + cancellable, error)) + return NULL; + + g_info ("Wrote digested summary at %s for %s", path, name); + return g_steal_pointer (&digest); +} + +static gboolean +flatpak_repo_save_digested_summary_delta (OstreeRepo *repo, + const char *from_digest, + const char *to_digest, + GBytes *delta, + GCancellable *cancellable, + GError **error) +{ + int repo_dfd = ostree_repo_get_dfd (repo); + g_autofree char *path = NULL; + g_autofree char *filename = g_strconcat (from_digest, "-", to_digest, ".delta", NULL); + struct stat stbuf; + + if (!glnx_shutil_mkdir_p_at (repo_dfd, "summaries", + 0775, + cancellable, + error)) + return FALSE; + + path = g_build_filename ("summaries", filename, NULL); + + /* Check for pre-existing copy of same size and avoid re-writing it */ + if (fstatat (repo_dfd, path, &stbuf, 0) == 0 && + stbuf.st_size == g_bytes_get_size (delta)) + { + g_info ("Reusing digested summary-diff for %s", filename); + return TRUE; + } + + if (!glnx_file_replace_contents_at (repo_dfd, path, + g_bytes_get_data (delta, NULL), + g_bytes_get_size (delta), + ostree_repo_get_disable_fsync (repo) ? GLNX_FILE_REPLACE_NODATASYNC : GLNX_FILE_REPLACE_DATASYNC_NEW, + cancellable, error)) + return FALSE; + + g_info ("Wrote digested summary delta at %s", path); + return TRUE; +} + +typedef struct +{ + guint64 installed_size; + guint64 download_size; + char *metadata_contents; + GPtrArray *subsets; + GVariant *sparse_data; + gsize commit_size; + guint64 commit_timestamp; +} CommitData; + +static void +commit_data_free (gpointer data) +{ + CommitData *rev_data = data; + + if (rev_data->subsets) + g_ptr_array_unref (rev_data->subsets); + g_free (rev_data->metadata_contents); + if (rev_data->sparse_data) + g_variant_unref (rev_data->sparse_data); + g_free (rev_data); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (CommitData, commit_data_free); + +static GHashTable * +commit_data_cache_new (void) +{ + return g_hash_table_new_full (g_str_hash, g_str_equal, g_free, commit_data_free); +} + +static GHashTable * +populate_commit_data_cache (OstreeRepo *repo, + GVariant *index_v) +{ + + VarSummaryIndexRef index = var_summary_index_from_gvariant (index_v); + VarMetadataRef index_metadata = var_summary_index_get_metadata (index); + VarSummaryIndexSubsummariesRef subsummaries = var_summary_index_get_subsummaries (index); + gsize n_subsummaries = var_summary_index_subsummaries_get_length (subsummaries); + guint32 cache_version; + g_autoptr(GHashTable) commit_data_cache = commit_data_cache_new (); + + cache_version = GUINT32_FROM_LE (var_metadata_lookup_uint32 (index_metadata, "xa.cache-version", 0)); + if (cache_version < FLATPAK_XA_CACHE_VERSION) + { + /* Need to re-index to get all data */ + g_info ("Old summary cache version %d, not using cache", cache_version); + return NULL; + } + + for (gsize i = 0; i < n_subsummaries; i++) + { + VarSummaryIndexSubsummariesEntryRef entry = var_summary_index_subsummaries_get_at (subsummaries, i); + const char *name = var_summary_index_subsummaries_entry_get_key (entry); + const char *s; + g_autofree char *subset = NULL; + VarSubsummaryRef subsummary = var_summary_index_subsummaries_entry_get_value (entry); + gsize checksum_bytes_len; + const guchar *checksum_bytes; + g_autofree char *digest = NULL; + g_autoptr(GVariant) summary_v = NULL; + VarSummaryRef summary; + VarRefMapRef ref_map; + gsize n_refs; + + checksum_bytes = var_subsummary_peek_checksum (subsummary, &checksum_bytes_len); + if (G_UNLIKELY (checksum_bytes_len != OSTREE_SHA256_DIGEST_LEN)) + { + g_info ("Invalid checksum for digested summary, not using cache"); + return NULL; + } + digest = ostree_checksum_from_bytes (checksum_bytes); + + s = strrchr (name, '-'); + if (s != NULL) + subset = g_strndup (name, s - name); + else + subset = g_strdup (""); + + summary_v = flatpak_repo_load_digested_summary (repo, digest, NULL); + if (summary_v == NULL) + { + g_info ("Failed to load digested summary %s, not using cache", digest); + return NULL; + } + + /* Note that all summaries refered to by the index is in new format */ + summary = var_summary_from_gvariant (summary_v); + ref_map = var_summary_get_ref_map (summary); + n_refs = var_ref_map_get_length (ref_map); + for (gsize j = 0; j < n_refs; j++) + { + VarRefMapEntryRef e = var_ref_map_get_at (ref_map, j); + const char *ref = var_ref_map_entry_get_ref (e); + VarRefInfoRef info = var_ref_map_entry_get_info (e); + VarMetadataRef commit_metadata = var_ref_info_get_metadata (info); + guint64 commit_size = var_ref_info_get_commit_size (info); + const guchar *commit_bytes; + gsize commit_bytes_len; + g_autofree char *rev = NULL; + CommitData *rev_data; + VarVariantRef xa_data_v; + VarCacheDataRef xa_data; + + if (!flatpak_is_app_runtime_or_appstream_ref (ref)) + continue; + + commit_bytes = var_ref_info_peek_checksum (info, &commit_bytes_len); + if (G_UNLIKELY (commit_bytes_len != OSTREE_SHA256_DIGEST_LEN)) + continue; + + if (!var_metadata_lookup (commit_metadata, "xa.data", NULL, &xa_data_v) || + !var_variant_is_type (xa_data_v, G_VARIANT_TYPE ("(tts)"))) + { + g_info ("Missing xa.data for ref %s, not using cache", ref); + return NULL; + } + + xa_data = var_cache_data_from_variant (xa_data_v); + + rev = ostree_checksum_from_bytes (commit_bytes); + rev_data = g_hash_table_lookup (commit_data_cache, rev); + if (rev_data == NULL) + { + g_auto(GVariantBuilder) sparse_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER; + g_variant_builder_init (&sparse_builder, G_VARIANT_TYPE_VARDICT); + gboolean has_sparse = FALSE; + + rev_data = g_new0 (CommitData, 1); + rev_data->installed_size = var_cache_data_get_installed_size (xa_data); + rev_data->download_size = var_cache_data_get_download_size (xa_data); + rev_data->metadata_contents = g_strdup (var_cache_data_get_metadata (xa_data)); + rev_data->commit_size = commit_size; + rev_data->commit_timestamp = GUINT64_FROM_BE (var_metadata_lookup_uint64 (commit_metadata, OSTREE_COMMIT_TIMESTAMP2, 0)); + + /* Get sparse data */ + gsize len = var_metadata_get_length (commit_metadata); + for (gsize k = 0; k < len; k++) + { + VarMetadataEntryRef m = var_metadata_get_at (commit_metadata, k); + const char *m_key = var_metadata_entry_get_key (m); + if (!g_str_has_prefix (m_key, "ot.") && + !g_str_has_prefix (m_key, "ostree.") && + strcmp (m_key, "xa.data") != 0) + { + VarVariantRef v = var_metadata_entry_get_value (m); + g_autoptr(GVariant) vv = g_variant_ref_sink (var_variant_dup_to_gvariant (v)); + g_autoptr(GVariant) child = g_variant_get_child_value (vv, 0); + g_variant_builder_add (&sparse_builder, "{sv}", m_key, child); + has_sparse = TRUE; + } + } + + if (has_sparse) + rev_data->sparse_data = g_variant_ref_sink (g_variant_builder_end (&sparse_builder)); + + g_hash_table_insert (commit_data_cache, g_strdup (rev), (CommitData *)rev_data); + } + + if (*subset != 0) + { + if (rev_data->subsets == NULL) + rev_data->subsets = g_ptr_array_new_with_free_func (g_free); + + if (!flatpak_g_ptr_array_contains_string (rev_data->subsets, subset)) + g_ptr_array_add (rev_data->subsets, g_strdup (subset)); + } + } + } + + return g_steal_pointer (&commit_data_cache); +} + +static CommitData * +read_commit_data (OstreeRepo *repo, + const char *ref, + const char *rev, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GFile) root = NULL; + g_autoptr(GFile) metadata = NULL; + guint64 installed_size = 0; + guint64 download_size = 0; + g_autofree char *metadata_contents = NULL; + g_autofree char *commit = NULL; + g_autoptr(GVariant) commit_v = NULL; + g_autoptr(GVariant) commit_metadata = NULL; + g_autoptr(GPtrArray) subsets = NULL; + CommitData *rev_data; + const char *eol = NULL; + const char *eol_rebase = NULL; + int token_type = -1; + g_autoptr(GVariant) extra_data_sources = NULL; + guint32 n_extra_data = 0; + guint64 total_extra_data_download_size = 0; + g_autoptr(GVariantIter) subsets_iter = NULL; + + if (!ostree_repo_read_commit (repo, rev, &root, &commit, NULL, error)) + return NULL; + + if (!ostree_repo_load_commit (repo, commit, &commit_v, NULL, error)) + return NULL; + + commit_metadata = g_variant_get_child_value (commit_v, 0); + if (!g_variant_lookup (commit_metadata, "xa.metadata", "s", &metadata_contents)) + { + metadata = g_file_get_child (root, "metadata"); + if (!g_file_load_contents (metadata, cancellable, &metadata_contents, NULL, NULL, NULL)) + metadata_contents = g_strdup (""); + } + + if (g_variant_lookup (commit_metadata, "xa.installed-size", "t", &installed_size) && + g_variant_lookup (commit_metadata, "xa.download-size", "t", &download_size)) + { + installed_size = GUINT64_FROM_BE (installed_size); + download_size = GUINT64_FROM_BE (download_size); + } + else + { + if (!flatpak_repo_collect_sizes (repo, root, &installed_size, &download_size, cancellable, error)) + return NULL; + } + + if (g_variant_lookup (commit_metadata, "xa.subsets", "as", &subsets_iter)) + { + const char *subset; + subsets = g_ptr_array_new_with_free_func (g_free); + while (g_variant_iter_next (subsets_iter, "&s", &subset)) + g_ptr_array_add (subsets, g_strdup (subset)); + } + + flatpak_repo_collect_extra_data_sizes (repo, rev, &installed_size, &download_size); + + rev_data = g_new0 (CommitData, 1); + rev_data->installed_size = installed_size; + rev_data->download_size = download_size; + rev_data->metadata_contents = g_steal_pointer (&metadata_contents); + rev_data->subsets = g_steal_pointer (&subsets); + rev_data->commit_size = g_variant_get_size (commit_v); + rev_data->commit_timestamp = ostree_commit_get_timestamp (commit_v); + + g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_ENDOFLIFE, "&s", &eol); + g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_ENDOFLIFE_REBASE, "&s", &eol_rebase); + if (g_variant_lookup (commit_metadata, "xa.token-type", "i", &token_type)) + token_type = GINT32_FROM_LE(token_type); + + extra_data_sources = flatpak_commit_get_extra_data_sources (commit_v, NULL); + if (extra_data_sources) + { + n_extra_data = g_variant_n_children (extra_data_sources); + for (int i = 0; i < n_extra_data; i++) + { + guint64 extra_download_size; + flatpak_repo_parse_extra_data_sources (extra_data_sources, i, + NULL, + &extra_download_size, + NULL, + NULL, + NULL); + total_extra_data_download_size += extra_download_size; + } + } + + if (eol || eol_rebase || token_type >= 0 || n_extra_data > 0) + { + g_auto(GVariantBuilder) sparse_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER; + g_variant_builder_init (&sparse_builder, G_VARIANT_TYPE_VARDICT); + if (eol) + g_variant_builder_add (&sparse_builder, "{sv}", FLATPAK_SPARSE_CACHE_KEY_ENDOFLINE, g_variant_new_string (eol)); + if (eol_rebase) + g_variant_builder_add (&sparse_builder, "{sv}", FLATPAK_SPARSE_CACHE_KEY_ENDOFLINE_REBASE, g_variant_new_string (eol_rebase)); + if (token_type >= 0) + g_variant_builder_add (&sparse_builder, "{sv}", FLATPAK_SPARSE_CACHE_KEY_TOKEN_TYPE, g_variant_new_int32 (GINT32_TO_LE(token_type))); + if (n_extra_data > 0) + g_variant_builder_add (&sparse_builder, "{sv}", FLATPAK_SPARSE_CACHE_KEY_EXTRA_DATA_SIZE, + g_variant_new ("(ut)", GUINT32_TO_LE(n_extra_data), GUINT64_TO_LE(total_extra_data_download_size))); + + rev_data->sparse_data = g_variant_ref_sink (g_variant_builder_end (&sparse_builder)); + } + + return rev_data; +} + +static void +_ostree_parse_delta_name (const char *delta_name, + char **out_from, + char **out_to) +{ + g_auto(GStrv) parts = g_strsplit (delta_name, "-", 2); + + if (parts[0] && parts[1]) + { + *out_from = g_steal_pointer (&parts[0]); + *out_to = g_steal_pointer (&parts[1]); + } + else + { + *out_from = NULL; + *out_to = g_steal_pointer (&parts[0]); + } +} + +static GString * +static_delta_path_base (const char *dir, + const char *from, + const char *to) +{ + guint8 csum_to[OSTREE_SHA256_DIGEST_LEN]; + char to_b64[44]; + guint8 csum_to_copy[OSTREE_SHA256_DIGEST_LEN]; + GString *ret = g_string_new (dir); + + ostree_checksum_inplace_to_bytes (to, csum_to); + ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64); + ostree_checksum_b64_inplace_to_bytes (to_b64, csum_to_copy); + + g_assert (memcmp (csum_to, csum_to_copy, OSTREE_SHA256_DIGEST_LEN) == 0); + + if (from != NULL) + { + guint8 csum_from[OSTREE_SHA256_DIGEST_LEN]; + char from_b64[44]; + + ostree_checksum_inplace_to_bytes (from, csum_from); + ostree_checksum_b64_inplace_from_bytes (csum_from, from_b64); + + g_string_append_c (ret, from_b64[0]); + g_string_append_c (ret, from_b64[1]); + g_string_append_c (ret, '/'); + g_string_append (ret, from_b64 + 2); + g_string_append_c (ret, '-'); + } + + g_string_append_c (ret, to_b64[0]); + g_string_append_c (ret, to_b64[1]); + if (from == NULL) + g_string_append_c (ret, '/'); + g_string_append (ret, to_b64 + 2); + + return ret; +} + +static char * +_ostree_get_relative_static_delta_path (const char *from, + const char *to, + const char *target) +{ + GString *ret = static_delta_path_base ("deltas/", from, to); + + if (target != NULL) + { + g_string_append_c (ret, '/'); + g_string_append (ret, target); + } + + return g_string_free (ret, FALSE); +} + +static char * +_ostree_get_relative_static_delta_superblock_path (const char *from, + const char *to) +{ + return _ostree_get_relative_static_delta_path (from, to, "superblock"); +} + +static GVariant * +_ostree_repo_static_delta_superblock_digest (OstreeRepo *repo, + const char *from, + const char *to, + GCancellable *cancellable, + GError **error) +{ + g_autofree char *superblock = _ostree_get_relative_static_delta_superblock_path ((from && from[0]) ? from : NULL, to); + glnx_autofd int fd = -1; + guint8 digest[OSTREE_SHA256_DIGEST_LEN]; + gsize len; + gpointer data = NULL; + + if (!glnx_openat_rdonly (ostree_repo_get_dfd (repo), superblock, TRUE, &fd, error)) + return NULL; + + g_autoptr(GBytes) superblock_content = glnx_fd_readall_bytes (fd, cancellable, error); + if (!superblock_content) + return NULL; + + g_autoptr(GChecksum) checksum = g_checksum_new (G_CHECKSUM_SHA256); + g_checksum_update (checksum, g_bytes_get_data (superblock_content, NULL), g_bytes_get_size (superblock_content)); + len = sizeof digest; + g_checksum_get_digest (checksum, digest, &len); + + data = g_memdup2 (digest, len); + return g_variant_new_from_data (G_VARIANT_TYPE ("ay"), + data, len, + FALSE, g_free, data); +} + +typedef enum { + DIFF_OP_KIND_RESUSE_OLD, + DIFF_OP_KIND_SKIP_OLD, + DIFF_OP_KIND_DATA, +} DiffOpKind; + +typedef struct { + DiffOpKind kind; + gsize size; +} DiffOp; + +typedef struct { + const guchar *old_data; + const guchar *new_data; + + GArray *ops; + GArray *data; + + gsize last_old_offset; + gsize last_new_offset; +} DiffData; + +static gsize +match_bytes_at_start (const guchar *data1, + gsize data1_len, + const guchar *data2, + gsize data2_len) +{ + gsize len = 0; + gsize max_len = MIN (data1_len, data2_len); + + while (len < max_len) + { + if (*data1 != *data2) + break; + data1++; + data2++; + len++; + } + return len; +} + +static gsize +match_bytes_at_end (const guchar *data1, + gsize data1_len, + const guchar *data2, + gsize data2_len) +{ + gsize len = 0; + gsize max_len = MIN (data1_len, data2_len); + + data1 += data1_len - 1; + data2 += data2_len - 1; + + while (len < max_len) + { + if (*data1 != *data2) + break; + data1--; + data2--; + len++; + } + return len; +} + +static DiffOp * +diff_ensure_op (DiffData *data, + DiffOpKind kind) +{ + if (data->ops->len == 0 || + g_array_index (data->ops, DiffOp, data->ops->len-1).kind != kind) + { + DiffOp op = {kind, 0}; + g_array_append_val (data->ops, op); + } + + return &g_array_index (data->ops, DiffOp, data->ops->len-1); +} + +static void +diff_emit_reuse (DiffData *data, + gsize size) +{ + DiffOp *op; + + if (size == 0) + return; + + op = diff_ensure_op (data, DIFF_OP_KIND_RESUSE_OLD); + op->size += size; +} + +static void +diff_emit_skip (DiffData *data, + gsize size) +{ + DiffOp *op; + + if (size == 0) + return; + + op = diff_ensure_op (data, DIFF_OP_KIND_SKIP_OLD); + op->size += size; +} + +static void +diff_emit_data (DiffData *data, + gsize size, + const guchar *new_data) +{ + DiffOp *op; + + if (size == 0) + return; + + op = diff_ensure_op (data, DIFF_OP_KIND_DATA); + op->size += size; + + g_array_append_vals (data->data, new_data, size); +} + +static GBytes * +diff_encode (DiffData *data, GError **error) +{ + g_autoptr(GOutputStream) mem = g_memory_output_stream_new_resizable (); + g_autoptr(GDataOutputStream) out = g_data_output_stream_new (mem); + gsize ops_count = 0; + + g_data_output_stream_set_byte_order (out, G_DATA_STREAM_BYTE_ORDER_LITTLE_ENDIAN); + + /* Header */ + if (!g_output_stream_write_all (G_OUTPUT_STREAM (out), + FLATPAK_SUMMARY_DIFF_HEADER, 4, + NULL, NULL, error)) + return NULL; + + /* Write the ops count placeholder */ + if (!g_data_output_stream_put_uint32 (out, 0, NULL, error)) + return NULL; + + for (gsize i = 0; i < data->ops->len; i++) + { + DiffOp *op = &g_array_index (data->ops, DiffOp, i); + gsize size = op->size; + + while (size > 0) + { + /* We leave a nibble at the top for the op */ + guint32 opdata = (guint64)size & 0x0fffffff; + size -= opdata; + + opdata = opdata | ((0xf & op->kind) << 28); + + if (!g_data_output_stream_put_uint32 (out, opdata, NULL, error)) + return NULL; + ops_count++; + } + } + + /* Then add the data */ + if (data->data->len > 0 && + !g_output_stream_write_all (G_OUTPUT_STREAM (out), + data->data->data, data->data->len, + NULL, NULL, error)) + return NULL; + + /* Back-patch in the ops count */ + if (!g_seekable_seek (G_SEEKABLE(out), 4, G_SEEK_SET, NULL, error)) + return NULL; + + if (!g_data_output_stream_put_uint32 (out, ops_count, NULL, error)) + return NULL; + + if (!g_output_stream_close (G_OUTPUT_STREAM (out), NULL, error)) + return NULL; + + return g_memory_output_stream_steal_as_bytes (G_MEMORY_OUTPUT_STREAM (mem)); +} + +static void +diff_consume_block2 (DiffData *data, + gsize consume_old_offset, + gsize consume_old_size, + gsize produce_new_offset, + gsize produce_new_size) +{ + /* We consumed $consume_old_size bytes from $consume_old_offset to + produce $produce_new_size bytes at $produce_new_size */ + + /* First we copy old data for any matching prefix of the block */ + + gsize prefix_len = match_bytes_at_start (data->old_data + consume_old_offset, consume_old_size, + data->new_data + produce_new_offset, produce_new_size); + diff_emit_reuse (data, prefix_len); + + consume_old_size -= prefix_len; + consume_old_offset += prefix_len; + + produce_new_size -= prefix_len; + produce_new_offset += prefix_len; + + /* Then we find the matching suffix for the rest */ + gsize suffix_len = match_bytes_at_end (data->old_data + consume_old_offset, consume_old_size, + data->new_data + produce_new_offset, produce_new_size); + + /* Skip source data until suffix match */ + diff_emit_skip (data, consume_old_size - suffix_len); + + /* Copy new data until suffix match */ + diff_emit_data (data, produce_new_size - suffix_len, data->new_data + produce_new_offset); + + diff_emit_reuse (data, suffix_len); +} + +static void +diff_consume_block (DiffData *data, + gssize consume_old_offset, + gsize consume_old_size, + gssize produce_new_offset, + gsize produce_new_size) +{ + if (consume_old_offset == -1) + consume_old_offset = data->last_old_offset; + if (produce_new_offset == -1) + produce_new_offset = data->last_new_offset; + + /* We consumed $consume_old_size bytes from $consume_old_offset to + * produce $produce_new_size bytes at $produce_new_size, however + * while the emitted blocks are in order they may not cover the + * every byte, so we emit the inbetwen blocks separately. */ + + if (consume_old_offset != data->last_old_offset || + produce_new_offset != data->last_new_offset) + diff_consume_block2 (data, + data->last_old_offset, consume_old_offset - data->last_old_offset , + data->last_new_offset, produce_new_offset - data->last_new_offset); + + diff_consume_block2 (data, + consume_old_offset, consume_old_size, + produce_new_offset, produce_new_size); + + data->last_old_offset = consume_old_offset + consume_old_size; + data->last_new_offset = produce_new_offset + produce_new_size; +} + +GBytes * +flatpak_summary_apply_diff (GBytes *old, + GBytes *diff, + GError **error) +{ + g_autoptr(GBytes) uncompressed = NULL; + const guchar *diffdata; + gsize diff_size; + guint32 *ops; + guint32 n_ops; + gsize data_offset; + gsize data_size; + const guchar *data; + const guchar *old_data = g_bytes_get_data (old, NULL); + gsize old_size = g_bytes_get_size (old); + g_autoptr(GByteArray) res = g_byte_array_new (); + + uncompressed = flatpak_zlib_decompress_bytes (diff, error); + if (uncompressed == NULL) + { + g_prefix_error (error, "Invalid summary diff: "); + return NULL; + } + + diffdata = g_bytes_get_data (uncompressed, NULL); + diff_size = g_bytes_get_size (uncompressed); + + if (diff_size < 8 || + memcmp (diffdata, FLATPAK_SUMMARY_DIFF_HEADER, 4) != 0) + { + flatpak_fail (error, "Invalid summary diff"); + return NULL; + } + + n_ops = GUINT32_FROM_LE (*(guint32 *)(diffdata+4)); + ops = (guint32 *)(diffdata+8); + + data_offset = 4 + 4 + 4 * n_ops; + + /* All ops must fit in diff, and avoid wrapping the multiply */ + if (data_offset > diff_size || + (data_offset - 4 - 4) / 4 != n_ops) + { + flatpak_fail (error, "Invalid summary diff"); + return NULL; + } + + data = diffdata + data_offset; + data_size = diff_size - data_offset; + + for (gsize i = 0; i < n_ops; i++) + { + guint32 opdata = GUINT32_FROM_LE (ops[i]); + guint32 kind = (opdata & 0xf0000000) >> 28; + guint32 size = opdata & 0x0fffffff; + + switch (kind) + { + case DIFF_OP_KIND_RESUSE_OLD: + if (size > old_size) + { + flatpak_fail (error, "Invalid summary diff"); + return NULL; + } + g_byte_array_append (res, old_data, size); + old_data += size; + old_size -= size; + break; + case DIFF_OP_KIND_SKIP_OLD: + if (size > old_size) + { + flatpak_fail (error, "Invalid summary diff"); + return NULL; + } + old_data += size; + old_size -= size; + break; + case DIFF_OP_KIND_DATA: + if (size > data_size) + { + flatpak_fail (error, "Invalid summary diff"); + return NULL; + } + g_byte_array_append (res, data, size); + data += size; + data_size -= size; + break; + default: + flatpak_fail (error, "Invalid summary diff"); + return NULL; + } + } + + return g_byte_array_free_to_bytes (g_steal_pointer (&res)); +} + + +static GBytes * +flatpak_summary_generate_diff (GVariant *old_v, + GVariant *new_v, + GError **error) +{ + VarSummaryRef new, old; + VarRefMapRef new_refs, old_refs; + VarRefMapEntryRef new_entry, old_entry; + gsize new_len, old_len; + int new_i, old_i; + const char *old_ref, *new_ref; + g_autoptr(GArray) ops = g_array_new (FALSE, TRUE, sizeof (DiffOp)); + g_autoptr(GArray) data_bytes = g_array_new (FALSE, TRUE, 1); + g_autoptr(GBytes) diff_uncompressed = NULL; + g_autoptr(GBytes) diff_compressed = NULL; + DiffData data = { + g_variant_get_data (old_v), + g_variant_get_data (new_v), + ops, + data_bytes, + }; + + new = var_summary_from_gvariant (new_v); + old = var_summary_from_gvariant (old_v); + + new_refs = var_summary_get_ref_map (new); + old_refs = var_summary_get_ref_map (old); + + new_len = var_ref_map_get_length (new_refs); + old_len = var_ref_map_get_length (old_refs); + + new_i = old_i = 0; + while (new_i < new_len && old_i < old_len) + { + if (new_i == new_len) + { + /* Just old left */ + old_entry = var_ref_map_get_at (old_refs, old_i); + old_ref = var_ref_map_entry_get_ref (old_entry); + old_i++; + diff_consume_block (&data, + -1, 0, + (const guchar *)new_entry.base - (const guchar *)new.base, new_entry.size); + } + else if (old_i == old_len) + { + /* Just new left */ + new_entry = var_ref_map_get_at (new_refs, new_i); + new_ref = var_ref_map_entry_get_ref (new_entry); + diff_consume_block (&data, + (const guchar *)old_entry.base - (const guchar *)old.base, old_entry.size, + -1, 0); + + new_i++; + } + else + { + new_entry = var_ref_map_get_at (new_refs, new_i); + new_ref = var_ref_map_entry_get_ref (new_entry); + + old_entry = var_ref_map_get_at (old_refs, old_i); + old_ref = var_ref_map_entry_get_ref (old_entry); + + int cmp = strcmp (new_ref, old_ref); + if (cmp == 0) + { + /* same ref */ + diff_consume_block (&data, + (const guchar *)old_entry.base - (const guchar *)old.base, old_entry.size, + (const guchar *)new_entry.base - (const guchar *)new.base, new_entry.size); + old_i++; + new_i++; + } + else if (cmp < 0) + { + /* new added */ + diff_consume_block (&data, + -1, 0, + (const guchar *)new_entry.base - (const guchar *)new.base, new_entry.size); + new_i++; + } + else + { + /* old removed */ + diff_consume_block (&data, + (const guchar *)old_entry.base - (const guchar *)old.base, old_entry.size, + -1, 0); + old_i++; + } + } + } + + /* Flush till the end */ + diff_consume_block2 (&data, + data.last_old_offset, old.size - data.last_old_offset, + data.last_new_offset, new.size - data.last_new_offset); + + diff_uncompressed = diff_encode (&data, error); + if (diff_uncompressed == NULL) + return NULL; + + diff_compressed = flatpak_zlib_compress_bytes (diff_uncompressed, 9, error); + if (diff_compressed == NULL) + return NULL; + +#ifdef VALIDATE_DIFF + { + g_autoptr(GError) apply_error = NULL; + g_autoptr(GBytes) old_bytes = g_variant_get_data_as_bytes (old_v); + g_autoptr(GBytes) new_bytes = g_variant_get_data_as_bytes (new_v); + g_autoptr(GBytes) applied = flatpak_summary_apply_diff (old_bytes, diff_compressed, &apply_error); + g_assert (applied != NULL); + g_assert (g_bytes_equal (applied, new_bytes)); + } +#endif + + return g_steal_pointer (&diff_compressed); +} + +static void +variant_dict_merge (GVariantDict *dict, + GVariant *to_merge) +{ + GVariantIter iter; + gchar *key; + GVariant *value; + + if (to_merge) + { + g_variant_iter_init (&iter, to_merge); + while (g_variant_iter_next (&iter, "{sv}", &key, &value)) + { + g_variant_dict_insert_value (dict, key, value); + g_variant_unref (value); + g_free (key); + } + } +} + +static void +add_summary_metadata (OstreeRepo *repo, + GVariantBuilder *metadata_builder) +{ + GKeyFile *config; + g_autofree char *title = NULL; + g_autofree char *comment = NULL; + g_autofree char *description = NULL; + g_autofree char *homepage = NULL; + g_autofree char *icon = NULL; + g_autofree char *redirect_url = NULL; + g_autofree char *default_branch = NULL; + g_autofree char *remote_mode_str = NULL; + g_autofree char *authenticator_name = NULL; + g_autofree char *gpg_keys = NULL; + g_auto(GStrv) config_keys = NULL; + int authenticator_install = -1; + const char *collection_id; + gboolean deploy_collection_id = FALSE; + gboolean deploy_sideload_collection_id = FALSE; + gboolean tombstone_commits = FALSE; + + config = ostree_repo_get_config (repo); + + if (config) + { + remote_mode_str = g_key_file_get_string (config, "core", "mode", NULL); + tombstone_commits = g_key_file_get_boolean (config, "core", "tombstone-commits", NULL); + + title = g_key_file_get_string (config, "flatpak", "title", NULL); + comment = g_key_file_get_string (config, "flatpak", "comment", NULL); + description = g_key_file_get_string (config, "flatpak", "description", NULL); + homepage = g_key_file_get_string (config, "flatpak", "homepage", NULL); + icon = g_key_file_get_string (config, "flatpak", "icon", NULL); + default_branch = g_key_file_get_string (config, "flatpak", "default-branch", NULL); + gpg_keys = g_key_file_get_string (config, "flatpak", "gpg-keys", NULL); + redirect_url = g_key_file_get_string (config, "flatpak", "redirect-url", NULL); + deploy_sideload_collection_id = g_key_file_get_boolean (config, "flatpak", "deploy-sideload-collection-id", NULL); + deploy_collection_id = g_key_file_get_boolean (config, "flatpak", "deploy-collection-id", NULL); + authenticator_name = g_key_file_get_string (config, "flatpak", "authenticator-name", NULL); + if (g_key_file_has_key (config, "flatpak", "authenticator-install", NULL)) + authenticator_install = g_key_file_get_boolean (config, "flatpak", "authenticator-install", NULL); + + config_keys = g_key_file_get_keys (config, "flatpak", NULL, NULL); + } + + collection_id = ostree_repo_get_collection_id (repo); + + g_variant_builder_add (metadata_builder, "{sv}", "ostree.summary.mode", + g_variant_new_string (remote_mode_str ? remote_mode_str : "bare")); + g_variant_builder_add (metadata_builder, "{sv}", "ostree.summary.tombstone-commits", + g_variant_new_boolean (tombstone_commits)); + g_variant_builder_add (metadata_builder, "{sv}", "ostree.summary.indexed-deltas", + g_variant_new_boolean (TRUE)); + g_variant_builder_add (metadata_builder, "{sv}", "ostree.summary.last-modified", + g_variant_new_uint64 (GUINT64_TO_BE (g_get_real_time () / G_USEC_PER_SEC))); + + if (collection_id) + g_variant_builder_add (metadata_builder, "{sv}", "ostree.summary.collection-id", + g_variant_new_string (collection_id)); + + if (title) + g_variant_builder_add (metadata_builder, "{sv}", "xa.title", + g_variant_new_string (title)); + + if (comment) + g_variant_builder_add (metadata_builder, "{sv}", "xa.comment", + g_variant_new_string (comment)); + + if (description) + g_variant_builder_add (metadata_builder, "{sv}", "xa.description", + g_variant_new_string (description)); + + if (homepage) + g_variant_builder_add (metadata_builder, "{sv}", "xa.homepage", + g_variant_new_string (homepage)); + + if (icon) + g_variant_builder_add (metadata_builder, "{sv}", "xa.icon", + g_variant_new_string (icon)); + + if (redirect_url) + g_variant_builder_add (metadata_builder, "{sv}", "xa.redirect-url", + g_variant_new_string (redirect_url)); + + if (default_branch) + g_variant_builder_add (metadata_builder, "{sv}", "xa.default-branch", + g_variant_new_string (default_branch)); + + if (deploy_collection_id && collection_id != NULL) + g_variant_builder_add (metadata_builder, "{sv}", OSTREE_META_KEY_DEPLOY_COLLECTION_ID, + g_variant_new_string (collection_id)); + else if (deploy_sideload_collection_id && collection_id != NULL) + g_variant_builder_add (metadata_builder, "{sv}", "xa.deploy-collection-id", + g_variant_new_string (collection_id)); + else if (deploy_collection_id) + g_info ("Ignoring deploy-collection-id=true because no collection ID is set."); + + if (authenticator_name) + g_variant_builder_add (metadata_builder, "{sv}", "xa.authenticator-name", + g_variant_new_string (authenticator_name)); + + if (authenticator_install != -1) + g_variant_builder_add (metadata_builder, "{sv}", "xa.authenticator-install", + g_variant_new_boolean (authenticator_install)); + + g_variant_builder_add (metadata_builder, "{sv}", "xa.cache-version", + g_variant_new_uint32 (GUINT32_TO_LE (FLATPAK_XA_CACHE_VERSION))); + + if (config_keys != NULL) + { + for (int i = 0; config_keys[i] != NULL; i++) + { + const char *key = config_keys[i]; + g_autofree char *xa_key = NULL; + g_autofree char *value = NULL; + + if (!g_str_has_prefix (key, "authenticator-options.")) + continue; + + value = g_key_file_get_string (config, "flatpak", key, NULL); + if (value == NULL) + continue; + + xa_key = g_strconcat ("xa.", key, NULL); + g_variant_builder_add (metadata_builder, "{sv}", xa_key, + g_variant_new_string (value)); + } + } + + if (gpg_keys) + { + guchar *decoded; + gsize decoded_len; + + gpg_keys = g_strstrip (gpg_keys); + decoded = g_base64_decode (gpg_keys, &decoded_len); + + g_variant_builder_add (metadata_builder, "{sv}", "xa.gpg-keys", + g_variant_new_from_data (G_VARIANT_TYPE ("ay"), decoded, decoded_len, + TRUE, (GDestroyNotify) g_free, decoded)); + } +} + +static char * +appstream_ref_get_subset (const char *ref) +{ + if (!g_str_has_prefix (ref, "appstream2/")) + return NULL; + + const char *rest = ref + strlen ("appstream2/"); + const char *dash = strrchr (rest, '-'); + if (dash == NULL) + return NULL; + + return g_strndup (rest, dash - rest); +} + +static GVariant * +generate_summary (OstreeRepo *repo, + gboolean compat_format, + GHashTable *refs, + GHashTable *commit_data_cache, + GPtrArray *delta_names, + const char *subset, + const char **summary_arches, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GVariantBuilder) metadata_builder = g_variant_builder_new (G_VARIANT_TYPE_VARDICT); + g_autoptr(GVariantBuilder) ref_data_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{s(tts)}")); + g_autoptr(GVariantBuilder) ref_sparse_data_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sa{sv}}")); + g_autoptr(GVariantBuilder) refs_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(s(taya{sv}))")); + g_autoptr(GVariantBuilder) summary_builder = g_variant_builder_new (OSTREE_SUMMARY_GVARIANT_FORMAT); + g_autoptr(GHashTable) summary_arches_ht = NULL; + g_autoptr(GHashTable) commits = NULL; + g_autoptr(GList) ordered_keys = NULL; + GList *l = NULL; + + /* In the new format this goes in the summary index instead */ + if (compat_format) + add_summary_metadata (repo, metadata_builder); + + ordered_keys = g_hash_table_get_keys (refs); + ordered_keys = g_list_sort (ordered_keys, (GCompareFunc) strcmp); + + if (summary_arches) + { + summary_arches_ht = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL); + for (int i = 0; summary_arches[i] != NULL; i++) + { + const char *arch = summary_arches[i]; + + g_hash_table_add (summary_arches_ht, (char *)arch); + } + } + + /* Compute which commits to keep */ + commits = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL); /* strings owned by ref */ + for (l = ordered_keys; l; l = l->next) + { + const char *ref = l->data; + const char *rev = g_hash_table_lookup (refs, ref); + g_autofree char *arch = NULL; + const CommitData *rev_data = NULL; + + if (summary_arches) + { + /* NOTE: Non-arched (unknown) refs get into all summary versions */ + arch = flatpak_get_arch_for_ref (ref); + if (arch != NULL && !g_hash_table_contains (summary_arches_ht, arch)) + continue; /* Filter this ref by arch */ + } + + rev_data = g_hash_table_lookup (commit_data_cache, rev); + if (*subset != 0) + { + /* Subset summaries keep the appstream2/$subset-$arch, and have no appstream/ compat branch */ + + if (g_str_has_prefix (ref, "appstream/")) + { + continue; /* No compat branch in subsets */ + } + else if (g_str_has_prefix (ref, "appstream2/")) + { + g_autofree char *ref_subset = appstream_ref_get_subset (ref); + if (ref_subset == NULL) + continue; /* Non-subset, ignore */ + + if (strcmp (subset, ref_subset) != 0) + continue; /* Different subset, ignore */ + + /* Otherwise, keep */ + } + else if (rev_data) + { + if (rev_data->subsets == NULL || + !flatpak_g_ptr_array_contains_string (rev_data->subsets, subset)) + continue; /* Ref is not in this subset */ + } + } + else + { + /* non-subset, keep everything but subset appstream refs */ + + g_autofree char *ref_subset = appstream_ref_get_subset (ref); + if (ref_subset != NULL) + continue; /* Subset appstream ref, ignore */ + } + + g_hash_table_add (commits, (char *)rev); + } + + /* Create refs list, metadata and sparse_data */ + for (l = ordered_keys; l; l = l->next) + { + const char *ref = l->data; + const char *rev = g_hash_table_lookup (refs, ref); + const CommitData *rev_data = NULL; + g_auto(GVariantDict) commit_metadata_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER; + guint64 commit_size; + guint64 commit_timestamp; + + if (!g_hash_table_contains (commits, rev)) + continue; /* Filter out commit (by arch & subset) */ + + if (flatpak_is_app_runtime_or_appstream_ref (ref)) + rev_data = g_hash_table_lookup (commit_data_cache, rev); + + if (rev_data != NULL) + { + commit_size = rev_data->commit_size; + commit_timestamp = rev_data->commit_timestamp; + } + else + { + g_autoptr(GVariant) commit_obj = NULL; + if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, rev, &commit_obj, error)) + return NULL; + commit_size = g_variant_get_size (commit_obj); + commit_timestamp = ostree_commit_get_timestamp (commit_obj); + } + + g_variant_dict_init (&commit_metadata_builder, NULL); + if (!compat_format && rev_data) + { + g_variant_dict_insert (&commit_metadata_builder, "xa.data", "(tts)", + GUINT64_TO_BE (rev_data->installed_size), + GUINT64_TO_BE (rev_data->download_size), + rev_data->metadata_contents); + variant_dict_merge (&commit_metadata_builder, rev_data->sparse_data); + } + + /* For the new format summary we use a shorter name for the timestamp to save space */ + g_variant_dict_insert_value (&commit_metadata_builder, + compat_format ? OSTREE_COMMIT_TIMESTAMP : OSTREE_COMMIT_TIMESTAMP2, + g_variant_new_uint64 (GUINT64_TO_BE (commit_timestamp))); + + g_variant_builder_add_value (refs_builder, + g_variant_new ("(s(t@ay@a{sv}))", ref, + commit_size, + ostree_checksum_to_bytes_v (rev), + g_variant_dict_end (&commit_metadata_builder))); + + if (compat_format && rev_data) + { + g_variant_builder_add (ref_data_builder, "{s(tts)}", + ref, + GUINT64_TO_BE (rev_data->installed_size), + GUINT64_TO_BE (rev_data->download_size), + rev_data->metadata_contents); + if (rev_data->sparse_data) + g_variant_builder_add (ref_sparse_data_builder, "{s@a{sv}}", + ref, rev_data->sparse_data); + } + } + + if (delta_names) + { + g_auto(GVariantDict) deltas_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER; + + g_variant_dict_init (&deltas_builder, NULL); + for (guint i = 0; i < delta_names->len; i++) + { + g_autofree char *from = NULL; + g_autofree char *to = NULL; + GVariant *digest; + + _ostree_parse_delta_name (delta_names->pdata[i], &from, &to); + + /* Only keep deltas going to a ref that is in the summary + * (i.e. not arch filtered or random) */ + if (!g_hash_table_contains (commits, to)) + continue; + + digest = _ostree_repo_static_delta_superblock_digest (repo, + (from && from[0]) ? from : NULL, + to, cancellable, error); + if (digest == NULL) + return FALSE; + + g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], digest); + } + + if (delta_names->len > 0) + g_variant_builder_add (metadata_builder, "{sv}", "ostree.static-deltas", g_variant_dict_end (&deltas_builder)); + } + + if (compat_format) + { + /* Note: xa.cache doesn’t need to support collection IDs for the refs listed + * in it, because the xa.cache metadata is stored on the ostree-metadata ref, + * which is itself strongly bound to a collection ID — so that collection ID + * is bound to all the refs in xa.cache. If a client is using the xa.cache + * data from a summary file (rather than an ostree-metadata branch), they are + * too old to care about collection IDs anyway. */ + g_variant_builder_add (metadata_builder, "{sv}", "xa.cache", + g_variant_new_variant (g_variant_builder_end (ref_data_builder))); + g_variant_builder_add (metadata_builder, "{sv}", "xa.sparse-cache", + g_variant_builder_end (ref_sparse_data_builder)); + } + else + { + g_variant_builder_add (metadata_builder, "{sv}", "xa.summary-version", + g_variant_new_uint32 (GUINT32_TO_LE (FLATPAK_XA_SUMMARY_VERSION))); + } + + g_variant_builder_add_value (summary_builder, g_variant_builder_end (refs_builder)); + g_variant_builder_add_value (summary_builder, g_variant_builder_end (metadata_builder)); + + return g_variant_ref_sink (g_variant_builder_end (summary_builder)); +} + +static GVariant * +read_digested_summary (OstreeRepo *repo, + const char *digest, + GHashTable *digested_summary_cache, + GCancellable *cancellable, + GError **error) +{ + GVariant *cached; + g_autoptr(GVariant) loaded = NULL; + + cached = g_hash_table_lookup (digested_summary_cache, digest); + if (cached) + return g_variant_ref (cached); + + loaded = flatpak_repo_load_digested_summary (repo, digest, error); + if (loaded == NULL) + return NULL; + + g_hash_table_insert (digested_summary_cache, g_strdup (digest), g_variant_ref (loaded)); + + return g_steal_pointer (&loaded); +} + +static gboolean +add_to_history (OstreeRepo *repo, + GVariantBuilder *history_builder, + VarChecksumRef old_digest_vv, + GVariant *current_digest_v, + GVariant *current_content, + GHashTable *digested_summary_cache, + guint *history_len, + guint max_history_length, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GVariant) old_digest_v = g_variant_ref_sink (var_checksum_dup_to_gvariant (old_digest_vv)); + g_autofree char *old_digest = NULL; + g_autoptr(GVariant) old_content = NULL; + g_autofree char *current_digest = NULL; + g_autoptr(GBytes) subsummary_diff = NULL; + + /* Limit history length */ + if (*history_len >= max_history_length) + return TRUE; + + /* Avoid repeats in the history (in case nothing changed in subsummary) */ + if (g_variant_equal (old_digest_v, current_digest_v)) + return TRUE; + + old_digest = ostree_checksum_from_bytes_v (old_digest_v); + old_content = read_digested_summary (repo, old_digest, digested_summary_cache, cancellable, NULL); + if (old_content == NULL) + return TRUE; /* Only add parents that still exist */ + + subsummary_diff = flatpak_summary_generate_diff (old_content, current_content, error); + if (subsummary_diff == NULL) + return FALSE; + + current_digest = ostree_checksum_from_bytes_v (current_digest_v); + + if (!flatpak_repo_save_digested_summary_delta (repo, old_digest, current_digest, + subsummary_diff, cancellable, error)) + return FALSE; + + *history_len += 1; + g_variant_builder_add_value (history_builder, old_digest_v); + + return TRUE; +} + +static GVariant * +generate_summary_index (OstreeRepo *repo, + GVariant *old_index_v, + GHashTable *summaries, + GHashTable *digested_summaries, + GHashTable *digested_summary_cache, + const char **gpg_key_ids, + const char *gpg_homedir, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GVariantBuilder) metadata_builder = g_variant_builder_new (G_VARIANT_TYPE_VARDICT); + g_autoptr(GVariantBuilder) subsummary_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{s(ayaaya{sv})}")); + g_autoptr(GVariantBuilder) index_builder = g_variant_builder_new (FLATPAK_SUMMARY_INDEX_GVARIANT_FORMAT); + g_autoptr(GVariant) index = NULL; + g_autoptr(GList) ordered_summaries = NULL; + guint max_history_length = flatpak_repo_get_summary_history_length (repo); + GList *l; + + add_summary_metadata (repo, metadata_builder); + + ordered_summaries = g_hash_table_get_keys (summaries); + ordered_summaries = g_list_sort (ordered_summaries, (GCompareFunc) strcmp); + for (l = ordered_summaries; l; l = l->next) + { + g_auto(GVariantDict) subsummary_metadata_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER; + const char *subsummary = l->data; + const char *digest = g_hash_table_lookup (summaries, subsummary); + g_autoptr(GVariant) digest_v = g_variant_ref_sink (ostree_checksum_to_bytes_v (digest)); + g_autoptr(GVariantBuilder) history_builder = g_variant_builder_new (G_VARIANT_TYPE ("aay")); + g_autoptr(GVariant) subsummary_content = NULL; + + subsummary_content = read_digested_summary (repo, digest, digested_summary_cache, cancellable, error); + if (subsummary_content == NULL) + return NULL; /* This really should always be there as we're supposed to index it */ + + if (old_index_v) + { + VarSummaryIndexRef old_index = var_summary_index_from_gvariant (old_index_v); + VarSummaryIndexSubsummariesRef old_subsummaries = var_summary_index_get_subsummaries (old_index); + VarSubsummaryRef old_subsummary; + guint history_len = 0; + + if (var_summary_index_subsummaries_lookup (old_subsummaries, subsummary, NULL, &old_subsummary)) + { + VarChecksumRef parent = var_subsummary_get_checksum (old_subsummary); + + /* Add current as first in history */ + if (!add_to_history (repo, history_builder, parent, digest_v, subsummary_content, digested_summary_cache, + &history_len, max_history_length, cancellable, error)) + return FALSE; + + /* Add previous history */ + VarArrayofChecksumRef history = var_subsummary_get_history (old_subsummary); + gsize len = var_arrayof_checksum_get_length (history); + for (gsize i = 0; i < len; i++) + { + VarChecksumRef c = var_arrayof_checksum_get_at (history, i); + if (!add_to_history (repo, history_builder, c, digest_v, subsummary_content, digested_summary_cache, + &history_len, max_history_length, cancellable, error)) + return FALSE; + } + } + } + + g_variant_dict_init (&subsummary_metadata_builder, NULL); + g_variant_builder_add (subsummary_builder, "{s(@ay@aay@a{sv})}", + subsummary, + digest_v, + g_variant_builder_end (history_builder), + g_variant_dict_end (&subsummary_metadata_builder)); + } + + g_variant_builder_add_value (index_builder, g_variant_builder_end (subsummary_builder)); + g_variant_builder_add_value (index_builder, g_variant_builder_end (metadata_builder)); + + index = g_variant_ref_sink (g_variant_builder_end (index_builder)); + + return g_steal_pointer (&index); +} + +static gboolean +flatpak_repo_gc_digested_summaries (OstreeRepo *repo, + const char *index_digest, /* The digest of the current (new) index (if any) */ + const char *old_index_digest, /* The digest of the previous index (if any) */ + GHashTable *digested_summaries, /* generated */ + GHashTable *digested_summary_cache, /* generated + referenced */ + GCancellable *cancellable, + GError **error) +{ + g_auto(GLnxDirFdIterator) iter = {0}; + int repo_fd = ostree_repo_get_dfd (repo); + struct dirent *dent; + const char *ext; + g_autoptr(GError) local_error = NULL; + + if (!glnx_dirfd_iterator_init_at (repo_fd, "summaries", FALSE, &iter, &local_error)) + { + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + return TRUE; + + g_propagate_error (error, g_steal_pointer (&local_error)); + return FALSE; + } + + while (TRUE) + { + gboolean remove = FALSE; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&iter, &dent, cancellable, error)) + return FALSE; + + if (dent == NULL) + break; + + if (dent->d_type != DT_REG) + continue; + + /* Keep it if its an unexpected type */ + ext = strchr (dent->d_name, '.'); + if (ext != NULL) + { + if (strcmp (ext, ".gz") == 0 && strlen (dent->d_name) == 64 + 3) + { + g_autofree char *sha256 = g_strndup (dent->d_name, 64); + + /* Keep all the referenced summaries */ + if (g_hash_table_contains (digested_summary_cache, sha256)) + { + g_info ("Keeping referenced summary %s", dent->d_name); + continue; + } + /* Remove rest */ + remove = TRUE; + } + else if (strcmp (ext, ".delta") == 0) + { + const char *dash = strchr (dent->d_name, '-'); + if (dash != NULL && dash < ext && (ext - dash) == 1 + 64) + { + g_autofree char *to_sha256 = g_strndup (dash + 1, 64); + + /* Only keep deltas going to a generated summary */ + if (g_hash_table_contains (digested_summaries, to_sha256)) + { + g_info ("Keeping delta to generated summary %s", dent->d_name); + continue; + } + /* Remove rest */ + remove = TRUE; + } + } + else if (strcmp (ext, ".idx.sig") == 0) + { + g_autofree char *digest = g_strndup (dent->d_name, strlen (dent->d_name) - strlen (".idx.sig")); + + if (g_strcmp0 (digest, index_digest) == 0) + continue; /* Always keep current */ + + if (g_strcmp0 (digest, old_index_digest) == 0) + continue; /* Always keep previous one, to avoid some races */ + + /* Remove the rest */ + remove = TRUE; + } + } + + if (remove) + { + g_info ("Removing old digested summary file %s", dent->d_name); + if (unlinkat (iter.fd, dent->d_name, 0) != 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } + } + else + g_info ("Keeping unexpected summary file %s", dent->d_name); + } + + return TRUE; +} + +/* Update the metadata in the summary file for @repo, and then re-sign the file. + * If the repo has a collection ID set, additionally store the metadata on a + * contentless commit in a well-known branch, which is the preferred way of + * broadcasting per-repo metadata (putting it in the summary file is deprecated, + * but kept for backwards compatibility). + * + * Note that there are two keys for the collection ID: collection-id, and + * ostree.deploy-collection-id. If a client does not currently have a + * collection ID configured for this remote, it will *only* update its + * configuration from ostree.deploy-collection-id. This allows phased + * deployment of collection-based repositories. Clients will only update their + * configuration from an unset to a set collection ID once (otherwise the + * security properties of collection IDs are broken). */ +gboolean +flatpak_repo_update (OstreeRepo *repo, + FlatpakRepoUpdateFlags flags, + const char **gpg_key_ids, + const char *gpg_homedir, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GHashTable) commit_data_cache = NULL; + g_autoptr(GVariant) compat_summary = NULL; + g_autoptr(GVariant) summary_index = NULL; + g_autoptr(GVariant) old_index = NULL; + g_autoptr(GPtrArray) delta_names = NULL; + g_auto(GStrv) summary_arches = NULL; + g_autoptr(GHashTable) refs = NULL; + g_autoptr(GHashTable) arches = NULL; + g_autoptr(GHashTable) subsets = NULL; + g_autoptr(GHashTable) summaries = NULL; + g_autoptr(GHashTable) digested_summaries = NULL; + g_autoptr(GHashTable) digested_summary_cache = NULL; + g_autoptr(GBytes) index_sig = NULL; + time_t old_compat_sig_mtime; + GKeyFile *config; + gboolean disable_index = (flags & FLATPAK_REPO_UPDATE_FLAG_DISABLE_INDEX) != 0; + g_autofree char *index_digest = NULL; + g_autofree char *old_index_digest = NULL; + + config = ostree_repo_get_config (repo); + + if (!ostree_repo_list_refs_ext (repo, NULL, &refs, + OSTREE_REPO_LIST_REFS_EXT_EXCLUDE_REMOTES | OSTREE_REPO_LIST_REFS_EXT_EXCLUDE_MIRRORS, + cancellable, error)) + return FALSE; + + old_index = flatpak_repo_load_summary_index (repo, NULL); + if (old_index) + commit_data_cache = populate_commit_data_cache (repo, old_index); + + if (commit_data_cache == NULL) /* No index or failed to load it */ + commit_data_cache = commit_data_cache_new (); + + if (!ostree_repo_list_static_delta_names (repo, &delta_names, cancellable, error)) + return FALSE; + + if (config) + summary_arches = g_key_file_get_string_list (config, "flatpak", "summary-arches", NULL, NULL); + + summaries = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); + /* These are the ones we generated */ + digested_summaries = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)g_variant_unref); + /* These are the ones generated or references */ + digested_summary_cache = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)g_variant_unref); + + arches = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + subsets = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + g_hash_table_add (subsets, g_strdup ("")); /* Always have everything subset */ + + GLNX_HASH_TABLE_FOREACH_KV (refs, const char *, ref, const char *, rev) + { + g_autofree char *arch = flatpak_get_arch_for_ref (ref); + CommitData *rev_data = NULL; + + if (arch != NULL && + !g_hash_table_contains (arches, arch)) + g_hash_table_add (arches, g_steal_pointer (&arch)); + + /* Add CommitData for flatpak refs that we didn't already pre-populate */ + if (flatpak_is_app_runtime_or_appstream_ref (ref)) + { + rev_data = g_hash_table_lookup (commit_data_cache, rev); + if (rev_data == NULL) + { + rev_data = read_commit_data (repo, ref, rev, cancellable, error); + if (rev_data == NULL) + return FALSE; + + g_hash_table_insert (commit_data_cache, g_strdup (rev), (CommitData *)rev_data); + } + + for (int i = 0; rev_data->subsets != NULL && i < rev_data->subsets->len; i++) + { + const char *subset = g_ptr_array_index (rev_data->subsets, i); + if (!g_hash_table_contains (subsets, subset)) + g_hash_table_add (subsets, g_strdup (subset)); + } + } + } + + compat_summary = generate_summary (repo, TRUE, refs, commit_data_cache, delta_names, + "", (const char **)summary_arches, + cancellable, error); + if (compat_summary == NULL) + return FALSE; + + if (!disable_index) + { + GLNX_HASH_TABLE_FOREACH (subsets, const char *, subset) + { + GLNX_HASH_TABLE_FOREACH (arches, const char *, arch) + { + const char *arch_v[] = { arch, NULL }; + g_autofree char *name = NULL; + g_autofree char *digest = NULL; + + if (*subset == 0) + name = g_strdup (arch); + else + name = g_strconcat (subset, "-", arch, NULL); + + g_autoptr(GVariant) arch_summary = generate_summary (repo, FALSE, refs, commit_data_cache, NULL, subset, arch_v, + cancellable, error); + if (arch_summary == NULL) + return FALSE; + + digest = flatpak_repo_save_digested_summary (repo, name, arch_summary, cancellable, error); + if (digest == NULL) + return FALSE; + + g_hash_table_insert (digested_summaries, g_strdup (digest), g_variant_ref (arch_summary)); + /* Prime summary cache with generated summaries */ + g_hash_table_insert (digested_summary_cache, g_strdup (digest), g_variant_ref (arch_summary)); + g_hash_table_insert (summaries, g_steal_pointer (&name), g_steal_pointer (&digest)); + } + } + + summary_index = generate_summary_index (repo, old_index, summaries, digested_summaries, digested_summary_cache, + gpg_key_ids, gpg_homedir, + cancellable, error); + if (summary_index == NULL) + return FALSE; + } + + if (!ostree_repo_static_delta_reindex (repo, 0, NULL, cancellable, error)) + return FALSE; + + if (summary_index && gpg_key_ids) + { + g_autoptr(GBytes) index_bytes = g_variant_get_data_as_bytes (summary_index); + + if (!ostree_repo_gpg_sign_data (repo, index_bytes, + NULL, + gpg_key_ids, + gpg_homedir, + &index_sig, + cancellable, + error)) + return FALSE; + } + + if (summary_index) + index_digest = g_compute_checksum_for_data (G_CHECKSUM_SHA256, + g_variant_get_data (summary_index), + g_variant_get_size (summary_index)); + if (old_index) + old_index_digest = g_compute_checksum_for_data (G_CHECKSUM_SHA256, + g_variant_get_data (old_index), + g_variant_get_size (old_index)); + + /* Release the memory-mapped summary index file before replacing it, + to avoid failure on filesystems like cifs */ + g_clear_pointer (&old_index, g_variant_unref); + + if (!flatpak_repo_save_summary_index (repo, summary_index, index_digest, index_sig, cancellable, error)) + return FALSE; + + if (!flatpak_repo_save_compat_summary (repo, compat_summary, &old_compat_sig_mtime, cancellable, error)) + return FALSE; + + if (gpg_key_ids) + { + if (!ostree_repo_add_gpg_signature_summary (repo, + gpg_key_ids, + gpg_homedir, + cancellable, + error)) + return FALSE; + + + if (old_compat_sig_mtime != 0) + { + int repo_dfd = ostree_repo_get_dfd (repo); + struct stat stbuf; + + /* Ensure we increase (in sec precision) */ + if (fstatat (repo_dfd, "summary.sig", &stbuf, AT_SYMLINK_NOFOLLOW) == 0 && + stbuf.st_mtime <= old_compat_sig_mtime) + { + struct timespec ts[2] = { {0, UTIME_OMIT}, {old_compat_sig_mtime + 1, 0} }; + (void) utimensat (repo_dfd, "summary.sig", ts, AT_SYMLINK_NOFOLLOW); + } + } + } + + if (!disable_index && + !flatpak_repo_gc_digested_summaries (repo, index_digest, old_index_digest, digested_summaries, digested_summary_cache, cancellable, error)) + return FALSE; + + return TRUE; +} + +/* Wrapper that uses ostree_repo_resolve_collection_ref() and on failure falls + * back to using ostree_repo_resolve_rev() for backwards compatibility. This + * means we support refs/heads/, refs/remotes/, and refs/mirrors/. */ +gboolean +flatpak_repo_resolve_rev (OstreeRepo *repo, + const char *collection_id, /* nullable */ + const char *remote_name, /* nullable */ + const char *ref_name, + gboolean allow_noent, + char **out_rev, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GError) local_error = NULL; + + if (collection_id != NULL) + { + /* Do a version check to ensure we have these: + * https://github.com/ostreedev/ostree/pull/1821 + * https://github.com/ostreedev/ostree/pull/1825 */ +#if OSTREE_CHECK_VERSION (2019, 2) + const OstreeCollectionRef c_r = + { + .collection_id = (char *) collection_id, + .ref_name = (char *) ref_name, + }; + OstreeRepoResolveRevExtFlags flags = remote_name == NULL ? + OSTREE_REPO_RESOLVE_REV_EXT_LOCAL_ONLY : + OSTREE_REPO_RESOLVE_REV_EXT_NONE; + if (ostree_repo_resolve_collection_ref (repo, &c_r, + allow_noent, + flags, + out_rev, + cancellable, NULL)) + return TRUE; +#endif + } + + /* There may be several remotes with the same branch (if we for + * instance changed the origin) so prepend the current origin to + * make sure we get the right one */ + if (remote_name != NULL) + { + g_autofree char *refspec = g_strdup_printf ("%s:%s", remote_name, ref_name); + ostree_repo_resolve_rev (repo, refspec, allow_noent, out_rev, &local_error); + } + else + ostree_repo_resolve_rev_ext (repo, ref_name, allow_noent, + OSTREE_REPO_RESOLVE_REV_EXT_NONE, out_rev, &local_error); + + if (local_error != NULL) + { + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, "%s", local_error->message); + else + g_propagate_error (error, g_steal_pointer (&local_error)); + + return FALSE; + } + + return TRUE; +} diff --git a/common/flatpak-utils-private.h b/common/flatpak-utils-private.h index 1d787aa2..f18e62e4 100644 --- a/common/flatpak-utils-private.h +++ b/common/flatpak-utils-private.h @@ -114,13 +114,6 @@ gboolean flatpak_variant_save (GFile *dest, GVariant *variant, GCancellable *cancellable, GError **error); -GVariant *flatpak_repo_load_summary (OstreeRepo *repo, - GError **error); -GVariant *flatpak_repo_load_summary_index (OstreeRepo *repo, - GError **error); -GVariant *flatpak_repo_load_digested_summary (OstreeRepo *repo, - const char *digest, - GError **error); GPtrArray *flatpak_summary_match_subrefs (GVariant *summary, const char *collection_id, FlatpakDecomposed *ref); @@ -198,40 +191,6 @@ GBytes *flatpak_zlib_compress_bytes (GBytes *bytes, GBytes *flatpak_zlib_decompress_bytes (GBytes *bytes, GError **error); -GBytes *flatpak_summary_apply_diff (GBytes *old, - GBytes *diff, - GError **error); - -typedef enum { - FLATPAK_REPO_UPDATE_FLAG_NONE = 0, - FLATPAK_REPO_UPDATE_FLAG_DISABLE_INDEX = 1 << 0, -} FlatpakRepoUpdateFlags; - -gboolean flatpak_repo_update (OstreeRepo *repo, - FlatpakRepoUpdateFlags flags, - const char **gpg_key_ids, - const char *gpg_homedir, - GCancellable *cancellable, - GError **error); -gboolean flatpak_repo_collect_sizes (OstreeRepo *repo, - GFile *root, - guint64 *installed_size, - guint64 *download_size, - GCancellable *cancellable, - GError **error); -GVariant *flatpak_commit_get_extra_data_sources (GVariant *commitv, - GError **error); -GVariant *flatpak_repo_get_extra_data_sources (OstreeRepo *repo, - const char *rev, - GCancellable *cancellable, - GError **error); -void flatpak_repo_parse_extra_data_sources (GVariant *extra_data_sources, - int index, - const char **name, - guint64 *download_size, - guint64 *installed_size, - const guchar **sha256, - const char **uri); gboolean flatpak_mtree_ensure_dir_metadata (OstreeRepo *repo, OstreeMutableTree *mtree, GCancellable *cancellable, diff --git a/common/flatpak-utils.c b/common/flatpak-utils.c index b070df4e..2e89edc5 100644 --- a/common/flatpak-utils.c +++ b/common/flatpak-utils.c @@ -41,7 +41,6 @@ #include #include -#include #include "flatpak-error.h" #include "flatpak-repo-utils-private.h" @@ -1978,2232 +1977,6 @@ flatpak_parse_repofile (const char *remote_name, return g_steal_pointer (&config); } -GVariant * -flatpak_commit_get_extra_data_sources (GVariant *commitv, - GError **error) -{ - g_autoptr(GVariant) commit_metadata = NULL; - g_autoptr(GVariant) extra_data_sources = NULL; - - commit_metadata = g_variant_get_child_value (commitv, 0); - extra_data_sources = g_variant_lookup_value (commit_metadata, - "xa.extra-data-sources", - G_VARIANT_TYPE ("a(ayttays)")); - - if (extra_data_sources == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - _("No extra data sources")); - return NULL; - } - - return g_steal_pointer (&extra_data_sources); -} - - -GVariant * -flatpak_repo_get_extra_data_sources (OstreeRepo *repo, - const char *rev, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(GVariant) commitv = NULL; - - if (!ostree_repo_load_variant (repo, - OSTREE_OBJECT_TYPE_COMMIT, - rev, &commitv, error)) - return NULL; - - return flatpak_commit_get_extra_data_sources (commitv, error); -} - -void -flatpak_repo_parse_extra_data_sources (GVariant *extra_data_sources, - int index, - const char **name, - guint64 *download_size, - guint64 *installed_size, - const guchar **sha256, - const char **uri) -{ - g_autoptr(GVariant) sha256_v = NULL; - g_variant_get_child (extra_data_sources, index, "(^aytt@ay&s)", - name, - download_size, - installed_size, - &sha256_v, - uri); - - if (download_size) - *download_size = GUINT64_FROM_BE (*download_size); - - if (installed_size) - *installed_size = GUINT64_FROM_BE (*installed_size); - - if (sha256) - *sha256 = ostree_checksum_bytes_peek (sha256_v); -} - -#define OSTREE_GIO_FAST_QUERYINFO ("standard::name,standard::type,standard::size,standard::is-symlink,standard::symlink-target," \ - "unix::device,unix::inode,unix::mode,unix::uid,unix::gid,unix::rdev") - -static gboolean -_flatpak_repo_collect_sizes (OstreeRepo *repo, - GFile *file, - GFileInfo *file_info, - guint64 *installed_size, - guint64 *download_size, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(GFileEnumerator) dir_enum = NULL; - GFileInfo *child_info_tmp; - g_autoptr(GError) temp_error = NULL; - - if (file_info != NULL && g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) - { - const char *checksum = ostree_repo_file_get_checksum (OSTREE_REPO_FILE (file)); - guint64 obj_size; - guint64 file_size = g_file_info_get_size (file_info); - - if (installed_size) - *installed_size += ((file_size + 511) / 512) * 512; - - if (download_size) - { - g_autoptr(GInputStream) input = NULL; - GInputStream *base_input; - g_autoptr(GError) local_error = NULL; - - if (!ostree_repo_query_object_storage_size (repo, - OSTREE_OBJECT_TYPE_FILE, checksum, - &obj_size, cancellable, &local_error)) - { - int fd; - struct stat stbuf; - - /* Ostree does not look at the staging directory when querying storage - size, so may return a NOT_FOUND error here. We work around this - by loading the object and walking back until we find the original - fd which we can fstat(). */ - if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - return FALSE; - - if (!ostree_repo_load_file (repo, checksum, &input, NULL, NULL, NULL, error)) - return FALSE; - - base_input = input; - while (G_IS_FILTER_INPUT_STREAM (base_input)) - base_input = g_filter_input_stream_get_base_stream (G_FILTER_INPUT_STREAM (base_input)); - - if (!G_IS_UNIX_INPUT_STREAM (base_input)) - return flatpak_fail (error, "Unable to find size of commit %s, not an unix stream", checksum); - - fd = g_unix_input_stream_get_fd (G_UNIX_INPUT_STREAM (base_input)); - - if (fstat (fd, &stbuf) != 0) - return glnx_throw_errno_prefix (error, "Can't find commit size: "); - - obj_size = stbuf.st_size; - } - - *download_size += obj_size; - } - } - - if (file_info == NULL || g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) - { - dir_enum = g_file_enumerate_children (file, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, error); - if (!dir_enum) - return FALSE; - - - while ((child_info_tmp = g_file_enumerator_next_file (dir_enum, cancellable, &temp_error))) - { - g_autoptr(GFileInfo) child_info = child_info_tmp; - const char *name = g_file_info_get_name (child_info); - g_autoptr(GFile) child = g_file_get_child (file, name); - - if (!_flatpak_repo_collect_sizes (repo, child, child_info, installed_size, download_size, cancellable, error)) - return FALSE; - } - } - - return TRUE; -} - -gboolean -flatpak_repo_collect_sizes (OstreeRepo *repo, - GFile *root, - guint64 *installed_size, - guint64 *download_size, - GCancellable *cancellable, - GError **error) -{ - /* Initialize the sums */ - if (installed_size) - *installed_size = 0; - if (download_size) - *download_size = 0; - return _flatpak_repo_collect_sizes (repo, root, NULL, installed_size, download_size, cancellable, error); -} - - -static void -flatpak_repo_collect_extra_data_sizes (OstreeRepo *repo, - const char *rev, - guint64 *installed_size, - guint64 *download_size) -{ - g_autoptr(GVariant) extra_data_sources = NULL; - gsize n_extra_data; - int i; - - extra_data_sources = flatpak_repo_get_extra_data_sources (repo, rev, NULL, NULL); - if (extra_data_sources == NULL) - return; - - n_extra_data = g_variant_n_children (extra_data_sources); - if (n_extra_data == 0) - return; - - for (i = 0; i < n_extra_data; i++) - { - guint64 extra_download_size; - guint64 extra_installed_size; - - flatpak_repo_parse_extra_data_sources (extra_data_sources, i, - NULL, - &extra_download_size, - &extra_installed_size, - NULL, NULL); - if (installed_size) - *installed_size += extra_installed_size; - if (download_size) - *download_size += extra_download_size; - } -} - -/* Loads the old compat summary file from a local repo */ -GVariant * -flatpak_repo_load_summary (OstreeRepo *repo, - GError **error) -{ - glnx_autofd int fd = -1; - g_autoptr(GMappedFile) mfile = NULL; - g_autoptr(GBytes) bytes = NULL; - - fd = openat (ostree_repo_get_dfd (repo), "summary", O_RDONLY | O_CLOEXEC); - if (fd < 0) - { - glnx_set_error_from_errno (error); - return NULL; - } - - mfile = g_mapped_file_new_from_fd (fd, FALSE, error); - if (!mfile) - return NULL; - - bytes = g_mapped_file_get_bytes (mfile); - - return g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes, TRUE)); -} - -GVariant * -flatpak_repo_load_summary_index (OstreeRepo *repo, - GError **error) -{ - glnx_autofd int fd = -1; - g_autoptr(GMappedFile) mfile = NULL; - g_autoptr(GBytes) bytes = NULL; - - fd = openat (ostree_repo_get_dfd (repo), "summary.idx", O_RDONLY | O_CLOEXEC); - if (fd < 0) - { - glnx_set_error_from_errno (error); - return NULL; - } - - mfile = g_mapped_file_new_from_fd (fd, FALSE, error); - if (!mfile) - return NULL; - - bytes = g_mapped_file_get_bytes (mfile); - - return g_variant_ref_sink (g_variant_new_from_bytes (FLATPAK_SUMMARY_INDEX_GVARIANT_FORMAT, bytes, TRUE)); -} - -static gboolean -flatpak_repo_save_compat_summary (OstreeRepo *repo, - GVariant *summary, - time_t *out_old_sig_mtime, - GCancellable *cancellable, - GError **error) -{ - int repo_dfd = ostree_repo_get_dfd (repo); - struct stat stbuf; - time_t old_sig_mtime = 0; - GLnxFileReplaceFlags flags; - - flags = GLNX_FILE_REPLACE_INCREASING_MTIME; - if (ostree_repo_get_disable_fsync (repo)) - flags |= GLNX_FILE_REPLACE_NODATASYNC; - else - flags |= GLNX_FILE_REPLACE_DATASYNC_NEW; - - if (!glnx_file_replace_contents_at (repo_dfd, "summary", - g_variant_get_data (summary), - g_variant_get_size (summary), - flags, - cancellable, error)) - return FALSE; - - if (fstatat (repo_dfd, "summary.sig", &stbuf, AT_SYMLINK_NOFOLLOW) == 0) - old_sig_mtime = stbuf.st_mtime; - - if (unlinkat (repo_dfd, "summary.sig", 0) != 0 && - G_UNLIKELY (errno != ENOENT)) - { - glnx_set_error_from_errno (error); - return FALSE; - } - - *out_old_sig_mtime = old_sig_mtime; - return TRUE; -} - -static gboolean -flatpak_repo_save_summary_index (OstreeRepo *repo, - GVariant *index, - const char *index_digest, - GBytes *index_sig, - GCancellable *cancellable, - GError **error) -{ - int repo_dfd = ostree_repo_get_dfd (repo); - GLnxFileReplaceFlags flags; - - if (index == NULL) - { - if (unlinkat (repo_dfd, "summary.idx", 0) != 0 && - G_UNLIKELY (errno != ENOENT)) - { - glnx_set_error_from_errno (error); - return FALSE; - } - if (unlinkat (repo_dfd, "summary.idx.sig", 0) != 0 && - G_UNLIKELY (errno != ENOENT)) - { - glnx_set_error_from_errno (error); - return FALSE; - } - - return TRUE; - } - - flags = GLNX_FILE_REPLACE_INCREASING_MTIME; - if (ostree_repo_get_disable_fsync (repo)) - flags |= GLNX_FILE_REPLACE_NODATASYNC; - else - flags |= GLNX_FILE_REPLACE_DATASYNC_NEW; - - if (index_sig) - { - g_autofree char *path = g_strconcat ("summaries/", index_digest, ".idx.sig", NULL); - - if (!glnx_shutil_mkdir_p_at (repo_dfd, "summaries", - 0775, cancellable, error)) - return FALSE; - - if (!glnx_file_replace_contents_at (repo_dfd, path, - g_bytes_get_data (index_sig, NULL), - g_bytes_get_size (index_sig), - flags, - cancellable, error)) - return FALSE; - } - - if (!glnx_file_replace_contents_at (repo_dfd, "summary.idx", - g_variant_get_data (index), - g_variant_get_size (index), - flags, - cancellable, error)) - return FALSE; - - /* Update the non-indexed summary.idx.sig file that was introduced in 1.9.1 but - * was made unnecessary in 1.9.3. Lets keep it for a while until everyone updates - */ - if (index_sig) - { - if (!glnx_file_replace_contents_at (repo_dfd, "summary.idx.sig", - g_bytes_get_data (index_sig, NULL), - g_bytes_get_size (index_sig), - flags, - cancellable, error)) - return FALSE; - } - else - { - if (unlinkat (repo_dfd, "summary.idx.sig", 0) != 0 && - G_UNLIKELY (errno != ENOENT)) - { - glnx_set_error_from_errno (error); - return FALSE; - } - } - - return TRUE; -} - -GVariant * -flatpak_repo_load_digested_summary (OstreeRepo *repo, - const char *digest, - GError **error) -{ - glnx_autofd int fd = -1; - g_autoptr(GMappedFile) mfile = NULL; - g_autoptr(GBytes) bytes = NULL; - g_autoptr(GBytes) compressed_bytes = NULL; - g_autofree char *path = NULL; - g_autofree char *filename = NULL; - - filename = g_strconcat (digest, ".gz", NULL); - path = g_build_filename ("summaries", filename, NULL); - - fd = openat (ostree_repo_get_dfd (repo), path, O_RDONLY | O_CLOEXEC); - if (fd < 0) - { - glnx_set_error_from_errno (error); - return NULL; - } - - mfile = g_mapped_file_new_from_fd (fd, FALSE, error); - if (!mfile) - return NULL; - - compressed_bytes = g_mapped_file_get_bytes (mfile); - bytes = flatpak_zlib_decompress_bytes (compressed_bytes, error); - if (bytes == NULL) - return NULL; - - return g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes, TRUE)); -} - -static char * -flatpak_repo_save_digested_summary (OstreeRepo *repo, - const char *name, - GVariant *summary, - GCancellable *cancellable, - GError **error) -{ - int repo_dfd = ostree_repo_get_dfd (repo); - g_autofree char *digest = NULL; - g_autofree char *filename = NULL; - g_autofree char *path = NULL; - g_autoptr(GBytes) data = NULL; - g_autoptr(GBytes) compressed_data = NULL; - struct stat stbuf; - - if (!glnx_shutil_mkdir_p_at (repo_dfd, "summaries", - 0775, - cancellable, - error)) - return NULL; - - digest = g_compute_checksum_for_data (G_CHECKSUM_SHA256, - g_variant_get_data (summary), - g_variant_get_size (summary)); - filename = g_strconcat (digest, ".gz", NULL); - - path = g_build_filename ("summaries", filename, NULL); - - /* Check for pre-existing (non-truncated) copy and avoid re-writing it */ - if (fstatat (repo_dfd, path, &stbuf, 0) == 0 && - stbuf.st_size != 0) - { - g_info ("Reusing digested summary at %s for %s", path, name); - return g_steal_pointer (&digest); - } - - data = g_variant_get_data_as_bytes (summary); - compressed_data = flatpak_zlib_compress_bytes (data, -1, error); - if (compressed_data == NULL) - return NULL; - - if (!glnx_file_replace_contents_at (repo_dfd, path, - g_bytes_get_data (compressed_data, NULL), - g_bytes_get_size (compressed_data), - ostree_repo_get_disable_fsync (repo) ? GLNX_FILE_REPLACE_NODATASYNC : GLNX_FILE_REPLACE_DATASYNC_NEW, - cancellable, error)) - return NULL; - - g_info ("Wrote digested summary at %s for %s", path, name); - return g_steal_pointer (&digest); -} - -static gboolean -flatpak_repo_save_digested_summary_delta (OstreeRepo *repo, - const char *from_digest, - const char *to_digest, - GBytes *delta, - GCancellable *cancellable, - GError **error) -{ - int repo_dfd = ostree_repo_get_dfd (repo); - g_autofree char *path = NULL; - g_autofree char *filename = g_strconcat (from_digest, "-", to_digest, ".delta", NULL); - struct stat stbuf; - - if (!glnx_shutil_mkdir_p_at (repo_dfd, "summaries", - 0775, - cancellable, - error)) - return FALSE; - - path = g_build_filename ("summaries", filename, NULL); - - /* Check for pre-existing copy of same size and avoid re-writing it */ - if (fstatat (repo_dfd, path, &stbuf, 0) == 0 && - stbuf.st_size == g_bytes_get_size (delta)) - { - g_info ("Reusing digested summary-diff for %s", filename); - return TRUE; - } - - if (!glnx_file_replace_contents_at (repo_dfd, path, - g_bytes_get_data (delta, NULL), - g_bytes_get_size (delta), - ostree_repo_get_disable_fsync (repo) ? GLNX_FILE_REPLACE_NODATASYNC : GLNX_FILE_REPLACE_DATASYNC_NEW, - cancellable, error)) - return FALSE; - - g_info ("Wrote digested summary delta at %s", path); - return TRUE; -} - - -typedef struct -{ - guint64 installed_size; - guint64 download_size; - char *metadata_contents; - GPtrArray *subsets; - GVariant *sparse_data; - gsize commit_size; - guint64 commit_timestamp; -} CommitData; - -static void -commit_data_free (gpointer data) -{ - CommitData *rev_data = data; - - if (rev_data->subsets) - g_ptr_array_unref (rev_data->subsets); - g_free (rev_data->metadata_contents); - if (rev_data->sparse_data) - g_variant_unref (rev_data->sparse_data); - g_free (rev_data); -} - -G_DEFINE_AUTOPTR_CLEANUP_FUNC (CommitData, commit_data_free); - -static GHashTable * -commit_data_cache_new (void) -{ - return g_hash_table_new_full (g_str_hash, g_str_equal, g_free, commit_data_free); -} - -static GHashTable * -populate_commit_data_cache (OstreeRepo *repo, - GVariant *index_v) -{ - - VarSummaryIndexRef index = var_summary_index_from_gvariant (index_v); - VarMetadataRef index_metadata = var_summary_index_get_metadata (index); - VarSummaryIndexSubsummariesRef subsummaries = var_summary_index_get_subsummaries (index); - gsize n_subsummaries = var_summary_index_subsummaries_get_length (subsummaries); - guint32 cache_version; - g_autoptr(GHashTable) commit_data_cache = commit_data_cache_new (); - - cache_version = GUINT32_FROM_LE (var_metadata_lookup_uint32 (index_metadata, "xa.cache-version", 0)); - if (cache_version < FLATPAK_XA_CACHE_VERSION) - { - /* Need to re-index to get all data */ - g_info ("Old summary cache version %d, not using cache", cache_version); - return NULL; - } - - for (gsize i = 0; i < n_subsummaries; i++) - { - VarSummaryIndexSubsummariesEntryRef entry = var_summary_index_subsummaries_get_at (subsummaries, i); - const char *name = var_summary_index_subsummaries_entry_get_key (entry); - const char *s; - g_autofree char *subset = NULL; - VarSubsummaryRef subsummary = var_summary_index_subsummaries_entry_get_value (entry); - gsize checksum_bytes_len; - const guchar *checksum_bytes; - g_autofree char *digest = NULL; - g_autoptr(GVariant) summary_v = NULL; - VarSummaryRef summary; - VarRefMapRef ref_map; - gsize n_refs; - - checksum_bytes = var_subsummary_peek_checksum (subsummary, &checksum_bytes_len); - if (G_UNLIKELY (checksum_bytes_len != OSTREE_SHA256_DIGEST_LEN)) - { - g_info ("Invalid checksum for digested summary, not using cache"); - return NULL; - } - digest = ostree_checksum_from_bytes (checksum_bytes); - - s = strrchr (name, '-'); - if (s != NULL) - subset = g_strndup (name, s - name); - else - subset = g_strdup (""); - - summary_v = flatpak_repo_load_digested_summary (repo, digest, NULL); - if (summary_v == NULL) - { - g_info ("Failed to load digested summary %s, not using cache", digest); - return NULL; - } - - /* Note that all summaries refered to by the index is in new format */ - summary = var_summary_from_gvariant (summary_v); - ref_map = var_summary_get_ref_map (summary); - n_refs = var_ref_map_get_length (ref_map); - for (gsize j = 0; j < n_refs; j++) - { - VarRefMapEntryRef e = var_ref_map_get_at (ref_map, j); - const char *ref = var_ref_map_entry_get_ref (e); - VarRefInfoRef info = var_ref_map_entry_get_info (e); - VarMetadataRef commit_metadata = var_ref_info_get_metadata (info); - guint64 commit_size = var_ref_info_get_commit_size (info); - const guchar *commit_bytes; - gsize commit_bytes_len; - g_autofree char *rev = NULL; - CommitData *rev_data; - VarVariantRef xa_data_v; - VarCacheDataRef xa_data; - - if (!flatpak_is_app_runtime_or_appstream_ref (ref)) - continue; - - commit_bytes = var_ref_info_peek_checksum (info, &commit_bytes_len); - if (G_UNLIKELY (commit_bytes_len != OSTREE_SHA256_DIGEST_LEN)) - continue; - - if (!var_metadata_lookup (commit_metadata, "xa.data", NULL, &xa_data_v) || - !var_variant_is_type (xa_data_v, G_VARIANT_TYPE ("(tts)"))) - { - g_info ("Missing xa.data for ref %s, not using cache", ref); - return NULL; - } - - xa_data = var_cache_data_from_variant (xa_data_v); - - rev = ostree_checksum_from_bytes (commit_bytes); - rev_data = g_hash_table_lookup (commit_data_cache, rev); - if (rev_data == NULL) - { - g_auto(GVariantBuilder) sparse_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER; - g_variant_builder_init (&sparse_builder, G_VARIANT_TYPE_VARDICT); - gboolean has_sparse = FALSE; - - rev_data = g_new0 (CommitData, 1); - rev_data->installed_size = var_cache_data_get_installed_size (xa_data); - rev_data->download_size = var_cache_data_get_download_size (xa_data); - rev_data->metadata_contents = g_strdup (var_cache_data_get_metadata (xa_data)); - rev_data->commit_size = commit_size; - rev_data->commit_timestamp = GUINT64_FROM_BE (var_metadata_lookup_uint64 (commit_metadata, OSTREE_COMMIT_TIMESTAMP2, 0)); - - /* Get sparse data */ - gsize len = var_metadata_get_length (commit_metadata); - for (gsize k = 0; k < len; k++) - { - VarMetadataEntryRef m = var_metadata_get_at (commit_metadata, k); - const char *m_key = var_metadata_entry_get_key (m); - if (!g_str_has_prefix (m_key, "ot.") && - !g_str_has_prefix (m_key, "ostree.") && - strcmp (m_key, "xa.data") != 0) - { - VarVariantRef v = var_metadata_entry_get_value (m); - g_autoptr(GVariant) vv = g_variant_ref_sink (var_variant_dup_to_gvariant (v)); - g_autoptr(GVariant) child = g_variant_get_child_value (vv, 0); - g_variant_builder_add (&sparse_builder, "{sv}", m_key, child); - has_sparse = TRUE; - } - } - - if (has_sparse) - rev_data->sparse_data = g_variant_ref_sink (g_variant_builder_end (&sparse_builder)); - - g_hash_table_insert (commit_data_cache, g_strdup (rev), (CommitData *)rev_data); - } - - if (*subset != 0) - { - if (rev_data->subsets == NULL) - rev_data->subsets = g_ptr_array_new_with_free_func (g_free); - - if (!flatpak_g_ptr_array_contains_string (rev_data->subsets, subset)) - g_ptr_array_add (rev_data->subsets, g_strdup (subset)); - } - } - } - - return g_steal_pointer (&commit_data_cache); -} - -static CommitData * -read_commit_data (OstreeRepo *repo, - const char *ref, - const char *rev, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(GFile) root = NULL; - g_autoptr(GFile) metadata = NULL; - guint64 installed_size = 0; - guint64 download_size = 0; - g_autofree char *metadata_contents = NULL; - g_autofree char *commit = NULL; - g_autoptr(GVariant) commit_v = NULL; - g_autoptr(GVariant) commit_metadata = NULL; - g_autoptr(GPtrArray) subsets = NULL; - CommitData *rev_data; - const char *eol = NULL; - const char *eol_rebase = NULL; - int token_type = -1; - g_autoptr(GVariant) extra_data_sources = NULL; - guint32 n_extra_data = 0; - guint64 total_extra_data_download_size = 0; - g_autoptr(GVariantIter) subsets_iter = NULL; - - if (!ostree_repo_read_commit (repo, rev, &root, &commit, NULL, error)) - return NULL; - - if (!ostree_repo_load_commit (repo, commit, &commit_v, NULL, error)) - return NULL; - - commit_metadata = g_variant_get_child_value (commit_v, 0); - if (!g_variant_lookup (commit_metadata, "xa.metadata", "s", &metadata_contents)) - { - metadata = g_file_get_child (root, "metadata"); - if (!g_file_load_contents (metadata, cancellable, &metadata_contents, NULL, NULL, NULL)) - metadata_contents = g_strdup (""); - } - - if (g_variant_lookup (commit_metadata, "xa.installed-size", "t", &installed_size) && - g_variant_lookup (commit_metadata, "xa.download-size", "t", &download_size)) - { - installed_size = GUINT64_FROM_BE (installed_size); - download_size = GUINT64_FROM_BE (download_size); - } - else - { - if (!flatpak_repo_collect_sizes (repo, root, &installed_size, &download_size, cancellable, error)) - return NULL; - } - - if (g_variant_lookup (commit_metadata, "xa.subsets", "as", &subsets_iter)) - { - const char *subset; - subsets = g_ptr_array_new_with_free_func (g_free); - while (g_variant_iter_next (subsets_iter, "&s", &subset)) - g_ptr_array_add (subsets, g_strdup (subset)); - } - - flatpak_repo_collect_extra_data_sizes (repo, rev, &installed_size, &download_size); - - rev_data = g_new0 (CommitData, 1); - rev_data->installed_size = installed_size; - rev_data->download_size = download_size; - rev_data->metadata_contents = g_steal_pointer (&metadata_contents); - rev_data->subsets = g_steal_pointer (&subsets); - rev_data->commit_size = g_variant_get_size (commit_v); - rev_data->commit_timestamp = ostree_commit_get_timestamp (commit_v); - - g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_ENDOFLIFE, "&s", &eol); - g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_ENDOFLIFE_REBASE, "&s", &eol_rebase); - if (g_variant_lookup (commit_metadata, "xa.token-type", "i", &token_type)) - token_type = GINT32_FROM_LE(token_type); - - extra_data_sources = flatpak_commit_get_extra_data_sources (commit_v, NULL); - if (extra_data_sources) - { - n_extra_data = g_variant_n_children (extra_data_sources); - for (int i = 0; i < n_extra_data; i++) - { - guint64 extra_download_size; - flatpak_repo_parse_extra_data_sources (extra_data_sources, i, - NULL, - &extra_download_size, - NULL, - NULL, - NULL); - total_extra_data_download_size += extra_download_size; - } - } - - if (eol || eol_rebase || token_type >= 0 || n_extra_data > 0) - { - g_auto(GVariantBuilder) sparse_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER; - g_variant_builder_init (&sparse_builder, G_VARIANT_TYPE_VARDICT); - if (eol) - g_variant_builder_add (&sparse_builder, "{sv}", FLATPAK_SPARSE_CACHE_KEY_ENDOFLINE, g_variant_new_string (eol)); - if (eol_rebase) - g_variant_builder_add (&sparse_builder, "{sv}", FLATPAK_SPARSE_CACHE_KEY_ENDOFLINE_REBASE, g_variant_new_string (eol_rebase)); - if (token_type >= 0) - g_variant_builder_add (&sparse_builder, "{sv}", FLATPAK_SPARSE_CACHE_KEY_TOKEN_TYPE, g_variant_new_int32 (GINT32_TO_LE(token_type))); - if (n_extra_data > 0) - g_variant_builder_add (&sparse_builder, "{sv}", FLATPAK_SPARSE_CACHE_KEY_EXTRA_DATA_SIZE, - g_variant_new ("(ut)", GUINT32_TO_LE(n_extra_data), GUINT64_TO_LE(total_extra_data_download_size))); - - rev_data->sparse_data = g_variant_ref_sink (g_variant_builder_end (&sparse_builder)); - } - - return rev_data; -} - -static void -_ostree_parse_delta_name (const char *delta_name, - char **out_from, - char **out_to) -{ - g_auto(GStrv) parts = g_strsplit (delta_name, "-", 2); - - if (parts[0] && parts[1]) - { - *out_from = g_steal_pointer (&parts[0]); - *out_to = g_steal_pointer (&parts[1]); - } - else - { - *out_from = NULL; - *out_to = g_steal_pointer (&parts[0]); - } -} - -static GString * -static_delta_path_base (const char *dir, - const char *from, - const char *to) -{ - guint8 csum_to[OSTREE_SHA256_DIGEST_LEN]; - char to_b64[44]; - guint8 csum_to_copy[OSTREE_SHA256_DIGEST_LEN]; - GString *ret = g_string_new (dir); - - ostree_checksum_inplace_to_bytes (to, csum_to); - ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64); - ostree_checksum_b64_inplace_to_bytes (to_b64, csum_to_copy); - - g_assert (memcmp (csum_to, csum_to_copy, OSTREE_SHA256_DIGEST_LEN) == 0); - - if (from != NULL) - { - guint8 csum_from[OSTREE_SHA256_DIGEST_LEN]; - char from_b64[44]; - - ostree_checksum_inplace_to_bytes (from, csum_from); - ostree_checksum_b64_inplace_from_bytes (csum_from, from_b64); - - g_string_append_c (ret, from_b64[0]); - g_string_append_c (ret, from_b64[1]); - g_string_append_c (ret, '/'); - g_string_append (ret, from_b64 + 2); - g_string_append_c (ret, '-'); - } - - g_string_append_c (ret, to_b64[0]); - g_string_append_c (ret, to_b64[1]); - if (from == NULL) - g_string_append_c (ret, '/'); - g_string_append (ret, to_b64 + 2); - - return ret; -} - -static char * -_ostree_get_relative_static_delta_path (const char *from, - const char *to, - const char *target) -{ - GString *ret = static_delta_path_base ("deltas/", from, to); - - if (target != NULL) - { - g_string_append_c (ret, '/'); - g_string_append (ret, target); - } - - return g_string_free (ret, FALSE); -} - -static char * -_ostree_get_relative_static_delta_superblock_path (const char *from, - const char *to) -{ - return _ostree_get_relative_static_delta_path (from, to, "superblock"); -} - -static GVariant * -_ostree_repo_static_delta_superblock_digest (OstreeRepo *repo, - const char *from, - const char *to, - GCancellable *cancellable, - GError **error) -{ - g_autofree char *superblock = _ostree_get_relative_static_delta_superblock_path ((from && from[0]) ? from : NULL, to); - glnx_autofd int fd = -1; - guint8 digest[OSTREE_SHA256_DIGEST_LEN]; - gsize len; - gpointer data = NULL; - - if (!glnx_openat_rdonly (ostree_repo_get_dfd (repo), superblock, TRUE, &fd, error)) - return NULL; - - g_autoptr(GBytes) superblock_content = glnx_fd_readall_bytes (fd, cancellable, error); - if (!superblock_content) - return NULL; - - g_autoptr(GChecksum) checksum = g_checksum_new (G_CHECKSUM_SHA256); - g_checksum_update (checksum, g_bytes_get_data (superblock_content, NULL), g_bytes_get_size (superblock_content)); - len = sizeof digest; - g_checksum_get_digest (checksum, digest, &len); - - data = g_memdup2 (digest, len); - return g_variant_new_from_data (G_VARIANT_TYPE ("ay"), - data, len, - FALSE, g_free, data); -} - -static char * -appstream_ref_get_subset (const char *ref) -{ - if (!g_str_has_prefix (ref, "appstream2/")) - return NULL; - - const char *rest = ref + strlen ("appstream2/"); - const char *dash = strrchr (rest, '-'); - if (dash == NULL) - return NULL; - - return g_strndup (rest, dash - rest); -} - -typedef enum { - DIFF_OP_KIND_RESUSE_OLD, - DIFF_OP_KIND_SKIP_OLD, - DIFF_OP_KIND_DATA, -} DiffOpKind; - -typedef struct { - DiffOpKind kind; - gsize size; -} DiffOp; - -typedef struct { - const guchar *old_data; - const guchar *new_data; - - GArray *ops; - GArray *data; - - gsize last_old_offset; - gsize last_new_offset; -} DiffData; - -static gsize -match_bytes_at_start (const guchar *data1, - gsize data1_len, - const guchar *data2, - gsize data2_len) -{ - gsize len = 0; - gsize max_len = MIN (data1_len, data2_len); - - while (len < max_len) - { - if (*data1 != *data2) - break; - data1++; - data2++; - len++; - } - return len; -} - -static gsize -match_bytes_at_end (const guchar *data1, - gsize data1_len, - const guchar *data2, - gsize data2_len) -{ - gsize len = 0; - gsize max_len = MIN (data1_len, data2_len); - - data1 += data1_len - 1; - data2 += data2_len - 1; - - while (len < max_len) - { - if (*data1 != *data2) - break; - data1--; - data2--; - len++; - } - return len; -} - -static DiffOp * -diff_ensure_op (DiffData *data, - DiffOpKind kind) -{ - if (data->ops->len == 0 || - g_array_index (data->ops, DiffOp, data->ops->len-1).kind != kind) - { - DiffOp op = {kind, 0}; - g_array_append_val (data->ops, op); - } - - return &g_array_index (data->ops, DiffOp, data->ops->len-1); -} - -static void -diff_emit_reuse (DiffData *data, - gsize size) -{ - DiffOp *op; - - if (size == 0) - return; - - op = diff_ensure_op (data, DIFF_OP_KIND_RESUSE_OLD); - op->size += size; -} - -static void -diff_emit_skip (DiffData *data, - gsize size) -{ - DiffOp *op; - - if (size == 0) - return; - - op = diff_ensure_op (data, DIFF_OP_KIND_SKIP_OLD); - op->size += size; -} - -static void -diff_emit_data (DiffData *data, - gsize size, - const guchar *new_data) -{ - DiffOp *op; - - if (size == 0) - return; - - op = diff_ensure_op (data, DIFF_OP_KIND_DATA); - op->size += size; - - g_array_append_vals (data->data, new_data, size); -} - -static GBytes * -diff_encode (DiffData *data, GError **error) -{ - g_autoptr(GOutputStream) mem = g_memory_output_stream_new_resizable (); - g_autoptr(GDataOutputStream) out = g_data_output_stream_new (mem); - gsize ops_count = 0; - - g_data_output_stream_set_byte_order (out, G_DATA_STREAM_BYTE_ORDER_LITTLE_ENDIAN); - - /* Header */ - if (!g_output_stream_write_all (G_OUTPUT_STREAM (out), - FLATPAK_SUMMARY_DIFF_HEADER, 4, - NULL, NULL, error)) - return NULL; - - /* Write the ops count placeholder */ - if (!g_data_output_stream_put_uint32 (out, 0, NULL, error)) - return NULL; - - for (gsize i = 0; i < data->ops->len; i++) - { - DiffOp *op = &g_array_index (data->ops, DiffOp, i); - gsize size = op->size; - - while (size > 0) - { - /* We leave a nibble at the top for the op */ - guint32 opdata = (guint64)size & 0x0fffffff; - size -= opdata; - - opdata = opdata | ((0xf & op->kind) << 28); - - if (!g_data_output_stream_put_uint32 (out, opdata, NULL, error)) - return NULL; - ops_count++; - } - } - - /* Then add the data */ - if (data->data->len > 0 && - !g_output_stream_write_all (G_OUTPUT_STREAM (out), - data->data->data, data->data->len, - NULL, NULL, error)) - return NULL; - - /* Back-patch in the ops count */ - if (!g_seekable_seek (G_SEEKABLE(out), 4, G_SEEK_SET, NULL, error)) - return NULL; - - if (!g_data_output_stream_put_uint32 (out, ops_count, NULL, error)) - return NULL; - - if (!g_output_stream_close (G_OUTPUT_STREAM (out), NULL, error)) - return NULL; - - return g_memory_output_stream_steal_as_bytes (G_MEMORY_OUTPUT_STREAM (mem)); -} - -static void -diff_consume_block2 (DiffData *data, - gsize consume_old_offset, - gsize consume_old_size, - gsize produce_new_offset, - gsize produce_new_size) -{ - /* We consumed $consume_old_size bytes from $consume_old_offset to - produce $produce_new_size bytes at $produce_new_size */ - - /* First we copy old data for any matching prefix of the block */ - - gsize prefix_len = match_bytes_at_start (data->old_data + consume_old_offset, consume_old_size, - data->new_data + produce_new_offset, produce_new_size); - diff_emit_reuse (data, prefix_len); - - consume_old_size -= prefix_len; - consume_old_offset += prefix_len; - - produce_new_size -= prefix_len; - produce_new_offset += prefix_len; - - /* Then we find the matching suffix for the rest */ - gsize suffix_len = match_bytes_at_end (data->old_data + consume_old_offset, consume_old_size, - data->new_data + produce_new_offset, produce_new_size); - - /* Skip source data until suffix match */ - diff_emit_skip (data, consume_old_size - suffix_len); - - /* Copy new data until suffix match */ - diff_emit_data (data, produce_new_size - suffix_len, data->new_data + produce_new_offset); - - diff_emit_reuse (data, suffix_len); -} - -static void -diff_consume_block (DiffData *data, - gssize consume_old_offset, - gsize consume_old_size, - gssize produce_new_offset, - gsize produce_new_size) -{ - if (consume_old_offset == -1) - consume_old_offset = data->last_old_offset; - if (produce_new_offset == -1) - produce_new_offset = data->last_new_offset; - - /* We consumed $consume_old_size bytes from $consume_old_offset to - * produce $produce_new_size bytes at $produce_new_size, however - * while the emitted blocks are in order they may not cover the - * every byte, so we emit the inbetwen blocks separately. */ - - if (consume_old_offset != data->last_old_offset || - produce_new_offset != data->last_new_offset) - diff_consume_block2 (data, - data->last_old_offset, consume_old_offset - data->last_old_offset , - data->last_new_offset, produce_new_offset - data->last_new_offset); - - diff_consume_block2 (data, - consume_old_offset, consume_old_size, - produce_new_offset, produce_new_size); - - data->last_old_offset = consume_old_offset + consume_old_size; - data->last_new_offset = produce_new_offset + produce_new_size; -} - -GBytes * -flatpak_summary_apply_diff (GBytes *old, - GBytes *diff, - GError **error) -{ - g_autoptr(GBytes) uncompressed = NULL; - const guchar *diffdata; - gsize diff_size; - guint32 *ops; - guint32 n_ops; - gsize data_offset; - gsize data_size; - const guchar *data; - const guchar *old_data = g_bytes_get_data (old, NULL); - gsize old_size = g_bytes_get_size (old); - g_autoptr(GByteArray) res = g_byte_array_new (); - - uncompressed = flatpak_zlib_decompress_bytes (diff, error); - if (uncompressed == NULL) - { - g_prefix_error (error, "Invalid summary diff: "); - return NULL; - } - - diffdata = g_bytes_get_data (uncompressed, NULL); - diff_size = g_bytes_get_size (uncompressed); - - if (diff_size < 8 || - memcmp (diffdata, FLATPAK_SUMMARY_DIFF_HEADER, 4) != 0) - { - flatpak_fail (error, "Invalid summary diff"); - return NULL; - } - - n_ops = GUINT32_FROM_LE (*(guint32 *)(diffdata+4)); - ops = (guint32 *)(diffdata+8); - - data_offset = 4 + 4 + 4 * n_ops; - - /* All ops must fit in diff, and avoid wrapping the multiply */ - if (data_offset > diff_size || - (data_offset - 4 - 4) / 4 != n_ops) - { - flatpak_fail (error, "Invalid summary diff"); - return NULL; - } - - data = diffdata + data_offset; - data_size = diff_size - data_offset; - - for (gsize i = 0; i < n_ops; i++) - { - guint32 opdata = GUINT32_FROM_LE (ops[i]); - guint32 kind = (opdata & 0xf0000000) >> 28; - guint32 size = opdata & 0x0fffffff; - - switch (kind) - { - case DIFF_OP_KIND_RESUSE_OLD: - if (size > old_size) - { - flatpak_fail (error, "Invalid summary diff"); - return NULL; - } - g_byte_array_append (res, old_data, size); - old_data += size; - old_size -= size; - break; - case DIFF_OP_KIND_SKIP_OLD: - if (size > old_size) - { - flatpak_fail (error, "Invalid summary diff"); - return NULL; - } - old_data += size; - old_size -= size; - break; - case DIFF_OP_KIND_DATA: - if (size > data_size) - { - flatpak_fail (error, "Invalid summary diff"); - return NULL; - } - g_byte_array_append (res, data, size); - data += size; - data_size -= size; - break; - default: - flatpak_fail (error, "Invalid summary diff"); - return NULL; - } - } - - return g_byte_array_free_to_bytes (g_steal_pointer (&res)); -} - - -static GBytes * -flatpak_summary_generate_diff (GVariant *old_v, - GVariant *new_v, - GError **error) -{ - VarSummaryRef new, old; - VarRefMapRef new_refs, old_refs; - VarRefMapEntryRef new_entry, old_entry; - gsize new_len, old_len; - int new_i, old_i; - const char *old_ref, *new_ref; - g_autoptr(GArray) ops = g_array_new (FALSE, TRUE, sizeof (DiffOp)); - g_autoptr(GArray) data_bytes = g_array_new (FALSE, TRUE, 1); - g_autoptr(GBytes) diff_uncompressed = NULL; - g_autoptr(GBytes) diff_compressed = NULL; - DiffData data = { - g_variant_get_data (old_v), - g_variant_get_data (new_v), - ops, - data_bytes, - }; - - new = var_summary_from_gvariant (new_v); - old = var_summary_from_gvariant (old_v); - - new_refs = var_summary_get_ref_map (new); - old_refs = var_summary_get_ref_map (old); - - new_len = var_ref_map_get_length (new_refs); - old_len = var_ref_map_get_length (old_refs); - - new_i = old_i = 0; - while (new_i < new_len && old_i < old_len) - { - if (new_i == new_len) - { - /* Just old left */ - old_entry = var_ref_map_get_at (old_refs, old_i); - old_ref = var_ref_map_entry_get_ref (old_entry); - old_i++; - diff_consume_block (&data, - -1, 0, - (const guchar *)new_entry.base - (const guchar *)new.base, new_entry.size); - } - else if (old_i == old_len) - { - /* Just new left */ - new_entry = var_ref_map_get_at (new_refs, new_i); - new_ref = var_ref_map_entry_get_ref (new_entry); - diff_consume_block (&data, - (const guchar *)old_entry.base - (const guchar *)old.base, old_entry.size, - -1, 0); - - new_i++; - } - else - { - new_entry = var_ref_map_get_at (new_refs, new_i); - new_ref = var_ref_map_entry_get_ref (new_entry); - - old_entry = var_ref_map_get_at (old_refs, old_i); - old_ref = var_ref_map_entry_get_ref (old_entry); - - int cmp = strcmp (new_ref, old_ref); - if (cmp == 0) - { - /* same ref */ - diff_consume_block (&data, - (const guchar *)old_entry.base - (const guchar *)old.base, old_entry.size, - (const guchar *)new_entry.base - (const guchar *)new.base, new_entry.size); - old_i++; - new_i++; - } - else if (cmp < 0) - { - /* new added */ - diff_consume_block (&data, - -1, 0, - (const guchar *)new_entry.base - (const guchar *)new.base, new_entry.size); - new_i++; - } - else - { - /* old removed */ - diff_consume_block (&data, - (const guchar *)old_entry.base - (const guchar *)old.base, old_entry.size, - -1, 0); - old_i++; - } - } - } - - /* Flush till the end */ - diff_consume_block2 (&data, - data.last_old_offset, old.size - data.last_old_offset, - data.last_new_offset, new.size - data.last_new_offset); - - diff_uncompressed = diff_encode (&data, error); - if (diff_uncompressed == NULL) - return NULL; - - diff_compressed = flatpak_zlib_compress_bytes (diff_uncompressed, 9, error); - if (diff_compressed == NULL) - return NULL; - -#ifdef VALIDATE_DIFF - { - g_autoptr(GError) apply_error = NULL; - g_autoptr(GBytes) old_bytes = g_variant_get_data_as_bytes (old_v); - g_autoptr(GBytes) new_bytes = g_variant_get_data_as_bytes (new_v); - g_autoptr(GBytes) applied = flatpak_summary_apply_diff (old_bytes, diff_compressed, &apply_error); - g_assert (applied != NULL); - g_assert (g_bytes_equal (applied, new_bytes)); - } -#endif - - return g_steal_pointer (&diff_compressed); -} - -static void -variant_dict_merge (GVariantDict *dict, - GVariant *to_merge) -{ - GVariantIter iter; - gchar *key; - GVariant *value; - - if (to_merge) - { - g_variant_iter_init (&iter, to_merge); - while (g_variant_iter_next (&iter, "{sv}", &key, &value)) - { - g_variant_dict_insert_value (dict, key, value); - g_variant_unref (value); - g_free (key); - } - } -} - -static void -add_summary_metadata (OstreeRepo *repo, - GVariantBuilder *metadata_builder) -{ - GKeyFile *config; - g_autofree char *title = NULL; - g_autofree char *comment = NULL; - g_autofree char *description = NULL; - g_autofree char *homepage = NULL; - g_autofree char *icon = NULL; - g_autofree char *redirect_url = NULL; - g_autofree char *default_branch = NULL; - g_autofree char *remote_mode_str = NULL; - g_autofree char *authenticator_name = NULL; - g_autofree char *gpg_keys = NULL; - g_auto(GStrv) config_keys = NULL; - int authenticator_install = -1; - const char *collection_id; - gboolean deploy_collection_id = FALSE; - gboolean deploy_sideload_collection_id = FALSE; - gboolean tombstone_commits = FALSE; - - config = ostree_repo_get_config (repo); - - if (config) - { - remote_mode_str = g_key_file_get_string (config, "core", "mode", NULL); - tombstone_commits = g_key_file_get_boolean (config, "core", "tombstone-commits", NULL); - - title = g_key_file_get_string (config, "flatpak", "title", NULL); - comment = g_key_file_get_string (config, "flatpak", "comment", NULL); - description = g_key_file_get_string (config, "flatpak", "description", NULL); - homepage = g_key_file_get_string (config, "flatpak", "homepage", NULL); - icon = g_key_file_get_string (config, "flatpak", "icon", NULL); - default_branch = g_key_file_get_string (config, "flatpak", "default-branch", NULL); - gpg_keys = g_key_file_get_string (config, "flatpak", "gpg-keys", NULL); - redirect_url = g_key_file_get_string (config, "flatpak", "redirect-url", NULL); - deploy_sideload_collection_id = g_key_file_get_boolean (config, "flatpak", "deploy-sideload-collection-id", NULL); - deploy_collection_id = g_key_file_get_boolean (config, "flatpak", "deploy-collection-id", NULL); - authenticator_name = g_key_file_get_string (config, "flatpak", "authenticator-name", NULL); - if (g_key_file_has_key (config, "flatpak", "authenticator-install", NULL)) - authenticator_install = g_key_file_get_boolean (config, "flatpak", "authenticator-install", NULL); - - config_keys = g_key_file_get_keys (config, "flatpak", NULL, NULL); - } - - collection_id = ostree_repo_get_collection_id (repo); - - g_variant_builder_add (metadata_builder, "{sv}", "ostree.summary.mode", - g_variant_new_string (remote_mode_str ? remote_mode_str : "bare")); - g_variant_builder_add (metadata_builder, "{sv}", "ostree.summary.tombstone-commits", - g_variant_new_boolean (tombstone_commits)); - g_variant_builder_add (metadata_builder, "{sv}", "ostree.summary.indexed-deltas", - g_variant_new_boolean (TRUE)); - g_variant_builder_add (metadata_builder, "{sv}", "ostree.summary.last-modified", - g_variant_new_uint64 (GUINT64_TO_BE (g_get_real_time () / G_USEC_PER_SEC))); - - if (collection_id) - g_variant_builder_add (metadata_builder, "{sv}", "ostree.summary.collection-id", - g_variant_new_string (collection_id)); - - if (title) - g_variant_builder_add (metadata_builder, "{sv}", "xa.title", - g_variant_new_string (title)); - - if (comment) - g_variant_builder_add (metadata_builder, "{sv}", "xa.comment", - g_variant_new_string (comment)); - - if (description) - g_variant_builder_add (metadata_builder, "{sv}", "xa.description", - g_variant_new_string (description)); - - if (homepage) - g_variant_builder_add (metadata_builder, "{sv}", "xa.homepage", - g_variant_new_string (homepage)); - - if (icon) - g_variant_builder_add (metadata_builder, "{sv}", "xa.icon", - g_variant_new_string (icon)); - - if (redirect_url) - g_variant_builder_add (metadata_builder, "{sv}", "xa.redirect-url", - g_variant_new_string (redirect_url)); - - if (default_branch) - g_variant_builder_add (metadata_builder, "{sv}", "xa.default-branch", - g_variant_new_string (default_branch)); - - if (deploy_collection_id && collection_id != NULL) - g_variant_builder_add (metadata_builder, "{sv}", OSTREE_META_KEY_DEPLOY_COLLECTION_ID, - g_variant_new_string (collection_id)); - else if (deploy_sideload_collection_id && collection_id != NULL) - g_variant_builder_add (metadata_builder, "{sv}", "xa.deploy-collection-id", - g_variant_new_string (collection_id)); - else if (deploy_collection_id) - g_info ("Ignoring deploy-collection-id=true because no collection ID is set."); - - if (authenticator_name) - g_variant_builder_add (metadata_builder, "{sv}", "xa.authenticator-name", - g_variant_new_string (authenticator_name)); - - if (authenticator_install != -1) - g_variant_builder_add (metadata_builder, "{sv}", "xa.authenticator-install", - g_variant_new_boolean (authenticator_install)); - - g_variant_builder_add (metadata_builder, "{sv}", "xa.cache-version", - g_variant_new_uint32 (GUINT32_TO_LE (FLATPAK_XA_CACHE_VERSION))); - - if (config_keys != NULL) - { - for (int i = 0; config_keys[i] != NULL; i++) - { - const char *key = config_keys[i]; - g_autofree char *xa_key = NULL; - g_autofree char *value = NULL; - - if (!g_str_has_prefix (key, "authenticator-options.")) - continue; - - value = g_key_file_get_string (config, "flatpak", key, NULL); - if (value == NULL) - continue; - - xa_key = g_strconcat ("xa.", key, NULL); - g_variant_builder_add (metadata_builder, "{sv}", xa_key, - g_variant_new_string (value)); - } - } - - if (gpg_keys) - { - guchar *decoded; - gsize decoded_len; - - gpg_keys = g_strstrip (gpg_keys); - decoded = g_base64_decode (gpg_keys, &decoded_len); - - g_variant_builder_add (metadata_builder, "{sv}", "xa.gpg-keys", - g_variant_new_from_data (G_VARIANT_TYPE ("ay"), decoded, decoded_len, - TRUE, (GDestroyNotify) g_free, decoded)); - } -} - -static GVariant * -generate_summary (OstreeRepo *repo, - gboolean compat_format, - GHashTable *refs, - GHashTable *commit_data_cache, - GPtrArray *delta_names, - const char *subset, - const char **summary_arches, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(GVariantBuilder) metadata_builder = g_variant_builder_new (G_VARIANT_TYPE_VARDICT); - g_autoptr(GVariantBuilder) ref_data_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{s(tts)}")); - g_autoptr(GVariantBuilder) ref_sparse_data_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sa{sv}}")); - g_autoptr(GVariantBuilder) refs_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(s(taya{sv}))")); - g_autoptr(GVariantBuilder) summary_builder = g_variant_builder_new (OSTREE_SUMMARY_GVARIANT_FORMAT); - g_autoptr(GHashTable) summary_arches_ht = NULL; - g_autoptr(GHashTable) commits = NULL; - g_autoptr(GList) ordered_keys = NULL; - GList *l = NULL; - - /* In the new format this goes in the summary index instead */ - if (compat_format) - add_summary_metadata (repo, metadata_builder); - - ordered_keys = g_hash_table_get_keys (refs); - ordered_keys = g_list_sort (ordered_keys, (GCompareFunc) strcmp); - - if (summary_arches) - { - summary_arches_ht = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL); - for (int i = 0; summary_arches[i] != NULL; i++) - { - const char *arch = summary_arches[i]; - - g_hash_table_add (summary_arches_ht, (char *)arch); - } - } - - /* Compute which commits to keep */ - commits = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL); /* strings owned by ref */ - for (l = ordered_keys; l; l = l->next) - { - const char *ref = l->data; - const char *rev = g_hash_table_lookup (refs, ref); - g_autofree char *arch = NULL; - const CommitData *rev_data = NULL; - - if (summary_arches) - { - /* NOTE: Non-arched (unknown) refs get into all summary versions */ - arch = flatpak_get_arch_for_ref (ref); - if (arch != NULL && !g_hash_table_contains (summary_arches_ht, arch)) - continue; /* Filter this ref by arch */ - } - - rev_data = g_hash_table_lookup (commit_data_cache, rev); - if (*subset != 0) - { - /* Subset summaries keep the appstream2/$subset-$arch, and have no appstream/ compat branch */ - - if (g_str_has_prefix (ref, "appstream/")) - { - continue; /* No compat branch in subsets */ - } - else if (g_str_has_prefix (ref, "appstream2/")) - { - g_autofree char *ref_subset = appstream_ref_get_subset (ref); - if (ref_subset == NULL) - continue; /* Non-subset, ignore */ - - if (strcmp (subset, ref_subset) != 0) - continue; /* Different subset, ignore */ - - /* Otherwise, keep */ - } - else if (rev_data) - { - if (rev_data->subsets == NULL || - !flatpak_g_ptr_array_contains_string (rev_data->subsets, subset)) - continue; /* Ref is not in this subset */ - } - } - else - { - /* non-subset, keep everything but subset appstream refs */ - - g_autofree char *ref_subset = appstream_ref_get_subset (ref); - if (ref_subset != NULL) - continue; /* Subset appstream ref, ignore */ - } - - g_hash_table_add (commits, (char *)rev); - } - - /* Create refs list, metadata and sparse_data */ - for (l = ordered_keys; l; l = l->next) - { - const char *ref = l->data; - const char *rev = g_hash_table_lookup (refs, ref); - const CommitData *rev_data = NULL; - g_auto(GVariantDict) commit_metadata_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER; - guint64 commit_size; - guint64 commit_timestamp; - - if (!g_hash_table_contains (commits, rev)) - continue; /* Filter out commit (by arch & subset) */ - - if (flatpak_is_app_runtime_or_appstream_ref (ref)) - rev_data = g_hash_table_lookup (commit_data_cache, rev); - - if (rev_data != NULL) - { - commit_size = rev_data->commit_size; - commit_timestamp = rev_data->commit_timestamp; - } - else - { - g_autoptr(GVariant) commit_obj = NULL; - if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, rev, &commit_obj, error)) - return NULL; - commit_size = g_variant_get_size (commit_obj); - commit_timestamp = ostree_commit_get_timestamp (commit_obj); - } - - g_variant_dict_init (&commit_metadata_builder, NULL); - if (!compat_format && rev_data) - { - g_variant_dict_insert (&commit_metadata_builder, "xa.data", "(tts)", - GUINT64_TO_BE (rev_data->installed_size), - GUINT64_TO_BE (rev_data->download_size), - rev_data->metadata_contents); - variant_dict_merge (&commit_metadata_builder, rev_data->sparse_data); - } - - /* For the new format summary we use a shorter name for the timestamp to save space */ - g_variant_dict_insert_value (&commit_metadata_builder, - compat_format ? OSTREE_COMMIT_TIMESTAMP : OSTREE_COMMIT_TIMESTAMP2, - g_variant_new_uint64 (GUINT64_TO_BE (commit_timestamp))); - - g_variant_builder_add_value (refs_builder, - g_variant_new ("(s(t@ay@a{sv}))", ref, - commit_size, - ostree_checksum_to_bytes_v (rev), - g_variant_dict_end (&commit_metadata_builder))); - - if (compat_format && rev_data) - { - g_variant_builder_add (ref_data_builder, "{s(tts)}", - ref, - GUINT64_TO_BE (rev_data->installed_size), - GUINT64_TO_BE (rev_data->download_size), - rev_data->metadata_contents); - if (rev_data->sparse_data) - g_variant_builder_add (ref_sparse_data_builder, "{s@a{sv}}", - ref, rev_data->sparse_data); - } - } - - if (delta_names) - { - g_auto(GVariantDict) deltas_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER; - - g_variant_dict_init (&deltas_builder, NULL); - for (guint i = 0; i < delta_names->len; i++) - { - g_autofree char *from = NULL; - g_autofree char *to = NULL; - GVariant *digest; - - _ostree_parse_delta_name (delta_names->pdata[i], &from, &to); - - /* Only keep deltas going to a ref that is in the summary - * (i.e. not arch filtered or random) */ - if (!g_hash_table_contains (commits, to)) - continue; - - digest = _ostree_repo_static_delta_superblock_digest (repo, - (from && from[0]) ? from : NULL, - to, cancellable, error); - if (digest == NULL) - return FALSE; - - g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], digest); - } - - if (delta_names->len > 0) - g_variant_builder_add (metadata_builder, "{sv}", "ostree.static-deltas", g_variant_dict_end (&deltas_builder)); - } - - if (compat_format) - { - /* Note: xa.cache doesn’t need to support collection IDs for the refs listed - * in it, because the xa.cache metadata is stored on the ostree-metadata ref, - * which is itself strongly bound to a collection ID — so that collection ID - * is bound to all the refs in xa.cache. If a client is using the xa.cache - * data from a summary file (rather than an ostree-metadata branch), they are - * too old to care about collection IDs anyway. */ - g_variant_builder_add (metadata_builder, "{sv}", "xa.cache", - g_variant_new_variant (g_variant_builder_end (ref_data_builder))); - g_variant_builder_add (metadata_builder, "{sv}", "xa.sparse-cache", - g_variant_builder_end (ref_sparse_data_builder)); - } - else - { - g_variant_builder_add (metadata_builder, "{sv}", "xa.summary-version", - g_variant_new_uint32 (GUINT32_TO_LE (FLATPAK_XA_SUMMARY_VERSION))); - } - - g_variant_builder_add_value (summary_builder, g_variant_builder_end (refs_builder)); - g_variant_builder_add_value (summary_builder, g_variant_builder_end (metadata_builder)); - - return g_variant_ref_sink (g_variant_builder_end (summary_builder)); -} - -static GVariant * -read_digested_summary (OstreeRepo *repo, - const char *digest, - GHashTable *digested_summary_cache, - GCancellable *cancellable, - GError **error) -{ - GVariant *cached; - g_autoptr(GVariant) loaded = NULL; - - cached = g_hash_table_lookup (digested_summary_cache, digest); - if (cached) - return g_variant_ref (cached); - - loaded = flatpak_repo_load_digested_summary (repo, digest, error); - if (loaded == NULL) - return NULL; - - g_hash_table_insert (digested_summary_cache, g_strdup (digest), g_variant_ref (loaded)); - - return g_steal_pointer (&loaded); -} - -static gboolean -add_to_history (OstreeRepo *repo, - GVariantBuilder *history_builder, - VarChecksumRef old_digest_vv, - GVariant *current_digest_v, - GVariant *current_content, - GHashTable *digested_summary_cache, - guint *history_len, - guint max_history_length, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(GVariant) old_digest_v = g_variant_ref_sink (var_checksum_dup_to_gvariant (old_digest_vv)); - g_autofree char *old_digest = NULL; - g_autoptr(GVariant) old_content = NULL; - g_autofree char *current_digest = NULL; - g_autoptr(GBytes) subsummary_diff = NULL; - - /* Limit history length */ - if (*history_len >= max_history_length) - return TRUE; - - /* Avoid repeats in the history (in case nothing changed in subsummary) */ - if (g_variant_equal (old_digest_v, current_digest_v)) - return TRUE; - - old_digest = ostree_checksum_from_bytes_v (old_digest_v); - old_content = read_digested_summary (repo, old_digest, digested_summary_cache, cancellable, NULL); - if (old_content == NULL) - return TRUE; /* Only add parents that still exist */ - - subsummary_diff = flatpak_summary_generate_diff (old_content, current_content, error); - if (subsummary_diff == NULL) - return FALSE; - - current_digest = ostree_checksum_from_bytes_v (current_digest_v); - - if (!flatpak_repo_save_digested_summary_delta (repo, old_digest, current_digest, - subsummary_diff, cancellable, error)) - return FALSE; - - *history_len += 1; - g_variant_builder_add_value (history_builder, old_digest_v); - - return TRUE; -} - -static GVariant * -generate_summary_index (OstreeRepo *repo, - GVariant *old_index_v, - GHashTable *summaries, - GHashTable *digested_summaries, - GHashTable *digested_summary_cache, - const char **gpg_key_ids, - const char *gpg_homedir, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(GVariantBuilder) metadata_builder = g_variant_builder_new (G_VARIANT_TYPE_VARDICT); - g_autoptr(GVariantBuilder) subsummary_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{s(ayaaya{sv})}")); - g_autoptr(GVariantBuilder) index_builder = g_variant_builder_new (FLATPAK_SUMMARY_INDEX_GVARIANT_FORMAT); - g_autoptr(GVariant) index = NULL; - g_autoptr(GList) ordered_summaries = NULL; - guint max_history_length = flatpak_repo_get_summary_history_length (repo); - GList *l; - - add_summary_metadata (repo, metadata_builder); - - ordered_summaries = g_hash_table_get_keys (summaries); - ordered_summaries = g_list_sort (ordered_summaries, (GCompareFunc) strcmp); - for (l = ordered_summaries; l; l = l->next) - { - g_auto(GVariantDict) subsummary_metadata_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER; - const char *subsummary = l->data; - const char *digest = g_hash_table_lookup (summaries, subsummary); - g_autoptr(GVariant) digest_v = g_variant_ref_sink (ostree_checksum_to_bytes_v (digest)); - g_autoptr(GVariantBuilder) history_builder = g_variant_builder_new (G_VARIANT_TYPE ("aay")); - g_autoptr(GVariant) subsummary_content = NULL; - - subsummary_content = read_digested_summary (repo, digest, digested_summary_cache, cancellable, error); - if (subsummary_content == NULL) - return NULL; /* This really should always be there as we're supposed to index it */ - - if (old_index_v) - { - VarSummaryIndexRef old_index = var_summary_index_from_gvariant (old_index_v); - VarSummaryIndexSubsummariesRef old_subsummaries = var_summary_index_get_subsummaries (old_index); - VarSubsummaryRef old_subsummary; - guint history_len = 0; - - if (var_summary_index_subsummaries_lookup (old_subsummaries, subsummary, NULL, &old_subsummary)) - { - VarChecksumRef parent = var_subsummary_get_checksum (old_subsummary); - - /* Add current as first in history */ - if (!add_to_history (repo, history_builder, parent, digest_v, subsummary_content, digested_summary_cache, - &history_len, max_history_length, cancellable, error)) - return FALSE; - - /* Add previous history */ - VarArrayofChecksumRef history = var_subsummary_get_history (old_subsummary); - gsize len = var_arrayof_checksum_get_length (history); - for (gsize i = 0; i < len; i++) - { - VarChecksumRef c = var_arrayof_checksum_get_at (history, i); - if (!add_to_history (repo, history_builder, c, digest_v, subsummary_content, digested_summary_cache, - &history_len, max_history_length, cancellable, error)) - return FALSE; - } - } - } - - g_variant_dict_init (&subsummary_metadata_builder, NULL); - g_variant_builder_add (subsummary_builder, "{s(@ay@aay@a{sv})}", - subsummary, - digest_v, - g_variant_builder_end (history_builder), - g_variant_dict_end (&subsummary_metadata_builder)); - } - - g_variant_builder_add_value (index_builder, g_variant_builder_end (subsummary_builder)); - g_variant_builder_add_value (index_builder, g_variant_builder_end (metadata_builder)); - - index = g_variant_ref_sink (g_variant_builder_end (index_builder)); - - return g_steal_pointer (&index); -} - -static gboolean -flatpak_repo_gc_digested_summaries (OstreeRepo *repo, - const char *index_digest, /* The digest of the current (new) index (if any) */ - const char *old_index_digest, /* The digest of the previous index (if any) */ - GHashTable *digested_summaries, /* generated */ - GHashTable *digested_summary_cache, /* generated + referenced */ - GCancellable *cancellable, - GError **error) -{ - g_auto(GLnxDirFdIterator) iter = {0}; - int repo_fd = ostree_repo_get_dfd (repo); - struct dirent *dent; - const char *ext; - g_autoptr(GError) local_error = NULL; - - if (!glnx_dirfd_iterator_init_at (repo_fd, "summaries", FALSE, &iter, &local_error)) - { - if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - return TRUE; - - g_propagate_error (error, g_steal_pointer (&local_error)); - return FALSE; - } - - while (TRUE) - { - gboolean remove = FALSE; - - if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&iter, &dent, cancellable, error)) - return FALSE; - - if (dent == NULL) - break; - - if (dent->d_type != DT_REG) - continue; - - /* Keep it if its an unexpected type */ - ext = strchr (dent->d_name, '.'); - if (ext != NULL) - { - if (strcmp (ext, ".gz") == 0 && strlen (dent->d_name) == 64 + 3) - { - g_autofree char *sha256 = g_strndup (dent->d_name, 64); - - /* Keep all the referenced summaries */ - if (g_hash_table_contains (digested_summary_cache, sha256)) - { - g_info ("Keeping referenced summary %s", dent->d_name); - continue; - } - /* Remove rest */ - remove = TRUE; - } - else if (strcmp (ext, ".delta") == 0) - { - const char *dash = strchr (dent->d_name, '-'); - if (dash != NULL && dash < ext && (ext - dash) == 1 + 64) - { - g_autofree char *to_sha256 = g_strndup (dash + 1, 64); - - /* Only keep deltas going to a generated summary */ - if (g_hash_table_contains (digested_summaries, to_sha256)) - { - g_info ("Keeping delta to generated summary %s", dent->d_name); - continue; - } - /* Remove rest */ - remove = TRUE; - } - } - else if (strcmp (ext, ".idx.sig") == 0) - { - g_autofree char *digest = g_strndup (dent->d_name, strlen (dent->d_name) - strlen (".idx.sig")); - - if (g_strcmp0 (digest, index_digest) == 0) - continue; /* Always keep current */ - - if (g_strcmp0 (digest, old_index_digest) == 0) - continue; /* Always keep previous one, to avoid some races */ - - /* Remove the rest */ - remove = TRUE; - } - } - - if (remove) - { - g_info ("Removing old digested summary file %s", dent->d_name); - if (unlinkat (iter.fd, dent->d_name, 0) != 0) - { - glnx_set_error_from_errno (error); - return FALSE; - } - } - else - g_info ("Keeping unexpected summary file %s", dent->d_name); - } - - return TRUE; -} - - -/* Update the metadata in the summary file for @repo, and then re-sign the file. - * If the repo has a collection ID set, additionally store the metadata on a - * contentless commit in a well-known branch, which is the preferred way of - * broadcasting per-repo metadata (putting it in the summary file is deprecated, - * but kept for backwards compatibility). - * - * Note that there are two keys for the collection ID: collection-id, and - * ostree.deploy-collection-id. If a client does not currently have a - * collection ID configured for this remote, it will *only* update its - * configuration from ostree.deploy-collection-id. This allows phased - * deployment of collection-based repositories. Clients will only update their - * configuration from an unset to a set collection ID once (otherwise the - * security properties of collection IDs are broken). */ -gboolean -flatpak_repo_update (OstreeRepo *repo, - FlatpakRepoUpdateFlags flags, - const char **gpg_key_ids, - const char *gpg_homedir, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(GHashTable) commit_data_cache = NULL; - g_autoptr(GVariant) compat_summary = NULL; - g_autoptr(GVariant) summary_index = NULL; - g_autoptr(GVariant) old_index = NULL; - g_autoptr(GPtrArray) delta_names = NULL; - g_auto(GStrv) summary_arches = NULL; - g_autoptr(GHashTable) refs = NULL; - g_autoptr(GHashTable) arches = NULL; - g_autoptr(GHashTable) subsets = NULL; - g_autoptr(GHashTable) summaries = NULL; - g_autoptr(GHashTable) digested_summaries = NULL; - g_autoptr(GHashTable) digested_summary_cache = NULL; - g_autoptr(GBytes) index_sig = NULL; - time_t old_compat_sig_mtime; - GKeyFile *config; - gboolean disable_index = (flags & FLATPAK_REPO_UPDATE_FLAG_DISABLE_INDEX) != 0; - g_autofree char *index_digest = NULL; - g_autofree char *old_index_digest = NULL; - - config = ostree_repo_get_config (repo); - - if (!ostree_repo_list_refs_ext (repo, NULL, &refs, - OSTREE_REPO_LIST_REFS_EXT_EXCLUDE_REMOTES | OSTREE_REPO_LIST_REFS_EXT_EXCLUDE_MIRRORS, - cancellable, error)) - return FALSE; - - old_index = flatpak_repo_load_summary_index (repo, NULL); - if (old_index) - commit_data_cache = populate_commit_data_cache (repo, old_index); - - if (commit_data_cache == NULL) /* No index or failed to load it */ - commit_data_cache = commit_data_cache_new (); - - if (!ostree_repo_list_static_delta_names (repo, &delta_names, cancellable, error)) - return FALSE; - - if (config) - summary_arches = g_key_file_get_string_list (config, "flatpak", "summary-arches", NULL, NULL); - - summaries = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); - /* These are the ones we generated */ - digested_summaries = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)g_variant_unref); - /* These are the ones generated or references */ - digested_summary_cache = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)g_variant_unref); - - arches = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - subsets = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - g_hash_table_add (subsets, g_strdup ("")); /* Always have everything subset */ - - GLNX_HASH_TABLE_FOREACH_KV (refs, const char *, ref, const char *, rev) - { - g_autofree char *arch = flatpak_get_arch_for_ref (ref); - CommitData *rev_data = NULL; - - if (arch != NULL && - !g_hash_table_contains (arches, arch)) - g_hash_table_add (arches, g_steal_pointer (&arch)); - - /* Add CommitData for flatpak refs that we didn't already pre-populate */ - if (flatpak_is_app_runtime_or_appstream_ref (ref)) - { - rev_data = g_hash_table_lookup (commit_data_cache, rev); - if (rev_data == NULL) - { - rev_data = read_commit_data (repo, ref, rev, cancellable, error); - if (rev_data == NULL) - return FALSE; - - g_hash_table_insert (commit_data_cache, g_strdup (rev), (CommitData *)rev_data); - } - - for (int i = 0; rev_data->subsets != NULL && i < rev_data->subsets->len; i++) - { - const char *subset = g_ptr_array_index (rev_data->subsets, i); - if (!g_hash_table_contains (subsets, subset)) - g_hash_table_add (subsets, g_strdup (subset)); - } - } - } - - compat_summary = generate_summary (repo, TRUE, refs, commit_data_cache, delta_names, - "", (const char **)summary_arches, - cancellable, error); - if (compat_summary == NULL) - return FALSE; - - if (!disable_index) - { - GLNX_HASH_TABLE_FOREACH (subsets, const char *, subset) - { - GLNX_HASH_TABLE_FOREACH (arches, const char *, arch) - { - const char *arch_v[] = { arch, NULL }; - g_autofree char *name = NULL; - g_autofree char *digest = NULL; - - if (*subset == 0) - name = g_strdup (arch); - else - name = g_strconcat (subset, "-", arch, NULL); - - g_autoptr(GVariant) arch_summary = generate_summary (repo, FALSE, refs, commit_data_cache, NULL, subset, arch_v, - cancellable, error); - if (arch_summary == NULL) - return FALSE; - - digest = flatpak_repo_save_digested_summary (repo, name, arch_summary, cancellable, error); - if (digest == NULL) - return FALSE; - - g_hash_table_insert (digested_summaries, g_strdup (digest), g_variant_ref (arch_summary)); - /* Prime summary cache with generated summaries */ - g_hash_table_insert (digested_summary_cache, g_strdup (digest), g_variant_ref (arch_summary)); - g_hash_table_insert (summaries, g_steal_pointer (&name), g_steal_pointer (&digest)); - } - } - - summary_index = generate_summary_index (repo, old_index, summaries, digested_summaries, digested_summary_cache, - gpg_key_ids, gpg_homedir, - cancellable, error); - if (summary_index == NULL) - return FALSE; - } - - if (!ostree_repo_static_delta_reindex (repo, 0, NULL, cancellable, error)) - return FALSE; - - if (summary_index && gpg_key_ids) - { - g_autoptr(GBytes) index_bytes = g_variant_get_data_as_bytes (summary_index); - - if (!ostree_repo_gpg_sign_data (repo, index_bytes, - NULL, - gpg_key_ids, - gpg_homedir, - &index_sig, - cancellable, - error)) - return FALSE; - } - - if (summary_index) - index_digest = g_compute_checksum_for_data (G_CHECKSUM_SHA256, - g_variant_get_data (summary_index), - g_variant_get_size (summary_index)); - if (old_index) - old_index_digest = g_compute_checksum_for_data (G_CHECKSUM_SHA256, - g_variant_get_data (old_index), - g_variant_get_size (old_index)); - - /* Release the memory-mapped summary index file before replacing it, - to avoid failure on filesystems like cifs */ - g_clear_pointer (&old_index, g_variant_unref); - - if (!flatpak_repo_save_summary_index (repo, summary_index, index_digest, index_sig, cancellable, error)) - return FALSE; - - if (!flatpak_repo_save_compat_summary (repo, compat_summary, &old_compat_sig_mtime, cancellable, error)) - return FALSE; - - if (gpg_key_ids) - { - if (!ostree_repo_add_gpg_signature_summary (repo, - gpg_key_ids, - gpg_homedir, - cancellable, - error)) - return FALSE; - - - if (old_compat_sig_mtime != 0) - { - int repo_dfd = ostree_repo_get_dfd (repo); - struct stat stbuf; - - /* Ensure we increase (in sec precision) */ - if (fstatat (repo_dfd, "summary.sig", &stbuf, AT_SYMLINK_NOFOLLOW) == 0 && - stbuf.st_mtime <= old_compat_sig_mtime) - { - struct timespec ts[2] = { {0, UTIME_OMIT}, {old_compat_sig_mtime + 1, 0} }; - (void) utimensat (repo_dfd, "summary.sig", ts, AT_SYMLINK_NOFOLLOW); - } - } - } - - if (!disable_index && - !flatpak_repo_gc_digested_summaries (repo, index_digest, old_index_digest, digested_summaries, digested_summary_cache, cancellable, error)) - return FALSE; - - return TRUE; -} - gboolean flatpak_mtree_create_dir (OstreeRepo *repo, OstreeMutableTree *parent, @@ -5632,69 +3405,6 @@ flatpak_levenshtein_distance (const char *s, return dist (s, ls, t, lt, 0, 0, d); } -/* Wrapper that uses ostree_repo_resolve_collection_ref() and on failure falls - * back to using ostree_repo_resolve_rev() for backwards compatibility. This - * means we support refs/heads/, refs/remotes/, and refs/mirrors/. */ -gboolean -flatpak_repo_resolve_rev (OstreeRepo *repo, - const char *collection_id, /* nullable */ - const char *remote_name, /* nullable */ - const char *ref_name, - gboolean allow_noent, - char **out_rev, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(GError) local_error = NULL; - - if (collection_id != NULL) - { - /* Do a version check to ensure we have these: - * https://github.com/ostreedev/ostree/pull/1821 - * https://github.com/ostreedev/ostree/pull/1825 */ -#if OSTREE_CHECK_VERSION (2019, 2) - const OstreeCollectionRef c_r = - { - .collection_id = (char *) collection_id, - .ref_name = (char *) ref_name, - }; - OstreeRepoResolveRevExtFlags flags = remote_name == NULL ? - OSTREE_REPO_RESOLVE_REV_EXT_LOCAL_ONLY : - OSTREE_REPO_RESOLVE_REV_EXT_NONE; - if (ostree_repo_resolve_collection_ref (repo, &c_r, - allow_noent, - flags, - out_rev, - cancellable, NULL)) - return TRUE; -#endif - } - - /* There may be several remotes with the same branch (if we for - * instance changed the origin) so prepend the current origin to - * make sure we get the right one */ - if (remote_name != NULL) - { - g_autofree char *refspec = g_strdup_printf ("%s:%s", remote_name, ref_name); - ostree_repo_resolve_rev (repo, refspec, allow_noent, out_rev, &local_error); - } - else - ostree_repo_resolve_rev_ext (repo, ref_name, allow_noent, - OSTREE_REPO_RESOLVE_REV_EXT_NONE, out_rev, &local_error); - - if (local_error != NULL) - { - if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, "%s", local_error->message); - else - g_propagate_error (error, g_steal_pointer (&local_error)); - - return FALSE; - } - - return TRUE; -} - /* Convert an app id to a dconf path in the obvious way. */ char * diff --git a/po/POTFILES.in b/po/POTFILES.in index 5f28f1bd..24148de0 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -59,6 +59,7 @@ common/flatpak-oci-registry.c common/flatpak-progress.c common/flatpak-ref-utils.c common/flatpak-remote.c +common/flatpak-repo-utils.c common/flatpak-run.c common/flatpak-run-dbus.c common/flatpak-transaction.c