From b1fdf6c4ef9d10b90a5d1004963f9c13606e2c15 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 19 Mar 2020 14:00:21 +0100 Subject: [PATCH 01/36] tests: Add and use "ok" helper This allows us to print some separators for the logs also. tests: Convert more tests to ok helper --- tests/libtest.sh | 5 ++ tests/test-auth.sh | 6 +- tests/test-basic.sh | 20 +++--- tests/test-build-update-repo.sh | 10 +-- tests/test-bundle.sh | 16 ++--- tests/test-completion.sh | 34 +++++----- tests/test-config.sh | 10 +-- tests/test-default-remotes.sh | 10 +-- tests/test-extensions.sh | 4 +- tests/test-http-utils.sh | 16 ++--- tests/test-info.sh | 14 ++--- tests/test-oci-registry.sh | 28 ++++----- tests/test-oci.sh | 4 +- tests/test-override.sh | 30 ++++----- tests/test-repo.sh | 76 +++++++++++------------ tests/test-run.sh | 32 +++++----- tests/test-unsigned-summaries.sh | 14 ++--- tests/test-update-portal.sh | 12 ++-- tests/test-update-remote-configuration.sh | 6 +- 19 files changed, 176 insertions(+), 171 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 795c69e1..ce1f41fe 100644 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -34,6 +34,11 @@ assert_not_reached () { echo $@ 1>&2; exit 1 } +ok () { + echo "ok $@" + echo "============================================================" +} + test_tmpdir=$(pwd) # Sanity check that we're in a tmpdir that has diff --git a/tests/test-auth.sh b/tests/test-auth.sh index a0f71a3b..d9e7c426 100755 --- a/tests/test-auth.sh +++ b/tests/test-auth.sh @@ -99,7 +99,7 @@ assert_failed_with_401 echo -n the-secret > ${XDG_RUNTIME_DIR}/required-token ${FLATPAK} ${U} update -y org.test.Hello -echo "ok installed build-exported token-type app" +ok "installed build-exported token-type app" # Drop token-type on main version make_updated_app test "" master UPDATE3 @@ -121,7 +121,7 @@ assert_failed_with_401 echo -n the-secret > ${XDG_RUNTIME_DIR}/required-token ${FLATPAK} ${U} install -y test-repo org.test.Hello//copy -echo "ok installed build-commit-from token-type app" +ok "installed build-commit-from token-type app" EXPORT_ARGS="--token-type=2" make_updated_app test "" master UPDATE4 mark_need_token app/org.test.Hello/$ARCH/master the-secret @@ -151,4 +151,4 @@ export BROWSER=curl EXPORT_ARGS="--token-type=2" make_updated_app test "" master UPDATE5 mark_need_token app/org.test.Hello/$ARCH/master the-secret -echo "ok update with webflow" +ok "update with webflow" diff --git a/tests/test-basic.sh b/tests/test-basic.sh index d4ab5a7c..88d2f8e8 100755 --- a/tests/test-basic.sh +++ b/tests/test-basic.sh @@ -31,13 +31,13 @@ ${FLATPAK} --version > version_out VERSION=`cat "$test_builddir/package_version.txt"` assert_file_has_content version_out "^Flatpak $VERSION$" -echo "ok version" +ok "version" ${FLATPAK} --help > help_out assert_file_has_content help_out "^Usage:$" -echo "ok help" +ok "help" ${FLATPAK} --default-arch > arch @@ -45,14 +45,14 @@ ${FLATPAK} --supported-arches > arches assert_streq `head -1 arches` `cat arch` -echo "ok default arch" +ok "default arch" ${FLATPAK} --gl-drivers > drivers assert_file_has_content drivers "^default$"; assert_file_has_content drivers "^host$"; -echo "ok gl drivers" +ok "gl drivers" for cmd in install update uninstall list info config repair create-usb \ search run override make-current enter ps document-export \ @@ -69,7 +69,7 @@ do assert_file_has_content help_out "flatpak $cmd" done -echo "ok command help" +ok "command help" for cmd in list ps remote-ls remotes documents history; do @@ -80,28 +80,28 @@ do assert_file_has_content help_out "^ help" done -echo "ok columns help" +ok "columns help" ${FLATPAK} >out 2>&1 || true assert_file_has_content out "^error: No command specified$" assert_file_has_content out "flatpak --help" -echo "ok missing command" +ok "missing command" ${FLATPAK} indo >out 2>&1 || true assert_file_has_content out "^error: .* 'info'" assert_file_has_content out "flatpak --help" -echo "ok misspelt command" +ok "misspelt command" ${FLATPAK} info >out 2>&1 || true assert_file_has_content out "^error: NAME must be specified$" assert_file_has_content out "flatpak info --help" -echo "ok info missing NAME" +ok "info missing NAME" for cmd in config make-current override remote-add repair; do ${FLATPAK} $cmd --system --user >out 2>&1 || true @@ -114,4 +114,4 @@ for cmd in config make-current override remote-add repair; do assert_file_has_content out "^error: Multiple installations specified" done -echo "ok ONE_DIR commands" +ok "ONE_DIR commands" diff --git a/tests/test-build-update-repo.sh b/tests/test-build-update-repo.sh index e3c2ab53..f8af29c1 100755 --- a/tests/test-build-update-repo.sh +++ b/tests/test-build-update-repo.sh @@ -38,7 +38,7 @@ ${FLATPAK} build-update-repo --collection-id=org.test.Collection repos/test assert_file_has_content repos/test/config '^collection-id=org\.test\.Collection$' -echo "ok 1 update repo to add collection ID" +ok "1 update repo to add collection ID" # Test that you’re not allowed to change the collection ID once it’s already set. if ${FLATPAK} build-update-repo --collection-id=org.test.Collection2 repos/test 2> build-update-repo-error-log; then @@ -48,22 +48,22 @@ fi assert_file_has_content repos/test/config '^collection-id=org\.test\.Collection$' assert_not_file_has_content repos/test/config '^collection-id=org\.test\.Collection2$' -echo "ok 2 collection ID cannot be changed" +ok "2 collection ID cannot be changed" ${FLATPAK} build-update-repo --title="My little repo" repos/test assert_file_has_content repos/test/config '^title=My little repo$' -echo "ok can update repo title" +ok "can update repo title" ${FLATPAK} build-update-repo --redirect-url=http://no.where/ repos/test assert_file_has_content repos/test/config '^redirect-url=http://no\.where/$' -echo "ok can update redirect url" +ok "can update redirect url" ${FLATPAK} build-update-repo --default-branch=no-such-branch repos/test assert_file_has_content repos/test/config '^default-branch=no-such-branch$' -echo "ok can update default branch" +ok "can update default branch" diff --git a/tests/test-bundle.sh b/tests/test-bundle.sh index d1682344..487b0735 100755 --- a/tests/test-bundle.sh +++ b/tests/test-bundle.sh @@ -35,14 +35,14 @@ assert_has_file bundles/hello.flatpak ${FLATPAK} build-bundle repos/test --runtime --repo-url=file://`pwd`/repos/test --gpg-keys=${FL_GPG_HOMEDIR}/pubring.gpg bundles/platform.flatpak org.test.Platform assert_has_file bundles/platform.flatpak -echo "ok create bundles server-side" +ok "create bundles server-side" rm bundles/hello.flatpak ${FLATPAK} ${U} install -y test-repo org.test.Hello ${FLATPAK} build-bundle $FL_DIR/repo --repo-url=file://`pwd`/repos/test --gpg-keys=${FL_GPG_HOMEDIR}/pubring.gpg bundles/hello.flatpak org.test.Hello assert_has_file bundles/hello.flatpak -echo "ok create bundles client-side" +ok "create bundles client-side" ${FLATPAK} uninstall ${U} -y org.test.Hello ${FLATPAK} install ${U} -y --bundle bundles/hello.flatpak @@ -90,7 +90,7 @@ $FLATPAK remote-list ${U} -d | grep hello-origin > /dev/null $FLATPAK remote-list ${U} -d | grep hello-origin | grep no-enumerate > /dev/null assert_has_file $FL_DIR/repo/hello-origin.trustedkeys.gpg -echo "ok install app bundle" +ok "install app bundle" ${FLATPAK} uninstall -y --force-remove ${U} org.test.Platform @@ -123,12 +123,12 @@ $FLATPAK remote-list ${U} -d | grep platform-origin > /dev/null $FLATPAK remote-list ${U} -d | grep platform-origin | grep no-enumerate > /dev/null assert_has_file $FL_DIR/repo/platform-origin.trustedkeys.gpg -echo "ok install runtime bundle" +ok "install runtime bundle" run org.test.Hello > hello_out assert_file_has_content hello_out '^Hello world, from a sandbox$' -echo "ok run" +ok "run" OLD_COMMIT=`${FLATPAK} ${U} info --show-commit org.test.Hello` @@ -140,7 +140,7 @@ if [ x${USE_SYSTEMDIR-} != xyes ] ; then assert_streq "$OLD_COMMIT" "$ALSO_OLD_COMMIT" fi -echo "ok null update" +ok "null update" make_updated_app @@ -153,7 +153,7 @@ assert_not_streq "$OLD_COMMIT" "$NEW_COMMIT" run org.test.Hello > hello_out assert_file_has_content hello_out '^Hello world, from a sandboxUPDATED$' -echo "ok update" +ok "update" make_updated_app test org.test.Collection.test master UPDATED2 @@ -169,4 +169,4 @@ assert_not_streq "$NEW_COMMIT" "$NEW2_COMMIT" run org.test.Hello > hello_out assert_file_has_content hello_out '^Hello world, from a sandboxUPDATED2$' -echo "ok update as bundle" +ok "update as bundle" diff --git a/tests/test-completion.sh b/tests/test-completion.sh index 483afc66..bf5f0395 100755 --- a/tests/test-completion.sh +++ b/tests/test-completion.sh @@ -36,7 +36,7 @@ ${FLATPAK} complete "flatpak a" 9 "a" | sort > complete_out (diff -u complete_out - || exit 1) < complete_out (diff -u complete_out - || exit 1) < complete_out (diff -u complete_out - || exit 1) < complete_out (diff -u complete_out - || exit 1) < complete_out --version EOF -echo "ok complete global options" +ok "complete global options" ${FLATPAK} complete "flatpak list --" 15 "--" | sort > complete_out (diff -u complete_out - || exit 1) < complete_out --verbose EOF -echo "ok complete list options" +ok "complete list options" ${FLATPAK} complete "flatpak create-usb /" 20 "/" | sort > complete_out #(diff -u complete_out - || echo "fail") < complete_out (diff -u complete_out - || exit 1) < complete_out (diff -u complete_out - || exit 1) < complete_out (diff -u complete_out - || exit 1) < complete_out assert_file_empty complete-out -echo "ok complete list --columns=all" +ok "complete list --columns=all" ${FLATPAK} complete "flatpak list --columns=hel" 27 "--columns=hel" | sort > complete_out (diff -u complete_out - || exit 1) < complete_out (diff -u complete_out - || exit 1) < list_out assert_file_has_content list_out "^languages:" -echo "ok config list" +ok "config list" ${FLATPAK} config --set languages "de;fr" ${FLATPAK} config --get languages > get_out assert_file_has_content get_out "^de;fr" -echo "ok config set" +ok "config set" ${FLATPAK} config --set languages "*" ${FLATPAK} config --get languages > get_out assert_file_has_content get_out "*" -echo "ok config languages *" +ok "config languages *" ${FLATPAK} config --set languages "all" ${FLATPAK} config --get languages > get_out assert_file_has_content get_out "all" -echo "ok config languages *" +ok "config languages *" ${FLATPAK} config --unset languages ${FLATPAK} config --get languages > get_out assert_file_has_content get_out "^[*]unset[*]" -echo "ok config unset" +ok "config unset" diff --git a/tests/test-default-remotes.sh b/tests/test-default-remotes.sh index ee7502d1..fdb1b0a9 100755 --- a/tests/test-default-remotes.sh +++ b/tests/test-default-remotes.sh @@ -65,14 +65,14 @@ assert_remote_has_no_config added-default xa.icon-is-set assert_remote_has_no_config added-default xa.noenumerate assert_remote_has_no_config added-default xa.filter -echo "ok pre-existing installation" +ok "pre-existing installation" rm -rf $FL_DIR ${FLATPAK} --system remotes > remotes assert_file_has_content remotes "added-default" -echo "ok non-existing installation" +ok "non-existing installation" ${FLATPAK} --system remotes > remotes assert_file_has_content remotes "added-default" @@ -83,7 +83,7 @@ ${FLATPAK} --system remote-delete added-default ${FLATPAK} --system remotes > remotes assert_not_file_has_content remotes "added-default" -echo "ok allow remove" +ok "allow remove" rm -rf $FL_DIR rm -rf $FLATPAK_CONFIG_DIR/remotes.d/* @@ -109,7 +109,7 @@ assert_remote_has_no_config added-default xa.comment assert_remote_has_no_config added-default xa.description assert_remote_has_no_config added-default xa.homepage -echo "ok pre-existing remote" +ok "pre-existing remote" rm -rf $FL_DIR rm -rf $FLATPAK_CONFIG_DIR/remotes.d/* @@ -130,4 +130,4 @@ assert_file_has_content remotes "added-default" assert_remote_has_no_config added-default xa.filter -echo "ok override default filter" +ok "override default filter" diff --git a/tests/test-extensions.sh b/tests/test-extensions.sh index 4ae8f9a1..e3326e09 100755 --- a/tests/test-extensions.sh +++ b/tests/test-extensions.sh @@ -150,7 +150,7 @@ run_sh org.test.Hello "ls -lR /usr/foo/multiversion" assert_has_extension_file /usr multiversion/master/extension-org.test.Multiversion.master:master assert_has_extension_file /usr multiversion/notmaster/extension-org.test.Multiversion.notmaster:not-master -echo "ok runtime extensions" +ok "runtime extensions" # Modify app metadata ostree checkout -U --repo=repos/test app/org.test.Hello/${ARCH}/master hello @@ -174,4 +174,4 @@ assert_not_has_extension_file /app dir2/foo/exists assert_has_extension_file /app multiversion/master/extension-org.test.Multiversion.master:master assert_has_extension_file /app multiversion/notmaster/extension-org.test.Multiversion.notmaster:not-master -echo "ok app extensions" +ok "app extensions" diff --git a/tests/test-http-utils.sh b/tests/test-http-utils.sh index 8ff3e169..8c62cf11 100755 --- a/tests/test-http-utils.sh +++ b/tests/test-http-utils.sh @@ -96,7 +96,7 @@ assert_ok "/?expires-past" $test_tmpdir/output assert_ok "/?expires-past" $test_tmpdir/output rm -f $test_tmpdir/output* -echo 'ok http cache lifetimes' +ok 'http cache lifetimes' # Revalation with an etag assert_ok "/?etag&no-cache" $test_tmpdir/output @@ -108,7 +108,7 @@ assert_ok "/?modified-time&no-cache" $test_tmpdir/output assert_304 "/?modified-time&no-cache" $test_tmpdir/output rm -f $test_tmpdir/output* -echo 'ok http revalidation' +ok 'http revalidation' # Test compressd downloading and storage assert_ok --compressed "/compress" $test_tmpdir/output @@ -116,7 +116,7 @@ contents=$(gunzip -c < $test_tmpdir/output) assert_streq $contents path=/compress rm -f $test_tmpdir/output* -echo 'ok compressed download' +ok 'compressed download' # Test uncompressed downloading with compressed storage assert_ok --compressed "/compress?ignore-accept-encoding" $test_tmpdir/output @@ -124,18 +124,18 @@ contents=$(gunzip -c < $test_tmpdir/output) assert_streq $contents path=/compress?ignore-accept-encoding rm -f $test_tmpdir/output* -echo 'ok compress after download' +ok 'compress after download' # Testing that things work without xattr support if have_xattrs $test_tmpdir ; then - echo "ok no-xattrs # skip /tmp doesn't have user xattr support" + ok "no-xattrs # skip /tmp doesn't have user xattr support" else assert_ok "/?etag&no-cache" $test_tmpdir/output assert_has_file $test_tmpdir/output.flatpak.http assert_304 "/?etag&no-cache" $test_tmpdir/output rm -f $test_tmpdir/output* - echo "ok no-xattrs" + ok "no-xattrs" fi # Testing with xattr support @@ -152,7 +152,7 @@ if have_xattrs $xattrs_tempdir ; then assert_not_has_file $xattrs_tempdir/output.flatpak.http assert_304 "/?etag&no-cache" $xattrs_tempdir/output rm -f $xattrs_tempdir/output* - echo "ok xattrs" + ok "xattrs" else - echo "ok xattrs # skip /var/tmp has user no xattr support" + ok "xattrs # skip /var/tmp has user no xattr support" fi diff --git a/tests/test-info.sh b/tests/test-info.sh index af17ecaa..4a247bdb 100644 --- a/tests/test-info.sh +++ b/tests/test-info.sh @@ -17,40 +17,40 @@ ${FLATPAK} info -rcos org.test.Hello > info assert_file_has_content info "^app/org\.test\.Hello/$(flatpak --default-arch)/master test-repo ${COMMIT}" -echo "ok info -rcos" +ok "info -rcos" ${FLATPAK} info --show-permissions org.test.Hello > info assert_file_empty info -echo "ok info --show-permissions" +ok "info --show-permissions" ${FLATPAK} info --show-location org.test.Hello > info assert_file_has_content info "app/org\.test\.Hello/$(flatpak --default-arch)/master/${COMMIT}" -echo "ok info --show-location" +ok "info --show-location" ${FLATPAK} info --show-runtime org.test.Hello > info assert_file_has_content info "^org\.test\.Platform/$(flatpak --default-arch)/master$" -echo "ok info --show-runtime" +ok "info --show-runtime" ${FLATPAK} info --show-sdk org.test.Hello > info assert_file_has_content info "^org\.test\.Platform/$(flatpak --default-arch)/master$" -echo "ok info --show-sdk" +ok "info --show-sdk" ${FLATPAK} info --show-extensions org.test.Hello > info assert_file_has_content info "Extension: runtime/org\.test\.Hello\.Locale/$(flatpak --default-arch)/master$" -echo "ok info --show-extensions" +ok "info --show-extensions" ${FLATPAK} info --file-access=home org.test.Hello > info assert_file_has_content info "^hidden$" -echo "ok info --file-access" +ok "info --file-access" diff --git a/tests/test-oci-registry.sh b/tests/test-oci-registry.sh index b3994d94..f66a0b2b 100755 --- a/tests/test-oci-registry.sh +++ b/tests/test-oci-registry.sh @@ -49,7 +49,7 @@ ${FLATPAK} remote-add ${U} oci-registry "oci+http://127.0.0.1:${port}" images=$(${FLATPAK} remote-ls ${U} --columns=app oci-registry | sort | tr '\n' ' ' | sed 's/ $//') assert_streq "$images" "org.test.Hello org.test.Platform" -echo "ok list remote" +ok "list remote" # Pull appstream data @@ -69,13 +69,13 @@ gunzip -c $appstream > appstream-uncompressed assert_file_has_content appstream-uncompressed 'org\.test\.Hello\.desktop' assert_has_file $icondir/64x64/org.test.Hello.png -echo "ok appstream" +ok "appstream" # Test that 'flatpak search' works ${FLATPAK} search org.test.Hello > search-results assert_file_has_content search-results "Print a greeting" -echo "ok search" +ok "search" # Replace with the app image with detached icons, check that the icons work @@ -88,7 +88,7 @@ assert_has_file $icondir/64x64/org.test.Hello.png new_icon_hash=(md5sum $icondir/64x64/org.test.Hello.png) assert_streq $old_icon_hash $new_icon_hash -echo "ok detached icons" +ok "detached icons" # Try installing from the remote @@ -97,7 +97,7 @@ ${FLATPAK} ${U} install -y oci-registry org.test.Hello run org.test.Hello > hello_out assert_file_has_content hello_out '^Hello world, from a sandbox$' -echo "ok install" +ok "install" make_updated_app oci @@ -116,7 +116,7 @@ assert_not_streq "$OLD_COMMIT" "$NEW_COMMIT" run org.test.Hello > hello_out assert_file_has_content hello_out '^Hello world, from a sandboxUPDATED$' -echo "ok update" +ok "update" # Remove the app from the registry, check that things were removed properly @@ -131,7 +131,7 @@ assert_not_file_has_content $appstream 'org\.test\.Hello\.desktop' assert_not_has_file $icondir/64x64/org.test.Hello.png assert_not_has_file $icondir/64x64 -echo "ok appstream change" +ok "appstream change" # Change the remote to a non-OCI remote, check that we cleaned up @@ -149,7 +149,7 @@ assert_not_has_file $base/oci/oci-registry.index.gz assert_not_has_file $base/oci/oci-registry.summary assert_not_has_dir $base/appstream/oci-registry -echo "ok change remote to non-OCI" +ok "change remote to non-OCI" # Change it back and refetch @@ -168,7 +168,7 @@ assert_not_has_file $base/oci/oci-registry.index.gz assert_not_has_file $base/oci/oci-registry.summary assert_not_has_dir $base/appstream/oci-registry -echo "ok delete remote" +ok "delete remote" # Try installing the platform via a flatpakref file. @@ -188,7 +188,7 @@ assert_file_has_content remotes-list '^platform-origin' assert_has_file $base/oci/platform-origin.index.gz -echo "ok install via flatpakref" +ok "install via flatpakref" # Uninstall, check that the origin remote was pruned, and files were # cleaned up properly @@ -200,7 +200,7 @@ assert_not_file_has_content remotes '^platform-origin' assert_not_has_file $base/oci/platform-origin.index.gz -echo "ok prune origin remote" +ok "prune origin remote" # Install from a (non-OCI) bundle, check that the repo-url is respected @@ -213,7 +213,7 @@ assert_file_has_content remotes-list "^platform-origin.*[ ]oci+http://127\.0\.0 assert_has_file $base/oci/platform-origin.index.gz -echo "ok install via bundle" +ok "install via bundle" # Install an app from a bundle @@ -226,7 +226,7 @@ assert_file_has_content remotes-list "^hello-origin.*[ ]oci+http://127\.0\.0\.1 assert_has_file $base/oci/hello-origin.index.gz -echo "ok app install via bundle" +ok "app install via bundle" # Install an updated app bundle with a different origin @@ -240,4 +240,4 @@ assert_file_has_content remotes-list "^hello-origin.*[ ]http://127\.0\.0\.1:${p assert_not_has_file $base/oci/hello-origin.index.gz -echo "ok change remote origin via bundle" +ok "change remote origin via bundle" diff --git a/tests/test-oci.sh b/tests/test-oci.sh index d8858fa1..50e45042 100755 --- a/tests/test-oci.sh +++ b/tests/test-oci.sh @@ -53,7 +53,7 @@ assert_file_has_content $image "org\.freedesktop\.appstream\.appdata.*P assert_file_has_content $image "org\.freedesktop\.appstream\.icon-64" assert_file_has_content $image org.flatpak.ref.*app/org.test.Hello/x86_64/master -echo "ok export oci" +ok "export oci" ostree --repo=repo2 init --mode=archive-z2 @@ -65,4 +65,4 @@ assert_has_dir checked-out/files assert_has_file checked-out/files/bin/hello.sh assert_has_file checked-out/metadata -echo "ok import oci" +ok "import oci" diff --git a/tests/test-override.sh b/tests/test-override.sh index 93009ee3..31f28bd7 100755 --- a/tests/test-override.sh +++ b/tests/test-override.sh @@ -26,7 +26,7 @@ ${FLATPAK} override --user --show org.test.Hello > override assert_file_has_content override "^\[Context\]$" assert_file_has_content override "^sockets=wayland;!ssh-auth;$" -echo "ok override --socket" +ok "override --socket" reset_overrides @@ -37,7 +37,7 @@ ${FLATPAK} override --user --show org.test.Hello > override assert_file_has_content override "^\[Context\]$" assert_file_has_content override "^devices=dri;!kvm;$" -echo "ok override --device" +ok "override --device" reset_overrides @@ -48,7 +48,7 @@ ${FLATPAK} override --user --show org.test.Hello > override assert_file_has_content override "^\[Context\]$" assert_file_has_content override "^shared=network;!ipc;$" -echo "ok override --share" +ok "override --share" reset_overrides @@ -59,7 +59,7 @@ ${FLATPAK} override --user --show org.test.Hello > override assert_file_has_content override "^\[Context\]$" assert_file_has_content override "^features=multiarch;!bluetooth;$" -echo "ok override --allow" +ok "override --allow" reset_overrides @@ -71,7 +71,7 @@ assert_file_has_content override "^\[Environment\]$" assert_file_has_content override "^FOO=BAR$" assert_file_has_content override "^BAR=$" -echo "ok override --env" +ok "override --env" ${FLATPAK} override --user --filesystem=home org.test.Hello ${FLATPAK} override --user --filesystem=xdg-desktop/foo:create org.test.Hello @@ -87,7 +87,7 @@ assert_file_has_content override "^filesystems=.*xdg-documents;.*$" assert_file_has_content override "^filesystems=.*xdg-desktop/foo:create;.*$" assert_file_has_content override "^filesystems=.*xdg-config:ro;.*$" -echo "ok override --filesystem" +ok "override --filesystem" reset_overrides @@ -102,7 +102,7 @@ assert_file_has_content override "^org\.foo\.Own=own$" assert_file_has_content override "^org\.foo\.Talk=talk$" assert_file_has_content override "^org\.foo\.NoTalk=none$" -echo "ok override session bus names" +ok "override session bus names" reset_overrides @@ -117,7 +117,7 @@ assert_file_has_content override "^org\.foo\.Own\.System=own$" assert_file_has_content override "^org\.foo\.Talk\.System=talk$" assert_file_has_content override "^org\.foo\.NoTalk\.System=none$" -echo "ok override system bus names" +ok "override system bus names" reset_overrides @@ -132,9 +132,9 @@ elif [ -S "${XDG_RUNTIME_DIR}/wayland-0" ]; then ${FLATPAK} run --command=ls org.test.Hello -- /run/user/1000 > out assert_not_file_has_content out "wayland-0" - echo "ok sandbox wayland socket" + ok "sandbox wayland socket" else - echo "ok sandbox wayland socket # skip not supported without Wayland" + ok "sandbox wayland socket # skip not supported without Wayland" fi reset_overrides @@ -150,9 +150,9 @@ elif [ -d "/dev/dri" ]; then ${FLATPAK} run --command=ls org.test.Hello -- /dev > out assert_not_file_has_content out "dri" - echo "ok sandbox dri device" + ok "sandbox dri device" else - echo "ok sandbox dri device # skip not supported without /dev/dri" + ok "sandbox dri device # skip not supported without /dev/dri" fi reset_overrides @@ -165,7 +165,7 @@ if ! skip_one_without_bwrap "sandbox dri device"; then FOO=bar ${FLATPAK} run --command=sh org.test.Hello -c 'echo $FOO' > out assert_file_has_content out "BAR" - echo "ok sandbox env" + ok "sandbox env" fi reset_overrides @@ -183,7 +183,7 @@ if ! skip_one_without_bwrap "sandbox filesystem"; then rm $HOME/example - echo "ok sandbox filesystem" + ok "sandbox filesystem" fi reset_overrides @@ -193,5 +193,5 @@ if ! skip_one_without_bwrap "persist"; then ${FLATPAK} run --command=sh org.test.Hello -c "echo goodbye > $HOME/example/bye" assert_file_has_content $HOME/.var/app/org.test.Hello/example/bye goodbye - echo "ok persist" + ok "persist" fi diff --git a/tests/test-repo.sh b/tests/test-repo.sh index c9192440..f8cce9e2 100644 --- a/tests/test-repo.sh +++ b/tests/test-repo.sh @@ -89,7 +89,7 @@ assert_has_symlink $FL_DIR/appstream/test-repo/$ARCH/active assert_has_file $FL_DIR/appstream/test-repo/$ARCH/active/appstream.xml assert_has_file $FL_DIR/appstream/test-repo/$ARCH/active/appstream.xml.gz -echo "ok update compat appstream" +ok "update compat appstream" # Then regenerate new appstream branch and verify that we update to it update_repo @@ -103,33 +103,33 @@ assert_has_symlink $FL_DIR/appstream/test-repo/$ARCH/active assert_has_file $FL_DIR/appstream/test-repo/$ARCH/active/appstream.xml assert_has_file $FL_DIR/appstream/test-repo/$ARCH/active/appstream.xml.gz -echo "ok update appstream" +ok "update appstream" if [ x${USE_COLLECTIONS_IN_CLIENT-} != xyes ] ; then install_repo test-no-gpg - echo "ok install without gpg key" + ok "install without gpg key" ${FLATPAK} ${U} uninstall -y org.test.Platform org.test.Hello else - echo "ok install without gpg key # skip not supported for collections" + ok "install without gpg key # skip not supported for collections" fi install_repo local-test-no-gpg ${FLATPAK} ${U} uninstall -y org.test.Platform org.test.Hello ${FLATPAK} ${U} update --appstream local-test-no-gpg-repo -echo "ok local without gpg key" +ok "local without gpg key" install_repo test-gpg2 -echo "ok with alternative gpg key" +ok "with alternative gpg key" if ${FLATPAK} ${U} install -y test-repo org.test.Platform 2> install-error-log; then assert_not_reached "Should not be able to install again from different remote without reinstall" fi -echo "ok failed to install again from different remote" +ok "failed to install again from different remote" ${FLATPAK} ${U} install -y --reinstall test-repo org.test.Platform -echo "ok re-install" +ok "re-install" ${FLATPAK} ${U} uninstall -y org.test.Hello @@ -140,7 +140,7 @@ assert_file_has_content install-log "org\.test\.Hello" ${FLATPAK} ${U} list -d > list-log assert_file_has_content list-log "org\.test\.Hello" -echo "ok typo correction works for install" +ok "typo correction works for install" ${FLATPAK} ${U} uninstall -y org.test.Hello @@ -168,7 +168,7 @@ if [ x${USE_COLLECTIONS_IN_CLIENT-} != xyes ] ; then ${FLATPAK} ${U} remote-modify --enable test-no-gpg-repo fi -echo "ok missing remote name auto-corrects for install" +ok "missing remote name auto-corrects for install" port=$(cat httpd-port) if ${FLATPAK} ${U} install -y http://127.0.0.1:${port}/nonexistent.flatpakref 2> install-error-log; then @@ -176,7 +176,7 @@ if ${FLATPAK} ${U} install -y http://127.0.0.1:${port}/nonexistent.flatpakref 2> fi assert_file_has_content install-error-log "Server returned status 404: Not Found" -echo "ok install fails gracefully for 404 URLs" +ok "install fails gracefully for 404 URLs" # Use a new remote so we can be sure it doesn't match any existing one's URL setup_repo_no_add flatpakref org.test.Collection.Flatpakref @@ -214,7 +214,7 @@ if [ $NUM_REMOTES_AFTER -ne $((NUM_REMOTES_BEFORE + 1)) ]; then assert_not_reached "install of flatpakref should only add one remote" fi -echo "ok install flatpakref normalizes remote URL trailing slash" +ok "install flatpakref normalizes remote URL trailing slash" ${FLATPAK} ${U} uninstall -y org.test.Platform org.test.Hello @@ -228,7 +228,7 @@ if ${FLATPAK} ${U} install test-missing-gpg-repo org.test.Hello 2> install-error fi assert_log_has_gpg_signature_error install-error-log -echo "ok fail with missing gpg key" +ok "fail with missing gpg key" if ${FLATPAK} ${U} install test-wrong-gpg-repo org.test.Platform 2> install-error-log; then assert_not_reached "Should not be able to install with wrong gpg key" @@ -240,7 +240,7 @@ if ${FLATPAK} ${U} install test-wrong-gpg-repo org.test.Hello 2> install-error-l fi assert_log_has_gpg_signature_error install-error-log -echo "ok fail with wrong gpg key" +ok "fail with wrong gpg key" make_required_version_app () { APP_ID=${1} @@ -303,7 +303,7 @@ assert_file_has_content install-error-log "needs a later flatpak version" ${FLATPAK} ${U} uninstall -y --all -echo "ok handles version requirements" +ok "handles version requirements" ${FLATPAK} ${U} remotes -d | grep ^test-repo > repo-info assert_not_file_has_content repo-info "new-title" @@ -315,7 +315,7 @@ ${FLATPAK} ${U} install -y test-repo org.test.Platform ${FLATPAK} ${U} remotes -d | grep ^test-repo > repo-info assert_file_has_content repo-info "new-title" -echo "ok update metadata" +ok "update metadata" if [ x${USE_COLLECTIONS_IN_SERVER-} == xyes ] ; then COPY_COLLECTION_ID=org.test.Collection.test @@ -339,7 +339,7 @@ assert_not_file_has_content appdata.xml "org\.test\.Hello\.desktop" ${FLATPAK} repo --branches repos/test-copy > branches-log assert_file_has_content branches-log "^app/org\.test\.Hello/.*eol=Reason1" -echo "ok eol build-commit-from" +ok "eol build-commit-from" ${FLATPAK} ${U} install -y test-repo org.test.Hello @@ -376,7 +376,7 @@ ${FLATPAK} ${U} uninstall -y org.test.Hello # Remove eol for future tests EXPORT_ARGS="" make_updated_app -echo "ok eol build-export" +ok "eol build-export" if [ x${USE_COLLECTIONS_IN_SERVER-} == xyes ] ; then REBASE_COLLECTION_ID=org.test.Collection.rebase @@ -422,7 +422,7 @@ assert_has_file $HOME/.var/app/org.test.Hello/data/another-file ${FLATPAK} ${U} uninstall -y org.test.NewHello org.test.Platform -echo "ok eol-rebase" +ok "eol-rebase" ${FLATPAK} ${U} install -y test-repo org.test.Platform @@ -442,7 +442,7 @@ update_repo test-gpg3 org.test.Collection.test ${FLATPAK} ${U} install -y test-repo org.test.Hello assert_file_has_content $FL_DIR/app/org.test.Hello/$ARCH/master/active/files/bin/hello.sh UPDATED -echo "ok redirect url and gpg key" +ok "redirect url and gpg key" # Test https://github.com/flatpak/flatpak/issues/3222 mkdir -p $FL_DIR/repo/refs/mirrors/org.test.Collection.test/app/org.test.Hello/$ARCH/ @@ -452,13 +452,13 @@ ${FLATPAK} ${U} update -y org.test.Hello assert_not_has_file $FL_DIR/repo/refs/mirrors/org.test.Collection.test/app/org.test.Hello/$ARCH/master assert_has_file $FL_DIR/repo/refs/remotes/test-repo/app/org.test.Hello/$ARCH/master -echo "ok mirror ref deletion on update" +ok "mirror ref deletion on update" ${FLATPAK} ${U} list --arch=$ARCH --columns=ref > list-log assert_file_has_content list-log "org\.test\.Hello" assert_file_has_content list-log "org\.test\.Platform" -echo "ok flatpak list --arch --columns works" +ok "flatpak list --arch --columns works" if ${FLATPAK} ${INVERT_U} uninstall -y org.test.Platform org.test.Hello; then assert_not_reached "Should not be able to uninstall ${INVERT_U} when installed ${U}" @@ -471,7 +471,7 @@ ${FLATPAK} ${U} list -d > list-log assert_not_file_has_content list-log "org\.test\.Hello" assert_not_file_has_content list-log "org\.test\.Platform" -echo "ok uninstall vs installations" +ok "uninstall vs installations" ${FLATPAK} ${U} install -y test-repo org.test.Hello @@ -490,7 +490,7 @@ ${FLATPAK} ${U} list -d > list-log assert_not_file_has_content list-log "org\.test\.Hello" assert_not_file_has_content list-log "org\.test\.Platform" -echo "ok uninstall dependencies" +ok "uninstall dependencies" ${FLATPAK} ${U} install -y test-repo org.test.Hello @@ -498,14 +498,14 @@ ${FLATPAK} ${U} install -y test-repo org.test.Hello ${FLATPAK} ${U} uninstall -y hello ${FLATPAK} ${U} uninstall -y platform -echo "ok typo correction works for uninstall" +ok "typo correction works for uninstall" ${FLATPAK} ${U} install -y test-repo org.test.Hello master ${FLATPAK} ${U} uninstall -y org.test.Hello master ${FLATPAK} ${U} uninstall -y org.test.Platform master -echo "ok install and uninstall support 'NAME BRANCH' syntax" +ok "install and uninstall support 'NAME BRANCH' syntax" ${FLATPAK} ${U} install -y --no-deploy test-repo org.test.Hello @@ -526,7 +526,7 @@ ${FLATPAK} ${U} list -d > list-log assert_file_has_content list-log "org\.test\.Hello" assert_file_has_content list-log "org\.test\.Platform" -echo "ok install with --no-deploy and then --no-pull" +ok "install with --no-deploy and then --no-pull" ${FLATPAK} ${U} uninstall -y org.test.Hello org.test.Platform @@ -550,7 +550,7 @@ ${FLATPAK} ${U} list -d > list-log assert_file_has_content list-log "org\.test\.Hello" assert_file_has_content list-log "org\.test\.Platform" -echo "ok install with --no-deploy and then --no-pull works with typo correction" +ok "install with --no-deploy and then --no-pull works with typo correction" ${FLATPAK} uninstall -y --all @@ -558,7 +558,7 @@ ${FLATPAK} ${U} list -d > list-log assert_not_file_has_content list-log "org\.test\.Hello" assert_not_file_has_content list-log "org\.test\.Platform" -echo "ok uninstall --all" +ok "uninstall --all" ${FLATPAK} ${U} install -y test-repo org.test.Hello @@ -575,7 +575,7 @@ assert_not_file_has_content list-log "org\.test\.Hello\.Locale" setup_repo -echo "ok uninstall with missing remote" +ok "uninstall with missing remote" ${FLATPAK} ${U} list -a --columns=application > list-log assert_file_has_content list-log "org\.test\.Platform" @@ -585,7 +585,7 @@ ${FLATPAK} ${U} uninstall -y --unused ${FLATPAK} ${U} list -a --columns=application > list-log assert_not_file_has_content list-log "org\.test\.Platform" -echo "ok uninstall --unused" +ok "uninstall --unused" # Test that remote-ls works in all of the following cases: # * system remote, and --system is used @@ -619,14 +619,14 @@ else assert_file_has_content remote-ls-error-log "Remote \"test-repo\" not found" fi -echo "ok remote-ls" +ok "remote-ls" # Test that remote-ls can take a file:// URI ostree --repo=repos/test summary -u ${FLATPAK} remote-ls file://`pwd`/repos/test > repo-list assert_file_has_content repo-list "org\.test\.Hello" -echo "ok remote-ls URI" +ok "remote-ls URI" # Test that remote-modify works in all of the following cases: # * system remote, and --system is used @@ -668,7 +668,7 @@ else assert_file_has_content remote-modify-error-log "Remote \"test-repo\" not found" fi -echo "ok remote-modify" +ok "remote-modify" # Test that remote-delete works in all of the following cases: # * system remote, and --system is used @@ -710,7 +710,7 @@ else assert_file_has_content remote-delete-error-log "Remote \"test-repo\" not found" fi -echo "ok remote-delete" +ok "remote-delete" # Test that remote-info works in all of the following cases: # * system remote, and --system is used @@ -744,7 +744,7 @@ else assert_file_has_content remote-info-error-log "Remote \"test-repo\" not found" fi -echo "ok remote-info" +ok "remote-info" ${FLATPAK} ${U} remote-ls -d -a test-repo > remote-ls-log assert_file_has_content remote-ls-log "app/org\.test\.Hello" @@ -803,7 +803,7 @@ assert_file_has_content remote-ls-log "app/org\.test\.Hello" assert_file_has_content remote-ls-log "runtime/org\.test\.Hello\.Locale" assert_file_has_content remote-ls-log "runtime/org\.test\.Platform" -echo "ok filter" +ok "filter" # Try installing it from a flatpakrepo file. Don’t uninstall afterwards because # we need it for the next test. @@ -877,4 +877,4 @@ assert_remote_has_config new-repo xa.filter "${test_srcdir}/test.filter" ${FLATPAK} ${U} remote-add --if-not-exists new-repo test.flatpakrepo assert_remote_has_no_config new-repo xa.filter -echo "ok flatpakrepo" +ok "flatpakrepo" diff --git a/tests/test-run.sh b/tests/test-run.sh index fecb756e..8212e50a 100644 --- a/tests/test-run.sh +++ b/tests/test-run.sh @@ -67,18 +67,18 @@ $FLATPAK info ${U} org.test.Hello > /dev/null $FLATPAK info ${U} org.test.Hello | grep test-repo > /dev/null $FLATPAK info ${U} org.test.Hello | grep $ID > /dev/null -echo "ok install" +ok "install" run org.test.Hello > hello_out assert_file_has_content hello_out '^Hello world, from a sandbox$' -echo "ok hello" +ok "hello" run_sh org.test.Platform cat /.flatpak-info >runtime-fpi assert_file_has_content runtime-fpi "[Runtime]" assert_file_has_content runtime-fpi "^runtime=runtime/org\.test\.Platform/$ARCH/stable$" -echo "ok run a runtime" +ok "run a runtime" if run org.test.Nonexistent 2> run-error-log; then assert_not_reached "Unexpectedly able to run non-existent runtime" @@ -95,12 +95,12 @@ if run runtime/org.test.Nonexistent 2> run-error-log; then fi assert_file_has_content run-error-log "error: runtime/org\.test\.Nonexistent/\*unspecified\*/\*unspecified\* not installed" -echo "ok error handling for invalid refs" +ok "error handling for invalid refs" run_sh org.test.Hello cat /run/user/`id -u`/flatpak-info > fpi assert_file_has_content fpi '^name=org\.test\.Hello$' -echo "ok flatpak-info" +ok "flatpak-info" run_sh org.test.Hello readlink /proc/self/ns/net > unshared_net_ns ARGS="--share=network" run_sh org.test.Hello readlink /proc/self/ns/net > shared_net_ns @@ -165,7 +165,7 @@ else echo "not testing \$HOME binding, \$HOME/.flatpak-tests/ does not exist" >&2 fi -echo "ok namespaces" +ok "namespaces" test_overrides () { local dir="$1" @@ -207,7 +207,7 @@ else echo "not testing \$HOME binding overrides, \$HOME/.flatpak-tests/ does not exist" >&2 fi -echo "ok overrides" +ok "overrides" OLD_COMMIT=`${FLATPAK} ${U} info --show-commit org.test.Hello` @@ -218,7 +218,7 @@ if [ x${USE_SYSTEMDIR-} != xyes ] ; then assert_streq "$OLD_COMMIT" "$ALSO_OLD_COMMIT" fi -echo "ok null update" +ok "null update" make_updated_app "" "" stable @@ -231,7 +231,7 @@ assert_not_streq "$OLD_COMMIT" "$NEW_COMMIT" run org.test.Hello > hello_out assert_file_has_content hello_out '^Hello world, from a sandboxUPDATED$' -echo "ok update" +ok "update" ostree --repo=repos/test reset app/org.test.Hello/$ARCH/stable "$OLD_COMMIT" update_repo @@ -244,7 +244,7 @@ NEW_NEW_COMMIT=`${FLATPAK} ${U} info --show-commit org.test.Hello` assert_streq "$NEW_COMMIT" "$NEW_NEW_COMMIT" -echo "ok backwards update" +ok "backwards update" make_updated_app "" "" stable UPDATED2 @@ -265,7 +265,7 @@ NEW_COMMIT=`${FLATPAK} ${U} info --show-commit org.test.Hello` assert_not_streq "$OLD_COMMIT" "$NEW_COMMIT" -echo "ok install --or-update" +ok "install --or-update" DIR=`mktemp -d` ${FLATPAK} build-init ${DIR} org.test.Split org.test.Platform org.test.Platform stable @@ -348,7 +348,7 @@ assert_has_file $FL_DIR/app/org.test.Split/$ARCH/stable/active/files/e/data assert_not_has_file $FL_DIR/app/org.test.Split/$ARCH/stable/active/files/f assert_not_has_file $FL_DIR/app/org.test.Split/$ARCH/stable/active/files/nope -echo "ok subpaths" +ok "subpaths" VERSION=`cat "$test_builddir/package_version.txt"` @@ -408,7 +408,7 @@ assert_file_has_content err_version.txt "needs a later flatpak version" assert_not_has_file $FL_DIR/app/org.test.CurrentVersion/$ARCH/stable/active/files/updated assert_has_file $FL_DIR/app/org.test.OldVersion/$ARCH/stable/active/files/updated -echo "ok version checks" +ok "version checks" rm -rf app flatpak build-init app org.test.Writable org.test.Platform org.test.Platform stable @@ -426,7 +426,7 @@ if ${FLATPAK} ${U} install -y test-repo org.test.Writable; then assert_file_has_mode $FL_DIR/app/org.test.Writable/$ARCH/stable/active/files/a-dir 775 fi -echo "ok no world writable dir" +ok "no world writable dir" rm -rf app flatpak build-init app org.test.Setuid org.test.Platform org.test.Platform stable @@ -443,7 +443,7 @@ if ${FLATPAK} ${U} install -y test-repo org.test.Setuid &> err2.txt; then fi assert_file_has_content err2.txt [Ii]nvalid -echo "ok no setuid" +ok "no setuid" rm -rf app flatpak build-init app org.test.App org.test.Platform org.test.Platform stable @@ -458,4 +458,4 @@ ${FLATPAK} ${U} info -m org.test.App > out assert_file_has_content out "^sdk=org\.test\.Sdk/$(flatpak --default-arch)/stable$" -echo "ok --sdk option" +ok "--sdk option" diff --git a/tests/test-unsigned-summaries.sh b/tests/test-unsigned-summaries.sh index 0701473d..63daa12d 100755 --- a/tests/test-unsigned-summaries.sh +++ b/tests/test-unsigned-summaries.sh @@ -43,7 +43,7 @@ assert_file_has_content ${FL_DIR}/repo/config '^collection-id=org\.test\.Collect assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify=true$' assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify=false$' -echo "ok 1 repo config with collections" +ok "1 repo config with collections" # Test that building an app with a collection ID set produces the right # metadata in the resulting repository. @@ -74,13 +74,13 @@ assert_file_has_content metadata "'xa\.cache': " assert_file_has_content metadata "'ostree\.collection-binding': <'org\.test\.Collection'>" assert_file_has_content metadata "'ostree\.ref-binding': <\['ostree-metadata'\]>" -echo "ok 2 create app with collections" +ok "2 create app with collections" # Try installing the app. ${FLATPAK} ${U} install -y test-repo org.test.App master ${FLATPAK} ${U} uninstall -y org.test.App -echo "ok 3 install app with collections" +ok "3 install app with collections" # Regenerate the summary so it doesn’t contain xa.cache and is unsigned; try installing again. ostree --repo=repos/test summary --update @@ -92,7 +92,7 @@ assert_not_file_has_content summary '^xa\.cache: ' ${FLATPAK} ${U} install -y test-repo org.test.App master ${FLATPAK} ${U} uninstall -y org.test.App -echo "ok 4 install app with collections from unsigned summary" +ok "4 install app with collections from unsigned summary" # Try installing it from a flatpakref file. Don’t uninstall afterwards because # we need it for the next test. @@ -111,7 +111,7 @@ EOF ${FLATPAK} ${U} install -y --from ./org.test.App.flatpakref ${FLATPAK} ${U} uninstall -y org.test.App -echo "ok 5 install app with collections from flatpakref" +ok "5 install app with collections from flatpakref" # Update the repo metadata and check that it changes in the ostree-metadata branch # and the summary file. @@ -126,11 +126,11 @@ assert_file_has_content metadata "'xa\.title': " assert_file_has_content metadata "'ostree\.collection-binding': <'org\.test\.Collection'>" assert_file_has_content metadata "'ostree\.ref-binding': <\['ostree-metadata'\]>" -echo "ok 6 update repo metadata" +ok "6 update repo metadata" # Try to install the app again, which should pull the updated repository # metadata as a side effect. ${FLATPAK} ${U} install -y test-repo org.test.App master assert_file_has_content ${FL_DIR}/repo/config '^xa\.title=New title$' -echo "ok 7 pull updated repo metadata" +ok "7 pull updated repo metadata" diff --git a/tests/test-update-portal.sh b/tests/test-update-portal.sh index ffe162ea..7870b4fd 100755 --- a/tests/test-update-portal.sh +++ b/tests/test-update-portal.sh @@ -66,15 +66,15 @@ done # Make sure monitor is dead kill -9 $MONITOR_PID -echo "ok monitor updates" +ok "monitor updates" run_with_sandboxed_bus ${test_builddir}/test-update-portal update monitor.pid -echo "ok update self" +ok "update self" run_with_sandboxed_bus ${test_builddir}/test-update-portal update-null monitor.pid -echo "ok null-update self" +ok "null-update self" make_updated_app test "" master UPDATE3 @@ -88,7 +88,7 @@ run_with_sandboxed_bus ${test_builddir}/test-update-portal update-fail monitor.p rm -rf repos/test/objects mv repos/test/orig-objects repos/test/objects -echo "ok update fail" +ok "update fail" ${FLATPAK} ${U} mask "org.test.Hello*" @@ -108,7 +108,7 @@ run_with_sandboxed_bus ${test_builddir}/test-update-portal update-null monitor.p ${FLATPAK} ${U} mask --remove "org.test.Hello*" -echo "ok update vs masked" +ok "update vs masked" BUILD_FINISH_ARGS="--filesystem=host" make_updated_app test "" master UPDATE41 run_with_sandboxed_bus ${test_builddir}/test-update-portal update-notsupp monitor.pid @@ -125,4 +125,4 @@ run_with_sandboxed_bus ${test_builddir}/test-update-portal update-notsupp monito make_updated_app test "" master UPDATE45 run_with_sandboxed_bus ${test_builddir}/test-update-portal update monitor.pid -echo "ok update with changed permissions" +ok "update with changed permissions" diff --git a/tests/test-update-remote-configuration.sh b/tests/test-update-remote-configuration.sh index 3262db5f..9396d19d 100755 --- a/tests/test-update-remote-configuration.sh +++ b/tests/test-update-remote-configuration.sh @@ -64,7 +64,7 @@ assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify=true$' assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify=false$' assert_not_file_has_content ${FL_DIR}/repo/config '^collection-id=' -echo "ok 1 update repo config without deploying collection ID" +ok "1 update repo config without deploying collection ID" # Now mark the collection ID as to be deployed. The client configuration should # be updated. @@ -77,7 +77,7 @@ assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify=true$' assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify=false$' assert_file_has_content ${FL_DIR}/repo/config '^collection-id=org\.test\.Collection$' -echo "ok 2 update repo config to deploy collection ID" +ok "2 update repo config to deploy collection ID" # Try updating the collection ID to some other non-empty value on the server. # The client should ignore the update (otherwise we have a security vulnerability). @@ -93,4 +93,4 @@ ${FLATPAK} ${U} update org.test.App master assert_file_has_content ${FL_DIR}/repo/config '^collection-id=org\.test\.Collection$' assert_not_file_has_content ${FL_DIR}/repo/config '^collection-id=net\.malicious\.NewCollection$' -echo "ok 3 update repo config with different collection ID" +ok "3 update repo config with different collection ID" From 241c5c7f40ee56d5b87dfd3559f3fb4cf3654cc6 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 20 Mar 2020 09:34:13 +0100 Subject: [PATCH 02/36] tests: Hide unnecessary trace output from the test logs We use some BASH_XTRACEFD hackery to hide the trace output from the internals of the assert (and ok) functions, so that the log output can focus on what is important, i.e. what asserts are checked and what errors do they print. --- tests/libtest.sh | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index ce1f41fe..80085ce2 100644 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -30,13 +30,23 @@ else test_builddir=$(dirname $0) fi +# All the asserts and ok functions below are wrapped such that they +# don't output any set -x traces of their internals (but still echo +# errors to stderr). This way the log output focuses on tracing what +# is essential to the test (the asserts being run and errors from them) + assert_not_reached () { + { { local BASH_XTRACEFD=3; } 2> /dev/null echo $@ 1>&2; exit 1 + } 3> /dev/null } ok () { - echo "ok $@" - echo "============================================================" + # Wrap this to avoid set -x showing the echo commands + { + echo "ok $@"; + echo "========================================================================"; + } 2> /dev/null } test_tmpdir=$(pwd) @@ -114,64 +124,85 @@ fi export FLATPAK="${CMD_PREFIX} flatpak" assert_streq () { + { { local BASH_XTRACEFD=3; } 2> /dev/null test "$1" = "$2" || (echo 1>&2 "$1 != $2"; exit 1) + } 3> /dev/null } assert_not_streq () { + { { local BASH_XTRACEFD=3; } 2> /dev/null (! test "$1" = "$2") || (echo 1>&2 "$1 == $2"; exit 1) + } 3> /dev/null } assert_has_file () { + { { local BASH_XTRACEFD=3; } 2> /dev/null test -f "$1" || (echo 1>&2 "Couldn't find '$1'"; exit 1) + } 3> /dev/null } assert_has_symlink () { + { { local BASH_XTRACEFD=3; } 2> /dev/null test -L "$1" || (echo 1>&2 "Couldn't find '$1'"; exit 1) + } 3> /dev/null } assert_has_dir () { + { { local BASH_XTRACEFD=3; } 2> /dev/null test -d "$1" || (echo 1>&2 "Couldn't find '$1'"; exit 1) + } 3> /dev/null } assert_not_has_file () { + { { local BASH_XTRACEFD=3; } 2> /dev/null if test -f "$1"; then sed -e 's/^/# /' < "$1" >&2 echo 1>&2 "File '$1' exists" exit 1 fi + } 3> /dev/null } assert_not_file_has_content () { + { { local BASH_XTRACEFD=3; } 2> /dev/null if grep -q -e "$2" "$1"; then sed -e 's/^/# /' < "$1" >&2 echo 1>&2 "File '$1' incorrectly matches regexp '$2'" exit 1 fi + } 3> /dev/null } assert_file_has_mode () { + { { local BASH_XTRACEFD=3; } 2> /dev/null mode=$(stat -c '%a' $1) if [ "$mode" != "$2" ]; then echo 1>&2 "File '$1' has wrong mode: expected $2, but got $mode" exit 1 fi + } 3> /dev/null } assert_not_has_dir () { + { { local BASH_XTRACEFD=3; } 2> /dev/null if test -d "$1"; then echo 1>&2 "Directory '$1' exists"; exit 1 fi + } 3> /dev/null } assert_file_has_content () { + { { local BASH_XTRACEFD=3; } 2> /dev/null if ! grep -q -e "$2" "$1"; then sed -e 's/^/# /' < "$1" >&2 echo 1>&2 "File '$1' doesn't match regexp '$2'" exit 1 fi + } 3> /dev/null } assert_log_has_gpg_signature_error () { + { { local BASH_XTRACEFD=3; } 2> /dev/null if ! grep -q -e "GPG signatures found, but none are in trusted keyring" "$1"; then if ! grep -q -e "Can't check signature: public key not found" "$1"; then sed -e 's/^/# /' < "$1" >&2 @@ -179,34 +210,43 @@ assert_log_has_gpg_signature_error () { exit 1 fi fi + } 3> /dev/null } assert_symlink_has_content () { + { { local BASH_XTRACEFD=3; } 2> /dev/null if ! readlink "$1" | grep -q -e "$2"; then readlink "$1" |sed -e 's/^/# /' >&2 echo 1>&2 "Symlink '$1' doesn't match regexp '$2'" exit 1 fi + } 3> /dev/null } assert_file_empty() { + { { local BASH_XTRACEFD=3; } 2> /dev/null if test -s "$1"; then sed -e 's/^/# /' < "$1" >&2 echo 1>&2 "File '$1' is not empty" exit 1 fi + } 3> /dev/null } assert_remote_has_config () { + { { local BASH_XTRACEFD=3; } 2> /dev/null ostree config --repo=$FL_DIR/repo get --group 'remote "'"$1"'"' "$2" > key-output assert_file_has_content key-output "$3" + } 3> /dev/null } assert_remote_has_no_config () { + { { local BASH_XTRACEFD=3; } 2> /dev/null if ostree config --repo=$FL_DIR/repo get --group 'remote "'"$1"'"' "$2" > /dev/null; then echo 1>&2 "Remote '$1' unexpectedly has key '$2'" exit 1 fi + } 3> /dev/null } export FL_GPG_HOMEDIR=${TEST_DATA_DIR}/gpghome From ee93324333783ea882acb5c82ca61f8e88640f38 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 20 Mar 2020 11:44:50 +0100 Subject: [PATCH 03/36] tests: Avoid showing stderr for commands expected to fail --- tests/test-repo.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-repo.sh b/tests/test-repo.sh index f8cce9e2..bf5ef2e4 100644 --- a/tests/test-repo.sh +++ b/tests/test-repo.sh @@ -767,11 +767,11 @@ assert_not_file_has_content remote-ls-log "app/org\.test\.Hello" assert_not_file_has_content remote-ls-log "runtime/org\.test\.Hello\.Locale" assert_file_has_content remote-ls-log "runtime/org\.test\.Platform" -if ${FLATPAK} ${U} remote-info test-repo org.test.Hello > remote-ref-info; then +if ${FLATPAK} ${U} remote-info test-repo org.test.Hello > remote-ref-info 2> /dev/null; then assert_not_reached "flatpak remote-info test-repo org.test.Hello should fail due to filter" fi -if ${FLATPAK} ${U} install -y test-repo org.test.Hello; then +if ${FLATPAK} ${U} install -y test-repo org.test.Hello 2> /dev/null; then assert_not_reached "should not be able to install org.test.Hello should fail due to filter" fi From 224b60016f770a738ede372c285bf211bf61eba7 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 23 Mar 2020 12:20:31 +0100 Subject: [PATCH 04/36] tests: Print calling file+lineno when assert fails --- tests/libtest.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 80085ce2..1706dec9 100644 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -45,7 +45,7 @@ ok () { # Wrap this to avoid set -x showing the echo commands { echo "ok $@"; - echo "========================================================================"; + echo "================ $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]} ================"; } 2> /dev/null } @@ -125,31 +125,31 @@ export FLATPAK="${CMD_PREFIX} flatpak" assert_streq () { { { local BASH_XTRACEFD=3; } 2> /dev/null - test "$1" = "$2" || (echo 1>&2 "$1 != $2"; exit 1) + test "$1" = "$2" || (echo 1>&2 "$1 != $2 at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}"; exit 1) } 3> /dev/null } assert_not_streq () { { { local BASH_XTRACEFD=3; } 2> /dev/null - (! test "$1" = "$2") || (echo 1>&2 "$1 == $2"; exit 1) + (! test "$1" = "$2") || (echo 1>&2 "$1 == $2 at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}"; exit 1) } 3> /dev/null } assert_has_file () { { { local BASH_XTRACEFD=3; } 2> /dev/null - test -f "$1" || (echo 1>&2 "Couldn't find '$1'"; exit 1) + test -f "$1" || (echo 1>&2 "Couldn't find '$1' at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}"; exit 1) } 3> /dev/null } assert_has_symlink () { { { local BASH_XTRACEFD=3; } 2> /dev/null - test -L "$1" || (echo 1>&2 "Couldn't find '$1'"; exit 1) + test -L "$1" || (echo 1>&2 "Couldn't find '$1' at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}"; exit 1) } 3> /dev/null } assert_has_dir () { { { local BASH_XTRACEFD=3; } 2> /dev/null - test -d "$1" || (echo 1>&2 "Couldn't find '$1'"; exit 1) + test -d "$1" || (echo 1>&2 "Couldn't find '$1' at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}"; exit 1) } 3> /dev/null } @@ -157,7 +157,7 @@ assert_not_has_file () { { { local BASH_XTRACEFD=3; } 2> /dev/null if test -f "$1"; then sed -e 's/^/# /' < "$1" >&2 - echo 1>&2 "File '$1' exists" + echo 1>&2 "File '$1' exists at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}" exit 1 fi } 3> /dev/null @@ -167,7 +167,7 @@ assert_not_file_has_content () { { { local BASH_XTRACEFD=3; } 2> /dev/null if grep -q -e "$2" "$1"; then sed -e 's/^/# /' < "$1" >&2 - echo 1>&2 "File '$1' incorrectly matches regexp '$2'" + echo 1>&2 "File '$1' incorrectly matches regexp '$2' at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}" exit 1 fi } 3> /dev/null @@ -177,7 +177,7 @@ assert_file_has_mode () { { { local BASH_XTRACEFD=3; } 2> /dev/null mode=$(stat -c '%a' $1) if [ "$mode" != "$2" ]; then - echo 1>&2 "File '$1' has wrong mode: expected $2, but got $mode" + echo 1>&2 "File '$1' has wrong mode: expected $2, but got $mode at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}" exit 1 fi } 3> /dev/null @@ -186,7 +186,7 @@ assert_file_has_mode () { assert_not_has_dir () { { { local BASH_XTRACEFD=3; } 2> /dev/null if test -d "$1"; then - echo 1>&2 "Directory '$1' exists"; exit 1 + echo 1>&2 "Directory '$1' exists at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}"; exit 1 fi } 3> /dev/null } @@ -195,7 +195,7 @@ assert_file_has_content () { { { local BASH_XTRACEFD=3; } 2> /dev/null if ! grep -q -e "$2" "$1"; then sed -e 's/^/# /' < "$1" >&2 - echo 1>&2 "File '$1' doesn't match regexp '$2'" + echo 1>&2 "File '$1' doesn't match regexp '$2' at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}" exit 1 fi } 3> /dev/null @@ -206,7 +206,7 @@ assert_log_has_gpg_signature_error () { if ! grep -q -e "GPG signatures found, but none are in trusted keyring" "$1"; then if ! grep -q -e "Can't check signature: public key not found" "$1"; then sed -e 's/^/# /' < "$1" >&2 - echo 1>&2 "File '$1' doesn't have gpg signature error" + echo 1>&2 "File '$1' doesn't have gpg signature error at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}" exit 1 fi fi @@ -217,7 +217,7 @@ assert_symlink_has_content () { { { local BASH_XTRACEFD=3; } 2> /dev/null if ! readlink "$1" | grep -q -e "$2"; then readlink "$1" |sed -e 's/^/# /' >&2 - echo 1>&2 "Symlink '$1' doesn't match regexp '$2'" + echo 1>&2 "Symlink '$1' doesn't match regexp '$2' at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}" exit 1 fi } 3> /dev/null @@ -227,7 +227,7 @@ assert_file_empty() { { { local BASH_XTRACEFD=3; } 2> /dev/null if test -s "$1"; then sed -e 's/^/# /' < "$1" >&2 - echo 1>&2 "File '$1' is not empty" + echo 1>&2 "File '$1' is not empty at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}" exit 1 fi } 3> /dev/null @@ -243,7 +243,7 @@ assert_remote_has_config () { assert_remote_has_no_config () { { { local BASH_XTRACEFD=3; } 2> /dev/null if ostree config --repo=$FL_DIR/repo get --group 'remote "'"$1"'"' "$2" > /dev/null; then - echo 1>&2 "Remote '$1' unexpectedly has key '$2'" + echo 1>&2 "Remote '$1' unexpectedly has key '$2' at $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[0]}" exit 1 fi } 3> /dev/null From 83ef8015373f95ffec86affa896d941b0333830d Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 19 Mar 2020 12:38:05 +0100 Subject: [PATCH 05/36] dir: Add flatpak_deploy_data_has_subpaths() helper --- common/flatpak-dir-private.h | 1 + common/flatpak-dir.c | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index c21c332f..35aaa886 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -366,6 +366,7 @@ const char * flatpak_deploy_data_get_commit (GBytes *deploy_data); const char * flatpak_deploy_data_get_appdata_content_rating_type (GBytes *deploy_data); GHashTable * flatpak_deploy_data_get_appdata_content_rating (GBytes *deploy_data); const char ** flatpak_deploy_data_get_subpaths (GBytes *deploy_data); +gboolean flatpak_deploy_data_has_subpaths (GBytes *deploy_data); guint64 flatpak_deploy_data_get_installed_size (GBytes *deploy_data); const char * flatpak_deploy_data_get_alt_id (GBytes *deploy_data); const char * flatpak_deploy_data_get_eol (GBytes *deploy_data); diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index 954b2930..a8b4ffa0 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -2471,6 +2471,15 @@ flatpak_deploy_data_get_subpaths (GBytes *deploy_data) return var_arrayofstring_to_strv (var_deploy_data_get_subpaths (ref), NULL); } +gboolean +flatpak_deploy_data_has_subpaths (GBytes *deploy_data) +{ + VarDeployDataRef ref = var_deploy_data_from_bytes (deploy_data); + VarArrayofstringRef subpaths = var_deploy_data_get_subpaths (ref); + + return var_arrayofstring_get_length (subpaths) != 0; +} + guint64 flatpak_deploy_data_get_installed_size (GBytes *deploy_data) { From 32194f2d29490d809323054b0d87988fff07d764 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 19 Mar 2020 12:38:24 +0100 Subject: [PATCH 06/36] create-usb: Detect the case of partial deploys and warn/fail We don't want to make a usb repo with e.g. just one translation. --- app/flatpak-builtins-create-usb.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/app/flatpak-builtins-create-usb.c b/app/flatpak-builtins-create-usb.c index 5693060e..23e4af07 100644 --- a/app/flatpak-builtins-create-usb.c +++ b/app/flatpak-builtins-create-usb.c @@ -120,6 +120,9 @@ add_related (GHashTable *all_refs, if (deploy_data == NULL) return FALSE; + if (flatpak_deploy_data_has_subpaths (deploy_data)) + return flatpak_fail (error, _("Related ref '%s' is only partially installed"), ref); + commit = flatpak_deploy_data_get_commit (deploy_data); deploy = flatpak_dir_load_deployed (dir, ref, commit, cancellable, error); @@ -154,6 +157,13 @@ add_related (GHashTable *all_refs, continue; } + if (flatpak_deploy_data_has_subpaths (ext_deploy_data)) + { + g_printerr (_("Warning: Omitting related ref ‘%s’ because it is partially installed.\n"), + ext->ref); + continue; + } + ext_remote = flatpak_deploy_data_get_origin (ext_deploy_data); if (ext_remote == NULL) return FALSE; @@ -628,6 +638,11 @@ flatpak_builtin_create_usb (int argc, char **argv, GCancellable *cancellable, GE deploy_data = flatpak_dir_get_deploy_data (dir, installed_ref, FLATPAK_DEPLOY_VERSION_ANY, cancellable, error); if (deploy_data == NULL) return FALSE; + + if (flatpak_deploy_data_has_subpaths (deploy_data)) + return flatpak_fail (error, + _("Ref '%s' is only partially installed"), installed_ref); + commit = flatpak_deploy_data_get_commit (deploy_data); c_s = commit_and_subpaths_new (commit, NULL); From 116a8b848788c582fb2c8577f1250438b100da25 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 17 Mar 2020 14:13:14 +0100 Subject: [PATCH 07/36] transaction: Ensure the metadata in the pulled commit matches what we resolved We're using the metadata from the summary, ostree-metadata or available commit when making security sensitive decisions, so lets verify this matches what we get in the actual commit we pulled. We already did check that this then actually also matches what gets deployed, so the new check shares code with that. Note, we don't do this for OCI installs, because it seems the current fedora flatpaks don't have this set, and we don't want to break existing remotes. --- common/flatpak-dir-private.h | 3 ++ common/flatpak-dir.c | 76 +++++++++++++++++++-------- common/flatpak-installation.c | 4 +- common/flatpak-transaction.c | 2 + system-helper/flatpak-system-helper.c | 6 +-- tests/test-extensions.sh | 4 +- tests/test-run.sh | 6 ++- 7 files changed, 71 insertions(+), 30 deletions(-) diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index 35aaa886..4a22b387 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -563,6 +563,7 @@ gboolean flatpak_dir_pull (FlatpakDir *self, const char *opt_rev, const OstreeRepoFinderResult * const *results, const char **subpaths, + GBytes *require_metadata, const char *token, OstreeRepo *repo, FlatpakPullFlags flatpak_flags, @@ -668,6 +669,7 @@ gboolean flatpak_dir_install (FlatpakDir *self, const char *opt_commit, const char **subpaths, const char **previous_ids, + GBytes *require_metadata, const char *token, OstreeAsyncProgress *progress, GCancellable *cancellable, @@ -714,6 +716,7 @@ gboolean flatpak_dir_update (FlatpakDir *self, const OstreeRepoFinderResult * const *results, const char **opt_subpaths, const char **opt_previous_ids, + GBytes *require_metadata, const char *token, OstreeAsyncProgress *progress, GCancellable *cancellable, diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index a8b4ffa0..52138e79 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -1072,6 +1072,32 @@ flatpak_get_user_base_dir_location (void) return g_object_ref ((GFile *) file); } +static gboolean +validate_commit_metadata (GVariant *commit_data, + const char *ref, + const char *required_metadata, + gboolean require_xa_metadata, + GError **error) +{ + g_autoptr(GVariant) commit_metadata = NULL; + const char *xa_metadata = NULL; + + commit_metadata = g_variant_get_child_value (commit_data, 0); + + if (commit_metadata != NULL) + g_variant_lookup (commit_metadata, "xa.metadata", "&s", &xa_metadata); + + if ((xa_metadata == NULL && require_xa_metadata) || + (xa_metadata != NULL && g_strcmp0 (required_metadata, xa_metadata) != 0)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_PERMISSION_DENIED, + _("Commit metadata for %s not matching expected metadata"), ref); + return FALSE; + } + + return TRUE; +} + /* This is a cache directory similar to ~/.cache/flatpak/system-cache, * but in /var/tmp. This is useful for things like the system child * repos, because it is more likely to be on the same filesystem as @@ -4513,12 +4539,12 @@ flatpak_dir_update_appstream (FlatpakDir *self, /* No need to use an existing OstreeRepoFinderResult array, since * appstream updates do not need to be atomic wrt other updates. */ used_branch = new_branch; - if (!flatpak_dir_pull (self, state, used_branch, NULL, NULL, NULL, NULL, + if (!flatpak_dir_pull (self, state, used_branch, NULL, NULL, NULL, NULL, NULL, child_repo, FLATPAK_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_MIRROR, progress, cancellable, &first_error)) { used_branch = old_branch; - if (!flatpak_dir_pull (self, state, used_branch, NULL, NULL, NULL, NULL, + if (!flatpak_dir_pull (self, state, used_branch, NULL, NULL, NULL, NULL, NULL, child_repo, FLATPAK_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_MIRROR, progress, cancellable, &second_error)) { @@ -4570,12 +4596,12 @@ flatpak_dir_update_appstream (FlatpakDir *self, /* No need to use an existing OstreeRepoFinderResult array, since * appstream updates do not need to be atomic wrt other updates. */ used_branch = new_branch; - if (!flatpak_dir_pull (self, state, used_branch, NULL, NULL, NULL, NULL, NULL, + if (!flatpak_dir_pull (self, state, used_branch, NULL, NULL, NULL, NULL, NULL, NULL, FLATPAK_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_NONE, progress, cancellable, &first_error)) { used_branch = old_branch; - if (!flatpak_dir_pull (self, state, used_branch, NULL, NULL, NULL, NULL, NULL, + if (!flatpak_dir_pull (self, state, used_branch, NULL, NULL, NULL, NULL, NULL, NULL, FLATPAK_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_NONE, progress, cancellable, &second_error)) { @@ -5462,6 +5488,7 @@ flatpak_dir_pull (FlatpakDir *self, const char *opt_rev, const OstreeRepoFinderResult * const *opt_results, const char **subpaths, + GBytes *require_metadata, const char *token, OstreeRepo *repo, FlatpakPullFlags flatpak_flags, @@ -5650,6 +5677,15 @@ flatpak_dir_pull (FlatpakDir *self, goto out; } + + if (require_metadata) + { + g_autoptr(GVariant) commit_data = NULL; + if (!ostree_repo_load_commit (repo, rev, &commit_data, NULL, error) || + !validate_commit_metadata (commit_data, ref, (const char *)g_bytes_get_data (require_metadata, NULL), TRUE, error)) + return FALSE; + } + if (!flatpak_dir_pull_extra_data (self, repo, state->remote_name, ref, rev, @@ -7956,7 +7992,6 @@ flatpak_dir_deploy (FlatpakDir *self, glnx_autofd int checkoutdir_dfd = -1; g_autoptr(GFile) tmp_dir_template = NULL; g_autofree char *tmp_dir_path = NULL; - const char *xa_metadata = NULL; const char *xa_ref = NULL; g_autofree char *checkout_basename = NULL; gboolean created_extra_data = FALSE; @@ -7966,6 +8001,7 @@ flatpak_dir_deploy (FlatpakDir *self, g_autofree char *metadata_contents = NULL; g_auto(GStrv) ref_parts = NULL; gboolean is_app; + gboolean is_oci; if (!flatpak_dir_ensure_repo (self, cancellable, error)) return FALSE; @@ -8221,18 +8257,14 @@ flatpak_dir_deploy (FlatpakDir *self, } /* Check the metadata in the commit to make sure it matches the actual - deployed metadata, in case we relied on the one in the commit for - a decision */ - g_variant_lookup (commit_metadata, "xa.metadata", "&s", &xa_metadata); - if (xa_metadata != NULL) - { - if (g_strcmp0 (metadata_contents, xa_metadata) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_PERMISSION_DENIED, - _("Deployed metadata does not match commit")); - return FALSE; - } - } + * deployed metadata, in case we relied on the one in the commit for + * a decision + * Note: For historical reason we don't enforce commits to contain xa.metadata + * since this was lacking in fedora builds. + */ + is_oci = flatpak_dir_get_remote_oci (self, origin); + if (!validate_commit_metadata (commit_data, ref, metadata_contents, !is_oci, error)) + return FALSE; dotref = g_file_resolve_relative_path (checkoutdir, "files/.ref"); if (!g_file_replace_contents (dotref, "", 0, NULL, FALSE, @@ -8887,6 +8919,7 @@ flatpak_dir_install (FlatpakDir *self, const char *opt_commit, const char **opt_subpaths, const char **opt_previous_ids, + GBytes *require_metadata, const char *token, OstreeAsyncProgress *progress, GCancellable *cancellable, @@ -9047,7 +9080,7 @@ flatpak_dir_install (FlatpakDir *self, flatpak_flags |= FLATPAK_PULL_FLAGS_SIDELOAD_EXTRA_DATA; - if (!flatpak_dir_pull (self, state, ref, opt_commit, NULL, subpaths, token, + if (!flatpak_dir_pull (self, state, ref, opt_commit, NULL, subpaths, require_metadata, token, child_repo, flatpak_flags, OSTREE_REPO_PULL_FLAGS_MIRROR, @@ -9123,7 +9156,7 @@ flatpak_dir_install (FlatpakDir *self, if (!no_pull) { - if (!flatpak_dir_pull (self, state, ref, opt_commit, NULL, opt_subpaths, token, NULL, + if (!flatpak_dir_pull (self, state, ref, opt_commit, NULL, opt_subpaths, require_metadata, token, NULL, flatpak_flags, OSTREE_REPO_PULL_FLAGS_NONE, progress, cancellable, error)) return FALSE; @@ -9568,6 +9601,7 @@ flatpak_dir_update (FlatpakDir *self, const OstreeRepoFinderResult * const *results, const char **opt_subpaths, const char **opt_previous_ids, + GBytes *require_metadata, const char *token, OstreeAsyncProgress *progress, GCancellable *cancellable, @@ -9740,7 +9774,7 @@ flatpak_dir_update (FlatpakDir *self, } flatpak_flags |= FLATPAK_PULL_FLAGS_SIDELOAD_EXTRA_DATA; - if (!flatpak_dir_pull (self, state, ref, commit, results, subpaths, token, + if (!flatpak_dir_pull (self, state, ref, commit, results, subpaths, require_metadata, token, child_repo, flatpak_flags, OSTREE_REPO_PULL_FLAGS_MIRROR, progress, cancellable, error)) @@ -9812,7 +9846,7 @@ flatpak_dir_update (FlatpakDir *self, if (!no_pull) { - if (!flatpak_dir_pull (self, state, ref, commit, results, subpaths, token, + if (!flatpak_dir_pull (self, state, ref, commit, results, subpaths, require_metadata, token, NULL, flatpak_flags, OSTREE_REPO_PULL_FLAGS_NONE, progress, cancellable, error)) return FALSE; diff --git a/common/flatpak-installation.c b/common/flatpak-installation.c index 42d6806f..ce2e9a26 100644 --- a/common/flatpak-installation.c +++ b/common/flatpak-installation.c @@ -2140,7 +2140,7 @@ flatpak_installation_install_full (FlatpakInstallation *self, (flags & FLATPAK_INSTALL_FLAGS_NO_DEPLOY) != 0, (flags & FLATPAK_INSTALL_FLAGS_NO_STATIC_DELTAS) != 0, FALSE, FALSE, state, - ref, NULL, (const char **) subpaths, NULL, NULL, + ref, NULL, (const char **) subpaths, NULL, NULL, NULL, ostree_progress, cancellable, error)) return NULL; @@ -2305,7 +2305,7 @@ flatpak_installation_update_full (FlatpakInstallation *self, FALSE, FALSE, FALSE, state, ref, target_commit, (const OstreeRepoFinderResult * const *) check_results, - (const char **) subpaths, NULL, NULL, + (const char **) subpaths, NULL, NULL, NULL, ostree_progress, cancellable, error)) return NULL; diff --git a/common/flatpak-transaction.c b/common/flatpak-transaction.c index acd05265..c635b379 100644 --- a/common/flatpak-transaction.c +++ b/common/flatpak-transaction.c @@ -3764,6 +3764,7 @@ _run_op_kind (FlatpakTransaction *self, remote_state, op->ref, op->resolved_commit, (const char **) op->subpaths, (const char **) op->previous_ids, + op->resolved_metadata, op->resolved_token, progress->ostree_progress, cancellable, error); @@ -3814,6 +3815,7 @@ _run_op_kind (FlatpakTransaction *self, NULL, (const char **) op->subpaths, (const char **) op->previous_ids, + op->resolved_metadata, op->resolved_token, progress->ostree_progress, cancellable, &local_error); diff --git a/system-helper/flatpak-system-helper.c b/system-helper/flatpak-system-helper.c index f69b9b40..61f82e30 100644 --- a/system-helper/flatpak-system-helper.c +++ b/system-helper/flatpak-system-helper.c @@ -674,7 +674,7 @@ handle_deploy (FlatpakSystemHelper *object, ostree_progress = ostree_async_progress_new_and_connect (no_progress_cb, NULL); - if (!flatpak_dir_pull (system, state, arg_ref, NULL, NULL, (const char **) arg_subpaths, NULL, NULL, + if (!flatpak_dir_pull (system, state, arg_ref, NULL, NULL, (const char **) arg_subpaths, NULL, NULL, NULL, FLATPAK_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_UNTRUSTED, ostree_progress, NULL, &error)) { @@ -911,11 +911,11 @@ handle_deploy_appstream (FlatpakSystemHelper *object, ostree_progress = ostree_async_progress_new_and_connect (no_progress_cb, NULL); - if (!flatpak_dir_pull (system, state, new_branch, NULL, NULL, NULL, NULL, NULL, + if (!flatpak_dir_pull (system, state, new_branch, NULL, NULL, NULL, NULL, NULL, NULL, FLATPAK_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_UNTRUSTED, ostree_progress, NULL, &first_error)) { - if (!flatpak_dir_pull (system, state, old_branch, NULL, NULL, NULL, NULL, NULL, + if (!flatpak_dir_pull (system, state, old_branch, NULL, NULL, NULL, NULL, NULL, NULL, FLATPAK_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_UNTRUSTED, ostree_progress, NULL, &second_error)) { diff --git a/tests/test-extensions.sh b/tests/test-extensions.sh index e3326e09..ee859c60 100755 --- a/tests/test-extensions.sh +++ b/tests/test-extensions.sh @@ -101,7 +101,7 @@ $(dirname $0)/make-test-app.sh repos/test "" master "" > /dev/null # Modify platform metadata ostree checkout -U --repo=repos/test runtime/org.test.Platform/${ARCH}/master platform add_extensions platform -ostree commit --repo=repos/test --owner-uid=0 --owner-gid=0 --no-xattrs --canonical-permissions --branch=runtime/org.test.Platform/${ARCH}/master -s "modified metadata" platform +${FLATPAK} build-export --disable-sandbox repos/test platform --files=files master ${FLATPAK} build-update-repo repos/test ${FLATPAK} remote-add --user --no-gpg-verify test-repo repos/test @@ -155,7 +155,7 @@ ok "runtime extensions" # Modify app metadata ostree checkout -U --repo=repos/test app/org.test.Hello/${ARCH}/master hello add_extensions hello -ostree commit --repo=repos/test --owner-uid=0 --owner-gid=0 --no-xattrs --canonical-permissions --branch=app/org.test.Hello/${ARCH}/master -s "modified metadata" hello +${FLATPAK} build-export --disable-sandbox repos/test hello master ${FLATPAK} build-update-repo repos/test ${FLATPAK} --user update -y org.test.Hello master diff --git a/tests/test-run.sh b/tests/test-run.sh index 8212e50a..fe6e9add 100644 --- a/tests/test-run.sh +++ b/tests/test-run.sh @@ -416,7 +416,8 @@ mkdir -p app/files/a-dir chmod a+rwx app/files/a-dir flatpak build-finish --command=hello.sh app # Note: not --canonical-permissions -ostree --repo=repos/test commit --owner-uid=0 --owner-gid=0 --no-xattrs ${FL_GPGARGS} --branch=app/org.test.Writable/$ARCH/stable app +${FLATPAK} build-export -vv --disable-sandbox --files=files repos/test app stable +ostree --repo=repos/test commit --keep-metadata=xa.metadata --owner-uid=0 --owner-gid=0 --no-xattrs ${FL_GPGARGS} --branch=app/org.test.Writable/$ARCH/stable app update_repo # In the system-helper case this fails to install due to the permission canonicalization happening in the @@ -435,7 +436,8 @@ touch app/files/exe chmod u+s app/files/exe flatpak build-finish --command=hello.sh app # Note: not --canonical-permissions -ostree --repo=repos/test commit --owner-uid=0 --owner-gid=0 --no-xattrs ${FL_GPGARGS} --branch=app/org.test.Setuid/$ARCH/stable app +${FLATPAK} build-export -vv --disable-sandbox --files=files repos/test app stable +ostree -v --repo=repos/test commit --keep-metadata=xa.metadata --owner-uid=0 --owner-gid=0 --no-xattrs ${FL_GPGARGS} --branch=app/org.test.Setuid/$ARCH/stable app update_repo if ${FLATPAK} ${U} install -y test-repo org.test.Setuid &> err2.txt; then From 9061b855d86fe44519c7728a146321abba6ad836 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 16 Mar 2020 15:19:58 +0100 Subject: [PATCH 08/36] Initial remove of p2p codepaths This removes the most basic codepaths for p2p installation, as well as the tests for it. There still remains various codepaths that looks as the collection id, these will be removed later. This is the first step in dropping the p2p code and replacing it with a simpler approach that focuses on the sideloading case only. --- common/flatpak-dir-private.h | 20 - common/flatpak-dir.c | 828 +----------------------------- common/flatpak-transaction.c | 176 +------ tests/Makefile-test-matrix.am.inc | 6 +- tests/Makefile.am.inc | 4 +- tests/test-wrapper.sh | 12 - 6 files changed, 19 insertions(+), 1027 deletions(-) diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index 4a22b387..0a493dc4 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -962,11 +962,6 @@ typedef struct char *ref; char *opt_commit; - /* Used during p2p resolve */ - OstreeCollectionRef collection_ref; /* owns the collection_id member only, ref_name is from above */ - char *local_commit; - char *latest_remote_commit; - /* out */ char *resolved_commit; GBytes *resolved_metadata; @@ -986,21 +981,6 @@ void flatpak_dir_resolve_free (FlatpakDirResolve *resolve); gboolean flatpak_dir_resolve_maybe_resolve_from_metadata (FlatpakDirResolve *resolve, FlatpakRemoteState *state); -typedef struct _FlatpakDirP2PState FlatpakDirP2PState; -void flatpak_dir_p2p_state_free (FlatpakDirP2PState *state); -G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakDirP2PState, flatpak_dir_p2p_state_free) - -FlatpakDirP2PState *flatpak_dir_prepare_resolve_p2p_refs (FlatpakDir *self, - FlatpakDirResolve **resolves, - GCancellable *cancellable, - GError **error); -gboolean flatpak_dir_finish_resolve_p2p_refs (FlatpakDir *self, - FlatpakDirResolve **resolves, - const char *token, - FlatpakDirP2PState *state, - GCancellable *cancellable, - GError **error); - char ** flatpak_dir_get_default_locales (FlatpakDir *self); char ** flatpak_dir_get_default_locale_languages (FlatpakDir *self); char ** flatpak_dir_get_locales (FlatpakDir *self); diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index 52138e79..2f387285 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -130,11 +130,6 @@ static gboolean flatpak_dir_cleanup_remote_for_url_change (FlatpakDir *self, GCancellable *cancellable, GError **error); -static gboolean _flatpak_dir_fetch_remote_state_metadata_branch (FlatpakDir *self, - FlatpakRemoteState *state, - gboolean only_cached, - GCancellable *cancellable, - GError **error); static gboolean flatpak_dir_lookup_remote_filter (FlatpakDir *self, const char *name, gboolean force_load, @@ -3676,75 +3671,15 @@ flatpak_dir_find_latest_rev (FlatpakDir *self, g_return_val_if_fail (out_rev != NULL, FALSE); - if (state->collection_id != NULL) - { - /* Find the latest rev from the remote and its available mirrors, including - * LAN and USB sources. */ - g_auto(GVariantBuilder) find_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER; - g_autoptr(GVariant) find_options = NULL; - g_autoptr(GAsyncResult) find_result = NULL; - g_auto(OstreeRepoFinderResultv) results = NULL; - OstreeCollectionRef collection_ref = { state->collection_id, (char *) ref }; - OstreeCollectionRef *collection_refs_to_fetch[2] = { &collection_ref, NULL }; - gsize i; - g_autoptr(GMainContextPopDefault) context = NULL; + if (!flatpak_remote_state_lookup_ref (state, ref, &latest_rev, NULL, NULL, error)) + return FALSE; + if (latest_rev == NULL) + return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, + _("Couldn't find latest checksum for ref %s in remote %s"), + ref, state->remote_name); - /* Find options */ - g_variant_builder_init (&find_builder, G_VARIANT_TYPE ("a{sv}")); - - if (checksum_or_latest != NULL) - { - g_variant_builder_add (&find_builder, "{s@v}", "override-commit-ids", - g_variant_new_variant (g_variant_new_strv (&checksum_or_latest, 1))); - } - - find_options = g_variant_ref_sink (g_variant_builder_end (&find_builder)); - - context = flatpak_main_context_new_default (); - - ostree_repo_find_remotes_async (self->repo, (const OstreeCollectionRef * const *) collection_refs_to_fetch, - find_options, - NULL /* default finders */, - NULL /* no progress reporting */, - cancellable, async_result_cb, &find_result); - - while (find_result == NULL) - g_main_context_iteration (context, TRUE); - - results = ostree_repo_find_remotes_finish (self->repo, find_result, error); - - if (results == NULL) - return FALSE; - - for (i = 0; results[i] != NULL && latest_rev == NULL; i++) - latest_rev = g_strdup (g_hash_table_lookup (results[i]->ref_to_checksum, &collection_ref)); - - if (latest_rev == NULL) - { - return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, - _("No such ref (%s, %s) in remote %s or elsewhere"), - collection_ref.collection_id, collection_ref.ref_name, - state->remote_name); - } - - if (out_results != NULL) - *out_results = g_steal_pointer (&results); - - if (out_rev != NULL) - *out_rev = g_steal_pointer (&latest_rev); - } - else - { - if (!flatpak_remote_state_lookup_ref (state, ref, &latest_rev, NULL, NULL, error)) - return FALSE; - if (latest_rev == NULL) - return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, - _("Couldn't find latest checksum for ref %s in remote %s"), - ref, state->remote_name); - - if (out_rev != NULL) - *out_rev = g_steal_pointer (&latest_rev); - } + if (out_rev != NULL) + *out_rev = g_steal_pointer (&latest_rev); return TRUE; } @@ -3774,471 +3709,11 @@ flatpak_dir_resolve_free (FlatpakDirResolve *resolve) g_bytes_unref (resolve->resolved_metadata); g_free (resolve->eol); g_free (resolve->eol_rebase); - g_free (resolve->collection_ref.collection_id); - g_free (resolve->local_commit); - g_free (resolve->latest_remote_commit); g_free (resolve); } } -static const char * -find_latest_p2p_result (OstreeRepoFinderResult **results, OstreeCollectionRef *cr) -{ - const char *latest_rev = NULL; - int i; - - for (i = 0; results[i] != NULL && latest_rev == NULL; i++) - latest_rev = g_hash_table_lookup (results[i]->ref_to_checksum, cr); - - return latest_rev; -} - -static void -remove_ref_from_p2p_results (OstreeRepoFinderResult **results, OstreeCollectionRef *cr) -{ - int i; - - for (i = 0; results[i] != NULL; i++) - g_hash_table_remove (results[i]->ref_to_checksum, cr); -} - -gboolean -flatpak_dir_resolve_maybe_resolve_from_metadata (FlatpakDirResolve *resolve, - FlatpakRemoteState *state) -{ - guint64 download_size = 0; - guint64 installed_size = 0; - const char *xa_metadata = NULL; - const guchar *metadata_checksum_bytes = NULL; - g_autofree char *metadata_checksum = NULL; - VarMetadataRef sparse_cache; - - if (!flatpak_remote_state_lookup_cache (state, resolve->ref, &download_size, &installed_size, &xa_metadata, &metadata_checksum_bytes, NULL)) - return FALSE; - - if (metadata_checksum_bytes == NULL) - return FALSE; /* Commit unknown, old server version */ - - metadata_checksum = ostree_checksum_from_bytes (metadata_checksum_bytes); - - /* If the latest available commit is the same as the one we have info on in the ostree-metadata - then we can use the ostree-metadata to resolve the op without having to download the commit */ - if (g_strcmp0 (metadata_checksum, resolve->latest_remote_commit) != 0) - return FALSE; - - resolve->resolved_commit = g_strdup (metadata_checksum); - resolve->resolved_metadata = g_bytes_new (xa_metadata, strlen (xa_metadata) + 1); - resolve->installed_size = installed_size; - resolve->download_size = download_size; - - if (flatpak_remote_state_lookup_sparse_cache (state, resolve->ref, &sparse_cache, NULL)) - { - resolve->eol = g_strdup (var_metadata_lookup_string (sparse_cache, FLATPAK_SPARSE_CACHE_KEY_ENDOFLINE, NULL)); - resolve->eol_rebase = g_strdup (var_metadata_lookup_string (sparse_cache, FLATPAK_SPARSE_CACHE_KEY_ENDOFLINE_REBASE, NULL)); - resolve->token_type = GINT32_FROM_LE(var_metadata_lookup_int32 (sparse_cache, FLATPAK_SPARSE_CACHE_KEY_TOKEN_TYPE, 0)); - } - - return TRUE; /* Resolved */ -} - -static void -resolve_p2p_update_from_commit (FlatpakDirResolve *resolve, - GVariant *commit_data) -{ - g_autoptr(GVariant) commit_metadata = NULL; - const char *xa_metadata = NULL; - guint64 download_size = 0; - guint64 installed_size = 0; - - commit_metadata = g_variant_get_child_value (commit_data, 0); - - g_variant_lookup (commit_metadata, "xa.metadata", "&s", &xa_metadata); - if (xa_metadata == NULL) - g_message ("Warning: No xa.metadata in commit %s ref %s", resolve->resolved_commit, resolve->ref); - else - resolve->resolved_metadata = g_bytes_new (xa_metadata, strlen (xa_metadata) + 1); - - if (g_variant_lookup (commit_metadata, "xa.download-size", "t", &download_size)) - resolve->download_size = GUINT64_FROM_BE (download_size); - - if (g_variant_lookup (commit_metadata, "xa.installed-size", "t", &installed_size)) - resolve->installed_size = GUINT64_FROM_BE (installed_size); - - g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_ENDOFLIFE, "s", &resolve->eol); - g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_ENDOFLIFE_REBASE, "s", &resolve->eol_rebase); - /* NOTE: The transaction code already default or stored token_type from ostree-metadata here, but fix it up from the commit anyway */ - if (g_variant_lookup (commit_metadata, "xa.token-type", "i", &resolve->token_type)) - resolve->token_type = GINT32_FROM_LE (resolve->token_type); -} - -struct _FlatpakDirP2PState { - OstreeRepo *child_repo; - GLnxLockFile child_repo_lock; - GPtrArray *results; - GPtrArray *results_refs; /* contains the original ref_to_checksum hashes from the items in results */ -}; - -void -flatpak_dir_p2p_state_free (FlatpakDirP2PState *state) -{ - if (state->child_repo) - { - flatpak_rm_rf (ostree_repo_get_path (state->child_repo), NULL, NULL); - g_object_unref (state->child_repo); - glnx_release_lock_file (&state->child_repo_lock); - } - - if (state->results) - g_ptr_array_unref (state->results); - - if (state->results_refs) - g_ptr_array_unref (state->results_refs); - - g_free (state); -} - -static void -finder_result_maybe_free (OstreeRepoFinderResult *result) -{ - if (result) - ostree_repo_finder_result_free (result); -} - -static FlatpakDirP2PState * -flatpak_dir_create_p2p_state (FlatpakDir *self, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(FlatpakDirP2PState) state = g_new0 (FlatpakDirP2PState, 1); - g_autoptr(GFile) user_cache_dir = NULL; - - /* We create a temporary child repo in the user homedir so that we can just blow it away when we're done. - * This lets us always write to the directory in the system-helper case, but also lets us properly clean up - * the transaction state directory, as that doesn't happen on abort. */ - user_cache_dir = flatpak_ensure_user_cache_dir_location (error); - if (user_cache_dir == NULL) - return NULL; - - state->child_repo = flatpak_dir_create_child_repo (self, user_cache_dir, &state->child_repo_lock, NULL, error); - if (state->child_repo == NULL) - return NULL; - - state->results = g_ptr_array_new_with_free_func ((GDestroyNotify) finder_result_maybe_free); - state->results_refs = g_ptr_array_new_with_free_func ((GDestroyNotify) g_hash_table_unref); - - return g_steal_pointer (&state); -} - -static gboolean -flatpak_dir_prepare_resolve_p2p_refs_helper (FlatpakDir *self, - GPtrArray *resolves, - gboolean with_commit_ids, - FlatpakDirP2PState *state, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(GPtrArray) collection_refs_to_fetch = g_ptr_array_new (); - g_autoptr(GPtrArray) commit_ids_to_fetch = NULL; - g_autoptr(GAsyncResult) find_result = NULL; - g_auto(OstreeRepoFinderResultv) results = NULL; - g_autoptr(GVariant) find_options = NULL; - g_auto(GVariantBuilder) find_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER; - g_autoptr(GMainContextPopDefault) main_context = NULL; - g_autoptr(GString) refs_str = NULL; - int i; - - if (with_commit_ids) - commit_ids_to_fetch = g_ptr_array_new (); - - refs_str = g_string_new (""); - for (i = 0; i < resolves->len; i++) - { - FlatpakDirResolve *resolve = resolves->pdata[i]; - - if (i != 0) - g_string_append (refs_str, ", "); - g_string_append_printf (refs_str, "(%s, %s)", - resolve->collection_ref.collection_id, - resolve->collection_ref.ref_name); - - g_ptr_array_add (collection_refs_to_fetch, &resolve->collection_ref); - if (commit_ids_to_fetch) - { - g_assert (resolve->opt_commit != NULL); - g_ptr_array_add (commit_ids_to_fetch, resolve->opt_commit); - } - } - - g_ptr_array_add (collection_refs_to_fetch, NULL); - - g_debug ("Resolving these collection-refs: [%s]", refs_str->str); - - g_variant_builder_init (&find_builder, G_VARIANT_TYPE ("a{sv}")); - if (commit_ids_to_fetch) - g_variant_builder_add (&find_builder, "{s@v}", "override-commit-ids", - g_variant_new_variant (g_variant_new_strv ((const char * const *) commit_ids_to_fetch->pdata, - commit_ids_to_fetch->len))); - find_options = g_variant_ref_sink (g_variant_builder_end (&find_builder)); - - main_context = flatpak_main_context_new_default (); - - ostree_repo_find_remotes_async (state->child_repo, - (const OstreeCollectionRef * const *) collection_refs_to_fetch->pdata, - find_options, - NULL /* default finders */, - NULL /* no progress reporting */, - cancellable, async_result_cb, &find_result); - - while (find_result == NULL) - g_main_context_iteration (main_context, TRUE); - - results = ostree_repo_find_remotes_finish (state->child_repo, find_result, error); - if (results == NULL) - return FALSE; - - if (results[0] == NULL) - return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, - _("No remotes found which provide these refs: [%s]"), - refs_str->str); - - /* Drop from the results all ops that are no-op updates */ - for (i = 0; i < resolves->len; i++) - { - FlatpakDirResolve *resolve = resolves->pdata[i]; - const char *latest_rev = NULL; - - latest_rev = find_latest_p2p_result (results, &resolve->collection_ref); - resolve->latest_remote_commit = g_strdup (latest_rev); - - if (resolve->local_commit == NULL) - continue; - - if (g_strcmp0 (latest_rev, resolve->local_commit) == 0) - { - g_autoptr(GVariant) commit_data = NULL; - - /* We already have the latest commit, so resolve it from - * the local commit and remove from all results. This way we - * avoid pulling this ref from all remotes. */ - - if (!ostree_repo_load_commit (state->child_repo, resolve->local_commit, &commit_data, NULL, NULL)) - return FALSE; - - resolve->resolved_commit = g_strdup (resolve->local_commit); - resolve_p2p_update_from_commit (resolve, commit_data); - remove_ref_from_p2p_results (results, &resolve->collection_ref); - } - } - - /* Copy any (now) non-empty FinderResults into the state */ - for (i = 0; results[i] != NULL; i++) - { - OstreeRepoFinderResult *result = results[i]; - - if (g_hash_table_size (result->ref_to_checksum) > 0) - g_ptr_array_add (state->results, result); - else - ostree_repo_finder_result_free (result); - } - - g_free (g_steal_pointer (&results)); /* content was freed or stolen above */ - - return TRUE; -} - -/* Unfortunately the p2p resolve case has to be split into two phases. The first (prepare) - * phase only works on public data such as the summary and ostree-metadata branch, whereas the second - * phace (_finish) actually needs to look at the commit objects, and before that is possible we have - * to externally any require bearer tokens for this to be possible. - */ -FlatpakDirP2PState * -flatpak_dir_prepare_resolve_p2p_refs (FlatpakDir *self, - FlatpakDirResolve **resolves, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(GPtrArray) latest_resolves = g_ptr_array_new (); - g_autoptr(GPtrArray) specific_resolves = g_ptr_array_new (); - g_autoptr(FlatpakDirP2PState) state = NULL; - int i; - - for (i = 0; resolves[i] != NULL; i++) - { - FlatpakDirResolve *resolve = resolves[i]; - - g_assert (resolve->ref != NULL); - g_assert (resolve->remote != NULL); - - resolve->collection_ref.ref_name = resolve->ref; - resolve->collection_ref.collection_id = flatpak_dir_get_remote_collection_id (self, resolve->remote); - - g_assert (resolve->collection_ref.collection_id != NULL); - - if (resolve->opt_commit == NULL) - { - flatpak_repo_resolve_rev (self->repo, resolve->collection_ref.collection_id, resolve->remote, - resolve->ref, TRUE, &resolve->local_commit, cancellable, NULL); - } - - /* The ostree p2p api doesn't let you mix pulls with specific commit IDs - * and HEAD (https://github.com/ostreedev/ostree/issues/1622) so we need - * to split these into two pull ops */ - if (resolve->opt_commit) - g_ptr_array_add (specific_resolves, resolve); - else - g_ptr_array_add (latest_resolves, resolve); - } - - state = flatpak_dir_create_p2p_state (self, cancellable, error); - if (state == NULL) - return NULL; - - if (specific_resolves->len > 0) - { - if (!flatpak_dir_prepare_resolve_p2p_refs_helper (self, specific_resolves, TRUE, state, cancellable, error)) - return NULL; - } - - if (latest_resolves->len > 0) - { - if (!flatpak_dir_prepare_resolve_p2p_refs_helper (self, latest_resolves, FALSE, state, cancellable, error)) - return NULL; - } - - for (i = 0; i < state->results->len; i++) - { - OstreeRepoFinderResult *result = state->results->pdata[i]; - - /* Squirrel away the original ref_to_checksum hash so that we can reuse it multiple times - each with a different subset */ - g_ptr_array_add (state->results_refs, g_steal_pointer (&result->ref_to_checksum)); - result->ref_to_checksum = g_hash_table_new_full (ostree_collection_ref_hash, - ostree_collection_ref_equal, - NULL, g_free); - } - - g_ptr_array_add (state->results, NULL); /* NULL terminate */ - - return g_steal_pointer (&state); -} - -gboolean -flatpak_dir_finish_resolve_p2p_refs (FlatpakDir *self, - FlatpakDirResolve **resolves, // This can be a subset of the prepare_resolve list - const char *token, - FlatpakDirP2PState *state, - GCancellable *cancellable, - GError **error) -{ - OstreeRepoPullFlags flags = OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY; - GVariantBuilder pull_builder, ref_keyring_map_builder; - g_autoptr(GVariant) pull_options = NULL; - g_autoptr(GAsyncResult) pull_result = NULL; - g_autoptr(FlatpakRepoTransaction) transaction = NULL; - g_autoptr(GMainContextPopDefault) main_context = NULL; - int i; - - if (state->results->pdata[0] == NULL) - return TRUE; - - /* Do a version check to ensure we have these: - * https://github.com/ostreedev/ostree/pull/1821 - * https://github.com/ostreedev/ostree/pull/1825 */ -#if OSTREE_CHECK_VERSION (2019, 2) - flags |= OSTREE_REPO_PULL_FLAGS_MIRROR; -#endif - - g_variant_builder_init (&pull_builder, G_VARIANT_TYPE ("a{sv}")); - g_variant_builder_add (&pull_builder, "{s@v}", "flags", - g_variant_new_variant (g_variant_new_int32 (flags))); - g_variant_builder_add (&pull_builder, "{s@v}", "inherit-transaction", - g_variant_new_variant (g_variant_new_boolean (TRUE))); - g_variant_builder_add (&pull_builder, "{s@v}", "append-user-agent", - g_variant_new_variant (g_variant_new_string ("flatpak/" PACKAGE_VERSION))); - - if (token) - { - GVariantBuilder hdr_builder; - g_variant_builder_init (&hdr_builder, G_VARIANT_TYPE ("a(ss)")); - g_autofree char *bearer_token = g_strdup_printf ("Bearer %s", token); - g_variant_builder_add (&hdr_builder, "(ss)", "Authorization", bearer_token); - g_variant_builder_add (&pull_builder, "{s@v}", "http-headers", - g_variant_new_variant (g_variant_builder_end (&hdr_builder))); - } - - /* Ensure the results are signed with the GPG keys associated with the correct remote */ - g_variant_builder_init (&ref_keyring_map_builder, G_VARIANT_TYPE ("a(sss)")); - for (i = 0; resolves[i] != NULL; i++) - { - FlatpakDirResolve *resolve = resolves[i]; - - g_variant_builder_add (&ref_keyring_map_builder, "(sss)", - resolve->collection_ref.collection_id, - resolve->collection_ref.ref_name, - resolve->remote); - } - g_variant_builder_add (&pull_builder, "{s@v}", "ref-keyring-map", - g_variant_new_variant (g_variant_builder_end (&ref_keyring_map_builder))); - - pull_options = g_variant_ref_sink (g_variant_builder_end (&pull_builder)); - - main_context = flatpak_main_context_new_default (); - - transaction = flatpak_repo_transaction_start (state->child_repo, cancellable, error); - if (transaction == NULL) - return FALSE; - - /* Update the ref_to_checksum array to only contain the original mappings that intersect - the set of refs we're pulling now. */ - for (i = 0; state->results->pdata[i] != NULL; i++) - { - GHashTable *orig_ref_to_checksum = state->results_refs->pdata[i]; - OstreeRepoFinderResult *result = state->results->pdata[i]; - - g_hash_table_remove_all (result->ref_to_checksum); - for (int j = 0; resolves[j] != NULL; j++) - { - FlatpakDirResolve *resolve = resolves[j]; - const char *checksum = g_hash_table_lookup (orig_ref_to_checksum, &resolve->collection_ref); - if (checksum) - g_hash_table_insert (result->ref_to_checksum, &resolve->collection_ref, g_strdup (checksum)); - } - } - - ostree_repo_pull_from_remotes_async (state->child_repo, (const OstreeRepoFinderResult * const *) state->results->pdata, - pull_options, NULL, - cancellable, async_result_cb, - &pull_result); - - while (pull_result == NULL) - g_main_context_iteration (main_context, TRUE); - - if (!ostree_repo_pull_from_remotes_finish (state->child_repo, pull_result, error)) - return FALSE; - - for (i = 0; resolves[i] != NULL; i++) - { - FlatpakDirResolve *resolve = resolves[i]; - g_autoptr(GVariant) commit_data = NULL; - - if (resolve->resolved_commit != NULL) - continue; - - if (!flatpak_repo_resolve_rev (state->child_repo, resolve->collection_ref.collection_id, resolve->remote, - resolve->ref, FALSE, &resolve->resolved_commit, - cancellable, error)) - return FALSE; - - if (!ostree_repo_load_commit (state->child_repo, resolve->resolved_commit, &commit_data, NULL, error)) - return FALSE; - - resolve_p2p_update_from_commit (resolve, commit_data); - } - - return TRUE; -} - static gboolean child_repo_ensure_summary (OstreeRepo *child_repo, FlatpakRemoteState *state, @@ -5547,81 +5022,12 @@ flatpak_dir_pull (FlatpakDir *self, } else { - if (state->collection_id) - { - GVariantBuilder find_builder; - g_autoptr(GVariant) find_options = NULL; - g_autoptr(GAsyncResult) find_result = NULL; - OstreeCollectionRef collection_ref; - OstreeCollectionRef *collection_refs_to_fetch[2]; - gboolean force_disable_deltas = (flatpak_flags & FLATPAK_PULL_FLAGS_NO_STATIC_DELTAS) != 0; - guint update_interval = 0; - gsize i; - g_autoptr(GMainContextPopDefault) context = NULL; - g_autoptr(FlatpakAsyncProgressChained) chained_progress = NULL; + flatpak_remote_state_lookup_ref (state, ref, &rev, NULL, NULL, error); + if (rev == NULL && error != NULL && *error == NULL) + flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, _("Couldn't find latest checksum for ref %s in remote %s"), + ref, state->remote_name); - /* FIXME: It would be nice to break out a helper function from - * flatpak_dir_do_resolve_p2p_refs() that would resolve refs to - * commits and reuse it here */ - - g_variant_builder_init (&find_builder, G_VARIANT_TYPE ("a{sv}")); - - if (force_disable_deltas) - { - g_variant_builder_add (&find_builder, "{s@v}", "disable-static-deltas", - g_variant_new_variant (g_variant_new_boolean (TRUE))); - } - - collection_ref.collection_id = state->collection_id; - collection_ref.ref_name = (char *) ref; - - collection_refs_to_fetch[0] = &collection_ref; - collection_refs_to_fetch[1] = NULL; - - if (progress != NULL) - update_interval = GPOINTER_TO_UINT (g_object_get_data (G_OBJECT (progress), "update-interval")); - if (update_interval == 0) - update_interval = FLATPAK_DEFAULT_UPDATE_INTERVAL_MS; - - g_variant_builder_add (&find_builder, "{s@v}", "update-frequency", - g_variant_new_variant (g_variant_new_uint32 (update_interval))); - - find_options = g_variant_ref_sink (g_variant_builder_end (&find_builder)); - - context = flatpak_main_context_new_default (); - chained_progress = flatpak_progress_chain (progress); - - ostree_repo_find_remotes_async (self->repo, (const OstreeCollectionRef * const *) collection_refs_to_fetch, - find_options, - NULL /* default finders */, - chained_progress, cancellable, - async_result_cb, &find_result); - - while (find_result == NULL) - g_main_context_iteration (context, TRUE); - - allocated_results = ostree_repo_find_remotes_finish (self->repo, find_result, error); - - results = (const OstreeRepoFinderResult * const *) allocated_results; - if (results == NULL) - return FALSE; - - for (i = 0, rev = NULL; results[i] != NULL && rev == NULL; i++) - rev = g_strdup (g_hash_table_lookup (results[i]->ref_to_checksum, &collection_ref)); - - if (rev == NULL) - return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, _("No such ref (%s, %s) in remote %s or elsewhere"), - collection_ref.collection_id, collection_ref.ref_name, state->remote_name); - } - else - { - flatpak_remote_state_lookup_ref (state, ref, &rev, NULL, NULL, error); - if (rev == NULL && error != NULL && *error == NULL) - flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, _("Couldn't find latest checksum for ref %s in remote %s"), - ref, state->remote_name); - - results = NULL; - } + results = NULL; if (rev == NULL) { @@ -11258,69 +10664,8 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, } } - if (state->collection_id == NULL) - { - if (state->summary != NULL) /* In the optional case we might not have a summary */ - state->metadata = g_variant_get_child_value (state->summary, 1); - } - else - { - g_autofree char *latest_rev = NULL; - g_autoptr(GVariant) commit_v = NULL; - g_autoptr(GError) local_error = NULL; - - /* Make sure the branch is up to date, but ignore downgrade errors (see - * below for the explanation). */ - if (!_flatpak_dir_fetch_remote_state_metadata_branch (self, state, only_cached, cancellable, &local_error) && - !g_error_matches (local_error, FLATPAK_ERROR, FLATPAK_ERROR_DOWNGRADE)) - { - if (optional && !g_cancellable_is_cancelled (cancellable)) - { - /* This happens for instance in the case where a p2p remote is invalid (wrong signature) - and we should just silently fail to update to it. */ - state->metadata_fetch_error = g_steal_pointer (&local_error); - g_debug ("Failed to download optional metadata"); - } - else - { - g_propagate_error (error, g_steal_pointer (&local_error)); - return NULL; - } - } - else - { - if (g_error_matches (local_error, FLATPAK_ERROR, FLATPAK_ERROR_DOWNGRADE)) - { - /* The latest metadata available is a downgrade, which means we're offline and using a - * LAN/USB source. Downgrading the metadata in the system repo would be a security - * risk, so instead ignore the downgrade and use the later metadata. There's some - * chance its information won't be accurate for the refs that are pulled, but using - * the old metadata wouldn't always be correct either because there's no guarantee the - * refs will be pulled from the same peer source as the metadata. Long term, we should - * figure out how to rely less on it. */ - g_debug ("Ignoring downgrade of ostree-metadata; using the newer one instead"); - } - - /* Look up the commit containing the latest repository metadata. */ - latest_rev = flatpak_dir_read_latest (self, remote_or_uri, OSTREE_REPO_METADATA_REF, - NULL, cancellable, error); - if (latest_rev == NULL) - { - if (only_cached) - { - g_clear_error (error); - g_set_error (error, FLATPAK_ERROR, FLATPAK_ERROR_NOT_CACHED, - _("No repo metadata cached for remote '%s'"), remote_or_uri); - } - return NULL; - } - - if (!ostree_repo_load_commit (self->repo, latest_rev, &commit_v, NULL, error)) - return NULL; - - state->metadata = g_variant_get_child_value (commit_v, 0); - } - } + if (state->summary != NULL) /* In the optional case we might not have a summary */ + state->metadata = g_variant_get_child_value (state->summary, 1); if (state->metadata) { @@ -13544,149 +12889,6 @@ flatpak_dir_list_remote_refs (FlatpakDir *self, return TRUE; } -gboolean -_flatpak_dir_fetch_remote_state_metadata_branch (FlatpakDir *self, - FlatpakRemoteState *state, /* This state does not have metadata filled out yet */ - gboolean only_cached, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(OstreeAsyncProgressFinish) progress = ostree_async_progress_new (); - FlatpakPullFlags flatpak_flags; - gboolean gpg_verify; - g_autofree char *checksum_from_summary = NULL; - g_autofree char *checksum_from_repo = NULL; - - g_assert (state->collection_id != NULL); - - /* We can only fetch metadata if we’re going to verify it with GPG. */ - if (!ostree_repo_remote_get_gpg_verify (self->repo, state->remote_name, - &gpg_verify, error)) - return FALSE; - - if (!gpg_verify) - return flatpak_fail_error (error, FLATPAK_ERROR_UNTRUSTED, _("Can't pull from untrusted non-gpg verified remote")); - - /* Look up the checksum as advertised by the summary file. If it differs from - * what we currently have on disk, try and pull the updated ostree-metadata ref. - * This is how we implement caching. Ignore failure and pull the ref anyway. */ - if (state->summary != NULL) - flatpak_summary_lookup_ref (state->summary, state->collection_id, - OSTREE_REPO_METADATA_REF, - &checksum_from_summary, NULL); - - if (!flatpak_repo_resolve_rev (self->repo, state->collection_id, state->remote_name, - OSTREE_REPO_METADATA_REF, TRUE, &checksum_from_repo, - cancellable, error)) - return FALSE; - - g_debug ("%s: Comparing %s from summary and %s from repo", - G_STRFUNC, checksum_from_summary, checksum_from_repo); - - if (checksum_from_summary != NULL && checksum_from_repo != NULL && - g_str_equal (checksum_from_summary, checksum_from_repo)) - return TRUE; - - /* Do the pull into the local repository. */ - flatpak_flags = FLATPAK_PULL_FLAGS_DOWNLOAD_EXTRA_DATA; - flatpak_flags |= FLATPAK_PULL_FLAGS_NO_STATIC_DELTAS; - - if (flatpak_dir_use_system_helper (self, NULL)) - { - g_autoptr(OstreeRepo) child_repo = NULL; - g_auto(GLnxLockFile) child_repo_lock = { 0, }; - const char *installation = flatpak_dir_get_id (self); - const char *subpaths[] = {NULL}; - const char * const *previous_ids = {NULL}; - g_autofree char *child_repo_path = NULL; - FlatpakHelperDeployFlags helper_flags = 0; - g_autofree char *url = NULL; - gboolean gpg_verify_summary; - gboolean gpg_verify; - gboolean is_oci; - - if (!ostree_repo_remote_get_url (self->repo, - state->remote_name, - &url, - error)) - return FALSE; - - if (!ostree_repo_remote_get_gpg_verify_summary (self->repo, state->remote_name, - &gpg_verify_summary, error)) - return FALSE; - - if (!ostree_repo_remote_get_gpg_verify (self->repo, state->remote_name, - &gpg_verify, error)) - return FALSE; - - is_oci = flatpak_dir_get_remote_oci (self, state->remote_name); - if ((!gpg_verify_summary && state->collection_id == NULL) || !gpg_verify) - { - /* The remote is not gpg verified, so we don't want to allow installation via - a download in the home directory, as there is no way to verify you're not - injecting anything into the remote. However, in the case of a remote - configured to a local filesystem we can just let the system helper do - the installation, as it can then avoid network i/o and be certain the - data comes from the right place. - - If a collection ID is available, we can verify the refs in commit - metadata. */ - if (g_str_has_prefix (url, "file:")) - helper_flags |= FLATPAK_HELPER_DEPLOY_FLAGS_LOCAL_PULL; - else - return flatpak_fail_error (error, FLATPAK_ERROR_UNTRUSTED, _("Can't pull from untrusted non-gpg verified remote")); - } - else if (is_oci) - { - return flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("No metadata branch for OCI")); - } - else - { - /* We're pulling from a remote source, we do the network mirroring pull as a - user and hand back the resulting data to the system-helper, that trusts us - due to the GPG signatures in the repo */ - child_repo = flatpak_dir_create_system_child_repo (self, &child_repo_lock, NULL, error); - if (child_repo == NULL) - return FALSE; - - if (!flatpak_dir_pull (self, state, OSTREE_REPO_METADATA_REF, NULL, NULL, NULL, NULL, - child_repo, - flatpak_flags, - 0, - progress, cancellable, error)) - return FALSE; - - if (!child_repo_ensure_summary (child_repo, state, cancellable, error)) - return FALSE; - - child_repo_path = g_file_get_path (ostree_repo_get_path (child_repo)); - } - - helper_flags |= FLATPAK_HELPER_DEPLOY_FLAGS_NO_DEPLOY; - - if (!flatpak_dir_system_helper_call_deploy (self, - child_repo_path ? child_repo_path : "", - helper_flags, OSTREE_REPO_METADATA_REF, state->remote_name, - (const char * const *) subpaths, previous_ids, - installation ? installation : "", - cancellable, - error)) - return FALSE; - - if (child_repo_path) - (void) glnx_shutil_rm_rf_at (AT_FDCWD, child_repo_path, NULL, NULL); - - return TRUE; - } - - if (!flatpak_dir_pull (self, state, OSTREE_REPO_METADATA_REF, NULL, NULL, NULL, NULL, NULL, - flatpak_flags, OSTREE_REPO_PULL_FLAGS_NONE, - progress, cancellable, error)) - return FALSE; - - return TRUE; -} - static gboolean strv_contains_prefix (const gchar * const *strv, const gchar *str) diff --git a/common/flatpak-transaction.c b/common/flatpak-transaction.c index c635b379..629accbe 100644 --- a/common/flatpak-transaction.c +++ b/common/flatpak-transaction.c @@ -2543,171 +2543,6 @@ op_may_need_token (FlatpakTransactionOperation *op) op->kind == FLATPAK_TRANSACTION_OPERATION_INSTALL_OR_UPDATE); } -static gboolean -resolve_p2p_ops (FlatpakTransaction *self, - GList *p2p_ops, - GCancellable *cancellable, - GError **error) -{ - FlatpakTransactionPrivate *priv = flatpak_transaction_get_instance_private (self); - g_autoptr(GPtrArray) resolves = g_ptr_array_new_with_free_func ((GDestroyNotify) flatpak_dir_resolve_free); - g_autoptr(FlatpakDirP2PState) state = NULL; - gboolean got_new_need_token; - GList *l; - int i; - - for (l = p2p_ops; l != NULL; l = l->next) - { - FlatpakTransactionOperation *op = l->data; - FlatpakDirResolve *resolve; - - g_debug ("resolving %s using p2p", op->ref); - - g_assert (op->kind != FLATPAK_TRANSACTION_OPERATION_UNINSTALL); - g_assert (op->kind != FLATPAK_TRANSACTION_OPERATION_INSTALL_BUNDLE); - g_assert (!op->resolved); - - resolve = flatpak_dir_resolve_new (op->remote, op->ref, op->commit); - g_ptr_array_add (resolves, resolve); - } - - g_ptr_array_add (resolves, NULL); - - /* This does the metadata checks and resolving of no-op updates. */ - state = flatpak_dir_prepare_resolve_p2p_refs (priv->dir, (FlatpakDirResolve **) resolves->pdata, - cancellable, error); - if (state == NULL) - return FALSE; - - /* Resolve any ops that we can resolve from the ostree-metadata info */ - for (i = 0, l = p2p_ops; l != NULL; i++, l = l->next) - { - FlatpakTransactionOperation *op = l->data; - FlatpakDirResolve *resolve = g_ptr_array_index (resolves, i); - - if (resolve->resolved_commit == NULL) - { - g_autoptr(FlatpakRemoteState) state = flatpak_transaction_ensure_remote_state (self, op->kind, op->remote, NULL); - if (state != NULL) - flatpak_dir_resolve_maybe_resolve_from_metadata (resolve, state); - } - } - - /* We now pre-resolved all p2p operations, which is all we can do without full access - to the commit objects. To get the commit objects we might need a token, but the information - about whether we need a token or not is in the commit object. This is a catch-22, which we - break by using the extracted commit info in the ostree-metadata branch. There is a minor - risk that this is out-of-sync with the exact information in the commit if using a p2p mirror - with an old version of the app, but the latest ostree-metadata version. The need-token info - is unlikely to change though, and the worst case issue here is some permission error during - download, so this is not *unsafe*, just kinda iffy. - */ - got_new_need_token = FALSE; - for (i = 0, l = p2p_ops; l != NULL; i++, l = l->next) - { - FlatpakTransactionOperation *op = l->data; - FlatpakDirResolve *resolve = g_ptr_array_index (resolves, i); - - if (resolve->resolved_commit != NULL) - continue; /* Already resolved this (as no-op) */ - - if (op_may_need_token (op)) - { - g_autoptr(FlatpakRemoteState) state = NULL; - VarMetadataRef sparse_cache; - gint32 token_type; - - state = flatpak_transaction_ensure_remote_state (self, op->kind, op->remote, error); - if (state == NULL) - return FALSE; - - token_type = state->default_token_type; - - if (flatpak_remote_state_lookup_sparse_cache (state, op->ref, &sparse_cache, NULL)) - token_type = GINT32_FROM_LE (var_metadata_lookup_int32 (sparse_cache, FLATPAK_SPARSE_CACHE_KEY_TOKEN_TYPE, token_type)); - - if (token_type > 0) - { - /* We set op->need_token here from the ostree-repo - so we query a token for this, later we'll - override it with the resolved value */ - op->token_type = token_type; - got_new_need_token = TRUE; - } - } - } - - if (got_new_need_token) - { - if (!request_required_tokens (self, NULL, cancellable, error)) - return FALSE; - } - - /* We can't get all the resolve commit objects in one pull request, - because the token can be different for each ref, so we need to - split things up by token.*/ - - g_autoptr(GPtrArray) non_token_resolves = g_ptr_array_new (); - g_autoptr(GHashTable) token_resolves_ht = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, (GDestroyNotify) g_ptr_array_unref); - - for (i = 0, l = p2p_ops; l != NULL; i++, l = l->next) - { - FlatpakTransactionOperation *op = l->data; - FlatpakDirResolve *resolve = g_ptr_array_index (resolves, i); - - if (resolve->resolved_commit != NULL) - continue; /* Already resolved this (as no-op) */ - - if (op->resolved_token) - { - GPtrArray *token_resolves = g_hash_table_lookup (token_resolves_ht, op->resolved_token); - if (token_resolves == NULL) - { - token_resolves = g_ptr_array_new (); - g_hash_table_insert (token_resolves_ht, op->resolved_token, token_resolves); - } - g_ptr_array_add (token_resolves, resolve); - } - else - { - g_ptr_array_add (non_token_resolves, resolve); - } - } - - /* This does the downloads of the actual commit objects that are needed. */ - - /* First refs that need no token */ - g_ptr_array_add (non_token_resolves, NULL); // NULL terminate - if (!flatpak_dir_finish_resolve_p2p_refs (priv->dir, (FlatpakDirResolve **) non_token_resolves->pdata, NULL, - state, cancellable, error)) - return FALSE; - - /* Then once per token */ - GLNX_HASH_TABLE_FOREACH_KV(token_resolves_ht, const char *, token, GPtrArray *, token_resolves) - { - g_ptr_array_add (token_resolves, NULL); // NULL terminate - if (!flatpak_dir_finish_resolve_p2p_refs (priv->dir, (FlatpakDirResolve **) token_resolves->pdata, token, - state, cancellable, error)) - return FALSE; - } - - for (i = 0, l = p2p_ops; l != NULL; i++, l = l->next) - { - FlatpakTransactionOperation *op = l->data; - FlatpakDirResolve *resolve = g_ptr_array_index (resolves, i); - - op->download_size = resolve->download_size; - op->installed_size = resolve->installed_size; - op->eol = g_strdup (resolve->eol); - op->eol_rebase = g_strdup (resolve->eol_rebase); - op->token_type = resolve->token_type; - - resolve_op_end (self, op, resolve->resolved_commit, resolve->resolved_metadata); - } - - return TRUE; -} - /* Resolving an operation means figuring out the target commit checksum and the metadata for that commit, so that we can handle dependencies from it, and verify versions. */ @@ -2775,7 +2610,7 @@ resolve_ops (FlatpakTransaction *self, resolve_op_from_commit (self, op, checksum, commit_data); } - else if (state->collection_id == NULL) /* In the non-p2p case we have all the info available in the summary, so use it */ + else { g_autoptr(GError) local_error = NULL; @@ -2803,17 +2638,8 @@ resolve_ops (FlatpakTransaction *self, where the user specified a commit, or p2p doesn't have the latest commit available */ resolve_op_from_metadata (self, op, checksum, state); } - else - { - /* This is a (potential) p2p operation, so rather than do these individually we queue them up in an operation later */ - collection_id_ops = g_list_prepend (collection_id_ops, op); - } } - if (collection_id_ops != NULL && - !resolve_p2p_ops (self, collection_id_ops, cancellable, error)) - return FALSE; - return TRUE; } diff --git a/tests/Makefile-test-matrix.am.inc b/tests/Makefile-test-matrix.am.inc index 9b02f1a9..7ce305d5 100644 --- a/tests/Makefile-test-matrix.am.inc +++ b/tests/Makefile-test-matrix.am.inc @@ -11,15 +11,11 @@ TEST_MATRIX= \ tests/test-repo@user.wrap \ tests/test-repo@system.wrap \ tests/test-repo@system-norevokefs.wrap \ - tests/test-repo@collections.wrap \ - tests/test-repo@collections-server-only.wrap \ tests/test-bundle@user.wrap \ tests/test-bundle@system.wrap \ tests/test-bundle@system-norevokefs.wrap \ tests/test-oci-registry@user.wrap \ tests/test-oci-registry@system.wrap \ - tests/test-auth@nocollections.wrap \ - tests/test-auth@collections.wrap \ $(NULL) TEST_MATRIX_DIST= \ tests/test-basic.sh \ @@ -34,6 +30,7 @@ TEST_MATRIX_DIST= \ tests/test-update-remote-configuration.sh \ tests/test-override.sh \ tests/test-update-portal.sh \ + tests/test-auth.sh \ $(NULL) TEST_MATRIX_EXTRA_DIST= \ tests/test-run.sh \ @@ -41,5 +38,4 @@ TEST_MATRIX_EXTRA_DIST= \ tests/test-repo.sh \ tests/test-bundle.sh \ tests/test-oci-registry.sh \ - tests/test-auth.sh \ $(NULL) diff --git a/tests/Makefile.am.inc b/tests/Makefile.am.inc index 7c2e8271..29977272 100644 --- a/tests/Makefile.am.inc +++ b/tests/Makefile.am.inc @@ -177,7 +177,7 @@ TEST_MATRIX_SOURCE = \ tests/test-http-utils.sh \ tests/test-run.sh{{user+system+system-norevokefs},{nodeltas+deltas}} \ tests/test-info.sh{user+system} \ - tests/test-repo.sh{user+system+system-norevokefs+collections+collections-server-only} \ + tests/test-repo.sh{user+system+system-norevokefs} \ tests/test-default-remotes.sh \ tests/test-extensions.sh \ tests/test-bundle.sh{user+system+system-norevokefs} \ @@ -187,7 +187,7 @@ TEST_MATRIX_SOURCE = \ tests/test-update-remote-configuration.sh \ tests/test-override.sh \ tests/test-update-portal.sh \ - tests/test-auth.sh{nocollections+collections} \ + tests/test-auth.sh \ $(NULL) update-test-matrix: diff --git a/tests/test-wrapper.sh b/tests/test-wrapper.sh index a7cc6e10..e2cc6286 100755 --- a/tests/test-wrapper.sh +++ b/tests/test-wrapper.sh @@ -18,18 +18,6 @@ for feature in $(echo $1 | sed "s/^.*@\(.*\).wrap/\1/" | tr "," "\n"); do nodeltas) export USE_DELTAS=no ;; - nocollections) - export USE_COLLECTIONS_IN_SERVER=no - export USE_COLLECTIONS_IN_CLIENT=no - ;; - collections) - export USE_COLLECTIONS_IN_SERVER=yes - export USE_COLLECTIONS_IN_CLIENT=yes - ;; - collections-server-only) - export USE_COLLECTIONS_IN_SERVER=yes - export USE_COLLECTIONS_IN_CLIENT=no - ;; labels) export USE_OCI_LABELS=yes ;; From 336a127f5515f69c7d5c6aa7943076035c59bf84 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 17 Mar 2020 17:19:36 +0100 Subject: [PATCH 09/36] Support sideloaded repos This adds a xa.sideload-collection-id option to the remote configuration and a global xa.sideload-repos option (which is a list of paths to local repos). When resolving or listing refs, if we fail to download the real remote summary (i.e. we're offline) then we instead look into the configured sideloaded repos for refs that match ref and the sideloaded collection id for the remote. For the transaction to resolve the ref we need more metadata. In the regular summary case we use the metadata from the summary, but that is not available in the (partial) summary in the sideload repo, so there we load the actual commit object and use the data from there. (The ostree-metadata branch is not used/needed.) This actually also fixes a longstanding issue when you "flatpak update --checksum=XYZ" because we now handle this correctly by downloading the commit object from the remote. Before we used the metadata in the summary which is not right for non-HEAD commits. To handle the sideloading we record the path to the sideload repo when sideloading and pass the url to the repo as the remote name when pulling, which will do a direct local pull. We avoid using sideloaded refs when offline if the timestamp in the commits is older than what is already installed locally. --- app/flatpak-builtins-remote-info.c | 2 +- app/flatpak-builtins-remote-ls.c | 9 +- common/flatpak-dir-private.h | 27 +- common/flatpak-dir.c | 824 +++++++++++++------------- common/flatpak-installation.c | 239 +------- common/flatpak-remote-ref-private.h | 2 +- common/flatpak-remote-ref.c | 4 +- common/flatpak-transaction.c | 142 +++-- system-helper/flatpak-system-helper.c | 8 +- tests/testlibrary.c | 7 - 10 files changed, 536 insertions(+), 728 deletions(-) diff --git a/app/flatpak-builtins-remote-info.c b/app/flatpak-builtins-remote-info.c index e107acd4..fa0d332e 100644 --- a/app/flatpak-builtins-remote-info.c +++ b/app/flatpak-builtins-remote-info.c @@ -150,7 +150,7 @@ flatpak_builtin_remote_info (int argc, char **argv, GCancellable *cancellable, G commit = g_strdup (opt_commit); else { - flatpak_remote_state_lookup_ref (state, ref, &commit, NULL, NULL, error); + flatpak_remote_state_lookup_ref (state, ref, &commit, NULL, NULL, NULL, error); if (commit == NULL) { if (error != NULL && *error == NULL) diff --git a/app/flatpak-builtins-remote-ls.c b/app/flatpak-builtins-remote-ls.c index 4b0040ed..ce64ddf9 100644 --- a/app/flatpak-builtins-remote-ls.c +++ b/app/flatpak-builtins-remote-ls.c @@ -161,8 +161,7 @@ ls_remote (GHashTable *refs_hash, const char **arches, const char *app_runtime, g_hash_table_iter_init (&iter, refs); while (g_hash_table_iter_next (&iter, &key, &value)) { - FlatpakCollectionRef *coll_ref = key; - char *ref = coll_ref->ref_name; + char *ref = key; char *partial_ref; const char *slash = strchr (ref, '/'); @@ -179,8 +178,7 @@ ls_remote (GHashTable *refs_hash, const char **arches, const char *app_runtime, g_hash_table_iter_init (&iter, refs); while (g_hash_table_iter_next (&iter, &key, &value)) { - FlatpakCollectionRef *coll_ref = key; - const char *ref = coll_ref->ref_name; + const char *ref = key; const char *checksum = value; g_auto(GStrv) parts = NULL; @@ -234,8 +232,7 @@ ls_remote (GHashTable *refs_hash, const char **arches, const char *app_runtime, strcmp (arches[0], parts[2]) != 0) { g_autofree char *alt_arch_ref = g_strconcat (parts[0], "/", parts[1], "/", arches[0], "/", parts[3], NULL); - g_autoptr(FlatpakCollectionRef) alt_arch_coll_ref = flatpak_collection_ref_new (coll_ref->collection_id, alt_arch_ref); - if (g_hash_table_lookup (refs, alt_arch_coll_ref)) + if (g_hash_table_lookup (refs, alt_arch_ref)) continue; } diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index 0a493dc4..0c7637c7 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -104,6 +104,10 @@ typedef struct void flatpak_related_free (FlatpakRelated *related); +typedef struct { + OstreeRepo *repo; + GVariant *summary; +} FlatpakSideloadState; /* The remote state represent the state of the remote at a particular time, including the summary file and the metadata (which may be from @@ -114,6 +118,7 @@ typedef struct { char *remote_name; char *collection_id; + char *sideload_collection_id; GVariant *summary; GBytes *summary_sig_bytes; GError *summary_fetch_error; @@ -123,6 +128,7 @@ typedef struct GRegex *deny_refs; int refcount; gint32 default_token_type; + GPtrArray *sideload_repos; } FlatpakRemoteState; FlatpakRemoteState *flatpak_remote_state_ref (FlatpakRemoteState *remote_state); @@ -136,8 +142,9 @@ gboolean flatpak_remote_state_allow_ref (FlatpakRemoteState *self, gboolean flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, const char *ref, char **out_checksum, - gboolean *out_has_info, + guint64 *out_timestamp, VarRefInfoRef *out_info, + GFile **out_sideload_path, GError **error); char **flatpak_remote_state_match_subrefs (FlatpakRemoteState *self, const char *ref); @@ -152,6 +159,11 @@ gboolean flatpak_remote_state_lookup_sparse_cache (FlatpakRemoteState *self, const char *ref, VarMetadataRef *out_metadata, GError **error); +GVariant *flatpak_remote_state_load_ref_commit (FlatpakRemoteState *self, + FlatpakDir *dir, + const char *ref, + const char *commit, + GError **error); G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakDir, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakDeploy, g_object_unref) @@ -561,8 +573,8 @@ gboolean flatpak_dir_pull (FlatpakDir *self, FlatpakRemoteState *state, const char *ref, const char *opt_rev, - const OstreeRepoFinderResult * const *results, const char **subpaths, + GFile *sideload_repo, GBytes *require_metadata, const char *token, OstreeRepo *repo, @@ -669,6 +681,7 @@ gboolean flatpak_dir_install (FlatpakDir *self, const char *opt_commit, const char **subpaths, const char **previous_ids, + GFile *sideload_repo, GBytes *require_metadata, const char *token, OstreeAsyncProgress *progress, @@ -700,7 +713,6 @@ char * flatpak_dir_check_for_update (FlatpakDir *self, const char *checksum_or_latest, const char **opt_subpaths, gboolean no_pull, - OstreeRepoFinderResult ***out_results, GCancellable *cancellable, GError **error); gboolean flatpak_dir_update (FlatpakDir *self, @@ -713,9 +725,9 @@ gboolean flatpak_dir_update (FlatpakDir *self, FlatpakRemoteState *state, const char *ref, const char *checksum_or_latest, - const OstreeRepoFinderResult * const *results, const char **opt_subpaths, const char **opt_previous_ids, + GFile *sideload_repo, GBytes *require_metadata, const char *token, OstreeAsyncProgress *progress, @@ -804,8 +816,7 @@ gboolean flatpak_dir_create_suggested_remote_for_ref_file (FlatpakDir *self, GBytes *data, GError **error); char *flatpak_dir_find_remote_by_uri (FlatpakDir *self, - const char *uri, - const char *collection_id); + const char *uri); gboolean flatpak_dir_has_remote (FlatpakDir *self, const char *remote_name, GError **error); @@ -826,6 +837,7 @@ gboolean flatpak_dir_remove_remote (FlatpakDir *self, const char *remote_name, GCancellable *cancellable, GError **error); +char ** flatpak_dir_get_sideload_repo_paths (FlatpakDir *self); char ** flatpak_dir_list_remote_config_keys (FlatpakDir *self, const char *remote_name); char *flatpak_dir_get_remote_title (FlatpakDir *self, @@ -949,7 +961,8 @@ gboolean flatpak_dir_find_latest_rev (FlatpakDir *self, const char *ref, const char *checksum_or_latest, char **out_rev, - OstreeRepoFinderResult ***out_results, + guint64 *out_timestamp, + GFile **out_sideload_path, GCancellable *cancellable, GError **error); GPtrArray * flatpak_dir_find_remote_auto_install_refs (FlatpakDir *self, diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index 2f387285..1140986e 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -67,6 +67,7 @@ #include #endif +#define OSTREE_COMMIT_TIMESTAMP "ostree.commit.timestamp" #define NO_SYSTEM_HELPER ((FlatpakSystemHelper *) (gpointer) 1) @@ -137,6 +138,12 @@ static gboolean flatpak_dir_lookup_remote_filter (FlatpakDir *self, GRegex **allow_regex, GRegex **deny_regex, GError **error); +static GBytes * flatpak_dir_fetch_remote_object (FlatpakDir *self, + const char *remote_name, + const char *checksum, + const char *type, + GCancellable *cancellable, + GError **error); static void ensure_soup_session (FlatpakDir *self); @@ -270,6 +277,13 @@ get_config_dir_location (void) return (const char *) path; } +static void +flatpak_sideload_state_free (FlatpakSideloadState *sideload_state) +{ + g_object_unref (sideload_state->repo); + g_variant_unref (sideload_state->summary); + g_free (sideload_state); +} static FlatpakRemoteState * flatpak_remote_state_new (void) @@ -277,6 +291,7 @@ flatpak_remote_state_new (void) FlatpakRemoteState *state = g_new0 (FlatpakRemoteState, 1); state->refcount = 1; + state->sideload_repos = g_ptr_array_new_with_free_func ((GDestroyNotify)flatpak_sideload_state_free); return state; } @@ -298,6 +313,7 @@ flatpak_remote_state_unref (FlatpakRemoteState *remote_state) { g_free (remote_state->remote_name); g_free (remote_state->collection_id); + g_free (remote_state->sideload_collection_id); g_clear_pointer (&remote_state->summary, g_variant_unref); g_clear_pointer (&remote_state->summary_sig_bytes, g_bytes_unref); g_clear_error (&remote_state->summary_fetch_error); @@ -305,6 +321,7 @@ flatpak_remote_state_unref (FlatpakRemoteState *remote_state) g_clear_error (&remote_state->metadata_fetch_error); g_clear_pointer (&remote_state->allow_refs, g_regex_unref); g_clear_pointer (&remote_state->deny_refs, g_regex_unref); + g_clear_pointer (&remote_state->sideload_repos, g_ptr_array_unref); g_free (remote_state); } @@ -351,15 +368,24 @@ flatpak_remote_state_allow_ref (FlatpakRemoteState *self, return flatpak_filters_allow_ref (self->allow_refs, self->deny_refs, ref); } -/* Returns TRUE if the ref is found in the summary or cache. out_checksum and - * out_variant are not guaranteed to be set even when the ref is found. - * Only use out_info if out_has_info is set. */ + +static guint64 +get_timestamp_from_ref_info (VarRefInfoRef info) +{ + VarMetadataRef metadata = var_ref_info_get_metadata (info); + return GUINT64_FROM_BE(var_metadata_lookup_uint64 (metadata, OSTREE_COMMIT_TIMESTAMP, 0)); + } + +/* Returns TRUE if the ref is found in the summary or cache. + * out_checksum and out_variant are only set when the ref is found. + */ gboolean flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, const char *ref, char **out_checksum, - gboolean *out_has_info, + guint64 *out_timestamp, VarRefInfoRef *out_info, + GFile **out_sideload_path, GError **error) { if (!flatpak_remote_state_allow_ref (self, ref)) @@ -369,37 +395,89 @@ flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, ref, self->remote_name); } - if (self->collection_id == NULL || self->summary != NULL) + /* If there is a summary we use it for metadata and for latest. We may later install from a sideloaded source though */ + if (self->summary != NULL) { - if (!flatpak_remote_state_ensure_summary (self, error)) - return FALSE; + VarRefInfoRef info; + g_autofree char *checksum = NULL; - if (out_has_info) - *out_has_info = TRUE; - if (!flatpak_summary_lookup_ref (self->summary, self->collection_id, ref, out_checksum, out_info)) + if (!flatpak_summary_lookup_ref (self->summary, self->collection_id, ref, &checksum, &info)) + return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, + _("No such ref '%s' in remote %s"), + ref, self->remote_name); + + /* Even if its available in the summary we want to install it from a sideload repo if available */ + + if (out_sideload_path) { - if (self->collection_id != NULL) - return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, - _("No such ref (%s, %s) in remote %s"), - self->collection_id, ref, self->remote_name); - else - return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, - _("No such ref '%s' in remote %s"), - ref, self->remote_name); + g_autoptr(GFile) found_sideload_path = NULL; + + for (int i = 0; i < self->sideload_repos->len; i++) + { + FlatpakSideloadState *ss = g_ptr_array_index (self->sideload_repos, i); + OstreeRepoCommitState commit_state; + + if (ostree_repo_load_commit (ss->repo, checksum, NULL, &commit_state, NULL) && + commit_state == OSTREE_REPO_COMMIT_STATE_NORMAL) + { + found_sideload_path = g_object_ref (ostree_repo_get_path (ss->repo)); + break; + } + } + + *out_sideload_path = g_steal_pointer (&found_sideload_path); } + + if (out_info) + *out_info = info; + if (out_checksum) + *out_checksum = g_steal_pointer (&checksum); + if (out_timestamp) + *out_timestamp = get_timestamp_from_ref_info (info); } else { - if (!flatpak_remote_state_ensure_metadata (self, error)) - return FALSE; + g_autofree char *latest_checksum = NULL; + guint64 latest_timestamp = 0; + VarRefInfoRef latest_sideload_info; + g_autoptr(GFile) latest_sideload_path = NULL; - if (!flatpak_remote_state_lookup_cache (self, ref, NULL, NULL, NULL, NULL, error)) - return FALSE; + for (int i = 0; i < self->sideload_repos->len; i++) + { + FlatpakSideloadState *ss = g_ptr_array_index (self->sideload_repos, i); + g_autofree char *sideload_checksum = NULL; + VarRefInfoRef sideload_info; - if (out_checksum) - *out_checksum = NULL; - if (out_has_info) - *out_has_info = FALSE; + if (flatpak_summary_lookup_ref (ss->summary, self->sideload_collection_id, ref, &sideload_checksum, &sideload_info)) + { + guint64 timestamp = get_timestamp_from_ref_info (sideload_info); + + if (latest_checksum == NULL || latest_timestamp < timestamp) + { + g_free (latest_checksum); + latest_checksum = g_steal_pointer (&sideload_checksum); + latest_timestamp = timestamp; + latest_sideload_info = sideload_info; + g_set_object (&latest_sideload_path, ostree_repo_get_path (ss->repo)); + } + } + } + + if (latest_checksum) + { + if (out_checksum) + *out_checksum = g_steal_pointer (&latest_checksum); + if (out_timestamp) + *out_timestamp = latest_timestamp; + if (out_info) + *out_info = latest_sideload_info; + if (out_sideload_path) + *out_sideload_path = g_steal_pointer (&latest_sideload_path); + } + else + return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, + _("No such ref '%s' in remote %s"), + ref, self->remote_name); } return TRUE; @@ -434,8 +512,11 @@ flatpak_remote_state_get_cache (FlatpakRemoteState *self, meta = var_metadata_from_gvariant (self->metadata); if (!var_metadata_lookup (meta, "xa.cache", NULL, &cache_vv)) - return flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("No summary or Flatpak cache available for remote %s"), - self->remote_name); + { + flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("No summary or Flatpak cache available for remote %s"), + self->remote_name); + return FALSE; + } /* For stupid historical reasons the xa.cache is double-wrapped in a variant */ cache_v = var_variant_from_variant (cache_vv); @@ -494,6 +575,69 @@ flatpak_remote_state_lookup_cache (FlatpakRemoteState *self, return TRUE; } +/* Tries to load the specified commit object that we resolved from + this remote. This either comes from the already available local + repo, or from one of the sideloading repos, and if not available we + download it from the actual remote. */ +GVariant * +flatpak_remote_state_load_ref_commit (FlatpakRemoteState *self, + FlatpakDir *dir, + const char *ref, + const char *commit, + GError **error) +{ + g_autoptr(GBytes) commit_bytes = NULL; + g_autoptr(GVariant) commit_data = NULL; + g_autoptr(GVariant) commit_metadata = NULL; + + /* First try local availability */ + if (ostree_repo_load_commit (dir->repo, commit, &commit_data, NULL, NULL)) + return g_steal_pointer (&commit_data); + + for (int i = 0; i < self->sideload_repos->len; i++) + { + FlatpakSideloadState *ss = g_ptr_array_index (self->sideload_repos, i); + + if (ostree_repo_load_commit (ss->repo, commit, &commit_data, NULL, NULL)) + return g_steal_pointer (&commit_data); + } + + commit_bytes = flatpak_dir_fetch_remote_object (dir, self->remote_name, + commit, "commit", + NULL, error); + if (commit_bytes == NULL) + return NULL; + + commit_data = g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_COMMIT_GVARIANT_FORMAT, + commit_bytes, FALSE)); + + /* We downloaded this without validating the signature, so we do some basic verification + of it. However, the signature will be checked when the download is done, and the final + metadata is compared to what we got here, so its pretty ok to use it for resolving + the transaction op. However, we do some basic checks. */ + if (!ostree_validate_structureof_commit (commit_data, error)) + return NULL; + + commit_metadata = g_variant_get_child_value (commit_data, 0); + if (ref != NULL) + { + const char *xa_ref = NULL; + g_autofree const char **commit_refs = NULL; + + if ((g_variant_lookup (commit_metadata, "xa.ref", "&s", &xa_ref) && + g_strcmp0 (xa_ref, ref) != 0) || + (g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_REF_BINDING, "^a&s", &commit_refs) && + !g_strv_contains ((const char * const *) commit_refs, ref))) + { + flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("Commit has no requested ref ‘%s’ in ref binding metadata"), ref); + return NULL; + } + } + + return g_steal_pointer (&commit_data); +} + + gboolean flatpak_remote_state_lookup_sparse_cache (FlatpakRemoteState *self, const char *ref, @@ -3647,15 +3791,6 @@ static gboolean repo_get_remote_collection_id (OstreeRepo *repo, char **collection_id_out, GError **error); -static void -async_result_cb (GObject *obj, - GAsyncResult *result, - gpointer user_data) -{ - GAsyncResult **result_out = user_data; - - *result_out = g_object_ref (result); -} gboolean flatpak_dir_find_latest_rev (FlatpakDir *self, @@ -3663,7 +3798,8 @@ flatpak_dir_find_latest_rev (FlatpakDir *self, const char *ref, const char *checksum_or_latest, char **out_rev, - OstreeRepoFinderResult ***out_results, + guint64 *out_timestamp, + GFile **out_sideload_path, GCancellable *cancellable, GError **error) { @@ -3671,7 +3807,7 @@ flatpak_dir_find_latest_rev (FlatpakDir *self, g_return_val_if_fail (out_rev != NULL, FALSE); - if (!flatpak_remote_state_lookup_ref (state, ref, &latest_rev, NULL, NULL, error)) + if (!flatpak_remote_state_lookup_ref (state, ref, &latest_rev, out_timestamp, NULL, out_sideload_path, error)) return FALSE; if (latest_rev == NULL) return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, @@ -3940,6 +4076,8 @@ flatpak_dir_update_appstream (FlatpakDir *self, g_autoptr(GError) first_error = NULL; g_autoptr(GError) second_error = NULL; g_autoptr(FlatpakRemoteState) state = NULL; + g_autofree char *appstream_commit = NULL; + g_autoptr(GFile) appstream_sideload_path = NULL; const char *installation; gboolean is_oci; @@ -3958,6 +4096,22 @@ flatpak_dir_update_appstream (FlatpakDir *self, if (state == NULL) return FALSE; + used_branch = new_branch; + if (!is_oci) + { + if (!flatpak_dir_find_latest_rev (self, state, used_branch, NULL, &appstream_commit, NULL, &appstream_sideload_path, cancellable, &first_error)) + { + used_branch = old_branch; + if (!flatpak_dir_find_latest_rev (self, state, used_branch, NULL, &appstream_commit, NULL, &appstream_sideload_path, cancellable, &second_error)) + { + g_prefix_error (&first_error, "Error updating appstream2: "); + g_prefix_error (&second_error, "Error updating appstream: "); + g_propagate_prefixed_error (error, g_steal_pointer (&second_error), "%s; ", first_error->message); + return FALSE; + } + } + } + if (flatpak_dir_use_system_helper (self, NULL)) { g_auto(GLnxLockFile) child_repo_lock = { 0, }; @@ -4011,23 +4165,12 @@ flatpak_dir_update_appstream (FlatpakDir *self, if (child_repo == NULL) return FALSE; - /* No need to use an existing OstreeRepoFinderResult array, since - * appstream updates do not need to be atomic wrt other updates. */ - used_branch = new_branch; - if (!flatpak_dir_pull (self, state, used_branch, NULL, NULL, NULL, NULL, NULL, + if (!flatpak_dir_pull (self, state, used_branch, appstream_commit, NULL, appstream_sideload_path, NULL, NULL, child_repo, FLATPAK_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_MIRROR, - progress, cancellable, &first_error)) + progress, cancellable, error)) { - used_branch = old_branch; - if (!flatpak_dir_pull (self, state, used_branch, NULL, NULL, NULL, NULL, NULL, - child_repo, FLATPAK_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_MIRROR, - progress, cancellable, &second_error)) - { - g_prefix_error (&first_error, "Error updating appstream2: "); - g_prefix_error (&second_error, "Error updating appstream: "); - g_propagate_prefixed_error (error, g_steal_pointer (&second_error), "%s; ", first_error->message); - return FALSE; - } + g_prefix_error (&first_error, "Error updating appstream: "); + return FALSE; } if (!child_repo_ensure_summary (child_repo, state, cancellable, error)) @@ -4068,23 +4211,13 @@ flatpak_dir_update_appstream (FlatpakDir *self, error); } - /* No need to use an existing OstreeRepoFinderResult array, since - * appstream updates do not need to be atomic wrt other updates. */ - used_branch = new_branch; - if (!flatpak_dir_pull (self, state, used_branch, NULL, NULL, NULL, NULL, NULL, NULL, + + if (!flatpak_dir_pull (self, state, used_branch, appstream_commit, NULL, appstream_sideload_path, NULL, NULL, NULL, FLATPAK_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_NONE, progress, - cancellable, &first_error)) + cancellable, error)) { - used_branch = old_branch; - if (!flatpak_dir_pull (self, state, used_branch, NULL, NULL, NULL, NULL, NULL, NULL, - FLATPAK_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_NONE, progress, - cancellable, &second_error)) - { - g_prefix_error (&first_error, "Error updating appstream2: "); - g_prefix_error (&second_error, "Error updating appstream: "); - g_propagate_prefixed_error (error, g_steal_pointer (&second_error), "%s; ", first_error->message); - return FALSE; - } + g_prefix_error (&first_error, "Error updating appstream: "); + return FALSE; } if (!flatpak_repo_resolve_rev (self->repo, state->collection_id, remote, used_branch, TRUE, @@ -4120,6 +4253,24 @@ repo_get_remote_collection_id (OstreeRepo *repo, return TRUE; } +static gboolean +repo_get_remote_sideload_collection_id (OstreeRepo *repo, + const char *remote_name, + char **collection_id_out, + GError **error) +{ + if (collection_id_out != NULL) + { + if (!ostree_repo_get_remote_option (repo, remote_name, "xa.sideload-collection-id", + NULL, collection_id_out, error)) + return FALSE; + if (*collection_id_out != NULL && **collection_id_out == '\0') + g_clear_pointer (collection_id_out, g_free); + } + + return TRUE; +} + /* Get options for the OSTree pull operation which can be shared between * collection-based and normal pulls. Update @builder in place. */ static void @@ -4200,8 +4351,9 @@ repo_pull (OstreeRepo *self, const char **dirs_to_pull, const char *ref_to_fetch, const char *rev_to_fetch, /* (nullable) */ + const char *sideload_collection_id, + GFile *sideload_repo, const char *token, - const OstreeRepoFinderResult * const *results_to_fetch, /* (nullable) */ FlatpakPullFlags flatpak_flags, OstreeRepoPullFlags flags, OstreeAsyncProgress *progress, @@ -4213,17 +4365,17 @@ repo_pull (OstreeRepo *self, g_autoptr(GVariant) old_commit = NULL; g_autoptr(GVariant) new_commit = NULL; const char *revs_to_fetch[2]; - gboolean res = FALSE; g_autofree gchar *collection_id = NULL; g_autoptr(GError) dummy_error = NULL; + GVariantBuilder builder; + g_autoptr(GVariant) options = NULL; + const char *refs_to_fetch[2]; + g_autofree char *sideload_url = NULL; /* The ostree fetcher asserts if error is NULL */ if (error == NULL) error = &dummy_error; - /* If @results_to_fetch is set, @rev_to_fetch must be. */ - g_assert (results_to_fetch == NULL || rev_to_fetch != NULL); - /* We always want this on for every type of pull */ flags |= OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES; @@ -4238,126 +4390,31 @@ repo_pull (OstreeRepo *self, !ostree_repo_load_commit (self, current_checksum, &old_commit, NULL, error)) return FALSE; - if (collection_id != NULL) + /* Pull options */ + g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); + get_common_pull_options (&builder, ref_to_fetch, token, dirs_to_pull, current_checksum, + force_disable_deltas, flags, progress); + + if (sideload_repo) { - g_autoptr(GAsyncResult) find_result = NULL, pull_result = NULL; - g_auto(OstreeRepoFinderResultv) results = NULL; - OstreeCollectionRef collection_ref; - OstreeCollectionRef *collection_refs_to_fetch[2]; - guint32 update_interval = 0; - g_autoptr(GMainContextPopDefault) context = NULL; - g_autoptr(FlatpakAsyncProgressChained) chained_progress = NULL; + GVariantBuilder colref_builder; - context = flatpak_main_context_new_default (); - chained_progress = flatpak_progress_chain (progress); + sideload_url = g_file_get_uri (sideload_repo); - if (results_to_fetch == NULL) - { - GVariantBuilder find_builder; - g_autoptr(GVariant) find_options = NULL; + g_debug ("Sideloading %s from %s during", ref_to_fetch, sideload_url); - /* Find options */ - g_variant_builder_init (&find_builder, G_VARIANT_TYPE ("a{sv}")); + g_assert (sideload_collection_id != NULL); - if (force_disable_deltas) - { - g_variant_builder_add (&find_builder, "{s@v}", "disable-static-deltas", - g_variant_new_variant (g_variant_new_boolean (TRUE))); - } + g_variant_builder_init (&colref_builder, G_VARIANT_TYPE ("a(sss)")); + g_variant_builder_add (&colref_builder, "(sss)", sideload_collection_id, ref_to_fetch, rev_to_fetch); - collection_ref.collection_id = collection_id; - collection_ref.ref_name = (char *) ref_to_fetch; - - collection_refs_to_fetch[0] = &collection_ref; - collection_refs_to_fetch[1] = NULL; - - if (progress != NULL) - update_interval = GPOINTER_TO_UINT (g_object_get_data (G_OBJECT (progress), "update-interval")); - if (update_interval == 0) - update_interval = FLATPAK_DEFAULT_UPDATE_INTERVAL_MS; - - g_variant_builder_add (&find_builder, "{s@v}", "update-frequency", - g_variant_new_variant (g_variant_new_uint32 (update_interval))); - - if (rev_to_fetch != NULL) - { - g_variant_builder_add (&find_builder, "{s@v}", "override-commit-ids", - g_variant_new_variant (g_variant_new_strv (&rev_to_fetch, 1))); - } - - find_options = g_variant_ref_sink (g_variant_builder_end (&find_builder)); - - ostree_repo_find_remotes_async (self, (const OstreeCollectionRef * const *) collection_refs_to_fetch, - find_options, - NULL /* default finders */, - chained_progress, cancellable, - async_result_cb, &find_result); - - while (find_result == NULL) - g_main_context_iteration (context, TRUE); - - results = ostree_repo_find_remotes_finish (self, find_result, error); - results_to_fetch = (const OstreeRepoFinderResult * const *) results; - } - - if (results_to_fetch != NULL && results_to_fetch[0] == NULL) - { - flatpak_fail (error, _("No remotes found which provide the ref (%s, %s)"), - collection_ref.collection_id, collection_ref.ref_name); - res = FALSE; - } - else if (results_to_fetch != NULL) - { - GVariantBuilder pull_builder, ref_keyring_map_builder; - g_autoptr(GVariant) pull_options = NULL; - - /* Pull options */ - g_variant_builder_init (&pull_builder, G_VARIANT_TYPE ("a{sv}")); - get_common_pull_options (&pull_builder, ref_to_fetch, token, dirs_to_pull, current_checksum, - force_disable_deltas, flags, progress); - - /* Ensure the results are signed with the GPG keys associated with the correct remote */ - g_variant_builder_init (&ref_keyring_map_builder, G_VARIANT_TYPE ("a(sss)")); - g_variant_builder_add (&ref_keyring_map_builder, "(sss)", collection_id, ref_to_fetch, remote_name); - g_variant_builder_add (&pull_builder, "{s@v}", "ref-keyring-map", - g_variant_new_variant (g_variant_builder_end (&ref_keyring_map_builder))); - - pull_options = g_variant_ref_sink (g_variant_builder_end (&pull_builder)); - - ostree_repo_pull_from_remotes_async (self, results_to_fetch, - pull_options, chained_progress, - cancellable, async_result_cb, - &pull_result); - - while (pull_result == NULL) - g_main_context_iteration (context, TRUE); - - res = ostree_repo_pull_from_remotes_finish (self, pull_result, error); - } - else - res = FALSE; + g_variant_builder_add (&builder, "{s@v}", "collection-refs", + g_variant_new_variant (g_variant_builder_end (&colref_builder))); + g_variant_builder_add (&builder, "{s@v}", "override-remote-name", + g_variant_new_variant (g_variant_new_string (remote_name))); } else - res = FALSE; - - if (!res) { - if (error != NULL && *error != NULL) - g_debug ("Failed to pull using find-remotes; falling back to normal pull: %s", (*error)->message); - g_clear_error (error); - } - - if (!res) - { - GVariantBuilder builder; - g_autoptr(GVariant) options = NULL; - const char *refs_to_fetch[2]; - - /* Pull options */ - g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); - get_common_pull_options (&builder, ref_to_fetch, token, dirs_to_pull, current_checksum, - force_disable_deltas, flags, progress); - refs_to_fetch[0] = ref_to_fetch; refs_to_fetch[1] = NULL; g_variant_builder_add (&builder, "{s@v}", "refs", @@ -4367,14 +4424,15 @@ repo_pull (OstreeRepo *self, revs_to_fetch[1] = NULL; g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", g_variant_new_variant (g_variant_new_strv ((const char * const *) revs_to_fetch, -1))); - - options = g_variant_ref_sink (g_variant_builder_end (&builder)); - - if (!ostree_repo_pull_with_options (self, remote_name, options, - progress, cancellable, error)) - return translate_ostree_repo_pull_errors (error); } + options = g_variant_ref_sink (g_variant_builder_end (&builder)); + + if (!ostree_repo_pull_with_options (self, + sideload_url ? sideload_url : remote_name, + options, progress, cancellable, error)) + return translate_ostree_repo_pull_errors (error); + if (old_commit && (flatpak_flags & FLATPAK_PULL_FLAGS_ALLOW_DOWNGRADE) == 0) { @@ -4431,7 +4489,6 @@ flatpak_dir_setup_extra_data (FlatpakDir *self, const char *ref, const char *rev, const char *token, - const OstreeRepoFinderResult * const *results, FlatpakPullFlags flatpak_flags, OstreeAsyncProgress *progress, GCancellable *cancellable, @@ -4442,8 +4499,6 @@ flatpak_dir_setup_extra_data (FlatpakDir *self, guint64 n_extra_data; guint64 total_download_size; - /* If @results is set, @rev must be. */ - g_assert (results == NULL || rev != NULL); /* ostree-metadata and appstreams never have extra data, so ignore those */ if (g_str_has_prefix ("app/", ref) || g_str_has_prefix ("runtime/", ref)) @@ -4468,8 +4523,8 @@ flatpak_dir_setup_extra_data (FlatpakDir *self, NULL, ref, rev, + NULL, NULL, token, - results, flatpak_flags, OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY, NULL, @@ -4805,14 +4860,13 @@ flatpak_dir_mirror_oci (FlatpakDir *self, g_autofree char *latest_rev = NULL; VarRefInfoRef latest_rev_info; VarMetadataRef metadata; - gboolean has_info; const char *oci_repository = NULL; gboolean res; /* We use the summary so that we can reuse any cached json */ - if (!flatpak_remote_state_lookup_ref (state, ref, &latest_rev, &has_info, &latest_rev_info, error)) + if (!flatpak_remote_state_lookup_ref (state, ref, &latest_rev, NULL, &latest_rev_info, NULL, error)) return FALSE; - if (latest_rev == NULL || !has_info) + if (latest_rev == NULL) return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, _("Couldn't find latest checksum for ref %s in remote %s"), ref, state->remote_name); @@ -4877,15 +4931,14 @@ flatpak_dir_pull_oci (FlatpakDir *self, g_autofree char *latest_alt_commit = NULL; VarMetadataRef metadata; g_autofree char *latest_rev = NULL; - gboolean has_info; G_GNUC_UNUSED g_autofree char *latest_commit = flatpak_dir_read_latest (self, state->remote_name, ref, &latest_alt_commit, cancellable, NULL); g_autofree char *name = NULL; /* We use the summary so that we can reuse any cached json */ - if (!flatpak_remote_state_lookup_ref (state, ref, &latest_rev, &has_info, &latest_rev_info, error)) + if (!flatpak_remote_state_lookup_ref (state, ref, &latest_rev, NULL, &latest_rev_info, NULL, error)) return FALSE; - if (latest_rev == NULL || !has_info) + if (latest_rev == NULL) return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, _("Couldn't find latest checksum for ref %s in remote %s"), ref, state->remote_name); @@ -4961,8 +5014,8 @@ flatpak_dir_pull (FlatpakDir *self, FlatpakRemoteState *state, const char *ref, const char *opt_rev, - const OstreeRepoFinderResult * const *opt_results, const char **subpaths, + GFile *sideload_repo, GBytes *require_metadata, const char *token, OstreeRepo *repo, @@ -4976,15 +5029,10 @@ flatpak_dir_pull (FlatpakDir *self, g_autofree char *rev = NULL; g_autofree char *url = NULL; g_autoptr(GPtrArray) subdirs_arg = NULL; - g_auto(OstreeRepoFinderResultv) allocated_results = NULL; - const OstreeRepoFinderResult * const *results; g_auto(GLnxLockFile) lock = { 0, }; g_autofree char *name = NULL; g_autofree char *current_checksum = NULL; - /* If @opt_results is set, @opt_rev must be. */ - g_return_val_if_fail (opt_results == NULL || opt_rev != NULL, FALSE); - if (!flatpak_dir_ensure_repo (self, cancellable, error)) return FALSE; @@ -5018,17 +5066,14 @@ flatpak_dir_pull (FlatpakDir *self, if (opt_rev != NULL) { rev = g_strdup (opt_rev); - results = opt_results; } else { - flatpak_remote_state_lookup_ref (state, ref, &rev, NULL, NULL, error); + flatpak_remote_state_lookup_ref (state, ref, &rev, NULL, NULL, NULL, error); if (rev == NULL && error != NULL && *error == NULL) flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, _("Couldn't find latest checksum for ref %s in remote %s"), ref, state->remote_name); - results = NULL; - if (rev == NULL) { g_assert (error == NULL || *error != NULL); @@ -5059,7 +5104,7 @@ flatpak_dir_pull (FlatpakDir *self, /* Setup extra data information before starting to pull, so we can have precise * progress reports */ if (!flatpak_dir_setup_extra_data (self, repo, state->remote_name, - ref, rev, token, results, + ref, rev, token, flatpak_flags, progress, cancellable, @@ -5075,7 +5120,7 @@ flatpak_dir_pull (FlatpakDir *self, if (!repo_pull (repo, state->remote_name, subdirs_arg ? (const char **) subdirs_arg->pdata : NULL, - ref, rev, token, results, flatpak_flags, flags, + ref, rev, state->sideload_collection_id, sideload_repo, token, flatpak_flags, flags, progress, cancellable, error)) { @@ -8325,6 +8370,7 @@ flatpak_dir_install (FlatpakDir *self, const char *opt_commit, const char **opt_subpaths, const char **opt_previous_ids, + GFile *sideload_repo, GBytes *require_metadata, const char *token, OstreeAsyncProgress *progress, @@ -8486,7 +8532,7 @@ flatpak_dir_install (FlatpakDir *self, flatpak_flags |= FLATPAK_PULL_FLAGS_SIDELOAD_EXTRA_DATA; - if (!flatpak_dir_pull (self, state, ref, opt_commit, NULL, subpaths, require_metadata, token, + if (!flatpak_dir_pull (self, state, ref, opt_commit, subpaths, sideload_repo, require_metadata, token, child_repo, flatpak_flags, OSTREE_REPO_PULL_FLAGS_MIRROR, @@ -8562,7 +8608,7 @@ flatpak_dir_install (FlatpakDir *self, if (!no_pull) { - if (!flatpak_dir_pull (self, state, ref, opt_commit, NULL, opt_subpaths, require_metadata, token, NULL, + if (!flatpak_dir_pull (self, state, ref, opt_commit, opt_subpaths, sideload_repo, require_metadata, token, NULL, flatpak_flags, OSTREE_REPO_PULL_FLAGS_NONE, progress, cancellable, error)) return FALSE; @@ -8956,7 +9002,6 @@ flatpak_dir_check_for_update (FlatpakDir *self, const char *checksum_or_latest, const char **opt_subpaths, gboolean no_pull, - OstreeRepoFinderResult ***out_results, GCancellable *cancellable, GError **error) { @@ -8976,7 +9021,7 @@ flatpak_dir_check_for_update (FlatpakDir *self, else { if (!flatpak_dir_find_latest_rev (self, state, ref, checksum_or_latest, &latest_rev, - out_results, cancellable, error)) + NULL, NULL, cancellable, error)) return NULL; } @@ -9004,9 +9049,9 @@ flatpak_dir_update (FlatpakDir *self, FlatpakRemoteState *state, const char *ref, const char *commit, - const OstreeRepoFinderResult * const *results, const char **opt_subpaths, const char **opt_previous_ids, + GFile *sideload_repo, GBytes *require_metadata, const char *token, OstreeAsyncProgress *progress, @@ -9020,8 +9065,7 @@ flatpak_dir_update (FlatpakDir *self, g_autofree const char **old_subpaths = NULL; gboolean is_oci; - /* This and @results are calculated in check_for_update. @results will be - * %NULL if we don’t support collections. */ + /* This is calculated in check_for_update */ g_assert (commit != NULL); flatpak_flags = FLATPAK_PULL_FLAGS_DOWNLOAD_EXTRA_DATA; @@ -9180,7 +9224,7 @@ flatpak_dir_update (FlatpakDir *self, } flatpak_flags |= FLATPAK_PULL_FLAGS_SIDELOAD_EXTRA_DATA; - if (!flatpak_dir_pull (self, state, ref, commit, results, subpaths, require_metadata, token, + if (!flatpak_dir_pull (self, state, ref, commit, subpaths, sideload_repo, require_metadata, token, child_repo, flatpak_flags, OSTREE_REPO_PULL_FLAGS_MIRROR, progress, cancellable, error)) @@ -9252,7 +9296,7 @@ flatpak_dir_update (FlatpakDir *self, if (!no_pull) { - if (!flatpak_dir_pull (self, state, ref, commit, results, subpaths, require_metadata, token, + if (!flatpak_dir_pull (self, state, ref, commit, subpaths, sideload_repo, require_metadata, token, NULL, flatpak_flags, OSTREE_REPO_PULL_FLAGS_NONE, progress, cancellable, error)) return FALSE; @@ -10603,6 +10647,8 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, return NULL; if (!repo_get_remote_collection_id (self->repo, remote_or_uri, &state->collection_id, error)) return NULL; + if (!repo_get_remote_sideload_collection_id (self->repo, remote_or_uri, &state->sideload_collection_id, error)) + return NULL; if (!flatpak_dir_lookup_remote_filter (self, remote_or_uri, FALSE, NULL, &state->allow_refs, &state->deny_refs, error)) return NULL; @@ -10653,8 +10699,8 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, { if (optional && !g_cancellable_is_cancelled (cancellable)) { + g_debug ("Failed to download optional summary: %s", local_error->message); state->summary_fetch_error = g_steal_pointer (&local_error); - g_debug ("Failed to download optional summary"); } else { @@ -10664,6 +10710,33 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, } } + if (state->sideload_collection_id) + { + g_auto(GStrv) sideload_paths = flatpak_dir_get_sideload_repo_paths (self); + for (int i = 0; sideload_paths != NULL && sideload_paths[i] != NULL; i++) + { + g_autoptr(GFile) f = g_file_new_for_path (sideload_paths[i]); + g_autoptr(GFile) summary_path = g_file_get_child (f, "summary"); + g_autoptr(GMappedFile) mfile = NULL; + g_autoptr(OstreeRepo) sideload_repo = ostree_repo_new (f); + + /* TODO: The sideloadstates are duplicated for each remote with the collection id set, we should maybe reuse them */ + + mfile = g_mapped_file_new (flatpak_file_get_path_cached (summary_path), FALSE, NULL); + if (mfile != NULL && ostree_repo_open (sideload_repo, NULL, NULL)) + { + g_autoptr(GBytes) summary_bytes = g_mapped_file_get_bytes (mfile); + FlatpakSideloadState *ss = g_new0 (FlatpakSideloadState, 1); + + ss->repo = g_steal_pointer (&sideload_repo); + ss->summary = g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, summary_bytes, TRUE)); + g_ptr_array_add (state->sideload_repos, ss); + + g_debug ("Using sideloaded repo %s for remote %s", sideload_paths[i], remote_or_uri); + } + } + } + if (state->summary != NULL) /* In the optional case we might not have a summary */ state->metadata = g_variant_get_child_value (state->summary, 1); @@ -10750,12 +10823,13 @@ flatpak_dir_remote_has_ref (FlatpakDir *self, return FALSE; } - return flatpak_remote_state_lookup_ref (state, ref, NULL, NULL, NULL, NULL); + return flatpak_remote_state_lookup_ref (state, ref, NULL, NULL, NULL, NULL, NULL); } static void -populate_hash_table_from_refs_map (GHashTable *ret_all_refs, VarRefMapRef ref_map, - const gchar *collection_id, +populate_hash_table_from_refs_map (GHashTable *ret_all_refs, + GHashTable *ref_timestamps, + VarRefMapRef ref_map, FlatpakRemoteState *state) { gsize len, i; @@ -10768,7 +10842,8 @@ populate_hash_table_from_refs_map (GHashTable *ret_all_refs, VarRefMapRef ref_ma const guint8 *csum_bytes; gsize csum_len; VarRefInfoRef info; - FlatpakCollectionRef *ref; + char *ref_name_dup; + guint64 *new_timestamp = NULL; if (!flatpak_remote_state_allow_ref (state, ref_name)) continue; @@ -10779,13 +10854,31 @@ populate_hash_table_from_refs_map (GHashTable *ret_all_refs, VarRefMapRef ref_ma if (csum_len != OSTREE_SHA256_DIGEST_LEN) continue; - ref = flatpak_collection_ref_new (collection_id, ref_name); - g_hash_table_insert (ret_all_refs, ref, ostree_checksum_from_bytes (csum_bytes)); + if (ref_timestamps) + { + guint64 timestamp = get_timestamp_from_ref_info (info); + gpointer value; + + if (g_hash_table_lookup_extended (ref_timestamps, ref_name, NULL, &value)) + { + guint64 *old_timestamp = value; + if (*old_timestamp >= timestamp) + continue; /* New timestamp is older, skip this commit */ + } + + new_timestamp = g_memdup (×tamp, sizeof (guint64)); + } + + ref_name_dup = g_strdup (ref_name); + g_hash_table_replace (ret_all_refs, ref_name_dup, ostree_checksum_from_bytes (csum_bytes)); + if (new_timestamp) + g_hash_table_replace (ref_timestamps, ref_name_dup, new_timestamp); } } -/* This duplicates ostree_repo_remote_list_refs so it can use - * flatpak_remote_state_ensure_summary and get caching. */ +/* This tries to list all available remote refs but also tries to keep + * working when offline, so it looks in sideloaded repos. Also it uses + * in-memory cached summaries which ostree doesn't. */ gboolean flatpak_dir_list_all_remote_refs (FlatpakDir *self, FlatpakRemoteState *state, @@ -10794,80 +10887,57 @@ flatpak_dir_list_all_remote_refs (FlatpakDir *self, GError **error) { g_autoptr(GHashTable) ret_all_refs = NULL; - const gchar *collection_id; VarSummaryRef summary; VarMetadataRef exts; VarRefMapRef ref_map; VarVariantRef v; - ret_all_refs = g_hash_table_new_full (flatpak_collection_ref_hash, - flatpak_collection_ref_equal, - (GDestroyNotify) flatpak_collection_ref_free, - g_free); + /* This is ref->commit */ + ret_all_refs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); - /* If the remote has P2P enabled and we're offline, get the refs list from - * xa.cache in ostree-metadata (although it's inferior to the summary refs - * list in that it lacks checksums). */ - if (state->collection_id != NULL && state->summary == NULL) + if (state->summary != NULL) { - VarCacheRef cache; - gsize i, n; + /* We're online, so report only the refs from the summary */ - if (!flatpak_remote_state_get_cache (state, &cache, error)) + summary = var_summary_from_gvariant (state->summary); + + exts = var_summary_get_metadata (summary); + + /* refs that match the main collection-id */ + ref_map = var_summary_get_ref_map (summary); + populate_hash_table_from_refs_map (ret_all_refs, NULL, ref_map, state); + } + else if (state->sideload_collection_id) + { + g_autoptr(GHashTable) ref_mtimes = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, g_free); /* Keys owned by ret_all_refs */ + + /* No main summary, add just all sideloded refs, with the latest version of each checksum */ + + for (int i = 0; i < state->sideload_repos->len; i++) + { + FlatpakSideloadState *ss = g_ptr_array_index (state->sideload_repos, i); + + summary = var_summary_from_gvariant (ss->summary); + exts = var_summary_get_metadata (summary); + + if (var_metadata_lookup (exts, "ostree.summary.collection-map", NULL, &v)) + { + VarCollectionMapRef map = var_collection_map_from_variant (v); + + if (var_collection_map_lookup (map, state->sideload_collection_id, NULL, &ref_map)) + populate_hash_table_from_refs_map (ret_all_refs, ref_mtimes, ref_map, state); + } + } + } + + if (state->summary == NULL && + g_hash_table_size (ret_all_refs) == 0) + { + /* No sideloaded refs, might as well return the summary error */ + if (!flatpak_remote_state_ensure_summary (state, error)) return FALSE; - - n = var_cache_get_length (cache); - for (i = 0; i < n; i++) - { - g_autoptr(FlatpakCollectionRef) coll_ref = NULL; - VarCacheEntryRef entry; - const char *ref; - - entry = var_cache_get_at (cache, i); - ref = var_cache_entry_get_key (entry); - - if (!flatpak_remote_state_allow_ref (state, ref)) - continue; - - coll_ref = flatpak_collection_ref_new (state->collection_id, ref); - g_hash_table_insert (ret_all_refs, g_steal_pointer (&coll_ref), NULL); - } - - goto out; } - if (!flatpak_remote_state_ensure_summary (state, error)) - return FALSE; - - summary = var_summary_from_gvariant (state->summary); - - exts = var_summary_get_metadata (summary); - - collection_id = var_metadata_lookup_string (exts, "ostree.summary.collection-id", NULL); - - /* refs that match the main collection-id */ - ref_map = var_summary_get_ref_map (summary); - populate_hash_table_from_refs_map (ret_all_refs, ref_map, collection_id, state); - - /* refs that match other collection-ids */ - if (var_metadata_lookup (exts, "ostree.summary.collection-map", NULL, &v)) - { - VarCollectionMapRef map = var_collection_map_from_variant (v); - gsize i, map_len; - - map_len = var_collection_map_get_length (map); - for (i = 0; i < map_len; i++) - { - VarCollectionMapEntryRef entry = var_collection_map_get_at (map, i); - const char *collection_id = var_collection_map_entry_get_key (entry); - VarRefMapRef collection_ref_map; - - collection_ref_map = var_collection_map_entry_get_value (entry); - populate_hash_table_from_refs_map (ret_all_refs, collection_ref_map, collection_id, state); - } - } - -out: *out_all_refs = g_steal_pointer (&ret_all_refs); return TRUE; @@ -10881,7 +10951,6 @@ find_matching_refs (GHashTable *refs, const char *opt_default_branch, const char *opt_arch, const char *opt_default_arch, - const char *opt_collection_id, FlatpakKinds kinds, FindMatchingRefsFlags flags, GError **error) @@ -10917,13 +10986,13 @@ find_matching_refs (GHashTable *refs, g_hash_table_iter_init (&hash_iter, refs); while (g_hash_table_iter_next (&hash_iter, &key, NULL)) { + const char *ref_name = key; g_autofree char *ref = NULL; g_auto(GStrv) parts = NULL; gboolean is_app, is_runtime; - FlatpakCollectionRef *coll_ref = key; /* Unprefix any remote name if needed */ - ostree_parse_refspec (coll_ref->ref_name, NULL, &ref, NULL); + ostree_parse_refspec (ref_name, NULL, &ref, NULL); if (ref == NULL) continue; @@ -10959,9 +11028,6 @@ find_matching_refs (GHashTable *refs, if (opt_branch != NULL && strcmp (opt_branch, parts[3]) != 0) continue; - if (opt_collection_id != NULL && strcmp (opt_collection_id, coll_ref->collection_id)) - continue; - if (opt_name != NULL && strcmp (opt_name, parts[1]) == 0) found_exact_name_match = TRUE; @@ -10972,7 +11038,7 @@ find_matching_refs (GHashTable *refs, found_default_branch_match = TRUE; if (flags & FIND_MATCHING_REFS_FLAGS_KEEP_REMOTE) - g_ptr_array_add (matched_refs, g_strdup (coll_ref->ref_name)); + g_ptr_array_add (matched_refs, g_strdup (ref_name)); else g_ptr_array_add (matched_refs, g_steal_pointer (&ref)); } @@ -11012,7 +11078,6 @@ find_matching_ref (GHashTable *refs, const char *opt_branch, const char *opt_default_branch, const char *opt_arch, - const char *opt_collection_id, FlatpakKinds kinds, GError **error) { @@ -11035,7 +11100,6 @@ find_matching_ref (GHashTable *refs, opt_default_branch, arches[i], NULL, - opt_collection_id, kinds, FIND_MATCHING_REFS_FLAGS_NONE, error); @@ -11102,7 +11166,6 @@ flatpak_dir_find_remote_refs (FlatpakDir *self, GCancellable *cancellable, GError **error) { - g_autofree char *collection_id = NULL; g_autoptr(GHashTable) remote_refs = NULL; g_autoptr(FlatpakRemoteState) state = NULL; GPtrArray *matched_refs; @@ -11115,20 +11178,26 @@ flatpak_dir_find_remote_refs (FlatpakDir *self, &remote_refs, cancellable, error)) return NULL; - collection_id = flatpak_dir_get_remote_collection_id (self, remote); + matched_refs = find_matching_refs (remote_refs, name, opt_branch, opt_default_branch, opt_arch, opt_default_arch, - collection_id, kinds, flags, error); if (matched_refs == NULL) return NULL; + /* If we can't match anything and we had an error downloading (offline?), report that as its more helpful */ + if (matched_refs->len == 0 && state->summary_fetch_error) + { + g_propagate_error (error, g_error_copy (state->summary_fetch_error)); + return NULL; + } + g_ptr_array_add (matched_refs, NULL); return (char **) g_ptr_array_free (matched_refs, FALSE); } @@ -11139,7 +11208,6 @@ find_ref_for_refs_set (GHashTable *refs, const char *opt_branch, const char *opt_default_branch, const char *opt_arch, - const char *collection_id, FlatpakKinds kinds, FlatpakKinds *out_kind, GError **error) @@ -11150,7 +11218,6 @@ find_ref_for_refs_set (GHashTable *refs, opt_branch, opt_default_branch, opt_arch, - collection_id, kinds, &my_error); if (ref == NULL) @@ -11198,7 +11265,6 @@ flatpak_dir_find_remote_ref (FlatpakDir *self, GCancellable *cancellable, GError **error) { - g_autofree char *collection_id = NULL; g_autofree char *remote_ref = NULL; g_autoptr(GHashTable) remote_refs = NULL; g_autoptr(FlatpakRemoteState) state = NULL; @@ -11212,9 +11278,8 @@ flatpak_dir_find_remote_ref (FlatpakDir *self, &remote_refs, cancellable, error)) return NULL; - collection_id = flatpak_dir_get_remote_collection_id (self, remote); remote_ref = find_ref_for_refs_set (remote_refs, name, opt_branch, - opt_default_branch, opt_arch, collection_id, + opt_default_branch, opt_arch, kinds, out_kind, &my_error); if (!remote_ref) { @@ -11236,40 +11301,6 @@ flatpak_dir_find_remote_ref (FlatpakDir *self, return g_steal_pointer (&remote_ref); } -static gboolean -list_collection_refs_from_ostree_repo (OstreeRepo *repo, - const char *refspec_prefix, - const char *opt_collection_id, - GHashTable **out_all_refs, - GCancellable *cancellable, - GError **error) -{ - GHashTableIter iter; - gpointer key; - GHashTable *coll_refs = NULL; - g_autoptr(GHashTable) refs = NULL; - - /* FIXME: Use ostree_repo_list_collection_refs when it's public */ - if (!ostree_repo_list_refs (repo, refspec_prefix, &refs, cancellable, error)) - return FALSE; - - coll_refs = g_hash_table_new_full (flatpak_collection_ref_hash, - flatpak_collection_ref_equal, - (GDestroyNotify) flatpak_collection_ref_free, - NULL); - - g_hash_table_iter_init (&iter, refs); - while (g_hash_table_iter_next (&iter, &key, NULL)) - { - FlatpakCollectionRef *ref = flatpak_collection_ref_new (opt_collection_id, key); - g_hash_table_add (coll_refs, ref); - } - - *out_all_refs = coll_refs; - - return TRUE; -} - char ** flatpak_dir_find_local_refs (FlatpakDir *self, const char *remote, @@ -11283,7 +11314,6 @@ flatpak_dir_find_local_refs (FlatpakDir *self, GCancellable *cancellable, GError **error) { - g_autofree char *collection_id = NULL; g_autoptr(GHashTable) local_refs = NULL; g_autoptr(GError) my_error = NULL; g_autofree char *refspec_prefix = g_strconcat (remote, ":.", NULL); @@ -11292,9 +11322,9 @@ flatpak_dir_find_local_refs (FlatpakDir *self, if (!flatpak_dir_ensure_repo (self, NULL, error)) return NULL; - collection_id = flatpak_dir_get_remote_collection_id (self, remote); - if (!list_collection_refs_from_ostree_repo (self->repo, refspec_prefix, collection_id, - &local_refs, cancellable, error)) + if (!ostree_repo_list_refs (self->repo, + refspec_prefix, + &local_refs, cancellable, error)) return NULL; matched_refs = find_matching_refs (local_refs, @@ -11303,7 +11333,6 @@ flatpak_dir_find_local_refs (FlatpakDir *self, opt_default_branch, opt_arch, opt_default_arch, - collection_id, kinds, flags, &my_error); @@ -11338,10 +11367,7 @@ flatpak_dir_get_all_installed_refs (FlatpakDir *self, if (!flatpak_dir_maybe_ensure_repo (self, NULL, error)) return NULL; - local_refs = g_hash_table_new_full (flatpak_collection_ref_hash, - flatpak_collection_ref_equal, - (GDestroyNotify) flatpak_collection_ref_free, - NULL); + local_refs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); if (kinds & FLATPAK_KINDS_APP) { g_auto(GStrv) app_refs = NULL; @@ -11350,15 +11376,7 @@ flatpak_dir_get_all_installed_refs (FlatpakDir *self, return NULL; for (i = 0; app_refs[i] != NULL; i++) - { - g_autofree char *remote = NULL; - g_autofree char *collection_id = NULL; - remote = flatpak_dir_get_origin (self, app_refs[i], NULL, NULL); - if (remote != NULL) - collection_id = flatpak_dir_get_remote_collection_id (self, remote); - FlatpakCollectionRef *ref = flatpak_collection_ref_new (collection_id, app_refs[i]); - g_hash_table_add (local_refs, ref); - } + g_hash_table_add (local_refs, g_strdup (app_refs[i])); } if (kinds & FLATPAK_KINDS_RUNTIME) { @@ -11368,15 +11386,7 @@ flatpak_dir_get_all_installed_refs (FlatpakDir *self, return NULL; for (i = 0; runtime_refs[i] != NULL; i++) - { - g_autofree char *remote = NULL; - g_autofree char *collection_id = NULL; - remote = flatpak_dir_get_origin (self, runtime_refs[i], NULL, NULL); - if (remote != NULL) - collection_id = flatpak_dir_get_remote_collection_id (self, remote); - FlatpakCollectionRef *ref = flatpak_collection_ref_new (collection_id, runtime_refs[i]); - g_hash_table_add (local_refs, ref); - } + g_hash_table_add (local_refs, g_strdup (runtime_refs[i])); } return g_steal_pointer (&local_refs); @@ -11404,7 +11414,6 @@ flatpak_dir_find_installed_refs (FlatpakDir *self, NULL, /* default branch */ opt_arch, NULL, /* default arch */ - NULL, kinds, flags, error); @@ -11433,7 +11442,7 @@ flatpak_dir_find_installed_ref (FlatpakDir *self, return NULL; local_ref = find_matching_ref (local_refs, opt_name, opt_branch, NULL, - opt_arch, NULL, kinds, &my_error); + opt_arch, kinds, &my_error); if (local_ref == NULL) { if (g_error_matches (my_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) @@ -11524,12 +11533,11 @@ flatpak_dir_cleanup_undeployed_refs (FlatpakDir *self, g_autoptr(GPtrArray) undeployed_refs = NULL; gsize i = 0; - if (!list_collection_refs_from_ostree_repo (self->repo, NULL, NULL, &local_refspecs, - cancellable, error)) + if (!ostree_repo_list_refs (self->repo, NULL, &local_refspecs, cancellable, error)) return FALSE; local_flatpak_refspecs = find_matching_refs (local_refspecs, - NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, FLATPAK_KINDS_APP | FLATPAK_KINDS_RUNTIME, FIND_MATCHING_REFS_FLAGS_KEEP_REMOTE, @@ -11739,6 +11747,18 @@ flatpak_dir_list_remote_config_keys (FlatpakDir *self, return NULL; } +char ** +flatpak_dir_get_sideload_repo_paths (FlatpakDir *self) +{ + GKeyFile *config = flatpak_dir_get_repo_config (self); + + if (config) + return g_key_file_get_string_list (config, "core", "xa.sideload-repos", NULL, NULL); + + return NULL; +} + + char * flatpak_dir_get_remote_title (FlatpakDir *self, const char *remote_name) @@ -11983,8 +12003,7 @@ flatpak_dir_remote_has_deploys (FlatpakDir *self, g_hash_table_iter_init (&hash_iter, refs); while (g_hash_table_iter_next (&hash_iter, &key, NULL)) { - FlatpakCollectionRef *coll_ref = key; - const char *ref = coll_ref->ref_name; + const char *ref = key; g_autofree char *origin = flatpak_dir_get_origin (self, ref, NULL, NULL); if (strcmp (remote, origin) == 0) @@ -12303,7 +12322,7 @@ flatpak_dir_create_remote_for_ref_file (FlatpakDir *self, } /* First try to reuse existing remote */ - remote = flatpak_dir_find_remote_by_uri (self, url, collection_id); + remote = flatpak_dir_find_remote_by_uri (self, url); if (remote == NULL) { @@ -12349,27 +12368,13 @@ _flatpak_uri_equal (const char *uri1, return g_strcmp0 (uri1_norm, uri2_norm) == 0; } -/* This tries to find a pre-configured remote for the specified uri - * and (optionally) collection id. This is a bit more complex than it - * sounds, because a local remote could be configured in different - * ways for a remote repo (i.e. it could be not using collection ids, - * even though the remote specifies it, or the flatpakrepo might lack - * the collection id details). So, we use these rules: +/* This tries to find a pre-configured remote for the specified uri. * - * If the url is the same, it is a match even if one part lacks - * collection ids. However, if both collection ids are specified and - * differ there is no match. - * - * If the collection id is the same (and specified), its going to be - * the same remote, even if the url is different (because it could be - * some other mirror of the same repo). - * - * We also consider non-OCI URLs equal even if one lacks a trailing slash. + * We consider non-OCI URLs equal even if one lacks a trailing slash. */ char * flatpak_dir_find_remote_by_uri (FlatpakDir *self, - const char *uri, - const char *collection_id) + const char *uri) { g_auto(GStrv) remotes = NULL; @@ -12395,21 +12400,8 @@ flatpak_dir_find_remote_by_uri (FlatpakDir *self, &remote_uri, NULL)) continue; - if (!repo_get_remote_collection_id (self->repo, remote, &remote_collection_id, NULL)) - continue; - /* Exact collection ids always match, independent of the uris used */ - if (collection_id != NULL && - remote_collection_id != NULL && - strcmp (collection_id, remote_collection_id) == 0) - return g_strdup (remote); - - /* Same repo if uris match, unless both have collection-id - specified but different */ - if (_flatpak_uri_equal (uri, remote_uri) && - !(collection_id != NULL && - remote_collection_id != NULL && - strcmp (collection_id, remote_collection_id) != 0)) + if (_flatpak_uri_equal (uri, remote_uri)) return g_strdup (remote); } } @@ -12833,9 +12825,9 @@ remove_unless_in_hash (gpointer key, gpointer user_data) { GHashTable *table = user_data; - FlatpakCollectionRef *ref = key; + const char *ref_name = key; - return !g_hash_table_contains (table, ref->ref_name); + return !g_hash_table_contains (table, ref_name); } gboolean @@ -13226,7 +13218,7 @@ flatpak_dir_fetch_remote_commit (FlatpakDir *self, if (state == NULL) return NULL; - if (!flatpak_remote_state_lookup_ref (state, ref, &latest_commit, NULL, NULL, error)) + if (!flatpak_remote_state_lookup_ref (state, ref, &latest_commit, NULL, NULL, NULL, error)) return NULL; if (latest_commit == NULL) { @@ -13528,7 +13520,7 @@ flatpak_dir_find_remote_related_for_metadata (FlatpakDir *self, extension_ref = g_build_filename ("runtime", extension, parts[2], branch, NULL); - if (flatpak_remote_state_lookup_ref (state, extension_ref, &checksum, NULL, NULL, NULL)) + if (flatpak_remote_state_lookup_ref (state, extension_ref, &checksum, NULL, NULL, NULL, NULL)) { if (flatpak_filters_allow_ref (NULL, masked, extension_ref)) add_related (self, related, extension, extension_collection_id, extension_ref, checksum, @@ -13542,7 +13534,7 @@ flatpak_dir_find_remote_related_for_metadata (FlatpakDir *self, { g_autofree char *subref_checksum = NULL; - if (flatpak_remote_state_lookup_ref (state, refs[j], &subref_checksum, NULL, NULL, NULL) && + if (flatpak_remote_state_lookup_ref (state, refs[j], &subref_checksum, NULL, NULL, NULL, NULL) && flatpak_filters_allow_ref (NULL, masked, refs[j])) add_related (self, related, extension, extension_collection_id, refs[j], subref_checksum, no_autodownload, download_if, autoprune_unless, autodelete, locale_subset); diff --git a/common/flatpak-installation.c b/common/flatpak-installation.c index ce2e9a26..0d21504a 100644 --- a/common/flatpak-installation.c +++ b/common/flatpak-installation.c @@ -1055,11 +1055,6 @@ flatpak_installation_list_installed_refs_for_update (FlatpakInstallation *self, if (flatpak_remote_get_disabled (remote)) continue; - /* Remotes with collection IDs will be handled separately below */ - collection_id = flatpak_remote_get_collection_id (remote); - if (collection_id != NULL) - continue; - /* We ignore errors here. we don't want one remote to fail us */ refs = flatpak_installation_list_remote_refs_sync (self, remote_name, @@ -1178,120 +1173,6 @@ flatpak_installation_list_installed_refs_for_update (FlatpakInstallation *self, } } - collection_refs = g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_collection_ref_free0); - - refs_str = g_string_new (""); - for (i = 0; i < installed->len; i++) - { - FlatpakInstalledRef *installed_ref = g_ptr_array_index (installed, i); - g_autofree char *ref = flatpak_ref_format_ref (FLATPAK_REF (installed_ref)); - g_autofree char *collection_id = NULL; - const char *remote_name = flatpak_installed_ref_get_origin (installed_ref); - - if (flatpak_dir_ref_is_masked (dir, ref)) - continue; - - collection_id = flatpak_dir_get_remote_collection_id (dir, remote_name); - if (collection_id != NULL) - { - OstreeCollectionRef *c_r = ostree_collection_ref_new (collection_id, ref); - g_ptr_array_add (collection_refs, c_r); - - if (refs_str->len > 0) - g_string_append (refs_str, ", "); - g_string_append_printf (refs_str, "(%s, %s)", collection_id, ref); - } - } - - /* if we do not have any collection refs, then we shouldn't try to find - * dynamic remotes for them, to avoid extra unnecessary processing, and also - * because the refs array cannot be empty in ostree_repo_find_remotes_async - * (otherwise it early returns and we never get our callback called) */ - if (collection_refs->len > 0) - { - g_autoptr(GMainContextPopDefault) context = NULL; - - g_ptr_array_add (collection_refs, NULL); - - context = flatpak_main_context_new_default (); - - ostree_repo_find_remotes_async (flatpak_dir_get_repo (dir), - (const OstreeCollectionRef * const *) collection_refs->pdata, - NULL, /* no options */ - NULL, /* default finders */ - NULL, /* no progress */ - cancellable, - async_result_cb, - &result); - - while (result == NULL) - g_main_context_iteration (context, TRUE); - - results = ostree_repo_find_remotes_finish (flatpak_dir_get_repo (dir), result, error); - - if (results == NULL) - return NULL; - - if (results[0] == NULL) - g_debug ("No remotes found which provide these refs: [%s]", refs_str->str); - } - - for (i = 0; i < installed->len && results != NULL && results[0] != NULL; i++) - { - FlatpakInstalledRef *installed_ref = g_ptr_array_index (installed, i); - const char *remote_name = flatpak_installed_ref_get_origin (installed_ref); - g_autofree char *ref = flatpak_ref_format_ref (FLATPAK_REF (installed_ref)); - g_autofree char *collection_id = NULL; - g_autoptr(OstreeCollectionRef) collection_ref = NULL; - - collection_id = flatpak_dir_get_remote_collection_id (dir, remote_name); - collection_ref = ostree_collection_ref_new (collection_id, ref); - - /* Look for matching remote refs that are updates */ - for (j = 0; results[j] != NULL; j++) - { - const char *local_commit, *remote_commit; - - local_commit = flatpak_installed_ref_get_latest_commit (installed_ref); - remote_commit = g_hash_table_lookup (results[j]->ref_to_checksum, collection_ref); - if (remote_commit == NULL || g_strcmp0 (remote_commit, local_commit) == 0) - continue; - - /* The ref_to_checksum map only tells us if this remote is offering - * the latest commit of the available remotes; we have to check - * ref_to_timestamp to know if the commit is an update or a - * downgrade. If local_commit is NULL assume it's an update until - * proven otherwise. - */ - if (local_commit != NULL) - { - guint64 local_timestamp = 0; - guint64 *remote_timestamp; - g_autoptr(GVariant) commit_v = NULL; - - if (ostree_repo_load_commit (flatpak_dir_get_repo (dir), local_commit, &commit_v, NULL, NULL)) - local_timestamp = ostree_commit_get_timestamp (commit_v); - - remote_timestamp = g_hash_table_lookup (results[j]->ref_to_timestamp, collection_ref); - *remote_timestamp = GUINT64_FROM_BE (*remote_timestamp); - - g_debug ("%s: Comparing local timestamp %" G_GUINT64_FORMAT " to remote timestamp %" - G_GUINT64_FORMAT " on ref (%s, %s)", G_STRFUNC, local_timestamp, *remote_timestamp, - collection_ref->collection_id, collection_ref->ref_name); - - /* The timestamp could be 0 due to an error reading it. Assume - * it's an update until proven otherwise. */ - if (*remote_timestamp != 0 && *remote_timestamp <= local_timestamp) - continue; - } - - g_ptr_array_add (updates, g_object_ref (installed_ref)); - - /* Move on to the next ref so we don't add duplicates */ - break; - } - } - return g_steal_pointer (&updates); } @@ -2049,8 +1930,7 @@ flatpak_installation_install_ref_file (FlatpakInstallation *self, if (!flatpak_installation_drop_caches (self, cancellable, error)) return NULL; - coll_ref = flatpak_collection_ref_new (collection_id, ref); - return flatpak_remote_ref_new (coll_ref, NULL, remote, NULL); + return flatpak_remote_ref_new (ref, NULL, remote, NULL); } /** @@ -2140,7 +2020,7 @@ flatpak_installation_install_full (FlatpakInstallation *self, (flags & FLATPAK_INSTALL_FLAGS_NO_DEPLOY) != 0, (flags & FLATPAK_INSTALL_FLAGS_NO_STATIC_DELTAS) != 0, FALSE, FALSE, state, - ref, NULL, (const char **) subpaths, NULL, NULL, NULL, + ref, NULL, (const char **) subpaths, NULL, NULL, NULL, NULL, ostree_progress, cancellable, error)) return NULL; @@ -2249,7 +2129,6 @@ flatpak_installation_update_full (FlatpakInstallation *self, g_autofree char *remote_name = NULL; FlatpakInstalledRef *result = NULL; g_autofree char *target_commit = NULL; - g_auto(OstreeRepoFinderResultv) check_results = NULL; g_autoptr(FlatpakRemoteState) state = NULL; g_autoptr(GMainContextPopDefault) main_context = NULL; @@ -2280,7 +2159,6 @@ flatpak_installation_update_full (FlatpakInstallation *self, target_commit = flatpak_dir_check_for_update (dir, state, ref, NULL, (const char **) subpaths, (flags & FLATPAK_UPDATE_FLAGS_NO_PULL) != 0, - &check_results, cancellable, error); if (target_commit == NULL) return NULL; @@ -2304,8 +2182,7 @@ flatpak_installation_update_full (FlatpakInstallation *self, (flags & FLATPAK_UPDATE_FLAGS_NO_STATIC_DELTAS) != 0, FALSE, FALSE, FALSE, state, ref, target_commit, - (const OstreeRepoFinderResult * const *) check_results, - (const char **) subpaths, NULL, NULL, NULL, + (const char **) subpaths, NULL, NULL, NULL, NULL, ostree_progress, cancellable, error)) return NULL; @@ -2623,11 +2500,11 @@ flatpak_installation_list_remote_refs_sync_full (FlatpakInstallation *self, g_hash_table_iter_init (&iter, ht); while (g_hash_table_iter_next (&iter, &key, &value)) { - FlatpakRemoteRef *ref; - FlatpakCollectionRef *coll_ref = key; + const char *ref_name = key; const gchar *ref_commit = value; + FlatpakRemoteRef *ref; - ref = flatpak_remote_ref_new (coll_ref, ref_commit, remote_or_uri, state); + ref = flatpak_remote_ref_new (ref_name, ref_commit, remote_or_uri, state); if (ref) g_ptr_array_add (refs, ref); @@ -2699,8 +2576,6 @@ flatpak_installation_fetch_remote_ref_sync_full (FlatpakInstallation *self, g_autoptr(GHashTable) ht = NULL; g_autoptr(FlatpakRemoteState) state = NULL; g_autofree char *ref = NULL; - g_autoptr(FlatpakCollectionRef) coll_ref = NULL; - g_autofree gchar *collection_id = NULL; const char *checksum; if (branch == NULL) @@ -2718,12 +2593,6 @@ flatpak_installation_fetch_remote_ref_sync_full (FlatpakInstallation *self, cancellable, error)) return NULL; - /* FIXME: Rework to accept the collection ID as an input argument instead */ - if (!ostree_repo_get_remote_option (flatpak_dir_get_repo (dir), - remote_name, "collection-id", - NULL, &collection_id, error)) - return FALSE; - if (kind == FLATPAK_REF_KIND_APP) ref = flatpak_build_app_ref (name, branch, @@ -2733,102 +2602,10 @@ flatpak_installation_fetch_remote_ref_sync_full (FlatpakInstallation *self, branch, arch); - coll_ref = flatpak_collection_ref_new (collection_id, ref); - checksum = g_hash_table_lookup (ht, coll_ref); - - /* Check LAN/USB sources too in case we're offline */ - if (checksum == NULL && collection_id != NULL && *collection_id != '\0') - { - OstreeRepo *repo; - const char * const *default_repo_finders; - g_autoptr(GAsyncResult) result = NULL; - OstreeCollectionRef ostree_coll_ref; - const OstreeCollectionRef *refs[2] = { NULL, }; - OstreeRepoFinder *finders[3] = { NULL, }; - guint finder_index = 0; - gsize i; - g_autoptr(GMainContextPopDefault) context = NULL; - g_autoptr(OstreeRepoFinder) finder_mount = NULL, finder_avahi = NULL; - - context = flatpak_main_context_new_default (); - - ostree_coll_ref.collection_id = collection_id; - ostree_coll_ref.ref_name = ref; - refs[0] = &ostree_coll_ref; - - if (!flatpak_dir_ensure_repo (dir, cancellable, error)) - return NULL; - repo = flatpak_dir_get_repo (dir); - default_repo_finders = ostree_repo_get_default_repo_finders (repo); - if (default_repo_finders == NULL || g_strv_contains (default_repo_finders, "mount")) - { - finder_mount = OSTREE_REPO_FINDER (ostree_repo_finder_mount_new (NULL)); - finders[finder_index++] = finder_mount; - } - - if (default_repo_finders == NULL || g_strv_contains (default_repo_finders, "lan")) - { - g_autoptr(GError) local_error = NULL; - finder_avahi = OSTREE_REPO_FINDER (ostree_repo_finder_avahi_new (context)); - finders[finder_index++] = finder_avahi; - - /* The Avahi finder may fail to start on, for example, a CI server. */ - ostree_repo_finder_avahi_start (OSTREE_REPO_FINDER_AVAHI (finder_avahi), &local_error); - if (local_error != NULL) - { - finders[--finder_index] = NULL; - g_clear_object (&finder_avahi); - } - } - - if (finders[0] != NULL) - { - g_auto(OstreeRepoFinderResultv) results = NULL; - - ostree_repo_find_remotes_async (repo, (const OstreeCollectionRef * const *)refs, - NULL, finders, NULL, cancellable, async_result_cb, &result); - while (result == NULL) - g_main_context_iteration (context, TRUE); - - results = ostree_repo_find_remotes_finish (repo, result, error); - - if (finder_avahi != NULL) - ostree_repo_finder_avahi_stop (OSTREE_REPO_FINDER_AVAHI (finder_avahi)); - - if (results == NULL) - return NULL; - - for (i = 0; results[i] != NULL; i++) - { - checksum = g_hash_table_lookup (results[i]->ref_to_checksum, &ostree_coll_ref); - if (checksum != NULL) - break; - } - } - } - - /* If there was not a match, it may be because the collection ID is - * not set in the local configuration, or it is wrong, so we resort to - * trying to match just the ref name */ - if (checksum == NULL) - { - GHashTableIter iter; - gpointer key, value; - - g_hash_table_iter_init (&iter, ht); - while (g_hash_table_iter_next (&iter, &key, &value)) - { - FlatpakCollectionRef *current = (FlatpakCollectionRef *) key; - if (g_strcmp0 (current->ref_name, ref) == 0) - { - checksum = (const gchar *) value; - break; - } - } - } + checksum = g_hash_table_lookup (ht, ref); if (checksum != NULL) - return flatpak_remote_ref_new (coll_ref, checksum, remote_name, state); + return flatpak_remote_ref_new (ref, checksum, remote_name, state); g_set_error (error, FLATPAK_ERROR, FLATPAK_ERROR_REF_NOT_FOUND, "Reference %s doesn't exist in remote", ref); diff --git a/common/flatpak-remote-ref-private.h b/common/flatpak-remote-ref-private.h index 71522c42..792dd7ee 100644 --- a/common/flatpak-remote-ref-private.h +++ b/common/flatpak-remote-ref-private.h @@ -28,7 +28,7 @@ #include #include -FlatpakRemoteRef *flatpak_remote_ref_new (FlatpakCollectionRef *coll_ref, +FlatpakRemoteRef *flatpak_remote_ref_new (const char *ref, const char *commit, const char *remote_name, FlatpakRemoteState *remote_state); diff --git a/common/flatpak-remote-ref.c b/common/flatpak-remote-ref.c index d31b6dca..828c4be3 100644 --- a/common/flatpak-remote-ref.c +++ b/common/flatpak-remote-ref.c @@ -319,7 +319,7 @@ flatpak_remote_ref_get_eol_rebase (FlatpakRemoteRef *self) } FlatpakRemoteRef * -flatpak_remote_ref_new (FlatpakCollectionRef *coll_ref, +flatpak_remote_ref_new (const char *full_ref, const char *commit, const char *remote_name, FlatpakRemoteState *state) @@ -331,7 +331,6 @@ flatpak_remote_ref_new (FlatpakCollectionRef *coll_ref, g_auto(GStrv) parts = NULL; FlatpakRemoteRef *ref; VarMetadataRef sparse_cache; - const char *full_ref = coll_ref->ref_name; const char *eol = NULL; const char *eol_rebase = NULL; @@ -365,7 +364,6 @@ flatpak_remote_ref_new (FlatpakCollectionRef *coll_ref, "name", parts[1], "arch", parts[2], "branch", parts[3], - "collection-id", coll_ref->collection_id, "commit", commit, "remote-name", remote_name, "installed-size", installed_size, diff --git a/common/flatpak-transaction.c b/common/flatpak-transaction.c index 629accbe..4aad4c2c 100644 --- a/common/flatpak-transaction.c +++ b/common/flatpak-transaction.c @@ -109,6 +109,7 @@ struct _FlatpakTransactionOperation gboolean resolved; char *resolved_commit; + GFile *resolved_sideload_path; GBytes *resolved_metadata; GKeyFile *resolved_metakey; GBytes *resolved_old_metadata; @@ -588,6 +589,8 @@ flatpak_transaction_operation_finalize (GObject *object) if (self->external_metadata) g_bytes_unref (self->external_metadata); g_free (self->resolved_commit); + if (self->resolved_sideload_path) + g_object_unref (self->resolved_sideload_path); if (self->resolved_metadata) g_bytes_unref (self->resolved_metadata); if (self->resolved_metakey) @@ -2316,7 +2319,7 @@ flatpak_transaction_add_auto_install (FlatpakTransaction *self, g_autoptr(FlatpakRemoteState) state = flatpak_transaction_ensure_remote_state (self, FLATPAK_TRANSACTION_OPERATION_UPDATE, remote, NULL); if (state != NULL && - flatpak_remote_state_lookup_ref (state, ref, NULL, NULL, NULL, NULL)) + flatpak_remote_state_lookup_ref (state, ref, NULL, NULL, NULL, NULL, NULL)) { g_debug ("Auto adding install of %s from remote %s", ref, remote); if (!flatpak_transaction_add_ref (self, remote, ref, NULL, NULL, NULL, @@ -2413,6 +2416,7 @@ emit_eol_and_maybe_skip (FlatpakTransaction *self, static void mark_op_resolved (FlatpakTransactionOperation *op, const char *commit, + GFile *sideload_path, GBytes *metadata, GBytes *old_metadata) { @@ -2424,6 +2428,10 @@ mark_op_resolved (FlatpakTransactionOperation *op, op->resolved = TRUE; op->resolved_commit = g_strdup (commit); + + if (sideload_path) + op->resolved_sideload_path = g_object_ref (sideload_path); + if (metadata) { g_autoptr(GKeyFile) metakey = g_key_file_new (); @@ -2452,12 +2460,13 @@ static void resolve_op_end (FlatpakTransaction *self, FlatpakTransactionOperation *op, const char *checksum, + GFile *sideload_path, GBytes *metadata_bytes) { g_autoptr(GBytes) old_metadata_bytes = NULL; old_metadata_bytes = load_deployed_metadata (self, op->ref, NULL); - mark_op_resolved (op, checksum, metadata_bytes, old_metadata_bytes); + mark_op_resolved (op, checksum, sideload_path, metadata_bytes, old_metadata_bytes); emit_eol_and_maybe_skip (self, op); } @@ -2466,6 +2475,7 @@ static void resolve_op_from_commit (FlatpakTransaction *self, FlatpakTransactionOperation *op, const char *checksum, + GFile *sideload_path, GVariant *commit_data) { g_autoptr(GBytes) metadata_bytes = NULL; @@ -2489,14 +2499,15 @@ resolve_op_from_commit (FlatpakTransaction *self, g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_ENDOFLIFE, "s", &op->eol); g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_ENDOFLIFE_REBASE, "s", &op->eol_rebase); - resolve_op_end (self, op, checksum, metadata_bytes); + resolve_op_end (self, op, checksum, sideload_path, metadata_bytes); } -static void -resolve_op_from_metadata (FlatpakTransaction *self, - FlatpakTransactionOperation *op, - const char *checksum, - FlatpakRemoteState *state) +static gboolean +try_resolve_op_from_metadata (FlatpakTransaction *self, + FlatpakTransactionOperation *op, + const char *checksum, + GFile *sideload_path, + FlatpakRemoteState *state) { g_autoptr(GBytes) metadata_bytes = NULL; guint64 download_size = 0; @@ -2505,17 +2516,22 @@ resolve_op_from_metadata (FlatpakTransaction *self, VarMetadataRef sparse_cache; g_autoptr(GError) local_error = NULL; VarRefInfoRef info; - gboolean has_info; + g_autofree char *summary_checksum = NULL; - if (!flatpak_remote_state_lookup_cache (state, op->ref, &download_size, &installed_size, &metadata, NULL, &local_error)) - { - g_message (_("Warning: Can't find %s metadata for dependencies: %s"), op->ref, local_error->message); - g_clear_error (&local_error); - } - else - metadata_bytes = g_bytes_new (metadata, strlen (metadata) + 1); + /* Ref has to match the actual commit in the summary */ + if (state->summary == NULL || + !flatpak_summary_lookup_ref (state->summary, NULL, op->ref, &summary_checksum, NULL) || + strcmp (summary_checksum, checksum) != 0) + return FALSE; - if (flatpak_remote_state_lookup_ref (state, op->ref, NULL, &has_info, &info, NULL) && has_info) + /* And, we must have the actual cached data in the summary */ + if (!flatpak_remote_state_lookup_cache (state, op->ref, + &download_size, &installed_size, &metadata, NULL, NULL)) + return FALSE; + + metadata_bytes = g_bytes_new (metadata, strlen (metadata) + 1); + + if (flatpak_remote_state_lookup_ref (state, op->ref, NULL, NULL, &info, NULL, NULL)) op->summary_metadata = var_metadata_dup_to_gvariant (var_ref_info_get_metadata (info)); op->installed_size = installed_size; @@ -2530,7 +2546,8 @@ resolve_op_from_metadata (FlatpakTransaction *self, op->token_type = GINT32_FROM_LE (var_metadata_lookup_int32 (sparse_cache, FLATPAK_SPARSE_CACHE_KEY_TOKEN_TYPE, op->token_type)); } - resolve_op_end (self, op, checksum, metadata_bytes); + resolve_op_end (self, op, checksum, sideload_path, metadata_bytes); + return TRUE; } static gboolean @@ -2553,14 +2570,12 @@ resolve_ops (FlatpakTransaction *self, { FlatpakTransactionPrivate *priv = flatpak_transaction_get_instance_private (self); GList *l; - g_autoptr(GList) collection_id_ops = NULL; for (l = priv->ops; l != NULL; l = l->next) { FlatpakTransactionOperation *op = l->data; g_autoptr(FlatpakRemoteState) state = NULL; g_autofree char *checksum = NULL; - g_autoptr(GVariant) commit_data = NULL; g_autoptr(GBytes) metadata_bytes = NULL; if (op->resolved) @@ -2571,14 +2586,14 @@ resolve_ops (FlatpakTransaction *self, /* We resolve to the deployed metadata, because we need it to uninstall related ops */ metadata_bytes = load_deployed_metadata (self, op->ref, &checksum); - mark_op_resolved (op, checksum, metadata_bytes, NULL); + mark_op_resolved (op, checksum, NULL, metadata_bytes, NULL); continue; } if (op->kind == FLATPAK_TRANSACTION_OPERATION_INSTALL_BUNDLE) { g_assert (op->commit != NULL); - mark_op_resolved (op, op->commit, op->external_metadata, NULL); + mark_op_resolved (op, op->commit, NULL, op->external_metadata, NULL); continue; } @@ -2604,39 +2619,76 @@ resolve_ops (FlatpakTransaction *self, /* Should we use local state */ if (transaction_is_local_only (self, op->kind)) { - commit_data = flatpak_dir_read_latest_commit (priv->dir, op->remote, op->ref, &checksum, NULL, error); + g_autoptr(GVariant) commit_data = flatpak_dir_read_latest_commit (priv->dir, op->remote, op->ref, &checksum, NULL, error); if (commit_data == NULL) return FALSE; - resolve_op_from_commit (self, op, checksum, commit_data); + resolve_op_from_commit (self, op, checksum, NULL, commit_data); } else { g_autoptr(GError) local_error = NULL; + g_autoptr(GFile) sideload_path = NULL; if (op->commit != NULL) checksum = g_strdup (op->commit); - else if (!flatpak_dir_find_latest_rev (priv->dir, state, op->ref, op->commit, &checksum, - NULL, cancellable, &local_error)) + else { - /* An unavailable remote summary shouldn't be fatal if we already have the ref */ - commit_data = flatpak_dir_read_latest_commit (priv->dir, op->remote, op->ref, &checksum, NULL, NULL); - if (commit_data == NULL) + g_autofree char *latest_checksum = NULL; + g_autoptr(GFile) latest_sideload_path = NULL; + g_autofree char *local_checksum = NULL; + guint64 latest_timestamp; + g_autoptr(GVariant) local_commit_data = flatpak_dir_read_latest_commit (priv->dir, op->remote, op->ref, &local_checksum, NULL, NULL); + + if (flatpak_dir_find_latest_rev (priv->dir, state, op->ref, op->commit, + &latest_checksum, &latest_timestamp, &latest_sideload_path, + cancellable, &local_error)) { - g_propagate_error (error, g_steal_pointer (&local_error)); - return FALSE; + /* If we found the latest in a sideload repo, it may be older that what is locally available, check timestamps. + * Note: If the timestamps are equal (timestamp granularity issue), assume we want to update */ + if (latest_sideload_path != NULL && local_commit_data && + ostree_commit_get_timestamp (local_commit_data) > latest_timestamp) + { + g_debug ("Installed commit %s newer than sideloaded %s, ignoring", local_checksum, latest_checksum); + checksum = g_steal_pointer (&local_checksum); + } + else + { + /* Otherwise, use whatever we found */ + checksum = g_steal_pointer (&latest_checksum); + sideload_path = g_steal_pointer (&latest_sideload_path); + } } else { + /* Ref not available in the remote (maybe offline), resolve to local version if installed */ + if (local_commit_data == NULL) + { + g_propagate_error (error, g_steal_pointer (&local_error)); + return FALSE; + } + g_message (_("Warning: Treating remote fetch error as non-fatal since %s is already installed: %s"), op->ref, local_error->message); g_clear_error (&local_error); + + checksum = g_steal_pointer (&local_checksum); } } - /* TODO: This only gets the metadata for the latest only, we need to handle the case - where the user specified a commit, or p2p doesn't have the latest commit available */ - resolve_op_from_metadata (self, op, checksum, state); + /* First try to resolve via metadata (if remote is available and its metadata matches the commit version) */ + if (!try_resolve_op_from_metadata (self, op, checksum, sideload_path, state)) + { + /* Else try to load the commit object */ + g_autoptr(GVariant) commit_data = NULL; + + commit_data = flatpak_remote_state_load_ref_commit (state, priv->dir, + op->ref, checksum, error); + if (commit_data == NULL) + return FALSE; + + resolve_op_from_commit (self, op, checksum, sideload_path, commit_data); + } } } @@ -2927,9 +2979,6 @@ request_tokens_for_remote (FlatpakTransaction *self, copy_summary_data (extra_builder, state->summary, "xa.oci-registry-uri"); } - if (state->collection_id) - g_variant_builder_add (extra_builder, "{sv}", "collection-id", g_variant_new_string (state->collection_id)); - if (flatpak_dir_get_no_interaction (priv->dir)) g_variant_builder_add (extra_builder, "{sv}", "no-interaction", g_variant_new_boolean (TRUE)); @@ -3197,13 +3246,12 @@ flatpak_transaction_get_installation (FlatpakTransaction *self) static gboolean remote_is_already_configured (FlatpakTransaction *self, - const char *url, - const char *collection_id) + const char *url) { FlatpakTransactionPrivate *priv = flatpak_transaction_get_instance_private (self); g_autofree char *old_remote = NULL; - old_remote = flatpak_dir_find_remote_by_uri (priv->dir, url, collection_id); + old_remote = flatpak_dir_find_remote_by_uri (priv->dir, url); /* Note: we don't check priv->extra_dependency_dirs because the transaction * can only operate on one installation so any install/update ops need to @@ -3219,7 +3267,6 @@ handle_suggested_remote_name (FlatpakTransaction *self, GKeyFile *keyfile, GErro g_autofree char *suggested_name = NULL; g_autofree char *name = NULL; g_autofree char *url = NULL; - g_autofree char *collection_id = NULL; g_autoptr(GKeyFile) config = NULL; g_autoptr(GBytes) gpg_key = NULL; gboolean res; @@ -3237,11 +3284,7 @@ handle_suggested_remote_name (FlatpakTransaction *self, GKeyFile *keyfile, GErro if (url == NULL) return TRUE; - collection_id = g_key_file_get_string (keyfile, FLATPAK_REF_GROUP, FLATPAK_REF_DEPLOY_COLLECTION_ID_KEY, NULL); - if (collection_id == NULL || *collection_id == '\0') - collection_id = g_key_file_get_string (keyfile, FLATPAK_REF_GROUP, FLATPAK_REF_COLLECTION_ID_KEY, NULL); - - if (remote_is_already_configured (self, url, collection_id)) + if (remote_is_already_configured (self, url)) return TRUE; /* The name is already used, ignore */ @@ -3288,7 +3331,6 @@ handle_runtime_repo_deps (FlatpakTransaction *self, g_autoptr(GBytes) gpg_key = NULL; g_autofree char *group = NULL; g_autoptr(GError) local_error = NULL; - g_autofree char *runtime_collection_id = NULL; g_autoptr(SoupSession) soup_session = NULL; char *t; int i; @@ -3367,9 +3409,8 @@ handle_runtime_repo_deps (FlatpakTransaction *self, group = g_strdup_printf ("remote \"%s\"", new_remote); runtime_url = g_key_file_get_string (config, group, "url", NULL); g_assert (runtime_url != NULL); - runtime_collection_id = g_key_file_get_string (config, group, "collection-id", NULL); - if (remote_is_already_configured (self, runtime_url, runtime_collection_id)) + if (remote_is_already_configured (self, runtime_url)) return TRUE; res = FALSE; @@ -3590,6 +3631,7 @@ _run_op_kind (FlatpakTransaction *self, remote_state, op->ref, op->resolved_commit, (const char **) op->subpaths, (const char **) op->previous_ids, + op->resolved_sideload_path, op->resolved_metadata, op->resolved_token, progress->ostree_progress, @@ -3638,9 +3680,9 @@ _run_op_kind (FlatpakTransaction *self, priv->max_op >= APP_UPDATE, priv->max_op == APP_INSTALL || priv->max_op == RUNTIME_INSTALL, remote_state, op->ref, op->resolved_commit, - NULL, (const char **) op->subpaths, (const char **) op->previous_ids, + op->resolved_sideload_path, op->resolved_metadata, op->resolved_token, progress->ostree_progress, diff --git a/system-helper/flatpak-system-helper.c b/system-helper/flatpak-system-helper.c index 61f82e30..37ba8c76 100644 --- a/system-helper/flatpak-system-helper.c +++ b/system-helper/flatpak-system-helper.c @@ -515,7 +515,6 @@ handle_deploy (FlatpakSystemHelper *object, g_autoptr(FlatpakOciVersioned) versioned = NULL; g_autoptr(FlatpakOciImage) image_config = NULL; g_autoptr(FlatpakRemoteState) state = NULL; - FlatpakCollectionRef collection_ref; g_autoptr(GHashTable) remote_refs = NULL; g_autofree char *checksum = NULL; const char *verified_digest; @@ -594,10 +593,7 @@ handle_deploy (FlatpakSystemHelper *object, return TRUE; } - collection_ref.collection_id = state->collection_id; - collection_ref.ref_name = (char *) arg_ref; - - verified_digest = g_hash_table_lookup (remote_refs, &collection_ref); + verified_digest = g_hash_table_lookup (remote_refs, arg_ref); if (!verified_digest) { g_dbus_method_invocation_return_error (invocation, G_DBUS_ERROR, G_DBUS_ERROR_FAILED, @@ -674,7 +670,7 @@ handle_deploy (FlatpakSystemHelper *object, ostree_progress = ostree_async_progress_new_and_connect (no_progress_cb, NULL); - if (!flatpak_dir_pull (system, state, arg_ref, NULL, NULL, (const char **) arg_subpaths, NULL, NULL, NULL, + if (!flatpak_dir_pull (system, state, arg_ref, NULL, (const char **) arg_subpaths, NULL, NULL, NULL, NULL, FLATPAK_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_UNTRUSTED, ostree_progress, NULL, &error)) { diff --git a/tests/testlibrary.c b/tests/testlibrary.c index c3f64959..721a6fa4 100644 --- a/tests/testlibrary.c +++ b/tests/testlibrary.c @@ -933,9 +933,6 @@ test_list_refs_in_remotes (void) g_autoptr(FlatpakRemote) remote = NULL; g_autofree char *repo_dir = g_build_filename (testdir, repo_name, NULL); g_autofree char *repo_uri = NULL; - g_autoptr(GHashTable) collection_ids = g_hash_table_new_full (g_str_hash, - g_str_equal, - NULL, NULL); g_autoptr(GHashTable) ref_specs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, @@ -971,13 +968,9 @@ test_list_refs_in_remotes (void) for (guint i = 0; i < refs1->len; ++i) { FlatpakRef *ref = g_ptr_array_index (refs1, i); - g_hash_table_add (collection_ids, (gchar *) flatpak_ref_get_collection_id (ref)); g_hash_table_add (ref_specs, flatpak_ref_format_ref (ref)); } - /* we have a locale extension for each app, thus the 2 */ - g_assert_cmpuint (2 * g_hash_table_size (collection_ids), ==, refs1->len); - /* Ensure that listing the refs by using a remote's URI will get us the * same results as using the name */ repo_uri = flatpak_remote_get_url (remote); From 124b0be92e0b033c35e04c5dcbea5a9da6915723 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 18 Mar 2020 11:06:50 +0100 Subject: [PATCH 10/36] Drop more collection_id use Most code that looks for a regular collection id set on the remote is removed, as these should never happen in flatpak repo setups now. Some is replaces with looking at xa.sideload-collection-id: * The libflatpak FlatpakRef::collection-id property now comes comes from the sideload id * Various CLI commands showing or changing the collection-id for a remote now uses the sideload id * Collection id deploy in update now sets the sideload-collection-id instead * Setting the collection id for a remote in libflatpak now sets the sideload id Additionally we now delete the code that allows unsigned summaries when there is a collection id (because there is none). create-usb now uses the sideload id as as collection id source when exporting. The direct repo operations (export, bundle, commit-from) still support collection ids, because on the server we do want to set it so that we can sideload. --- app/flatpak-builtins-create-usb.c | 35 +- app/flatpak-builtins-remote-add.c | 5 +- app/flatpak-builtins-remote-list.c | 2 +- app/flatpak-builtins-remote-modify.c | 3 +- common/flatpak-dir-private.h | 23 +- common/flatpak-dir.c | 369 +++++----------------- common/flatpak-installation.c | 32 +- common/flatpak-installed-ref.c | 4 +- common/flatpak-related-ref-private.h | 3 +- common/flatpak-related-ref.c | 5 +- common/flatpak-remote-ref-private.h | 1 + common/flatpak-remote-ref.c | 2 + common/flatpak-remote.c | 23 +- common/flatpak-transaction.c | 1 - system-helper/flatpak-system-helper.c | 2 +- tests/test-update-portal.c | 1 - tests/test-update-remote-configuration.sh | 14 +- tests/testlibrary.c | 4 +- 18 files changed, 130 insertions(+), 399 deletions(-) diff --git a/app/flatpak-builtins-create-usb.c b/app/flatpak-builtins-create-usb.c index 23e4af07..dad99a36 100644 --- a/app/flatpak-builtins-create-usb.c +++ b/app/flatpak-builtins-create-usb.c @@ -167,7 +167,7 @@ add_related (GHashTable *all_refs, ext_remote = flatpak_deploy_data_get_origin (ext_deploy_data); if (ext_remote == NULL) return FALSE; - ext_collection_id = flatpak_dir_get_remote_collection_id (dir, ext_remote); + ext_collection_id = flatpak_dir_get_remote_sideload_collection_id (dir, ext_remote); if (ext_collection_id == NULL) { g_printerr (_("Warning: Omitting related ref ‘%s’ because its remote ‘%s’ does not have a collection ID set.\n"), @@ -239,7 +239,7 @@ add_runtime (GHashTable *all_refs, runtime_remote = flatpak_dir_get_origin (dir, runtime_ref, cancellable, error); if (runtime_remote == NULL) return FALSE; - runtime_collection_id = flatpak_dir_get_remote_collection_id (dir, runtime_remote); + runtime_collection_id = flatpak_dir_get_remote_sideload_collection_id (dir, runtime_remote); if (runtime_collection_id == NULL) return flatpak_fail (error, _("Remote ‘%s’ does not have a collection ID set, which is required for P2P distribution of ‘%s’."), @@ -605,7 +605,7 @@ flatpak_builtin_create_usb (int argc, char **argv, GCancellable *cancellable, GE if (remote == NULL) return FALSE; - ref_collection_id = flatpak_dir_get_remote_collection_id (dir, remote); + ref_collection_id = flatpak_dir_get_remote_sideload_collection_id (dir, remote); if (ref_collection_id == NULL) return flatpak_fail (error, _("Remote ‘%s’ does not have a collection ID set, which is required for P2P distribution of ‘%s’."), @@ -670,6 +670,8 @@ flatpak_builtin_create_usb (int argc, char **argv, GCancellable *cancellable, GE g_autoptr(OstreeCollectionRef) appstream2_collection_ref = NULL; g_autoptr(FlatpakRemoteState) state = NULL; g_autoptr(GError) local_error = NULL; + g_autofree char *appstream_refspec = NULL; + g_autofree char *appstream2_refspec = NULL; g_autofree char *appstream_ref = NULL; g_autofree char *appstream2_ref = NULL; const char **remote_arches; @@ -684,10 +686,13 @@ flatpak_builtin_create_usb (int argc, char **argv, GCancellable *cancellable, GE g_clear_error (&local_error); } - /* Add the ostree-metadata ref to the list */ + /* Add the ostree-metadata ref to the list if available */ metadata_collection_ref = ostree_collection_ref_new (collection_id, OSTREE_REPO_METADATA_REF); - g_hash_table_insert (all_refs, g_steal_pointer (&metadata_collection_ref), - commit_and_subpaths_new (NULL, NULL)); + if (ostree_repo_resolve_collection_ref (src_repo, metadata_collection_ref, FALSE, + OSTREE_REPO_RESOLVE_REV_EXT_NONE, + NULL, NULL, NULL)) + g_hash_table_insert (all_refs, g_steal_pointer (&metadata_collection_ref), + commit_and_subpaths_new (NULL, NULL)); /* Add whatever appstream data is available for each arch */ remote_arches = g_hash_table_lookup (remote_arch_map, remote_name); @@ -697,6 +702,8 @@ flatpak_builtin_create_usb (int argc, char **argv, GCancellable *cancellable, GE g_autoptr(GPtrArray) dirs = NULL; g_autoptr(GError) appstream_error = NULL; g_autoptr(GError) appstream2_error = NULL; + g_autofree char *commit = NULL; + g_autofree char *commit2 = NULL; /* Try to update the appstream data, but don't fail on error because we * want this to work offline. */ @@ -712,25 +719,25 @@ flatpak_builtin_create_usb (int argc, char **argv, GCancellable *cancellable, GE /* Copy the appstream data if it exists. It's optional because without it * the USB will still be useful to the flatpak CLI even if GNOME Software * wouldn't display the contents. */ + appstream_refspec = g_strdup_printf ("%s:appstream/%s", remote_name, current_arch); appstream_ref = g_strdup_printf ("appstream/%s", current_arch); appstream_collection_ref = ostree_collection_ref_new (collection_id, appstream_ref); - if (ostree_repo_resolve_collection_ref (src_repo, appstream_collection_ref, FALSE, - OSTREE_REPO_RESOLVE_REV_EXT_NONE, - NULL, cancellable, &appstream_error)) + if (ostree_repo_resolve_rev (src_repo, appstream_refspec, FALSE, + &commit, &appstream_error)) { g_hash_table_insert (all_refs, g_steal_pointer (&appstream_collection_ref), - commit_and_subpaths_new (NULL, NULL)); + commit_and_subpaths_new (commit, NULL)); } /* Copy the appstream2 data if it exists. */ + appstream2_refspec = g_strdup_printf ("%s:appstream2/%s", remote_name, current_arch); appstream2_ref = g_strdup_printf ("appstream2/%s", current_arch); appstream2_collection_ref = ostree_collection_ref_new (collection_id, appstream2_ref); - if (ostree_repo_resolve_collection_ref (src_repo, appstream2_collection_ref, FALSE, - OSTREE_REPO_RESOLVE_REV_EXT_NONE, - NULL, cancellable, &appstream2_error)) + if (ostree_repo_resolve_rev (src_repo, appstream2_refspec, FALSE, + &commit2, &appstream2_error)) { g_hash_table_insert (all_refs, g_steal_pointer (&appstream2_collection_ref), - commit_and_subpaths_new (NULL, NULL)); + commit_and_subpaths_new (commit2, NULL)); } else { diff --git a/app/flatpak-builtins-remote-add.c b/app/flatpak-builtins-remote-add.c index 1023a3a5..f0932ddb 100644 --- a/app/flatpak-builtins-remote-add.c +++ b/app/flatpak-builtins-remote-add.c @@ -115,10 +115,7 @@ get_config_from_opts (GKeyFile *config, } if (opt_collection_id) - { - g_key_file_set_string (config, group, "collection-id", opt_collection_id); - g_key_file_set_boolean (config, group, "gpg-verify-summary", FALSE); - } + g_key_file_set_string (config, group, "xa.sideload-collection-id", opt_collection_id); if (opt_title) { diff --git a/app/flatpak-builtins-remote-list.c b/app/flatpak-builtins-remote-list.c index dbfeef01..5f01ec91 100644 --- a/app/flatpak-builtins-remote-list.c +++ b/app/flatpak-builtins-remote-list.c @@ -154,7 +154,7 @@ list_remotes (GPtrArray *dirs, Column *columns, GCancellable *cancellable, GErro } else if (strcmp (columns[k].name, "collection") == 0) { - g_autofree char *id = flatpak_dir_get_remote_collection_id (dir, remote_name); + g_autofree char *id = flatpak_dir_get_remote_sideload_collection_id (dir, remote_name); if (id != NULL) flatpak_table_printer_add_column (printer, id); else diff --git a/app/flatpak-builtins-remote-modify.c b/app/flatpak-builtins-remote-modify.c index 5d629c23..d13ddb7d 100644 --- a/app/flatpak-builtins-remote-modify.c +++ b/app/flatpak-builtins-remote-modify.c @@ -130,8 +130,7 @@ get_config_from_opts (FlatpakDir *dir, const char *remote_name, gboolean *change if (opt_collection_id) { - g_key_file_set_string (config, group, "collection-id", opt_collection_id); - g_key_file_set_boolean (config, group, "gpg-verify-summary", FALSE); + g_key_file_set_string (config, group, "xa.sideload-collection-id", opt_collection_id); *changed = TRUE; } diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index 0c7637c7..6f19b39c 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -93,7 +93,6 @@ GType flatpak_deploy_get_type (void); typedef struct { - char *collection_id; /* (nullable) */ char *ref; char *commit; char **subpaths; @@ -117,7 +116,6 @@ typedef struct { typedef struct { char *remote_name; - char *collection_id; char *sideload_collection_id; GVariant *summary; GBytes *summary_sig_bytes; @@ -170,21 +168,6 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakDeploy, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakRelated, flatpak_related_free) G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakRemoteState, flatpak_remote_state_unref) -typedef struct -{ - char *collection_id; - char *ref_name; -} FlatpakCollectionRef; - -FlatpakCollectionRef * flatpak_collection_ref_new (const char *collection_id, - const char *ref_name); -void flatpak_collection_ref_free (FlatpakCollectionRef *ref); -guint flatpak_collection_ref_hash (gconstpointer ref); -gboolean flatpak_collection_ref_equal (gconstpointer ref1, - gconstpointer ref2); - -G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakCollectionRef, flatpak_collection_ref_free) - typedef enum { FLATPAK_HELPER_DEPLOY_FLAGS_NONE = 0, FLATPAK_HELPER_DEPLOY_FLAGS_UPDATE = 1 << 0, @@ -799,7 +782,7 @@ char *flatpak_dir_create_origin_remote (FlatpakDir *self, const char *title, const char *main_ref, GBytes *gpg_data, - const char *collection_id, + const char *sideload_collection_id, gboolean *changed_config, GCancellable *cancellable, GError **error); @@ -809,7 +792,7 @@ gboolean flatpak_dir_create_remote_for_ref_file (FlatpakDir *self, GKeyFile *keyfile, const char *default_arch, char **remote_name_out, - char **collection_id_out, + char **sideload_collection_id_out, char **ref_out, GError **error); gboolean flatpak_dir_create_suggested_remote_for_ref_file (FlatpakDir *self, @@ -854,6 +837,8 @@ char *flatpak_dir_get_remote_icon (FlatpakDir *self, const char *remote_name); char *flatpak_dir_get_remote_collection_id (FlatpakDir *self, const char *remote_name); +char *flatpak_dir_get_remote_sideload_collection_id (FlatpakDir *self, + const char *remote_name); char *flatpak_dir_get_remote_main_ref (FlatpakDir *self, const char *remote_name); gboolean flatpak_dir_get_remote_oci (FlatpakDir *self, diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index 1140986e..d8e37da5 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -312,7 +312,6 @@ flatpak_remote_state_unref (FlatpakRemoteState *remote_state) if (remote_state->refcount == 0) { g_free (remote_state->remote_name); - g_free (remote_state->collection_id); g_free (remote_state->sideload_collection_id); g_clear_pointer (&remote_state->summary, g_variant_unref); g_clear_pointer (&remote_state->summary_sig_bytes, g_bytes_unref); @@ -349,7 +348,7 @@ flatpak_remote_state_ensure_metadata (FlatpakRemoteState *self, /* If the collection ID is NULL the metadata comes from the summary */ if (self->metadata_fetch_error != NULL) error_msg = g_strdup (self->metadata_fetch_error->message); - else if (self->collection_id == NULL && self->summary_fetch_error != NULL) + else if (self->summary_fetch_error != NULL) error_msg = g_strdup_printf ("summary fetch error: %s", self->summary_fetch_error->message); return flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, @@ -401,7 +400,7 @@ flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, VarRefInfoRef info; g_autofree char *checksum = NULL; - if (!flatpak_summary_lookup_ref (self->summary, self->collection_id, ref, &checksum, &info)) + if (!flatpak_summary_lookup_ref (self->summary, NULL, ref, &checksum, &info)) return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, _("No such ref '%s' in remote %s"), ref, self->remote_name); @@ -494,7 +493,7 @@ flatpak_remote_state_match_subrefs (FlatpakRemoteState *self, return g_strdupv ((char **) empty); } - return flatpak_summary_match_subrefs (self->summary, self->collection_id, ref); + return flatpak_summary_match_subrefs (self->summary, NULL, ref); } @@ -3542,7 +3541,6 @@ flatpak_dir_deploy_appstream (FlatpakDir *self, g_autofree char *filter_checksum = NULL; g_autoptr(GRegex) allow_refs = NULL; g_autoptr(GRegex) deny_refs = NULL; - g_autofree char *collection_id = NULL; /* Keep a shared repo lock to avoid prunes removing objects we're relying on * while we do the checkout. This could happen if the ref changes after we @@ -3576,9 +3574,8 @@ flatpak_dir_deploy_appstream (FlatpakDir *self, if (file_info != NULL) old_dir = g_file_info_get_symlink_target (file_info); - collection_id = flatpak_dir_get_remote_collection_id (self, remote); branch = g_strdup_printf ("appstream2/%s", arch); - if (!flatpak_repo_resolve_rev (self->repo, collection_id, remote, branch, TRUE, + if (!flatpak_repo_resolve_rev (self->repo, NULL, remote, branch, TRUE, &new_checksum, cancellable, error)) return FALSE; @@ -3587,7 +3584,7 @@ flatpak_dir_deploy_appstream (FlatpakDir *self, /* Fall back to old branch */ g_clear_pointer (&branch, g_free); branch = g_strdup_printf ("appstream/%s", arch); - if (!flatpak_repo_resolve_rev (self->repo, collection_id, remote, branch, TRUE, + if (!flatpak_repo_resolve_rev (self->repo, NULL, remote, branch, TRUE, &new_checksum, cancellable, error)) return FALSE; do_compress = FALSE; @@ -3856,24 +3853,9 @@ child_repo_ensure_summary (OstreeRepo *child_repo, GCancellable *cancellable, GError **error) { - if (state->collection_id != NULL) - { - /* Regenerate the summary in the child repo because the summary copied - * into the repo by flatpak_dir_pull() is reflective of the refs on the - * remote that was pulled from, which might be a peer remote and might not - * have the full set of refs that was pulled. It's also possible that - * ostree didn't copy the remote summary into the repo at all if the - * "branches" key is set in the remote config. See - * https://github.com/ostreedev/ostree/issues/1461 */ - if (!ostree_repo_regenerate_summary (child_repo, NULL, cancellable, error)) - return FALSE; - } - else - { - if (!flatpak_remote_state_save_summary (state, ostree_repo_get_path (child_repo), - cancellable, error)) - return FALSE; - } + if (!flatpak_remote_state_save_summary (state, ostree_repo_get_path (child_repo), + cancellable, error)) + return FALSE; return TRUE; } @@ -4145,17 +4127,14 @@ flatpak_dir_update_appstream (FlatpakDir *self, * if necessary. */ } - else if ((!gpg_verify_summary && state->collection_id == NULL) || !gpg_verify) + else if (!gpg_verify_summary || !gpg_verify) { /* The remote is not gpg verified, so we don't want to allow installation via a download in the home directory, as there is no way to verify you're not injecting anything into the remote. However, in the case of a remote configured to a local filesystem we can just let the system helper do the installation, as it can then avoid network i/o and be certain the - data comes from the right place. - - If @collection_id is non-%NULL, we can verify the refs in commit - metadata, so don’t need to verify the summary. */ + data comes from the right place. */ if (!g_str_has_prefix (url, "file:")) return flatpak_fail_error (error, FLATPAK_ERROR_UNTRUSTED, _("Can't pull from untrusted non-gpg verified remote")); } @@ -4176,7 +4155,7 @@ flatpak_dir_update_appstream (FlatpakDir *self, if (!child_repo_ensure_summary (child_repo, state, cancellable, error)) return FALSE; - if (!flatpak_repo_resolve_rev (child_repo, state->collection_id, remote, used_branch, TRUE, + if (!flatpak_repo_resolve_rev (child_repo, NULL, remote, used_branch, TRUE, &new_checksum, cancellable, error)) return FALSE; @@ -4220,7 +4199,7 @@ flatpak_dir_update_appstream (FlatpakDir *self, return FALSE; } - if (!flatpak_repo_resolve_rev (self->repo, state->collection_id, remote, used_branch, TRUE, + if (!flatpak_repo_resolve_rev (self->repo, NULL, remote, used_branch, TRUE, &new_checksum, cancellable, error)) return FALSE; @@ -4365,7 +4344,6 @@ repo_pull (OstreeRepo *self, g_autoptr(GVariant) old_commit = NULL; g_autoptr(GVariant) new_commit = NULL; const char *revs_to_fetch[2]; - g_autofree gchar *collection_id = NULL; g_autoptr(GError) dummy_error = NULL; GVariantBuilder builder; g_autoptr(GVariant) options = NULL; @@ -4379,10 +4357,7 @@ repo_pull (OstreeRepo *self, /* We always want this on for every type of pull */ flags |= OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES; - if (!repo_get_remote_collection_id (self, remote_name, &collection_id, NULL)) - g_clear_pointer (&collection_id, g_free); - - if (!flatpak_repo_resolve_rev (self, collection_id, remote_name, ref_to_fetch, TRUE, + if (!flatpak_repo_resolve_rev (self, NULL, remote_name, ref_to_fetch, TRUE, ¤t_checksum, cancellable, error)) return FALSE; @@ -5115,7 +5090,7 @@ flatpak_dir_pull (FlatpakDir *self, if (!ostree_repo_prepare_transaction (repo, NULL, cancellable, error)) goto out; - flatpak_repo_resolve_rev (repo, state->collection_id, state->remote_name, ref, TRUE, + flatpak_repo_resolve_rev (repo, NULL, state->remote_name, ref, TRUE, ¤t_checksum, NULL, NULL); if (!repo_pull (repo, state->remote_name, @@ -5190,12 +5165,10 @@ repo_pull_local_untrusted (FlatpakDir *self, const OstreeRepoPullFlags flags = OSTREE_REPO_PULL_FLAGS_UNTRUSTED | OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES; GVariantBuilder builder; g_autoptr(GVariant) options = NULL; - g_auto(GVariantBuilder) refs_builder = FLATPAK_VARIANT_BUILDER_INITIALIZER; gboolean res; g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); const char *refs[2] = { NULL, NULL }; const char *commits[2] = { NULL, NULL }; - g_autofree char *collection_id = NULL; g_autoptr(GError) dummy_error = NULL; /* The ostree fetcher asserts if error is NULL */ @@ -5204,27 +5177,13 @@ repo_pull_local_untrusted (FlatpakDir *self, g_assert (progress != NULL); - if (!repo_get_remote_collection_id (repo, remote_name, &collection_id, error)) - return FALSE; + refs[0] = ref; + commits[0] = checksum; - if (collection_id != NULL) - { - g_variant_builder_init (&refs_builder, G_VARIANT_TYPE ("a(sss)")); - g_variant_builder_add (&refs_builder, "(sss)", collection_id, ref, checksum); - - g_variant_builder_add (&builder, "{s@v}", "collection-refs", - g_variant_new_variant (g_variant_builder_end (&refs_builder))); - } - else - { - refs[0] = ref; - commits[0] = checksum; - - g_variant_builder_add (&builder, "{s@v}", "refs", - g_variant_new_variant (g_variant_new_strv ((const char * const *) refs, -1))); - g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", - g_variant_new_variant (g_variant_new_strv ((const char * const *) commits, -1))); - } + g_variant_builder_add (&builder, "{s@v}", "refs", + g_variant_new_variant (g_variant_new_strv ((const char * const *) refs, -1))); + g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", + g_variant_new_variant (g_variant_new_strv ((const char * const *) commits, -1))); g_variant_builder_add (&builder, "{s@v}", "flags", g_variant_new_variant (g_variant_new_int32 (flags))); @@ -5233,7 +5192,7 @@ repo_pull_local_untrusted (FlatpakDir *self, g_variant_builder_add (&builder, "{s@v}", "gpg-verify", g_variant_new_variant (g_variant_new_boolean (TRUE))); g_variant_builder_add (&builder, "{s@v}", "gpg-verify-summary", - g_variant_new_variant (g_variant_new_boolean (collection_id == NULL))); + g_variant_new_variant (g_variant_new_boolean (TRUE))); g_variant_builder_add (&builder, "{s@v}", "inherit-transaction", g_variant_new_variant (g_variant_new_boolean (TRUE))); g_variant_builder_add (&builder, "{s@v}", "update-frequency", @@ -5274,7 +5233,6 @@ flatpak_dir_pull_untrusted_local (FlatpakDir *self, g_autofree char *current_checksum = NULL; gboolean gpg_verify_summary; gboolean gpg_verify; - g_autofree char *collection_id = NULL; char *summary_data = NULL; char *summary_sig_data = NULL; gsize summary_data_size, summary_sig_data_size; @@ -5306,15 +5264,12 @@ flatpak_dir_pull_untrusted_local (FlatpakDir *self, &gpg_verify_summary, error)) return FALSE; - if (!repo_get_remote_collection_id (self->repo, remote_name, &collection_id, error)) - return FALSE; - if (!ostree_repo_remote_get_gpg_verify (self->repo, remote_name, &gpg_verify, error)) return FALSE; /* This was verified in the client, but lets do it here too */ - if ((!gpg_verify_summary && collection_id == NULL) || !gpg_verify) + if (!gpg_verify_summary || !gpg_verify) return flatpak_fail_error (error, FLATPAK_ERROR_UNTRUSTED, _("Can't pull from untrusted non-gpg verified remote")); /* We verify the summary manually before anything else to make sure @@ -5349,7 +5304,7 @@ flatpak_dir_pull_untrusted_local (FlatpakDir *self, g_clear_object (&gpg_result); - if (!flatpak_repo_resolve_rev (self->repo, collection_id, remote_name, ref, TRUE, + if (!flatpak_repo_resolve_rev (self->repo, NULL, remote_name, ref, TRUE, ¤t_checksum, NULL, error)) return FALSE; @@ -5361,22 +5316,11 @@ flatpak_dir_pull_untrusted_local (FlatpakDir *self, if (!ostree_repo_open (src_repo, cancellable, error)) return FALSE; - if (collection_id == NULL) - { - summary = g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, summary_bytes, FALSE)); - if (!flatpak_summary_lookup_ref (summary, - NULL, - ref, - &checksum, NULL)) - return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, _("No such ref '%s' in remote %s"), - ref, remote_name); - } - else - { - if (!flatpak_repo_resolve_rev (src_repo, collection_id, remote_name, ref, FALSE, - &checksum, NULL, error)) - return FALSE; - } + summary = g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, summary_bytes, FALSE)); + if (!flatpak_summary_lookup_ref (summary, NULL, ref, &checksum, NULL)) + return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, + _("No such ref '%s' in remote %s"), + ref, remote_name); if (gpg_verify) { @@ -5393,48 +5337,6 @@ flatpak_dir_pull_untrusted_local (FlatpakDir *self, if (!ostree_repo_load_commit (src_repo, checksum, &new_commit, NULL, error)) return FALSE; - if (gpg_verify) - { - /* Verify the commit’s binding to the ref and to the repo. See - * verify_bindings() in libostree. */ - g_autoptr(GVariant) new_commit_metadata = g_variant_get_child_value (new_commit, 0); - g_autofree const char **commit_refs = NULL; - - if (!g_variant_lookup (new_commit_metadata, - OSTREE_COMMIT_META_KEY_REF_BINDING, - "^a&s", - &commit_refs)) - { - /* Early return here - if the remote collection ID is NULL, then - * we certainly will not verify the collection binding in the - * commit. - */ - if (collection_id != NULL) - return flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("Expected commit metadata to have ref binding information, found none")); - } - - if (collection_id != NULL && - !g_strv_contains ((const char * const *) commit_refs, ref)) - return flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("Commit has no requested ref ‘%s’ in ref binding metadata"), - ref); - - if (collection_id != NULL) - { - const char *commit_collection_id; - if (!g_variant_lookup (new_commit_metadata, - "ostree.collection-binding", - "&s", - &commit_collection_id)) - return flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("Expected commit metadata to have collection ID binding information, found none")); - if (!g_str_equal (commit_collection_id, collection_id)) - return flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, - _("Commit has collection ID ‘%s’ in collection binding " - "metadata, while the remote it came from has " - "collection ID ‘%s’"), - commit_collection_id, collection_id); - } - } - if (old_commit) { guint64 old_timestamp; @@ -5807,10 +5709,8 @@ flatpak_dir_read_latest_commit (FlatpakDir *self, { g_autofree char *res = NULL; g_autoptr(GVariant) commit_data = NULL; - g_autofree char *collection_id = NULL; - collection_id = flatpak_dir_get_remote_collection_id (self, remote); - if (!flatpak_repo_resolve_rev (self->repo, collection_id, remote, ref, FALSE, + if (!flatpak_repo_resolve_rev (self->repo, NULL, remote, ref, FALSE, &res, cancellable, error)) return NULL; @@ -5834,10 +5734,8 @@ flatpak_dir_read_latest (FlatpakDir *self, { g_autofree char *alt_id = NULL; g_autofree char *res = NULL; - g_autofree char *collection_id = NULL; - collection_id = flatpak_dir_get_remote_collection_id (self, remote); - if (!flatpak_repo_resolve_rev (self->repo, collection_id, remote, ref, FALSE, + if (!flatpak_repo_resolve_rev (self->repo, NULL, remote, ref, FALSE, &res, cancellable, error)) return NULL; @@ -8438,17 +8336,14 @@ flatpak_dir_install (FlatpakDir *self, if (!flatpak_dir_mirror_oci (self, registry, state, ref, NULL, token, progress, cancellable, error)) return FALSE; } - else if ((!gpg_verify_summary && state->collection_id == NULL) || !gpg_verify) + else if (!gpg_verify_summary || !gpg_verify) { /* The remote is not gpg verified, so we don't want to allow installation via a download in the home directory, as there is no way to verify you're not injecting anything into the remote. However, in the case of a remote configured to a local filesystem we can just let the system helper do the installation, as it can then avoid network i/o and be certain the - data comes from the right place. - - If a collection ID is available, we can verify the refs in commit - metadata. */ + data comes from the right place. */ if (g_str_has_prefix (url, "file:")) helper_flags |= FLATPAK_HELPER_DEPLOY_FLAGS_LOCAL_PULL; else @@ -8647,7 +8542,7 @@ flatpak_dir_ensure_bundle_remote (FlatpakDir *self, GBytes *gpg_data = NULL; g_autofree char *to_checksum = NULL; g_autofree char *remote = NULL; - g_autofree char *collection_id = NULL; + g_autofree char *sideload_collection_id = NULL; if (!flatpak_dir_ensure_repo (self, cancellable, error)) return NULL; @@ -8657,7 +8552,7 @@ flatpak_dir_ensure_bundle_remote (FlatpakDir *self, &origin, NULL, &fp_metadata, NULL, &included_gpg_data, - &collection_id, + &sideload_collection_id, error); if (metadata == NULL) return NULL; @@ -8695,7 +8590,7 @@ flatpak_dir_ensure_bundle_remote (FlatpakDir *self, basename, ref, gpg_data, - collection_id, + sideload_collection_id, &created_remote, cancellable, error); @@ -9010,7 +8905,7 @@ flatpak_dir_check_for_update (FlatpakDir *self, if (no_pull) { - if (!flatpak_repo_resolve_rev (self->repo, state->collection_id, state->remote_name, + if (!flatpak_repo_resolve_rev (self->repo, NULL, state->remote_name, ref, FALSE, &latest_rev, NULL, NULL)) { g_set_error (error, FLATPAK_ERROR, FLATPAK_ERROR_ALREADY_INSTALLED, @@ -9148,7 +9043,7 @@ flatpak_dir_update (FlatpakDir *self, if (!flatpak_dir_mirror_oci (self, registry, state, ref, NULL, token, progress, cancellable, error)) return FALSE; } - else if ((!gpg_verify_summary && state->collection_id == NULL) || !gpg_verify) + else if (!gpg_verify_summary || !gpg_verify) { /* The remote is not gpg verified, so we don't want to allow installation via a download in the home directory, as there is no way to verify you're not @@ -10645,8 +10540,6 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, { if (!flatpak_dir_has_remote (self, remote_or_uri, error)) return NULL; - if (!repo_get_remote_collection_id (self->repo, remote_or_uri, &state->collection_id, error)) - return NULL; if (!repo_get_remote_sideload_collection_id (self->repo, remote_or_uri, &state->sideload_collection_id, error)) return NULL; if (!flatpak_dir_lookup_remote_filter (self, remote_or_uri, FALSE, NULL, &state->allow_refs, &state->deny_refs, error)) @@ -11153,6 +11046,20 @@ flatpak_dir_get_remote_collection_id (FlatpakDir *self, return collection_id; } +char * +flatpak_dir_get_remote_sideload_collection_id (FlatpakDir *self, + const char *remote_name) +{ + char *collection_id = NULL; + + if (!flatpak_dir_ensure_repo (self, NULL, NULL)) + return NULL; + + repo_get_remote_sideload_collection_id (self->repo, remote_name, &collection_id, NULL); + + return collection_id; +} + char ** flatpak_dir_find_remote_refs (FlatpakDir *self, const char *remote, @@ -12038,12 +11945,10 @@ origin_remote_matches (OstreeRepo *repo, const char *remote_name, const char *url, const char *main_ref, - gboolean gpg_verify, - const char *collection_id) + gboolean gpg_verify) { g_autofree char *real_url = NULL; g_autofree char *real_main_ref = NULL; - g_autofree char *real_collection_id = NULL; gboolean noenumerate; gboolean real_gpg_verify; @@ -12082,14 +11987,6 @@ origin_remote_matches (OstreeRepo *repo, g_strcmp0 (main_ref, real_main_ref) != 0) return FALSE; - /* Must match collection ID */ - if (ostree_repo_get_remote_option (repo, remote_name, - "collection-id", - NULL, &real_collection_id, - NULL) && - g_strcmp0 (main_ref, real_main_ref) != 0) - return FALSE; - return TRUE; } @@ -12100,7 +11997,7 @@ create_origin_remote_config (OstreeRepo *repo, const char *title, const char *main_ref, gboolean gpg_verify, - const char *collection_id, + const char *sideload_collection_id, GKeyFile **new_config) { g_autofree char *remote = NULL; @@ -12124,7 +12021,7 @@ create_origin_remote_config (OstreeRepo *repo, name = g_strdup_printf ("%s%d-origin", prefix, version); version++; - if (origin_remote_matches (repo, name, url, main_ref, gpg_verify, collection_id)) + if (origin_remote_matches (repo, name, url, main_ref, gpg_verify)) return g_steal_pointer (&name); if (remotes == NULL || @@ -12142,15 +12039,13 @@ create_origin_remote_config (OstreeRepo *repo, g_key_file_set_string (*new_config, group, "xa.title", title); g_key_file_set_string (*new_config, group, "xa.noenumerate", "true"); g_key_file_set_string (*new_config, group, "xa.prio", "0"); - /* Don’t enable summary verification if a collection ID is set, as collection - * IDs enable the verification of refs from commit metadata instead. */ - g_key_file_set_string (*new_config, group, "gpg-verify-summary", (gpg_verify && collection_id == NULL) ? "true" : "false"); + g_key_file_set_string (*new_config, group, "gpg-verify-summary", gpg_verify ? "true" : "false"); g_key_file_set_string (*new_config, group, "gpg-verify", gpg_verify ? "true" : "false"); if (main_ref) g_key_file_set_string (*new_config, group, "xa.main-ref", main_ref); - if (collection_id) - g_key_file_set_string (*new_config, group, "collection-id", collection_id); + if (sideload_collection_id) + g_key_file_set_string (*new_config, group, "xa.sideload-collection-id", sideload_collection_id); return g_steal_pointer (&remote); } @@ -12162,7 +12057,7 @@ flatpak_dir_create_origin_remote (FlatpakDir *self, const char *title, const char *main_ref, GBytes *gpg_data, - const char *collection_id, + const char *sideload_collection_id, gboolean *changed_config, GCancellable *cancellable, GError **error) @@ -12170,7 +12065,7 @@ flatpak_dir_create_origin_remote (FlatpakDir *self, g_autoptr(GKeyFile) new_config = NULL; g_autofree char *remote = NULL; - remote = create_origin_remote_config (self->repo, url, id, title, main_ref, gpg_data != NULL, collection_id, &new_config); + remote = create_origin_remote_config (self->repo, url, id, title, main_ref, gpg_data != NULL, sideload_collection_id, &new_config); if (new_config && !flatpak_dir_modify_remote (self, remote, new_config, @@ -12289,7 +12184,7 @@ flatpak_dir_create_remote_for_ref_file (FlatpakDir *self, GKeyFile *keyfile, const char *default_arch, char **remote_name_out, - char **collection_id_out, + char **sideload_collection_id_out, char **ref_out, GError **error) { @@ -12332,8 +12227,8 @@ flatpak_dir_create_remote_for_ref_file (FlatpakDir *self, return FALSE; } - if (collection_id_out != NULL) - *collection_id_out = g_steal_pointer (&collection_id); + if (sideload_collection_id_out != NULL) + *sideload_collection_id_out = g_steal_pointer (&collection_id); *remote_name_out = g_steal_pointer (&remote); *ref_out = (char *) g_steal_pointer (&ref); @@ -12393,7 +12288,6 @@ flatpak_dir_find_remote_by_uri (FlatpakDir *self, { const char *remote = remotes[i]; g_autofree char *remote_uri = NULL; - g_autofree char *remote_collection_id = NULL; if (!ostree_repo_remote_get_url (self->repo, remote, @@ -12970,7 +12864,7 @@ flatpak_dir_update_remote_configuration_for_state (FlatpakDir *self, if (strcmp (key, "xa.redirect-url") == 0) g_ptr_array_add (updated_params, g_strdup ("url")); else if (strcmp (key, OSTREE_META_KEY_DEPLOY_COLLECTION_ID) == 0) - g_ptr_array_add (updated_params, g_strdup ("collection-id")); + g_ptr_array_add (updated_params, g_strdup ("xa.sideload-collection-id")); else g_ptr_array_add (updated_params, g_strdup (key)); g_ptr_array_add (updated_params, g_strdup (value)); @@ -13102,7 +12996,7 @@ flatpak_dir_update_remote_configuration (FlatpakDir *self, if (!ostree_repo_remote_get_gpg_verify (self->repo, remote, &gpg_verify, error)) return FALSE; - if ((!gpg_verify_summary && state->collection_id == NULL) || !gpg_verify) + if (!gpg_verify_summary || !gpg_verify) { g_debug ("Ignoring automatic updates for system-helper remotes without gpg signatures"); return TRUE; @@ -13111,7 +13005,7 @@ flatpak_dir_update_remote_configuration (FlatpakDir *self, if (!flatpak_dir_update_remote_configuration_for_state (self, state, TRUE, &has_changed, cancellable, error)) return FALSE; - if (state->collection_id == NULL && state->summary_sig_bytes == NULL) + if (state->summary_sig_bytes == NULL) { g_debug ("Can't update remote configuration as user, no GPG signature"); return TRUE; @@ -13269,7 +13163,6 @@ flatpak_dir_fetch_remote_commit (FlatpakDir *self, void flatpak_related_free (FlatpakRelated *self) { - g_free (self->collection_id); g_free (self->ref); g_free (self->commit); g_strfreev (self->subpaths); @@ -13280,7 +13173,6 @@ static void add_related (FlatpakDir *self, GPtrArray *related, const char *extension, - const char *extension_collection_id, const char *extension_ref, const char *checksum, gboolean no_autodownload, @@ -13351,7 +13243,6 @@ add_related (FlatpakDir *self, subpaths = flatpak_subpaths_merge ((char **) old_subpaths, extra_subpaths); rel = g_new0 (FlatpakRelated, 1); - rel->collection_id = g_strdup (extension_collection_id); rel->ref = g_strdup (extension_ref); rel->commit = g_strdup (checksum); rel->subpaths = g_steal_pointer (&subpaths); @@ -13476,7 +13367,6 @@ flatpak_dir_find_remote_related_for_metadata (FlatpakDir *self, FLATPAK_METADATA_KEY_AUTODELETE, NULL); gboolean locale_subset = g_key_file_get_boolean (metakey, groups[i], FLATPAK_METADATA_KEY_LOCALE_SUBSET, NULL); - g_autofree char *extension_collection_id = NULL; const char *default_branches[] = { NULL, NULL}; const char **branches; int branch_i; @@ -13495,23 +13385,6 @@ flatpak_dir_find_remote_related_for_metadata (FlatpakDir *self, branches = default_branches; } - extension_collection_id = g_key_file_get_string (metakey, groups[i], - FLATPAK_METADATA_KEY_COLLECTION_ID, NULL); - - /* For the moment, none of the related ref machinery handles - * collection IDs which don’t match the original ref. */ - if (extension_collection_id != NULL && *extension_collection_id != '\0' && - g_strcmp0 (extension_collection_id, state->collection_id) != 0) - { - g_debug ("Skipping related extension ‘%s’ because it’s in collection " - "‘%s’ which does not match the current remote ‘%s’.", - extension, extension_collection_id, state->collection_id); - continue; - } - - g_clear_pointer (&extension_collection_id, g_free); - extension_collection_id = g_strdup (state->collection_id); - for (branch_i = 0; branches[branch_i] != NULL; branch_i++) { g_autofree char *extension_ref = NULL; @@ -13523,7 +13396,7 @@ flatpak_dir_find_remote_related_for_metadata (FlatpakDir *self, if (flatpak_remote_state_lookup_ref (state, extension_ref, &checksum, NULL, NULL, NULL, NULL)) { if (flatpak_filters_allow_ref (NULL, masked, extension_ref)) - add_related (self, related, extension, extension_collection_id, extension_ref, checksum, + add_related (self, related, extension, extension_ref, checksum, no_autodownload, download_if, autoprune_unless, autodelete, locale_subset); } else if (subdirectories) @@ -13536,7 +13409,7 @@ flatpak_dir_find_remote_related_for_metadata (FlatpakDir *self, if (flatpak_remote_state_lookup_ref (state, refs[j], &subref_checksum, NULL, NULL, NULL, NULL) && flatpak_filters_allow_ref (NULL, masked, refs[j])) - add_related (self, related, extension, extension_collection_id, refs[j], subref_checksum, + add_related (self, related, extension, refs[j], subref_checksum, no_autodownload, download_if, autoprune_unless, autodelete, locale_subset); } } @@ -13672,29 +13545,6 @@ local_match_prefix (FlatpakDir *self, return matches; } -static gboolean -repo_get_ref_collection_binding (OstreeRepo *repo, - const char *ref, - const char *commit, - char **out_collection_id, - GError **error) -{ - g_autoptr(GVariant) commit_v = NULL; - g_autoptr(GVariant) commit_metadata = NULL; - const char *collection_id = NULL; - - if (!ostree_repo_load_commit (repo, commit, &commit_v, NULL, error)) - return FALSE; - - commit_metadata = g_variant_get_child_value (commit_v, 0); - g_variant_lookup (commit_metadata, "ostree.collection-binding", "&s", &collection_id); - - if (out_collection_id) - *out_collection_id = g_strdup (collection_id); - - return TRUE; -} - GPtrArray * flatpak_dir_find_local_related_for_metadata (FlatpakDir *self, const char *ref, @@ -13707,17 +13557,11 @@ flatpak_dir_find_local_related_for_metadata (FlatpakDir *self, int i; g_auto(GStrv) parts = NULL; g_autoptr(GPtrArray) related = g_ptr_array_new_with_free_func ((GDestroyNotify) flatpak_related_free); - g_autofree char *collection_id = NULL; g_auto(GStrv) groups = NULL; if (!flatpak_dir_ensure_repo (self, cancellable, error)) return NULL; - /* Derive the collection ID from the commit metadata. This will act as - * a sanity check on the summary ref lookup. */ - if (!repo_get_ref_collection_binding (flatpak_dir_get_repo (self), ref, commit, &collection_id, error)) - return NULL; - parts = flatpak_decompose_ref (ref, error); if (parts == NULL) return NULL; @@ -13748,7 +13592,6 @@ flatpak_dir_find_local_related_for_metadata (FlatpakDir *self, FLATPAK_METADATA_KEY_AUTODELETE, NULL); gboolean locale_subset = g_key_file_get_boolean (metakey, groups[i], FLATPAK_METADATA_KEY_LOCALE_SUBSET, NULL); - g_autofree char *extension_collection_id = NULL; const char *default_branches[] = { NULL, NULL}; const char **branches; int branch_i; @@ -13767,24 +13610,6 @@ flatpak_dir_find_local_related_for_metadata (FlatpakDir *self, branches = default_branches; } - extension_collection_id = g_key_file_get_string (metakey, groups[i], - FLATPAK_METADATA_KEY_COLLECTION_ID, NULL); - - /* As we’re looking locally, we can’t support extension - * collection IDs which don’t match the current remote (since the - * associated refs could be anywhere). */ - if (extension_collection_id != NULL && *extension_collection_id != '\0' && - g_strcmp0 (extension_collection_id, collection_id) != 0) - { - g_debug ("Skipping related extension ‘%s’ because it’s in collection " - "‘%s’ which does not match the current remote ‘%s’.", - extension, extension_collection_id, collection_id); - continue; - } - - g_clear_pointer (&extension_collection_id, g_free); - extension_collection_id = g_strdup (collection_id); - for (branch_i = 0; branches[branch_i] != NULL; branch_i++) { g_autofree char *extension_ref = NULL; @@ -13794,7 +13619,7 @@ flatpak_dir_find_local_related_for_metadata (FlatpakDir *self, extension_ref = g_build_filename ("runtime", extension, parts[2], branch, NULL); if (flatpak_repo_resolve_rev (self->repo, - collection_id, + NULL, remote_name, extension_ref, FALSE, @@ -13802,7 +13627,7 @@ flatpak_dir_find_local_related_for_metadata (FlatpakDir *self, NULL, NULL)) { - add_related (self, related, extension, extension_collection_id, extension_ref, + add_related (self, related, extension, extension_ref, checksum, no_autodownload, download_if, autoprune_unless, autodelete, locale_subset); } else if ((deploy_data = flatpak_dir_get_deploy_data (self, extension_ref, @@ -13814,7 +13639,7 @@ flatpak_dir_find_local_related_for_metadata (FlatpakDir *self, * --force */ checksum = g_strdup (flatpak_deploy_data_get_commit (deploy_data)); - add_related (self, related, extension, extension_collection_id, extension_ref, + add_related (self, related, extension, extension_ref, checksum, no_autodownload, download_if, autoprune_unless, autodelete, locale_subset); } else if (subdirectories) @@ -13826,7 +13651,7 @@ flatpak_dir_find_local_related_for_metadata (FlatpakDir *self, g_autoptr(GBytes) match_deploy_data = NULL; if (flatpak_repo_resolve_rev (self->repo, - collection_id, + NULL, remote_name, match, FALSE, @@ -13834,8 +13659,7 @@ flatpak_dir_find_local_related_for_metadata (FlatpakDir *self, NULL, NULL)) { - add_related (self, related, extension, - extension_collection_id, match, match_checksum, + add_related (self, related, extension, match, match_checksum, no_autodownload, download_if, autoprune_unless, autodelete, locale_subset); } else if ((match_deploy_data = flatpak_dir_get_deploy_data (self, match, @@ -13846,8 +13670,7 @@ flatpak_dir_find_local_related_for_metadata (FlatpakDir *self, * not have a ref in the repo */ match_checksum = g_strdup (flatpak_deploy_data_get_commit (match_deploy_data)); - add_related (self, related, extension, - extension_collection_id, match, match_checksum, + add_related (self, related, extension, match, match_checksum, no_autodownload, download_if, autoprune_unless, autodelete, locale_subset); } } @@ -14228,52 +14051,6 @@ flatpak_dir_get_locale_subpaths (FlatpakDir *self) return subpaths; } -/* The flatpak_collection_ref_* methods were copied from the - * ostree_collection_ref_* ones */ -FlatpakCollectionRef * -flatpak_collection_ref_new (const gchar *collection_id, - const gchar *ref_name) -{ - g_autoptr(FlatpakCollectionRef) collection_ref = NULL; - - collection_ref = g_new0 (FlatpakCollectionRef, 1); - collection_ref->collection_id = g_strdup (collection_id); - collection_ref->ref_name = g_strdup (ref_name); - - return g_steal_pointer (&collection_ref); -} - -void -flatpak_collection_ref_free (FlatpakCollectionRef *ref) -{ - g_return_if_fail (ref != NULL); - - g_free (ref->collection_id); - g_free (ref->ref_name); - g_free (ref); -} - -guint -flatpak_collection_ref_hash (gconstpointer ref) -{ - const FlatpakCollectionRef *_ref = ref; - - if (_ref->collection_id != NULL) - return g_str_hash (_ref->collection_id) ^ g_str_hash (_ref->ref_name); - else - return g_str_hash (_ref->ref_name); -} - -gboolean -flatpak_collection_ref_equal (gconstpointer ref1, - gconstpointer ref2) -{ - const FlatpakCollectionRef *_ref1 = ref1, *_ref2 = ref2; - - return g_strcmp0 (_ref1->collection_id, _ref2->collection_id) == 0 && - g_strcmp0 (_ref1->ref_name, _ref2->ref_name) == 0; -} - void flatpak_dir_set_source_pid (FlatpakDir *self, pid_t pid) diff --git a/common/flatpak-installation.c b/common/flatpak-installation.c index 0d21504a..20573b89 100644 --- a/common/flatpak-installation.c +++ b/common/flatpak-installation.c @@ -739,7 +739,7 @@ get_ref (FlatpakDir *dir, g_autofree char *deploy_subdirname = NULL; g_autoptr(GBytes) deploy_data = NULL; g_autofree const char **subpaths = NULL; - g_autofree char *collection_id = NULL; + g_autofree char *sideload_collection_id = NULL; gboolean is_current = FALSE; guint64 installed_size = 0; @@ -769,12 +769,12 @@ get_ref (FlatpakDir *dir, latest_commit = flatpak_dir_read_latest (dir, origin, full_ref, &latest_alt_id, NULL, NULL); - collection_id = flatpak_dir_get_remote_collection_id (dir, origin); + sideload_collection_id = flatpak_dir_get_remote_sideload_collection_id (dir, origin); return flatpak_installed_ref_new (full_ref, alt_id ? alt_id : commit, latest_alt_id ? latest_alt_id : latest_commit, - origin, collection_id, subpaths, + origin, sideload_collection_id, subpaths, deploy_path, installed_size, is_current, @@ -992,16 +992,6 @@ async_result_cb (GObject *obj, *result_out = g_object_ref (result); } -/* Useful as the #GDestroyNotify in NULL-terminated pointer arrays. */ -static void -_ostree_collection_ref_free0 (OstreeCollectionRef *ref) -{ - if (ref == NULL) - return; - - ostree_collection_ref_free (ref); -} - /** * flatpak_installation_list_installed_refs_for_update: * @self: a #FlatpakInstallation @@ -1033,10 +1023,6 @@ flatpak_installation_list_installed_refs_for_update (FlatpakInstallation *self, g_autoptr(GHashTable) remote_states = NULL; /* (element-type utf8 FlatpakRemoteState) */ int i, j; g_autoptr(FlatpakDir) dir = NULL; - g_auto(OstreeRepoFinderResultv) results = NULL; - g_autoptr(GAsyncResult) result = NULL; - g_autoptr(GPtrArray) collection_refs = NULL; /* (element-type OstreeCollectionRef) */ - g_autoptr(GString) refs_str = NULL; remote_commits = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); @@ -1049,7 +1035,6 @@ flatpak_installation_list_installed_refs_for_update (FlatpakInstallation *self, FlatpakRemote *remote = g_ptr_array_index (remotes, i); g_autoptr(GPtrArray) refs = NULL; g_autoptr(GError) local_error = NULL; - g_autofree char *collection_id = NULL; const char *remote_name = flatpak_remote_get_name (remote); if (flatpak_remote_get_disabled (remote)) @@ -1912,7 +1897,6 @@ flatpak_installation_install_ref_file (FlatpakInstallation *self, g_autofree char *remote = NULL; g_autofree char *ref = NULL; g_autofree char *collection_id = NULL; - g_autoptr(FlatpakCollectionRef) coll_ref = NULL; g_autoptr(GKeyFile) keyfile = g_key_file_new (); dir = flatpak_installation_get_dir (self, error); @@ -1930,7 +1914,7 @@ flatpak_installation_install_ref_file (FlatpakInstallation *self, if (!flatpak_installation_drop_caches (self, cancellable, error)) return NULL; - return flatpak_remote_ref_new (ref, NULL, remote, NULL); + return flatpak_remote_ref_new (ref, NULL, remote, collection_id, NULL); } /** @@ -2504,7 +2488,7 @@ flatpak_installation_list_remote_refs_sync_full (FlatpakInstallation *self, const gchar *ref_commit = value; FlatpakRemoteRef *ref; - ref = flatpak_remote_ref_new (ref_name, ref_commit, remote_or_uri, state); + ref = flatpak_remote_ref_new (ref_name, ref_commit, remote_or_uri, state->sideload_collection_id, state); if (ref) g_ptr_array_add (refs, ref); @@ -2605,7 +2589,7 @@ flatpak_installation_fetch_remote_ref_sync_full (FlatpakInstallation *self, checksum = g_hash_table_lookup (ht, ref); if (checksum != NULL) - return flatpak_remote_ref_new (ref, checksum, remote_name, state); + return flatpak_remote_ref_new (ref, checksum, remote_name, state->sideload_collection_id, state); g_set_error (error, FLATPAK_ERROR, FLATPAK_ERROR_REF_NOT_FOUND, "Reference %s doesn't exist in remote", ref); @@ -2785,7 +2769,7 @@ flatpak_installation_list_remote_related_refs_sync (FlatpakInstallation *self, FlatpakRelated *rel = g_ptr_array_index (related, i); FlatpakRelatedRef *rel_ref; - rel_ref = flatpak_related_ref_new (rel->collection_id, rel->ref, rel->commit, + rel_ref = flatpak_related_ref_new (rel->ref, rel->commit, rel->subpaths, rel->download, rel->delete); if (rel_ref) @@ -2845,7 +2829,7 @@ flatpak_installation_list_installed_related_refs_sync (FlatpakInstallation *self FlatpakRelated *rel = g_ptr_array_index (related, i); FlatpakRelatedRef *rel_ref; - rel_ref = flatpak_related_ref_new (rel->collection_id, rel->ref, rel->commit, + rel_ref = flatpak_related_ref_new (rel->ref, rel->commit, rel->subpaths, rel->download, rel->delete); if (rel_ref) diff --git a/common/flatpak-installed-ref.c b/common/flatpak-installed-ref.c index 44c521f9..f32cfd72 100644 --- a/common/flatpak-installed-ref.c +++ b/common/flatpak-installed-ref.c @@ -698,7 +698,7 @@ flatpak_installed_ref_new (const char *full_ref, const char *commit, const char *latest_commit, const char *origin, - const char *collection_id, + const char *sideload_collection_id, const char **subpaths, const char *deploy_dir, guint64 installed_size, @@ -733,7 +733,7 @@ flatpak_installed_ref_new (const char *full_ref, "commit", commit, "latest-commit", latest_commit, "origin", origin, - "collection-id", collection_id, + "collection-id", sideload_collection_id, "subpaths", subpaths, "is-current", is_current, "installed-size", installed_size, diff --git a/common/flatpak-related-ref-private.h b/common/flatpak-related-ref-private.h index 4a610d1f..70ad3fb1 100644 --- a/common/flatpak-related-ref-private.h +++ b/common/flatpak-related-ref-private.h @@ -28,8 +28,7 @@ #include #include -FlatpakRelatedRef *flatpak_related_ref_new (const char *collection_id, - const char *full_ref, +FlatpakRelatedRef *flatpak_related_ref_new (const char *full_ref, const char *commit, char **subpaths, gboolean download, diff --git a/common/flatpak-related-ref.c b/common/flatpak-related-ref.c index 27f52f3f..bdf2106d 100644 --- a/common/flatpak-related-ref.c +++ b/common/flatpak-related-ref.c @@ -256,7 +256,6 @@ flatpak_related_ref_get_subpaths (FlatpakRelatedRef *self) /** * flatpak_related_ref_new: - * @collection_id: (nullable): the collection ID * @full_ref: a full ref to refer to * @commit: (nullable): a commit ID to refer to * @subpaths: (nullable): a nul-terminated array of subpaths @@ -268,8 +267,7 @@ flatpak_related_ref_get_subpaths (FlatpakRelatedRef *self) * Returns: a new ref */ FlatpakRelatedRef * -flatpak_related_ref_new (const char *collection_id, - const char *full_ref, +flatpak_related_ref_new (const char *full_ref, const char *commit, char **subpaths, gboolean download, @@ -297,7 +295,6 @@ flatpak_related_ref_new (const char *collection_id, "subpaths", subpaths, "should-download", download, "should-delete", delete, - "collection-id", collection_id, NULL); return ref; diff --git a/common/flatpak-remote-ref-private.h b/common/flatpak-remote-ref-private.h index 792dd7ee..35df3a31 100644 --- a/common/flatpak-remote-ref-private.h +++ b/common/flatpak-remote-ref-private.h @@ -31,6 +31,7 @@ FlatpakRemoteRef *flatpak_remote_ref_new (const char *ref, const char *commit, const char *remote_name, + const char *sideload_collection_id, FlatpakRemoteState *remote_state); #endif /* __FLATPAK_REMOTE_REF_PRIVATE_H__ */ diff --git a/common/flatpak-remote-ref.c b/common/flatpak-remote-ref.c index 828c4be3..b48672bd 100644 --- a/common/flatpak-remote-ref.c +++ b/common/flatpak-remote-ref.c @@ -322,6 +322,7 @@ FlatpakRemoteRef * flatpak_remote_ref_new (const char *full_ref, const char *commit, const char *remote_name, + const char *sideload_collection_id, FlatpakRemoteState *state) { FlatpakRefKind kind = FLATPAK_REF_KIND_APP; @@ -366,6 +367,7 @@ flatpak_remote_ref_new (const char *full_ref, "branch", parts[3], "commit", commit, "remote-name", remote_name, + "collection-id", sideload_collection_id, "installed-size", installed_size, "download-size", download_size, "metadata", metadata_bytes, diff --git a/common/flatpak-remote.c b/common/flatpak-remote.c index eb9aa221..f5c33be2 100644 --- a/common/flatpak-remote.c +++ b/common/flatpak-remote.c @@ -368,7 +368,7 @@ flatpak_remote_get_collection_id (FlatpakRemote *self) return g_strdup (priv->local_collection_id); if (priv->dir) - return flatpak_dir_get_remote_collection_id (priv->dir, priv->name); + return flatpak_dir_get_remote_sideload_collection_id (priv->dir, priv->name); return NULL; } @@ -1250,26 +1250,9 @@ flatpak_remote_commit (FlatpakRemote *self, if (priv->local_collection_id_set) { if (priv->local_collection_id != NULL) - { - g_key_file_set_string (config, group, "collection-id", priv->local_collection_id); - - /* When a collection ID is set, flatpak uses signed per-repo and - * per-commit metadata instead of summary signatures. */ - g_key_file_set_boolean (config, group, "gpg-verify-summary", FALSE); - } + g_key_file_set_string (config, group, "xa.sideload-collection-id", priv->local_collection_id); else - { - g_autoptr(GError) local_error = NULL; - gboolean gpg_verify_value; - - g_key_file_remove_key (config, group, "collection-id", NULL); - - /* Without a collection ID gpg-verify-summary should go back to - * matching gpg-verify. */ - gpg_verify_value = g_key_file_get_boolean (config, group, "gpg-verify", &local_error); - if (local_error == NULL) - g_key_file_set_boolean (config, group, "gpg-verify-summary", gpg_verify_value); - } + g_key_file_remove_key (config, group, "xa.sideload-collection-id", NULL); } if (priv->local_title_set) diff --git a/common/flatpak-transaction.c b/common/flatpak-transaction.c index 4aad4c2c..803c9fc1 100644 --- a/common/flatpak-transaction.c +++ b/common/flatpak-transaction.c @@ -2514,7 +2514,6 @@ try_resolve_op_from_metadata (FlatpakTransaction *self, guint64 installed_size = 0; const char *metadata = NULL; VarMetadataRef sparse_cache; - g_autoptr(GError) local_error = NULL; VarRefInfoRef info; g_autofree char *summary_checksum = NULL; diff --git a/system-helper/flatpak-system-helper.c b/system-helper/flatpak-system-helper.c index 37ba8c76..1f69a059 100644 --- a/system-helper/flatpak-system-helper.c +++ b/system-helper/flatpak-system-helper.c @@ -1231,7 +1231,7 @@ handle_update_remote (FlatpakSystemHelper *object, return TRUE; } - if (summary_sig_bytes == NULL && state->collection_id == NULL) + if (summary_sig_bytes == NULL) { g_dbus_method_invocation_return_error (invocation, G_DBUS_ERROR, G_DBUS_ERROR_INVALID_ARGS, "UpdateRemote requires a summary signature"); diff --git a/tests/test-update-portal.c b/tests/test-update-portal.c index 3513dfeb..b7940a6c 100644 --- a/tests/test-update-portal.c +++ b/tests/test-update-portal.c @@ -130,7 +130,6 @@ progress_cb (PortalFlatpakUpdateMonitor *object, GVariant *arg_info, UpdateData *data) { - g_autofree char *args = g_variant_print (arg_info, FALSE); guint32 op = 0; guint32 n_ops = 0; guint32 progress = 0; diff --git a/tests/test-update-remote-configuration.sh b/tests/test-update-remote-configuration.sh index 9396d19d..281edc67 100755 --- a/tests/test-update-remote-configuration.sh +++ b/tests/test-update-remote-configuration.sh @@ -47,7 +47,7 @@ assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=true$' assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=false$' assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify=true$' assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify=false$' -assert_not_file_has_content ${FL_DIR}/repo/config '^collection-id=' +assert_not_file_has_content ${FL_DIR}/repo/config '^xa.sideload-collection-id=' # Change its configuration to include a collection ID, update the repository, # but don’t mark the collection ID as to be deployed yet. Ensure it doesn’t @@ -63,6 +63,7 @@ assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=false$' assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify=true$' assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify=false$' assert_not_file_has_content ${FL_DIR}/repo/config '^collection-id=' +assert_not_file_has_content ${FL_DIR}/repo/config '^xa.sideload-collection-id=' ok "1 update repo config without deploying collection ID" @@ -71,11 +72,12 @@ ok "1 update repo config without deploying collection ID" UPDATE_REPO_ARGS="--collection-id=org.test.Collection --deploy-collection-id" update_repo ${FLATPAK} ${U} update -y org.test.App master -assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=false$' -assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=true$' +assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=true$' +assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=false$' assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify=true$' assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify=false$' -assert_file_has_content ${FL_DIR}/repo/config '^collection-id=org\.test\.Collection$' +assert_not_file_has_content ${FL_DIR}/repo/config '^collection-id=' +assert_file_has_content ${FL_DIR}/repo/config '^xa.sideload-collection-id=org\.test\.Collection$' ok "2 update repo config to deploy collection ID" @@ -90,7 +92,7 @@ ok "2 update repo config to deploy collection ID" ostree --repo=repos/test summary --update --add-metadata="ostree.deploy-collection-id='net.malicious.NewCollection'" ${FLATPAK} ${U} update org.test.App master -assert_file_has_content ${FL_DIR}/repo/config '^collection-id=org\.test\.Collection$' -assert_not_file_has_content ${FL_DIR}/repo/config '^collection-id=net\.malicious\.NewCollection$' +assert_file_has_content ${FL_DIR}/repo/config '^xa.sideload-collection-id=org\.test\.Collection$' +assert_not_file_has_content ${FL_DIR}/repo/config '^xa.sideload-collection-id=net\.malicious\.NewCollection$' ok "3 update repo config with different collection ID" diff --git a/tests/testlibrary.c b/tests/testlibrary.c index 721a6fa4..bab53f63 100644 --- a/tests/testlibrary.c +++ b/tests/testlibrary.c @@ -623,10 +623,10 @@ test_remote (void) res = ostree_repo_open (repo, NULL, &error); g_assert_no_error (error); g_assert_true (res); - res = ostree_repo_get_remote_boolean_option (repo, repo_name, "gpg-verify-summary", TRUE, &gpg_verify_summary, &error); + res = ostree_repo_get_remote_boolean_option (repo, repo_name, "gpg-verify-summary", FALSE, &gpg_verify_summary, &error); g_assert_no_error (error); g_assert_true (res); - g_assert_false (gpg_verify_summary); + g_assert_true (gpg_verify_summary); /* Temporarily unset the collection ID */ flatpak_remote_set_collection_id (remote, NULL); From bea879c0d0d758f280271b130ee35df2315462b4 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 18 Mar 2020 17:42:18 +0100 Subject: [PATCH 11/36] FlatpakRemoteState: Drop separate tracking of metadata We now always use the metadata from the summary, so no need to track it separately. --- common/flatpak-dir-private.h | 2 -- common/flatpak-dir.c | 61 +++++++++++++----------------------- 2 files changed, 22 insertions(+), 41 deletions(-) diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index 6f19b39c..57a92cdc 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -120,8 +120,6 @@ typedef struct GVariant *summary; GBytes *summary_sig_bytes; GError *summary_fetch_error; - GVariant *metadata; - GError *metadata_fetch_error; GRegex *allow_refs; GRegex *deny_refs; int refcount; diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index d8e37da5..d5bc6a9d 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -316,8 +316,6 @@ flatpak_remote_state_unref (FlatpakRemoteState *remote_state) g_clear_pointer (&remote_state->summary, g_variant_unref); g_clear_pointer (&remote_state->summary_sig_bytes, g_bytes_unref); g_clear_error (&remote_state->summary_fetch_error); - g_clear_pointer (&remote_state->metadata, g_variant_unref); - g_clear_error (&remote_state->metadata_fetch_error); g_clear_pointer (&remote_state->allow_refs, g_regex_unref); g_clear_pointer (&remote_state->deny_refs, g_regex_unref); g_clear_pointer (&remote_state->sideload_repos, g_ptr_array_unref); @@ -337,29 +335,6 @@ flatpak_remote_state_ensure_summary (FlatpakRemoteState *self, return TRUE; } -gboolean -flatpak_remote_state_ensure_metadata (FlatpakRemoteState *self, - GError **error) -{ - if (self->metadata == NULL) - { - g_autofree char *error_msg = NULL; - - /* If the collection ID is NULL the metadata comes from the summary */ - if (self->metadata_fetch_error != NULL) - error_msg = g_strdup (self->metadata_fetch_error->message); - else if (self->summary_fetch_error != NULL) - error_msg = g_strdup_printf ("summary fetch error: %s", self->summary_fetch_error->message); - - return flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, - _("Unable to load metadata from remote %s: %s"), - self->remote_name, - error_msg != NULL ? error_msg : "unknown error"); - } - - return TRUE; -} - gboolean flatpak_remote_state_allow_ref (FlatpakRemoteState *self, const char *ref) @@ -505,11 +480,13 @@ flatpak_remote_state_get_cache (FlatpakRemoteState *self, VarMetadataRef meta; VarVariantRef cache_vv; VarVariantRef cache_v; + VarSummaryRef summary; - if (!flatpak_remote_state_ensure_metadata (self, error)) + if (!flatpak_remote_state_ensure_summary (self, error)) return FALSE; - meta = var_metadata_from_gvariant (self->metadata); + summary = var_summary_from_gvariant (self->summary); + meta = var_summary_get_metadata (summary); if (!var_metadata_lookup (meta, "xa.cache", NULL, &cache_vv)) { flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("No summary or Flatpak cache available for remote %s"), @@ -555,7 +532,8 @@ flatpak_remote_state_lookup_cache (FlatpakRemoteState *self, if (maybe_commit_bytes) { - VarMetadataRef meta = var_metadata_from_gvariant (self->metadata); + VarSummaryRef summary = var_summary_from_gvariant (self->summary); + VarMetadataRef meta = var_summary_get_metadata (summary); VarVariantRef commits_v; *maybe_commit_bytes = NULL; @@ -643,13 +621,15 @@ flatpak_remote_state_lookup_sparse_cache (FlatpakRemoteState *self, VarMetadataRef *out_metadata, GError **error) { + VarSummaryRef summary; VarMetadataRef meta; VarVariantRef sparse_cache_v; - if (!flatpak_remote_state_ensure_metadata (self, error)) + if (!flatpak_remote_state_ensure_summary (self, error)) return FALSE; - meta = var_metadata_from_gvariant (self->metadata); + summary = var_summary_from_gvariant (self->summary); + meta = var_summary_get_metadata (summary); if (var_metadata_lookup (meta, "xa.sparse-cache", NULL, &sparse_cache_v)) { VarSparseCacheRef sparse_cache = var_sparse_cache_from_variant (sparse_cache_v); @@ -10551,7 +10531,6 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, if (local_only) { flatpak_fail (&state->summary_fetch_error, "Internal error, local_only state"); - flatpak_fail (&state->metadata_fetch_error, "Internal error, local_only state"); return g_steal_pointer (&state); } @@ -10631,13 +10610,14 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, } if (state->summary != NULL) /* In the optional case we might not have a summary */ - state->metadata = g_variant_get_child_value (state->summary, 1); - - if (state->metadata) { - gint32 token_type; - if (g_variant_lookup (state->metadata, "xa.default-token-type", "i", &token_type)) - state->default_token_type = GINT32_FROM_LE (token_type); + VarSummaryRef summary = var_summary_from_gvariant (state->summary); + VarMetadataRef meta = var_summary_get_metadata (summary); + VarVariantRef res; + + if (var_metadata_lookup (meta, "xa.default-token-type", NULL, &res) && + var_variant_is_type (res, G_VARIANT_TYPE_INT32)) + state->default_token_type = GINT32_FROM_LE (var_variant_get_int32 (res)); } return g_steal_pointer (&state); @@ -12821,15 +12801,18 @@ flatpak_dir_update_remote_configuration_for_state (FlatpakDir *self, NULL }; g_autoptr(GPtrArray) updated_params = NULL; + g_autoptr(GVariant) metadata = NULL; GVariantIter iter; g_autoptr(GBytes) gpg_keys = NULL; updated_params = g_ptr_array_new_with_free_func (g_free); - if (!flatpak_remote_state_ensure_metadata (remote_state, error)) + if (!flatpak_remote_state_ensure_summary (remote_state, error)) return FALSE; - g_variant_iter_init (&iter, remote_state->metadata); + metadata = g_variant_get_child_value (remote_state->summary, 1); + + g_variant_iter_init (&iter, metadata); if (g_variant_iter_n_children (&iter) > 0) { GVariant *value_var = NULL; From 1d1819840e01285eb39daf3f289e89e58caef3c8 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 19 Mar 2020 13:51:22 +0100 Subject: [PATCH 12/36] config: Support setting sideload-repos option --- app/flatpak-builtins-config.c | 13 +++++++++++++ system-helper/flatpak-system-helper.c | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/app/flatpak-builtins-config.c b/app/flatpak-builtins-config.c index 8b4855fd..d5b72a9f 100644 --- a/app/flatpak-builtins-config.c +++ b/app/flatpak-builtins-config.c @@ -199,6 +199,18 @@ get_lang_default (FlatpakDir *dir) return g_strjoinv (";", langs); } +static char * +parse_string (const char *value, GError **error) +{ + return g_strdup (value); +} + +static char * +print_string (const char *value) +{ + return g_strdup (value); +} + typedef struct { const char *name; @@ -210,6 +222,7 @@ typedef struct ConfigKey keys[] = { { "languages", parse_lang, print_lang, get_lang_default }, { "extra-languages", parse_locale, print_locale, NULL }, + { "sideload-repos", parse_string, print_string, NULL }, }; static ConfigKey * diff --git a/system-helper/flatpak-system-helper.c b/system-helper/flatpak-system-helper.c index 1f69a059..16232a3c 100644 --- a/system-helper/flatpak-system-helper.c +++ b/system-helper/flatpak-system-helper.c @@ -1138,7 +1138,9 @@ handle_configure (FlatpakSystemHelper *object, return TRUE; } - if ((strcmp (arg_key, "languages") != 0) && (strcmp (arg_key, "extra-languages") != 0)) + if ((strcmp (arg_key, "languages") != 0) && + (strcmp (arg_key, "extra-languages") != 0) && + (strcmp (arg_key, "sideload-repos") != 0)) { g_dbus_method_invocation_return_error (invocation, G_DBUS_ERROR, G_DBUS_ERROR_INVALID_ARGS, "Unsupported key: %s", arg_key); From 904e7b2e2f3e48ff93d2d5856b0f1ee9bb513f2f Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 20 Mar 2020 15:21:05 +0100 Subject: [PATCH 13/36] tests: Remove test-unsigned-summaries.sh We no longer support unsigned summaries since we dropped the use of the ostree-metadata branch for p2p. --- tests/Makefile-test-matrix.am.inc | 1 - tests/Makefile.am.inc | 1 - tests/test-unsigned-summaries.sh | 136 ------------------------------ 3 files changed, 138 deletions(-) delete mode 100755 tests/test-unsigned-summaries.sh diff --git a/tests/Makefile-test-matrix.am.inc b/tests/Makefile-test-matrix.am.inc index 7ce305d5..70f8e625 100644 --- a/tests/Makefile-test-matrix.am.inc +++ b/tests/Makefile-test-matrix.am.inc @@ -26,7 +26,6 @@ TEST_MATRIX_DIST= \ tests/test-default-remotes.sh \ tests/test-extensions.sh \ tests/test-oci.sh \ - tests/test-unsigned-summaries.sh \ tests/test-update-remote-configuration.sh \ tests/test-override.sh \ tests/test-update-portal.sh \ diff --git a/tests/Makefile.am.inc b/tests/Makefile.am.inc index 29977272..d1ea064a 100644 --- a/tests/Makefile.am.inc +++ b/tests/Makefile.am.inc @@ -183,7 +183,6 @@ TEST_MATRIX_SOURCE = \ tests/test-bundle.sh{user+system+system-norevokefs} \ tests/test-oci.sh \ tests/test-oci-registry.sh{{user+system} \ - tests/test-unsigned-summaries.sh \ tests/test-update-remote-configuration.sh \ tests/test-override.sh \ tests/test-update-portal.sh \ diff --git a/tests/test-unsigned-summaries.sh b/tests/test-unsigned-summaries.sh deleted file mode 100755 index 63daa12d..00000000 --- a/tests/test-unsigned-summaries.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/bash -# -# Copyright © 2017 Endless Mobile, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. -# -# Authors: -# - Philip Withnall - -set -euo pipefail - -. $(dirname $0)/libtest.sh - -skip_without_bwrap -skip_revokefs_without_fuse - -echo "1..7" - -# Configure a repository and set up a collection ID for it. Check that setting -# the collection ID in the remote config disables summary signature checking. -setup_repo -install_repo - -echo -e "[core]\ncollection-id=org.test.Collection" >> repos/test/config -${FLATPAK} remote-modify ${U} test-repo --collection-id org.test.Collection - -assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=false$' -assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=true$' -assert_file_has_content ${FL_DIR}/repo/config '^collection-id=org\.test\.Collection$' -assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify=true$' -assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify=false$' - -ok "1 repo config with collections" - -# Test that building an app with a collection ID set produces the right -# metadata in the resulting repository. -DIR=$(mktemp -d) -${FLATPAK} build-init ${DIR} org.test.App org.test.Platform org.test.Platform -mkdir -p ${DIR}/files/a -echo "a" > ${DIR}/files/a/data -${FLATPAK} build-finish ${DIR} --socket=x11 --share=network --command=true -${FLATPAK} build-export ${FL_GPGARGS} --update-appstream repos/test --collection-id org.test.Collection ${DIR} master -update_repo - -ostree --repo=repos/test refs > refs -assert_file_has_content refs "^app/org\.test\.App/$ARCH/master$" -assert_file_has_content refs '^ostree-metadata$' -assert_file_has_content refs "^appstream/${ARCH}$" -assert_file_has_content refs "^appstream2/${ARCH}$" -ostree --repo=repos/test refs --collections > refs-collections -assert_file_has_content refs-collections "^(org\.test\.Collection, app/org\.test\.App/$ARCH/master)$" -assert_file_has_content refs-collections '^(org\.test\.Collection, ostree-metadata)$' -assert_file_has_content refs-collections "^(org\.test\.Collection, appstream/${ARCH})$" -assert_file_has_content refs-collections "^(org\.test\.Collection, appstream2/${ARCH})$" -assert_has_file repos/test/summary.sig -ostree --repo=repos/test summary --view > summary -assert_file_has_content summary '^Collection ID (ostree\.summary\.collection-id): org\.test\.Collection$' -assert_file_has_content summary '^xa\.cache: ' -ostree --repo=repos/test show --raw ostree-metadata > metadata -assert_file_has_content metadata "'xa\.cache': " -assert_file_has_content metadata "'ostree\.collection-binding': <'org\.test\.Collection'>" -assert_file_has_content metadata "'ostree\.ref-binding': <\['ostree-metadata'\]>" - -ok "2 create app with collections" - -# Try installing the app. -${FLATPAK} ${U} install -y test-repo org.test.App master -${FLATPAK} ${U} uninstall -y org.test.App - -ok "3 install app with collections" - -# Regenerate the summary so it doesn’t contain xa.cache and is unsigned; try installing again. -ostree --repo=repos/test summary --update -assert_not_has_file repos/test/summary.sig -ostree --repo=repos/test summary --view > summary -assert_file_has_content summary '^Collection ID (ostree\.summary\.collection-id): org\.test\.Collection$' -assert_not_file_has_content summary '^xa\.cache: ' - -${FLATPAK} ${U} install -y test-repo org.test.App master -${FLATPAK} ${U} uninstall -y org.test.App - -ok "4 install app with collections from unsigned summary" - -# Try installing it from a flatpakref file. Don’t uninstall afterwards because -# we need it for the next test. -cat << EOF > org.test.App.flatpakref -[Flatpak Ref] -Title=Test App -Name=org.test.App -Branch=master -Url=http://127.0.0.1:$(cat httpd-port)/test -IsRuntime=False -GPGKey=${FL_GPG_BASE64} -#RuntimeRepo=http://127.0.0.1:$(cat httpd-port)/test -DeployCollectionID=org.test.Collection -EOF - -${FLATPAK} ${U} install -y --from ./org.test.App.flatpakref -${FLATPAK} ${U} uninstall -y org.test.App - -ok "5 install app with collections from flatpakref" - -# Update the repo metadata and check that it changes in the ostree-metadata branch -# and the summary file. -${FLATPAK} build-update-repo ${FL_GPGARGS} --title "New title" repos/test - -assert_has_file repos/test/summary.sig -ostree --repo=repos/test summary --view > summary -assert_file_has_content summary '^Collection ID (ostree\.summary\.collection-id): org\.test\.Collection$' -assert_file_has_content summary '^xa\.title: ' -ostree --repo=repos/test show --raw ostree-metadata > metadata -assert_file_has_content metadata "'xa\.title': " -assert_file_has_content metadata "'ostree\.collection-binding': <'org\.test\.Collection'>" -assert_file_has_content metadata "'ostree\.ref-binding': <\['ostree-metadata'\]>" - -ok "6 update repo metadata" - -# Try to install the app again, which should pull the updated repository -# metadata as a side effect. -${FLATPAK} ${U} install -y test-repo org.test.App master -assert_file_has_content ${FL_DIR}/repo/config '^xa\.title=New title$' - -ok "7 pull updated repo metadata" From 2ac6be91f39668df61e9650f0f6f9c5c50ae3e4d Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 20 Mar 2020 15:30:34 +0100 Subject: [PATCH 14/36] Change how we handle refs in child-repo for system installs We used to to do a MIRROR pull and make a copy of the upstream summary file in the child repo. However, now that we want to allow side-loading from a repo with a partial summary we can no longer do that. So, the new approach is that the child repo *always* contains "remote:ref" style refs (never heads or mirror/collection-id refs), but in order to not let you import a signel commit into the wrong ref name we now require the commits to have a ref-binding (they all do since a long time anyway). --- common/flatpak-dir.c | 157 ++++++++----------------------------------- 1 file changed, 28 insertions(+), 129 deletions(-) diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index d5bc6a9d..cf488126 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -642,40 +642,6 @@ flatpak_remote_state_lookup_sparse_cache (FlatpakRemoteState *self, return FALSE; } -static gboolean -flatpak_remote_state_save_summary (FlatpakRemoteState *self, - GFile *dir, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(GFile) summary_file = g_file_get_child (dir, "summary"); - g_autoptr(GBytes) summary_bytes = NULL; - - /* For non-p2p case we always require a summary */ - if (!flatpak_remote_state_ensure_summary (self, error)) - return FALSE; - - summary_bytes = g_variant_get_data_as_bytes (self->summary); - - if (!g_file_replace_contents (summary_file, - g_bytes_get_data (summary_bytes, NULL), - g_bytes_get_size (summary_bytes), - NULL, FALSE, 0, NULL, cancellable, error)) - return FALSE; - - if (self->summary_sig_bytes != NULL) - { - g_autoptr(GFile) summary_sig_file = g_file_get_child (dir, "summary.sig"); - if (!g_file_replace_contents (summary_sig_file, - g_bytes_get_data (self->summary_sig_bytes, NULL), - g_bytes_get_size (self->summary_sig_bytes), - NULL, FALSE, 0, NULL, cancellable, error)) - return FALSE; - } - - return TRUE; -} - static DirExtraData * dir_extra_data_new (const char *id, const char *display_name, @@ -3827,19 +3793,6 @@ flatpak_dir_resolve_free (FlatpakDirResolve *resolve) } } -static gboolean -child_repo_ensure_summary (OstreeRepo *child_repo, - FlatpakRemoteState *state, - GCancellable *cancellable, - GError **error) -{ - if (!flatpak_remote_state_save_summary (state, ostree_repo_get_path (child_repo), - cancellable, error)) - return FALSE; - - return TRUE; -} - static gboolean get_mtime (GFile *file, GTimeVal *result, @@ -4125,16 +4078,13 @@ flatpak_dir_update_appstream (FlatpakDir *self, return FALSE; if (!flatpak_dir_pull (self, state, used_branch, appstream_commit, NULL, appstream_sideload_path, NULL, NULL, - child_repo, FLATPAK_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_MIRROR, + child_repo, FLATPAK_PULL_FLAGS_NONE, 0, progress, cancellable, error)) { g_prefix_error (&first_error, "Error updating appstream: "); return FALSE; } - if (!child_repo_ensure_summary (child_repo, state, cancellable, error)) - return FALSE; - if (!flatpak_repo_resolve_rev (child_repo, NULL, remote, used_branch, TRUE, &new_checksum, cancellable, error)) return FALSE; @@ -4356,7 +4306,7 @@ repo_pull (OstreeRepo *self, sideload_url = g_file_get_uri (sideload_repo); - g_debug ("Sideloading %s from %s during", ref_to_fetch, sideload_url); + g_debug ("Sideloading %s from %s in pull", ref_to_fetch, sideload_url); g_assert (sideload_collection_id != NULL); @@ -5172,7 +5122,7 @@ repo_pull_local_untrusted (FlatpakDir *self, g_variant_builder_add (&builder, "{s@v}", "gpg-verify", g_variant_new_variant (g_variant_new_boolean (TRUE))); g_variant_builder_add (&builder, "{s@v}", "gpg-verify-summary", - g_variant_new_variant (g_variant_new_boolean (TRUE))); + g_variant_new_variant (g_variant_new_boolean (FALSE))); g_variant_builder_add (&builder, "{s@v}", "inherit-transaction", g_variant_new_variant (g_variant_new_boolean (TRUE))); g_variant_builder_add (&builder, "{s@v}", "update-frequency", @@ -5206,27 +5156,21 @@ flatpak_dir_pull_untrusted_local (FlatpakDir *self, GError **error) { g_autoptr(GFile) path_file = g_file_new_for_path (src_path); - g_autoptr(GFile) summary_file = g_file_get_child (path_file, "summary"); - g_autoptr(GFile) summary_sig_file = g_file_get_child (path_file, "summary.sig"); g_autofree char *url = g_file_get_uri (path_file); g_autofree char *checksum = NULL; g_autofree char *current_checksum = NULL; gboolean gpg_verify_summary; gboolean gpg_verify; - char *summary_data = NULL; - char *summary_sig_data = NULL; - gsize summary_data_size, summary_sig_data_size; - g_autoptr(GBytes) summary_bytes = NULL; - g_autoptr(GBytes) summary_sig_bytes = NULL; g_autoptr(OstreeGpgVerifyResult) gpg_result = NULL; - g_autoptr(GVariant) summary = NULL; g_autoptr(GVariant) old_commit = NULL; g_autoptr(OstreeRepo) src_repo = NULL; g_autoptr(GVariant) new_commit = NULL; + g_autoptr(GVariant) new_commit_metadata = NULL; g_autoptr(GVariant) extra_data_sources = NULL; g_autoptr(GPtrArray) subdirs_arg = NULL; g_auto(GLnxLockFile) lock = { 0, }; gboolean ret = FALSE; + g_autofree const char **ref_bindings = NULL; if (!flatpak_dir_ensure_repo (self, cancellable, error)) return FALSE; @@ -5252,38 +5196,6 @@ flatpak_dir_pull_untrusted_local (FlatpakDir *self, if (!gpg_verify_summary || !gpg_verify) return flatpak_fail_error (error, FLATPAK_ERROR_UNTRUSTED, _("Can't pull from untrusted non-gpg verified remote")); - /* We verify the summary manually before anything else to make sure - we've got something right before looking too hard at the repo and - so we can check for a downgrade before pulling and updating the - ref */ - - if (!g_file_load_contents (summary_file, cancellable, - &summary_data, &summary_data_size, NULL, NULL)) - return flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("No summary found")); - summary_bytes = g_bytes_new_take (summary_data, summary_data_size); - - if (gpg_verify_summary) - { - if (!g_file_load_contents (summary_sig_file, cancellable, - &summary_sig_data, &summary_sig_data_size, NULL, NULL)) - return flatpak_fail_error (error, FLATPAK_ERROR_UNTRUSTED, _("GPG verification enabled, but no summary signatures found for remote '%s'"), remote_name); - - summary_sig_bytes = g_bytes_new_take (summary_sig_data, summary_sig_data_size); - - gpg_result = ostree_repo_verify_summary (self->repo, - remote_name, - summary_bytes, - summary_sig_bytes, - cancellable, error); - if (gpg_result == NULL) - return FALSE; - - if (ostree_gpg_verify_result_count_valid (gpg_result) == 0) - return flatpak_fail_error (error, FLATPAK_ERROR_UNTRUSTED, _("GPG signatures found for remote '%s', but none are in trusted keyring"), remote_name); - } - - g_clear_object (&gpg_result); - if (!flatpak_repo_resolve_rev (self->repo, NULL, remote_name, ref, TRUE, ¤t_checksum, NULL, error)) return FALSE; @@ -5296,11 +5208,8 @@ flatpak_dir_pull_untrusted_local (FlatpakDir *self, if (!ostree_repo_open (src_repo, cancellable, error)) return FALSE; - summary = g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, summary_bytes, FALSE)); - if (!flatpak_summary_lookup_ref (summary, NULL, ref, &checksum, NULL)) - return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, - _("No such ref '%s' in remote %s"), - ref, remote_name); + if (!flatpak_repo_resolve_rev (src_repo, NULL, remote_name, ref, FALSE, &checksum, NULL, error)) + return FALSE; if (gpg_verify) { @@ -5317,6 +5226,23 @@ flatpak_dir_pull_untrusted_local (FlatpakDir *self, if (!ostree_repo_load_commit (src_repo, checksum, &new_commit, NULL, error)) return FALSE; + /* Here we check that there is actually a ref binding, otherwise we + could allow installing a ref as another app, because both would + pass gpg validation. Note that ostree pull actually also verifies + the ref-bindings, but only if the exist. We could do only the + ref-binding exist check, but if we got something weird might as + well stop handling it early. */ + + new_commit_metadata = g_variant_get_child_value (new_commit, 0); + if (!g_variant_lookup (new_commit_metadata, OSTREE_COMMIT_META_KEY_REF_BINDING, "^a&s", &ref_bindings)) + return flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("Commit for ‘%s’ has no ref binding"), ref); + + if (!g_strv_contains ((const char *const *) ref_bindings, ref)) + { + g_autofree char *as_string = g_strjoinv (", ", (char **)ref_bindings); + return flatpak_fail_error (error, FLATPAK_ERROR_INVALID_DATA, _("Commit for ‘%s’ is not in expected bound refs: %s"), ref, as_string); + } + if (old_commit) { guint64 old_timestamp; @@ -5326,7 +5252,8 @@ flatpak_dir_pull_untrusted_local (FlatpakDir *self, new_timestamp = ostree_commit_get_timestamp (new_commit); if (new_timestamp < old_timestamp) - return flatpak_fail_error (error, FLATPAK_ERROR_DOWNGRADE, "Not allowed to downgrade %s", ref); + return flatpak_fail_error (error, FLATPAK_ERROR_DOWNGRADE, "Not allowed to downgrade %s (old_commit: %s/%ld new_commit: %s/%ld ", + ref, current_checksum, old_timestamp, checksum, new_timestamp); } if (subpaths != NULL && subpaths[0] != NULL) @@ -8410,7 +8337,7 @@ flatpak_dir_install (FlatpakDir *self, if (!flatpak_dir_pull (self, state, ref, opt_commit, subpaths, sideload_repo, require_metadata, token, child_repo, flatpak_flags, - OSTREE_REPO_PULL_FLAGS_MIRROR, + 0, progress, cancellable, error)) { if (is_revokefs_pull) @@ -8425,20 +8352,6 @@ flatpak_dir_install (FlatpakDir *self, return FALSE; } - if (!child_repo_ensure_summary (child_repo, state, cancellable, error)) - { - if (is_revokefs_pull) - { - flatpak_dir_unmount_and_cancel_pull (self, - FLATPAK_HELPER_CANCEL_PULL_FLAGS_PRESERVE_PULL, - cancellable, - &child_repo, &child_repo_lock, - mnt_dir, src_dir); - } - - return FALSE; - } - g_assert (child_repo_path != NULL); if (is_revokefs_pull && @@ -9101,7 +9014,7 @@ flatpak_dir_update (FlatpakDir *self, flatpak_flags |= FLATPAK_PULL_FLAGS_SIDELOAD_EXTRA_DATA; if (!flatpak_dir_pull (self, state, ref, commit, subpaths, sideload_repo, require_metadata, token, child_repo, - flatpak_flags, OSTREE_REPO_PULL_FLAGS_MIRROR, + flatpak_flags, 0, progress, cancellable, error)) { if (is_revokefs_pull) @@ -9116,20 +9029,6 @@ flatpak_dir_update (FlatpakDir *self, return FALSE; } - if (!child_repo_ensure_summary (child_repo, state, cancellable, error)) - { - if (is_revokefs_pull) - { - flatpak_dir_unmount_and_cancel_pull (self, - FLATPAK_HELPER_CANCEL_PULL_FLAGS_PRESERVE_PULL, - cancellable, - &child_repo, &child_repo_lock, - mnt_dir, src_dir); - } - - return FALSE; - } - g_assert (child_repo_path != NULL); if (is_revokefs_pull && From 77105495965273c180f5852451e26ece2b913f7b Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 20 Mar 2020 15:35:07 +0100 Subject: [PATCH 15/36] tests: Add tests for sideloading apps --- tests/Makefile-test-matrix.am.inc | 3 ++ tests/Makefile.am.inc | 1 + tests/test-sideload.sh | 82 +++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 tests/test-sideload.sh diff --git a/tests/Makefile-test-matrix.am.inc b/tests/Makefile-test-matrix.am.inc index 70f8e625..ddf12dce 100644 --- a/tests/Makefile-test-matrix.am.inc +++ b/tests/Makefile-test-matrix.am.inc @@ -11,6 +11,8 @@ TEST_MATRIX= \ tests/test-repo@user.wrap \ tests/test-repo@system.wrap \ tests/test-repo@system-norevokefs.wrap \ + tests/test-sideload@user.wrap \ + tests/test-sideload@system.wrap \ tests/test-bundle@user.wrap \ tests/test-bundle@system.wrap \ tests/test-bundle@system-norevokefs.wrap \ @@ -35,6 +37,7 @@ TEST_MATRIX_EXTRA_DIST= \ tests/test-run.sh \ tests/test-info.sh \ tests/test-repo.sh \ + tests/test-sideload.sh \ tests/test-bundle.sh \ tests/test-oci-registry.sh \ $(NULL) diff --git a/tests/Makefile.am.inc b/tests/Makefile.am.inc index d1ea064a..a7b25cb0 100644 --- a/tests/Makefile.am.inc +++ b/tests/Makefile.am.inc @@ -178,6 +178,7 @@ TEST_MATRIX_SOURCE = \ tests/test-run.sh{{user+system+system-norevokefs},{nodeltas+deltas}} \ tests/test-info.sh{user+system} \ tests/test-repo.sh{user+system+system-norevokefs} \ + tests/test-sideload.sh{user+system} \ tests/test-default-remotes.sh \ tests/test-extensions.sh \ tests/test-bundle.sh{user+system+system-norevokefs} \ diff --git a/tests/test-sideload.sh b/tests/test-sideload.sh new file mode 100644 index 00000000..1781da57 --- /dev/null +++ b/tests/test-sideload.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# +# Copyright (C) 2011 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +USE_COLLECTIONS_IN_SERVER=yes +USE_COLLECTIONS_IN_CLIENT=yes + +. $(dirname $0)/libtest.sh + +skip_without_bwrap +skip_revokefs_without_fuse + +echo "1..3" + +#Regular repo +setup_repo + +${FLATPAK} ${U} install -y test-repo org.test.Hello +# Ensure we have the full locale extension: +${FLATPAK} ${U} update -y --subpath= + +mkdir usb_dir + +${FLATPAK} ${U} create-usb --destination-repo=repo usb_dir org.test.Hello + +assert_has_file usb_dir/repo/config +assert_has_file usb_dir/repo/summary +assert_has_file usb_dir/repo/refs/mirrors/org.test.Collection.test/app/org.test.Hello/${ARCH}/master +assert_has_file usb_dir/repo/refs/mirrors/org.test.Collection.test/runtime/org.test.Hello.Locale/${ARCH}/master +assert_has_file usb_dir/repo/refs/mirrors/org.test.Collection.test/runtime/org.test.Platform/${ARCH}/master +assert_has_file usb_dir/repo/refs/mirrors/org.test.Collection.test/appstream2/${ARCH} + +${FLATPAK} ${U} uninstall -y --all + +ok "created sideloaded repo" + +${FLATPAK} ${U} remote-modify --url="http://no.127.0.0.1:${port}/test" test-repo + +if ${FLATPAK} ${U} install -y test-repo org.test.Hello &> /dev/null; then + assert_not_reached "Should not be able to install with wrong url" +fi + +SIDELOAD_REPO=$(realpath usb_dir/repo) +${FLATPAK} ${U} config --set sideload-repos ${SIDELOAD_REPO} + +${FLATPAK} ${U} config --get sideload-repos > sideload-repos +assert_file_has_content sideload-repos ${SIDELOAD_REPO} + +${FLATPAK} ${U} install -y test-repo org.test.Hello +assert_has_file $FL_DIR/app/org.test.Hello/$ARCH/master/active/metadata +assert_has_file $FL_DIR/repo/refs/remotes/test-repo/app/org.test.Hello/${ARCH}/master +assert_not_has_file $FL_DIR/repo/refs/mirrors/org.test.Collection.test/app/org.test.Hello/${ARCH}/master + +ok "installed sideloaded app" + +# Remove old appstream checkouts so we can update from the sideload +rm -rf $FL_DIR/appstream/test-repo/$ARCH/ +rm -rf $FL_DIR/repo/refs/remotes/test-repo/appstream2/$ARCH +ostree prune --repo=$FL_DIR/repo --refs-only + +${FLATPAK} ${U} update --appstream test-repo + +assert_has_file $FL_DIR/appstream/test-repo/$ARCH/active/appstream.xml + +ok "updated sideloaded appstream" From 316baff539b23130dcd8ffff5196a88f9b32f4f7 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 23 Mar 2020 12:32:11 +0100 Subject: [PATCH 16/36] Don't enforce gpg-verify false when using collection id With the new sideload approach to collection ids it is fine to require gpg signed summaries. (Not for the child repos or the sideload repos though, but thoser are either trusted (sideload) or safe for other reasons like ref-bindings and signed commits). --- common/flatpak-dir.c | 12 ------------ common/flatpak-utils.c | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index cf488126..a7476b5e 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -12801,18 +12801,6 @@ flatpak_dir_update_remote_configuration_for_state (FlatpakDir *self, { has_changed = TRUE; g_key_file_set_string (config, group, key, new_val); - - /* Special case for collection-id: if it’s set, gpg-verify-summary - * must be set to false. The logic above ensures that the - * collection-id is only set if we’re transitioning from an - * unset to a set collection-ID. We *must not* allow the - * collection ID to be changed from one set value to another - * without the user manually verifying it; or a malicious - * repository could assume the collection ID of another without - * the user’s consent. */ - if (g_str_equal (key, "collection-id") && - new_val != NULL && *new_val != '\0') - g_key_file_set_boolean (config, group, "gpg-verify-summary", FALSE); } } diff --git a/common/flatpak-utils.c b/common/flatpak-utils.c index 93d3a85d..0f55e975 100644 --- a/common/flatpak-utils.c +++ b/common/flatpak-utils.c @@ -2876,7 +2876,7 @@ flatpak_parse_repofile (const char *remote_name, /* If a collection ID is set, refs are verified from commit metadata rather * than the summary file. */ g_key_file_set_boolean (config, group, "gpg-verify-summary", - (gpg_key != NULL && collection_id == NULL)); + (gpg_key != NULL)); authenticator_name = g_key_file_get_string (keyfile, FLATPAK_REPO_GROUP, FLATPAK_REPO_AUTHENTICATOR_NAME_KEY, NULL); From 9abbb00b33378f336d9961908054501474eebb48 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 23 Mar 2020 12:34:03 +0100 Subject: [PATCH 17/36] Drop xa.sideload-collection remote option in favour of regular collection-id Nothing fundamentally happens differently in ostree if the collection-id is set, as long as we don't call the p2p specific apis. So, lets keep using it instead of adding our own special magic. --- app/flatpak-builtins-create-usb.c | 6 +- app/flatpak-builtins-remote-add.c | 2 +- app/flatpak-builtins-remote-list.c | 2 +- app/flatpak-builtins-remote-modify.c | 2 +- common/flatpak-dir-private.h | 8 +-- common/flatpak-dir.c | 76 +++++++---------------- common/flatpak-installation.c | 10 +-- common/flatpak-installed-ref.c | 4 +- common/flatpak-remote-ref-private.h | 2 +- common/flatpak-remote-ref.c | 4 +- common/flatpak-remote.c | 6 +- tests/test-update-remote-configuration.sh | 11 ++-- 12 files changed, 49 insertions(+), 84 deletions(-) diff --git a/app/flatpak-builtins-create-usb.c b/app/flatpak-builtins-create-usb.c index dad99a36..6ab4175f 100644 --- a/app/flatpak-builtins-create-usb.c +++ b/app/flatpak-builtins-create-usb.c @@ -167,7 +167,7 @@ add_related (GHashTable *all_refs, ext_remote = flatpak_deploy_data_get_origin (ext_deploy_data); if (ext_remote == NULL) return FALSE; - ext_collection_id = flatpak_dir_get_remote_sideload_collection_id (dir, ext_remote); + ext_collection_id = flatpak_dir_get_remote_collection_id (dir, ext_remote); if (ext_collection_id == NULL) { g_printerr (_("Warning: Omitting related ref ‘%s’ because its remote ‘%s’ does not have a collection ID set.\n"), @@ -239,7 +239,7 @@ add_runtime (GHashTable *all_refs, runtime_remote = flatpak_dir_get_origin (dir, runtime_ref, cancellable, error); if (runtime_remote == NULL) return FALSE; - runtime_collection_id = flatpak_dir_get_remote_sideload_collection_id (dir, runtime_remote); + runtime_collection_id = flatpak_dir_get_remote_collection_id (dir, runtime_remote); if (runtime_collection_id == NULL) return flatpak_fail (error, _("Remote ‘%s’ does not have a collection ID set, which is required for P2P distribution of ‘%s’."), @@ -605,7 +605,7 @@ flatpak_builtin_create_usb (int argc, char **argv, GCancellable *cancellable, GE if (remote == NULL) return FALSE; - ref_collection_id = flatpak_dir_get_remote_sideload_collection_id (dir, remote); + ref_collection_id = flatpak_dir_get_remote_collection_id (dir, remote); if (ref_collection_id == NULL) return flatpak_fail (error, _("Remote ‘%s’ does not have a collection ID set, which is required for P2P distribution of ‘%s’."), diff --git a/app/flatpak-builtins-remote-add.c b/app/flatpak-builtins-remote-add.c index f0932ddb..ec36ea6f 100644 --- a/app/flatpak-builtins-remote-add.c +++ b/app/flatpak-builtins-remote-add.c @@ -115,7 +115,7 @@ get_config_from_opts (GKeyFile *config, } if (opt_collection_id) - g_key_file_set_string (config, group, "xa.sideload-collection-id", opt_collection_id); + g_key_file_set_string (config, group, "collection-id", opt_collection_id); if (opt_title) { diff --git a/app/flatpak-builtins-remote-list.c b/app/flatpak-builtins-remote-list.c index 5f01ec91..dbfeef01 100644 --- a/app/flatpak-builtins-remote-list.c +++ b/app/flatpak-builtins-remote-list.c @@ -154,7 +154,7 @@ list_remotes (GPtrArray *dirs, Column *columns, GCancellable *cancellable, GErro } else if (strcmp (columns[k].name, "collection") == 0) { - g_autofree char *id = flatpak_dir_get_remote_sideload_collection_id (dir, remote_name); + g_autofree char *id = flatpak_dir_get_remote_collection_id (dir, remote_name); if (id != NULL) flatpak_table_printer_add_column (printer, id); else diff --git a/app/flatpak-builtins-remote-modify.c b/app/flatpak-builtins-remote-modify.c index d13ddb7d..a7fa9e62 100644 --- a/app/flatpak-builtins-remote-modify.c +++ b/app/flatpak-builtins-remote-modify.c @@ -130,7 +130,7 @@ get_config_from_opts (FlatpakDir *dir, const char *remote_name, gboolean *change if (opt_collection_id) { - g_key_file_set_string (config, group, "xa.sideload-collection-id", opt_collection_id); + g_key_file_set_string (config, group, "collection-id", opt_collection_id); *changed = TRUE; } diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index 57a92cdc..77b6afd9 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -116,7 +116,7 @@ typedef struct { typedef struct { char *remote_name; - char *sideload_collection_id; + char *collection_id; GVariant *summary; GBytes *summary_sig_bytes; GError *summary_fetch_error; @@ -780,7 +780,7 @@ char *flatpak_dir_create_origin_remote (FlatpakDir *self, const char *title, const char *main_ref, GBytes *gpg_data, - const char *sideload_collection_id, + const char *collection_id, gboolean *changed_config, GCancellable *cancellable, GError **error); @@ -790,7 +790,7 @@ gboolean flatpak_dir_create_remote_for_ref_file (FlatpakDir *self, GKeyFile *keyfile, const char *default_arch, char **remote_name_out, - char **sideload_collection_id_out, + char **collection_id_out, char **ref_out, GError **error); gboolean flatpak_dir_create_suggested_remote_for_ref_file (FlatpakDir *self, @@ -835,8 +835,6 @@ char *flatpak_dir_get_remote_icon (FlatpakDir *self, const char *remote_name); char *flatpak_dir_get_remote_collection_id (FlatpakDir *self, const char *remote_name); -char *flatpak_dir_get_remote_sideload_collection_id (FlatpakDir *self, - const char *remote_name); char *flatpak_dir_get_remote_main_ref (FlatpakDir *self, const char *remote_name); gboolean flatpak_dir_get_remote_oci (FlatpakDir *self, diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index a7476b5e..2fd56e73 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -312,7 +312,7 @@ flatpak_remote_state_unref (FlatpakRemoteState *remote_state) if (remote_state->refcount == 0) { g_free (remote_state->remote_name); - g_free (remote_state->sideload_collection_id); + g_free (remote_state->collection_id); g_clear_pointer (&remote_state->summary, g_variant_unref); g_clear_pointer (&remote_state->summary_sig_bytes, g_bytes_unref); g_clear_error (&remote_state->summary_fetch_error); @@ -422,7 +422,7 @@ flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, g_autofree char *sideload_checksum = NULL; VarRefInfoRef sideload_info; - if (flatpak_summary_lookup_ref (ss->summary, self->sideload_collection_id, ref, &sideload_checksum, &sideload_info)) + if (flatpak_summary_lookup_ref (ss->summary, self->collection_id, ref, &sideload_checksum, &sideload_info)) { guint64 timestamp = get_timestamp_from_ref_info (sideload_info); @@ -4162,24 +4162,6 @@ repo_get_remote_collection_id (OstreeRepo *repo, return TRUE; } -static gboolean -repo_get_remote_sideload_collection_id (OstreeRepo *repo, - const char *remote_name, - char **collection_id_out, - GError **error) -{ - if (collection_id_out != NULL) - { - if (!ostree_repo_get_remote_option (repo, remote_name, "xa.sideload-collection-id", - NULL, collection_id_out, error)) - return FALSE; - if (*collection_id_out != NULL && **collection_id_out == '\0') - g_clear_pointer (collection_id_out, g_free); - } - - return TRUE; -} - /* Get options for the OSTree pull operation which can be shared between * collection-based and normal pulls. Update @builder in place. */ static void @@ -4260,7 +4242,7 @@ repo_pull (OstreeRepo *self, const char **dirs_to_pull, const char *ref_to_fetch, const char *rev_to_fetch, /* (nullable) */ - const char *sideload_collection_id, + const char *collection_id, GFile *sideload_repo, const char *token, FlatpakPullFlags flatpak_flags, @@ -4308,10 +4290,10 @@ repo_pull (OstreeRepo *self, g_debug ("Sideloading %s from %s in pull", ref_to_fetch, sideload_url); - g_assert (sideload_collection_id != NULL); + g_assert (collection_id != NULL); g_variant_builder_init (&colref_builder, G_VARIANT_TYPE ("a(sss)")); - g_variant_builder_add (&colref_builder, "(sss)", sideload_collection_id, ref_to_fetch, rev_to_fetch); + g_variant_builder_add (&colref_builder, "(sss)", collection_id, ref_to_fetch, rev_to_fetch); g_variant_builder_add (&builder, "{s@v}", "collection-refs", g_variant_new_variant (g_variant_builder_end (&colref_builder))); @@ -5025,7 +5007,7 @@ flatpak_dir_pull (FlatpakDir *self, if (!repo_pull (repo, state->remote_name, subdirs_arg ? (const char **) subdirs_arg->pdata : NULL, - ref, rev, state->sideload_collection_id, sideload_repo, token, flatpak_flags, flags, + ref, rev, state->collection_id, sideload_repo, token, flatpak_flags, flags, progress, cancellable, error)) { @@ -8435,7 +8417,7 @@ flatpak_dir_ensure_bundle_remote (FlatpakDir *self, GBytes *gpg_data = NULL; g_autofree char *to_checksum = NULL; g_autofree char *remote = NULL; - g_autofree char *sideload_collection_id = NULL; + g_autofree char *collection_id = NULL; if (!flatpak_dir_ensure_repo (self, cancellable, error)) return NULL; @@ -8445,7 +8427,7 @@ flatpak_dir_ensure_bundle_remote (FlatpakDir *self, &origin, NULL, &fp_metadata, NULL, &included_gpg_data, - &sideload_collection_id, + &collection_id, error); if (metadata == NULL) return NULL; @@ -8483,7 +8465,7 @@ flatpak_dir_ensure_bundle_remote (FlatpakDir *self, basename, ref, gpg_data, - sideload_collection_id, + collection_id, &created_remote, cancellable, error); @@ -10419,7 +10401,7 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, { if (!flatpak_dir_has_remote (self, remote_or_uri, error)) return NULL; - if (!repo_get_remote_sideload_collection_id (self->repo, remote_or_uri, &state->sideload_collection_id, error)) + if (!repo_get_remote_collection_id (self->repo, remote_or_uri, &state->collection_id, error)) return NULL; if (!flatpak_dir_lookup_remote_filter (self, remote_or_uri, FALSE, NULL, &state->allow_refs, &state->deny_refs, error)) return NULL; @@ -10481,7 +10463,7 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, } } - if (state->sideload_collection_id) + if (state->collection_id) { g_auto(GStrv) sideload_paths = flatpak_dir_get_sideload_repo_paths (self); for (int i = 0; sideload_paths != NULL && sideload_paths[i] != NULL; i++) @@ -10679,7 +10661,7 @@ flatpak_dir_list_all_remote_refs (FlatpakDir *self, ref_map = var_summary_get_ref_map (summary); populate_hash_table_from_refs_map (ret_all_refs, NULL, ref_map, state); } - else if (state->sideload_collection_id) + else if (state->collection_id) { g_autoptr(GHashTable) ref_mtimes = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, g_free); /* Keys owned by ret_all_refs */ @@ -10696,7 +10678,7 @@ flatpak_dir_list_all_remote_refs (FlatpakDir *self, { VarCollectionMapRef map = var_collection_map_from_variant (v); - if (var_collection_map_lookup (map, state->sideload_collection_id, NULL, &ref_map)) + if (var_collection_map_lookup (map, state->collection_id, NULL, &ref_map)) populate_hash_table_from_refs_map (ret_all_refs, ref_mtimes, ref_map, state); } } @@ -10925,20 +10907,6 @@ flatpak_dir_get_remote_collection_id (FlatpakDir *self, return collection_id; } -char * -flatpak_dir_get_remote_sideload_collection_id (FlatpakDir *self, - const char *remote_name) -{ - char *collection_id = NULL; - - if (!flatpak_dir_ensure_repo (self, NULL, NULL)) - return NULL; - - repo_get_remote_sideload_collection_id (self->repo, remote_name, &collection_id, NULL); - - return collection_id; -} - char ** flatpak_dir_find_remote_refs (FlatpakDir *self, const char *remote, @@ -11876,7 +11844,7 @@ create_origin_remote_config (OstreeRepo *repo, const char *title, const char *main_ref, gboolean gpg_verify, - const char *sideload_collection_id, + const char *collection_id, GKeyFile **new_config) { g_autofree char *remote = NULL; @@ -11923,8 +11891,8 @@ create_origin_remote_config (OstreeRepo *repo, if (main_ref) g_key_file_set_string (*new_config, group, "xa.main-ref", main_ref); - if (sideload_collection_id) - g_key_file_set_string (*new_config, group, "xa.sideload-collection-id", sideload_collection_id); + if (collection_id) + g_key_file_set_string (*new_config, group, "collection-id", collection_id); return g_steal_pointer (&remote); } @@ -11936,7 +11904,7 @@ flatpak_dir_create_origin_remote (FlatpakDir *self, const char *title, const char *main_ref, GBytes *gpg_data, - const char *sideload_collection_id, + const char *collection_id, gboolean *changed_config, GCancellable *cancellable, GError **error) @@ -11944,7 +11912,7 @@ flatpak_dir_create_origin_remote (FlatpakDir *self, g_autoptr(GKeyFile) new_config = NULL; g_autofree char *remote = NULL; - remote = create_origin_remote_config (self->repo, url, id, title, main_ref, gpg_data != NULL, sideload_collection_id, &new_config); + remote = create_origin_remote_config (self->repo, url, id, title, main_ref, gpg_data != NULL, collection_id, &new_config); if (new_config && !flatpak_dir_modify_remote (self, remote, new_config, @@ -12063,7 +12031,7 @@ flatpak_dir_create_remote_for_ref_file (FlatpakDir *self, GKeyFile *keyfile, const char *default_arch, char **remote_name_out, - char **sideload_collection_id_out, + char **collection_id_out, char **ref_out, GError **error) { @@ -12106,8 +12074,8 @@ flatpak_dir_create_remote_for_ref_file (FlatpakDir *self, return FALSE; } - if (sideload_collection_id_out != NULL) - *sideload_collection_id_out = g_steal_pointer (&collection_id); + if (collection_id_out != NULL) + *collection_id_out = g_steal_pointer (&collection_id); *remote_name_out = g_steal_pointer (&remote); *ref_out = (char *) g_steal_pointer (&ref); @@ -12746,7 +12714,7 @@ flatpak_dir_update_remote_configuration_for_state (FlatpakDir *self, if (strcmp (key, "xa.redirect-url") == 0) g_ptr_array_add (updated_params, g_strdup ("url")); else if (strcmp (key, OSTREE_META_KEY_DEPLOY_COLLECTION_ID) == 0) - g_ptr_array_add (updated_params, g_strdup ("xa.sideload-collection-id")); + g_ptr_array_add (updated_params, g_strdup ("collection-id")); else g_ptr_array_add (updated_params, g_strdup (key)); g_ptr_array_add (updated_params, g_strdup (value)); diff --git a/common/flatpak-installation.c b/common/flatpak-installation.c index 20573b89..5c660222 100644 --- a/common/flatpak-installation.c +++ b/common/flatpak-installation.c @@ -739,7 +739,7 @@ get_ref (FlatpakDir *dir, g_autofree char *deploy_subdirname = NULL; g_autoptr(GBytes) deploy_data = NULL; g_autofree const char **subpaths = NULL; - g_autofree char *sideload_collection_id = NULL; + g_autofree char *collection_id = NULL; gboolean is_current = FALSE; guint64 installed_size = 0; @@ -769,12 +769,12 @@ get_ref (FlatpakDir *dir, latest_commit = flatpak_dir_read_latest (dir, origin, full_ref, &latest_alt_id, NULL, NULL); - sideload_collection_id = flatpak_dir_get_remote_sideload_collection_id (dir, origin); + collection_id = flatpak_dir_get_remote_collection_id (dir, origin); return flatpak_installed_ref_new (full_ref, alt_id ? alt_id : commit, latest_alt_id ? latest_alt_id : latest_commit, - origin, sideload_collection_id, subpaths, + origin, collection_id, subpaths, deploy_path, installed_size, is_current, @@ -2488,7 +2488,7 @@ flatpak_installation_list_remote_refs_sync_full (FlatpakInstallation *self, const gchar *ref_commit = value; FlatpakRemoteRef *ref; - ref = flatpak_remote_ref_new (ref_name, ref_commit, remote_or_uri, state->sideload_collection_id, state); + ref = flatpak_remote_ref_new (ref_name, ref_commit, remote_or_uri, state->collection_id, state); if (ref) g_ptr_array_add (refs, ref); @@ -2589,7 +2589,7 @@ flatpak_installation_fetch_remote_ref_sync_full (FlatpakInstallation *self, checksum = g_hash_table_lookup (ht, ref); if (checksum != NULL) - return flatpak_remote_ref_new (ref, checksum, remote_name, state->sideload_collection_id, state); + return flatpak_remote_ref_new (ref, checksum, remote_name, state->collection_id, state); g_set_error (error, FLATPAK_ERROR, FLATPAK_ERROR_REF_NOT_FOUND, "Reference %s doesn't exist in remote", ref); diff --git a/common/flatpak-installed-ref.c b/common/flatpak-installed-ref.c index f32cfd72..44c521f9 100644 --- a/common/flatpak-installed-ref.c +++ b/common/flatpak-installed-ref.c @@ -698,7 +698,7 @@ flatpak_installed_ref_new (const char *full_ref, const char *commit, const char *latest_commit, const char *origin, - const char *sideload_collection_id, + const char *collection_id, const char **subpaths, const char *deploy_dir, guint64 installed_size, @@ -733,7 +733,7 @@ flatpak_installed_ref_new (const char *full_ref, "commit", commit, "latest-commit", latest_commit, "origin", origin, - "collection-id", sideload_collection_id, + "collection-id", collection_id, "subpaths", subpaths, "is-current", is_current, "installed-size", installed_size, diff --git a/common/flatpak-remote-ref-private.h b/common/flatpak-remote-ref-private.h index 35df3a31..0bd8e083 100644 --- a/common/flatpak-remote-ref-private.h +++ b/common/flatpak-remote-ref-private.h @@ -31,7 +31,7 @@ FlatpakRemoteRef *flatpak_remote_ref_new (const char *ref, const char *commit, const char *remote_name, - const char *sideload_collection_id, + const char *collection_id, FlatpakRemoteState *remote_state); #endif /* __FLATPAK_REMOTE_REF_PRIVATE_H__ */ diff --git a/common/flatpak-remote-ref.c b/common/flatpak-remote-ref.c index b48672bd..4fb44374 100644 --- a/common/flatpak-remote-ref.c +++ b/common/flatpak-remote-ref.c @@ -322,7 +322,7 @@ FlatpakRemoteRef * flatpak_remote_ref_new (const char *full_ref, const char *commit, const char *remote_name, - const char *sideload_collection_id, + const char *collection_id, FlatpakRemoteState *state) { FlatpakRefKind kind = FLATPAK_REF_KIND_APP; @@ -367,7 +367,7 @@ flatpak_remote_ref_new (const char *full_ref, "branch", parts[3], "commit", commit, "remote-name", remote_name, - "collection-id", sideload_collection_id, + "collection-id", collection_id, "installed-size", installed_size, "download-size", download_size, "metadata", metadata_bytes, diff --git a/common/flatpak-remote.c b/common/flatpak-remote.c index f5c33be2..7f404420 100644 --- a/common/flatpak-remote.c +++ b/common/flatpak-remote.c @@ -368,7 +368,7 @@ flatpak_remote_get_collection_id (FlatpakRemote *self) return g_strdup (priv->local_collection_id); if (priv->dir) - return flatpak_dir_get_remote_sideload_collection_id (priv->dir, priv->name); + return flatpak_dir_get_remote_collection_id (priv->dir, priv->name); return NULL; } @@ -1250,9 +1250,9 @@ flatpak_remote_commit (FlatpakRemote *self, if (priv->local_collection_id_set) { if (priv->local_collection_id != NULL) - g_key_file_set_string (config, group, "xa.sideload-collection-id", priv->local_collection_id); + g_key_file_set_string (config, group, "collection-id", priv->local_collection_id); else - g_key_file_remove_key (config, group, "xa.sideload-collection-id", NULL); + g_key_file_remove_key (config, group, "collection-id", NULL); } if (priv->local_title_set) diff --git a/tests/test-update-remote-configuration.sh b/tests/test-update-remote-configuration.sh index 281edc67..0aebe374 100755 --- a/tests/test-update-remote-configuration.sh +++ b/tests/test-update-remote-configuration.sh @@ -47,7 +47,7 @@ assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=true$' assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=false$' assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify=true$' assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify=false$' -assert_not_file_has_content ${FL_DIR}/repo/config '^xa.sideload-collection-id=' +assert_not_file_has_content ${FL_DIR}/repo/config '^collection-id=' # Change its configuration to include a collection ID, update the repository, # but don’t mark the collection ID as to be deployed yet. Ensure it doesn’t @@ -63,7 +63,7 @@ assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=false$' assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify=true$' assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify=false$' assert_not_file_has_content ${FL_DIR}/repo/config '^collection-id=' -assert_not_file_has_content ${FL_DIR}/repo/config '^xa.sideload-collection-id=' +assert_not_file_has_content ${FL_DIR}/repo/config '^collection-id=' ok "1 update repo config without deploying collection ID" @@ -76,8 +76,7 @@ assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=true$' assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=false$' assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify=true$' assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify=false$' -assert_not_file_has_content ${FL_DIR}/repo/config '^collection-id=' -assert_file_has_content ${FL_DIR}/repo/config '^xa.sideload-collection-id=org\.test\.Collection$' +assert_file_has_content ${FL_DIR}/repo/config '^collection-id=org\.test\.Collection$' ok "2 update repo config to deploy collection ID" @@ -92,7 +91,7 @@ ok "2 update repo config to deploy collection ID" ostree --repo=repos/test summary --update --add-metadata="ostree.deploy-collection-id='net.malicious.NewCollection'" ${FLATPAK} ${U} update org.test.App master -assert_file_has_content ${FL_DIR}/repo/config '^xa.sideload-collection-id=org\.test\.Collection$' -assert_not_file_has_content ${FL_DIR}/repo/config '^xa.sideload-collection-id=net\.malicious\.NewCollection$' +assert_file_has_content ${FL_DIR}/repo/config '^collection-id=org\.test\.Collection$' +assert_not_file_has_content ${FL_DIR}/repo/config '^collection-id=net\.malicious\.NewCollection$' ok "3 update repo config with different collection ID" From b3da5db1c3c5d8e04cb0b774dc2acddc08e0423c Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 23 Mar 2020 15:17:44 +0100 Subject: [PATCH 18/36] update: Allow update --commit to install from sideloaded repo --- common/flatpak-dir-private.h | 2 ++ common/flatpak-dir.c | 26 ++++++++++++++++++++++++-- common/flatpak-transaction.c | 6 +++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index 77b6afd9..bacc0d3e 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -144,6 +144,8 @@ gboolean flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, GError **error); char **flatpak_remote_state_match_subrefs (FlatpakRemoteState *self, const char *ref); +GFile *flatpak_remote_state_lookup_sideload_checksum (FlatpakRemoteState *self, + char *checksum); gboolean flatpak_remote_state_lookup_cache (FlatpakRemoteState *self, const char *ref, guint64 *download_size, diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index 2fd56e73..5cc074a4 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -350,6 +350,25 @@ get_timestamp_from_ref_info (VarRefInfoRef info) return GUINT64_FROM_BE(var_metadata_lookup_uint64 (metadata, OSTREE_COMMIT_TIMESTAMP, 0)); } + +GFile * +flatpak_remote_state_lookup_sideload_checksum (FlatpakRemoteState *self, + char *checksum) +{ + for (int i = 0; i < self->sideload_repos->len; i++) + { + FlatpakSideloadState *ss = g_ptr_array_index (self->sideload_repos, i); + OstreeRepoCommitState commit_state; + + if (ostree_repo_load_commit (ss->repo, checksum, NULL, &commit_state, NULL) && + commit_state == OSTREE_REPO_COMMIT_STATE_NORMAL) + return g_object_ref (ostree_repo_get_path (ss->repo)); + } + + return NULL; +} + + /* Returns TRUE if the ref is found in the summary or cache. * out_checksum and out_variant are only set when the ref is found. */ @@ -4968,8 +4987,11 @@ flatpak_dir_pull (FlatpakDir *self, } } - g_debug ("%s: Using commit %s for pull of ref %s from remote %s", - G_STRFUNC, rev, ref, state->remote_name); + g_debug ("%s: Using commit %s for pull of ref %s from remote %s%s%s", + G_STRFUNC, rev, ref, state->remote_name, + sideload_repo ? "sideloaded from " : "", + sideload_repo ? flatpak_file_get_path_cached (sideload_repo) : "" + ); if (repo == NULL) repo = self->repo; diff --git a/common/flatpak-transaction.c b/common/flatpak-transaction.c index 803c9fc1..4bf7d0a3 100644 --- a/common/flatpak-transaction.c +++ b/common/flatpak-transaction.c @@ -2630,7 +2630,11 @@ resolve_ops (FlatpakTransaction *self, g_autoptr(GFile) sideload_path = NULL; if (op->commit != NULL) - checksum = g_strdup (op->commit); + { + checksum = g_strdup (op->commit); + /* Check if this is available offline and if so, use that */ + sideload_path = flatpak_remote_state_lookup_sideload_checksum (state, op->commit); + } else { g_autofree char *latest_checksum = NULL; From cd88ddd2af035b4c7f178a087614f26f229747ef Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 23 Mar 2020 15:22:30 +0100 Subject: [PATCH 19/36] tests: Add more tests for sideloading * Test that we load from sideload repo even when online * Test that when offline we don't update to older version in sideload repo * Test update to explicit version in sideload repo * Test updates to new version from sideload repo --- tests/test-sideload.sh | 87 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/tests/test-sideload.sh b/tests/test-sideload.sh index 1781da57..379dea63 100644 --- a/tests/test-sideload.sh +++ b/tests/test-sideload.sh @@ -27,14 +27,15 @@ USE_COLLECTIONS_IN_CLIENT=yes skip_without_bwrap skip_revokefs_without_fuse -echo "1..3" +echo "1..7" #Regular repo setup_repo -${FLATPAK} ${U} install -y test-repo org.test.Hello # Ensure we have the full locale extension: -${FLATPAK} ${U} update -y --subpath= +${FLATPAK} ${U} config --set languages "*" + +${FLATPAK} ${U} install -y test-repo org.test.Hello mkdir usb_dir @@ -80,3 +81,83 @@ ${FLATPAK} ${U} update --appstream test-repo assert_has_file $FL_DIR/appstream/test-repo/$ARCH/active/appstream.xml ok "updated sideloaded appstream" + +${FLATPAK} ${U} remote-modify --url="http://127.0.0.1:${port}/test" test-repo +${FLATPAK} ${U} uninstall -y --all + +# Disable sideload repo and "break" online repo +${FLATPAK} ${U} config --unset sideload-repos +mv repos/test/objects repos/test/objects.disabled + +# Ensure this fails (but still loads summary) +if ${FLATPAK} ${U} install -y test-repo org.test.Hello &> install-error-log; then + assert_not_reached "Disabled online install broken" +fi +assert_file_has_content install-error-log "Server returned status 404: Not Found" + +${FLATPAK} ${U} config --set sideload-repos ${SIDELOAD_REPO} + +${FLATPAK} ${U} install -y test-repo org.test.Hello +assert_has_file $FL_DIR/app/org.test.Hello/$ARCH/master/active/metadata + +# Re-enable online repo +mv repos/test/objects.disabled repos/test/objects + +ok "installed sideloaded app when online" + +OLD_COMMIT=$(cat repos/test/refs/heads/app/org.test.Hello/${ARCH}/master) + +make_updated_app +update_repo + +NEW_COMMIT=$(cat repos/test/refs/heads/app/org.test.Hello/$ARCH/master) + +${FLATPAK} ${U} update -y + +# Prepare sideload repo for NEW_COMMIT +${FLATPAK} ${U} create-usb --destination-repo=repo2 usb_dir org.test.Hello + +UPDATED_COMMIT=$( ${FLATPAK} ${U} info --show-commit app/org.test.Hello/${ARCH}/master ) +assert_streq "$NEW_COMMIT" "$UPDATED_COMMIT" + +# Update again should do nothing +${FLATPAK} ${U} update -y +UPDATED_COMMIT=$( ${FLATPAK} ${U} info --show-commit app/org.test.Hello/${ARCH}/master ) +assert_streq "$NEW_COMMIT" "$UPDATED_COMMIT" + +# Ensure that offline update don't downgrade to older version +${FLATPAK} ${U} remote-modify --url="http://no.127.0.0.1:${port}/test" test-repo +${FLATPAK} ${U} update -y +UPDATED_COMMIT=$( ${FLATPAK} ${U} info --show-commit app/org.test.Hello/${ARCH}/master ) +assert_streq "$NEW_COMMIT" "$UPDATED_COMMIT" + +ok "updates from sideload don't go backwards" + +if [ x${USE_SYSTEMDIR-} == xyes ]; then + # --commit + --system works only as root, so lets just "fake it" by installing the current sideload version + ${FLATPAK} ${U} uninstall -y --no-related org.test.Hello + ${FLATPAK} ${U} install -y org.test.Hello +else + # Try (offline) to update to the old version + ${FLATPAK} ${U} update -y --commit $OLD_COMMIT org.test.Hello +fi + +UPDATED_COMMIT=$( ${FLATPAK} ${U} info --show-commit app/org.test.Hello/${ARCH}/master ) +assert_streq "$OLD_COMMIT" "$UPDATED_COMMIT" + +ok "update to explicit old version" + +# Switch to updated usb repo +mv usb_dir/repo usb_dir/repo.old +mv usb_dir/repo2 usb_dir/repo + +# And update to it (offline) +${FLATPAK} ${U} update -y + +UPDATED_COMMIT=$( ${FLATPAK} ${U} info --show-commit app/org.test.Hello/${ARCH}/master ) +assert_streq "$NEW_COMMIT" "$UPDATED_COMMIT" + +# Re enable (go online) +${FLATPAK} ${U} remote-modify --url="http://127.0.0.1:${port}/test" test-repo + +ok "update offline to new version" From 9208913664830a948e56a909fe61ef4287433a99 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 23 Mar 2020 16:22:04 +0100 Subject: [PATCH 20/36] Migrate config: Always enable gpg-verify-summary if collection id is set This used to not be set for collection-id remotes as we used the ostree-metadata branch for resolving. However, we now use the summary always when doing a remote install (and not ostree-metadata for local sideloads), so we still want to verify summary. The signature on the summary is a nice security feature, but it is also a very efficient small file to download to verify that no new summary needs to be downloaded in the no-op update case. --- common/flatpak-dir-private.h | 4 ++ common/flatpak-dir.c | 80 +++++++++++++++++++++++++++ common/flatpak-transaction.c | 3 + data/org.freedesktop.Flatpak.xml | 1 + system-helper/flatpak-system-helper.c | 4 ++ tests/test-sideload.sh | 17 +++++- 6 files changed, 108 insertions(+), 1 deletion(-) diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index bacc0d3e..e6652619 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -896,6 +896,10 @@ FlatpakRemoteState * flatpak_dir_get_remote_state_for_summary (FlatpakDir *sel GBytes *opt_summary_sig, GCancellable *cancellable, GError **error); +gboolean flatpak_dir_migrate_config (FlatpakDir *self, + gboolean *changed, + GCancellable *cancellable, + GError **error); gboolean flatpak_dir_remote_make_oci_summary (FlatpakDir *self, const char *remote, gboolean only_cached, diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index 5cc074a4..29f4311f 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -2925,6 +2925,86 @@ flatpak_dir_ensure_path (FlatpakDir *self, } } +gboolean +flatpak_dir_migrate_config (FlatpakDir *self, + gboolean *changed, + GCancellable *cancellable, + GError **error) +{ + g_auto(GStrv) remotes = NULL; + g_autoptr(GKeyFile) config = NULL; + int i; + + if (changed != NULL) + *changed = FALSE; + + /* Only do anything if it exists */ + if (!flatpak_dir_maybe_ensure_repo (self, NULL, NULL)) + return TRUE; + + remotes = flatpak_dir_list_remotes (self, cancellable, NULL); + if (remotes == NULL) + return TRUE; + + /* Enable gpg-verify-summary for all remotes with a collection id *and* gpg-verify set, because + * we want to use summary verification, but older versions of collection-id didn't work with it */ + for (i = 0; remotes != NULL && remotes[i] != NULL; i++) + { + g_autofree char *remote_collection_id = NULL; + const char *remote = remotes[i]; + gboolean gpg_verify_summary; + gboolean gpg_verify; + + if (flatpak_dir_get_remote_disabled (self, remote)) + continue; + + remote_collection_id = flatpak_dir_get_remote_collection_id (self, remotes[i]); + if (remote_collection_id == NULL) + continue; + + if (!ostree_repo_remote_get_gpg_verify_summary (self->repo, remote, &gpg_verify_summary, NULL)) + continue; + + if (!ostree_repo_remote_get_gpg_verify (self->repo, remote, &gpg_verify, NULL)) + continue; + + if (gpg_verify && !gpg_verify_summary) + { + g_autofree char *group = g_strdup_printf ("remote \"%s\"", remote); + if (config == NULL) + config = ostree_repo_copy_config (flatpak_dir_get_repo (self)); + + g_debug ("Migrating remote '%s' to gpg-verify-summary", remote); + g_key_file_set_boolean (config, group, "gpg-verify-summary", TRUE); + } + } + + if (config != NULL) + { + if (flatpak_dir_use_system_helper (self, NULL)) + { + g_autoptr(GError) local_error = NULL; + const char *installation = flatpak_dir_get_id (self); + + if (!flatpak_dir_system_helper_call_ensure_repo (self, + FLATPAK_HELPER_ENSURE_REPO_FLAGS_NONE, + installation ? installation : "", + NULL, &local_error)) + g_debug ("Failed to migrate system config: %s", local_error->message); + } + else + { + if (!ostree_repo_write_config (self->repo, config, error)) + return FALSE; + } + + if (changed != NULL) + *changed = FALSE; + } + + return TRUE; +} + /* Warning: This is not threadsafe, don't use in libflatpak */ gboolean flatpak_dir_recreate_repo (FlatpakDir *self, diff --git a/common/flatpak-transaction.c b/common/flatpak-transaction.c index 4bf7d0a3..00a2cbe5 100644 --- a/common/flatpak-transaction.c +++ b/common/flatpak-transaction.c @@ -2240,6 +2240,9 @@ flatpak_transaction_update_metadata (FlatpakTransaction *self, /* Collect all dir+remotes used in this transaction */ + if (!flatpak_dir_migrate_config (priv->dir, &some_updated, cancellable, error)) + return FALSE; + for (l = priv->ops; l != NULL; l = l->next) { FlatpakTransactionOperation *op = l->data; diff --git a/data/org.freedesktop.Flatpak.xml b/data/org.freedesktop.Flatpak.xml index 497a54a5..b9cd8b6d 100644 --- a/data/org.freedesktop.Flatpak.xml +++ b/data/org.freedesktop.Flatpak.xml @@ -167,6 +167,7 @@ + diff --git a/system-helper/flatpak-system-helper.c b/system-helper/flatpak-system-helper.c index 16232a3c..6c06e5a6 100644 --- a/system-helper/flatpak-system-helper.c +++ b/system-helper/flatpak-system-helper.c @@ -1353,6 +1353,7 @@ handle_ensure_repo (FlatpakSystemHelper *object, { g_autoptr(FlatpakDir) system = NULL; g_autoptr(GError) error = NULL; + g_autoptr(GError) local_error = NULL; g_debug ("EnsureRepo %u %s", arg_flags, arg_installation); @@ -1376,6 +1377,9 @@ handle_ensure_repo (FlatpakSystemHelper *object, return TRUE; } + if (!flatpak_dir_migrate_config (system, NULL, NULL, &local_error)) + g_warning ("Failed to migrate configuration for installation %s: %s", arg_installation, local_error->message); + flatpak_system_helper_complete_ensure_repo (object, invocation); return TRUE; diff --git a/tests/test-sideload.sh b/tests/test-sideload.sh index 379dea63..a351f380 100644 --- a/tests/test-sideload.sh +++ b/tests/test-sideload.sh @@ -27,7 +27,7 @@ USE_COLLECTIONS_IN_CLIENT=yes skip_without_bwrap skip_revokefs_without_fuse -echo "1..7" +echo "1..8" #Regular repo setup_repo @@ -161,3 +161,18 @@ assert_streq "$NEW_COMMIT" "$UPDATED_COMMIT" ${FLATPAK} ${U} remote-modify --url="http://127.0.0.1:${port}/test" test-repo ok "update offline to new version" + +assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=true$' +assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=false$' + +ostree --repo=$FL_DIR/repo config set --group='remote "test-repo"' gpg-verify-summary false + +assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=false$' +assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=true$' + +${FLATPAK} ${U} update -y + +assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=true$' +assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=false$' + +ok "migrate to gpg-verify-summary" From d30e902d9f5a37a5f838b48076cf42929699cd43 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 23 Mar 2020 17:19:58 +0100 Subject: [PATCH 21/36] Add build-update-repo --deploy-sideload-collection-id option This is a new version of --deploy-collection-id that only applies the collection id update for new (1.7.x+) version of flatpak clients. This allows you to enable collection ids for sideload use but not affect older clients where the p2p codepaths are not as tested. --- app/flatpak-builtins-build-update-repo.c | 6 ++++++ common/flatpak-dir.c | 3 +++ common/flatpak-utils-private.h | 3 +++ common/flatpak-utils.c | 17 +++++++++++++++++ doc/flatpak-build-update-repo.xml | 10 ++++++++++ tests/test-update-remote-configuration.sh | 16 ++++++++++++++++ 6 files changed, 55 insertions(+) diff --git a/app/flatpak-builtins-build-update-repo.c b/app/flatpak-builtins-build-update-repo.c index f08aca18..f466a561 100644 --- a/app/flatpak-builtins-build-update-repo.c +++ b/app/flatpak-builtins-build-update-repo.c @@ -41,6 +41,7 @@ static char *opt_icon; static char *opt_redirect_url; static char *opt_default_branch; static char *opt_collection_id = NULL; +static gboolean opt_deploy_sideload_collection_id = FALSE; static gboolean opt_deploy_collection_id = FALSE; static char **opt_gpg_import; static char *opt_generate_delta_from; @@ -68,6 +69,7 @@ static GOptionEntry options[] = { { "icon", 0, 0, G_OPTION_ARG_STRING, &opt_icon, N_("URL for an icon for this repository"), N_("URL") }, { "default-branch", 0, 0, G_OPTION_ARG_STRING, &opt_default_branch, N_("Default branch to use for this repository"), N_("BRANCH") }, { "collection-id", 0, 0, G_OPTION_ARG_STRING, &opt_collection_id, N_("Collection ID"), N_("COLLECTION-ID") }, + { "deploy-sideload-collection-id", 0, 0, G_OPTION_ARG_NONE, &opt_deploy_sideload_collection_id, N_("Permanently deploy collection ID to client remote configurations, only for sideload support"), NULL }, { "deploy-collection-id", 0, 0, G_OPTION_ARG_NONE, &opt_deploy_collection_id, N_("Permanently deploy collection ID to client remote configurations"), NULL }, { "authenticator-name", 0, 0, G_OPTION_ARG_STRING, &opt_authenticator_name, N_("Name of authenticator for this repository"), N_("NAME") }, { "authenticator-install", 0, 0, G_OPTION_ARG_NONE, &opt_authenticator_install, N_("Autoinstall authenticator for this repository"), NULL }, @@ -577,6 +579,10 @@ flatpak_builtin_build_update_repo (int argc, char **argv, return FALSE; } + if (opt_deploy_sideload_collection_id && + !flatpak_repo_set_deploy_sideload_collection_id (repo, TRUE, error)) + return FALSE; + if (opt_deploy_collection_id && !flatpak_repo_set_deploy_collection_id (repo, TRUE, error)) return FALSE; diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index 29f4311f..db7cbb1c 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -12763,6 +12763,7 @@ flatpak_dir_update_remote_configuration_for_state (FlatpakDir *self, "xa.authenticator-name", "xa.authenticator-install", OSTREE_META_KEY_DEPLOY_COLLECTION_ID, + "xa.deploy-collection-id", /* This is a new version only supported in post p2p flatpak (1.7) */ NULL }; static const char *const supported_param_prefixes[] = { @@ -12817,6 +12818,8 @@ flatpak_dir_update_remote_configuration_for_state (FlatpakDir *self, g_ptr_array_add (updated_params, g_strdup ("url")); else if (strcmp (key, OSTREE_META_KEY_DEPLOY_COLLECTION_ID) == 0) g_ptr_array_add (updated_params, g_strdup ("collection-id")); + else if (strcmp (key, "xa.deploy-collection-id") == 0) + g_ptr_array_add (updated_params, g_strdup ("collection-id")); else g_ptr_array_add (updated_params, g_strdup (key)); g_ptr_array_add (updated_params, g_strdup (value)); diff --git a/common/flatpak-utils-private.h b/common/flatpak-utils-private.h index f866655c..9038f2a8 100644 --- a/common/flatpak-utils-private.h +++ b/common/flatpak-utils-private.h @@ -450,6 +450,9 @@ gboolean flatpak_repo_set_collection_id (OstreeRepo *repo, gboolean flatpak_repo_set_deploy_collection_id (OstreeRepo *repo, gboolean deploy_collection_id, GError **error); +gboolean flatpak_repo_set_deploy_sideload_collection_id (OstreeRepo *repo, + gboolean deploy_collection_id, + GError **error); gboolean flatpak_repo_set_gpg_keys (OstreeRepo *repo, GBytes *bytes, GError **error); diff --git a/common/flatpak-utils.c b/common/flatpak-utils.c index 0f55e975..478a8449 100644 --- a/common/flatpak-utils.c +++ b/common/flatpak-utils.c @@ -3114,6 +3114,18 @@ flatpak_repo_set_deploy_collection_id (OstreeRepo *repo, return ostree_repo_write_config (repo, config, error); } +gboolean +flatpak_repo_set_deploy_sideload_collection_id (OstreeRepo *repo, + gboolean deploy_collection_id, + GError **error) +{ + g_autoptr(GKeyFile) config = NULL; + + config = ostree_repo_copy_config (repo); + g_key_file_set_boolean (config, "flatpak", "deploy-sideload-collection-id", deploy_collection_id); + return ostree_repo_write_config (repo, config, error); +} + gboolean flatpak_repo_set_gpg_keys (OstreeRepo *repo, GBytes *bytes, @@ -3534,6 +3546,7 @@ flatpak_repo_update (OstreeRepo *repo, g_autofree char *old_ostree_metadata_checksum = NULL; g_autoptr(GVariant) old_ostree_metadata_v = NULL; gboolean deploy_collection_id = FALSE; + gboolean deploy_sideload_collection_id = FALSE; config = ostree_repo_get_config (repo); @@ -3547,6 +3560,7 @@ flatpak_repo_update (OstreeRepo *repo, default_branch = g_key_file_get_string (config, "flatpak", "default-branch", NULL); gpg_keys = g_key_file_get_string (config, "flatpak", "gpg-keys", NULL); redirect_url = g_key_file_get_string (config, "flatpak", "redirect-url", NULL); + deploy_sideload_collection_id = g_key_file_get_boolean (config, "flatpak", "deploy-sideload-collection-id", NULL); deploy_collection_id = g_key_file_get_boolean (config, "flatpak", "deploy-collection-id", NULL); authenticator_name = g_key_file_get_string (config, "flatpak", "authenticator-name", NULL); if (g_key_file_has_key (config, "flatpak", "authenticator-install", NULL)) @@ -3588,6 +3602,9 @@ flatpak_repo_update (OstreeRepo *repo, if (deploy_collection_id && collection_id != NULL) g_variant_builder_add (builder, "{sv}", OSTREE_META_KEY_DEPLOY_COLLECTION_ID, g_variant_new_string (collection_id)); + else if (deploy_sideload_collection_id && collection_id != NULL) + g_variant_builder_add (builder, "{sv}", "xa.deploy-collection-id", + g_variant_new_string (collection_id)); else if (deploy_collection_id) g_debug ("Ignoring deploy-collection-id=true because no collection ID is set."); diff --git a/doc/flatpak-build-update-repo.xml b/doc/flatpak-build-update-repo.xml index 743d9b34..cadac19b 100644 --- a/doc/flatpak-build-update-repo.xml +++ b/doc/flatpak-build-update-repo.xml @@ -164,6 +164,16 @@ + + + + + This is similar to --deploy-collection-id, but it only applies the + deploy to clients newer than flatpak 1.7 which supports the new form + of sideloades. + + + diff --git a/tests/test-update-remote-configuration.sh b/tests/test-update-remote-configuration.sh index 0aebe374..b4e16d02 100755 --- a/tests/test-update-remote-configuration.sh +++ b/tests/test-update-remote-configuration.sh @@ -70,6 +70,8 @@ ok "1 update repo config without deploying collection ID" # Now mark the collection ID as to be deployed. The client configuration should # be updated. UPDATE_REPO_ARGS="--collection-id=org.test.Collection --deploy-collection-id" update_repo +assert_file_has_content repos/test/config '^deploy-collection-id=true$' + ${FLATPAK} ${U} update -y org.test.App master assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify-summary=true$' @@ -78,6 +80,20 @@ assert_file_has_content ${FL_DIR}/repo/config '^gpg-verify=true$' assert_not_file_has_content ${FL_DIR}/repo/config '^gpg-verify=false$' assert_file_has_content ${FL_DIR}/repo/config '^collection-id=org\.test\.Collection$' +# Try the deploy for sideload only method +sed -i "s/deploy-collection-id=true//" repos/test/config +assert_not_file_has_content repos/test/config '^deploy-collection-id=true$' + +flatpak remote-modify --collection-id= test-repo +assert_not_file_has_content ${FL_DIR}/repo/config '^collection-id=org\.test\.Collection$' + +UPDATE_REPO_ARGS="--collection-id=org.test.Collection --deploy-sideload-collection-id" update_repo +assert_file_has_content repos/test/config '^deploy-sideload-collection-id=true$' + +${FLATPAK} ${U} update -y org.test.App master + +assert_file_has_content ${FL_DIR}/repo/config '^collection-id=org\.test\.Collection$' + ok "2 update repo config to deploy collection ID" # Try updating the collection ID to some other non-empty value on the server. From 9cf2ee7324f711e859ef67af02e3a839aead8a82 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 24 Mar 2020 09:16:11 +0100 Subject: [PATCH 22/36] dir: Resolve extra-data setup from sideload repo too --- common/flatpak-dir.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index db7cbb1c..7234cbe4 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -4470,10 +4470,11 @@ extra_data_progress_report (guint64 downloaded_bytes, static gboolean flatpak_dir_setup_extra_data (FlatpakDir *self, + FlatpakRemoteState *state, OstreeRepo *repo, - const char *repository, const char *ref, const char *rev, + GFile *sideload_repo, const char *token, FlatpakPullFlags flatpak_flags, OstreeAsyncProgress *progress, @@ -4505,11 +4506,11 @@ flatpak_dir_setup_extra_data (FlatpakDir *self, /* Pull the commits (and only the commits) to check for extra data * again. Here we don't pass the progress because we don't want any * reports coming out of it. */ - if (!repo_pull (repo, repository, + if (!repo_pull (repo, state->remote_name, NULL, ref, rev, - NULL, NULL, + state->collection_id, sideload_repo, token, flatpak_flags, OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY, @@ -5092,8 +5093,8 @@ flatpak_dir_pull (FlatpakDir *self, /* Setup extra data information before starting to pull, so we can have precise * progress reports */ - if (!flatpak_dir_setup_extra_data (self, repo, state->remote_name, - ref, rev, token, + if (!flatpak_dir_setup_extra_data (self, state, repo, + ref, rev, sideload_repo, token, flatpak_flags, progress, cancellable, From e4df0fa6a6d7c4f4a8dfcbc1d386a518c80590ca Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 24 Mar 2020 11:46:33 +0100 Subject: [PATCH 23/36] sideload: Support pulling individual objects from sideload repos We use the localcache-repos option to ostree_repo_pull to make ostree directly import any files that are locally available in the sideload repo even when pulling the main commit from upstream. This also adds a test that verifies that such files are not pulled via http. --- common/flatpak-dir.c | 39 ++++++++++++++++++++++++++++----------- tests/libtest.sh | 12 +++++++++++- tests/test-sideload.sh | 16 +++++++++++++++- 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index 7234cbe4..2922c59d 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -4337,11 +4337,10 @@ translate_ostree_repo_pull_errors (GError **error) static gboolean repo_pull (OstreeRepo *self, - const char *remote_name, + FlatpakRemoteState *state, const char **dirs_to_pull, const char *ref_to_fetch, const char *rev_to_fetch, /* (nullable) */ - const char *collection_id, GFile *sideload_repo, const char *token, FlatpakPullFlags flatpak_flags, @@ -4368,7 +4367,7 @@ repo_pull (OstreeRepo *self, /* We always want this on for every type of pull */ flags |= OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES; - if (!flatpak_repo_resolve_rev (self, NULL, remote_name, ref_to_fetch, TRUE, + if (!flatpak_repo_resolve_rev (self, NULL, state->remote_name, ref_to_fetch, TRUE, ¤t_checksum, cancellable, error)) return FALSE; @@ -4389,15 +4388,15 @@ repo_pull (OstreeRepo *self, g_debug ("Sideloading %s from %s in pull", ref_to_fetch, sideload_url); - g_assert (collection_id != NULL); + g_assert (state->collection_id != NULL); g_variant_builder_init (&colref_builder, G_VARIANT_TYPE ("a(sss)")); - g_variant_builder_add (&colref_builder, "(sss)", collection_id, ref_to_fetch, rev_to_fetch); + g_variant_builder_add (&colref_builder, "(sss)", state->collection_id, ref_to_fetch, rev_to_fetch); g_variant_builder_add (&builder, "{s@v}", "collection-refs", g_variant_new_variant (g_variant_builder_end (&colref_builder))); g_variant_builder_add (&builder, "{s@v}", "override-remote-name", - g_variant_new_variant (g_variant_new_string (remote_name))); + g_variant_new_variant (g_variant_new_string (state->remote_name))); } else { @@ -4410,12 +4409,30 @@ repo_pull (OstreeRepo *self, revs_to_fetch[1] = NULL; g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", g_variant_new_variant (g_variant_new_strv ((const char * const *) revs_to_fetch, -1))); + + + if (state->sideload_repos->len > 0) + { + GVariantBuilder localcache_repos_builder; + + g_variant_builder_init (&localcache_repos_builder, G_VARIANT_TYPE ("as")); + for (int i = 0; i < state->sideload_repos->len; i++) + { + FlatpakSideloadState *ss = g_ptr_array_index (state->sideload_repos, i); + GFile *sideload_path = ostree_repo_get_path (ss->repo); + + g_variant_builder_add (&localcache_repos_builder, "s", + flatpak_file_get_path_cached (sideload_path)); + } + g_variant_builder_add (&builder, "{s@v}", "localcache-repos", + g_variant_new_variant (g_variant_builder_end (&localcache_repos_builder))); + } } options = g_variant_ref_sink (g_variant_builder_end (&builder)); if (!ostree_repo_pull_with_options (self, - sideload_url ? sideload_url : remote_name, + sideload_url ? sideload_url : state->remote_name, options, progress, cancellable, error)) return translate_ostree_repo_pull_errors (error); @@ -4506,11 +4523,11 @@ flatpak_dir_setup_extra_data (FlatpakDir *self, /* Pull the commits (and only the commits) to check for extra data * again. Here we don't pass the progress because we don't want any * reports coming out of it. */ - if (!repo_pull (repo, state->remote_name, + if (!repo_pull (repo, state, NULL, ref, rev, - state->collection_id, sideload_repo, + sideload_repo, token, flatpak_flags, OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY, @@ -5108,9 +5125,9 @@ flatpak_dir_pull (FlatpakDir *self, flatpak_repo_resolve_rev (repo, NULL, state->remote_name, ref, TRUE, ¤t_checksum, NULL, NULL); - if (!repo_pull (repo, state->remote_name, + if (!repo_pull (repo, state, subdirs_arg ? (const char **) subdirs_arg->pdata : NULL, - ref, rev, state->collection_id, sideload_repo, token, flatpak_flags, flags, + ref, rev, sideload_repo, token, flatpak_flags, flags, progress, cancellable, error)) { diff --git a/tests/libtest.sh b/tests/libtest.sh index 1706dec9..556ec42d 100644 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -302,10 +302,14 @@ httpd () { rm -f httpd-pipe mkfifo httpd-pipe - $(dirname $0)/$COMMAND "$DIR" 3> httpd-pipe & + PYTHONUNBUFFERED=1 $(dirname $0)/$COMMAND "$DIR" 3> httpd-pipe 2>&1 | tee --append httpd-log & read < httpd-pipe } +httpd_clear_log () { + truncate -s 0 httpd-log +} + setup_repo_no_add () { REPONAME=${1:-test} if [ x${USE_COLLECTIONS_IN_SERVER-} == xyes ] ; then @@ -503,6 +507,12 @@ gdb_bt () { gdb -batch -ex "run" -ex "thread apply all bt" -ex "quit 1" --args "$@" } +commit_to_path () { + COMMIT=$1 + EXT=$2 + echo "objects/$(echo $COMMIT | cut -b 1-2)/$(echo $COMMIT | cut -b 3-)".${EXT} +} + cleanup () { /bin/kill -9 $DBUS_SESSION_BUS_PID gpg-connect-agent --homedir "${FL_GPG_HOMEDIR}" killagent /bye || true diff --git a/tests/test-sideload.sh b/tests/test-sideload.sh index a351f380..91ea43ec 100644 --- a/tests/test-sideload.sh +++ b/tests/test-sideload.sh @@ -89,10 +89,15 @@ ${FLATPAK} ${U} uninstall -y --all ${FLATPAK} ${U} config --unset sideload-repos mv repos/test/objects repos/test/objects.disabled +httpd_clear_log + # Ensure this fails (but still loads summary) if ${FLATPAK} ${U} install -y test-repo org.test.Hello &> install-error-log; then assert_not_reached "Disabled online install broken" fi +assert_file_has_content install-error-log "Server returned status 404: Not Found" +assert_file_has_content httpd-log "GET .*commit .*404" + assert_file_has_content install-error-log "Server returned status 404: Not Found" ${FLATPAK} ${U} config --set sideload-repos ${SIDELOAD_REPO} @@ -107,12 +112,21 @@ ok "installed sideloaded app when online" OLD_COMMIT=$(cat repos/test/refs/heads/app/org.test.Hello/${ARCH}/master) +# Make new app version make_updated_app update_repo NEW_COMMIT=$(cat repos/test/refs/heads/app/org.test.Hello/$ARCH/master) -${FLATPAK} ${U} update -y +# (Re)install the new version (online), ensure we used sideload repo for objects shared between new and old version +SHARED_OBJECT=$(ostree ls --repo=repos/test -C app/org.test.Hello/$ARCH/master /files/share/applications/org.test.Hello.desktop | awk '{ print $5 }') +SHARED_OBJECT_PATH=$(commit_to_path $SHARED_OBJECT filez) +${FLATPAK} ${U} uninstall -y app/org.test.Hello/$ARCH/master + +httpd_clear_log +${FLATPAK} ${U} install -y app/org.test.Hello/$ARCH/master + +assert_not_file_has_content httpd-log "GET /test/${SHARED_OBJECT_PATH}" # Prepare sideload repo for NEW_COMMIT ${FLATPAK} ${U} create-usb --destination-repo=repo2 usb_dir org.test.Hello From 1d1703fcd825c5e5775ba9301cc80a3ecd64172c Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 24 Mar 2020 12:06:43 +0100 Subject: [PATCH 24/36] tests: Enable stderr/stdout merging This makes it a lot easier to read the error logs, and I don't think we have anything that writes something that confuses TAP to stderr. --- buildutil/glib-tap.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutil/glib-tap.mk b/buildutil/glib-tap.mk index b05ad5f1..943a31c4 100644 --- a/buildutil/glib-tap.mk +++ b/buildutil/glib-tap.mk @@ -7,7 +7,7 @@ TESTS_ENVIRONMENT= \ G_DEBUG=gc-friendly \ MALLOC_CHECK_=2 \ MALLOC_PERTURB_=$$(($${RANDOM:-256} % 256)) -LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) $(top_srcdir)/buildutil/tap-driver.sh +LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) $(top_srcdir)/buildutil/tap-driver.sh --merge LOG_COMPILER = $(top_srcdir)/buildutil/tap-test TESTS = From b8d2196c2e31814a0d55c70376a463dbbf9cf208 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 24 Mar 2020 12:51:12 +0100 Subject: [PATCH 25/36] create-usb: Add --allow-partial option If you really want to export partial refs you can use this switch. --- app/flatpak-builtins-create-usb.c | 6 ++++-- doc/flatpak-create-usb.xml | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/flatpak-builtins-create-usb.c b/app/flatpak-builtins-create-usb.c index 6ab4175f..981f6a5f 100644 --- a/app/flatpak-builtins-create-usb.c +++ b/app/flatpak-builtins-create-usb.c @@ -38,12 +38,14 @@ static char *opt_arch; static char *opt_destination_repo; static gboolean opt_runtime; static gboolean opt_app; +static gboolean opt_allow_partial; static GOptionEntry options[] = { { "app", 0, 0, G_OPTION_ARG_NONE, &opt_app, N_("Look for app with the specified name"), NULL }, { "arch", 0, 0, G_OPTION_ARG_STRING, &opt_arch, N_("Arch to copy"), N_("ARCH") }, { "destination-repo", 0, 0, G_OPTION_ARG_FILENAME, &opt_destination_repo, "Use custom repository directory within the mount", N_("DEST") }, { "runtime", 0, 0, G_OPTION_ARG_NONE, &opt_runtime, N_("Look for runtime with the specified name"), NULL }, + { "allow-partial", 0, 0, G_OPTION_ARG_NONE, &opt_allow_partial, N_("Allow partial commits in the created repo"), NULL }, { NULL } }; @@ -120,7 +122,7 @@ add_related (GHashTable *all_refs, if (deploy_data == NULL) return FALSE; - if (flatpak_deploy_data_has_subpaths (deploy_data)) + if (flatpak_deploy_data_has_subpaths (deploy_data) && !opt_allow_partial) return flatpak_fail (error, _("Related ref '%s' is only partially installed"), ref); commit = flatpak_deploy_data_get_commit (deploy_data); @@ -639,7 +641,7 @@ flatpak_builtin_create_usb (int argc, char **argv, GCancellable *cancellable, GE if (deploy_data == NULL) return FALSE; - if (flatpak_deploy_data_has_subpaths (deploy_data)) + if (flatpak_deploy_data_has_subpaths (deploy_data) && !opt_allow_partial) return flatpak_fail (error, _("Ref '%s' is only partially installed"), installed_ref); diff --git a/doc/flatpak-create-usb.xml b/doc/flatpak-create-usb.xml index 059abc7d..8ab7c526 100644 --- a/doc/flatpak-create-usb.xml +++ b/doc/flatpak-create-usb.xml @@ -159,6 +159,15 @@ + + + + + Allow exporting partially installed commits, for example locale extensions without all languages. These can cause + problems when installing them, for example if the language config is different on the installing side. + + + From a843d2d594b39f7149b1b23182726ba5802bd3ab Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 24 Mar 2020 14:01:20 +0100 Subject: [PATCH 26/36] sideload: Add api and CLI support to specify sideload repos dynamically --- app/flatpak-builtins-install.c | 15 ++++++- app/flatpak-builtins-remote-info.c | 2 +- app/flatpak-builtins-update.c | 5 +++ app/flatpak-complete.c | 2 +- common/flatpak-dir-private.h | 5 +++ common/flatpak-dir.c | 67 ++++++++++++++++++------------ common/flatpak-transaction.c | 35 +++++++++++++++- common/flatpak-transaction.h | 3 ++ doc/flatpak-install.xml | 10 +++++ doc/flatpak-update.xml | 10 +++++ tests/test-sideload.sh | 20 ++++++++- 11 files changed, 141 insertions(+), 33 deletions(-) diff --git a/app/flatpak-builtins-install.c b/app/flatpak-builtins-install.c index 09bf62d2..fa2d080a 100644 --- a/app/flatpak-builtins-install.c +++ b/app/flatpak-builtins-install.c @@ -42,6 +42,7 @@ static char *opt_arch; static char **opt_gpg_file; static char **opt_subpaths; +static char **opt_sideload_repos; static gboolean opt_no_pull; static gboolean opt_no_deploy; static gboolean opt_no_related; @@ -73,6 +74,7 @@ static GOptionEntry options[] = { { "reinstall", 0, 0, G_OPTION_ARG_NONE, &opt_reinstall, N_("Uninstall first if already installed"), NULL }, { "noninteractive", 0, 0, G_OPTION_ARG_NONE, &opt_noninteractive, N_("Produce minimal output and don't ask questions"), NULL }, { "or-update", 0, 0, G_OPTION_ARG_NONE, &opt_or_update, N_("Update install if already installed"), NULL }, + { "sideload-repo", 0, 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_sideload_repos, N_("Use this local repo for sideloads"), N_("PATH") }, { NULL } }; @@ -165,6 +167,9 @@ install_bundle (FlatpakDir *dir, flatpak_transaction_set_disable_related (transaction, opt_no_related); flatpak_transaction_set_reinstall (transaction, opt_reinstall); + for (int i = 0; opt_sideload_repos != NULL && opt_sideload_repos[i] != NULL; i++) + flatpak_transaction_add_sideload_repo (transaction, opt_sideload_repos[i]); + if (!flatpak_transaction_add_install_bundle (transaction, file, gpg_data, error)) return FALSE; @@ -239,6 +244,9 @@ install_from (FlatpakDir *dir, flatpak_transaction_set_reinstall (transaction, opt_reinstall); flatpak_transaction_set_default_arch (transaction, opt_arch); + for (int i = 0; opt_sideload_repos != NULL && opt_sideload_repos[i] != NULL; i++) + flatpak_transaction_add_sideload_repo (transaction, opt_sideload_repos[i]); + if (!flatpak_transaction_add_install_flatpakref (transaction, file_data, error)) return FALSE; @@ -392,7 +400,7 @@ flatpak_builtin_install (int argc, char **argv, GCancellable *cancellable, GErro matched_kinds, FIND_MATCHING_REFS_FLAGS_FUZZY, cancellable, &local_error); else - refs = flatpak_dir_find_remote_refs (this_dir, this_remote, id, branch, this_default_branch, arch, + refs = flatpak_dir_find_remote_refs (this_dir, this_remote, (const char **)opt_sideload_repos, id, branch, this_default_branch, arch, flatpak_get_default_arch (), matched_kinds, FIND_MATCHING_REFS_FLAGS_FUZZY, cancellable, &local_error); @@ -459,6 +467,9 @@ flatpak_builtin_install (int argc, char **argv, GCancellable *cancellable, GErro flatpak_transaction_set_disable_related (transaction, opt_no_related); flatpak_transaction_set_reinstall (transaction, opt_reinstall); + for (i = 0; opt_sideload_repos != NULL && opt_sideload_repos[i] != NULL; i++) + flatpak_transaction_add_sideload_repo (transaction, opt_sideload_repos[i]); + for (i = 0; i < n_prefs; i++) { const char *pref = prefs[i]; @@ -484,7 +495,7 @@ flatpak_builtin_install (int argc, char **argv, GCancellable *cancellable, GErro matched_kinds, FIND_MATCHING_REFS_FLAGS_FUZZY, cancellable, error); else - refs = flatpak_dir_find_remote_refs (dir, remote, id, branch, default_branch, arch, + refs = flatpak_dir_find_remote_refs (dir, remote, (const char **)opt_sideload_repos, id, branch, default_branch, arch, flatpak_get_default_arch (), matched_kinds, FIND_MATCHING_REFS_FLAGS_FUZZY, cancellable, error); diff --git a/app/flatpak-builtins-remote-info.c b/app/flatpak-builtins-remote-info.c index fa0d332e..d5eb81d7 100644 --- a/app/flatpak-builtins-remote-info.c +++ b/app/flatpak-builtins-remote-info.c @@ -135,7 +135,7 @@ flatpak_builtin_remote_info (int argc, char **argv, GCancellable *cancellable, G &matched_kinds, &id, &arch, &branch, error)) return FALSE; - ref = flatpak_dir_find_remote_ref (preferred_dir, remote, id, branch, default_branch, arch, + ref = flatpak_dir_find_remote_ref (preferred_dir, remote, NULL, id, branch, default_branch, arch, matched_kinds, &kind, cancellable, error); if (ref == NULL) return FALSE; diff --git a/app/flatpak-builtins-update.c b/app/flatpak-builtins-update.c index 163a6763..3da7a067 100644 --- a/app/flatpak-builtins-update.c +++ b/app/flatpak-builtins-update.c @@ -39,6 +39,7 @@ static char *opt_arch; static char *opt_commit; static char **opt_subpaths; +static char **opt_sideload_repos; static gboolean opt_force_remove; static gboolean opt_no_pull; static gboolean opt_no_deploy; @@ -66,6 +67,7 @@ static GOptionEntry options[] = { { "subpath", 0, 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_subpaths, N_("Only update this subpath"), N_("PATH") }, { "assumeyes", 'y', 0, G_OPTION_ARG_NONE, &opt_yes, N_("Automatically answer yes for all questions"), NULL }, { "noninteractive", 0, 0, G_OPTION_ARG_NONE, &opt_noninteractive, N_("Produce minimal output and don't ask questions"), NULL }, + { "sideload-repo", 0, 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_sideload_repos, N_("Use this local repo for sideloads"), N_("PATH") }, { NULL } }; @@ -141,6 +143,9 @@ flatpak_builtin_update (int argc, flatpak_transaction_set_disable_dependencies (transaction, opt_no_deps); flatpak_transaction_set_disable_related (transaction, opt_no_related); + for (int i = 0; opt_sideload_repos != NULL && opt_sideload_repos[i] != NULL; i++) + flatpak_transaction_add_sideload_repo (transaction, opt_sideload_repos[i]); + g_ptr_array_insert (transactions, 0, transaction); } diff --git a/app/flatpak-complete.c b/app/flatpak-complete.c index 5eb4b66e..9010b4b7 100644 --- a/app/flatpak-complete.c +++ b/app/flatpak-complete.c @@ -194,7 +194,7 @@ flatpak_complete_partial_ref (FlatpakCompletion *completion, if (remote) { - refs = flatpak_dir_find_remote_refs (dir, completion->argv[1], + refs = flatpak_dir_find_remote_refs (dir, completion->argv[1], NULL, (element > 1) ? id : NULL, (element > 3) ? branch : NULL, NULL, /* default branch */ diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index e6652619..b0b6211d 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -162,6 +162,9 @@ GVariant *flatpak_remote_state_load_ref_commit (FlatpakRemoteState *self, const char *ref, const char *commit, GError **error); +void flatpak_remote_state_add_sideload_repo (FlatpakRemoteState *self, + const char *path); + G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakDir, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakDeploy, g_object_unref) @@ -457,6 +460,7 @@ gboolean flatpak_dir_ref_is_masked (FlatpakDir *self, const char *ref); char * flatpak_dir_find_remote_ref (FlatpakDir *self, const char *remote, + const char **opt_sideload_repos, const char *name, const char *opt_branch, const char *opt_default_branch, @@ -467,6 +471,7 @@ char * flatpak_dir_find_remote_ref (FlatpakDir *self, GError **error); char ** flatpak_dir_find_remote_refs (FlatpakDir *self, const char *remote, + const char **opt_sideload_repos, const char *name, const char *opt_branch, const char *opt_default_branch, diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index 2922c59d..304218a8 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -324,6 +324,35 @@ flatpak_remote_state_unref (FlatpakRemoteState *remote_state) } } +void +flatpak_remote_state_add_sideload_repo (FlatpakRemoteState *self, + const char *path) +{ + g_autoptr(GFile) f = g_file_new_for_path (path); + g_autoptr(GFile) summary_path = g_file_get_child (f, "summary"); + g_autoptr(GMappedFile) mfile = NULL; + g_autoptr(OstreeRepo) sideload_repo = ostree_repo_new (f); + + /* Sideloading only works if collection id is set */ + if (self->collection_id == NULL) + return; + + /* TODO: The sideloadstates are duplicated for each remote with the collection id set, we should maybe reuse them */ + + mfile = g_mapped_file_new (flatpak_file_get_path_cached (summary_path), FALSE, NULL); + if (mfile != NULL && ostree_repo_open (sideload_repo, NULL, NULL)) + { + g_autoptr(GBytes) summary_bytes = g_mapped_file_get_bytes (mfile); + FlatpakSideloadState *ss = g_new0 (FlatpakSideloadState, 1); + + ss->repo = g_steal_pointer (&sideload_repo); + ss->summary = g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, summary_bytes, TRUE)); + g_ptr_array_add (self->sideload_repos, ss); + + g_debug ("Using sideloaded repo %s for remote %s", path, self->remote_name); + } +} + gboolean flatpak_remote_state_ensure_summary (FlatpakRemoteState *self, GError **error) @@ -10506,6 +10535,7 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, GError **error) { g_autoptr(FlatpakRemoteState) state = flatpak_remote_state_new (); + g_auto(GStrv) sideload_paths = NULL; g_autoptr(GError) my_error = NULL; gboolean is_local; @@ -10583,32 +10613,9 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, } } - if (state->collection_id) - { - g_auto(GStrv) sideload_paths = flatpak_dir_get_sideload_repo_paths (self); - for (int i = 0; sideload_paths != NULL && sideload_paths[i] != NULL; i++) - { - g_autoptr(GFile) f = g_file_new_for_path (sideload_paths[i]); - g_autoptr(GFile) summary_path = g_file_get_child (f, "summary"); - g_autoptr(GMappedFile) mfile = NULL; - g_autoptr(OstreeRepo) sideload_repo = ostree_repo_new (f); - - /* TODO: The sideloadstates are duplicated for each remote with the collection id set, we should maybe reuse them */ - - mfile = g_mapped_file_new (flatpak_file_get_path_cached (summary_path), FALSE, NULL); - if (mfile != NULL && ostree_repo_open (sideload_repo, NULL, NULL)) - { - g_autoptr(GBytes) summary_bytes = g_mapped_file_get_bytes (mfile); - FlatpakSideloadState *ss = g_new0 (FlatpakSideloadState, 1); - - ss->repo = g_steal_pointer (&sideload_repo); - ss->summary = g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, summary_bytes, TRUE)); - g_ptr_array_add (state->sideload_repos, ss); - - g_debug ("Using sideloaded repo %s for remote %s", sideload_paths[i], remote_or_uri); - } - } - } + sideload_paths = flatpak_dir_get_sideload_repo_paths (self); + for (int i = 0; sideload_paths != NULL && sideload_paths[i] != NULL; i++) + flatpak_remote_state_add_sideload_repo (state, sideload_paths[i]); if (state->summary != NULL) /* In the optional case we might not have a summary */ { @@ -11030,6 +11037,7 @@ flatpak_dir_get_remote_collection_id (FlatpakDir *self, char ** flatpak_dir_find_remote_refs (FlatpakDir *self, const char *remote, + const char **opt_sideload_repos, const char *name, const char *opt_branch, const char *opt_default_branch, @@ -11048,6 +11056,9 @@ flatpak_dir_find_remote_refs (FlatpakDir *self, if (state == NULL) return NULL; + for (int i = 0; opt_sideload_repos != NULL && opt_sideload_repos[i] != NULL; i++) + flatpak_remote_state_add_sideload_repo (state, opt_sideload_repos[i]); + if (!flatpak_dir_list_all_remote_refs (self, state, &remote_refs, cancellable, error)) return NULL; @@ -11130,6 +11141,7 @@ find_ref_for_refs_set (GHashTable *refs, char * flatpak_dir_find_remote_ref (FlatpakDir *self, const char *remote, + const char **opt_sideload_repos, const char *name, const char *opt_branch, const char *opt_default_branch, @@ -11148,6 +11160,9 @@ flatpak_dir_find_remote_ref (FlatpakDir *self, if (state == NULL) return NULL; + for (int i = 0; opt_sideload_repos != NULL && opt_sideload_repos[i] != NULL; i++) + flatpak_remote_state_add_sideload_repo (state, opt_sideload_repos[i]); + if (!flatpak_dir_list_all_remote_refs (self, state, &remote_refs, cancellable, error)) return NULL; diff --git a/common/flatpak-transaction.c b/common/flatpak-transaction.c index 00a2cbe5..cf2042a3 100644 --- a/common/flatpak-transaction.c +++ b/common/flatpak-transaction.c @@ -156,6 +156,7 @@ struct _FlatpakTransactionPrivate GHashTable *last_op_for_ref; GHashTable *remote_states; /* (element-type utf8 FlatpakRemoteState) */ GPtrArray *extra_dependency_dirs; + GPtrArray *extra_sideload_repos; GList *ops; GPtrArray *added_origin_remotes; @@ -480,6 +481,27 @@ flatpak_transaction_add_dependency_source (FlatpakTransaction *self, flatpak_installation_clone_dir_noensure (installation)); } +/** + * flatpak_transaction_add_sideload_repo: + * @self: a #FlatpakTransaction + * @path: a path to a local flatpak repository + * + * Adds an extra local ostree repo as source for installation. This is equivalent + * with setting the xa.sideload-repos global option, but can be done dynamically. + * If the option is set both sources are used. + * + * Since: 1.7.1 + */ +void +flatpak_transaction_add_sideload_repo (FlatpakTransaction *self, + const char *path) +{ + FlatpakTransactionPrivate *priv = flatpak_transaction_get_instance_private (self); + + g_ptr_array_add (priv->extra_sideload_repos, + g_strdup (path)); +} + /** * flatpak_transaction_add_default_dependency_sources: * @self: a #FlatpakTransaction @@ -873,6 +895,7 @@ flatpak_transaction_finalize (GObject *object) g_ptr_array_unref (priv->added_origin_remotes); g_ptr_array_free (priv->extra_dependency_dirs, TRUE); + g_ptr_array_free (priv->extra_sideload_repos, TRUE); G_OBJECT_CLASS (flatpak_transaction_parent_class)->finalize (object); } @@ -1244,6 +1267,7 @@ flatpak_transaction_init (FlatpakTransaction *self) priv->remote_states = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, (GDestroyNotify) flatpak_remote_state_unref); priv->added_origin_remotes = g_ptr_array_new_with_free_func (g_free); priv->extra_dependency_dirs = g_ptr_array_new_with_free_func (g_object_unref); + priv->extra_sideload_repos = g_ptr_array_new_with_free_func (g_free); priv->can_run = TRUE; } @@ -1622,8 +1646,17 @@ flatpak_transaction_ensure_remote_state (FlatpakTransaction *self, state = flatpak_dir_get_remote_state_optional (priv->dir, remote, FALSE, NULL, error); + g_printerr ("flatpak_transaction_ensure_remote_state\n"); if (state) - g_hash_table_insert (priv->remote_states, state->remote_name, flatpak_remote_state_ref (state)); + { + g_hash_table_insert (priv->remote_states, state->remote_name, flatpak_remote_state_ref (state)); + + for (int i = 0; i < priv->extra_sideload_repos->len; i++) + { + const char *path = g_ptr_array_index (priv->extra_sideload_repos, i); + flatpak_remote_state_add_sideload_repo (state, path); + } + } return state; } diff --git a/common/flatpak-transaction.h b/common/flatpak-transaction.h index cf4c811a..f66ddee2 100644 --- a/common/flatpak-transaction.h +++ b/common/flatpak-transaction.h @@ -230,6 +230,9 @@ FLATPAK_EXTERN void flatpak_transaction_add_dependency_source (FlatpakTransaction *self, FlatpakInstallation *installation); FLATPAK_EXTERN +void flatpak_transaction_add_sideload_repo (FlatpakTransaction *self, + const char *path); +FLATPAK_EXTERN void flatpak_transaction_add_default_dependency_sources (FlatpakTransaction *self); FLATPAK_EXTERN gboolean flatpak_transaction_run (FlatpakTransaction *transaction, diff --git a/doc/flatpak-install.xml b/doc/flatpak-install.xml index 0b04b0fb..bcccc180 100644 --- a/doc/flatpak-install.xml +++ b/doc/flatpak-install.xml @@ -256,6 +256,16 @@ Assume that all REFs are runtimes if not explicitly specified. + + + + + + Adds an extra local ostree repo as source for installation. This is equivalent + with setting the xa.sideload-repos global option, but can be done without modifying the configuration. + If the option is already set both sources are used. + + diff --git a/doc/flatpak-update.xml b/doc/flatpak-update.xml index d1c641ef..78959747 100644 --- a/doc/flatpak-update.xml +++ b/doc/flatpak-update.xml @@ -215,6 +215,16 @@ + + + + + Adds an extra local ostree repo as source for installation. This is equivalent + with setting the xa.sideload-repos global option, but can be done without modifying the configuration. + If the option is already set both sources are used. + + + diff --git a/tests/test-sideload.sh b/tests/test-sideload.sh index 91ea43ec..8ad79379 100644 --- a/tests/test-sideload.sh +++ b/tests/test-sideload.sh @@ -27,7 +27,7 @@ USE_COLLECTIONS_IN_CLIENT=yes skip_without_bwrap skip_revokefs_without_fuse -echo "1..8" +echo "1..9" #Regular repo setup_repo @@ -71,6 +71,19 @@ assert_not_has_file $FL_DIR/repo/refs/mirrors/org.test.Collection.test/app/org.t ok "installed sideloaded app" +${FLATPAK} ${U} config --unset sideload-repos + +${FLATPAK} ${U} uninstall -y org.test.Hello +${FLATPAK} ${U} install --sideload-repo=${SIDELOAD_REPO} -y test-repo org.test.Hello + +assert_has_file $FL_DIR/app/org.test.Hello/$ARCH/master/active/metadata +assert_has_file $FL_DIR/repo/refs/remotes/test-repo/app/org.test.Hello/${ARCH}/master +assert_not_has_file $FL_DIR/repo/refs/mirrors/org.test.Collection.test/app/org.test.Hello/${ARCH}/master + +${FLATPAK} ${U} config --set sideload-repos ${SIDELOAD_REPO} + +ok "installed w/ runtime sideload option" + # Remove old appstream checkouts so we can update from the sideload rm -rf $FL_DIR/appstream/test-repo/$ARCH/ rm -rf $FL_DIR/repo/refs/remotes/test-repo/appstream2/$ARCH @@ -165,8 +178,11 @@ ok "update to explicit old version" mv usb_dir/repo usb_dir/repo.old mv usb_dir/repo2 usb_dir/repo +# Try with --sideload-repo instead of config +${FLATPAK} ${U} config --unset sideload-repos + # And update to it (offline) -${FLATPAK} ${U} update -y +${FLATPAK} ${U} update -y --sideload-repo=${SIDELOAD_REPO} UPDATED_COMMIT=$( ${FLATPAK} ${U} info --show-commit app/org.test.Hello/${ARCH}/master ) assert_streq "$NEW_COMMIT" "$UPDATED_COMMIT" From 8999e70c4af6d56c4c53cfd393f4e22524f0358d Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 24 Mar 2020 16:13:24 +0100 Subject: [PATCH 27/36] RemoteState: Try the sideload paths even for local-only states This is so we can use these for listing sideloaded refs. --- common/flatpak-dir.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index 304218a8..adffefb7 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -10559,6 +10559,10 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, state->default_token_type = flatpak_dir_get_remote_default_token_type (self, remote_or_uri); } + sideload_paths = flatpak_dir_get_sideload_repo_paths (self); + for (int i = 0; sideload_paths != NULL && sideload_paths[i] != NULL; i++) + flatpak_remote_state_add_sideload_repo (state, sideload_paths[i]); + if (local_only) { flatpak_fail (&state->summary_fetch_error, "Internal error, local_only state"); @@ -10613,10 +10617,6 @@ _flatpak_dir_get_remote_state (FlatpakDir *self, } } - sideload_paths = flatpak_dir_get_sideload_repo_paths (self); - for (int i = 0; sideload_paths != NULL && sideload_paths[i] != NULL; i++) - flatpak_remote_state_add_sideload_repo (state, sideload_paths[i]); - if (state->summary != NULL) /* In the optional case we might not have a summary */ { VarSummaryRef summary = var_summary_from_gvariant (state->summary); From 9c6bdc5bfd62b5a7e041a019fe0ab8f35f083756 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 24 Mar 2020 16:15:23 +0100 Subject: [PATCH 28/36] dir: Add flatpak_remote_state_load_data() function This is similar to lookup_cache() but it also works for sideloaded refs. Additionally it returns an allocated metadata pointer rather than a pointer to the cache. Also convert some callers to use this when it makes sense. --- common/flatpak-dir-private.h | 7 +- common/flatpak-dir.c | 207 ++++++++++++++++++++++------------ common/flatpak-installation.c | 18 +-- common/flatpak-remote-ref.c | 13 ++- common/flatpak-transaction.c | 2 +- 5 files changed, 160 insertions(+), 87 deletions(-) diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index b0b6211d..a6563fc9 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -151,8 +151,13 @@ gboolean flatpak_remote_state_lookup_cache (FlatpakRemoteState *self, guint64 *download_size, guint64 *installed_size, const char **metadata, - const guchar **maybe_commit_bytes, GError **error); +gboolean flatpak_remote_state_load_data (FlatpakRemoteState *self, + const char *ref, + guint64 *out_download_size, + guint64 *out_installed_size, + char **out_metadata, + GError **error); gboolean flatpak_remote_state_lookup_sparse_cache (FlatpakRemoteState *self, const char *ref, VarMetadataRef *out_metadata, diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index adffefb7..f5d7c787 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -397,6 +397,58 @@ flatpak_remote_state_lookup_sideload_checksum (FlatpakRemoteState *self, return NULL; } +static gboolean +flatpak_remote_state_resolve_sideloaded_ref (FlatpakRemoteState *self, + const char *ref, + char **out_checksum, + guint64 *out_timestamp, + VarRefInfoRef *out_info, + FlatpakSideloadState **out_sideload_state, + GError **error) +{ + g_autofree char *latest_checksum = NULL; + guint64 latest_timestamp = 0; + FlatpakSideloadState *latest_ss = NULL; + VarRefInfoRef latest_sideload_info; + + for (int i = 0; i < self->sideload_repos->len; i++) + { + FlatpakSideloadState *ss = g_ptr_array_index (self->sideload_repos, i); + g_autofree char *sideload_checksum = NULL; + VarRefInfoRef sideload_info; + + if (flatpak_summary_lookup_ref (ss->summary, self->collection_id, ref, &sideload_checksum, &sideload_info)) + { + guint64 timestamp = get_timestamp_from_ref_info (sideload_info); + + if (latest_checksum == NULL || latest_timestamp < timestamp) + { + g_free (latest_checksum); + latest_checksum = g_steal_pointer (&sideload_checksum); + latest_timestamp = timestamp; + latest_sideload_info = sideload_info; + latest_ss = ss; + } + } + } + + if (latest_checksum == NULL) + return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, + _("No such ref '%s' in remote %s"), + ref, self->remote_name); + + if (out_checksum) + *out_checksum = g_steal_pointer (&latest_checksum); + if (out_timestamp) + *out_timestamp = latest_timestamp; + if (out_info) + *out_info = latest_sideload_info; + if (out_sideload_state) + *out_sideload_state = latest_ss; + + return TRUE; +} + /* Returns TRUE if the ref is found in the summary or cache. * out_checksum and out_variant are only set when the ref is found. @@ -459,47 +511,13 @@ flatpak_remote_state_lookup_ref (FlatpakRemoteState *self, } else { - g_autofree char *latest_checksum = NULL; - guint64 latest_timestamp = 0; - VarRefInfoRef latest_sideload_info; - g_autoptr(GFile) latest_sideload_path = NULL; + FlatpakSideloadState *ss = NULL; - for (int i = 0; i < self->sideload_repos->len; i++) - { - FlatpakSideloadState *ss = g_ptr_array_index (self->sideload_repos, i); - g_autofree char *sideload_checksum = NULL; - VarRefInfoRef sideload_info; + if (!flatpak_remote_state_resolve_sideloaded_ref (self, ref, out_checksum, out_timestamp, out_info, &ss, error)) + return FALSE; - if (flatpak_summary_lookup_ref (ss->summary, self->collection_id, ref, &sideload_checksum, &sideload_info)) - { - guint64 timestamp = get_timestamp_from_ref_info (sideload_info); - - if (latest_checksum == NULL || latest_timestamp < timestamp) - { - g_free (latest_checksum); - latest_checksum = g_steal_pointer (&sideload_checksum); - latest_timestamp = timestamp; - latest_sideload_info = sideload_info; - g_set_object (&latest_sideload_path, ostree_repo_get_path (ss->repo)); - } - } - } - - if (latest_checksum) - { - if (out_checksum) - *out_checksum = g_steal_pointer (&latest_checksum); - if (out_timestamp) - *out_timestamp = latest_timestamp; - if (out_info) - *out_info = latest_sideload_info; - if (out_sideload_path) - *out_sideload_path = g_steal_pointer (&latest_sideload_path); - } - else - return flatpak_fail_error (error, FLATPAK_ERROR_REF_NOT_FOUND, - _("No such ref '%s' in remote %s"), - ref, self->remote_name); + if (out_sideload_path) + *out_sideload_path = g_object_ref (ostree_repo_get_path (ss->repo)); } return TRUE; @@ -551,10 +569,9 @@ flatpak_remote_state_get_cache (FlatpakRemoteState *self, gboolean flatpak_remote_state_lookup_cache (FlatpakRemoteState *self, const char *ref, - guint64 *download_size, - guint64 *installed_size, - const char **metadata, - const guchar **maybe_commit_bytes, + guint64 *out_download_size, + guint64 *out_installed_size, + const char **out_metadata, GError **error) { VarCacheRef cache; @@ -569,37 +586,83 @@ flatpak_remote_state_lookup_cache (FlatpakRemoteState *self, _("No entry for %s in remote '%s' summary flatpak cache "), ref, self->remote_name); - if (installed_size) - *installed_size = var_cache_data_get_installed_size (cache_data); + if (out_installed_size) + *out_installed_size = var_cache_data_get_installed_size (cache_data); - if (download_size) - *download_size = var_cache_data_get_download_size (cache_data); + if (out_download_size) + *out_download_size = var_cache_data_get_download_size (cache_data); - if (metadata) - *metadata = var_cache_data_get_metadata (cache_data); - - if (maybe_commit_bytes) - { - VarSummaryRef summary = var_summary_from_gvariant (self->summary); - VarMetadataRef meta = var_summary_get_metadata (summary); - VarVariantRef commits_v; - - *maybe_commit_bytes = NULL; - if (var_metadata_lookup (meta, "xa.commits", NULL, &commits_v)) - { - VarCommitsCacheRef commits = var_commits_cache_from_variant (commits_v); - if (pos < var_commits_cache_get_length (commits)) - { - VarChecksumRef checksum = var_commits_cache_get_at (commits, pos); - if (var_checksum_get_length (checksum) == OSTREE_SHA256_DIGEST_LEN) - *maybe_commit_bytes = var_checksum_peek (checksum); - } - } - } + if (out_metadata) + *out_metadata = var_cache_data_get_metadata (cache_data); return TRUE; } +gboolean +flatpak_remote_state_load_data (FlatpakRemoteState *self, + const char *ref, + guint64 *out_download_size, + guint64 *out_installed_size, + char **out_metadata, + GError **error) +{ + if (self->summary) + { + const char *metadata = NULL; + if (!flatpak_remote_state_lookup_cache (self, ref, out_download_size, out_installed_size,&metadata, error)) + return FALSE; + + if (out_metadata) + *out_metadata = g_strdup (metadata); + } + else + { + /* Look up from sideload */ + g_autofree char *checksum = NULL; + guint64 timestamp; + VarRefInfoRef info; + FlatpakSideloadState *ss = NULL; + g_autoptr(GVariant) commit_data = NULL; + g_autoptr(GVariant) commit_metadata = NULL; + const char *xa_metadata = NULL; + guint64 download_size = 0; + guint64 installed_size = 0; + g_autoptr(GBytes) metadata_bytes = NULL; + + /* Use sideload refs if any */ + + if (!flatpak_remote_state_resolve_sideloaded_ref (self, ref, &checksum, ×tamp, + &info, &ss, error)) + return FALSE; + + if (!ostree_repo_load_commit (ss->repo, checksum, &commit_data, NULL, error)) + return FALSE; + + commit_metadata = g_variant_get_child_value (commit_data, 0); + g_variant_lookup (commit_metadata, "xa.metadata", "&s", &xa_metadata); + if (xa_metadata == NULL) + return flatpak_fail (error, "No xa.metadata in sideload commit %s ref %s", checksum, ref); + + if (g_variant_lookup (commit_metadata, "xa.download-size", "t", &download_size)) + download_size = GUINT64_FROM_BE (download_size); + if (g_variant_lookup (commit_metadata, "xa.installed-size", "t", &installed_size)) + installed_size = GUINT64_FROM_BE (installed_size); + + if (out_installed_size) + *out_installed_size = installed_size; + + if (out_download_size) + *out_download_size = download_size; + + if (out_metadata) + *out_metadata = g_strdup (xa_metadata); + } + return TRUE; +} + + + + /* Tries to load the specified commit object that we resolved from this remote. This either comes from the already available local repo, or from one of the sideloading repos, and if not available we @@ -13439,7 +13502,7 @@ flatpak_dir_find_remote_related (FlatpakDir *self, GCancellable *cancellable, GError **error) { - const char *metadata = NULL; + g_autofree char *metadata = NULL; g_autoptr(GKeyFile) metakey = g_key_file_new (); g_auto(GStrv) parts = NULL; g_autoptr(GPtrArray) related = g_ptr_array_new_with_free_func ((GDestroyNotify) flatpak_related_free); @@ -13458,9 +13521,9 @@ flatpak_dir_find_remote_related (FlatpakDir *self, if (*url == 0) return g_steal_pointer (&related); /* Empty url, silently disables updates */ - if (flatpak_remote_state_lookup_cache (state, ref, - NULL, NULL, &metadata, - NULL, NULL) && + if (flatpak_remote_state_load_data (state, ref, + NULL, NULL, &metadata, + NULL) && g_key_file_load_from_data (metakey, metadata, -1, 0, NULL)) { g_ptr_array_unref (related); diff --git a/common/flatpak-installation.c b/common/flatpak-installation.c index 5c660222..e91d2625 100644 --- a/common/flatpak-installation.c +++ b/common/flatpak-installation.c @@ -2369,9 +2369,9 @@ flatpak_installation_fetch_remote_size_sync (FlatpakInstallation *self, if (state == NULL) return FALSE; - return flatpak_remote_state_lookup_cache (state, full_ref, - download_size, installed_size, NULL, - NULL, error); + return flatpak_remote_state_load_data (state, full_ref, + download_size, installed_size, NULL, + error); } /** @@ -2400,7 +2400,8 @@ flatpak_installation_fetch_remote_metadata_sync (FlatpakInstallation *self, g_autoptr(FlatpakDir) dir = NULL; g_autoptr(FlatpakRemoteState) state = NULL; g_autofree char *full_ref = flatpak_ref_format_ref (ref); - const char *res = NULL; + g_autofree char *res = NULL; + gsize len; dir = flatpak_installation_get_dir (self, error); if (dir == NULL) @@ -2410,12 +2411,13 @@ flatpak_installation_fetch_remote_metadata_sync (FlatpakInstallation *self, if (state == NULL) return FALSE; - if (!flatpak_remote_state_lookup_cache (state, full_ref, - NULL, NULL, &res, - NULL, error)) + if (!flatpak_remote_state_load_data (state, full_ref, + NULL, NULL, &res, + error)) return NULL; - return g_bytes_new (res, strlen (res)); + len = strlen (res); + return g_bytes_new_take (g_steal_pointer (&res), len); } /** diff --git a/common/flatpak-remote-ref.c b/common/flatpak-remote-ref.c index 4fb44374..d0ec6850 100644 --- a/common/flatpak-remote-ref.c +++ b/common/flatpak-remote-ref.c @@ -327,7 +327,7 @@ flatpak_remote_ref_new (const char *full_ref, { FlatpakRefKind kind = FLATPAK_REF_KIND_APP; guint64 download_size = 0, installed_size = 0; - const char *metadata = NULL; + g_autofree char *metadata = NULL; g_autoptr(GBytes) metadata_bytes = NULL; g_auto(GStrv) parts = NULL; FlatpakRemoteRef *ref; @@ -340,15 +340,18 @@ flatpak_remote_ref_new (const char *full_ref, return NULL; if (state && - !flatpak_remote_state_lookup_cache (state, full_ref, - &download_size, &installed_size, &metadata, - NULL, NULL)) + !flatpak_remote_state_load_data (state, full_ref, + &download_size, &installed_size, &metadata, + NULL)) { g_debug ("Can't find metadata for ref %s", full_ref); } if (metadata) - metadata_bytes = g_bytes_new (metadata, strlen (metadata)); + { + metadata_bytes = g_bytes_new_take (metadata, strlen (metadata)); + metadata = NULL; /* steal */ + } if (state && flatpak_remote_state_lookup_sparse_cache (state, full_ref, &sparse_cache, NULL)) diff --git a/common/flatpak-transaction.c b/common/flatpak-transaction.c index cf2042a3..a8b9bd7c 100644 --- a/common/flatpak-transaction.c +++ b/common/flatpak-transaction.c @@ -2561,7 +2561,7 @@ try_resolve_op_from_metadata (FlatpakTransaction *self, /* And, we must have the actual cached data in the summary */ if (!flatpak_remote_state_lookup_cache (state, op->ref, - &download_size, &installed_size, &metadata, NULL, NULL)) + &download_size, &installed_size, &metadata, NULL)) return FALSE; metadata_bytes = g_bytes_new (metadata, strlen (metadata) + 1); From e5e498c34fea86cb569912ffdfe75dec0ed84800 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 24 Mar 2020 16:17:06 +0100 Subject: [PATCH 29/36] remote-ls/info: Add --sideloaded option This means we load the info from sideloaded repos only --- app/flatpak-builtins-remote-info.c | 4 +++- app/flatpak-builtins-remote-ls.c | 14 ++++++++------ app/flatpak-builtins-utils.c | 28 +++++++++++++++++++--------- app/flatpak-builtins-utils.h | 1 + 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/app/flatpak-builtins-remote-info.c b/app/flatpak-builtins-remote-info.c index d5eb81d7..04dba31a 100644 --- a/app/flatpak-builtins-remote-info.c +++ b/app/flatpak-builtins-remote-info.c @@ -47,6 +47,7 @@ static gboolean opt_log; static gboolean opt_show_runtime; static gboolean opt_show_sdk; static gboolean opt_cached; +static gboolean opt_sideloaded; static GOptionEntry options[] = { { "arch", 0, 0, G_OPTION_ARG_STRING, &opt_arch, N_("Arch to install for"), N_("ARCH") }, @@ -61,6 +62,7 @@ static GOptionEntry options[] = { { "show-runtime", 0, 0, G_OPTION_ARG_NONE, &opt_show_runtime, N_("Show runtime"), NULL }, { "show-sdk", 0, 0, G_OPTION_ARG_NONE, &opt_show_sdk, N_("Show sdk"), NULL }, { "cached", 0, 0, G_OPTION_ARG_NONE, &opt_cached, N_("Use local caches even if they are stale"), NULL }, + { "sideloaded", 0, 0, G_OPTION_ARG_NONE, &opt_sideloaded, N_("Only list refs available as sideloads"), NULL }, { NULL } }; @@ -140,7 +142,7 @@ flatpak_builtin_remote_info (int argc, char **argv, GCancellable *cancellable, G if (ref == NULL) return FALSE; - state = get_remote_state (preferred_dir, remote, opt_cached, cancellable, error); + state = get_remote_state (preferred_dir, remote, opt_cached, opt_sideloaded, cancellable, error); if (state == NULL) return FALSE; diff --git a/app/flatpak-builtins-remote-ls.c b/app/flatpak-builtins-remote-ls.c index ce64ddf9..b930caf1 100644 --- a/app/flatpak-builtins-remote-ls.c +++ b/app/flatpak-builtins-remote-ls.c @@ -41,6 +41,7 @@ static gboolean opt_app; static gboolean opt_all; static gboolean opt_only_updates; static gboolean opt_cached; +static gboolean opt_sideloaded; static char *opt_arch; static char *opt_app_runtime; static const char **opt_cols; @@ -55,6 +56,7 @@ static GOptionEntry options[] = { { "app-runtime", 0, 0, G_OPTION_ARG_STRING, &opt_app_runtime, N_("List all applications using RUNTIME"), N_("RUNTIME") }, { "columns", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_cols, N_("What information to show"), N_("FIELD,…") }, { "cached", 0, 0, G_OPTION_ARG_NONE, &opt_cached, N_("Use local caches even if they are stale"), NULL }, + { "sideloaded", 0, 0, G_OPTION_ARG_NONE, &opt_sideloaded, N_("Only list refs available as sideloads"), NULL }, { NULL } }; @@ -280,12 +282,12 @@ ls_remote (GHashTable *refs_hash, const char **arches, const char *app_runtime, if (need_cache_data) { - const char *metadata = NULL; + g_autofree char *metadata = NULL; g_autoptr(GKeyFile) metakey = NULL; - if (!flatpak_remote_state_lookup_cache (state, ref, - &download_size, &installed_size, &metadata, - NULL, error)) + if (!flatpak_remote_state_load_data (state, ref, + &download_size, &installed_size, &metadata, + error)) return FALSE; metakey = g_key_file_new (); @@ -439,7 +441,7 @@ flatpak_builtin_remote_ls (int argc, char **argv, GCancellable *cancellable, GEr return FALSE; } - state = get_remote_state (preferred_dir, argv[1], opt_cached, cancellable, error); + state = get_remote_state (preferred_dir, argv[1], opt_cached, opt_sideloaded, cancellable, error); if (state == NULL) return FALSE; @@ -474,7 +476,7 @@ flatpak_builtin_remote_ls (int argc, char **argv, GCancellable *cancellable, GEr if (flatpak_dir_get_remote_disabled (dir, remote_name)) continue; - state = get_remote_state (dir, remote_name, opt_cached, + state = get_remote_state (dir, remote_name, opt_cached, opt_sideloaded, cancellable, error); if (state == NULL) return FALSE; diff --git a/app/flatpak-builtins-utils.c b/app/flatpak-builtins-utils.c index 07ca765f..5e190800 100644 --- a/app/flatpak-builtins-utils.c +++ b/app/flatpak-builtins-utils.c @@ -1349,23 +1349,33 @@ FlatpakRemoteState * get_remote_state (FlatpakDir *dir, const char *remote, gboolean cached, + gboolean sideloaded, GCancellable *cancellable, GError **error) { g_autoptr(GError) local_error = NULL; - FlatpakRemoteState *state; + FlatpakRemoteState *state = NULL; - state = flatpak_dir_get_remote_state (dir, remote, cached, cancellable, &local_error); - if (state == NULL && g_error_matches (local_error, FLATPAK_ERROR, FLATPAK_ERROR_NOT_CACHED)) + if (sideloaded) { - g_clear_error (&local_error); - state = flatpak_dir_get_remote_state (dir, remote, FALSE, cancellable, &local_error); + state = flatpak_dir_get_remote_state_local_only (dir, remote, cancellable, error); + if (state == NULL) + return NULL; } - - if (state == NULL) + else { - g_propagate_error (error, g_steal_pointer (&local_error)); - return NULL; + state = flatpak_dir_get_remote_state (dir, remote, cached, cancellable, &local_error); + if (state == NULL && g_error_matches (local_error, FLATPAK_ERROR, FLATPAK_ERROR_NOT_CACHED)) + { + g_clear_error (&local_error); + state = flatpak_dir_get_remote_state (dir, remote, FALSE, cancellable, &local_error); + } + + if (state == NULL) + { + g_propagate_error (error, g_steal_pointer (&local_error)); + return NULL; + } } return state; diff --git a/app/flatpak-builtins-utils.h b/app/flatpak-builtins-utils.h index f909b6dc..2ef01680 100644 --- a/app/flatpak-builtins-utils.h +++ b/app/flatpak-builtins-utils.h @@ -176,6 +176,7 @@ void print_wrapped (int columns, FlatpakRemoteState * get_remote_state (FlatpakDir *dir, const char *remote, gboolean cached, + gboolean sideloaded, GCancellable *cancellable, GError **error); From 83795c210ca06a5448d36177c2fd93271e7931dc Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 24 Mar 2020 17:29:19 +0100 Subject: [PATCH 30/36] Remove unused variable --- common/flatpak-dir.c | 1 - 1 file changed, 1 deletion(-) diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index f5d7c787..7b3d2de2 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -627,7 +627,6 @@ flatpak_remote_state_load_data (FlatpakRemoteState *self, const char *xa_metadata = NULL; guint64 download_size = 0; guint64 installed_size = 0; - g_autoptr(GBytes) metadata_bytes = NULL; /* Use sideload refs if any */ From 56158a008766e9580b16590b96a3766b1009901e Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 24 Mar 2020 20:12:50 +0100 Subject: [PATCH 31/36] DeployData: Add commit timestamp to deploy data We want to use this to do quick checks if we need to do an update. --- common/flatpak-dir-private.h | 4 +++- common/flatpak-dir.c | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index a6563fc9..ba44849b 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -33,8 +33,9 @@ * * Version 1 added appdata-name/summary/version/license * Version 2 added extension-of/appdata-content-rating + * Version 3 added timestamp */ -#define FLATPAK_DEPLOY_VERSION_CURRENT 2 +#define FLATPAK_DEPLOY_VERSION_CURRENT 3 #define FLATPAK_DEPLOY_VERSION_ANY 0 #define FLATPAK_TYPE_DIR flatpak_dir_get_type () @@ -371,6 +372,7 @@ GHashTable * flatpak_deploy_data_get_appdata_content_rating (GBytes *depl const char ** flatpak_deploy_data_get_subpaths (GBytes *deploy_data); gboolean flatpak_deploy_data_has_subpaths (GBytes *deploy_data); guint64 flatpak_deploy_data_get_installed_size (GBytes *deploy_data); +guint64 flatpak_deploy_data_get_timestamp (GBytes *deploy_data); const char * flatpak_deploy_data_get_alt_id (GBytes *deploy_data); const char * flatpak_deploy_data_get_eol (GBytes *deploy_data); const char * flatpak_deploy_data_get_eol_rebase (GBytes *deploy_data); diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index 7b3d2de2..f91fa863 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -2528,6 +2528,16 @@ flatpak_deploy_data_get_version (GBytes *deploy_data) return var_metadata_lookup_int32 (metadata, "deploy-version", 0); } +/* Note: This will return 0 if this is unset, which happens on deloy data updates, so ensure we handle that in all callers */ +guint64 +flatpak_deploy_data_get_timestamp (GBytes *deploy_data) +{ + VarDeployDataRef ref = var_deploy_data_from_bytes (deploy_data); + VarMetadataRef metadata = var_deploy_data_get_metadata (ref); + + return var_metadata_lookup_uint64 (metadata, "timestamp", 0); +} + static const char * flatpak_deploy_data_get_string (GBytes *deploy_data, const char *key) { @@ -2822,6 +2832,7 @@ add_appdata_to_deploy_data (GVariantBuilder *metadata_builder, static GBytes * flatpak_dir_new_deploy_data (FlatpakDir *self, GFile *deploy_dir, + GVariant *commit_data, GVariant *commit_metadata, GKeyFile *metadata, const char *id, @@ -2854,6 +2865,8 @@ flatpak_dir_new_deploy_data (FlatpakDir *self, g_variant_builder_init (&metadata_builder, G_VARIANT_TYPE ("a{sv}")); g_variant_builder_add (&metadata_builder, "{s@v}", "deploy-version", g_variant_new_variant (g_variant_new_int32 (FLATPAK_DEPLOY_VERSION_CURRENT))); + g_variant_builder_add (&metadata_builder, "{s@v}", "timestamp", + g_variant_new_variant (g_variant_new_uint64 (ostree_commit_get_timestamp (commit_data)))); if (alt_id) g_variant_builder_add (&metadata_builder, "{s@v}", "alt-id", g_variant_new_variant (g_variant_new_string (alt_id))); @@ -2920,6 +2933,13 @@ upgrade_deploy_data (GBytes *deploy_data, GFile *deploy_dir, const char *ref) add_appdata_to_deploy_data (&metadata_builder, deploy_dir, ref_parts[1]); } + if (old_version < 3) + { + /* We don't know what timestamp to use here, use 0 and special case that for update checks */ + g_variant_builder_add (&metadata_builder, "{s@v}", "timestamp", + g_variant_new_variant (g_variant_new_uint64 (0))); + } + subpaths = flatpak_deploy_data_get_subpaths (deploy_data); res = g_variant_ref_sink (g_variant_new ("(ss^ast@a{sv})", flatpak_deploy_data_get_origin (deploy_data), @@ -7809,6 +7829,7 @@ flatpak_dir_deploy (FlatpakDir *self, deploy_data = flatpak_dir_new_deploy_data (self, checkoutdir, + commit_data, commit_metadata, keyfile, ref_parts[1], From 9648cc5eaac04b038db486efc865dce1b5286931 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 24 Mar 2020 20:14:29 +0100 Subject: [PATCH 32/36] tests: Revert the --merge option This is causing the tests to hang on errors sometimes. --- buildutil/glib-tap.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutil/glib-tap.mk b/buildutil/glib-tap.mk index 943a31c4..b05ad5f1 100644 --- a/buildutil/glib-tap.mk +++ b/buildutil/glib-tap.mk @@ -7,7 +7,7 @@ TESTS_ENVIRONMENT= \ G_DEBUG=gc-friendly \ MALLOC_CHECK_=2 \ MALLOC_PERTURB_=$$(($${RANDOM:-256} % 256)) -LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) $(top_srcdir)/buildutil/tap-driver.sh --merge +LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) $(top_srcdir)/buildutil/tap-driver.sh LOG_COMPILER = $(top_srcdir)/buildutil/tap-test TESTS = From 1117f0d872e538e094d841a5d4e5ae6cc4e70398 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 24 Mar 2020 20:16:50 +0100 Subject: [PATCH 33/36] lib: Add new FLATPAK_QUERY_FLAGS_ONLY_SIDELOADED option --- common/flatpak-installation.c | 26 ++++++++++++++++++++++---- common/flatpak-installation.h | 2 ++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/common/flatpak-installation.c b/common/flatpak-installation.c index e91d2625..b3c30238 100644 --- a/common/flatpak-installation.c +++ b/common/flatpak-installation.c @@ -2467,6 +2467,7 @@ flatpak_installation_list_remote_refs_sync_full (FlatpakInstallation *self, g_autoptr(GPtrArray) refs = g_ptr_array_new_with_free_func (g_object_unref); g_autoptr(FlatpakRemoteState) state = NULL; g_autoptr(GHashTable) ht = NULL; + g_autoptr(GError) local_error = NULL; GHashTableIter iter; gpointer key; gpointer value; @@ -2475,13 +2476,27 @@ flatpak_installation_list_remote_refs_sync_full (FlatpakInstallation *self, if (dir == NULL) return NULL; - state = flatpak_dir_get_remote_state (dir, remote_or_uri, (flags & FLATPAK_QUERY_FLAGS_ONLY_CACHED) != 0, cancellable, error); + if (flags & FLATPAK_QUERY_FLAGS_ONLY_SIDELOADED) + state = flatpak_dir_get_remote_state_local_only (dir, remote_or_uri, cancellable, error); + else + state = flatpak_dir_get_remote_state (dir, remote_or_uri, (flags & FLATPAK_QUERY_FLAGS_ONLY_CACHED) != 0, cancellable, error); if (state == NULL) return NULL; if (!flatpak_dir_list_remote_refs (dir, state, &ht, - cancellable, error)) - return NULL; + cancellable, &local_error)) + { + if (flags & FLATPAK_QUERY_FLAGS_ONLY_SIDELOADED) + { + /* Just return no sideloaded refs rather than a summary download failed error if there are none */ + return g_steal_pointer (&refs); + } + else + { + g_propagate_error (error, g_steal_pointer (&local_error)); + return NULL; + } + } g_hash_table_iter_init (&iter, ht); while (g_hash_table_iter_next (&iter, &key, &value)) @@ -2571,7 +2586,10 @@ flatpak_installation_fetch_remote_ref_sync_full (FlatpakInstallation *self, if (dir == NULL) return NULL; - state = flatpak_dir_get_remote_state_optional (dir, remote_name, (flags & FLATPAK_QUERY_FLAGS_ONLY_CACHED) != 0, cancellable, error); + if (flags & FLATPAK_QUERY_FLAGS_ONLY_SIDELOADED) + state = flatpak_dir_get_remote_state_local_only (dir, remote_name, cancellable, error); + else + state = flatpak_dir_get_remote_state (dir, remote_name, (flags & FLATPAK_QUERY_FLAGS_ONLY_CACHED) != 0, cancellable, error); if (state == NULL) return NULL; diff --git a/common/flatpak-installation.h b/common/flatpak-installation.h index 644434a7..13052fd9 100644 --- a/common/flatpak-installation.h +++ b/common/flatpak-installation.h @@ -127,6 +127,7 @@ typedef enum { * @FLATPAK_QUERY_FLAGS_ONLY_CACHED: Don't do any network i/o, but only return cached data. * This can return stale data, or a #FLATPAK_ERROR_NOT_CACHED error, however it is a * lot more efficient if you're doing many requests. + * @FLATPAK_QUERY_FLAGS_ONLY_SIDELOADED: Only list refs available from any eventuall sideload repos. (Snce: 1.7) * * Flags to alter the behavior of e.g flatpak_installation_list_remote_refs_sync_full(). * @@ -135,6 +136,7 @@ typedef enum { typedef enum { FLATPAK_QUERY_FLAGS_NONE = 0, FLATPAK_QUERY_FLAGS_ONLY_CACHED = (1 << 0), + FLATPAK_QUERY_FLAGS_ONLY_SIDELOADED = (1 << 1), } FlatpakQueryFlags; /** From 102c710b392b2e014577de47b7c91b88a8890582 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 25 Mar 2020 08:20:18 +0100 Subject: [PATCH 34/36] FlatpakInstallation: Mark non-transaction install/update ops deprecated These versions still work, but they don't do neearly as much as you'd want (depdendencies and whatnot), so document this. --- common/flatpak-installation.c | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/common/flatpak-installation.c b/common/flatpak-installation.c index b3c30238..0507305f 100644 --- a/common/flatpak-installation.c +++ b/common/flatpak-installation.c @@ -1285,6 +1285,10 @@ list_remotes_for_configured_remote (const gchar *remote_name, * * Lists only the remotes whose type is included in the @types argument. * + * Since flatpak 1.7 this will never return any types except FLATPAK_REMOTE_TYPE_STATIC. + * Equivalent functionallity to FLATPAK_REMOTE_TYPE_USB can be had by listing remote refs + * with FLATPAK_QUERY_FLAGS_ONLY_SIDELOADED. + * * Returns: (transfer container) (element-type FlatpakRemote): a GPtrArray of * #FlatpakRemote instances */ @@ -1816,6 +1820,10 @@ flatpak_installation_load_app_overrides (FlatpakInstallation *self, * @cancellable: (nullable): a #GCancellable * @error: return location for a #GError * + * This is and old deprecated function, you should use + * #FlatpakTransaction and flatpak_transaction_add_install_bundle() + * instead. It has a lot more interesting features. + * * Install an application or runtime from an flatpak bundle file. * See flatpak-build-bundle(1) for how to create bundles. * @@ -1874,6 +1882,10 @@ flatpak_installation_install_bundle (FlatpakInstallation *self, * @cancellable: (nullable): a #GCancellable * @error: return location for a #GError * + * This is and old deprecated function, you should use + * #FlatpakTransaction and flatpak_transaction_add_install_flatpakref() + * instead. It has a lot more interesting features. + * * Creates a remote based on the passed in .flatpakref file contents * in @ref_file_data and returns the #FlatpakRemoteRef that can be used * to install it. @@ -1932,6 +1944,9 @@ flatpak_installation_install_ref_file (FlatpakInstallation *self, * @cancellable: (nullable): a #GCancellable * @error: return location for a #GError * + * This is and old deprecated function, you should use + * #FlatpakTransaction and flatpak_transaction_add_install() + * instead. It has a lot more interesting features. * Install a new application or runtime. * * Note that this function was originally written to always return a @@ -2038,6 +2053,11 @@ flatpak_installation_install_full (FlatpakInstallation *self, * @cancellable: (nullable): a #GCancellable * @error: return location for a #GError * + * This is and old deprecated function, you should use + * #FlatpakTransaction and flatpak_transaction_add_install() + * instead. It has a lot more interesting features. + * Install a new application or runtime. + * * Install a new application or runtime. * * Note that this function was originally written to always return a @@ -2082,6 +2102,11 @@ flatpak_installation_install (FlatpakInstallation *self, * @cancellable: (nullable): a #GCancellable * @error: return location for a #GError * + * This is and old deprecated function, you should use + * #FlatpakTransaction and flatpak_transaction_add_update() + * instead. It has a lot more interesting features. + * Install a new application or runtime. + * * Update an application or runtime. * * If the specified package is not installed, then %FLATPAK_ERROR_NOT_INSTALLED @@ -2199,6 +2224,11 @@ flatpak_installation_update_full (FlatpakInstallation *self, * @cancellable: (nullable): a #GCancellable * @error: return location for a #GError * + * This is and old deprecated function, you should use + * #FlatpakTransaction and flatpak_transaction_add_update() + * instead. It has a lot more interesting features. + * Install a new application or runtime. + * * Update an application or runtime. * * If the specified package is not installed, then %FLATPAK_ERROR_NOT_INSTALLED @@ -2240,6 +2270,11 @@ flatpak_installation_update (FlatpakInstallation *self, * @cancellable: (nullable): a #GCancellable * @error: return location for a #GError * + * This is and old deprecated function, you should use + * #FlatpakTransaction and flatpak_transaction_add_unintstall() + * instead. It has a lot more interesting features. + * Install a new application or runtime. + * * Uninstall an application or runtime. * * Returns: %TRUE on success @@ -2276,6 +2311,11 @@ flatpak_installation_uninstall (FlatpakInstallation *self, * @cancellable: (nullable): a #GCancellable * @error: return location for a #GError * + * This is and old deprecated function, you should use + * #FlatpakTransaction and flatpak_transaction_add_unintstall() + * instead. It has a lot more interesting features. + * Install a new application or runtime. + * * Uninstall an application or runtime. * * Returns: %TRUE on success From c092fa4cb7c34cc8c72d19fcda7e48bbc3a32647 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 25 Mar 2020 08:27:18 +0100 Subject: [PATCH 35/36] installation: Reimplement flatpak_installation_list_installed_refs_for_update Instead of doing a lot of FlatpakInstallation calls we do lower level FlatpakDir calls, sharing a single RemoteState per remote for the entire operation. Also, some parts of the checks are moved to FlatpakDir as flatpak_dir_check_if_installed_ref_needs_update() --- common/flatpak-dir-private.h | 5 + common/flatpak-dir.c | 75 +++++++++++++ common/flatpak-installation.c | 200 +++++++++++++--------------------- 3 files changed, 155 insertions(+), 125 deletions(-) diff --git a/common/flatpak-dir-private.h b/common/flatpak-dir-private.h index ba44849b..1dc077e3 100644 --- a/common/flatpak-dir-private.h +++ b/common/flatpak-dir-private.h @@ -702,6 +702,11 @@ gboolean flatpak_dir_needs_update_for_commit_and_subpaths (FlatpakDir *self, const char *ref, const char *target_commit, const char **opt_subpaths); +gboolean flatpak_dir_check_if_installed_ref_needs_update (FlatpakDir *self, + FlatpakRemoteState *state, + const char *ref, + GBytes *deploy_data, + GCancellable *cancellable); char * flatpak_dir_check_for_update (FlatpakDir *self, FlatpakRemoteState *state, const char *ref, diff --git a/common/flatpak-dir.c b/common/flatpak-dir.c index f91fa863..6ffad270 100644 --- a/common/flatpak-dir.c +++ b/common/flatpak-dir.c @@ -8997,6 +8997,81 @@ flatpak_dir_needs_update_for_commit_and_subpaths (FlatpakDir *self, return FALSE; } +/* This returns true if the remote version of the installed version is + * newer than the installed, or if some related ref it relies on is + * missing (which will be fixed by an update run of FlatpakTransaction). */ +gboolean +flatpak_dir_check_if_installed_ref_needs_update (FlatpakDir *self, + FlatpakRemoteState *state, + const char *ref, + GBytes *deploy_data, + GCancellable *cancellable) +{ + g_autofree char *latest_commit = NULL; + guint64 latest_timestamp; + guint64 current_timestamp; + g_autoptr(GFile) deploy_dir = NULL; + const gchar *current_commit = NULL; + + if (flatpak_dir_ref_is_masked (self, ref)) + return FALSE; + + current_commit = flatpak_deploy_data_get_commit (deploy_data); + current_timestamp = flatpak_deploy_data_get_timestamp (deploy_data); + + if (!flatpak_remote_state_lookup_ref (state, ref, &latest_commit, &latest_timestamp, NULL, NULL, NULL)) + return FALSE; + + /* Check if the latest is newer than the current installed, if so update */ + if (current_timestamp == 0) + { + /* This happens during deploy data updates, fall back to commit comparisons */ + if (strcmp (current_commit, latest_commit) != 0) + return TRUE; + } + else + { + if (latest_timestamp > current_timestamp || + (latest_timestamp == current_timestamp && + strcmp (current_commit, latest_commit) != 0)) + return TRUE; + } + + /* The ref itself doesn't need update, but we do some extra checks + * for related refs that can trigger an update. */ + + /* Check if all "should-download" related refs for the ref are installed. + * If not, add the ref in @updates array so that it can be installed via + * FlatpakTransaction's update-op. + * + * This makes sure that the ref (maybe an app or runtime) remains in usable + * state and fixes itself through an update. + */ + if (flatpak_dir_check_installed_ref_missing_related_ref (self, state, ref, cancellable)) + return TRUE; + + /* This checks if an already installed app has a missing runtime. + * If so, return that installed ref in the updates list, so that FlatpakTransaction + * can resolve one of its operation to install the runtime instead. + * + * Runtime of an app can go missing if an app upgrade makes an app dependent on a new runtime + * entirely. We had couple of cases like that in the past, for example, before it was updated + * to use FlatpakTransaction, updating an app in GNOME Software to a version which needs a + * different runtime would not install that new runtime, leaving the app unusable. + */ + if (g_str_has_prefix (ref, "app/")) + { + const gchar *runtime = flatpak_deploy_data_get_runtime (deploy_data); + g_autofree gchar *full_runtime_ref = g_strconcat ("runtime/", runtime, NULL); + deploy_dir = flatpak_dir_get_if_deployed (self, full_runtime_ref, NULL, cancellable); + if (deploy_dir == NULL) + return TRUE; + } + + return FALSE; +} + +/* This is called by the old-school non-transaction flatpak_installation_update, so doesn't do a lot. */ char * flatpak_dir_check_for_update (FlatpakDir *self, FlatpakRemoteState *state, diff --git a/common/flatpak-installation.c b/common/flatpak-installation.c index 0507305f..7c7934e6 100644 --- a/common/flatpak-installation.c +++ b/common/flatpak-installation.c @@ -992,6 +992,29 @@ async_result_cb (GObject *obj, *result_out = g_object_ref (result); } + +static gboolean +ref_check_for_update (FlatpakDir *dir, + const char *ref, + GHashTable *remote_states, + GCancellable *cancellable) +{ + g_autoptr(GBytes) deploy_data = NULL; + FlatpakRemoteState *state; + const char *origin = NULL; + + deploy_data = flatpak_dir_get_deploy_data (dir, ref, FLATPAK_DEPLOY_VERSION_CURRENT, cancellable, NULL); + if (deploy_data == NULL) + return FALSE; + + origin = flatpak_deploy_data_get_origin (deploy_data); + state = g_hash_table_lookup (remote_states, origin); + if (state == NULL) + return FALSE; + + return flatpak_dir_check_if_installed_ref_needs_update (dir, state, ref, deploy_data, cancellable); +} + /** * flatpak_installation_list_installed_refs_for_update: * @self: a #FlatpakInstallation @@ -1016,146 +1039,73 @@ flatpak_installation_list_installed_refs_for_update (FlatpakInstallation *self, GCancellable *cancellable, GError **error) { - g_autoptr(GPtrArray) updates = NULL; /* (element-type FlatpakInstalledRef) */ - g_autoptr(GPtrArray) installed = NULL; /* (element-type FlatpakInstalledRef) */ - g_autoptr(GPtrArray) remotes = NULL; /* (element-type FlatpakRemote) */ - g_autoptr(GHashTable) remote_commits = NULL; /* (element-type utf8 utf8) */ - g_autoptr(GHashTable) remote_states = NULL; /* (element-type utf8 FlatpakRemoteState) */ - int i, j; + g_autoptr(FlatpakDir) dir_orig = flatpak_installation_get_dir_maybe_no_repo (self); g_autoptr(FlatpakDir) dir = NULL; + g_auto(GStrv) remote_names = NULL; + g_autoptr(GPtrArray) updates = NULL; /* (element-type FlatpakInstalledRef) */ + g_autoptr(GHashTable) remote_states = NULL; /* (element-type utf8 FlatpakRemoteState) */ + g_auto(GStrv) refs_app = NULL; + g_auto(GStrv) refs_runtime = NULL; - remote_commits = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); - - remotes = flatpak_installation_list_remotes (self, cancellable, error); - if (remotes == NULL) + /* We clone the dir here to make sure we re-read the latest ostree repo config, in case + it has local changes */ + dir = flatpak_dir_clone (dir_orig); + if (!flatpak_dir_maybe_ensure_repo (dir, cancellable, error)) return NULL; - for (i = 0; i < remotes->len; i++) - { - FlatpakRemote *remote = g_ptr_array_index (remotes, i); - g_autoptr(GPtrArray) refs = NULL; - g_autoptr(GError) local_error = NULL; - const char *remote_name = flatpak_remote_get_name (remote); + remote_names = flatpak_dir_list_remotes (dir, cancellable, error); + if (remote_names == NULL) + return NULL; - if (flatpak_remote_get_disabled (remote)) + remote_states = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, (GDestroyNotify)flatpak_remote_state_unref); + for (int i = 0; remote_names[i] != NULL; ++i) + { + const char *remote = remote_names[i]; + g_autoptr(FlatpakRemoteState) state = NULL; + g_autoptr(GError) local_error = NULL; + + if (flatpak_dir_get_remote_disabled (dir, remote)) continue; - /* We ignore errors here. we don't want one remote to fail us */ - refs = flatpak_installation_list_remote_refs_sync (self, - remote_name, - cancellable, &local_error); - if (refs != NULL) + state = flatpak_dir_get_remote_state_optional (dir, remote, FALSE, NULL, &local_error); + if (state == NULL) { - for (j = 0; j < refs->len; j++) - { - FlatpakRemoteRef *remote_ref = g_ptr_array_index (refs, j); - g_autofree char *full_ref = flatpak_ref_format_ref (FLATPAK_REF (remote_ref)); - g_autofree char *key = g_strdup_printf ("%s:%s", remote_name, full_ref); - - g_hash_table_insert (remote_commits, g_steal_pointer (&key), - g_strdup (flatpak_ref_get_commit (FLATPAK_REF (remote_ref)))); - } - } - else - { - g_debug ("Update: Failed to read remote %s: %s", - flatpak_remote_get_name (remote), - local_error->message); + g_debug ("Update: Failed to read remote %s: %s", remote, local_error->message); + continue; } + g_hash_table_insert (remote_states, (char *)remote, g_steal_pointer (&state)); } - installed = flatpak_installation_list_installed_refs (self, cancellable, error); - if (installed == NULL) - return NULL; - updates = g_ptr_array_new_with_free_func (g_object_unref); - dir = flatpak_installation_get_dir (self, error); - if (dir == NULL) - return NULL; - - remote_states = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify) flatpak_remote_state_unref); - - for (i = 0; i < installed->len; i++) + if (flatpak_dir_list_refs (dir, "app", &refs_app, cancellable, error)) { - FlatpakRemoteState *state; - FlatpakInstalledRef *installed_ref = g_ptr_array_index (installed, i); - const char *remote_name = flatpak_installed_ref_get_origin (installed_ref); - g_autofree char *full_ref = flatpak_ref_format_ref (FLATPAK_REF (installed_ref)); - g_autofree char *key = g_strdup_printf ("%s:%s", remote_name, full_ref); - const char *remote_commit = g_hash_table_lookup (remote_commits, key); - const char *local_commit = flatpak_installed_ref_get_latest_commit (installed_ref); - g_autoptr(GError) local_error = NULL; + for (int i = 0; refs_app[i] != NULL; i++) + { + const char *ref = refs_app[i]; + if (ref_check_for_update (dir, ref, remote_states, cancellable)) + { + g_printerr ("adding update %s\n", ref); + FlatpakInstalledRef *installed_ref = get_ref (dir, ref, cancellable, NULL); + if (installed_ref) + g_ptr_array_add (updates, g_object_ref (installed_ref)); + } + } + } - if (flatpak_dir_ref_is_masked (dir, full_ref)) - continue; - - /* Note: local_commit may be NULL here */ - if (remote_commit != NULL && - g_strcmp0 (remote_commit, local_commit) != 0) - { - g_ptr_array_add (updates, g_object_ref (installed_ref)); - - /* Don't check further, as we already added the installed_ref to @updates. */ - continue; - } - - /* Check if all "should-download" related refs for the ref are installed. - * If not, add the ref in @updates array so that it can be installed via - * FlatpakTransaction's update-op. - * - * This makes sure that the ref (maybe an app or runtime) remains in usable - * state and fixes itself through an update. - */ - state = g_hash_table_lookup (remote_states, remote_name); - if (state == NULL) - { - state = flatpak_dir_get_remote_state_optional (dir, remote_name, FALSE, cancellable, &local_error); - if (state == NULL) - { - g_debug ("Update: Failed to get remote state for %s: %s", - remote_name, local_error->message); - continue; - } - - g_hash_table_insert (remote_states, g_strdup (remote_name), state); - } - - if (flatpak_dir_check_installed_ref_missing_related_ref (dir, state, full_ref, cancellable)) - { - g_ptr_array_add (updates, g_object_ref (installed_ref)); - - /* Don't check for runtime, if we already added the installed_ref to @updates. */ - continue; - } - - if (flatpak_ref_get_kind (FLATPAK_REF (installed_ref)) == FLATPAK_REF_KIND_APP) - { - g_autoptr(GBytes) deploy_data = NULL; - - /* This checks if an already installed app has a missing runtime. - * If so, return that installed ref in the updates list, so that FlatpakTransaction - * can resolve one of its operation to install the runtime instead. - * - * Runtime of an app can go missing if an app upgrade makes an app dependent on a new runtime - * entirely. We had couple of cases like that in the past, for example, before it was updated - * to use FlatpakTransaction, updating an app in GNOME Software to a version which needs a - * different runtime would not install that new runtime, leaving the app unusable. - */ - deploy_data = flatpak_dir_get_deploy_data (dir, full_ref, FLATPAK_DEPLOY_VERSION_CURRENT, cancellable, NULL); - if (deploy_data != NULL) - { - g_autoptr(GFile) deploy_dir = NULL; - const gchar *runtime = NULL; - g_autofree gchar *full_runtime_ref = NULL; - - runtime = flatpak_deploy_data_get_runtime (deploy_data); - full_runtime_ref = g_strconcat ("runtime/", runtime, NULL); - deploy_dir = flatpak_dir_get_if_deployed (dir, full_runtime_ref, NULL, cancellable); - if (deploy_dir == NULL) - g_ptr_array_add (updates, g_object_ref (installed_ref)); - } - } + if (flatpak_dir_list_refs (dir, "runtime", &refs_runtime, cancellable, error)) + { + for (int i = 0; refs_runtime[i] != NULL; i++) + { + const char *ref = refs_runtime[i]; + if (ref_check_for_update (dir, ref, remote_states, cancellable)) + { + g_printerr ("adding update %s\n", ref); + FlatpakInstalledRef *installed_ref = get_ref (dir, ref, cancellable, NULL); + if (installed_ref) + g_ptr_array_add (updates, g_object_ref (installed_ref)); + } + } } return g_steal_pointer (&updates); From 59d34280b9da67c5f0230c31e667352d2d068257 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 25 Mar 2020 08:52:28 +0100 Subject: [PATCH 36/36] lib: Only support FLATPAK_REMOTE_TYPE_STATIC remote types With the new sideload support usb sideload works differently, they don't appear like separate remotes. Its just the normal remote and installs from it will automatically work. However, to list just the sideloadable refs we point the docs to the new ONLY_SIDELOADED flag. --- common/flatpak-installation.c | 161 +------------------------------- common/flatpak-remote-private.h | 4 - common/flatpak-remote.c | 34 ------- 3 files changed, 4 insertions(+), 195 deletions(-) diff --git a/common/flatpak-installation.c b/common/flatpak-installation.c index 7c7934e6..b080edb2 100644 --- a/common/flatpak-installation.c +++ b/common/flatpak-installation.c @@ -982,17 +982,6 @@ flatpak_installation_list_installed_refs_by_kind (FlatpakInstallation *self, return g_steal_pointer (&refs); } -static void -async_result_cb (GObject *obj, - GAsyncResult *result, - gpointer user_data) -{ - GAsyncResult **result_out = user_data; - - *result_out = g_object_ref (result); -} - - static gboolean ref_check_for_update (FlatpakDir *dir, const char *ref, @@ -1118,112 +1107,6 @@ flatpak_installation_list_installed_refs_for_update (FlatpakInstallation *self, * have a repo. * * FIXME: If this were async, the parallelisation could be handled in the caller. */ -static gboolean -list_remotes_for_configured_remote (const gchar *remote_name, - FlatpakDir *dir, - gboolean types_filter[], - GPtrArray *remotes /* (element-type FlatpakRemote) */, - GCancellable *cancellable, - GError **error) -{ - OstreeRepo *repo; - g_autofree gchar *collection_id = NULL; - OstreeCollectionRef ref; - OstreeCollectionRef ref2; - const OstreeCollectionRef *refs[3] = { NULL, }; - g_autofree gchar *appstream_ref = NULL; - g_autofree gchar *appstream2_ref = NULL; - g_auto(OstreeRepoFinderResultv) results = NULL; - g_autoptr(GAsyncResult) result = NULL; - g_autoptr(OstreeRepoFinder) finder_mount = NULL, finder_avahi = NULL; - OstreeRepoFinder *finders[3] = { NULL, }; - gsize i; - guint finder_index = 0; - g_autoptr(GMainContextPopDefault) context = NULL; - - if (!types_filter[FLATPAK_REMOTE_TYPE_USB] && - !types_filter[FLATPAK_REMOTE_TYPE_LAN]) - return TRUE; - - repo = flatpak_dir_get_repo (dir); - if (repo == NULL) - return TRUE; - - /* Find the collection ID for @remote_name, or bail if there is none. */ - if (!ostree_repo_get_remote_option (repo, - remote_name, "collection-id", - NULL, &collection_id, error)) - return FALSE; - if (collection_id == NULL || *collection_id == '\0') - return TRUE; - - context = flatpak_main_context_new_default (); - - appstream_ref = g_strdup_printf ("appstream/%s", flatpak_get_arch ()); - ref.collection_id = collection_id; - ref.ref_name = appstream_ref; - refs[0] = &ref; - appstream2_ref = g_strdup_printf ("appstream2/%s", flatpak_get_arch ()); - ref2.collection_id = collection_id; - ref2.ref_name = appstream2_ref; - refs[1] = &ref2; - - if (types_filter[FLATPAK_REMOTE_TYPE_USB]) - { - finder_mount = OSTREE_REPO_FINDER (ostree_repo_finder_mount_new (NULL)); - finders[finder_index++] = finder_mount; - } - - if (types_filter[FLATPAK_REMOTE_TYPE_LAN]) - { - g_autoptr(GError) local_error = NULL; - finder_avahi = OSTREE_REPO_FINDER (ostree_repo_finder_avahi_new (context)); - finders[finder_index++] = finder_avahi; - - /* The Avahi finder may fail to start on, for example, a CI server. */ - ostree_repo_finder_avahi_start (OSTREE_REPO_FINDER_AVAHI (finder_avahi), &local_error); - if (local_error != NULL) - { - if (finder_index == 1) - return TRUE; - else - { - finders[--finder_index] = NULL; - g_clear_object (&finder_avahi); - } - } - } - - ostree_repo_find_remotes_async (repo, - (const OstreeCollectionRef * const *) refs, - NULL, /* no options */ - finders, - NULL, /* no progress */ - cancellable, - async_result_cb, - &result); - - while (result == NULL) - g_main_context_iteration (context, TRUE); - - results = ostree_repo_find_remotes_finish (repo, result, error); - - if (finder_avahi != NULL) - ostree_repo_finder_avahi_stop (OSTREE_REPO_FINDER_AVAHI (finder_avahi)); - - if (results == NULL) - return FALSE; - - for (i = 0; results[i] != NULL; i++) - { - g_ptr_array_add (remotes, - flatpak_remote_new_from_ostree (results[i]->remote, - results[i]->finder, - dir)); - } - - return TRUE; -} /** * flatpak_installation_list_remotes_by_type: @@ -1256,8 +1139,6 @@ flatpak_installation_list_remotes_by_type (FlatpakInstallation *self, const guint NUM_FLATPAK_REMOTE_TYPES = 3; gboolean types_filter[NUM_FLATPAK_REMOTE_TYPES]; gsize i; - const char * const *default_repo_finders = NULL; - OstreeRepo *repo; remote_names = flatpak_dir_list_remotes (dir, cancellable, error); if (remote_names == NULL) @@ -1269,42 +1150,15 @@ flatpak_installation_list_remotes_by_type (FlatpakInstallation *self, if (!flatpak_dir_maybe_ensure_repo (dir_clone, cancellable, error)) return NULL; - repo = flatpak_dir_get_repo (dir_clone); - if (repo != NULL) - default_repo_finders = ostree_repo_get_default_repo_finders (repo); - - /* If NULL or an empty array of types is passed then we use the default set - * provided by ostree, or fall back to using all */ + /* If NULL or an empty array of types is passed then we list all types */ for (i = 0; i < NUM_FLATPAK_REMOTE_TYPES; ++i) { - if (num_types != 0) + if (types != NULL && num_types != 0) types_filter[i] = FALSE; - else if (default_repo_finders == NULL) + else types_filter[i] = TRUE; } - if (default_repo_finders != NULL && num_types == 0) - { - g_autofree char *default_repo_finders_str = g_strjoinv (" ", (gchar **) default_repo_finders); - g_debug ("Using default repo finder list: %s", default_repo_finders_str); - - for (const char * const *iter = default_repo_finders; iter && *iter; iter++) - { - const char *default_repo_finder = *iter; - - if (strcmp (default_repo_finder, "config") == 0) - types_filter[FLATPAK_REMOTE_TYPE_STATIC] = TRUE; - else if (strcmp (default_repo_finder, "lan") == 0) - types_filter[FLATPAK_REMOTE_TYPE_LAN] = TRUE; - else if (strcmp (default_repo_finder, "mount") == 0) - types_filter[FLATPAK_REMOTE_TYPE_USB] = TRUE; - else - g_debug ("Unknown value in list returned by " - "ostree_repo_get_default_repo_finders(): %s", - default_repo_finder); - } - } - for (i = 0; i < num_types; ++i) { g_return_val_if_fail (types[i] < NUM_FLATPAK_REMOTE_TYPES, NULL); @@ -1313,17 +1167,10 @@ flatpak_installation_list_remotes_by_type (FlatpakInstallation *self, for (i = 0; remote_names[i] != NULL; ++i) { - g_autoptr(GError) local_error = NULL; + /* These days we only support static remotes */ if (types_filter[FLATPAK_REMOTE_TYPE_STATIC]) g_ptr_array_add (remotes, flatpak_remote_new_with_dir (remote_names[i], dir_clone)); - - /* Add the dynamic mirrors of this remote. */ - if (!list_remotes_for_configured_remote (remote_names[i], dir_clone, - types_filter, remotes, - cancellable, &local_error)) - g_debug ("Couldn't find remotes for configured remote %s: %s", - remote_names[i], local_error->message); } return g_steal_pointer (&remotes); diff --git a/common/flatpak-remote-private.h b/common/flatpak-remote-private.h index 3f5f0418..fba6a027 100644 --- a/common/flatpak-remote-private.h +++ b/common/flatpak-remote-private.h @@ -32,10 +32,6 @@ FlatpakRemote *flatpak_remote_new_with_dir (const char *name, FlatpakDir *dir); -FlatpakRemote *flatpak_remote_new_from_ostree (OstreeRemote *remote, - OstreeRepoFinder *repo_finder, - FlatpakDir *dir); - gboolean flatpak_remote_commit (FlatpakRemote *self, FlatpakDir *dir, GCancellable *cancellable, diff --git a/common/flatpak-remote.c b/common/flatpak-remote.c index 7f404420..7ef4cf58 100644 --- a/common/flatpak-remote.c +++ b/common/flatpak-remote.c @@ -1037,40 +1037,6 @@ flatpak_remote_new_with_dir (const char *name, return self; } -static FlatpakRemoteType -repo_finder_to_remote_type (OstreeRepoFinder *repo_finder) -{ - if (OSTREE_IS_REPO_FINDER_AVAHI (repo_finder)) - return FLATPAK_REMOTE_TYPE_LAN; - else if (OSTREE_IS_REPO_FINDER_MOUNT (repo_finder)) - return FLATPAK_REMOTE_TYPE_USB; - else - return FLATPAK_REMOTE_TYPE_STATIC; -} - -FlatpakRemote * -flatpak_remote_new_from_ostree (OstreeRemote *remote, - OstreeRepoFinder *repo_finder, - FlatpakDir *dir) -{ - g_autofree gchar *url = NULL; - FlatpakRemotePrivate *priv; - FlatpakRemote *self = g_object_new (FLATPAK_TYPE_REMOTE, - "name", ostree_remote_get_name (remote), - "type", repo_finder_to_remote_type (repo_finder), - NULL); - - priv = flatpak_remote_get_instance_private (self); - if (dir) - priv->dir = g_object_ref (dir); - - url = ostree_remote_get_url (remote); - if (url != NULL) - flatpak_remote_set_url (self, url); - - return self; -} - /** * flatpak_remote_new: * @name: a name