diff --git a/Makefile.am b/Makefile.am
index cd3c8c8b..f2b608c2 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,7 +1,7 @@
 NULL =
 
 bin_PROGRAMS = $(NULL)
-dist_installed_test_scripts = $(NULL)
+dist_installed_test_extra_scripts = $(NULL)
 noinst_PROGRAMS = $(NULL)
 noinst_LTLIBRARIES = $(NULL)
 libexec_PROGRAMS = $(NULL)
diff --git a/common/flatpak-run.c b/common/flatpak-run.c
index 3c063713..d72f270a 100644
--- a/common/flatpak-run.c
+++ b/common/flatpak-run.c
@@ -2481,7 +2481,7 @@ add_document_portal_args (GPtrArray  *argv_array,
         {
           if (g_dbus_message_to_gerror (reply, &local_error))
             {
-              g_warning ("Can't get document portal: %s\n", local_error->message);
+              g_message ("Can't get document portal: %s\n", local_error->message);
             }
           else
             {
diff --git a/tests/Makefile.am.inc b/tests/Makefile.am.inc
index 1d4a1074..4bf16daf 100644
--- a/tests/Makefile.am.inc
+++ b/tests/Makefile.am.inc
@@ -59,7 +59,7 @@ tests/package_version.txt: Makefile
 
 tests/test-basic.sh: tests/package_version.txt
 
-dist_installed_test_scripts += \
+dist_installed_test_extra_scripts += \
 	buildutil/tap-driver.sh \
 	tests/test-configure \
 	tests/make-test-app.sh \
diff --git a/tests/libtest.sh b/tests/libtest.sh
index da7ab298..b314ef1c 100644
--- a/tests/libtest.sh
+++ b/tests/libtest.sh
@@ -70,11 +70,13 @@ fi
 # We need this to be in /var/tmp because /tmp has no xattr support
 TEST_DATA_DIR=`mktemp -d /var/tmp/test-flatpak-XXXXXX`
 mkdir -p ${TEST_DATA_DIR}/home
+mkdir -p ${TEST_DATA_DIR}/runtime
 mkdir -p ${TEST_DATA_DIR}/system
 export FLATPAK_SYSTEM_DIR=${TEST_DATA_DIR}/system
 export FLATPAK_SYSTEM_HELPER_ON_SESSION=1
 
 export XDG_DATA_HOME=${TEST_DATA_DIR}/home/share
+export XDG_RUNTIME_DIR=${TEST_DATA_DIR}/runtime
 
 export USERDIR=${TEST_DATA_DIR}/home/share/flatpak
 export SYSTEMDIR=${TEST_DATA_DIR}/system
@@ -220,4 +222,9 @@ skip_without_bwrap () {
 sed s#@testdir@#${test_builddir}# ${test_srcdir}/session.conf.in > session.conf
 eval `dbus-launch --config-file=session.conf --sh-syntax`
 
-trap "rm -rf $TEST_DATA_DIR; /bin/kill $DBUS_SESSION_BUS_PID" EXIT
+cleanup () {
+    /bin/kill $DBUS_SESSION_BUS_PID
+    fusermount -u $XDG_RUNTIME_DIR/doc || :
+    rm -rf $TEST_DATA_DIR
+}
+trap cleanup EXIT