run: Allow caller to replace /app and/or /usr

The pressure-vessel container tool in Steam will want to use this, to
replace /usr with a Steam Runtime container supplied by the Steam CDN,
instead of using the same Flatpak runtime that is used to run the Steam
client and non-containerized games.

If a custom /usr is used, the "official" Flatpak runtime is still the
one reflected in the metadata. It is also mounted at /run/parent,
with all its extensions, so that pressure-vessel has the option of using
its graphics drivers (by populating the custom /usr with symlinks into
/run/parent and/or /run/host).

When doing this, we need to put an empty directory on /app, because
the real /app expects to be run on top of the real runtime. It would
also be reasonable to substitute a custom replacement for /app, so
I've included support for that too.

Partially addresses #3797.

Signed-off-by: Simon McVittie <smcv@collabora.com>

app/flatpak-builtins-build.c

@@ -541,8 +541,8 @@ flatpak_builtin_build (int argc, char **argv, GCancellable *cancellable, GError
                             NULL);
 
   if (!flatpak_run_add_app_info_args (bwrap,
-                                      app_files, NULL, app_extensions,
+                                      app_files, app_files, NULL, app_extensions,
-                                      runtime_files, runtime_deploy_data, runtime_extensions,
+                                      runtime_files, runtime_files, runtime_deploy_data, runtime_extensions,
                                       id, NULL,
                                       runtime_ref,
                                       app_id_dir, app_context, NULL,

app/flatpak-builtins-run.c

@@ -59,6 +59,8 @@ static int opt_parent_pid;
 static gboolean opt_parent_expose_pids;
 static gboolean opt_parent_share_pids;
 static int opt_instance_id_fd = -1;
+static char *opt_app_path;
+static char *opt_usr_path;
 
 static GOptionEntry options[] = {
   { "arch", 0, 0, G_OPTION_ARG_STRING, &opt_arch, N_("Arch to use"), N_("ARCH") },
@@ -85,6 +87,8 @@ static GOptionEntry options[] = {
   { "parent-expose-pids", 0, 0, G_OPTION_ARG_NONE, &opt_parent_expose_pids, N_("Make processes visible in parent namespace"), NULL },
   { "parent-share-pids", 0, 0, G_OPTION_ARG_NONE, &opt_parent_share_pids, N_("Share process ID namespace with parent"), NULL },
   { "instance-id-fd", 0, 0, G_OPTION_ARG_INT, &opt_instance_id_fd, N_("Write the instance ID to the given file descriptor"), NULL },
+  { "app-path", 0, 0, G_OPTION_ARG_FILENAME, &opt_app_path, N_("Use PATH instead of the app's /app"), N_("PATH") },
+  { "usr-path", 0, 0, G_OPTION_ARG_FILENAME, &opt_usr_path, N_("Use PATH instead of the runtime's /usr"), N_("PATH") },
   { NULL }
 };
 
@@ -297,10 +301,12 @@ flatpak_builtin_run (int argc, char **argv, GCancellable *cancellable, GError **
 
   if (!flatpak_run_app (app_deploy ? app_ref : runtime_ref,
                         app_deploy,
+                        opt_app_path,
                         arg_context,
                         opt_runtime,
                         opt_runtime_version,
                         opt_runtime_commit,
+                        opt_usr_path,
                         opt_parent_pid,
                         flags,
                         opt_cwd,

common/flatpak-installation.c

@@ -700,8 +700,9 @@ flatpak_installation_launch_full (FlatpakInstallation *self,
 
   if (!flatpak_run_app (app_ref,
                         app_deploy,
+                        NULL,
                         NULL, NULL,
-                        NULL, NULL,
+                        NULL, NULL, NULL,
                         0,
                         run_flags,
                         NULL,

common/flatpak-run-private.h

@@ -54,12 +54,14 @@ gboolean flatpak_run_in_transient_unit (const char *app_id,
 #define FLATPAK_METADATA_GROUP_INSTANCE "Instance"
 #define FLATPAK_METADATA_KEY_INSTANCE_PATH "instance-path"
 #define FLATPAK_METADATA_KEY_INSTANCE_ID "instance-id"
+#define FLATPAK_METADATA_KEY_ORIGINAL_APP_PATH "original-app-path"
 #define FLATPAK_METADATA_KEY_APP_PATH "app-path"
 #define FLATPAK_METADATA_KEY_APP_COMMIT "app-commit"
 #define FLATPAK_METADATA_KEY_APP_EXTENSIONS "app-extensions"
 #define FLATPAK_METADATA_KEY_ARCH "arch"
 #define FLATPAK_METADATA_KEY_BRANCH "branch"
 #define FLATPAK_METADATA_KEY_FLATPAK_VERSION "flatpak-version"
+#define FLATPAK_METADATA_KEY_ORIGINAL_RUNTIME_PATH "original-runtime-path"
 #define FLATPAK_METADATA_KEY_RUNTIME_PATH "runtime-path"
 #define FLATPAK_METADATA_KEY_RUNTIME_COMMIT "runtime-commit"
 #define FLATPAK_METADATA_KEY_RUNTIME_EXTENSIONS "runtime-extensions"
@@ -155,9 +157,11 @@ gboolean flatpak_run_setup_base_argv (FlatpakBwrap   *bwrap,
                                       GError        **error);
 gboolean flatpak_run_add_app_info_args (FlatpakBwrap       *bwrap,
                                         GFile              *app_files,
+                                        GFile              *original_app_files,
                                         GBytes             *app_deploy_data,
                                         const char         *app_extensions,
                                         GFile              *runtime_files,
+                                        GFile              *original_runtime_files,
                                         GBytes             *runtime_deploy_data,
                                         const char         *runtime_extensions,
                                         const char         *app_id,
@@ -176,10 +180,12 @@ gboolean flatpak_run_add_app_info_args (FlatpakBwrap       *bwrap,
 
 gboolean flatpak_run_app (FlatpakDecomposed  *app_ref,
                           FlatpakDeploy      *app_deploy,
+                          const char         *custom_app_path,
                           FlatpakContext     *extra_context,
                           const char         *custom_runtime,
                           const char         *custom_runtime_version,
                           const char         *custom_runtime_commit,
+                          const char         *custom_usr_path,
                           int                 parent_pid,
                           FlatpakRunFlags     flags,
                           const char         *cwd,

common/flatpak-run.c

@@ -2272,9 +2272,11 @@ flatpak_run_add_dconf_args (FlatpakBwrap *bwrap,
 gboolean
 flatpak_run_add_app_info_args (FlatpakBwrap       *bwrap,
                                GFile              *app_files,
+                               GFile              *original_app_files,
                                GBytes             *app_deploy_data,
                                const char         *app_extensions,
                                GFile              *runtime_files,
+                               GFile              *original_runtime_files,
                                GBytes             *runtime_deploy_data,
                                const char         *runtime_extensions,
                                const char         *app_id,
@@ -2326,7 +2328,7 @@ flatpak_run_add_app_info_args (FlatpakBwrap       *bwrap,
 
   keyfile = g_key_file_new ();
 
-  if (app_files)
+  if (original_app_files)
     group = FLATPAK_METADATA_GROUP_APPLICATION;
   else
     group = FLATPAK_METADATA_GROUP_RUNTIME;
@@ -2350,6 +2352,14 @@ flatpak_run_add_app_info_args (FlatpakBwrap       *bwrap,
       g_key_file_set_string (keyfile, FLATPAK_METADATA_GROUP_INSTANCE,
                              FLATPAK_METADATA_KEY_APP_PATH, app_path);
     }
+
+  if (original_app_files != NULL && original_app_files != app_files)
+    {
+      g_autofree char *app_path = g_file_get_path (original_app_files);
+      g_key_file_set_string (keyfile, FLATPAK_METADATA_GROUP_INSTANCE,
+                             FLATPAK_METADATA_KEY_ORIGINAL_APP_PATH, app_path);
+    }
+
   if (app_deploy_data)
     g_key_file_set_string (keyfile, FLATPAK_METADATA_GROUP_INSTANCE,
                            FLATPAK_METADATA_KEY_APP_COMMIT, flatpak_deploy_data_get_commit (app_deploy_data));
@@ -2359,6 +2369,14 @@ flatpak_run_add_app_info_args (FlatpakBwrap       *bwrap,
   runtime_path = g_file_get_path (runtime_files);
   g_key_file_set_string (keyfile, FLATPAK_METADATA_GROUP_INSTANCE,
                          FLATPAK_METADATA_KEY_RUNTIME_PATH, runtime_path);
+
+  if (runtime_files != original_runtime_files)
+    {
+      g_autofree char *path = g_file_get_path (original_runtime_files);
+      g_key_file_set_string (keyfile, FLATPAK_METADATA_GROUP_INSTANCE,
+                             FLATPAK_METADATA_KEY_ORIGINAL_RUNTIME_PATH, path);
+    }
+
   if (runtime_deploy_data)
     g_key_file_set_string (keyfile, FLATPAK_METADATA_GROUP_INSTANCE,
                            FLATPAK_METADATA_KEY_RUNTIME_COMMIT, flatpak_deploy_data_get_commit (runtime_deploy_data));
@@ -3627,10 +3645,12 @@ check_sudo (GError **error)
 gboolean
 flatpak_run_app (FlatpakDecomposed *app_ref,
                  FlatpakDeploy     *app_deploy,
+                 const char        *custom_app_path,
                  FlatpakContext    *extra_context,
                  const char        *custom_runtime,
                  const char        *custom_runtime_version,
                  const char        *custom_runtime_commit,
+                 const char        *custom_usr_path,
                  int                parent_pid,
                  FlatpakRunFlags    flags,
                  const char        *cwd,
@@ -3646,7 +3666,9 @@ flatpak_run_app (FlatpakDecomposed *app_ref,
   g_autoptr(GBytes) runtime_deploy_data = NULL;
   g_autoptr(GBytes) app_deploy_data = NULL;
   g_autoptr(GFile) app_files = NULL;
+  g_autoptr(GFile) original_app_files = NULL;
   g_autoptr(GFile) runtime_files = NULL;
+  g_autoptr(GFile) original_runtime_files = NULL;
   g_autoptr(GFile) bin_ldconfig = NULL;
   g_autoptr(GFile) app_id_dir = NULL;
   g_autoptr(GFile) real_app_id_dir = NULL;
@@ -3682,6 +3704,7 @@ flatpak_run_app (FlatpakDecomposed *app_ref,
   gboolean sandboxed = (flags & FLATPAK_RUN_FLAG_SANDBOX) != 0;
   gboolean parent_expose_pids = (flags & FLATPAK_RUN_FLAG_PARENT_EXPOSE_PIDS) != 0;
   gboolean parent_share_pids = (flags & FLATPAK_RUN_FLAG_PARENT_SHARE_PIDS) != 0;
+  const char *app_target_path = "/app";
   const char *runtime_target_path = "/usr";
   struct stat s;
 
@@ -3794,11 +3817,31 @@ flatpak_run_app (FlatpakDecomposed *app_ref,
   if (extra_context)
     flatpak_context_merge (app_context, extra_context);
 
-  runtime_files = flatpak_deploy_get_files (runtime_deploy);
+  original_runtime_files = flatpak_deploy_get_files (runtime_deploy);
+
+  if (custom_usr_path != NULL)
+    {
+      runtime_files = g_file_new_for_path (custom_usr_path);
+      /* Mount the original runtime below here instead of /usr */
+      runtime_target_path = "/run/parent/usr";
+    }
+  else
+    {
+      runtime_files = g_object_ref (original_runtime_files);
+    }
+
   bin_ldconfig = g_file_resolve_relative_path (runtime_files, "bin/ldconfig");
   if (!g_file_query_exists (bin_ldconfig, NULL))
     use_ld_so_cache = FALSE;
 
+  /* We can't use the ld.so cache if we are using a custom /usr or /app,
+   * because we don't have a unique ID for the /usr or /app, so we can't
+   * do cache-invalidation correctly. The caller can either build their
+   * own ld.so.cache before supplying us with the runtime, or supply
+   * their own LD_LIBRARY_PATH. */
+  if (custom_usr_path != NULL || custom_app_path != NULL)
+    use_ld_so_cache = FALSE;
+
   if (app_deploy != NULL)
     {
       g_autofree const char **previous_ids = NULL;
@@ -3806,7 +3849,7 @@ flatpak_run_app (FlatpakDecomposed *app_ref,
       gboolean do_migrate;
 
       real_app_id_dir = flatpak_get_data_dir (app_id);
-      app_files = flatpak_deploy_get_files (app_deploy);
+      original_app_files = flatpak_deploy_get_files (app_deploy);
 
       previous_app_id_dirs = g_ptr_array_new_with_free_func (g_object_unref);
       previous_ids = flatpak_deploy_data_get_previous_ids (app_deploy_data, &len);
@@ -3887,6 +3930,21 @@ flatpak_run_app (FlatpakDecomposed *app_ref,
         app_id_dir = g_object_ref (real_app_id_dir);
     }
 
+  if (custom_app_path != NULL)
+    {
+      if (strcmp (custom_app_path, "") == 0)
+        app_files = NULL;
+      else
+        app_files = g_file_new_for_path (custom_app_path);
+
+      /* Mount the original app below here */
+      app_target_path = "/run/parent/app";
+    }
+  else if (original_app_files != NULL)
+    {
+      app_files = g_object_ref (original_app_files);
+    }
+
   flatpak_run_apply_env_default (bwrap, use_ld_so_cache);
   flatpak_run_apply_env_vars (bwrap, app_context);
   flatpak_run_apply_env_prompt (bwrap, app_id);
@@ -3899,22 +3957,95 @@ flatpak_run_app (FlatpakDecomposed *app_ref,
 
   flatpak_bwrap_add_args (bwrap,
                           "--ro-bind", flatpak_file_get_path_cached (runtime_files), "/usr",
-                          "--lock-file", "/usr/.ref",
                           NULL);
 
-  if (app_files != NULL)
+  if (runtime_files == original_runtime_files)
-    flatpak_bwrap_add_args (bwrap,
+    {
-                            "--ro-bind", flatpak_file_get_path_cached (app_files), "/app",
+      /* All true Flatpak runtimes have files/.ref */
-                            "--lock-file", "/app/.ref",
+      flatpak_bwrap_add_args (bwrap,
-                            NULL);
+                              "--lock-file", "/usr/.ref",
+                              NULL);
+    }
   else
-    flatpak_bwrap_add_args (bwrap,
+    {
-                            "--dir", "/app",
+      g_autoptr(GFile) runtime_child = NULL;
-                            NULL);
+
+      runtime_child = g_file_get_child (runtime_files, ".ref");
+
+      /* Lock ${usr}/.ref if it exists */
+      if (g_file_query_exists (runtime_child, NULL))
+        flatpak_bwrap_add_args (bwrap,
+                                "--lock-file", "/usr/.ref",
+                                NULL);
+
+      /* Put the real Flatpak runtime in /run/parent, so that the
+       * replacement /usr can have symlinks into /run/parent in order
+       * to use the Flatpak runtime's graphics drivers etc. if desired */
+      flatpak_bwrap_add_args (bwrap,
+                              "--ro-bind",
+                              flatpak_file_get_path_cached (original_runtime_files),
+                              "/run/parent/usr",
+                              "--lock-file", "/run/parent/usr/.ref",
+                              NULL);
+      flatpak_run_setup_usr_links (bwrap, original_runtime_files,
+                                   "/run/parent");
+
+      g_clear_object (&runtime_child);
+      runtime_child = g_file_get_child (original_runtime_files, "etc");
+
+      if (g_file_query_exists (runtime_child, NULL))
+        flatpak_bwrap_add_args (bwrap,
+                                "--symlink", "usr/etc", "/run/parent/etc",
+                                NULL);
+    }
+
+  if (app_files != NULL)
+    {
+      flatpak_bwrap_add_args (bwrap,
+                              "--ro-bind", flatpak_file_get_path_cached (app_files), "/app",
+                              NULL);
+
+      if (app_files == original_app_files)
+        {
+          /* All true Flatpak apps have files/.ref */
+          flatpak_bwrap_add_args (bwrap,
+                                  "--lock-file", "/app/.ref",
+                                  NULL);
+        }
+      else
+        {
+          g_autoptr(GFile) app_child = NULL;
+
+          app_child = g_file_get_child (app_files, ".ref");
+
+          /* Lock ${app}/.ref if it exists */
+          if (g_file_query_exists (app_child, NULL))
+            flatpak_bwrap_add_args (bwrap,
+                                    "--lock-file", "/app/.ref",
+                                    NULL);
+        }
+    }
+  else
+    {
+      flatpak_bwrap_add_args (bwrap,
+                              "--dir", "/app",
+                              NULL);
+    }
+
+  if (original_app_files != NULL && app_files != original_app_files)
+    {
+      /* Put the real Flatpak app in /run/parent/app */
+      flatpak_bwrap_add_args (bwrap,
+                              "--ro-bind",
+                              flatpak_file_get_path_cached (original_app_files),
+                              "/run/parent/app",
+                              "--lock-file", "/run/parent/app/.ref",
+                              NULL);
+    }
 
   if (metakey != NULL &&
       !flatpak_run_add_extension_args (bwrap, metakey, app_ref,
-                                       use_ld_so_cache, "/app",
+                                       use_ld_so_cache, app_target_path,
                                        &app_extensions, &app_ld_path,
                                        cancellable, error))
     return FALSE;
@@ -3925,7 +4056,11 @@ flatpak_run_app (FlatpakDecomposed *app_ref,
                                        cancellable, error))
     return FALSE;
 
-  flatpak_run_extend_ld_path (bwrap, app_ld_path, runtime_ld_path);
+  if (custom_usr_path == NULL)
+    flatpak_run_extend_ld_path (bwrap, NULL, runtime_ld_path);
+
+  if (custom_app_path == NULL)
+    flatpak_run_extend_ld_path (bwrap, app_ld_path, NULL);
 
   runtime_ld_so_conf = g_file_resolve_relative_path (runtime_files, "etc/ld.so.conf");
   if (lstat (flatpak_file_get_path_cached (runtime_ld_so_conf), &s) == 0)
@@ -3969,8 +4104,8 @@ flatpak_run_app (FlatpakDecomposed *app_ref,
     }
 
   if (!flatpak_run_add_app_info_args (bwrap,
-                                      app_files, app_deploy_data, app_extensions,
+                                      app_files, original_app_files, app_deploy_data, app_extensions,
-                                      runtime_files, runtime_deploy_data, runtime_extensions,
+                                      runtime_files, original_runtime_files, runtime_deploy_data, runtime_extensions,
                                       app_id, flatpak_decomposed_get_branch (app_ref),
                                       runtime_ref, app_id_dir, app_context, extra_context,
                                       sandboxed, FALSE, flags & FLATPAK_RUN_FLAG_DEVEL,

data/org.freedesktop.portal.Flatpak.xml

@@ -36,7 +36,7 @@
       bus name org.freedesktop.portal.Flatpak and the object path
       /org/freedesktop/portal/Flatpak.
 
-      This documentation describes version 5 of this interface.
+      This documentation describes version 6 of this interface.
   -->
   <interface name='org.freedesktop.portal.Flatpak'>
     <property name="version" type="u" access="read"/>
@@ -133,6 +133,23 @@
                This was added in version 5 of this interface (available from flatpak 1.10.0 and later).
              </para></listitem>
            </varlistentry>
+           <varlistentry>
+             <term>256 (FLATPAK_SPAWN_FLAGS_EMPTY_APP)</term>
+             <listitem><para>
+               Don't provide app files at <filename>/app</filename> in the
+               new sandbox. Instead, <filename>/app</filename> will be an
+               empty directory. This flag and the <option>app-path</option>
+               option are mutually exclusive.
+             </para><para>
+               As with the <option>app-path</option> option, the caller's
+               Flatpak app files and extensions will be mounted on
+               <filename>/run/parent/app</filename>, with
+               filenames like <filename>/run/parent/app/bin/myapp</filename>.
+             </para><para>
+               This was added in version 6 of this interface (available from
+               flatpak 1.12.0 and later).
+             </para></listitem>
+           </varlistentry>
          </variablelist>
 
         Unknown (unsupported) flags are an error and will cause Spawn()
@@ -231,6 +248,55 @@
              This was added in version 5 of this interface (available from flatpak 1.10.0 and later).
              </para></listitem>
            </varlistentry>
+           <varlistentry>
+             <term>usr-fd h</term>
+             <listitem><para>
+               A file descriptor for the directory that will be used as
+               <filename>/usr</filename> in the new sandbox, instead of the
+               <filename>files</filename> directory
+               from the caller's Flatpak runtime.
+               The new sandbox's <filename>/etc</filename> will be based
+               on the <filename>etc</filename> subdirectory of the given
+               directory, and compatibility symlinks in its
+               root directory (<filename>/lib</filename>,
+               <filename>/bin</filename> and so on) will point into the
+               given directory. The caller's Flatpak runtime and its
+               extensions will be mounted on
+               <filename>/run/parent/usr</filename>, with filenames like
+               <filename>/run/parent/usr/bin/env</filename>,
+               and compatibility symlinks like
+               <filename>/run/parent/bin</filename> →
+               <filename>usr/bin</filename>.
+             </para><para>
+               The file descriptor must be opened with O_PATH and
+               O_NOFOLLOW and cannot be a symlink.
+             </para><para>
+               This was added in version 6 of this interface (available from
+               flatpak 1.12.0 and later).
+             </para></listitem>
+           </varlistentry>
+           <varlistentry>
+             <term>app-fd h</term>
+             <listitem><para>
+               A file descriptor for the directory that will be used as
+               <filename>/app</filename> in the new sandbox, instead of the
+               <filename>files</filename> directory
+               from the caller's Flatpak app. The caller's Flatpak app
+               files and extensions will be
+               mounted on <filename>/run/parent/app</filename>, with
+               filenames like <filename>/run/parent/app/bin/myapp</filename>.
+             </para><para>
+               This option and the
+               <option>FLATPAK_SPAWN_FLAGS_EMPTY_APP</option>
+               flag are mutually exclusive.
+             </para><para>
+               The file descriptor must be opened with O_PATH and
+               O_NOFOLLOW and cannot be a symlink.
+             </para><para>
+               This was added in version 6 of this interface (available from
+               flatpak 1.12.0 and later).
+             </para></listitem>
+           </varlistentry>
          </variablelist>
 
     -->

doc/flatpak-metadata.xml

@@ -530,11 +530,42 @@
                         app files, as mounted at <filename>/app</filename>
                         inside the container. Available since 0.6.10.
                     </para></listitem>
+                    <listitem><para>
+                        Since 1.12.0, if <command>flatpak run</command>
+                        was run with the <option>--app-path</option> option,
+                        this key gives the absolute path of whatever files
+                        were mounted on <filename>/app</filename>, even if
+                        that differs from the app's normal app files.
+                    </para></listitem>
+                    <listitem><para>
+                        If <command>flatpak run</command> was run with
+                        <option>--app-path=</option> (resulting in an
+                        empty directory being mounted on
+                        <filename>/app</filename>), the value is set to
+                        the empty string.
+                    </para></listitem>
+                </varlistentry>
+                <varlistentry>
+                    <term><option>original-app-path</option> (string)</term>
+                    <listitem><para>
+                        If <command>flatpak run</command> was run with the
+                        <option>--app-path</option> option, this key gives
+                        the absolute path of the app's original files,
+                        as mounted at <filename>/run/parent/app</filename>
+                        inside the container. Available since 1.12.0.
+                    </para></listitem>
+                    <listitem><para>
+                        If this key is missing, the app files are given
+                        by <option>app-path</option>.
+                    </para></listitem>
                 </varlistentry>
                 <varlistentry>
                     <term><option>app-commit</option> (string)</term>
                     <listitem><para>
                         The commit ID of the application that is running.
+                        The filename of a deployment of this commit can
+                        be found in <option>original-app-path</option>
+                        if present, or <option>app-path</option> otherwise.
                     </para></listitem>
                 </varlistentry>
                 <varlistentry>
@@ -543,6 +574,10 @@
                         A list of app extensions that are mounted into
                         the running instance. The format for each list item is
                         <option>EXTENSION_ID=COMMIT</option>.
+                        If <option>original-app-path</option> is present,
+                        the extensions are mounted below
+                        <filename>/run/parent/app</filename>; otherwise,
+                        they are mounted below <filename>/app</filename>.
                     </para></listitem>
                 </varlistentry>
                 <varlistentry>
@@ -573,11 +608,36 @@
                         runtime files, as mounted at <filename>/usr</filename>
                         inside the container. Available since 0.6.10.
                     </para></listitem>
+                    <listitem><para>
+                        Since 1.12.0, if <command>flatpak run</command>
+                        was run with the <option>--usr-path</option> option,
+                        this key gives the absolute path of whatever files
+                        were mounted on <filename>/usr</filename>, even if
+                        that differs from the app's normal runtime files.
+                    </para></listitem>
+                </varlistentry>
+                <varlistentry>
+                    <term><option>original-runtime-path</option> (string)</term>
+                    <listitem><para>
+                        If <command>flatpak run</command> was run with the
+                        <option>--runtime-path</option> option, this key gives
+                        the absolute path of the app's original runtime,
+                        as mounted at <filename>/run/parent/usr</filename>
+                        inside the container. Available since 1.12.0.
+                    </para></listitem>
+                    <listitem><para>
+                        If this key is missing, the runtime files are given
+                        by <option>runtime-path</option>.
+                    </para></listitem>
                 </varlistentry>
                 <varlistentry>
                     <term><option>runtime-commit</option> (string)</term>
                     <listitem><para>
                         The commit ID of the runtime that is used.
+                        The filename of a deployment of this commit can be
+                        found in <option>original-runtime-path</option>
+                        if present, or <option>runtime-path</option>
+                        otherwise.
                     </para></listitem>
                 </varlistentry>
                 <varlistentry>
@@ -586,6 +646,10 @@
                         A list of runtime extensions that are mounted into
                         the running instance. The format for each list item is
                         <option>EXTENSION_ID=COMMIT</option>.
+                        If <option>original-app-path</option> is present,
+                        the extensions are mounted below
+                        <filename>/run/parent/usr</filename>; otherwise,
+                        they are mounted below <filename>/usr</filename>.
                     </para></listitem>
                 </varlistentry>
                 <varlistentry>

doc/flatpak-run.xml

@@ -636,7 +636,54 @@ key=v1;v2;
                     permissions for the application.
                 </para></listitem>
             </varlistentry>
+
+            <varlistentry>
+                <term><option>--app-path=<replaceable>PATH</replaceable></option></term>
+                <listitem><para>
+                    Instead of mounting the app's content on
+                    <filename>/app</filename> in the sandbox, mount
+                    <replaceable>PATH</replaceable> on <filename>/app</filename>,
+                    and the app's content on
+                    <filename>/run/parent/app</filename>.
+                    If the app has extensions, they will also be redirected
+                    into <filename>/run/parent/app</filename>, and will not
+                    be included in the <envar>LD_LIBRARY_PATH</envar> inside
+                    the sandbox.
+                </para></listitem>
+            </varlistentry>
+            <varlistentry>
+                <term><option>--app-path=</option></term>
+                <listitem><para>
+                    As a special case, <option>--app-path=</option>
+                    (with an empty <replaceable>PATH</replaceable>)
+                    results in an empty directory being mounted on
+                    <filename>/app</filename>.
+                </para></listitem>
+            </varlistentry>
+
+            <varlistentry>
+                <term><option>--usr-path=<replaceable>PATH</replaceable></option></term>
+                <listitem><para>
+                    Instead of mounting the runtime's files on
+                    <filename>/usr</filename> in the sandbox, mount
+                    <replaceable>PATH</replaceable> on
+                    <filename>/usr</filename>,
+                    and the runtime's normal files on
+                    <filename>/run/parent/usr</filename>.
+                    If the runtime has extensions, they will also be redirected
+                    into <filename>/run/parent/usr</filename>, and will not
+                    be included in the <envar>LD_LIBRARY_PATH</envar> inside
+                    the sandbox.
+                </para></listitem>
+                <listitem><para>
+                    This option will usually only be useful if it is
+                    combined with <option>--app-path=</option> and
+                    <option>--env=LD_LIBRARY_PATH=<replaceable>...</replaceable></option>.
+                </para></listitem>
+            </varlistentry>
+
         </variablelist>
+
     </refsect1>
 
     <refsect1>

portal/flatpak-portal.c

@@ -735,7 +735,9 @@ get_path_for_fd (int fd,
   if (access (proc_path, W_OK) == 0)
     writable = TRUE;
 
-  *writable_out = writable;
+  if (writable_out != NULL)
+    *writable_out = writable;
+
   return g_steal_pointer (&path);
 }
 
@@ -780,6 +782,8 @@ handle_spawn (PortalFlatpak         *object,
   g_auto(GStrv) sandbox_expose_ro = NULL;
   g_autoptr(GVariant) sandbox_expose_fd = NULL;
   g_autoptr(GVariant) sandbox_expose_fd_ro = NULL;
+  g_autoptr(GVariant) app_fd = NULL;
+  g_autoptr(GVariant) usr_fd = NULL;
   g_autoptr(GOutputStream) instance_id_out_stream = NULL;
   guint sandbox_flags = 0;
   gboolean sandboxed;
@@ -787,6 +791,7 @@ handle_spawn (PortalFlatpak         *object,
   gboolean share_pids;
   gboolean notify_start;
   gboolean devel;
+  gboolean empty_app;
   g_autoptr(GString) env_string = g_string_new ("");
 
   child_setup_data.instance_id_fd = -1;
@@ -876,6 +881,8 @@ handle_spawn (PortalFlatpak         *object,
   sandbox_expose_fd = g_variant_lookup_value (arg_options, "sandbox-expose-fd", G_VARIANT_TYPE ("ah"));
   sandbox_expose_fd_ro = g_variant_lookup_value (arg_options, "sandbox-expose-fd-ro", G_VARIANT_TYPE ("ah"));
   g_variant_lookup (arg_options, "unset-env", "^as", &unset_env);
+  app_fd = g_variant_lookup_value (arg_options, "app-fd", G_VARIANT_TYPE_HANDLE);
+  usr_fd = g_variant_lookup_value (arg_options, "usr-fd", G_VARIANT_TYPE_HANDLE);
 
   if ((sandbox_flags & ~FLATPAK_SPAWN_SANDBOX_FLAGS_ALL) != 0)
     {
@@ -1324,6 +1331,76 @@ handle_spawn (PortalFlatpak         *object,
         }
     }
 
+  empty_app = (arg_flags & FLATPAK_SPAWN_FLAGS_EMPTY_APP) != 0;
+
+  if (app_fd != NULL)
+    {
+      gint32 handle = g_variant_get_handle (app_fd);
+      g_autofree char *path = NULL;
+
+      if (empty_app)
+        {
+          g_dbus_method_invocation_return_error (invocation, G_DBUS_ERROR,
+                                                 G_DBUS_ERROR_INVALID_ARGS,
+                                                 "app-fd and EMPTY_APP cannot both be used");
+          return G_DBUS_METHOD_INVOCATION_HANDLED;
+        }
+
+      if (handle >= fds_len)
+        {
+          g_dbus_method_invocation_return_error (invocation, G_DBUS_ERROR,
+                                                 G_DBUS_ERROR_INVALID_ARGS,
+                                                 "No file descriptor for handle %d",
+                                                 handle);
+          return G_DBUS_METHOD_INVOCATION_HANDLED;
+        }
+
+      path = get_path_for_fd (fds[handle], NULL, &error);
+
+      if (path == NULL)
+        {
+          g_prefix_error (&error, "Unable to convert /app fd %d into path: ",
+                          fds[handle]);
+          g_dbus_method_invocation_return_gerror (invocation, error);
+          return G_DBUS_METHOD_INVOCATION_HANDLED;
+        }
+
+      g_debug ("Using %s as /app instead of app", path);
+      g_ptr_array_add (flatpak_argv, g_strdup_printf ("--app-path=%s", path));
+    }
+  else if (empty_app)
+    {
+      g_ptr_array_add (flatpak_argv, g_strdup ("--app-path="));
+    }
+
+  if (usr_fd != NULL)
+    {
+      gint32 handle = g_variant_get_handle (usr_fd);
+      g_autofree char *path = NULL;
+
+      if (handle >= fds_len)
+        {
+          g_dbus_method_invocation_return_error (invocation, G_DBUS_ERROR,
+                                                 G_DBUS_ERROR_INVALID_ARGS,
+                                                 "No file descriptor for handle %d",
+                                                 handle);
+          return G_DBUS_METHOD_INVOCATION_HANDLED;
+        }
+
+      path = get_path_for_fd (fds[handle], NULL, &error);
+
+      if (path == NULL)
+        {
+          g_prefix_error (&error, "Unable to convert /usr fd %d into path: ",
+                          fds[handle]);
+          g_dbus_method_invocation_return_gerror (invocation, error);
+          return G_DBUS_METHOD_INVOCATION_HANDLED;
+        }
+
+      g_debug ("Using %s as /usr instead of runtime", path);
+      g_ptr_array_add (flatpak_argv, g_strdup_printf ("--usr-path=%s", path));
+    }
+
   g_ptr_array_add (flatpak_argv, g_strdup_printf ("--runtime=%s", runtime_parts[1]));
   g_ptr_array_add (flatpak_argv, g_strdup_printf ("--runtime-version=%s", runtime_parts[3]));
 
@@ -2791,7 +2868,7 @@ on_bus_acquired (GDBusConnection *connection,
   g_dbus_interface_skeleton_set_flags (G_DBUS_INTERFACE_SKELETON (portal),
                                        G_DBUS_INTERFACE_SKELETON_FLAGS_HANDLE_METHOD_INVOCATIONS_IN_THREAD);
 
-  portal_flatpak_set_version (PORTAL_FLATPAK (portal), 5);
+  portal_flatpak_set_version (PORTAL_FLATPAK (portal), 6);
   portal_flatpak_set_supports (PORTAL_FLATPAK (portal), supports);
 
   g_signal_connect (portal, "handle-spawn", G_CALLBACK (handle_spawn), NULL);

portal/flatpak-portal.h

@@ -30,6 +30,7 @@ typedef enum {
   FLATPAK_SPAWN_FLAGS_EXPOSE_PIDS = 1 << 5,
   FLATPAK_SPAWN_FLAGS_NOTIFY_START = 1 << 6,
   FLATPAK_SPAWN_FLAGS_SHARE_PIDS = 1 << 7,
+  FLATPAK_SPAWN_FLAGS_EMPTY_APP = 1 << 8,
 } FlatpakSpawnFlags;
 
 typedef enum {
@@ -56,7 +57,8 @@ typedef enum {
                                  FLATPAK_SPAWN_FLAGS_WATCH_BUS | \
                                  FLATPAK_SPAWN_FLAGS_EXPOSE_PIDS | \
                                  FLATPAK_SPAWN_FLAGS_NOTIFY_START | \
-                                 FLATPAK_SPAWN_FLAGS_SHARE_PIDS)
+                                 FLATPAK_SPAWN_FLAGS_SHARE_PIDS | \
+                                 FLATPAK_SPAWN_FLAGS_EMPTY_APP)
 
 #define FLATPAK_SPAWN_SANDBOX_FLAGS_ALL (FLATPAK_SPAWN_SANDBOX_FLAGS_SHARE_DISPLAY | \
                                          FLATPAK_SPAWN_SANDBOX_FLAGS_SHARE_SOUND | \