From 63bf454d6686b68653e926b01e8b8ef908da2bfc Mon Sep 17 00:00:00 2001 From: Matt Heon Date: Sat, 14 Jun 2025 07:22:51 -0400 Subject: [PATCH] Refactor `podman export` to work with the remote client Previously, our approach was to inspect the volume, grab its mountpoint, and tar that up, all in the CLI code. There's no reason why that has to be in the CLI - if we move it into Libpod, and add a REST endpoint to stream the tar, we can enable it for the remote client as well. As a bonus, previously, we could not properly handle volumes that needed to be mounted. Now, we can mount the volume if necessary, and as such export works with more types of volumes, including volume drivers. Signed-off-by: Matt Heon --- cmd/podman/volumes/export.go | 54 +++---------------------- libpod/volume.go | 29 +++++++++++++ pkg/api/handlers/libpod/volumes.go | 19 +++++++++ pkg/api/server/register_volumes.go | 25 ++++++++++++ pkg/bindings/volumes/volumes.go | 35 ++++++++++++++++ pkg/domain/entities/engine_container.go | 1 + pkg/domain/entities/volumes.go | 9 +++-- pkg/domain/infra/abi/volumes.go | 31 ++++++++++++++ pkg/domain/infra/tunnel/volumes.go | 4 ++ test/e2e/volume_create_test.go | 4 -- test/system/160-volumes.bats | 2 - 11 files changed, 155 insertions(+), 58 deletions(-) diff --git a/cmd/podman/volumes/export.go b/cmd/podman/volumes/export.go index 6d00af6e92..24d5d3eb4d 100644 --- a/cmd/podman/volumes/export.go +++ b/cmd/podman/volumes/export.go @@ -3,15 +3,11 @@ package volumes import ( "context" "errors" - "fmt" "github.com/containers/common/pkg/completion" "github.com/containers/podman/v5/cmd/podman/common" "github.com/containers/podman/v5/cmd/podman/registry" "github.com/containers/podman/v5/pkg/domain/entities" - "github.com/containers/podman/v5/pkg/errorhandling" - "github.com/containers/podman/v5/utils" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -21,7 +17,6 @@ podman volume export Allow content of volume to be exported into external tar.` exportCommand = &cobra.Command{ - Annotations: map[string]string{registry.EngineMode: registry.ABIMode}, Use: "export [options] VOLUME", Short: "Export volumes", Args: cobra.ExactArgs(1), @@ -32,10 +27,7 @@ Allow content of volume to be exported into external tar.` ) var ( - // Temporary struct to hold cli values. - cliExportOpts = struct { - Output string - }{} + cliExportOpts entities.VolumeExportOptions ) func init() { @@ -46,54 +38,20 @@ func init() { flags := exportCommand.Flags() outputFlagName := "output" - flags.StringVarP(&cliExportOpts.Output, outputFlagName, "o", "/dev/stdout", "Write to a specified file (default: stdout, which must be redirected)") + flags.StringVarP(&cliExportOpts.OutputPath, outputFlagName, "o", "/dev/stdout", "Write to a specified file (default: stdout, which must be redirected)") _ = exportCommand.RegisterFlagCompletionFunc(outputFlagName, completion.AutocompleteDefault) } func export(cmd *cobra.Command, args []string) error { - var inspectOpts entities.InspectOptions containerEngine := registry.ContainerEngine() ctx := context.Background() - if cliExportOpts.Output == "" { + if cliExportOpts.OutputPath == "" { return errors.New("expects output path, use --output=[path]") } - inspectOpts.Type = common.VolumeType - volumeData, errs, err := containerEngine.VolumeInspect(ctx, args, inspectOpts) - if err != nil { + + if err := containerEngine.VolumeExport(ctx, args[0], cliExportOpts); err != nil { return err } - if len(errs) > 0 { - return errorhandling.JoinErrors(errs) - } - if len(volumeData) < 1 { - return errors.New("no volume data found") - } - mountPoint := volumeData[0].VolumeConfigResponse.Mountpoint - driver := volumeData[0].VolumeConfigResponse.Driver - volumeOptions := volumeData[0].VolumeConfigResponse.Options - volumeMountStatus, err := containerEngine.VolumeMounted(ctx, args[0]) - if err != nil { - return err - } - if mountPoint == "" { - return errors.New("volume is not mounted anywhere on host") - } - // Check if volume is using external plugin and export only if volume is mounted - if driver != "" && driver != "local" { - if !volumeMountStatus.Value { - return fmt.Errorf("volume is using a driver %s and volume is not mounted on %s", driver, mountPoint) - } - } - // Check if volume is using `local` driver and has mount options type other than tmpfs - if driver == "local" { - if mountOptionType, ok := volumeOptions["type"]; ok { - if mountOptionType != "tmpfs" && !volumeMountStatus.Value { - return fmt.Errorf("volume is using a driver %s and volume is not mounted on %s", driver, mountPoint) - } - } - } - logrus.Debugf("Exporting volume data from %s to %s", mountPoint, cliExportOpts.Output) - err = utils.CreateTarFromSrc(mountPoint, cliExportOpts.Output) - return err + return nil } diff --git a/libpod/volume.go b/libpod/volume.go index bf28bc5b5e..238d5f5bfd 100644 --- a/libpod/volume.go +++ b/libpod/volume.go @@ -3,12 +3,16 @@ package libpod import ( + "fmt" + "io" "time" "github.com/containers/podman/v5/libpod/define" "github.com/containers/podman/v5/libpod/lock" "github.com/containers/podman/v5/libpod/plugin" + "github.com/containers/podman/v5/utils" "github.com/containers/storage/pkg/directory" + "github.com/sirupsen/logrus" ) // Volume is a libpod named volume. @@ -294,3 +298,28 @@ func (v *Volume) Unmount() error { func (v *Volume) NeedsMount() bool { return v.needsMount() } + +// Returns a ReadCloser which points to a tar of all the volume's contents. +func (v *Volume) ExportVolume() (io.ReadCloser, error) { + v.lock.Lock() + err := v.mount() + v.lock.Unlock() + if err != nil { + return nil, err + } + defer func() { + v.lock.Lock() + defer v.lock.Unlock() + + if err := v.unmount(false); err != nil { + logrus.Errorf("Error unmounting volume %s: %v", v.Name(), err) + } + }() + + volContents, err := utils.TarWithChroot(v.mountPoint()) + if err != nil { + return nil, fmt.Errorf("creating tar of volume %s contents: %w", v.Name(), err) + } + + return volContents, nil +} diff --git a/pkg/api/handlers/libpod/volumes.go b/pkg/api/handlers/libpod/volumes.go index 545f3b3f55..ec7ed4fb30 100644 --- a/pkg/api/handlers/libpod/volumes.go +++ b/pkg/api/handlers/libpod/volumes.go @@ -215,3 +215,22 @@ func ExistsVolume(w http.ResponseWriter, r *http.Request) { } utils.WriteResponse(w, http.StatusNoContent, "") } + +// ExportVolume exports a volume +func ExportVolume(w http.ResponseWriter, r *http.Request) { + runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) + name := utils.GetName(r) + + vol, err := runtime.GetVolume(name) + if err != nil { + utils.Error(w, http.StatusNotFound, err) + return + } + + contents, err := vol.ExportVolume() + if err != nil { + utils.Error(w, http.StatusInternalServerError, err) + return + } + utils.WriteResponse(w, http.StatusOK, contents) +} diff --git a/pkg/api/server/register_volumes.go b/pkg/api/server/register_volumes.go index 867325d24e..9c67b83d05 100644 --- a/pkg/api/server/register_volumes.go +++ b/pkg/api/server/register_volumes.go @@ -148,6 +148,31 @@ func (s *APIServer) registerVolumeHandlers(r *mux.Router) error { // $ref: "#/responses/internalError" r.Handle(VersionedPath("/libpod/volumes/{name}"), s.APIHandler(libpod.RemoveVolume)).Methods(http.MethodDelete) + // swagger:operation GET /libpod/volumes/{name}/export libpod VolumeExportLibpod + // --- + // tags: + // - volumes + // summary: Export a volume + // parameters: + // - in: path + // name: name + // type: string + // required: true + // description: the name or ID of the volume + // produces: + // - application/x-tar + // responses: + // 200: + // description: no error + // schema: + // type: string + // format: binary + // 404: + // $ref: "#/responses/volumeNotFound" + // 500: + // $ref: "#/responses/internalError" + r.Handle(VersionedPath("/libpod/volumes/{name}/export"), s.APIHandler(libpod.ExportVolume)).Methods(http.MethodGet) + /* * Docker compatibility endpoints */ diff --git a/pkg/bindings/volumes/volumes.go b/pkg/bindings/volumes/volumes.go index 6ad1b9fb84..9440fd8176 100644 --- a/pkg/bindings/volumes/volumes.go +++ b/pkg/bindings/volumes/volumes.go @@ -2,10 +2,15 @@ package volumes import ( "context" + "errors" + "fmt" + "io" "net/http" + "os" "strings" "github.com/containers/podman/v5/pkg/bindings" + "github.com/containers/podman/v5/pkg/domain/entities" "github.com/containers/podman/v5/pkg/domain/entities/reports" entitiesTypes "github.com/containers/podman/v5/pkg/domain/entities/types" jsoniter "github.com/json-iterator/go" @@ -139,3 +144,33 @@ func Exists(ctx context.Context, nameOrID string, options *ExistsOptions) (bool, return response.IsSuccess(), nil } + +// Export exports a volume to the given path +func Export(ctx context.Context, nameOrID string, options entities.VolumeExportOptions) error { + if options.OutputPath == "" { + return errors.New("must provide valid path for file to write to") + } + + targetFile, err := os.Create(options.OutputPath) + if err != nil { + return fmt.Errorf("unable to create target file path %q: %w", options.OutputPath, err) + } + defer targetFile.Close() + + conn, err := bindings.GetClient(ctx) + if err != nil { + return err + } + response, err := conn.DoRequest(ctx, nil, http.MethodGet, "/volumes/%s/export", nil, nil, nameOrID) + if err != nil { + return err + } + defer response.Body.Close() + + if response.IsSuccess() || response.IsRedirection() { + if _, err := io.Copy(targetFile, response.Body); err != nil { + return fmt.Errorf("writing volume %s contents to file: %w", nameOrID, err) + } + } + return response.Process(nil) +} diff --git a/pkg/domain/entities/engine_container.go b/pkg/domain/entities/engine_container.go index 0e798be4e8..18a12e092d 100644 --- a/pkg/domain/entities/engine_container.go +++ b/pkg/domain/entities/engine_container.go @@ -116,4 +116,5 @@ type ContainerEngine interface { //nolint:interfacebloat VolumeRm(ctx context.Context, namesOrIds []string, opts VolumeRmOptions) ([]*VolumeRmReport, error) VolumeUnmount(ctx context.Context, namesOrIds []string) ([]*VolumeUnmountReport, error) VolumeReload(ctx context.Context) (*VolumeReloadReport, error) + VolumeExport(ctx context.Context, nameOrID string, options VolumeExportOptions) error } diff --git a/pkg/domain/entities/volumes.go b/pkg/domain/entities/volumes.go index 050e28ce0c..54fa8acba1 100644 --- a/pkg/domain/entities/volumes.go +++ b/pkg/domain/entities/volumes.go @@ -37,12 +37,13 @@ type VolumeListReport = types.VolumeListReport // VolumeReloadReport describes the response from reload volume plugins type VolumeReloadReport = types.VolumeReloadReport -/* - * Docker API compatibility types - */ - // VolumeMountReport describes the response from volume mount type VolumeMountReport = types.VolumeMountReport // VolumeUnmountReport describes the response from umounting a volume type VolumeUnmountReport = types.VolumeUnmountReport + +// VolumeExportOptions describes the options required to export a volume. +type VolumeExportOptions struct { + OutputPath string +} diff --git a/pkg/domain/infra/abi/volumes.go b/pkg/domain/infra/abi/volumes.go index 77c99559e5..4b57a1dc82 100644 --- a/pkg/domain/infra/abi/volumes.go +++ b/pkg/domain/infra/abi/volumes.go @@ -6,6 +6,8 @@ import ( "context" "errors" "fmt" + "io" + "os" "github.com/containers/podman/v5/libpod" "github.com/containers/podman/v5/libpod/define" @@ -239,3 +241,32 @@ func (ic *ContainerEngine) VolumeReload(ctx context.Context) (*entities.VolumeRe report := ic.Libpod.UpdateVolumePlugins(ctx) return &entities.VolumeReloadReport{VolumeReload: *report}, nil } + +func (ic *ContainerEngine) VolumeExport(ctx context.Context, nameOrID string, options entities.VolumeExportOptions) error { + if options.OutputPath == "" { + return errors.New("must provide valid path for file to write to") + } + + targetFile, err := os.Create(options.OutputPath) + if err != nil { + return fmt.Errorf("unable to create target file path %q: %w", options.OutputPath, err) + } + defer targetFile.Close() + + vol, err := ic.Libpod.GetVolume(nameOrID) + if err != nil { + return err + } + + contents, err := vol.ExportVolume() + if err != nil { + return err + } + defer contents.Close() + + if _, err := io.Copy(targetFile, contents); err != nil { + return fmt.Errorf("writing volume %s to file: %w", vol.Name(), err) + } + + return nil +} diff --git a/pkg/domain/infra/tunnel/volumes.go b/pkg/domain/infra/tunnel/volumes.go index 175d5b1465..bc14430d02 100644 --- a/pkg/domain/infra/tunnel/volumes.go +++ b/pkg/domain/infra/tunnel/volumes.go @@ -113,3 +113,7 @@ func (ic *ContainerEngine) VolumeUnmount(ctx context.Context, nameOrIDs []string func (ic *ContainerEngine) VolumeReload(ctx context.Context) (*entities.VolumeReloadReport, error) { return nil, errors.New("volume reload is not supported for remote clients") } + +func (ic *ContainerEngine) VolumeExport(ctx context.Context, nameOrID string, options entities.VolumeExportOptions) error { + return volumes.Export(ic.ClientCtx, nameOrID, options) +} diff --git a/test/e2e/volume_create_test.go b/test/e2e/volume_create_test.go index 7ea3e893bd..dfdee43fe2 100644 --- a/test/e2e/volume_create_test.go +++ b/test/e2e/volume_create_test.go @@ -65,10 +65,6 @@ var _ = Describe("Podman volume create", func() { }) It("podman create and export volume", func() { - if podmanTest.RemoteTest { - Skip("Volume export check does not work with a remote client") - } - volName := "my_vol_" + RandomString(10) session := podmanTest.Podman([]string{"volume", "create", volName}) session.WaitWithDefaultTimeout() diff --git a/test/system/160-volumes.bats b/test/system/160-volumes.bats index e1935e7d61..2438a535e4 100644 --- a/test/system/160-volumes.bats +++ b/test/system/160-volumes.bats @@ -260,8 +260,6 @@ EOF # stdout with NULs is easier to test here than in ginkgo @test "podman volume export to stdout" { - skip_if_remote "N/A on podman-remote" - local volname="myvol_$(random_string 10)" local mountpoint="/data$(random_string 8)"