Merge pull request #29767 from hihry/feat/AddSupporForOutputFlag

Add support for --output to podman build --remote

cmd/podman/common/build.go

@@ -129,7 +129,6 @@ func DefineBuildFlags(cmd *cobra.Command, buildOpts *BuildFlagsWrapper, isFarmBu
 		_ = flags.MarkHidden("sign-by")
 		_ = flags.MarkHidden("signature-policy")
 		_ = flags.MarkHidden("compress")
-		_ = flags.MarkHidden("output")
 		_ = flags.MarkHidden("logsplit")
 		_ = flags.MarkHidden("cw")
 		// Support for farm build in podman-remote
@@ -149,9 +148,6 @@ func ParseBuildOpts(cmd *cobra.Command, args []string, buildOpts *BuildFlagsWrap
 		return nil, errors.New("cannot specify --squash-all with --squash")
 	}
 
-	if cmd.Flag("output").Changed && registry.IsRemote() {
-		return nil, errors.New("'--output' option is not supported in remote mode")
-	}
 	if cmd.Flag("metadata-file").Changed && registry.IsRemote() {
 		return nil, errors.New("'--metadata-file' option is not supported in remote mode")
 	}

docs/source/markdown/podman-build.1.md.in

@@ -300,7 +300,7 @@ This option conflicts with **--add-host**.
 
 Output destination (format: type=local,dest=path)
 
-The --output (or -o) option extends the default behavior of building a container image by allowing users to export the contents of the image as files on the local filesystem, which can be useful for generating local binaries, code generation, etc. (This option is not available with the remote Podman client, including Mac and Windows (excluding WSL2) machines)
+The --output (or -o) option extends the default behavior of building a container image by allowing users to export the contents of the image as files on the local filesystem, which can be useful for generating local binaries, code generation, etc.
 
 The value for --output is a comma-separated sequence of key=value pairs, defining the output type and options.
 

pkg/api/handlers/libpod/images.go

@@ -19,6 +19,7 @@ import (
 	"github.com/moby/moby/api/types/jsonstream"
 	"github.com/sirupsen/logrus"
 	"go.podman.io/buildah"
+	"go.podman.io/buildah/copier"
 	"go.podman.io/common/libimage"
 	"go.podman.io/common/pkg/ssh"
 	"go.podman.io/image/v5/manifest"
@@ -43,6 +44,7 @@ import (
 	"go.podman.io/storage/pkg/archive"
 	"go.podman.io/storage/pkg/chrootarchive"
 	"go.podman.io/storage/pkg/idtools"
+	"go.podman.io/storage/pkg/unshare"
 )
 
 // Commit
@@ -253,6 +255,76 @@ func ExportImage(w http.ResponseWriter, r *http.Request) {
 	utils.WriteResponse(w, http.StatusOK, rdr)
 }
 
+func ExportImageRootfs(w http.ResponseWriter, r *http.Request) {
+	runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime)
+	name := utils.GetName(r)
+
+	image, _, err := runtime.LibimageRuntime().LookupImage(name, nil)
+	if err != nil {
+		utils.ImageNotFound(w, name, err)
+		return
+	}
+
+	ctx := r.Context()
+	mountPoint, err := image.Mount(ctx, nil, "")
+
+	cleanup := func() {
+		if err := image.Unmount(false); err != nil {
+			logrus.Errorf("failed to unmount image %s: %v", image.ID(), err)
+		}
+	}
+
+	// If the image is located in an additional read-only store, image.Mount()
+	// will fail with storage.ErrLayerUnknown because the storage driver expects
+	// a Read/Write layer to exist in order to mount it successfully.
+	// We fall back to creating a short-lived working container (exactly as Buildah does),
+	// which allocates a fresh, empty R/W layer in the primary store, allowing the mount to succeed.
+	if err != nil && errors.Is(err, storage.ErrLayerUnknown) {
+		store, storeErr := storage.GetStore(runtime.StorageConfig())
+		if storeErr == nil {
+			if b, err2 := buildah.NewBuilder(ctx, store, buildah.BuilderOptions{
+				FromImage:       image.ID(),
+				PullPolicy:      buildah.PullNever,
+				ContainerSuffix: "tmp",
+			}); err2 == nil {
+				mountPoint, err = b.Mount("")
+
+				cleanup = func() {
+					if err := b.Delete(); err != nil {
+						logrus.Errorf("failed to delete temporary container %q for image %q: %v", b.ContainerID, image.ID(), err)
+					}
+				}
+			}
+		}
+	}
+
+	if err != nil {
+		utils.InternalServerError(w, fmt.Errorf("mounting image to stream rootfs: %w", err))
+		return
+	}
+
+	defer cleanup()
+
+	w.Header().Set("Content-Type", "application/x-tar")
+
+	getOptions := copier.GetOptions{}
+	if unshare.IsRootless() {
+		// In order to maintain parity with buildah's native --output behavior,
+		// we strip setuid, setgid, and extended attributes. This avoids
+		// unsafe invocation of exported executables on the client side.
+		// Matches logic in go.podman.io/buildah/imagebuildah/stage_executor.go
+		getOptions.StripSetuidBit = true
+		getOptions.StripSetgidBit = true
+		getOptions.StripXattrs = true
+	}
+
+	err = copier.Get(mountPoint, mountPoint, getOptions, []string{"."}, w)
+	if err != nil {
+		utils.InternalServerError(w, fmt.Errorf("failed to stream image rootfs to client: %w", err))
+		return
+	}
+}
+
 func ExportImages(w http.ResponseWriter, r *http.Request) {
 	var output string
 	runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime)

pkg/api/server/register_images.go

@@ -1351,6 +1351,31 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error {
 	//   500:
 	//     $ref: '#/responses/internalError'
 	r.Handle(VersionedPath("/libpod/images/{name:.*}/get"), s.APIHandler(libpod.ExportImage)).Methods(http.MethodGet)
+	// swagger:operation GET /libpod/images/{name}/rootfs libpod ImageRootfsLibpod
+	// ---
+	// tags:
+	//  - images
+	// summary: Export an image rootfs
+	// description: Export an image's root filesystem as a tar archive
+	// parameters:
+	//  - in: path
+	//    name: name
+	//    type: string
+	//    required: true
+	//    description: the name or ID of the image
+	// produces:
+	// - application/x-tar
+	// responses:
+	//   200:
+	//     description: no error
+	//     schema:
+	//       type: string
+	//       format: binary
+	//   404:
+	//     $ref: '#/responses/imageNotFound'
+	//   500:
+	//     $ref: '#/responses/internalError'
+	r.Handle(VersionedPath("/libpod/images/{name:.*}/rootfs"), s.APIHandler(libpod.ExportImageRootfs)).Methods(http.MethodGet)
 	// swagger:operation GET /libpod/images/export libpod ImageExportLibpod
 	// ---
 	// tags:

pkg/bindings/images/images.go

@@ -343,3 +343,23 @@ func Scp(ctx context.Context, source, _ *string, options ScpOptions) (reports.Sc
 
 	return rep, response.Process(&rep)
 }
+
+// ExportRootfs streams the extracted root filesystem of an image as a tar archive.
+// this returns a flat filesystem tarball suitable for `podman build --output`.
+func ExportRootfs(ctx context.Context, nameOrID string, w io.Writer) error {
+	conn, err := bindings.GetClient(ctx)
+	if err != nil {
+		return err
+	}
+	response, err := conn.DoRequest(ctx, nil, http.MethodGet, "/images/%s/rootfs", nil, nil, nameOrID)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+
+	if response.IsSuccess() || response.IsRedirection() {
+		_, err = io.Copy(w, response.Body)
+		return err
+	}
+	return response.Process(nil)
+}

pkg/domain/infra/tunnel/images.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"strconv"
@@ -419,9 +420,12 @@ func (ir *ImageEngine) Build(_ context.Context, containerFiles []string, opts en
 	if err != nil {
 		logrus.Debugf("IsHyperVProvider check failed: %v", err)
 	}
+
 	// Local api is not supported on Windows Hyper-V, because 9p mounts don't translate all file attributes correctly.
 	// So we skip trying to use localapi in that case.
-	if !isHyperV {
+	// We also skip localapi if BuildOutputs are requested, because localapi doesn't translate output paths,
+	// and we want our client-side extraction to handle it over the REST API instead.
+	if !isHyperV && len(opts.BuildOutputs) == 0 {
 		if translatedContainerFiles, translatedOptions, ok := localapi.CheckIfImageBuildPathsOnRunningMachine(ir.ClientCtx, containerFiles, opts); ok {
 			report, err := images.BuildFromServerContext(ir.ClientCtx, translatedContainerFiles, translatedOptions)
 			if err == nil {
@@ -441,6 +445,21 @@ func (ir *ImageEngine) Build(_ context.Context, containerFiles []string, opts en
 		}
 	}
 
+	// Capture and strip BuildOutputs before sending to server
+	// The server should do normal image build; the client handlees
+	// The --output extraction afterward
+	buildOutputs := opts.BuildOutputs
+	opts.BuildOutputs = nil
+
+	var parsedOutputs []buildOutputOption
+	for _, buildOutput := range buildOutputs {
+		parsedOpt, err := parseBuildOutput(buildOutput)
+		if err != nil {
+			return nil, err
+		}
+		parsedOutputs = append(parsedOutputs, parsedOpt)
+	}
+
 	report, err := images.Build(ir.ClientCtx, containerFiles, opts)
 	if err != nil {
 		return nil, err
@@ -449,9 +468,157 @@ func (ir *ImageEngine) Build(_ context.Context, containerFiles []string, opts en
 	if opts.OutputFormat == bdefine.Dockerv2ImageManifest {
 		report.SaveFormat = define.V2s2Archive
 	}
+
+	// Handle --output extraction if requested
+	if len(buildOutputs) > 0 {
+		if err := ir.handleBuildOutputs(parsedOutputs, report.ID); err != nil {
+			return nil, err
+		}
+	}
 	return report, nil
 }
 
+// handleBuildOutputs downloads the rootfs of a built image and writes it
+// to the local destinations specified by the --output flag(s).
+func (ir *ImageEngine) handleBuildOutputs(parsedOutputs []buildOutputOption, imageID string) error {
+	for _, outputOpt := range parsedOutputs {
+		switch outputOpt.Type {
+		case buildOutputStdout:
+			err := images.ExportRootfs(ir.ClientCtx, imageID, os.Stdout)
+			if err != nil {
+				return fmt.Errorf("exporting image rootfs: %w", err)
+			}
+
+		case buildOutputTar:
+			f, err := os.Create(outputOpt.Path)
+			if err != nil {
+				return fmt.Errorf("creating output file %s: %w", outputOpt.Path, err)
+			}
+			err = images.ExportRootfs(ir.ClientCtx, imageID, f)
+			closeErr := f.Close()
+			if err != nil {
+				return fmt.Errorf("exporting image rootfs: %w", err)
+			}
+			if closeErr != nil {
+				return fmt.Errorf("closing output file %s: %w", outputOpt.Path, closeErr)
+			}
+
+		case buildOutputLocalDir:
+			if err := os.MkdirAll(outputOpt.Path, 0o755); err != nil {
+				return fmt.Errorf("creating output directory %s: %w", outputOpt.Path, err)
+			}
+
+			pr, pw := io.Pipe()
+			errCh := make(chan error, 1)
+			go func() {
+				err := images.ExportRootfs(ir.ClientCtx, imageID, pw)
+				pw.CloseWithError(err)
+				errCh <- err
+			}()
+
+			// The Server has already stripped setuid/xattrs if rootless,
+			// so archive.Untar is perfectly safe to use here!
+			if err := archive.Untar(pr, outputOpt.Path, &archive.TarOptions{NoLchown: true}); err != nil {
+				pr.CloseWithError(err)
+				return fmt.Errorf("extracting rootfs to %s: %w", outputOpt.Path, err)
+			}
+			pr.Close()
+			if err := <-errCh; err != nil {
+				return fmt.Errorf("exporting image rootfs: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+type buildOutputType int
+
+const (
+	buildOutputInvalid buildOutputType = iota
+	buildOutputStdout
+	buildOutputLocalDir
+	buildOutputTar
+)
+
+// buildOutputOption contains the outcome of parsing the value of a build --output flag
+type buildOutputOption struct {
+	Type buildOutputType
+	Path string // Only valid if Type is local dir or tar
+}
+
+// parseBuildOutput is responsible for parsing custom build output argument i.e `build --output` flag.
+// This logic perfectly mirrors go.podman.io/buildah/internal/output/build_output.go
+func parseBuildOutput(buildOutput string) (buildOutputOption, error) {
+	// Support simple values, in the form --output ./mydir
+	if !strings.Contains(buildOutput, ",") && !strings.Contains(buildOutput, "=") {
+		if buildOutput == "-" {
+			return buildOutputOption{Type: buildOutputStdout, Path: ""}, nil
+		}
+		return buildOutputOption{Type: buildOutputLocalDir, Path: buildOutput}, nil
+	}
+
+	// Support complex values, in the form --output type=local,dest=./mydir
+	typeSelected := buildOutputInvalid
+	pathSelected := ""
+	for option := range strings.SplitSeq(buildOutput, ",") {
+		key, value, found := strings.Cut(option, "=")
+		if !found {
+			return buildOutputOption{}, fmt.Errorf("invalid build output options %q, expected format key=value", buildOutput)
+		}
+		switch key {
+		case "type":
+			if typeSelected != buildOutputInvalid {
+				return buildOutputOption{}, fmt.Errorf("duplicate %q not supported", key)
+			}
+			switch value {
+			case "local":
+				typeSelected = buildOutputLocalDir
+			case "tar":
+				typeSelected = buildOutputTar
+			default:
+				return buildOutputOption{}, fmt.Errorf("invalid type %q selected for build output options %q", value, buildOutput)
+			}
+		case "dest":
+			if pathSelected != "" {
+				return buildOutputOption{}, fmt.Errorf("duplicate %q not supported", key)
+			}
+			pathSelected = value
+		default:
+			return buildOutputOption{}, fmt.Errorf("unrecognized key %q in build output option: %q", key, buildOutput)
+		}
+	}
+
+	// Validate there is a type
+	if typeSelected == buildOutputInvalid {
+		return buildOutputOption{}, fmt.Errorf("missing required key %q in build output option: %q", "type", buildOutput)
+	}
+
+	// Validate path
+	if typeSelected == buildOutputLocalDir || typeSelected == buildOutputTar {
+		if pathSelected == "" {
+			return buildOutputOption{}, fmt.Errorf("missing required key %q in build output option: %q", "dest", buildOutput)
+		}
+	} else {
+		// Clear path when not needed by type
+		pathSelected = ""
+	}
+
+	// Handle redirecting stdout for tar output
+	if pathSelected == "-" {
+		if typeSelected == buildOutputTar {
+			typeSelected = buildOutputStdout
+			pathSelected = ""
+		} else {
+			return buildOutputOption{}, fmt.Errorf(`invalid build output option %q, only "type=tar" can be used with "dest=-"`, buildOutput)
+		}
+	}
+
+	return buildOutputOption{
+		Type: typeSelected,
+		Path: pathSelected,
+	}, nil
+}
+
 func (ir *ImageEngine) Tree(_ context.Context, nameOrID string, opts entities.ImageTreeOptions) (*entities.ImageTreeReport, error) {
 	options := new(images.TreeOptions).WithWhatRequires(opts.WhatRequires)
 	return images.Tree(ir.ClientCtx, nameOrID, options)

test/e2e/build_test.go

@@ -1456,16 +1456,26 @@ COPY --from=img2 /etc/alpine-release /prefix-test/container-prefix.txt`
 	})
 
 	It("podman build --output type=local,dest=./folder outputs to ./folder", func() {
-		SkipIfRemote("--output is not supported in remote mode")
 		podmanTest.PodmanExitCleanly("build", "-f", "build/basicalpine/Containerfile", "--output", fmt.Sprintf("type=local,dest=%v", podmanTest.TempDir))
 		files, err := os.ReadDir(podmanTest.TempDir)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(len(files)).To(BeNumerically(">", 1))
 	})
 
+	It("podman build --output type=tar outputs a tar file", func() {
+		outfile := filepath.Join(podmanTest.TempDir, "output.tar")
+		podmanTest.PodmanExitCleanly("build", "-f", "build/basicalpine/Containerfile", "--output", fmt.Sprintf("type=tar,dest=%v", outfile))
+		_, err := os.Stat(outfile)
+		Expect(err).ToNot(HaveOccurred())
+		// Verify the tar contains rootfs contents (e.g., /etc)
+		session := podmanTest.Podman([]string{"run", "--rm", "-v", fmt.Sprintf("%s:/test.tar:z", outfile), ALPINE, "tar", "tf", "/test.tar"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(ExitCleanly())
+		Expect(session.OutputToString()).To(ContainSubstring("etc/"))
+	})
+
 	// Should error because no type
 	It("podman build --output dest=./folder must fail", func() {
-		SkipIfRemote("--output is not supported in remote mode")
 		session := podmanTest.Podman([]string{"build", "-f", "build/basicalpine/Containerfile", "--output", fmt.Sprintf("dest=%v", podmanTest.TempDir)})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitWithError(125, `missing required key "type"`))