sftp: run private SFTP server based on Docker (atmoz/sftp) (#796)

* sftp: run private SFTP server based on Docker (atmoz/sftp)

* sftp: disable test on GH Actions && !linux because it does not support Docker.

.gitignore

@@ -23,3 +23,4 @@ node_modules/
 .boto
 *.log
 *~
+.creds*

internal/testutil/dockertestutil.go

@@ -0,0 +1,71 @@
+package testutil
+
+import (
+	"bytes"
+	"net/url"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+// RunDockerAndGetOutputOrSkip runs Docker and returns the output as a string.
+func RunDockerAndGetOutputOrSkip(t *testing.T, args ...string) string {
+	t.Helper()
+	t.Logf("running docker %v", args)
+
+	c := exec.Command("docker", args...) //nolint:gosec
+
+	var stderr bytes.Buffer
+
+	c.Stderr = &stderr
+
+	out, err := c.Output()
+	if err != nil {
+		// skip or fail hard when running in CI environment.
+		TestSkipUnlessCI(t, "unable to run docker: %v %s (stderr %v)", err, out, stderr.String())
+	}
+
+	return strings.TrimSpace(string(out))
+}
+
+// RunContainerAndKillOnCloseOrSkip runs "docker run" and ensures that resulting container is killed
+// on exit. Returns containerID.
+func RunContainerAndKillOnCloseOrSkip(t *testing.T, args ...string) string {
+	t.Helper()
+
+	containerID := RunDockerAndGetOutputOrSkip(t, args...)
+
+	t.Cleanup(func() {
+		RunDockerAndGetOutputOrSkip(t, "kill", containerID)
+	})
+
+	return containerID
+}
+
+// GetContainerMappedPortAddress returns <host>:<port> that can be used to connect to
+// a given container and private port.
+func GetContainerMappedPortAddress(t *testing.T, containerID, privatePort string) string {
+	t.Helper()
+
+	portMapping := RunDockerAndGetOutputOrSkip(t, "port", containerID, privatePort)
+
+	p := strings.LastIndex(portMapping, ":")
+	if p < 0 {
+		t.Fatalf("invalid port mapping: %v", portMapping)
+	}
+
+	colonPort := portMapping[p:]
+
+	dockerhost := os.Getenv("DOCKER_HOST")
+	if dockerhost == "" {
+		return "localhost" + colonPort
+	}
+
+	u, err := url.Parse(dockerhost)
+	if err != nil {
+		t.Fatalf("unable to parse DOCKER_HOST: %v", err)
+	}
+
+	return u.Hostname() + colonPort
+}

internal/testutil/testutil.go

@@ -0,0 +1,33 @@
+package testutil
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"testing"
+)
+
+// TestSkipUnlessCI skips the current test with a provided message, except when running
+// in CI environment, in which case it causes hard failure.
+func TestSkipUnlessCI(t *testing.T, msg string, args ...interface{}) {
+	t.Helper()
+
+	if len(args) > 0 {
+		msg = fmt.Sprintf(msg, args...)
+	}
+
+	if os.Getenv("CI") != "" {
+		t.Fatal(msg)
+	} else {
+		t.Skip(msg)
+	}
+}
+
+// TestSkipOnCIUnlessLinuxAMD64 skips the current test if running on CI unless the environment is Linux/AMD64.
+func TestSkipOnCIUnlessLinuxAMD64(t *testing.T) {
+	t.Helper()
+
+	if os.Getenv("CI") != "" && runtime.GOOS+"/"+runtime.GOARCH != "linux/amd64" {
+		t.Skip("test not supported in this environment.")
+	}
+}

repo/blob/sftp/sftp_storage_test.go

@@ -4,23 +4,130 @@
 	"context"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
+	"os/exec"
+	"path/filepath"
 	"strconv"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/kopia/kopia/internal/blobtesting"
 	"github.com/kopia/kopia/internal/testlogging"
+	"github.com/kopia/kopia/internal/testutil"
 	"github.com/kopia/kopia/repo/blob"
 	"github.com/kopia/kopia/repo/blob/sftp"
 )
 
+const (
+	dockerImage  = "atmoz/sftp"
+	dialTimeout  = 10 * time.Second
+	sftpUsername = "foo"
+)
+
+func mustGetLocalTmpDir(t *testing.T) string {
+	t.Helper()
+
+	tmpDir, err := ioutil.TempDir(".", ".creds")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tmpDir, err = filepath.Abs(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		os.RemoveAll(tmpDir)
+	})
+
+	return tmpDir
+}
+
+func mustRunOrSkip(t *testing.T, cmd string, args ...string) []byte {
+	t.Helper()
+
+	o, err := exec.Command(cmd, args...).CombinedOutput()
+	if err != nil {
+		t.Fatalf("%s: %v", o, err)
+	}
+
+	t.Logf("output: %s", o)
+
+	return o
+}
+
+func startDockerSFTPServerOrSkip(t *testing.T, idRSA string) (host string, port int, knownHostsFile string) {
+	t.Helper()
+
+	tmpDir := mustGetLocalTmpDir(t)
+	sshHostED25519Key := filepath.Join(tmpDir, "ssh_host_ed25519_key")
+	sshHostRSAKey := filepath.Join(tmpDir, "ssh_host_rsa_key")
+
+	mustRunOrSkip(t, "ssh-keygen", "-t", "ed25519", "-P", "", "-f", sshHostED25519Key)
+	mustRunOrSkip(t, "ssh-keygen", "-t", "rsa", "-P", "", "-f", sshHostRSAKey)
+
+	// see https://github.com/atmoz/sftp for instructions
+	shortContainerID := testutil.RunContainerAndKillOnCloseOrSkip(t,
+		"run", "--rm", "-p", "0:22",
+		"-v", idRSA+".pub:/home/"+sftpUsername+"/.ssh/keys/id_rsa.pub:ro",
+		"-v", sshHostED25519Key+":/etc/ssh/ssh_host_ed25519_key:ro",
+		"-v", sshHostRSAKey+":/etc/ssh/ssh_host_rsa_key:ro",
+		"-d", dockerImage,
+		sftpUsername+"::::upload")
+	sftpEndpoint := testutil.GetContainerMappedPortAddress(t, shortContainerID, "22")
+
+	// wait for SFTP server to come up.
+	deadline := time.Now().Add(dialTimeout)
+	for time.Now().Before(deadline) {
+		t.Logf("waiting for SFTP server to come up on '%v'...", sftpEndpoint)
+
+		conn, err := net.DialTimeout("tcp", sftpEndpoint, time.Second)
+		if err != nil {
+			t.Logf("err: %v", err)
+			time.Sleep(time.Second)
+
+			continue
+		}
+
+		defer conn.Close()
+
+		parts := strings.Split(sftpEndpoint, ":")
+		host = parts[0]
+		port, _ = strconv.Atoi(parts[1])
+		knownHostsFile = filepath.Join(t.TempDir(), "known_hosts")
+		knownHostsData := mustRunOrSkip(t, "ssh-keyscan", "-t", "rsa", "-p", strconv.Itoa(port), host)
+
+		ioutil.WriteFile(knownHostsFile, knownHostsData, 0600)
+
+		t.Logf("SFTP server OK on host:%q port:%v. Known hosts file: %v", host, port, knownHostsFile)
+
+		return
+	}
+
+	t.Skipf("SFTP server did not start!")
+
+	return //nolint:nakedret
+}
+
 func TestSFTPStorageValid(t *testing.T) {
+	testutil.TestSkipOnCIUnlessLinuxAMD64(t)
+
+	tmpDir := mustGetLocalTmpDir(t)
+	idRSA := filepath.Join(tmpDir, "id_rsa")
+
+	mustRunOrSkip(t, "ssh-keygen", "-t", "rsa", "-P", "", "-f", idRSA)
+
+	host, port, knownHostsFile := startDockerSFTPServerOrSkip(t, idRSA)
+
 	for _, embedCreds := range []bool{false, true} {
 		embedCreds := embedCreds
 		t.Run(fmt.Sprintf("Embed=%v", embedCreds), func(t *testing.T) {
 			ctx := testlogging.Context(t)
 
-			st, err := createSFTPStorage(ctx, t, embedCreds)
+			st, err := createSFTPStorage(ctx, t, host, port, idRSA, knownHostsFile, embedCreds)
 			if err != nil {
 				t.Fatalf("unable to connect to SSH: %v", err)
 			}
@@ -50,50 +157,19 @@ func deleteBlobs(ctx context.Context, t *testing.T, st blob.Storage) {
 	}
 }
 
-func createSFTPStorage(ctx context.Context, t *testing.T, embed bool) (blob.Storage, error) {
+func createSFTPStorage(ctx context.Context, t *testing.T, host string, port int, idRSA, knownHostsFile string, embed bool) (blob.Storage, error) {
 	t.Helper()
 
-	host := os.Getenv("KOPIA_SFTP_TEST_HOST")
-	if host == "" {
-		t.Skip("KOPIA_SFTP_TEST_HOST not provided")
-	}
-
-	envPort := os.Getenv("KOPIA_SFTP_TEST_PORT")
-	if envPort == "" {
-		t.Skip("KOPIA_SFTP_TEST_PORT not provided")
-	}
-
-	port, err := strconv.ParseInt(envPort, 10, 64)
-	if err != nil {
-		t.Skip("skipping test because port is not numeric")
-	}
-
-	path := os.Getenv("KOPIA_SFTP_TEST_PATH")
-	if path == "" {
-		t.Skip("KOPIA_SFTP_TEST_PATH not provided")
-	}
-
-	keyfile := os.Getenv("KOPIA_SFTP_KEYFILE")
-	if _, err = os.Stat(keyfile); err != nil {
-		t.Skip("skipping test because SFTP keyfile can't be opened")
-	}
-
-	usr := os.Getenv("KOPIA_SFTP_TEST_USER")
-	if usr == "" {
-		t.Skip("KOPIA_SFTP_TEST_USER not provided")
-	}
-
-	knownHostsFile := os.Getenv("KOPIA_SFTP_KNOWN_HOSTS_FILE")
-	if _, err = os.Stat(knownHostsFile); err != nil {
-		t.Skip("skipping test because SFTP known hosts file can't be opened")
+	if _, err := os.Stat(knownHostsFile); err != nil {
+		t.Fatalf("skipping test because SFTP known hosts file can't be opened: %v", knownHostsFile)
 	}
 
 	opt := &sftp.Options{
-		Path:           path,
+		Path:           "/upload",
 		Host:           host,
-		Username:       usr,
-		Port:           int(port),
-		Keyfile:        keyfile,
+		Username:       sftpUsername,
+		Port:           port,
+		Keyfile:        idRSA,
 		KnownHostsFile: knownHostsFile,
 	}