diff --git a/.gitignore b/.gitignore index ca4001538..4533cad8a 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ node_modules/ .boto *.log *~ +.creds* diff --git a/internal/testutil/dockertestutil.go b/internal/testutil/dockertestutil.go new file mode 100644 index 000000000..15fc7a93d --- /dev/null +++ b/internal/testutil/dockertestutil.go @@ -0,0 +1,71 @@ +package testutil + +import ( + "bytes" + "net/url" + "os" + "os/exec" + "strings" + "testing" +) + +// RunDockerAndGetOutputOrSkip runs Docker and returns the output as a string. +func RunDockerAndGetOutputOrSkip(t *testing.T, args ...string) string { + t.Helper() + t.Logf("running docker %v", args) + + c := exec.Command("docker", args...) //nolint:gosec + + var stderr bytes.Buffer + + c.Stderr = &stderr + + out, err := c.Output() + if err != nil { + // skip or fail hard when running in CI environment. + TestSkipUnlessCI(t, "unable to run docker: %v %s (stderr %v)", err, out, stderr.String()) + } + + return strings.TrimSpace(string(out)) +} + +// RunContainerAndKillOnCloseOrSkip runs "docker run" and ensures that resulting container is killed +// on exit. Returns containerID. +func RunContainerAndKillOnCloseOrSkip(t *testing.T, args ...string) string { + t.Helper() + + containerID := RunDockerAndGetOutputOrSkip(t, args...) + + t.Cleanup(func() { + RunDockerAndGetOutputOrSkip(t, "kill", containerID) + }) + + return containerID +} + +// GetContainerMappedPortAddress returns : that can be used to connect to +// a given container and private port. +func GetContainerMappedPortAddress(t *testing.T, containerID, privatePort string) string { + t.Helper() + + portMapping := RunDockerAndGetOutputOrSkip(t, "port", containerID, privatePort) + + p := strings.LastIndex(portMapping, ":") + if p < 0 { + t.Fatalf("invalid port mapping: %v", portMapping) + } + + colonPort := portMapping[p:] + + dockerhost := os.Getenv("DOCKER_HOST") + if dockerhost == "" { + return "localhost" + colonPort + } + + u, err := url.Parse(dockerhost) + if err != nil { + t.Fatalf("unable to parse DOCKER_HOST: %v", err) + } + + return u.Hostname() + colonPort +} diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go new file mode 100644 index 000000000..f9965be2f --- /dev/null +++ b/internal/testutil/testutil.go @@ -0,0 +1,33 @@ +package testutil + +import ( + "fmt" + "os" + "runtime" + "testing" +) + +// TestSkipUnlessCI skips the current test with a provided message, except when running +// in CI environment, in which case it causes hard failure. +func TestSkipUnlessCI(t *testing.T, msg string, args ...interface{}) { + t.Helper() + + if len(args) > 0 { + msg = fmt.Sprintf(msg, args...) + } + + if os.Getenv("CI") != "" { + t.Fatal(msg) + } else { + t.Skip(msg) + } +} + +// TestSkipOnCIUnlessLinuxAMD64 skips the current test if running on CI unless the environment is Linux/AMD64. +func TestSkipOnCIUnlessLinuxAMD64(t *testing.T) { + t.Helper() + + if os.Getenv("CI") != "" && runtime.GOOS+"/"+runtime.GOARCH != "linux/amd64" { + t.Skip("test not supported in this environment.") + } +} diff --git a/repo/blob/sftp/sftp_storage_test.go b/repo/blob/sftp/sftp_storage_test.go index c8f27a42b..6da9464e9 100644 --- a/repo/blob/sftp/sftp_storage_test.go +++ b/repo/blob/sftp/sftp_storage_test.go @@ -4,23 +4,130 @@ "context" "fmt" "io/ioutil" + "net" "os" + "os/exec" + "path/filepath" "strconv" + "strings" "testing" + "time" "github.com/kopia/kopia/internal/blobtesting" "github.com/kopia/kopia/internal/testlogging" + "github.com/kopia/kopia/internal/testutil" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/sftp" ) +const ( + dockerImage = "atmoz/sftp" + dialTimeout = 10 * time.Second + sftpUsername = "foo" +) + +func mustGetLocalTmpDir(t *testing.T) string { + t.Helper() + + tmpDir, err := ioutil.TempDir(".", ".creds") + if err != nil { + t.Fatal(err) + } + + tmpDir, err = filepath.Abs(tmpDir) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + os.RemoveAll(tmpDir) + }) + + return tmpDir +} + +func mustRunOrSkip(t *testing.T, cmd string, args ...string) []byte { + t.Helper() + + o, err := exec.Command(cmd, args...).CombinedOutput() + if err != nil { + t.Fatalf("%s: %v", o, err) + } + + t.Logf("output: %s", o) + + return o +} + +func startDockerSFTPServerOrSkip(t *testing.T, idRSA string) (host string, port int, knownHostsFile string) { + t.Helper() + + tmpDir := mustGetLocalTmpDir(t) + sshHostED25519Key := filepath.Join(tmpDir, "ssh_host_ed25519_key") + sshHostRSAKey := filepath.Join(tmpDir, "ssh_host_rsa_key") + + mustRunOrSkip(t, "ssh-keygen", "-t", "ed25519", "-P", "", "-f", sshHostED25519Key) + mustRunOrSkip(t, "ssh-keygen", "-t", "rsa", "-P", "", "-f", sshHostRSAKey) + + // see https://github.com/atmoz/sftp for instructions + shortContainerID := testutil.RunContainerAndKillOnCloseOrSkip(t, + "run", "--rm", "-p", "0:22", + "-v", idRSA+".pub:/home/"+sftpUsername+"/.ssh/keys/id_rsa.pub:ro", + "-v", sshHostED25519Key+":/etc/ssh/ssh_host_ed25519_key:ro", + "-v", sshHostRSAKey+":/etc/ssh/ssh_host_rsa_key:ro", + "-d", dockerImage, + sftpUsername+"::::upload") + sftpEndpoint := testutil.GetContainerMappedPortAddress(t, shortContainerID, "22") + + // wait for SFTP server to come up. + deadline := time.Now().Add(dialTimeout) + for time.Now().Before(deadline) { + t.Logf("waiting for SFTP server to come up on '%v'...", sftpEndpoint) + + conn, err := net.DialTimeout("tcp", sftpEndpoint, time.Second) + if err != nil { + t.Logf("err: %v", err) + time.Sleep(time.Second) + + continue + } + + defer conn.Close() + + parts := strings.Split(sftpEndpoint, ":") + host = parts[0] + port, _ = strconv.Atoi(parts[1]) + knownHostsFile = filepath.Join(t.TempDir(), "known_hosts") + knownHostsData := mustRunOrSkip(t, "ssh-keyscan", "-t", "rsa", "-p", strconv.Itoa(port), host) + + ioutil.WriteFile(knownHostsFile, knownHostsData, 0600) + + t.Logf("SFTP server OK on host:%q port:%v. Known hosts file: %v", host, port, knownHostsFile) + + return + } + + t.Skipf("SFTP server did not start!") + + return //nolint:nakedret +} + func TestSFTPStorageValid(t *testing.T) { + testutil.TestSkipOnCIUnlessLinuxAMD64(t) + + tmpDir := mustGetLocalTmpDir(t) + idRSA := filepath.Join(tmpDir, "id_rsa") + + mustRunOrSkip(t, "ssh-keygen", "-t", "rsa", "-P", "", "-f", idRSA) + + host, port, knownHostsFile := startDockerSFTPServerOrSkip(t, idRSA) + for _, embedCreds := range []bool{false, true} { embedCreds := embedCreds t.Run(fmt.Sprintf("Embed=%v", embedCreds), func(t *testing.T) { ctx := testlogging.Context(t) - st, err := createSFTPStorage(ctx, t, embedCreds) + st, err := createSFTPStorage(ctx, t, host, port, idRSA, knownHostsFile, embedCreds) if err != nil { t.Fatalf("unable to connect to SSH: %v", err) } @@ -50,50 +157,19 @@ func deleteBlobs(ctx context.Context, t *testing.T, st blob.Storage) { } } -func createSFTPStorage(ctx context.Context, t *testing.T, embed bool) (blob.Storage, error) { +func createSFTPStorage(ctx context.Context, t *testing.T, host string, port int, idRSA, knownHostsFile string, embed bool) (blob.Storage, error) { t.Helper() - host := os.Getenv("KOPIA_SFTP_TEST_HOST") - if host == "" { - t.Skip("KOPIA_SFTP_TEST_HOST not provided") - } - - envPort := os.Getenv("KOPIA_SFTP_TEST_PORT") - if envPort == "" { - t.Skip("KOPIA_SFTP_TEST_PORT not provided") - } - - port, err := strconv.ParseInt(envPort, 10, 64) - if err != nil { - t.Skip("skipping test because port is not numeric") - } - - path := os.Getenv("KOPIA_SFTP_TEST_PATH") - if path == "" { - t.Skip("KOPIA_SFTP_TEST_PATH not provided") - } - - keyfile := os.Getenv("KOPIA_SFTP_KEYFILE") - if _, err = os.Stat(keyfile); err != nil { - t.Skip("skipping test because SFTP keyfile can't be opened") - } - - usr := os.Getenv("KOPIA_SFTP_TEST_USER") - if usr == "" { - t.Skip("KOPIA_SFTP_TEST_USER not provided") - } - - knownHostsFile := os.Getenv("KOPIA_SFTP_KNOWN_HOSTS_FILE") - if _, err = os.Stat(knownHostsFile); err != nil { - t.Skip("skipping test because SFTP known hosts file can't be opened") + if _, err := os.Stat(knownHostsFile); err != nil { + t.Fatalf("skipping test because SFTP known hosts file can't be opened: %v", knownHostsFile) } opt := &sftp.Options{ - Path: path, + Path: "/upload", Host: host, - Username: usr, - Port: int(port), - Keyfile: keyfile, + Username: sftpUsername, + Port: port, + Keyfile: idRSA, KnownHostsFile: knownHostsFile, }