From d9a1fc93b79219dddb101910408ec80e6fcb6bff Mon Sep 17 00:00:00 2001 From: "Juan B. Rodriguez" Date: Thu, 11 Jul 2019 07:33:28 -0500 Subject: [PATCH] Add e2e tests for sftp storage provider Uses environment variables from the CI server, to connect to a test sftp server This also removes the "localhost" test --- repo/blob/sftp/sftp_storage_test.go | 218 ++++++++-------------------- 1 file changed, 61 insertions(+), 157 deletions(-) diff --git a/repo/blob/sftp/sftp_storage_test.go b/repo/blob/sftp/sftp_storage_test.go index 6444b40a7..6c2e230f6 100644 --- a/repo/blob/sftp/sftp_storage_test.go +++ b/repo/blob/sftp/sftp_storage_test.go @@ -2,20 +2,11 @@ import ( "context" - "fmt" - "io" - "io/ioutil" - "log" - "net" "os" - "os/user" - "path/filepath" + "strconv" "testing" "time" - psftp "github.com/pkg/sftp" - "golang.org/x/crypto/ssh" - "github.com/kopia/kopia/internal/blobtesting" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/sftp" @@ -27,31 +18,10 @@ t3 = "0dae5918f83e6a24c8b3e274ca1026e43f24" ) -func TestSSHStorage(t *testing.T) { - go server(t) - - cwd, err := os.Getwd() - if err != nil { - t.Errorf("unable to getwd: %s", err) - } - - host := "localhost" - port := 2222 - keyfile := filepath.Join(cwd, "id_rsa") - usr, err := user.Current() - if err != nil { - t.Errorf("unable to get current user: %s", err) - } - +func TestSFTPStorageValid(t *testing.T) { ctx := context.Background() - st, err := sftp.New(ctx, &sftp.Options{ - Path: ".", - Host: host, - Username: usr.Username, - Port: port, - Keyfile: keyfile, - KnownHosts: filepath.Join(cwd, "known_hosts"), - }) + additionalPath := "" + st, err := createSFTPStorage(ctx, additionalPath, t) if err != nil { t.Fatalf("unable to connect to SSH: %v", err) @@ -66,6 +36,7 @@ func TestSSHStorage(t *testing.T) { deleteBlobs(ctx, t, st) blobtesting.VerifyStorage(ctx, t, st) + blobtesting.AssertConnectionInfoRoundTrips(ctx, t, st) // delete everything again deleteBlobs(ctx, t, st) @@ -75,78 +46,20 @@ func TestSSHStorage(t *testing.T) { } } -func server(t *testing.T) { - debugStream := os.Stderr +func TestSFTPStorageInvalid(t *testing.T) { + ctx := context.Background() + additionalPath := "-no-such-path" + st, err := createSFTPStorage(ctx, additionalPath, t) - c := createConfig(t) - - listener, _ := net.Listen("tcp", "127.0.0.1:2222") - for { - conn, _ := listener.Accept() - _, chans, reqs, _ := ssh.NewServerConn(conn, c) - go ssh.DiscardRequests(reqs) - - for newChannel := range chans { - // Channels have a type, depending on the application level - // protocol intended. In the case of an SFTP session, this is "subsystem" - // with a payload string of "sftp" - fmt.Fprintf(debugStream, "Incoming channel: %s\n", newChannel.ChannelType()) - if newChannel.ChannelType() != "session" { - _ = newChannel.Reject(ssh.UnknownChannelType, "unknown channel type") - fmt.Fprintf(debugStream, "Unknown channel type: %s\n", newChannel.ChannelType()) - continue - } - - channel, requests, err := newChannel.Accept() - if err != nil { - log.Fatalf("Could not accept channel: %v", err) - } - fmt.Fprintf(debugStream, "Channel accepted\n") - - // Sessions have out-of-band requests such as "shell", - // "pty-req" and "env". Here we handle only the - // "subsystem" request. - go func(in <-chan *ssh.Request) { - for req := range in { - fmt.Fprintf(debugStream, "Request: %v\n", req.Type) - ok := false - switch req.Type { - case "subsystem": - fmt.Fprintf(debugStream, "Subsystem: %s\n", req.Payload[4:]) - if string(req.Payload[4:]) == "sftp" { - ok = true - } - default: - ok = false - } - fmt.Fprintf(debugStream, " - accepted: %v\n", ok) - _ = req.Reply(ok, nil) - } - }(requests) - - serverOptions := []psftp.ServerOption{ - psftp.WithDebug(debugStream), - } - - fmt.Fprintf(debugStream, "Read write server\n") - - server, err := psftp.NewServer( - channel, - serverOptions..., - ) - if err != nil { - log.Fatal(err) - } - if err := server.Serve(); err == io.EOF { - channel.Close() - server.Close() - fmt.Fprintln(debugStream, "sftp client exited session.") - } else if err != nil { - fmt.Fprintf(debugStream, "sftp server completed with error: %s", err) - } - } + if err != nil { + t.Fatalf("unable to connect to SSH: %v", err) } + defer st.Close(ctx) + + if err := st.PutBlob(ctx, t1, []byte{1}); err == nil { + t.Errorf("unexpected success when adding to non-existent path") + } } func assertNoError(t *testing.T, err error) { @@ -156,60 +69,6 @@ func assertNoError(t *testing.T, err error) { } } -func createConfig(t *testing.T) *ssh.ServerConfig { - cwd, err := os.Getwd() - if err != nil { - t.Errorf("unable to getwd: %s", err) - } - - // Public key authentication is done by comparing - // the public key of a received connection - // with the entries in the authorized_keys file. - authorizedKeysBytes, err := ioutil.ReadFile(filepath.Join(cwd, "known_hosts")) - if err != nil { - t.Errorf("failed to load authorized_keys, err: %v", err) - } - - authorizedKeysMap := map[string]bool{} - for len(authorizedKeysBytes) > 0 { - pubKey, _, _, rest, e := ssh.ParseAuthorizedKey(authorizedKeysBytes) - if e != nil { - log.Fatal(e) - } - - authorizedKeysMap[string(pubKey.Marshal())] = true - authorizedKeysBytes = rest - } - - c := &ssh.ServerConfig{ - PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) { - if authorizedKeysMap[string(pubKey.Marshal())] { - return &ssh.Permissions{ - // Record the public key used for authentication. - Extensions: map[string]string{ - "pubkey-fp": ssh.FingerprintSHA256(pubKey), - }, - }, nil - } - return nil, fmt.Errorf("unknown public key for %q", c.User()) - }, - } - - privateBytes, err := ioutil.ReadFile(filepath.Join(cwd, "id_rsa")) - if err != nil { - log.Fatal("Failed to load private key: ", err) - } - - private, err := ssh.ParsePrivateKey(privateBytes) - if err != nil { - log.Fatal("Failed to parse private key: ", err) - } - - c.AddHostKey(private) - - return c -} - func deleteBlobs(ctx context.Context, t *testing.T, st blob.Storage) { if err := st.ListBlobs(ctx, "", func(bm blob.Metadata) error { return st.DeleteBlob(ctx, bm.BlobID) @@ -217,3 +76,48 @@ func deleteBlobs(ctx context.Context, t *testing.T, st blob.Storage) { t.Fatalf("unable to clear sftp storage: %v", err) } } + +func createSFTPStorage(ctx context.Context, additionalPath string, t *testing.T) (blob.Storage, error) { + host := os.Getenv("KOPIA_SFTP_TEST_HOST") + if host == "" { + t.Skip("KOPIA_SFTP_TEST_HOST not provided") + } + + envPort := os.Getenv("KOPIA_SFTP_TEST_PORT") + if envPort == "" { + t.Skip("KOPIA_SFTP_TEST_PORT not provided") + } + port, err := strconv.ParseInt(envPort, 10, 64) + if err != nil { + t.Skip("skipping test because port is not numeric") + } + + path := os.Getenv("KOPIA_SFTP_TEST_PATH") + if path == "" { + t.Skip("KOPIA_SFTP_TEST_PATH not provided") + } + + keyfile := os.Getenv("KOPIA_SFTP_KEYFILE") + if _, err = os.Stat(keyfile); err != nil { + t.Skip("skipping test because SFTP keyfile can't be opened") + } + + usr := os.Getenv("KOPIA_SFTP_TEST_USER") + if usr == "" { + t.Skip("KOPIA_SFTP_TEST_USER not provided") + } + + knownHosts := os.Getenv("KOPIA_SFTP_KNOWN_HOSTS_FILE") + if _, err = os.Stat(knownHosts); err != nil { + t.Skip("skipping test because SFTP known hosts file can't be opened") + } + + return sftp.New(ctx, &sftp.Options{ + Path: path + additionalPath, + Host: host, + Username: usr, + Port: int(port), + Keyfile: keyfile, + KnownHosts: knownHosts, + }) +}