mirror of
https://github.com/kopia/kopia.git
synced 2026-05-07 22:32:45 -04:00
Add e2e tests for sftp storage provider
Uses environment variables from the CI server, to connect to a test sftp server This also removes the "localhost" test
This commit is contained in:
Juan B. Rodriguez
committed by
Jarek Kowalski
Jarek Kowalski
parent
e414e7a4d1
commit
d9a1fc93b7
@@ -2,20 +2,11 @@
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
psftp "github.com/pkg/sftp"
|
||||
"golang.org/x/crypto/ssh"
|
||||
|
||||
"github.com/kopia/kopia/internal/blobtesting"
|
||||
"github.com/kopia/kopia/repo/blob"
|
||||
"github.com/kopia/kopia/repo/blob/sftp"
|
||||
@@ -27,31 +18,10 @@
|
||||
t3 = "0dae5918f83e6a24c8b3e274ca1026e43f24"
|
||||
)
|
||||
|
||||
func TestSSHStorage(t *testing.T) {
|
||||
go server(t)
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Errorf("unable to getwd: %s", err)
|
||||
}
|
||||
|
||||
host := "localhost"
|
||||
port := 2222
|
||||
keyfile := filepath.Join(cwd, "id_rsa")
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
t.Errorf("unable to get current user: %s", err)
|
||||
}
|
||||
|
||||
func TestSFTPStorageValid(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st, err := sftp.New(ctx, &sftp.Options{
|
||||
Path: ".",
|
||||
Host: host,
|
||||
Username: usr.Username,
|
||||
Port: port,
|
||||
Keyfile: keyfile,
|
||||
KnownHosts: filepath.Join(cwd, "known_hosts"),
|
||||
})
|
||||
additionalPath := ""
|
||||
st, err := createSFTPStorage(ctx, additionalPath, t)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("unable to connect to SSH: %v", err)
|
||||
@@ -66,6 +36,7 @@ func TestSSHStorage(t *testing.T) {
|
||||
deleteBlobs(ctx, t, st)
|
||||
|
||||
blobtesting.VerifyStorage(ctx, t, st)
|
||||
blobtesting.AssertConnectionInfoRoundTrips(ctx, t, st)
|
||||
|
||||
// delete everything again
|
||||
deleteBlobs(ctx, t, st)
|
||||
@@ -75,78 +46,20 @@ func TestSSHStorage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func server(t *testing.T) {
|
||||
debugStream := os.Stderr
|
||||
func TestSFTPStorageInvalid(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
additionalPath := "-no-such-path"
|
||||
st, err := createSFTPStorage(ctx, additionalPath, t)
|
||||
|
||||
c := createConfig(t)
|
||||
|
||||
listener, _ := net.Listen("tcp", "127.0.0.1:2222")
|
||||
for {
|
||||
conn, _ := listener.Accept()
|
||||
_, chans, reqs, _ := ssh.NewServerConn(conn, c)
|
||||
go ssh.DiscardRequests(reqs)
|
||||
|
||||
for newChannel := range chans {
|
||||
// Channels have a type, depending on the application level
|
||||
// protocol intended. In the case of an SFTP session, this is "subsystem"
|
||||
// with a payload string of "<length=4>sftp"
|
||||
fmt.Fprintf(debugStream, "Incoming channel: %s\n", newChannel.ChannelType())
|
||||
if newChannel.ChannelType() != "session" {
|
||||
_ = newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
|
||||
fmt.Fprintf(debugStream, "Unknown channel type: %s\n", newChannel.ChannelType())
|
||||
continue
|
||||
}
|
||||
|
||||
channel, requests, err := newChannel.Accept()
|
||||
if err != nil {
|
||||
log.Fatalf("Could not accept channel: %v", err)
|
||||
}
|
||||
fmt.Fprintf(debugStream, "Channel accepted\n")
|
||||
|
||||
// Sessions have out-of-band requests such as "shell",
|
||||
// "pty-req" and "env". Here we handle only the
|
||||
// "subsystem" request.
|
||||
go func(in <-chan *ssh.Request) {
|
||||
for req := range in {
|
||||
fmt.Fprintf(debugStream, "Request: %v\n", req.Type)
|
||||
ok := false
|
||||
switch req.Type {
|
||||
case "subsystem":
|
||||
fmt.Fprintf(debugStream, "Subsystem: %s\n", req.Payload[4:])
|
||||
if string(req.Payload[4:]) == "sftp" {
|
||||
ok = true
|
||||
}
|
||||
default:
|
||||
ok = false
|
||||
}
|
||||
fmt.Fprintf(debugStream, " - accepted: %v\n", ok)
|
||||
_ = req.Reply(ok, nil)
|
||||
}
|
||||
}(requests)
|
||||
|
||||
serverOptions := []psftp.ServerOption{
|
||||
psftp.WithDebug(debugStream),
|
||||
}
|
||||
|
||||
fmt.Fprintf(debugStream, "Read write server\n")
|
||||
|
||||
server, err := psftp.NewServer(
|
||||
channel,
|
||||
serverOptions...,
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := server.Serve(); err == io.EOF {
|
||||
channel.Close()
|
||||
server.Close()
|
||||
fmt.Fprintln(debugStream, "sftp client exited session.")
|
||||
} else if err != nil {
|
||||
fmt.Fprintf(debugStream, "sftp server completed with error: %s", err)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unable to connect to SSH: %v", err)
|
||||
}
|
||||
|
||||
defer st.Close(ctx)
|
||||
|
||||
if err := st.PutBlob(ctx, t1, []byte{1}); err == nil {
|
||||
t.Errorf("unexpected success when adding to non-existent path")
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoError(t *testing.T, err error) {
|
||||
@@ -156,60 +69,6 @@ func assertNoError(t *testing.T, err error) {
|
||||
}
|
||||
}
|
||||
|
||||
func createConfig(t *testing.T) *ssh.ServerConfig {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Errorf("unable to getwd: %s", err)
|
||||
}
|
||||
|
||||
// Public key authentication is done by comparing
|
||||
// the public key of a received connection
|
||||
// with the entries in the authorized_keys file.
|
||||
authorizedKeysBytes, err := ioutil.ReadFile(filepath.Join(cwd, "known_hosts"))
|
||||
if err != nil {
|
||||
t.Errorf("failed to load authorized_keys, err: %v", err)
|
||||
}
|
||||
|
||||
authorizedKeysMap := map[string]bool{}
|
||||
for len(authorizedKeysBytes) > 0 {
|
||||
pubKey, _, _, rest, e := ssh.ParseAuthorizedKey(authorizedKeysBytes)
|
||||
if e != nil {
|
||||
log.Fatal(e)
|
||||
}
|
||||
|
||||
authorizedKeysMap[string(pubKey.Marshal())] = true
|
||||
authorizedKeysBytes = rest
|
||||
}
|
||||
|
||||
c := &ssh.ServerConfig{
|
||||
PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
|
||||
if authorizedKeysMap[string(pubKey.Marshal())] {
|
||||
return &ssh.Permissions{
|
||||
// Record the public key used for authentication.
|
||||
Extensions: map[string]string{
|
||||
"pubkey-fp": ssh.FingerprintSHA256(pubKey),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unknown public key for %q", c.User())
|
||||
},
|
||||
}
|
||||
|
||||
privateBytes, err := ioutil.ReadFile(filepath.Join(cwd, "id_rsa"))
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load private key: ", err)
|
||||
}
|
||||
|
||||
private, err := ssh.ParsePrivateKey(privateBytes)
|
||||
if err != nil {
|
||||
log.Fatal("Failed to parse private key: ", err)
|
||||
}
|
||||
|
||||
c.AddHostKey(private)
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func deleteBlobs(ctx context.Context, t *testing.T, st blob.Storage) {
|
||||
if err := st.ListBlobs(ctx, "", func(bm blob.Metadata) error {
|
||||
return st.DeleteBlob(ctx, bm.BlobID)
|
||||
@@ -217,3 +76,48 @@ func deleteBlobs(ctx context.Context, t *testing.T, st blob.Storage) {
|
||||
t.Fatalf("unable to clear sftp storage: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func createSFTPStorage(ctx context.Context, additionalPath string, t *testing.T) (blob.Storage, error) {
|
||||
host := os.Getenv("KOPIA_SFTP_TEST_HOST")
|
||||
if host == "" {
|
||||
t.Skip("KOPIA_SFTP_TEST_HOST not provided")
|
||||
}
|
||||
|
||||
envPort := os.Getenv("KOPIA_SFTP_TEST_PORT")
|
||||
if envPort == "" {
|
||||
t.Skip("KOPIA_SFTP_TEST_PORT not provided")
|
||||
}
|
||||
port, err := strconv.ParseInt(envPort, 10, 64)
|
||||
if err != nil {
|
||||
t.Skip("skipping test because port is not numeric")
|
||||
}
|
||||
|
||||
path := os.Getenv("KOPIA_SFTP_TEST_PATH")
|
||||
if path == "" {
|
||||
t.Skip("KOPIA_SFTP_TEST_PATH not provided")
|
||||
}
|
||||
|
||||
keyfile := os.Getenv("KOPIA_SFTP_KEYFILE")
|
||||
if _, err = os.Stat(keyfile); err != nil {
|
||||
t.Skip("skipping test because SFTP keyfile can't be opened")
|
||||
}
|
||||
|
||||
usr := os.Getenv("KOPIA_SFTP_TEST_USER")
|
||||
if usr == "" {
|
||||
t.Skip("KOPIA_SFTP_TEST_USER not provided")
|
||||
}
|
||||
|
||||
knownHosts := os.Getenv("KOPIA_SFTP_KNOWN_HOSTS_FILE")
|
||||
if _, err = os.Stat(knownHosts); err != nil {
|
||||
t.Skip("skipping test because SFTP known hosts file can't be opened")
|
||||
}
|
||||
|
||||
return sftp.New(ctx, &sftp.Options{
|
||||
Path: path + additionalPath,
|
||||
Host: host,
|
||||
Username: usr,
|
||||
Port: int(port),
|
||||
Keyfile: keyfile,
|
||||
KnownHosts: knownHosts,
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user