diff --git a/cli/storage_sftp.go b/cli/storage_sftp.go index 6287e33b1..5e10d024d 100644 --- a/cli/storage_sftp.go +++ b/cli/storage_sftp.go @@ -31,36 +31,43 @@ func(cmd *kingpin.CmdClause) { cmd.Flag("known-hosts", "path to known_hosts file").StringVar(&options.KnownHostsFile) cmd.Flag("known-hosts-data", "known_hosts file entries").StringVar(&options.KnownHostsData) cmd.Flag("embed-credentials", "Embed key and known_hosts in Kopia configuration").BoolVar(&embedCredentials) + + cmd.Flag("external", "Launch external passwordless SSH command").BoolVar(&options.ExternalSSH) + cmd.Flag("ssh-command", "SSH command").Default("ssh").StringVar(&options.SSHCommand) + cmd.Flag("ssh-args", "Arguments to external SSH command").StringVar(&options.SSHArguments) + cmd.Flag("flat", "Use flat directory structure").BoolVar(&connectFlat) }, func(ctx context.Context, isNew bool) (blob.Storage, error) { sftpo := options // nolint:nestif - if embedCredentials { - if sftpo.KeyData == "" { - d, err := ioutil.ReadFile(sftpo.Keyfile) - if err != nil { - return nil, err + if !sftpo.ExternalSSH { + if embedCredentials { + if sftpo.KeyData == "" { + d, err := ioutil.ReadFile(sftpo.Keyfile) + if err != nil { + return nil, err + } + + sftpo.KeyData = string(d) + sftpo.Keyfile = "" } - sftpo.KeyData = string(d) - sftpo.Keyfile = "" - } + if sftpo.KnownHostsData == "" && sftpo.KnownHostsFile != "" { + d, err := ioutil.ReadFile(sftpo.KnownHostsFile) + if err != nil { + return nil, err + } - if sftpo.KnownHostsData == "" && sftpo.KnownHostsFile != "" { - d, err := ioutil.ReadFile(sftpo.KnownHostsFile) - if err != nil { - return nil, err + sftpo.KnownHostsData = string(d) + sftpo.KnownHostsFile = "" } - - sftpo.KnownHostsData = string(d) - sftpo.KnownHostsFile = "" } - } - if sftpo.KeyData == "" && sftpo.Keyfile == "" { - return nil, errors.Errorf("must provide either key file or key data") + if sftpo.KeyData == "" && sftpo.Keyfile == "" { + return nil, errors.Errorf("must provide either key file or key data") + } } if connectFlat { diff --git a/go.mod b/go.mod index d076b1085..dd5c78f49 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.2 // indirect github.com/Azure/azure-storage-blob-go v0.8.0 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect + github.com/alexhunt7/ssher v0.0.0-20190216204854-d36569cf7047 github.com/aws/aws-sdk-go v1.31.3 github.com/bgentry/speakeasy v0.1.0 github.com/chmduquesne/rollinghash v4.0.0+incompatible @@ -23,6 +24,7 @@ require ( github.com/google/uuid v1.1.1 github.com/google/wire v0.4.0 // indirect github.com/gorilla/mux v1.7.4 + github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect github.com/klauspost/compress v1.10.10 github.com/klauspost/pgzip v1.2.4 github.com/kylelemons/godebug v1.1.0 diff --git a/go.sum b/go.sum index 8b3b4da09..5183c09fa 100644 --- a/go.sum +++ b/go.sum @@ -88,6 +88,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1C github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alexhunt7/ssher v0.0.0-20190216204854-d36569cf7047 h1:zEgsMrLxdTDEIfdq6KANRC3NYbSYuKtu+PgSmBqFi/k= +github.com/alexhunt7/ssher v0.0.0-20190216204854-d36569cf7047/go.mod h1:YOadIa0pWX675SPsCaZ27boU9r6NCkmAzw6lc4ZAhK0= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -356,6 +358,9 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -452,6 +457,7 @@ github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA= github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -500,6 +506,7 @@ github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0C github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -641,6 +648,7 @@ gocloud.dev v0.19.0/go.mod h1:SmKwiR8YwIMMJvQBKLsC3fHNyMwXLw3PMDO+VVteJMI= golang.org/x/arch v0.0.0-20190909030613-46d78d1859ac/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190122013713-64072686203f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= @@ -657,6 +665,7 @@ golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -754,6 +763,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190122071731-054c452bb702/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/htmlui/src/SetupSFTP.js b/htmlui/src/SetupSFTP.js index 3caba5692..5fc8affc5 100644 --- a/htmlui/src/SetupSFTP.js +++ b/htmlui/src/SetupSFTP.js @@ -1,6 +1,6 @@ import React, { Component } from 'react'; import Form from 'react-bootstrap/Form'; -import { handleChange, OptionalField, OptionalNumberField, RequiredField, validateRequiredFields, hasExactlyOneOf } from './forms'; +import { handleChange, OptionalField, OptionalNumberField, RequiredField, validateRequiredFields, hasExactlyOneOf, RequiredBoolean } from './forms'; export class SetupSFTP extends Component { constructor(props) { @@ -8,16 +8,25 @@ export class SetupSFTP extends Component { this.state = { port: 22, + validated: false, ...props.initial }; this.handleChange = handleChange.bind(this); } validate() { + this.setState({ + validated: true, + }); + if (!validateRequiredFields(this, ["host", "port", "username", "path"])) { return false; } + if (this.state.externalSSH) { + return true + } + if (!hasExactlyOneOf(this, ["keyfile", "keyData"])) { return false; } @@ -32,13 +41,14 @@ export class SetupSFTP extends Component { render() { return <> - {RequiredField(this, "Host", "host", { autoFocus: true, placeholder: "host name" })} - {OptionalNumberField(this, "Port", "port", { placeholder: "port number (e.g. 22)" })} + {RequiredField(this, "Host", "host", { autoFocus: true, placeholder: "ssh host name (e.g. example.com)" })} {RequiredField(this, "User", "username", { placeholder: "user name" })} + {OptionalNumberField(this, "Port", "port", { placeholder: "port number (e.g. 22)" })} - {RequiredField(this, "Path", "path", { placeholder: "enter remote path" })} + {RequiredField(this, "Path", "path", { placeholder: "enter remote path to repository, e.g. '/mnt/data/repository'" })} + {!this.state.externalSSH && <> {OptionalField(this, "Path to key file", "keyfile", { placeholder: "enter path to the key file" })} {OptionalField(this, "Path to known_hosts File", "knownHostsFile", { placeholder: "enter path to known_hosts file" })} @@ -48,15 +58,22 @@ export class SetupSFTP extends Component { placeholder: "paste contents of the key file", as: "textarea", rows: 5, - isInvalid: !hasExactlyOneOf(this, ["keyfile", "keyData"]), + isInvalid: this.state.validated && !this.state.externalSSH && !hasExactlyOneOf(this, ["keyfile", "keyData"]), }, null, <>Either Key File or Key Data is required, but not both.)} {OptionalField(this, "Known Hosts Data", "knownHostsData", { placeholder: "paste contents of the known_hosts file", as: "textarea", rows: 5, - isInvalid: !hasExactlyOneOf(this, ["knownHostsFile", "knownHostsData"]), + isInvalid: this.state.validated && !this.state.externalSSH && !hasExactlyOneOf(this, ["knownHostsFile", "knownHostsData"]), }, null, <>Either Known Hosts File or Known Hosts Data is required, but not both.)} + } + {RequiredBoolean(this, "Launch external password-less SSH command", "externalSSH", "By default Kopia connects to the server using internal SSH client which supports limited options. Alternatively it may launch external password-less SSH command, which supports additional options.")} + {this.state.externalSSH && <> + {OptionalField(this, "SSH Command", "sshCommand", { placeholder: "provide enter passwordless SSH command to execute (typically 'ssh')" })} + {OptionalField(this, "SSH Arguments", "sshArguments", { placeholder: "enter SSH command arguments ('user@host -s sftp' will be appended automatically)" })} + } + ; } } diff --git a/htmlui/src/tests/SetupSFTP.test.js b/htmlui/src/tests/SetupSFTP.test.js index dbd918b50..a1c04a150 100644 --- a/htmlui/src/tests/SetupSFTP.test.js +++ b/htmlui/src/tests/SetupSFTP.test.js @@ -23,6 +23,7 @@ it('can set fields', async () => { "knownHostsFile": "some-knownHostsFile", "path": "some-path", "port": 22, + "validated": true, }); // now enter key data instead of key file, make sure validation triggers along the way @@ -44,5 +45,6 @@ it('can set fields', async () => { "knownHostsData": "some-knownHostsData", "path": "some-path", "port": 22, + "validated": true, }); }); diff --git a/repo/blob/sftp/sftp_options.go b/repo/blob/sftp/sftp_options.go index ce04da1c6..7abbcd327 100644 --- a/repo/blob/sftp/sftp_options.go +++ b/repo/blob/sftp/sftp_options.go @@ -17,6 +17,10 @@ type Options struct { KnownHostsFile string `json:"knownHostsFile,omitempty"` KnownHostsData string `json:"knownHostsData,omitempty"` + ExternalSSH bool `json:"externalSSH"` + SSHCommand string `json:"sshCommand,omitempty"` // default "ssh" + SSHArguments string `json:"sshArguments,omitempty"` + DirectoryShards []int `json:"dirShards"` } diff --git a/repo/blob/sftp/sftp_storage.go b/repo/blob/sftp/sftp_storage.go index 06edfe64e..880460f5f 100644 --- a/repo/blob/sftp/sftp_storage.go +++ b/repo/blob/sftp/sftp_storage.go @@ -2,26 +2,29 @@ package sftp import ( - "bufio" - "bytes" "context" "crypto/rand" "fmt" "io" "io/ioutil" "os" + "os/exec" "path" "strings" "time" "github.com/pkg/errors" - psftp "github.com/pkg/sftp" + "github.com/pkg/sftp" "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/sharded" + "github.com/kopia/kopia/repo/logging" ) +var log = logging.GetContextLoggerFunc("sftp") + const ( sftpStorageType = "sftp" fsStorageChunkSuffix = ".f" @@ -31,6 +34,8 @@ var sftpDefaultShards = []int{3, 3} +type closeFunc func() error + // sftpStorage implements blob.Storage on top of sftp. type sftpStorage struct { sharded.Storage @@ -39,8 +44,8 @@ type sftpStorage struct { type sftpImpl struct { Options - conn *ssh.Client - cli *psftp.Client + closeFunc func() error + cli *sftp.Client } func (s *sftpImpl) GetBlobFromPath(ctx context.Context, dirPath, fullPath string, offset, length int64) ([]byte, error) { @@ -54,29 +59,28 @@ func (s *sftpImpl) GetBlobFromPath(ctx context.Context, dirPath, fullPath string } defer r.Close() //nolint:errcheck - // pkg/sftp doesn't have a `ioutil.Readall`, so we WriteTo to a buffer - // and either return it all or return the offset/length bytes - buf := new(bytes.Buffer) - - n, err := r.WriteTo(buf) - if err != nil { - return nil, err - } - if length < 0 { - return buf.Bytes(), nil + // read entire blob + return ioutil.ReadAll(r) } - if offset > n || offset < 0 { - return nil, errors.New("invalid offset") + // parial read, seek to the provided offset and read given number of bytes. + if _, err = r.Seek(offset, io.SeekStart); err != nil { + return nil, errors.Wrap(err, "seek error") } - data := buf.Bytes()[offset:] - if int(length) > len(data) { - return nil, errors.New("invalid length") + b := make([]byte, length) + + n, err := r.Read(b) + if err != nil { + return nil, errors.Wrap(err, "read error") } - return data[0:length], nil + if n != len(b) { + return nil, errors.Errorf("truncated read") + } + + return b, nil } func (s *sftpImpl) GetMetadataFromPath(ctx context.Context, dirPath, fullPath string) (blob.Metadata, error) { @@ -140,7 +144,7 @@ func (s *sftpImpl) SetTimeInPath(ctx context.Context, dirPath, fullPath string, return s.cli.Chtimes(fullPath, n, n) } -func (s *sftpImpl) createTempFileAndDir(tempFile string) (*psftp.File, error) { +func (s *sftpImpl) createTempFileAndDir(tempFile string) (*sftp.File, error) { flags := os.O_CREATE | os.O_WRONLY | os.O_EXCL f, err := s.cli.OpenFile(tempFile, flags) @@ -161,6 +165,10 @@ func isNotExist(err error) bool { return false } + if errors.Is(err, os.ErrNotExist) { + return true + } + return strings.Contains(err.Error(), "does not exist") } @@ -194,66 +202,46 @@ func (s *sftpStorage) Close(ctx context.Context) error { return errors.Wrap(err, "closing SFTP client") } - if err := s.Impl.(*sftpImpl).conn.Close(); err != nil { + if err := s.Impl.(*sftpImpl).closeFunc(); err != nil { return errors.Wrap(err, "closing SFTP connection") } return nil } -// example host strings: [localhost]:2222, [xyz.example.com], [192.168.1.1]:2210, 192.168.1.1. -func cleanup(host string) string { - if index := strings.Index(host, ":"); index > 0 { - host = host[:index] +func writeKnownHostsDataStringToTempFile(data string) (string, error) { + tf, err := ioutil.TempFile("", "kopia-known-hosts") + if err != nil { + return "", err } - host = strings.ReplaceAll(host, "[", "") - host = strings.ReplaceAll(host, "]", "") + defer tf.Close() //nolint:errcheck,gosec - return host -} - -// given a list of hosts from a known_hosts entry, determine if the host is referenced. -func hostExists(host string, hosts []string) bool { - for _, entry := range hosts { - if host == cleanup(entry) { - return true - } + if _, err := io.WriteString(tf, data); err != nil { + return "", errors.Wrap(err, "error writing temporary file") } - return false + return tf.Name(), nil } -// getHostKey parses OpenSSH known_hosts file for a public key that matches the host -// The known_hosts file format is documented in the sshd(8) manual page. -func getHostKey(opt *Options) (ssh.PublicKey, error) { - var reader io.Reader - +// getHostKeyCallback returns a HostKeyCallback that validates the connected host based on KnownHostsFile or KnownHostsData. +func getHostKeyCallback(opt *Options) (ssh.HostKeyCallback, error) { if opt.KnownHostsData != "" { - reader = strings.NewReader(opt.KnownHostsData) - } else { - file, err := os.Open(opt.knownHostsFile()) + // if known hosts data is provided, it takes precedence of KnownHostsFile + // We need to write to temporary file so we can parse, unfortunately knownhosts.New() only accepts + // file names, but known_hosts data is not really sensitive so it can be briefly written to disk. + tmpFile, err := writeKnownHostsDataStringToTempFile(opt.KnownHostsData) if err != nil { return nil, err } - defer file.Close() //nolint:errcheck,gosec - reader = file + // this file is no longer needed after `knownhosts.New` returns, so we can delete it. + defer os.Remove(tmpFile) // nolint:errcheck + + return knownhosts.New(tmpFile) } - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - _, hosts, hostKey, _, _, err := ssh.ParseKnownHosts(scanner.Bytes()) - if err != nil { - return nil, errors.Wrapf(err, "error parsing %s", scanner.Text()) - } - - if hostExists(opt.Host, hosts) { - return hostKey, nil - } - } - - return nil, errors.Errorf("no hostkey found for %s", opt.Host) + return knownhosts.New(opt.knownHostsFile()) } // getSigner parses and returns a signer for the user-entered private key. @@ -283,8 +271,10 @@ func getSigner(opts *Options) (ssh.Signer, error) { return key, nil } -func createSSHConfig(opts *Options) (*ssh.ClientConfig, error) { - hostKey, err := getHostKey(opts) +func createSSHConfig(ctx context.Context, opts *Options) (*ssh.ClientConfig, error) { + log(ctx).Debugf("using built-in SSH connection") + + hostKeyCallback, err := getHostKeyCallback(opts) if err != nil { return nil, errors.Wrapf(err, "unable to getHostKey: %s", opts.Host) } @@ -294,32 +284,104 @@ func createSSHConfig(opts *Options) (*ssh.ClientConfig, error) { return nil, errors.Wrapf(err, "unable to getSigner") } - config := &ssh.ClientConfig{ + return &ssh.ClientConfig{ User: opts.Username, Auth: []ssh.AuthMethod{ ssh.PublicKeys(signer), }, - HostKeyCallback: ssh.FixedHostKey(hostKey), + HostKeyCallback: hostKeyCallback, + }, nil +} + +func getSFTPClientExternal(ctx context.Context, opt *Options) (*sftp.Client, closeFunc, error) { + var cmdArgs []string + + if opt.SSHArguments != "" { + cmdArgs = append(cmdArgs, strings.Split(opt.SSHArguments, " ")...) } - return config, nil + cmdArgs = append( + cmdArgs, + opt.Username+"@"+opt.Host, + "-s", "sftp", + ) + + sshCommand := opt.SSHCommand + if sshCommand == "" { + sshCommand = "ssh" + } + + log(ctx).Debugf("launching external SSH process %v %v", sshCommand, strings.Join(cmdArgs, " ")) + + cmd := exec.Command(sshCommand, cmdArgs...) //nolint:gosec + + // send errors from ssh to stderr + cmd.Stderr = os.Stderr + + // get stdin and stdout + wr, err := cmd.StdinPipe() + if err != nil { + return nil, nil, err + } + + rd, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + + if err = cmd.Start(); err != nil { + return nil, nil, errors.Wrap(err, "error starting SSH") + } + + closeFunc := func() error { + p := cmd.Process + if p != nil { + p.Kill() // nolint:errcheck + } + + return nil + } + + // open the SFTP session + c, err := sftp.NewClientPipe(rd, wr) + if err != nil { + closeFunc() // nolint:errcheck + + return nil, nil, err + } + + return c, closeFunc, nil +} + +func getSFTPClient(ctx context.Context, opt *Options) (*sftp.Client, closeFunc, error) { + if opt.ExternalSSH { + return getSFTPClientExternal(ctx, opt) + } + + config, err := createSSHConfig(ctx, opt) + if err != nil { + return nil, nil, err + } + + addr := fmt.Sprintf("%s:%d", opt.Host, opt.Port) + + conn, err := ssh.Dial("tcp", addr, config) + if err != nil { + return nil, nil, errors.Wrapf(err, "unable to dial [%s]: %+v", addr, config) + } + + c, err := sftp.NewClient(conn, sftp.MaxPacket(packetSize)) + if err != nil { + conn.Close() // nolint:errcheck + return nil, nil, errors.Wrapf(err, "unable to create sftp client") + } + + return c, conn.Close, nil } // New creates new ssh-backed storage in a specified host. func New(ctx context.Context, opts *Options) (blob.Storage, error) { - config, err := createSSHConfig(opts) - if err != nil { - return nil, err - } - - addr := fmt.Sprintf("%s:%d", opts.Host, opts.Port) - - conn, err := ssh.Dial("tcp", addr, config) - if err != nil { - return nil, errors.Wrapf(err, "unable to dial [%s]: %+v", addr, config) - } - - c, err := psftp.NewClient(conn, psftp.MaxPacket(packetSize)) + c, closeFunc, err := getSFTPClient(ctx, opts) if err != nil { return nil, errors.Wrapf(err, "unable to create sftp client") } @@ -337,9 +399,9 @@ func New(ctx context.Context, opts *Options) (blob.Storage, error) { r := &sftpStorage{ sharded.Storage{ Impl: &sftpImpl{ - Options: *opts, - conn: conn, - cli: c, + Options: *opts, + cli: c, + closeFunc: closeFunc, }, RootPath: opts.Path, Suffix: fsStorageChunkSuffix, diff --git a/site/content/docs/Repositories/_index.md b/site/content/docs/Repositories/_index.md index 6e8bc0e07..1aafab4f8 100644 --- a/site/content/docs/Repositories/_index.md +++ b/site/content/docs/Repositories/_index.md @@ -111,6 +111,66 @@ $ kopia repository connect b2 --- + +## SFTP + +The `SFTP` provider can be used to connect to a file server over SFTP/SSH protocol. + +You must first configure passwordless SFTP login by following [these instructions](https://www.redhat.com/sysadmin/passwordless-ssh). Make sure to choose empty passphrase, because Kopia does not support password prompts on each use. + +If everything is configured correctly, you should be able to connect to your SFTP server without any password by using: + +``` +$ sftp some-user@my-server +Connected to my-server. +sftp> +``` + + +### Creating a repository + +Assiuming passwordless connection worked, you can now create SFTP repository. Assuming you want the files to be stored under `/remote/path`, run the following command: + +```shell +$ kopia repository create sftp \ + --host my-server \ + --username some-user \ + --keyfile ~/.ssh/id_rsa \ + --known-hosts ~/.ssh/known_hosts \ + --path /remote/path +``` + +(adjust paths to the key file and known hosts file as necessary). + +When prompted, enter Kopia password to encrypt the repository contents. + +If everything is done correctly, you should be able to verify that SFTP server indeed has Kopia files in the provided location, including special file named `kopia.repository.f`: + +``` +$ sftp some-user@my-server +sftp> ls -al /remote/path +-rw-r--r-- 1 some-user some-user 661 Sep 18 16:12 kopia.repository.f +``` + +### Connecting To Repository + +To connect to an existing SFTP repository, simply use `connect` instead of `create`: + +```shell +$ kopia repository connect sftp \ + --host my-server \ + --username some-user \ + --keyfile ~/.ssh/id_rsa \ + --known-hosts ~/.ssh/known_hosts \ + --path /remote/path +``` + +If the connection to SFTP server does not work, try adding `--external` which will launch external `ssh` process, which supports more connectivity options which may be needed for some hosts. + +[Detailed information and settings](/docs/reference/command-line/common/repository-connect-sftp/) + +--- + ## Rclone Kopia can connect to certain backends supported by [Rclone](https://rclone.org) as long as they support