mirror of
https://github.com/kopia/kopia.git
synced 2026-04-20 14:08:32 -04:00
SFTP connectivity and docs improvements (#623)
* sftp: support for external SSH command and host verfication improvements - removed custom parsing of hostnames and verification and replaced with standard 'knownhosts' implementation. - added option to launch external SSH command which supports aliases, agent, etc. NOTE, we're still not supporting any cases where password needs to be entered on the command line, since that would be incompatible with the UI which uses client-server model. Fixes #500 Fixes #414 * site: updated SFTP repository connection instructions Fixes #590
This commit is contained in:
Jarek Kowalski
@@ -31,36 +31,43 @@ func(cmd *kingpin.CmdClause) {
|
||||
cmd.Flag("known-hosts", "path to known_hosts file").StringVar(&options.KnownHostsFile)
|
||||
cmd.Flag("known-hosts-data", "known_hosts file entries").StringVar(&options.KnownHostsData)
|
||||
cmd.Flag("embed-credentials", "Embed key and known_hosts in Kopia configuration").BoolVar(&embedCredentials)
|
||||
|
||||
cmd.Flag("external", "Launch external passwordless SSH command").BoolVar(&options.ExternalSSH)
|
||||
cmd.Flag("ssh-command", "SSH command").Default("ssh").StringVar(&options.SSHCommand)
|
||||
cmd.Flag("ssh-args", "Arguments to external SSH command").StringVar(&options.SSHArguments)
|
||||
|
||||
cmd.Flag("flat", "Use flat directory structure").BoolVar(&connectFlat)
|
||||
},
|
||||
func(ctx context.Context, isNew bool) (blob.Storage, error) {
|
||||
sftpo := options
|
||||
|
||||
// nolint:nestif
|
||||
if embedCredentials {
|
||||
if sftpo.KeyData == "" {
|
||||
d, err := ioutil.ReadFile(sftpo.Keyfile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if !sftpo.ExternalSSH {
|
||||
if embedCredentials {
|
||||
if sftpo.KeyData == "" {
|
||||
d, err := ioutil.ReadFile(sftpo.Keyfile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sftpo.KeyData = string(d)
|
||||
sftpo.Keyfile = ""
|
||||
}
|
||||
|
||||
sftpo.KeyData = string(d)
|
||||
sftpo.Keyfile = ""
|
||||
}
|
||||
if sftpo.KnownHostsData == "" && sftpo.KnownHostsFile != "" {
|
||||
d, err := ioutil.ReadFile(sftpo.KnownHostsFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if sftpo.KnownHostsData == "" && sftpo.KnownHostsFile != "" {
|
||||
d, err := ioutil.ReadFile(sftpo.KnownHostsFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
sftpo.KnownHostsData = string(d)
|
||||
sftpo.KnownHostsFile = ""
|
||||
}
|
||||
|
||||
sftpo.KnownHostsData = string(d)
|
||||
sftpo.KnownHostsFile = ""
|
||||
}
|
||||
}
|
||||
|
||||
if sftpo.KeyData == "" && sftpo.Keyfile == "" {
|
||||
return nil, errors.Errorf("must provide either key file or key data")
|
||||
if sftpo.KeyData == "" && sftpo.Keyfile == "" {
|
||||
return nil, errors.Errorf("must provide either key file or key data")
|
||||
}
|
||||
}
|
||||
|
||||
if connectFlat {
|
||||
|
||||
2
go.mod
2
go.mod
@@ -9,6 +9,7 @@ require (
|
||||
github.com/Azure/azure-pipeline-go v0.2.2 // indirect
|
||||
github.com/Azure/azure-storage-blob-go v0.8.0
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
|
||||
github.com/alexhunt7/ssher v0.0.0-20190216204854-d36569cf7047
|
||||
github.com/aws/aws-sdk-go v1.31.3
|
||||
github.com/bgentry/speakeasy v0.1.0
|
||||
github.com/chmduquesne/rollinghash v4.0.0+incompatible
|
||||
@@ -23,6 +24,7 @@ require (
|
||||
github.com/google/uuid v1.1.1
|
||||
github.com/google/wire v0.4.0 // indirect
|
||||
github.com/gorilla/mux v1.7.4
|
||||
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect
|
||||
github.com/klauspost/compress v1.10.10
|
||||
github.com/klauspost/pgzip v1.2.4
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
|
||||
10
go.sum
10
go.sum
@@ -88,6 +88,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1C
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/alexhunt7/ssher v0.0.0-20190216204854-d36569cf7047 h1:zEgsMrLxdTDEIfdq6KANRC3NYbSYuKtu+PgSmBqFi/k=
|
||||
github.com/alexhunt7/ssher v0.0.0-20190216204854-d36569cf7047/go.mod h1:YOadIa0pWX675SPsCaZ27boU9r6NCkmAzw6lc4ZAhK0=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
@@ -356,6 +358,9 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
@@ -452,6 +457,7 @@ github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA=
|
||||
github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
|
||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
|
||||
@@ -500,6 +506,7 @@ github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0C
|
||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
|
||||
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
|
||||
github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
|
||||
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
@@ -641,6 +648,7 @@ gocloud.dev v0.19.0/go.mod h1:SmKwiR8YwIMMJvQBKLsC3fHNyMwXLw3PMDO+VVteJMI=
|
||||
golang.org/x/arch v0.0.0-20190909030613-46d78d1859ac/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190122013713-64072686203f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
|
||||
@@ -657,6 +665,7 @@ golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPh
|
||||
golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw=
|
||||
golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig=
|
||||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -754,6 +763,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190122071731-054c452bb702/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import React, { Component } from 'react';
|
||||
import Form from 'react-bootstrap/Form';
|
||||
import { handleChange, OptionalField, OptionalNumberField, RequiredField, validateRequiredFields, hasExactlyOneOf } from './forms';
|
||||
import { handleChange, OptionalField, OptionalNumberField, RequiredField, validateRequiredFields, hasExactlyOneOf, RequiredBoolean } from './forms';
|
||||
|
||||
export class SetupSFTP extends Component {
|
||||
constructor(props) {
|
||||
@@ -8,16 +8,25 @@ export class SetupSFTP extends Component {
|
||||
|
||||
this.state = {
|
||||
port: 22,
|
||||
validated: false,
|
||||
...props.initial
|
||||
};
|
||||
this.handleChange = handleChange.bind(this);
|
||||
}
|
||||
|
||||
validate() {
|
||||
this.setState({
|
||||
validated: true,
|
||||
});
|
||||
|
||||
if (!validateRequiredFields(this, ["host", "port", "username", "path"])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (this.state.externalSSH) {
|
||||
return true
|
||||
}
|
||||
|
||||
if (!hasExactlyOneOf(this, ["keyfile", "keyData"])) {
|
||||
return false;
|
||||
}
|
||||
@@ -32,13 +41,14 @@ export class SetupSFTP extends Component {
|
||||
render() {
|
||||
return <>
|
||||
<Form.Row>
|
||||
{RequiredField(this, "Host", "host", { autoFocus: true, placeholder: "host name" })}
|
||||
{OptionalNumberField(this, "Port", "port", { placeholder: "port number (e.g. 22)" })}
|
||||
{RequiredField(this, "Host", "host", { autoFocus: true, placeholder: "ssh host name (e.g. example.com)" })}
|
||||
{RequiredField(this, "User", "username", { placeholder: "user name" })}
|
||||
{OptionalNumberField(this, "Port", "port", { placeholder: "port number (e.g. 22)" })}
|
||||
</Form.Row>
|
||||
<Form.Row>
|
||||
{RequiredField(this, "Path", "path", { placeholder: "enter remote path" })}
|
||||
{RequiredField(this, "Path", "path", { placeholder: "enter remote path to repository, e.g. '/mnt/data/repository'" })}
|
||||
</Form.Row>
|
||||
{!this.state.externalSSH && <>
|
||||
<Form.Row>
|
||||
{OptionalField(this, "Path to key file", "keyfile", { placeholder: "enter path to the key file" })}
|
||||
{OptionalField(this, "Path to known_hosts File", "knownHostsFile", { placeholder: "enter path to known_hosts file" })}
|
||||
@@ -48,15 +58,22 @@ export class SetupSFTP extends Component {
|
||||
placeholder: "paste contents of the key file",
|
||||
as: "textarea",
|
||||
rows: 5,
|
||||
isInvalid: !hasExactlyOneOf(this, ["keyfile", "keyData"]),
|
||||
isInvalid: this.state.validated && !this.state.externalSSH && !hasExactlyOneOf(this, ["keyfile", "keyData"]),
|
||||
}, null, <>Either <b>Key File</b> or <b>Key Data</b> is required, but not both.</>)}
|
||||
{OptionalField(this, "Known Hosts Data", "knownHostsData", {
|
||||
placeholder: "paste contents of the known_hosts file",
|
||||
as: "textarea",
|
||||
rows: 5,
|
||||
isInvalid: !hasExactlyOneOf(this, ["knownHostsFile", "knownHostsData"]),
|
||||
isInvalid: this.state.validated && !this.state.externalSSH && !hasExactlyOneOf(this, ["knownHostsFile", "knownHostsData"]),
|
||||
}, null, <>Either <b>Known Hosts File</b> or <b>Known Hosts Data</b> is required, but not both.</>)}
|
||||
</Form.Row>
|
||||
</>}
|
||||
{RequiredBoolean(this, "Launch external password-less SSH command", "externalSSH", "By default Kopia connects to the server using internal SSH client which supports limited options. Alternatively it may launch external password-less SSH command, which supports additional options.")}
|
||||
{this.state.externalSSH && <><Form.Row>
|
||||
{OptionalField(this, "SSH Command", "sshCommand", { placeholder: "provide enter passwordless SSH command to execute (typically 'ssh')" })}
|
||||
{OptionalField(this, "SSH Arguments", "sshArguments", { placeholder: "enter SSH command arguments ('user@host -s sftp' will be appended automatically)" })}
|
||||
</Form.Row></>}
|
||||
|
||||
</>;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ it('can set fields', async () => {
|
||||
"knownHostsFile": "some-knownHostsFile",
|
||||
"path": "some-path",
|
||||
"port": 22,
|
||||
"validated": true,
|
||||
});
|
||||
|
||||
// now enter key data instead of key file, make sure validation triggers along the way
|
||||
@@ -44,5 +45,6 @@ it('can set fields', async () => {
|
||||
"knownHostsData": "some-knownHostsData",
|
||||
"path": "some-path",
|
||||
"port": 22,
|
||||
"validated": true,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -17,6 +17,10 @@ type Options struct {
|
||||
KnownHostsFile string `json:"knownHostsFile,omitempty"`
|
||||
KnownHostsData string `json:"knownHostsData,omitempty"`
|
||||
|
||||
ExternalSSH bool `json:"externalSSH"`
|
||||
SSHCommand string `json:"sshCommand,omitempty"` // default "ssh"
|
||||
SSHArguments string `json:"sshArguments,omitempty"`
|
||||
|
||||
DirectoryShards []int `json:"dirShards"`
|
||||
}
|
||||
|
||||
|
||||
@@ -2,26 +2,29 @@
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
psftp "github.com/pkg/sftp"
|
||||
"github.com/pkg/sftp"
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
|
||||
"github.com/kopia/kopia/repo/blob"
|
||||
"github.com/kopia/kopia/repo/blob/sharded"
|
||||
"github.com/kopia/kopia/repo/logging"
|
||||
)
|
||||
|
||||
var log = logging.GetContextLoggerFunc("sftp")
|
||||
|
||||
const (
|
||||
sftpStorageType = "sftp"
|
||||
fsStorageChunkSuffix = ".f"
|
||||
@@ -31,6 +34,8 @@
|
||||
|
||||
var sftpDefaultShards = []int{3, 3}
|
||||
|
||||
type closeFunc func() error
|
||||
|
||||
// sftpStorage implements blob.Storage on top of sftp.
|
||||
type sftpStorage struct {
|
||||
sharded.Storage
|
||||
@@ -39,8 +44,8 @@ type sftpStorage struct {
|
||||
type sftpImpl struct {
|
||||
Options
|
||||
|
||||
conn *ssh.Client
|
||||
cli *psftp.Client
|
||||
closeFunc func() error
|
||||
cli *sftp.Client
|
||||
}
|
||||
|
||||
func (s *sftpImpl) GetBlobFromPath(ctx context.Context, dirPath, fullPath string, offset, length int64) ([]byte, error) {
|
||||
@@ -54,29 +59,28 @@ func (s *sftpImpl) GetBlobFromPath(ctx context.Context, dirPath, fullPath string
|
||||
}
|
||||
defer r.Close() //nolint:errcheck
|
||||
|
||||
// pkg/sftp doesn't have a `ioutil.Readall`, so we WriteTo to a buffer
|
||||
// and either return it all or return the offset/length bytes
|
||||
buf := new(bytes.Buffer)
|
||||
|
||||
n, err := r.WriteTo(buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if length < 0 {
|
||||
return buf.Bytes(), nil
|
||||
// read entire blob
|
||||
return ioutil.ReadAll(r)
|
||||
}
|
||||
|
||||
if offset > n || offset < 0 {
|
||||
return nil, errors.New("invalid offset")
|
||||
// parial read, seek to the provided offset and read given number of bytes.
|
||||
if _, err = r.Seek(offset, io.SeekStart); err != nil {
|
||||
return nil, errors.Wrap(err, "seek error")
|
||||
}
|
||||
|
||||
data := buf.Bytes()[offset:]
|
||||
if int(length) > len(data) {
|
||||
return nil, errors.New("invalid length")
|
||||
b := make([]byte, length)
|
||||
|
||||
n, err := r.Read(b)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "read error")
|
||||
}
|
||||
|
||||
return data[0:length], nil
|
||||
if n != len(b) {
|
||||
return nil, errors.Errorf("truncated read")
|
||||
}
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (s *sftpImpl) GetMetadataFromPath(ctx context.Context, dirPath, fullPath string) (blob.Metadata, error) {
|
||||
@@ -140,7 +144,7 @@ func (s *sftpImpl) SetTimeInPath(ctx context.Context, dirPath, fullPath string,
|
||||
return s.cli.Chtimes(fullPath, n, n)
|
||||
}
|
||||
|
||||
func (s *sftpImpl) createTempFileAndDir(tempFile string) (*psftp.File, error) {
|
||||
func (s *sftpImpl) createTempFileAndDir(tempFile string) (*sftp.File, error) {
|
||||
flags := os.O_CREATE | os.O_WRONLY | os.O_EXCL
|
||||
|
||||
f, err := s.cli.OpenFile(tempFile, flags)
|
||||
@@ -161,6 +165,10 @@ func isNotExist(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return true
|
||||
}
|
||||
|
||||
return strings.Contains(err.Error(), "does not exist")
|
||||
}
|
||||
|
||||
@@ -194,66 +202,46 @@ func (s *sftpStorage) Close(ctx context.Context) error {
|
||||
return errors.Wrap(err, "closing SFTP client")
|
||||
}
|
||||
|
||||
if err := s.Impl.(*sftpImpl).conn.Close(); err != nil {
|
||||
if err := s.Impl.(*sftpImpl).closeFunc(); err != nil {
|
||||
return errors.Wrap(err, "closing SFTP connection")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// example host strings: [localhost]:2222, [xyz.example.com], [192.168.1.1]:2210, 192.168.1.1.
|
||||
func cleanup(host string) string {
|
||||
if index := strings.Index(host, ":"); index > 0 {
|
||||
host = host[:index]
|
||||
func writeKnownHostsDataStringToTempFile(data string) (string, error) {
|
||||
tf, err := ioutil.TempFile("", "kopia-known-hosts")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
host = strings.ReplaceAll(host, "[", "")
|
||||
host = strings.ReplaceAll(host, "]", "")
|
||||
defer tf.Close() //nolint:errcheck,gosec
|
||||
|
||||
return host
|
||||
}
|
||||
|
||||
// given a list of hosts from a known_hosts entry, determine if the host is referenced.
|
||||
func hostExists(host string, hosts []string) bool {
|
||||
for _, entry := range hosts {
|
||||
if host == cleanup(entry) {
|
||||
return true
|
||||
}
|
||||
if _, err := io.WriteString(tf, data); err != nil {
|
||||
return "", errors.Wrap(err, "error writing temporary file")
|
||||
}
|
||||
|
||||
return false
|
||||
return tf.Name(), nil
|
||||
}
|
||||
|
||||
// getHostKey parses OpenSSH known_hosts file for a public key that matches the host
|
||||
// The known_hosts file format is documented in the sshd(8) manual page.
|
||||
func getHostKey(opt *Options) (ssh.PublicKey, error) {
|
||||
var reader io.Reader
|
||||
|
||||
// getHostKeyCallback returns a HostKeyCallback that validates the connected host based on KnownHostsFile or KnownHostsData.
|
||||
func getHostKeyCallback(opt *Options) (ssh.HostKeyCallback, error) {
|
||||
if opt.KnownHostsData != "" {
|
||||
reader = strings.NewReader(opt.KnownHostsData)
|
||||
} else {
|
||||
file, err := os.Open(opt.knownHostsFile())
|
||||
// if known hosts data is provided, it takes precedence of KnownHostsFile
|
||||
// We need to write to temporary file so we can parse, unfortunately knownhosts.New() only accepts
|
||||
// file names, but known_hosts data is not really sensitive so it can be briefly written to disk.
|
||||
tmpFile, err := writeKnownHostsDataStringToTempFile(opt.KnownHostsData)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close() //nolint:errcheck,gosec
|
||||
|
||||
reader = file
|
||||
// this file is no longer needed after `knownhosts.New` returns, so we can delete it.
|
||||
defer os.Remove(tmpFile) // nolint:errcheck
|
||||
|
||||
return knownhosts.New(tmpFile)
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(reader)
|
||||
for scanner.Scan() {
|
||||
_, hosts, hostKey, _, _, err := ssh.ParseKnownHosts(scanner.Bytes())
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "error parsing %s", scanner.Text())
|
||||
}
|
||||
|
||||
if hostExists(opt.Host, hosts) {
|
||||
return hostKey, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.Errorf("no hostkey found for %s", opt.Host)
|
||||
return knownhosts.New(opt.knownHostsFile())
|
||||
}
|
||||
|
||||
// getSigner parses and returns a signer for the user-entered private key.
|
||||
@@ -283,8 +271,10 @@ func getSigner(opts *Options) (ssh.Signer, error) {
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func createSSHConfig(opts *Options) (*ssh.ClientConfig, error) {
|
||||
hostKey, err := getHostKey(opts)
|
||||
func createSSHConfig(ctx context.Context, opts *Options) (*ssh.ClientConfig, error) {
|
||||
log(ctx).Debugf("using built-in SSH connection")
|
||||
|
||||
hostKeyCallback, err := getHostKeyCallback(opts)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to getHostKey: %s", opts.Host)
|
||||
}
|
||||
@@ -294,32 +284,104 @@ func createSSHConfig(opts *Options) (*ssh.ClientConfig, error) {
|
||||
return nil, errors.Wrapf(err, "unable to getSigner")
|
||||
}
|
||||
|
||||
config := &ssh.ClientConfig{
|
||||
return &ssh.ClientConfig{
|
||||
User: opts.Username,
|
||||
Auth: []ssh.AuthMethod{
|
||||
ssh.PublicKeys(signer),
|
||||
},
|
||||
HostKeyCallback: ssh.FixedHostKey(hostKey),
|
||||
HostKeyCallback: hostKeyCallback,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func getSFTPClientExternal(ctx context.Context, opt *Options) (*sftp.Client, closeFunc, error) {
|
||||
var cmdArgs []string
|
||||
|
||||
if opt.SSHArguments != "" {
|
||||
cmdArgs = append(cmdArgs, strings.Split(opt.SSHArguments, " ")...)
|
||||
}
|
||||
|
||||
return config, nil
|
||||
cmdArgs = append(
|
||||
cmdArgs,
|
||||
opt.Username+"@"+opt.Host,
|
||||
"-s", "sftp",
|
||||
)
|
||||
|
||||
sshCommand := opt.SSHCommand
|
||||
if sshCommand == "" {
|
||||
sshCommand = "ssh"
|
||||
}
|
||||
|
||||
log(ctx).Debugf("launching external SSH process %v %v", sshCommand, strings.Join(cmdArgs, " "))
|
||||
|
||||
cmd := exec.Command(sshCommand, cmdArgs...) //nolint:gosec
|
||||
|
||||
// send errors from ssh to stderr
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
// get stdin and stdout
|
||||
wr, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
rd, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if err = cmd.Start(); err != nil {
|
||||
return nil, nil, errors.Wrap(err, "error starting SSH")
|
||||
}
|
||||
|
||||
closeFunc := func() error {
|
||||
p := cmd.Process
|
||||
if p != nil {
|
||||
p.Kill() // nolint:errcheck
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// open the SFTP session
|
||||
c, err := sftp.NewClientPipe(rd, wr)
|
||||
if err != nil {
|
||||
closeFunc() // nolint:errcheck
|
||||
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return c, closeFunc, nil
|
||||
}
|
||||
|
||||
func getSFTPClient(ctx context.Context, opt *Options) (*sftp.Client, closeFunc, error) {
|
||||
if opt.ExternalSSH {
|
||||
return getSFTPClientExternal(ctx, opt)
|
||||
}
|
||||
|
||||
config, err := createSSHConfig(ctx, opt)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
addr := fmt.Sprintf("%s:%d", opt.Host, opt.Port)
|
||||
|
||||
conn, err := ssh.Dial("tcp", addr, config)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrapf(err, "unable to dial [%s]: %+v", addr, config)
|
||||
}
|
||||
|
||||
c, err := sftp.NewClient(conn, sftp.MaxPacket(packetSize))
|
||||
if err != nil {
|
||||
conn.Close() // nolint:errcheck
|
||||
return nil, nil, errors.Wrapf(err, "unable to create sftp client")
|
||||
}
|
||||
|
||||
return c, conn.Close, nil
|
||||
}
|
||||
|
||||
// New creates new ssh-backed storage in a specified host.
|
||||
func New(ctx context.Context, opts *Options) (blob.Storage, error) {
|
||||
config, err := createSSHConfig(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
addr := fmt.Sprintf("%s:%d", opts.Host, opts.Port)
|
||||
|
||||
conn, err := ssh.Dial("tcp", addr, config)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to dial [%s]: %+v", addr, config)
|
||||
}
|
||||
|
||||
c, err := psftp.NewClient(conn, psftp.MaxPacket(packetSize))
|
||||
c, closeFunc, err := getSFTPClient(ctx, opts)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to create sftp client")
|
||||
}
|
||||
@@ -337,9 +399,9 @@ func New(ctx context.Context, opts *Options) (blob.Storage, error) {
|
||||
r := &sftpStorage{
|
||||
sharded.Storage{
|
||||
Impl: &sftpImpl{
|
||||
Options: *opts,
|
||||
conn: conn,
|
||||
cli: c,
|
||||
Options: *opts,
|
||||
cli: c,
|
||||
closeFunc: closeFunc,
|
||||
},
|
||||
RootPath: opts.Path,
|
||||
Suffix: fsStorageChunkSuffix,
|
||||
|
||||
@@ -111,6 +111,66 @@ $ kopia repository connect b2
|
||||
|
||||
---
|
||||
|
||||
|
||||
## SFTP
|
||||
|
||||
The `SFTP` provider can be used to connect to a file server over SFTP/SSH protocol.
|
||||
|
||||
You must first configure passwordless SFTP login by following [these instructions](https://www.redhat.com/sysadmin/passwordless-ssh). Make sure to choose empty passphrase, because Kopia does not support password prompts on each use.
|
||||
|
||||
If everything is configured correctly, you should be able to connect to your SFTP server without any password by using:
|
||||
|
||||
```
|
||||
$ sftp some-user@my-server
|
||||
Connected to my-server.
|
||||
sftp>
|
||||
```
|
||||
|
||||
|
||||
### Creating a repository
|
||||
|
||||
Assiuming passwordless connection worked, you can now create SFTP repository. Assuming you want the files to be stored under `/remote/path`, run the following command:
|
||||
|
||||
```shell
|
||||
$ kopia repository create sftp \
|
||||
--host my-server \
|
||||
--username some-user \
|
||||
--keyfile ~/.ssh/id_rsa \
|
||||
--known-hosts ~/.ssh/known_hosts \
|
||||
--path /remote/path
|
||||
```
|
||||
|
||||
(adjust paths to the key file and known hosts file as necessary).
|
||||
|
||||
When prompted, enter Kopia password to encrypt the repository contents.
|
||||
|
||||
If everything is done correctly, you should be able to verify that SFTP server indeed has Kopia files in the provided location, including special file named `kopia.repository.f`:
|
||||
|
||||
```
|
||||
$ sftp some-user@my-server
|
||||
sftp> ls -al /remote/path
|
||||
-rw-r--r-- 1 some-user some-user 661 Sep 18 16:12 kopia.repository.f
|
||||
```
|
||||
|
||||
### Connecting To Repository
|
||||
|
||||
To connect to an existing SFTP repository, simply use `connect` instead of `create`:
|
||||
|
||||
```shell
|
||||
$ kopia repository connect sftp \
|
||||
--host my-server \
|
||||
--username some-user \
|
||||
--keyfile ~/.ssh/id_rsa \
|
||||
--known-hosts ~/.ssh/known_hosts \
|
||||
--path /remote/path
|
||||
```
|
||||
|
||||
If the connection to SFTP server does not work, try adding `--external` which will launch external `ssh` process, which supports more connectivity options which may be needed for some hosts.
|
||||
|
||||
[Detailed information and settings](/docs/reference/command-line/common/repository-connect-sftp/)
|
||||
|
||||
---
|
||||
|
||||
## Rclone
|
||||
|
||||
Kopia can connect to certain backends supported by [Rclone](https://rclone.org) as long as they support
|
||||
|
||||
Reference in New Issue
Block a user