mirror of
https://github.com/rclone/rclone.git
synced 2026-10-10 15:01:37 -04:00
Nick Craig-Wood
5bc26e57b4
job/status returns the complete output of the job so it is as
sensitive as the call which started the job, e.g. config/dump started
with _async returns the config file including any secrets. It was
possible to read this without authentication where an authenticated
and an unauthenticated rc server share the same process, e.g. rclone
gui --rc.
An unauthenticated client can't start jobs on a server which needs
authentication, other than with the calls which don't need it, so
this shouldn't affect existing users.
(cherry picked from commit 939f82d908)
1208 lines
31 KiB
Go
1208 lines
31 KiB
Go
package rcserver
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
|
|
_ "github.com/rclone/rclone/backend/local"
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/config/configfile"
|
|
"github.com/rclone/rclone/fs/rc"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
const (
|
|
testBindAddress = "localhost:0"
|
|
defaultTestTemplate = "testdata/golden/testindex.html"
|
|
testFs = "testdata/files"
|
|
remoteURL = "[" + testFs + "]/" // initial URL path to fetch from that remote
|
|
)
|
|
|
|
func TestMain(m *testing.M) {
|
|
// Pretend to be rclone version if we have a version string parameter
|
|
if os.Args[len(os.Args)-1] == "version" {
|
|
fmt.Printf("rclone %s\n", fs.Version)
|
|
os.Exit(0)
|
|
}
|
|
// Pretend to error if we have an unknown command
|
|
if os.Args[len(os.Args)-1] == "unknown_command" {
|
|
fmt.Printf("rclone %s\n", fs.Version)
|
|
fmt.Fprintf(os.Stderr, "Unknown command\n")
|
|
os.Exit(1)
|
|
}
|
|
os.Exit(m.Run())
|
|
}
|
|
|
|
// Test the RC server runs and we can do HTTP fetches from it.
|
|
// We'll do the majority of the testing with the httptest framework
|
|
func TestRcServer(t *testing.T) {
|
|
opt := rc.Opt
|
|
opt.HTTP.ListenAddr = []string{testBindAddress}
|
|
opt.Template.Path = defaultTestTemplate
|
|
opt.Enabled = true
|
|
opt.Serve = true
|
|
opt.Files = testFs
|
|
mux := http.NewServeMux()
|
|
rcServer, err := newServer(context.Background(), &opt, mux)
|
|
require.NoError(t, err)
|
|
assert.NoError(t, rcServer.Serve())
|
|
defer func() {
|
|
assert.NoError(t, rcServer.Shutdown())
|
|
rcServer.Wait()
|
|
}()
|
|
testURL := rcServer.server.URLs()[0]
|
|
|
|
// Do the simplest possible test to check the server is alive
|
|
// Do it a few times to wait for the server to start
|
|
var resp *http.Response
|
|
for range 10 {
|
|
resp, err = http.Get(testURL + "file.txt")
|
|
if err == nil {
|
|
break
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
|
|
require.NoError(t, err)
|
|
body, err := io.ReadAll(resp.Body)
|
|
_ = resp.Body.Close()
|
|
|
|
require.NoError(t, err)
|
|
require.NoError(t, resp.Body.Close())
|
|
|
|
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
|
assert.Equal(t, "this is file1.txt\n", string(body))
|
|
}
|
|
|
|
type testRun struct {
|
|
Name string
|
|
URL string
|
|
User string
|
|
Pass string
|
|
Status int
|
|
Method string
|
|
Range string
|
|
Body string
|
|
ContentType string
|
|
Expected string
|
|
Contains *regexp.Regexp
|
|
Headers map[string]string
|
|
RequestHeaders map[string]string
|
|
}
|
|
|
|
// Run a suite of tests
|
|
func testServer(t *testing.T, tests []testRun, opt *rc.Options) {
|
|
t.Helper()
|
|
|
|
ctx := context.Background()
|
|
configfile.Install()
|
|
if opt.Template.Path == "" {
|
|
opt.Template.Path = defaultTestTemplate
|
|
}
|
|
rcServer, err := newServer(ctx, opt, http.DefaultServeMux)
|
|
require.NoError(t, err)
|
|
testURL := rcServer.server.URLs()[0]
|
|
mux := rcServer.server.Router()
|
|
emulateCalls(t, tests, mux, testURL)
|
|
}
|
|
|
|
func emulateCalls(t *testing.T, tests []testRun, mux chi.Router, testURL string) {
|
|
for _, test := range tests {
|
|
t.Run(test.Name, func(t *testing.T) {
|
|
t.Helper()
|
|
|
|
method := test.Method
|
|
if method == "" {
|
|
method = "GET"
|
|
}
|
|
var inBody io.Reader
|
|
if test.Body != "" {
|
|
buf := bytes.NewBufferString(test.Body)
|
|
inBody = buf
|
|
}
|
|
req, err := http.NewRequest(method, "http://1.2.3.4/"+test.URL, inBody)
|
|
require.NoError(t, err)
|
|
if test.Range != "" {
|
|
req.Header.Add("Range", test.Range)
|
|
}
|
|
if test.ContentType != "" {
|
|
req.Header.Add("Content-Type", test.ContentType)
|
|
}
|
|
if test.User != "" && test.Pass != "" {
|
|
req.SetBasicAuth(test.User, test.Pass)
|
|
}
|
|
for k, v := range test.RequestHeaders {
|
|
req.Header.Set(k, v)
|
|
}
|
|
|
|
w := httptest.NewRecorder()
|
|
mux.ServeHTTP(w, req)
|
|
resp := w.Result()
|
|
|
|
assert.Equal(t, test.Status, resp.StatusCode)
|
|
body, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
|
|
if test.ContentType == "application/json" && test.Expected != "" {
|
|
expectedNormalized := normalizeJSON(t, test.Expected)
|
|
actualNormalized := normalizeJSON(t, string(body))
|
|
assert.Equal(t, expectedNormalized, actualNormalized, "Normalized JSON does not match")
|
|
} else if test.Contains == nil {
|
|
// go1.23 started putting an html wrapper
|
|
bodyNormalized := strings.TrimPrefix(string(body), "<!doctype html>\n<meta name=\"viewport\" content=\"width=device-width\">\n")
|
|
assert.Equal(t, test.Expected, bodyNormalized)
|
|
} else {
|
|
assert.True(t, test.Contains.Match(body), fmt.Sprintf("body didn't match: %v: %v", test.Contains, string(body)))
|
|
}
|
|
|
|
for k, v := range test.Headers {
|
|
if v == "testURL" {
|
|
v = testURL
|
|
}
|
|
assert.Equal(t, v, resp.Header.Get(k), k)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// return an enabled rc
|
|
func newTestOpt() rc.Options {
|
|
opt := rc.Opt
|
|
opt.Enabled = true
|
|
opt.HTTP.ListenAddr = []string{testBindAddress}
|
|
return opt
|
|
}
|
|
|
|
func TestFileServing(t *testing.T) {
|
|
tests := []testRun{{
|
|
Name: "index",
|
|
URL: "",
|
|
Status: http.StatusOK,
|
|
Expected: `<pre>
|
|
<a href="dir/">dir/</a>
|
|
<a href="file.txt">file.txt</a>
|
|
<a href="modtime/">modtime/</a>
|
|
</pre>
|
|
`,
|
|
}, {
|
|
Name: "notfound",
|
|
URL: "notfound",
|
|
Status: http.StatusNotFound,
|
|
Expected: "404 page not found\n",
|
|
}, {
|
|
Name: "dirnotfound",
|
|
URL: "dirnotfound/",
|
|
Status: http.StatusNotFound,
|
|
Expected: "404 page not found\n",
|
|
}, {
|
|
Name: "dir",
|
|
URL: "dir/",
|
|
Status: http.StatusOK,
|
|
Expected: `<pre>
|
|
<a href="file2.txt">file2.txt</a>
|
|
</pre>
|
|
`,
|
|
}, {
|
|
Name: "file",
|
|
URL: "file.txt",
|
|
Status: http.StatusOK,
|
|
Expected: "this is file1.txt\n",
|
|
Headers: map[string]string{
|
|
"Content-Length": "18",
|
|
},
|
|
}, {
|
|
Name: "file2",
|
|
URL: "dir/file2.txt",
|
|
Status: http.StatusOK,
|
|
Expected: "this is dir/file2.txt\n",
|
|
}, {
|
|
Name: "file-head",
|
|
URL: "file.txt",
|
|
Method: "HEAD",
|
|
Status: http.StatusOK,
|
|
Expected: ``,
|
|
Headers: map[string]string{
|
|
"Content-Length": "18",
|
|
},
|
|
}, {
|
|
Name: "file-range",
|
|
URL: "file.txt",
|
|
Status: http.StatusPartialContent,
|
|
Range: "bytes=8-12",
|
|
Expected: `file1`,
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = testFs
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
func TestRemoteServing(t *testing.T) {
|
|
tests := []testRun{
|
|
// Test serving files from the test remote
|
|
{
|
|
Name: "index",
|
|
URL: remoteURL + "",
|
|
Status: http.StatusOK,
|
|
Expected: `<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<title>Directory listing of /</title>
|
|
</head>
|
|
<body>
|
|
<h1>Directory listing of /</h1>
|
|
<a href="dir/">dir/</a><br />
|
|
<a href="modtime/">modtime/</a><br />
|
|
<a href="file.txt">file.txt</a><br />
|
|
</body>
|
|
</html>
|
|
`,
|
|
}, {
|
|
Name: "notfound-index",
|
|
URL: "[notfound]/",
|
|
Status: http.StatusNotFound,
|
|
Expected: `{
|
|
"error": "failed to list directory: directory not found",
|
|
"input": null,
|
|
"path": "",
|
|
"status": 404
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "notfound",
|
|
URL: remoteURL + "notfound",
|
|
Status: http.StatusNotFound,
|
|
Expected: `{
|
|
"error": "failed to find object: object not found",
|
|
"input": null,
|
|
"path": "notfound",
|
|
"status": 404
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "dirnotfound",
|
|
URL: remoteURL + "dirnotfound/",
|
|
Status: http.StatusNotFound,
|
|
Expected: `{
|
|
"error": "failed to list directory: directory not found",
|
|
"input": null,
|
|
"path": "dirnotfound",
|
|
"status": 404
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "dir",
|
|
URL: remoteURL + "dir/",
|
|
Status: http.StatusOK,
|
|
Expected: `<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<title>Directory listing of /dir</title>
|
|
</head>
|
|
<body>
|
|
<h1>Directory listing of /dir</h1>
|
|
<a href="file2.txt">file2.txt</a><br />
|
|
</body>
|
|
</html>
|
|
`,
|
|
}, {
|
|
Name: "file",
|
|
URL: remoteURL + "file.txt",
|
|
Status: http.StatusOK,
|
|
Expected: "this is file1.txt\n",
|
|
Headers: map[string]string{
|
|
"Content-Length": "18",
|
|
},
|
|
}, {
|
|
Name: "file with no slash after ]",
|
|
URL: strings.TrimRight(remoteURL, "/") + "file.txt",
|
|
Status: http.StatusOK,
|
|
Expected: "this is file1.txt\n",
|
|
Headers: map[string]string{
|
|
"Content-Length": "18",
|
|
},
|
|
}, {
|
|
Name: "file2",
|
|
URL: remoteURL + "dir/file2.txt",
|
|
Status: http.StatusOK,
|
|
Expected: "this is dir/file2.txt\n",
|
|
}, {
|
|
Name: "file-head",
|
|
URL: remoteURL + "file.txt",
|
|
Method: "HEAD",
|
|
Status: http.StatusOK,
|
|
Expected: ``,
|
|
Headers: map[string]string{
|
|
"Content-Length": "18",
|
|
},
|
|
}, {
|
|
Name: "file-range",
|
|
URL: remoteURL + "file.txt",
|
|
Status: http.StatusPartialContent,
|
|
Range: "bytes=8-12",
|
|
Expected: `file1`,
|
|
}, {
|
|
Name: "bad-remote",
|
|
URL: "[notfoundremote:]/",
|
|
Status: http.StatusInternalServerError,
|
|
Expected: `{
|
|
"error": "failed to make Fs: didn't find section in config file (\"notfoundremote\")",
|
|
"input": null,
|
|
"path": "/",
|
|
"status": 500
|
|
}
|
|
`,
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = testFs
|
|
opt.NoAuth = true
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
// TestCheckServeRemote must reject request-derived backend instantiation on the
|
|
// unauthenticated file-serving path, and global.* config mutation.
|
|
func TestCheckServeRemote(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
fsName string
|
|
authenticated bool
|
|
wantErr string // substring of expected error, "" means no error
|
|
}{
|
|
// Plain named remotes are always allowed
|
|
{name: "named remote unauth", fsName: "remote:path", wantErr: ""},
|
|
{name: "named remote auth", fsName: "remote:path", authenticated: true, wantErr: ""},
|
|
// Inline backend definitions can run commands (webdav bearer_token_command, sftp ssh)
|
|
{name: "inline webdav unauth", fsName: ":webdav,url=http://localhost,bearer_token_command=id:", wantErr: "inline remotes"},
|
|
{name: "inline sftp unauth", fsName: ":sftp,ssh=id:", wantErr: "inline remotes"},
|
|
{name: "inline local unauth", fsName: ":local:/etc", wantErr: "inline remotes"},
|
|
{name: "inline allowed when authed", fsName: ":local:/etc", authenticated: true, wantErr: ""},
|
|
// Bare local paths allow arbitrary local file access
|
|
{name: "bare path unauth", fsName: "/etc/passwd", wantErr: "local paths"},
|
|
{name: "bare path allowed when authed", fsName: "/etc/passwd", authenticated: true, wantErr: ""},
|
|
// Connection string overrides can set command-executing options
|
|
{name: "connection string unauth", fsName: "remote,vendor=other:path", wantErr: "connection string"},
|
|
{name: "connection string allowed when authed", fsName: "remote,vendor=other:path", authenticated: true, wantErr: ""},
|
|
// global.* mutates process-wide config and is never allowed
|
|
{name: "global config unauth", fsName: "remote,global.http_proxy=x:path", wantErr: "global.http_proxy"},
|
|
{name: "global config blocked even when authed", fsName: "remote,global.http_proxy=x:path", authenticated: true, wantErr: "global.http_proxy"},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
err := checkServeRemote(test.fsName, test.authenticated)
|
|
if test.wantErr == "" {
|
|
assert.NoError(t, err)
|
|
} else {
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), test.wantErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// On an unauthenticated server the serve path must not instantiate
|
|
// attacker-supplied inline remotes or local paths, but must still allow
|
|
// configured named remotes.
|
|
func TestServeRemoteUnauthenticated(t *testing.T) {
|
|
forbidden := regexp.MustCompile(`"status": 403`)
|
|
tests := []testRun{{
|
|
// An inline remote could run a command during initialisation
|
|
Name: "inline-remote-rejected",
|
|
URL: "[:local:" + testFs + "]/file.txt",
|
|
Status: http.StatusForbidden,
|
|
Contains: forbidden,
|
|
}, {
|
|
Name: "bare-local-path-rejected",
|
|
URL: remoteURL + "file.txt",
|
|
Status: http.StatusForbidden,
|
|
Contains: forbidden,
|
|
}, {
|
|
// A configured (here non-existent) named remote is allowed through to
|
|
// the backend, which then reports it isn't in the config file
|
|
Name: "named-remote-allowed-through",
|
|
URL: "[notfoundremote:]/",
|
|
Status: http.StatusInternalServerError,
|
|
Contains: regexp.MustCompile(`didn't find section in config file`),
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
// With authentication configured the serve path may instantiate inline remotes
|
|
// (the request is authenticated) but must still refuse to mutate process-wide
|
|
// config via global.* options.
|
|
func TestServeRemoteWithAuth(t *testing.T) {
|
|
const user, pass = "user", "pass"
|
|
tests := []testRun{{
|
|
// Authenticated requests can serve bare local paths again
|
|
Name: "bare-local-path-served-when-authenticated",
|
|
URL: remoteURL + "file.txt",
|
|
User: user,
|
|
Pass: pass,
|
|
Status: http.StatusOK,
|
|
Expected: "this is file1.txt\n",
|
|
}, {
|
|
// global.* is blocked even for an authenticated request
|
|
Name: "global-config-blocked-when-authenticated",
|
|
URL: "[notfoundremote,global.http_proxy=x:]/",
|
|
User: user,
|
|
Pass: pass,
|
|
Status: http.StatusForbidden,
|
|
Contains: regexp.MustCompile(`global\.http_proxy`),
|
|
}, {
|
|
// A request without credentials is rejected before reaching the handler
|
|
Name: "unauthenticated-request-rejected",
|
|
URL: remoteURL + "file.txt",
|
|
Status: http.StatusUnauthorized,
|
|
Contains: regexp.MustCompile(``),
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = testFs
|
|
opt.Auth.BasicUser = user
|
|
opt.Auth.BasicPass = pass
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
// The serve path must mark backend creation as a remote control (rc) request
|
|
// so a request-supplied remote can't change process-wide config via global.*
|
|
// options.
|
|
func TestServeRemoteMarksRCRequest(t *testing.T) {
|
|
configfile.Install()
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.NoAuth = true // allow inline remotes so we reach backend creation
|
|
opt.Template.Path = defaultTestTemplate
|
|
rcServer, err := newServer(context.Background(), &opt, http.NewServeMux())
|
|
require.NoError(t, err)
|
|
|
|
ctx := context.Background()
|
|
original := fs.GetConfig(ctx).UserAgent
|
|
defer func() { fs.GetConfig(ctx).UserAgent = original }()
|
|
|
|
// serveRemote uses s.ctx, marked as an rc request, so global.* must not
|
|
// change the process-wide config.
|
|
rcServer.serveRemote(httptest.NewRecorder(), httptest.NewRequest("GET", "/file.txt", nil),
|
|
"file.txt", ":local,global.user_agent=rcservertest:"+testFs)
|
|
assert.Equal(t, original, fs.GetConfig(ctx).UserAgent,
|
|
"the serve path must not let global.* change the process-wide config")
|
|
}
|
|
|
|
func TestRC(t *testing.T) {
|
|
tests := []testRun{{
|
|
Name: "rc-root",
|
|
URL: "",
|
|
Method: "POST",
|
|
Status: http.StatusNotFound,
|
|
Expected: `{
|
|
"error": "couldn't find method \"\"",
|
|
"input": {},
|
|
"path": "",
|
|
"status": 404
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "rc-noop",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
Status: http.StatusOK,
|
|
Expected: "{}\n",
|
|
}, {
|
|
Name: "rc-error",
|
|
URL: "rc/error",
|
|
Method: "POST",
|
|
Status: http.StatusInternalServerError,
|
|
Expected: `{
|
|
"error": "arbitrary error on input map[]",
|
|
"input": {},
|
|
"path": "rc/error",
|
|
"status": 500
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "core-gc",
|
|
URL: "core/gc", // now requires auth
|
|
Method: "POST",
|
|
Status: http.StatusForbidden,
|
|
Expected: `{
|
|
"error": "authentication must be set up on the rc server to use \"core/gc\" or the --rc-no-auth flag must be in use",
|
|
"input": {},
|
|
"path": "core/gc",
|
|
"status": 403
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "url-params",
|
|
URL: "rc/noop?param1=potato¶m2=sausage",
|
|
Method: "POST",
|
|
Status: http.StatusOK,
|
|
Expected: `{
|
|
"param1": "potato",
|
|
"param2": "sausage"
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "json",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
Body: `{ "param1":"string", "param2":true }`,
|
|
ContentType: "application/json",
|
|
Status: http.StatusOK,
|
|
Expected: `{
|
|
"param1": "string",
|
|
"param2": true
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "json-mixed-case-content-type",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
Body: `{ "param1":"string", "param2":true }`,
|
|
ContentType: "ApplicAtion/JsOn",
|
|
Status: http.StatusOK,
|
|
Expected: `{
|
|
"param1": "string",
|
|
"param2": true
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "json-and-url-params",
|
|
URL: "rc/noop?param1=potato¶m2=sausage",
|
|
Method: "POST",
|
|
Body: `{ "param1":"string", "param3":true }`,
|
|
ContentType: "application/json",
|
|
Status: http.StatusOK,
|
|
Expected: `{
|
|
"param1": "string",
|
|
"param2": "sausage",
|
|
"param3": true
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "json-bad",
|
|
URL: "rc/noop?param1=potato¶m2=sausage",
|
|
Method: "POST",
|
|
Body: `{ param1":"string", "param3":true }`,
|
|
ContentType: "application/json",
|
|
Status: http.StatusBadRequest,
|
|
Expected: `{
|
|
"error": "failed to read input JSON: invalid character 'p' looking for beginning of object key string",
|
|
"input": {
|
|
"param1": "potato",
|
|
"param2": "sausage"
|
|
},
|
|
"path": "rc/noop",
|
|
"status": 400
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "json-charset",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
Body: `{ "param1":"string", "param2":true }`,
|
|
ContentType: "application/json; charset=utf-8",
|
|
Status: http.StatusOK,
|
|
Expected: `{
|
|
"param1": "string",
|
|
"param2": true
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "json-mixed-case-charset",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
Body: `{ "param1":"string", "param2":true }`,
|
|
ContentType: "aPPlication/jSoN; charset=UtF-8",
|
|
Status: http.StatusOK,
|
|
Expected: `{
|
|
"param1": "string",
|
|
"param2": true
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "json-bad-charset",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
Body: `{ "param1":"string", "param2":true }`,
|
|
ContentType: "application/json; charset=latin1",
|
|
Status: http.StatusBadRequest,
|
|
Expected: `{
|
|
"error": "unsupported charset \"latin1\" for JSON input",
|
|
"input": {},
|
|
"path": "rc/noop",
|
|
"status": 400
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "form",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
Body: `param1=string¶m2=true`,
|
|
ContentType: "application/x-www-form-urlencoded",
|
|
Status: http.StatusOK,
|
|
Expected: `{
|
|
"param1": "string",
|
|
"param2": "true"
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "form-and-url-params",
|
|
URL: "rc/noop?param1=potato¶m2=sausage",
|
|
Method: "POST",
|
|
Body: `param1=string¶m3=true`,
|
|
ContentType: "application/x-www-form-urlencoded",
|
|
Status: http.StatusOK,
|
|
Expected: `{
|
|
"param1": "potato",
|
|
"param2": "sausage",
|
|
"param3": "true"
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "form-bad",
|
|
URL: "rc/noop?param1=potato¶m2=sausage",
|
|
Method: "POST",
|
|
Body: `%zz`,
|
|
ContentType: "application/x-www-form-urlencoded",
|
|
Status: http.StatusBadRequest,
|
|
Expected: `{
|
|
"error": "failed to parse form/URL parameters: invalid URL escape \"%zz\"",
|
|
"input": null,
|
|
"path": "rc/noop",
|
|
"status": 400
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "malformed-content-type",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
ContentType: "malformed/",
|
|
Status: http.StatusBadRequest,
|
|
Expected: `{
|
|
"error": "failed to parse Content-Type: mime: expected token after slash",
|
|
"input": null,
|
|
"path": "rc/noop",
|
|
"status": 400
|
|
}
|
|
`,
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = testFs
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
func TestRCWithAuth(t *testing.T) {
|
|
tests := []testRun{{
|
|
Name: "core-command",
|
|
URL: "core/command",
|
|
Method: "POST",
|
|
Body: `command=version`,
|
|
ContentType: "application/x-www-form-urlencoded",
|
|
Status: http.StatusOK,
|
|
Expected: fmt.Sprintf(`{
|
|
"error": false,
|
|
"result": "rclone %s\n"
|
|
}
|
|
`, fs.Version),
|
|
}, {
|
|
Name: "core-command-bad-returnType",
|
|
URL: "core/command",
|
|
Method: "POST",
|
|
Body: `command=version&returnType=POTATO`,
|
|
ContentType: "application/x-www-form-urlencoded",
|
|
Status: http.StatusInternalServerError,
|
|
Expected: `{
|
|
"error": "unknown returnType \"POTATO\"",
|
|
"input": {
|
|
"command": "version",
|
|
"returnType": "POTATO"
|
|
},
|
|
"path": "core/command",
|
|
"status": 500
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "core-command-stream",
|
|
URL: "core/command",
|
|
Method: "POST",
|
|
Body: `command=version&returnType=STREAM`,
|
|
ContentType: "application/x-www-form-urlencoded",
|
|
Status: http.StatusOK,
|
|
Expected: fmt.Sprintf(`rclone %s
|
|
{}
|
|
`, fs.Version),
|
|
}, {
|
|
Name: "core-command-stream-error",
|
|
URL: "core/command",
|
|
Method: "POST",
|
|
Body: `command=unknown_command&returnType=STREAM`,
|
|
ContentType: "application/x-www-form-urlencoded",
|
|
Status: http.StatusOK,
|
|
Expected: fmt.Sprintf(`rclone %s
|
|
Unknown command
|
|
{
|
|
"error": "exit status 1",
|
|
"input": {
|
|
"command": "unknown_command",
|
|
"returnType": "STREAM"
|
|
},
|
|
"path": "core/command",
|
|
"status": 500
|
|
}
|
|
`, fs.Version),
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = testFs
|
|
opt.NoAuth = true
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
var matchRemoteDirListing = regexp.MustCompile(`<title>Directory listing of /</title>`)
|
|
|
|
func TestServingRoot(t *testing.T) {
|
|
tests := []testRun{{
|
|
Name: "rootlist",
|
|
URL: "*",
|
|
Status: http.StatusOK,
|
|
Contains: matchRemoteDirListing,
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = testFs
|
|
opt.NoAuth = true
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
func TestServingRootNoFiles(t *testing.T) {
|
|
tests := []testRun{{
|
|
Name: "rootlist",
|
|
URL: "",
|
|
Status: http.StatusOK,
|
|
Contains: matchRemoteDirListing,
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = ""
|
|
opt.NoAuth = true
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
// On a server with no authentication configured and without --rc-no-auth the
|
|
// remote listing must not disclose the configured remote names.
|
|
func TestServingRootFailClosed(t *testing.T) {
|
|
forbidden := regexp.MustCompile(`"status": 403`)
|
|
tests := []testRun{{
|
|
Name: "rootlist-star",
|
|
URL: "*",
|
|
Status: http.StatusForbidden,
|
|
Contains: forbidden,
|
|
}, {
|
|
Name: "rootlist-empty",
|
|
URL: "",
|
|
Status: http.StatusForbidden,
|
|
Contains: forbidden,
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = ""
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
// With authentication configured the remote listing is available to
|
|
// authenticated requests only.
|
|
func TestServingRootWithAuth(t *testing.T) {
|
|
const user, pass = "user", "pass"
|
|
tests := []testRun{{
|
|
Name: "rootlist-authed",
|
|
URL: "",
|
|
User: user,
|
|
Pass: pass,
|
|
Status: http.StatusOK,
|
|
Contains: matchRemoteDirListing,
|
|
}, {
|
|
Name: "rootlist-unauthed",
|
|
URL: "",
|
|
Status: http.StatusUnauthorized,
|
|
Contains: regexp.MustCompile(`Unauthorized`),
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = ""
|
|
opt.Auth.BasicUser = user
|
|
opt.Auth.BasicPass = pass
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
func TestNoFiles(t *testing.T) {
|
|
tests := []testRun{{
|
|
Name: "file",
|
|
URL: "file.txt",
|
|
Status: http.StatusNotFound,
|
|
Expected: "Not Found\n",
|
|
}, {
|
|
Name: "dir",
|
|
URL: "dir/",
|
|
Status: http.StatusNotFound,
|
|
Expected: "Not Found\n",
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = ""
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
func TestNoServe(t *testing.T) {
|
|
tests := []testRun{{
|
|
Name: "file",
|
|
URL: remoteURL + "file.txt",
|
|
Status: http.StatusNotFound,
|
|
Expected: "404 page not found\n",
|
|
}, {
|
|
Name: "dir",
|
|
URL: remoteURL + "dir/",
|
|
Status: http.StatusNotFound,
|
|
Expected: "404 page not found\n",
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = false
|
|
opt.Files = testFs
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
func TestAuthRequired(t *testing.T) {
|
|
tests := []testRun{{
|
|
Name: "auth",
|
|
URL: "rc/noopauth",
|
|
Method: "POST",
|
|
Body: `{}`,
|
|
ContentType: "application/javascript",
|
|
Status: http.StatusForbidden,
|
|
Expected: `{
|
|
"error": "authentication must be set up on the rc server to use \"rc/noopauth\" or the --rc-no-auth flag must be in use",
|
|
"input": {},
|
|
"path": "rc/noopauth",
|
|
"status": 403
|
|
}
|
|
`,
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = false
|
|
opt.Files = ""
|
|
opt.NoAuth = false
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
// job/status returns the output of any job so it must need
|
|
// authentication, e.g. so an unauthenticated user can't read the
|
|
// output of config/dump started with _async by an authenticated one.
|
|
func TestJobAuthRequired(t *testing.T) {
|
|
tests := []testRun{{
|
|
Name: "status",
|
|
URL: "job/status",
|
|
Method: "POST",
|
|
Body: `{"jobid":1}`,
|
|
ContentType: "application/json",
|
|
Status: http.StatusForbidden,
|
|
Expected: `{
|
|
"error": "authentication must be set up on the rc server to use \"job/status\" or the --rc-no-auth flag must be in use",
|
|
"input": {
|
|
"jobid": 1
|
|
},
|
|
"path": "job/status",
|
|
"status": 403
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "list",
|
|
URL: "job/list",
|
|
Method: "POST",
|
|
Body: `{}`,
|
|
ContentType: "application/json",
|
|
Status: http.StatusForbidden,
|
|
Expected: `{
|
|
"error": "authentication must be set up on the rc server to use \"job/list\" or the --rc-no-auth flag must be in use",
|
|
"input": {},
|
|
"path": "job/list",
|
|
"status": 403
|
|
}
|
|
`,
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = false
|
|
opt.Files = ""
|
|
opt.NoAuth = false
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
func TestNoAuth(t *testing.T) {
|
|
tests := []testRun{{
|
|
Name: "auth",
|
|
URL: "rc/noopauth",
|
|
Method: "POST",
|
|
Body: `{}`,
|
|
ContentType: "application/javascript",
|
|
Status: http.StatusOK,
|
|
Expected: "{}\n",
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = false
|
|
opt.Files = ""
|
|
opt.NoAuth = true
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
func TestWithUserPass(t *testing.T) {
|
|
tests := []testRun{{
|
|
Name: "authMissing",
|
|
URL: "rc/noopauth",
|
|
Method: "POST",
|
|
Body: `{}`,
|
|
ContentType: "application/javascript",
|
|
Status: http.StatusUnauthorized,
|
|
Expected: "401 Unauthorized\n",
|
|
}, {
|
|
Name: "authWrong",
|
|
URL: "rc/noopauth",
|
|
Method: "POST",
|
|
Body: `{}`,
|
|
ContentType: "application/javascript",
|
|
Status: http.StatusUnauthorized,
|
|
Expected: "401 Unauthorized\n",
|
|
User: "user1",
|
|
Pass: "pass2",
|
|
}, {
|
|
Name: "authOK",
|
|
URL: "rc/noopauth",
|
|
Method: "POST",
|
|
Body: `{}`,
|
|
ContentType: "application/javascript",
|
|
Status: http.StatusOK,
|
|
Expected: "{}\n",
|
|
User: "user",
|
|
Pass: "pass",
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = false
|
|
opt.Files = ""
|
|
opt.NoAuth = false
|
|
opt.Auth.BasicUser = "user"
|
|
opt.Auth.BasicPass = "pass"
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
func TestRCAsync(t *testing.T) {
|
|
tests := []testRun{{
|
|
Name: "ok",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
ContentType: "application/json",
|
|
Body: `{ "_async":true }`,
|
|
Status: http.StatusOK,
|
|
Contains: regexp.MustCompile(`(?s)\{.*\"jobid\":.*\}`),
|
|
}, {
|
|
Name: "prefer-respond-async",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
ContentType: "application/json",
|
|
Body: `{ "_async":true }`,
|
|
RequestHeaders: map[string]string{"Prefer": "respond-async"},
|
|
Status: http.StatusAccepted,
|
|
Contains: regexp.MustCompile(`(?s)\{.*\"jobid\":.*\}`),
|
|
}, {
|
|
Name: "prefer-without-async",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
ContentType: "application/json",
|
|
Body: `{}`,
|
|
RequestHeaders: map[string]string{"Prefer": "respond-async"},
|
|
Status: http.StatusAccepted,
|
|
Contains: regexp.MustCompile(`(?s)\{.*\"jobid\":.*\}`),
|
|
}, {
|
|
Name: "prefer-respond-async-mixed",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
ContentType: "application/json",
|
|
Body: `{}`,
|
|
RequestHeaders: map[string]string{"Prefer": "wait=10, respond-async"},
|
|
Status: http.StatusAccepted,
|
|
Contains: regexp.MustCompile(`(?s)\{.*\"jobid\":.*\}`),
|
|
}, {
|
|
Name: "bad",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
ContentType: "application/json",
|
|
Body: `{ "_async":"truthy" }`,
|
|
Status: http.StatusBadRequest,
|
|
Expected: `{
|
|
"error": "couldn't parse key \"_async\" (truthy) as bool: strconv.ParseBool: parsing \"truthy\": invalid syntax",
|
|
"input": {
|
|
"_async": "truthy"
|
|
},
|
|
"path": "rc/noop",
|
|
"status": 400
|
|
}
|
|
`,
|
|
}}
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = ""
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
// The debug/pprof handlers expose the process command line, heap and
|
|
// goroutine dumps, so require auth.
|
|
func TestRCDebug(t *testing.T) {
|
|
index := testRun{
|
|
Name: "index",
|
|
URL: "debug/pprof/",
|
|
Method: "GET",
|
|
ContentType: "text/html",
|
|
Status: http.StatusOK,
|
|
Contains: regexp.MustCompile(`Types of profiles available`),
|
|
}
|
|
goroutines := testRun{
|
|
Name: "goroutines",
|
|
URL: "debug/pprof/goroutine?debug=1",
|
|
Method: "GET",
|
|
ContentType: "text/html",
|
|
Status: http.StatusOK,
|
|
Contains: regexp.MustCompile(`goroutine profile`),
|
|
}
|
|
|
|
// With --rc-no-auth the debug handlers are attached and open
|
|
t.Run("NoAuth", func(t *testing.T) {
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = ""
|
|
opt.NoAuth = true
|
|
testServer(t, []testRun{index, goroutines}, &opt)
|
|
})
|
|
|
|
// With auth configured the debug handlers are attached but require credentials
|
|
t.Run("WithAuth", func(t *testing.T) {
|
|
authed := index
|
|
authed.User, authed.Pass = "user", "pass"
|
|
unauthed := index
|
|
unauthed.Name = "index-no-credentials"
|
|
unauthed.Status = http.StatusUnauthorized
|
|
unauthed.Contains = regexp.MustCompile(`Unauthorized`)
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = ""
|
|
opt.Auth.BasicUser = "user"
|
|
opt.Auth.BasicPass = "pass"
|
|
testServer(t, []testRun{authed, unauthed}, &opt)
|
|
})
|
|
|
|
// On a default unauthenticated server the debug handlers must not be
|
|
// mounted, so they bypass nothing - they simply aren't there
|
|
t.Run("FailClosed", func(t *testing.T) {
|
|
blocked := index
|
|
blocked.Status = http.StatusNotFound
|
|
blocked.Contains = regexp.MustCompile(`Not Found`)
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Files = ""
|
|
testServer(t, []testRun{blocked}, &opt)
|
|
})
|
|
}
|
|
|
|
func TestServeModTime(t *testing.T) {
|
|
for file, mtime := range map[string]time.Time{
|
|
"dir": time.Date(2023, 4, 12, 21, 15, 17, 0, time.UTC),
|
|
"modtime.txt": time.Date(2021, 1, 18, 5, 2, 28, 0, time.UTC),
|
|
} {
|
|
path := filepath.Join(testFs, "modtime", file)
|
|
err := os.Chtimes(path, mtime, mtime)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
opt := newTestOpt()
|
|
opt.Serve = true
|
|
opt.Template.Path = "testdata/golden/testmodtime.html"
|
|
opt.NoAuth = true
|
|
|
|
tests := []testRun{{
|
|
Name: "modtime",
|
|
Method: "GET",
|
|
URL: remoteURL + "modtime/",
|
|
Status: http.StatusOK,
|
|
Expected: "* dir/ - 2023-04-12T21:15:17Z\n* modtime.txt - 2021-01-18T05:02:28Z\n",
|
|
}}
|
|
testServer(t, tests, &opt)
|
|
|
|
opt.ServeNoModTime = true
|
|
tests = []testRun{{
|
|
Name: "no modtime",
|
|
Method: "GET",
|
|
URL: remoteURL + "modtime/",
|
|
Status: http.StatusOK,
|
|
Expected: "* dir/ - 0001-01-01T00:00:00Z\n* modtime.txt - 0001-01-01T00:00:00Z\n",
|
|
}}
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
func TestContentTypeJSON(t *testing.T) {
|
|
tests := []testRun{
|
|
{
|
|
Name: "Check Content-Type for JSON response",
|
|
URL: "rc/noop",
|
|
Method: "POST",
|
|
Body: `{}`,
|
|
ContentType: "application/json",
|
|
Status: http.StatusOK,
|
|
Expected: "{}\n",
|
|
Headers: map[string]string{
|
|
"Content-Type": "application/json",
|
|
},
|
|
},
|
|
{
|
|
Name: "Check Content-Type for JSON error response",
|
|
URL: "rc/error",
|
|
Method: "POST",
|
|
Body: `{}`,
|
|
ContentType: "application/json",
|
|
Status: http.StatusInternalServerError,
|
|
Expected: `{
|
|
"error": "arbitrary error on input map[]",
|
|
"input": {},
|
|
"path": "rc/error",
|
|
"status": 500
|
|
}
|
|
`,
|
|
Headers: map[string]string{
|
|
"Content-Type": "application/json",
|
|
},
|
|
},
|
|
}
|
|
opt := newTestOpt()
|
|
testServer(t, tests, &opt)
|
|
}
|
|
|
|
func normalizeJSON(t *testing.T, jsonStr string) string {
|
|
var jsonObj map[string]any
|
|
err := json.Unmarshal([]byte(jsonStr), &jsonObj)
|
|
require.NoError(t, err, "JSON unmarshalling failed")
|
|
normalizedJSON, err := json.Marshal(jsonObj)
|
|
require.NoError(t, err, "JSON marshalling failed")
|
|
return string(normalizedJSON)
|
|
}
|