lib/http: add static rendering support to the directory listing

- .Static hides the "up" link at the root
- .SetLinkIndex makes directory links point at an index doc
- .Render writes the listing to an io.Writer
- .Path, .IsRoot, .UpLink, .NumDirs, .NumFiles, .TotalSize and .MimeType.
- Sorting is now stable so the rendered output is deterministic

lib/http/serve/dir.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"html/template"
+	"io"
 	"net/http"
 	"net/url"
 	"path"
@@ -19,13 +20,14 @@ import (
 
 // DirEntry is a directory entry
 type DirEntry struct {
-	remote  string
-	URL     string
-	ZipURL  string
-	Leaf    string
-	IsDir   bool
-	Size    int64
-	ModTime time.Time
+	remote   string
+	URL      string
+	ZipURL   string
+	Leaf     string
+	IsDir    bool
+	Size     int64
+	ModTime  time.Time
+	MimeType string
 }
 
 // Directory represents a directory
@@ -41,6 +43,8 @@ type Directory struct {
 	Breadcrumb   []Crumb
 	Sort         string
 	Order        string
+	Static       bool   // rendering for a static site, so no "up" link at the root
+	LinkIndex    string // if set, appended to directory links, e.g. "index.html"
 }
 
 // Crumb is a breadcrumb entry
@@ -80,6 +84,64 @@ func NewDirectory(dirRemote string, htmlTemplate *template.Template) *Directory
 	return d
 }
 
+// SetLinkIndex makes directory links point at name inside the
+// directory, e.g. "dir/index.html" rather than "dir/", for hosts which
+// don't serve index documents. Call it before adding entries.
+func (d *Directory) SetLinkIndex(name string) *Directory {
+	d.LinkIndex = name
+	for i := range d.Breadcrumb {
+		d.Breadcrumb[i].Link += name
+	}
+	return d
+}
+
+// Path returns the path of the directory from the root with leading
+// and trailing slashes, e.g. "/dir/" or "/" for the root.
+func (d *Directory) Path() string {
+	if d.IsRoot() {
+		return "/"
+	}
+	return "/" + strings.Trim(d.DirRemote, "/") + "/"
+}
+
+// IsRoot returns true if this is the root directory
+func (d *Directory) IsRoot() bool {
+	return strings.Trim(d.DirRemote, "/") == ""
+}
+
+// UpLink returns the link to the parent directory
+func (d *Directory) UpLink() string {
+	if d.LinkIndex == "" {
+		return ".."
+	}
+	return "../" + d.LinkIndex
+}
+
+// NumDirs returns the number of directory entries
+func (d *Directory) NumDirs() (n int) {
+	for i := range d.Entries {
+		if d.Entries[i].IsDir {
+			n++
+		}
+	}
+	return n
+}
+
+// NumFiles returns the number of file entries
+func (d *Directory) NumFiles() int {
+	return len(d.Entries) - d.NumDirs()
+}
+
+// TotalSize returns the total size of the file entries
+func (d *Directory) TotalSize() (size int64) {
+	for i := range d.Entries {
+		if !d.Entries[i].IsDir {
+			size += d.Entries[i].Size
+		}
+	}
+	return size
+}
+
 // SetQuery sets the query parameters for each URL
 func (d *Directory) SetQuery(queryParams url.Values) *Directory {
 	d.Query = ""
@@ -96,22 +158,25 @@ func (d *Directory) AddHTMLEntry(remote string, isDir bool, size int64, modTime
 		leaf = ""
 	}
 	urlRemote := leaf
+	mimeType := fs.MimeTypeFromName(leaf)
+	linkIndex, zipURL := "", ""
 	if isDir {
 		leaf += "/"
 		urlRemote += "/"
+		mimeType = "inode/directory"
+		linkIndex = d.LinkIndex
+		zipURL = rest.URLPathEscape(urlRemote) + "?download=zip"
 	}
 	d.Entries = append(d.Entries, DirEntry{
-		remote:  remote,
-		URL:     rest.URLPathEscape(urlRemote) + d.Query,
-		ZipURL:  "",
-		Leaf:    leaf,
-		IsDir:   isDir,
-		Size:    size,
-		ModTime: modTime,
+		remote:   remote,
+		URL:      rest.URLPathEscape(urlRemote) + linkIndex + d.Query,
+		ZipURL:   zipURL,
+		Leaf:     leaf,
+		IsDir:    isDir,
+		Size:     size,
+		ModTime:  modTime,
+		MimeType: mimeType,
 	})
-	if isDir {
-		d.Entries[len(d.Entries)-1].ZipURL = rest.URLPathEscape(urlRemote) + "?download=zip"
-	}
 }
 
 // AddEntry adds an entry to that directory
@@ -164,7 +229,7 @@ func (d *Directory) ProcessQueryParams(sortParm string, orderParm string) *Direc
 		toSort = sort.Reverse(toSort)
 	}
 	if toSort != nil {
-		sort.Sort(toSort)
+		sort.Stable(toSort)
 	}
 
 	return d
@@ -230,6 +295,11 @@ const (
 	sortByTime         = "time"
 )
 
+// Render writes the directory listing to w using the HTML template
+func (d *Directory) Render(w io.Writer) error {
+	return d.HTMLTemplate.Execute(w, d)
+}
+
 // Serve serves a directory
 func (d *Directory) Serve(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
@@ -240,7 +310,7 @@ func (d *Directory) Serve(w http.ResponseWriter, r *http.Request) {
 	fs.Infof(d.DirRemote, "%s: Serving directory", r.RemoteAddr)
 
 	buf := &bytes.Buffer{}
-	err := d.HTMLTemplate.Execute(buf, d)
+	err := d.Render(buf)
 	if err != nil {
 		Error(ctx, d.DirRemote, w, "Failed to render template", err)
 		return

lib/http/serve/dir_test.go

@@ -1,6 +1,7 @@
 package serve
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"html/template"
@@ -46,11 +47,11 @@ func TestAddHTMLEntry(t *testing.T) {
 	d.AddHTMLEntry("a/b/c/colon:colon.txt", false, 64, modtime)
 	d.AddHTMLEntry("\"quotes\".txt", false, 64, modtime)
 	assert.Equal(t, []DirEntry{
-		{remote: "", URL: "/", ZipURL: "/?download=zip", Leaf: "/", IsDir: true, Size: 0, ModTime: modtime},
-		{remote: "dir", URL: "dir/", ZipURL: "dir/?download=zip", Leaf: "dir/", IsDir: true, Size: 0, ModTime: modtime},
-		{remote: "a/b/c/d.txt", URL: "d.txt", ZipURL: "", Leaf: "d.txt", IsDir: false, Size: 64, ModTime: modtime},
-		{remote: "a/b/c/colon:colon.txt", URL: "./colon:colon.txt", ZipURL: "", Leaf: "colon:colon.txt", IsDir: false, Size: 64, ModTime: modtime},
-		{remote: "\"quotes\".txt", URL: "%22quotes%22.txt", ZipURL: "", Leaf: "\"quotes\".txt", Size: 64, IsDir: false, ModTime: modtime},
+		{remote: "", URL: "/", ZipURL: "/?download=zip", Leaf: "/", IsDir: true, Size: 0, ModTime: modtime, MimeType: "inode/directory"},
+		{remote: "dir", URL: "dir/", ZipURL: "dir/?download=zip", Leaf: "dir/", IsDir: true, Size: 0, ModTime: modtime, MimeType: "inode/directory"},
+		{remote: "a/b/c/d.txt", URL: "d.txt", ZipURL: "", Leaf: "d.txt", IsDir: false, Size: 64, ModTime: modtime, MimeType: "text/plain; charset=utf-8"},
+		{remote: "a/b/c/colon:colon.txt", URL: "./colon:colon.txt", ZipURL: "", Leaf: "colon:colon.txt", IsDir: false, Size: 64, ModTime: modtime, MimeType: "text/plain; charset=utf-8"},
+		{remote: "\"quotes\".txt", URL: "%22quotes%22.txt", ZipURL: "", Leaf: "\"quotes\".txt", Size: 64, IsDir: false, ModTime: modtime, MimeType: "text/plain; charset=utf-8"},
 	}, d.Entries)
 
 	// Now test with a query parameter
@@ -58,9 +59,65 @@ func TestAddHTMLEntry(t *testing.T) {
 	d.AddHTMLEntry("file", false, 64, modtime)
 	d.AddHTMLEntry("dir", true, 0, modtime)
 	assert.Equal(t, []DirEntry{
-		{remote: "file", URL: "file?potato=42", ZipURL: "", Leaf: "file", IsDir: false, Size: 64, ModTime: modtime},
-		{remote: "dir", URL: "dir/?potato=42", ZipURL: "dir/?download=zip", Leaf: "dir/", IsDir: true, Size: 0, ModTime: modtime},
+		{remote: "file", URL: "file?potato=42", ZipURL: "", Leaf: "file", IsDir: false, Size: 64, ModTime: modtime, MimeType: "application/octet-stream"},
+		{remote: "dir", URL: "dir/?potato=42", ZipURL: "dir/?download=zip", Leaf: "dir/", IsDir: true, Size: 0, ModTime: modtime, MimeType: "inode/directory"},
 	}, d.Entries)
+
+	// Now test with a link index
+	d = NewDirectory("z", GetTemplate(t)).SetLinkIndex("index.html")
+	d.AddHTMLEntry("file", false, 64, modtime)
+	d.AddHTMLEntry("dir", true, 0, modtime)
+	assert.Equal(t, []DirEntry{
+		{remote: "file", URL: "file", ZipURL: "", Leaf: "file", IsDir: false, Size: 64, ModTime: modtime, MimeType: "application/octet-stream"},
+		{remote: "dir", URL: "dir/index.html", ZipURL: "dir/?download=zip", Leaf: "dir/", IsDir: true, Size: 0, ModTime: modtime, MimeType: "inode/directory"},
+	}, d.Entries)
+	assert.Equal(t, []Crumb{{Link: "../index.html", Text: "/"}, {Link: "index.html", Text: "z"}}, d.Breadcrumb)
+	assert.Equal(t, "../index.html", d.UpLink())
+}
+
+func TestDirectoryPath(t *testing.T) {
+	d := NewDirectory("", GetTemplate(t))
+	assert.Equal(t, "/", d.Path())
+	assert.True(t, d.IsRoot())
+	assert.Equal(t, "..", d.UpLink())
+
+	d = NewDirectory("a/b", GetTemplate(t))
+	assert.Equal(t, "/a/b/", d.Path())
+	assert.False(t, d.IsRoot())
+
+	d = NewDirectory("a/b/", GetTemplate(t))
+	assert.Equal(t, "/a/b/", d.Path())
+}
+
+func TestDirectorySummary(t *testing.T) {
+	d := NewDirectory("z", GetTemplate(t))
+	d.AddHTMLEntry("file1", false, 64, time.Time{})
+	d.AddHTMLEntry("file2", false, 100, time.Time{})
+	d.AddHTMLEntry("dir", true, 0, time.Time{})
+	assert.Equal(t, 1, d.NumDirs())
+	assert.Equal(t, 2, d.NumFiles())
+	assert.Equal(t, int64(164), d.TotalSize())
+}
+
+func TestRenderStatic(t *testing.T) {
+	htmlTemplate, err := libhttp.GetTemplate("")
+	require.NoError(t, err)
+	render := func(dirRemote string, static bool) string {
+		d := NewDirectory(dirRemote, htmlTemplate)
+		d.Static = static
+		d.AddHTMLEntry(dirRemote+"/file", false, 64, time.Time{})
+		var buf bytes.Buffer
+		require.NoError(t, d.Render(&buf))
+		return buf.String()
+	}
+
+	// serve http output has an up link everywhere
+	assert.Contains(t, render("z", false), "Go up")
+	assert.Contains(t, render("", false), "Go up")
+
+	// static output has no up link at the root but has one below it
+	assert.NotContains(t, render("", true), "Go up")
+	assert.Contains(t, render("z", true), "Go up")
 }
 
 func TestAddEntry(t *testing.T) {

lib/http/template.go

@@ -30,6 +30,13 @@ to be used within the template to server pages:
 | .Sort       |              | The current sort used. This is changeable via '?sort=' parameter. Possible values: namedirfirst, name, size, time (default namedirfirst). |
 | .Order      |              | The current ordering used. This is changeable via '?order=' parameter. Possible values: asc, desc (default asc). |
 | .Query      |              | Currently unused. |
+| .Path       |              | The path of the directory from the root with leading and trailing slashes, e.g. '/dir/'. |
+| .IsRoot     |              | Boolean for if this is the root directory. |
+| .UpLink     |              | The link to the parent directory. |
+| .Static     |              | Boolean for if the listing is being written as a static page by rclone index. |
+| .NumDirs    |              | The number of directories in the listing. |
+| .NumFiles   |              | The number of files in the listing. |
+| .TotalSize  |              | The total size in bytes of the files in the listing. |
 | .Breadcrumb |              | Allows for creating a relative navigation. |
 |             | .Link        | The link of the Text relative to the root. |
 |             | .Text        | The Name of the directory. |
@@ -39,6 +46,7 @@ to be used within the template to server pages:
 |             | .IsDir       | Boolean for if an entry is a directory or not. |
 |             | .Size        | Size in bytes of the entry. |
 |             | .ModTime     | The UTC timestamp of an entry. |
+|             | .MimeType    | The MIME type of an entry guessed from its name, or 'inode/directory'. |
 
 The server also makes the following functions available so that they can be used
 within the template. These functions help extend the options for dynamic

lib/http/templates/index.html

@@ -318,10 +318,11 @@ tr.file:hover td .zip {
 					</tr>
 					</thead>
 					<tbody>
+					{{- if not (and .Static .IsRoot)}}
 					<tr>
 						<td></td>
 						<td>
-							<a href="..">
+							<a href="{{html .UpLink}}">
 								<span class="goup">Go up</span>
 							</a>
 						</td>
@@ -329,6 +330,7 @@ tr.file:hover td .zip {
 						<td class="hideable">&mdash;</td>
 						<td class="hideable"></td>
 					</tr>
+					{{- end}}
 					{{- range .Entries}}
 					<tr class="file{{if .IsDir}} dir{{end}}">
 						<td>