mirror of
https://github.com/rclone/rclone.git
synced 2026-10-09 22:45:26 -04:00
index: add a command to write static directory listings into a remote
This makes directory listings for buckets and other remotes served as static websites, for example S3 website endpoints or R2 behind Cloudflare, so they can be browsed without directory listing support on the host.
This commit is contained in:
Nick Craig-Wood committed
1 parent
ae9a50cacd
commit
1897074971
3 files changed
+212
No files matched your search
@@ -33,6 +33,7 @@ import (
|
||||
_ "github.com/rclone/rclone/cmd/gitannex"
|
||||
_ "github.com/rclone/rclone/cmd/gui"
|
||||
_ "github.com/rclone/rclone/cmd/hashsum"
|
||||
_ "github.com/rclone/rclone/cmd/index"
|
||||
_ "github.com/rclone/rclone/cmd/link"
|
||||
_ "github.com/rclone/rclone/cmd/listremotes"
|
||||
_ "github.com/rclone/rclone/cmd/ls"
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
// Package index provides the index command.
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
opt = operations.IndexOptDefault
|
||||
printTemplate string
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.StringArrayVarP(cmdFlags, &opt.Outputs, "output", "", opt.Outputs, "Listing to write in each directory as NAME=FORMAT", "")
|
||||
flags.StringArrayVarP(cmdFlags, &opt.Rules.FilterRule, "index-filter", "", nil, "Add a rule for which directories get listings", "")
|
||||
flags.StringArrayVarP(cmdFlags, &opt.Rules.FilterFrom, "index-filter-from", "", nil, "Read directory rules from a file (use - to read from stdin)", "")
|
||||
flags.StringArrayVarP(cmdFlags, &opt.Rules.ExcludeRule, "index-exclude", "", nil, "Don't write listings in directories matching pattern", "")
|
||||
flags.StringArrayVarP(cmdFlags, &opt.Rules.ExcludeFrom, "index-exclude-from", "", nil, "Read directory exclude patterns from file (use - to read from stdin)", "")
|
||||
flags.StringArrayVarP(cmdFlags, &opt.Rules.IncludeRule, "index-include", "", nil, "Only write listings in directories matching pattern", "")
|
||||
flags.StringArrayVarP(cmdFlags, &opt.Rules.IncludeFrom, "index-include-from", "", nil, "Read directory include patterns from file (use - to read from stdin)", "")
|
||||
flags.IntVarP(cmdFlags, &opt.MaxDepth, "index-max-depth", "", opt.MaxDepth, "Only write listings this many directories deep (-1 for no limit)", "")
|
||||
flags.BoolVarP(cmdFlags, &opt.LinkIndex, "link-index", "", opt.LinkIndex, "Link to dir/NAME rather than dir/ for hosts without index documents", "")
|
||||
flags.FVarP(cmdFlags, &opt.DirTime, "dir-time", "", "How to work out the time shown for a directory", "")
|
||||
flags.BoolVarP(cmdFlags, &opt.NoModTime, "no-modtime", "", opt.NoModTime, "Don't show modification times in listings", "")
|
||||
flags.BoolVarP(cmdFlags, &opt.Rewrite, "index-rewrite", "", opt.Rewrite, "Write every listing even if it is unchanged", "")
|
||||
flags.StringVarP(cmdFlags, &printTemplate, "print-template", "", "", "Print the built-in template for FORMAT and exit", "")
|
||||
}
|
||||
|
||||
//go:embed index.md
|
||||
var indexHelp string
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "index remote:path",
|
||||
Short: `Write static directory listings into a remote.`,
|
||||
Long: strings.TrimSpace(indexHelp),
|
||||
Annotations: map[string]string{
|
||||
"versionIntroduced": "v1.76",
|
||||
"groups": "Copy,Filter,Listing,Important",
|
||||
},
|
||||
RunE: func(command *cobra.Command, args []string) error {
|
||||
if printTemplate != "" {
|
||||
text, err := operations.IndexTemplate(printTemplate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Print(text)
|
||||
return nil
|
||||
}
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
fdst := cmd.NewFsDir(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
return operations.Index(context.Background(), fdst, &opt)
|
||||
})
|
||||
return nil
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
Write a directory listing, `index.html` by default, into every
|
||||
directory of `remote:path` so that it can be browsed when it is served
|
||||
as a static website, for example from an S3 website endpoint, a bucket
|
||||
behind a CDN, or a plain web server.
|
||||
|
||||
This works like `rclone sync`: each run makes the minimum changes needed
|
||||
to bring the listings into line with the files. Listings which haven't
|
||||
changed aren't rewritten, so a run with no changes makes no uploads. The
|
||||
listings never show the listing files themselves.
|
||||
|
||||
**Note** that `rclone index` treats every file with an output name
|
||||
(`index.html` by default) in the directories it indexes as its own, and
|
||||
will overwrite it or delete it without checking where it came from. To
|
||||
protect a hand-written index, exclude its directory with
|
||||
`--index-exclude`.
|
||||
|
||||
### Outputs
|
||||
|
||||
`--output NAME=FORMAT` says what to write in each directory and can be
|
||||
repeated. NAME is the file name, and FORMAT is one of the built-in
|
||||
formats below or the path of a Go template file.
|
||||
|
||||
| Format | MIME type | Contents |
|
||||
|:-------|:----------|:---------|
|
||||
| `html` | `text/html; charset=utf-8` | The HTML listing from the `rclone serve http` template |
|
||||
| `json` | `application/json` | JSON in the format of `rclone lsjson` for the directory |
|
||||
| `caddy` | `application/json` | JSON in the format of Caddy's `file_server browse` |
|
||||
|
||||
The default is `--output index.html=html`. The built-in formats are
|
||||
templates, so they can be printed with `--print-template FORMAT`, copied
|
||||
and changed. Outputs whose name ends in `.html` or `.htm` are rendered
|
||||
with Go's `html/template` and everything else with `text/template`, which
|
||||
has a `json` function for safe encoding. The template data is that of
|
||||
`rclone serve http` with `.Static` set (see
|
||||
[rclone serve http](/commands/rclone_serve_http/#template)). Outputs are
|
||||
uploaded with the MIME type of the built-in format, or that of the
|
||||
output name's extension for template outputs, since static hosts serve
|
||||
the stored type.
|
||||
|
||||
### Filters
|
||||
|
||||
The normal filters (`--include`, `--exclude`, `--filter-from`, `--max-age`, ...)
|
||||
control what appears in the listings. An excluded file isn't listed, and
|
||||
an excluded directory isn't listed or indexed.
|
||||
|
||||
The `--index-include`, `--index-exclude`, `--index-filter` flags and their
|
||||
`-from` variants control which directories get listings. They take the same
|
||||
patterns as the normal filters, applied to directory paths, and
|
||||
`--index-include` implies excluding all other directories. A directory
|
||||
excluded here still appears in its parent's listing, but nothing is written
|
||||
or deleted inside it, so it can have its own hand-made index.
|
||||
`--index-max-depth` limits how deep listings are written.
|
||||
|
||||
### Modification times
|
||||
|
||||
Listings show the modification time of each file. When using the s3,
|
||||
oracleobjectstorage and swift backends rclone stores the modification time as
|
||||
metadata which isn't returned by the bucket listing, so **every object needs
|
||||
a HEAD request**, which can mean thousands of transactions per run. Avoid
|
||||
this with one of:
|
||||
|
||||
- `--use-server-modtime` uses the time the object was uploaded instead,
|
||||
which is what the web server sends as `Last-Modified`. Recommended for s3
|
||||
unless the original modification times matter.
|
||||
- `--no-modtime` shows no times at all, which is the cheapest option.
|
||||
|
||||
`--dir-time` sets the time shown for directories:
|
||||
|
||||
- `newest` (default): the newest modification time of anything below the
|
||||
directory. Note that this means one new file changes the listing of
|
||||
every directory above it.
|
||||
- `dir`: the directory's own modification time. On most file systems
|
||||
writing a listing into a directory changes its time, so this can take
|
||||
more than one run to settle.
|
||||
- `none`: no time.
|
||||
|
||||
The listing files are given the newest modification time below their
|
||||
directory, so the `Last-Modified` header of a listing is meaningful.
|
||||
|
||||
### Links
|
||||
|
||||
Directory links are `dir/` by default, which works on hosts that serve
|
||||
index documents: S3 website endpoints, GCS and Azure static websites,
|
||||
nginx and Apache, and R2 custom domains with a URL rewrite rule. For
|
||||
hosts that don't (plain S3 REST URLs, B2 friendly URLs) use
|
||||
`--link-index` to make directory links `dir/index.html` instead.
|
||||
|
||||
### Site icon
|
||||
|
||||
The listings carry no icon or branding of their own. To give a site an
|
||||
icon, upload a `favicon.ico` to the root of the remote, which browsers
|
||||
fetch from there for every page. Exclude it from the listings with
|
||||
`--exclude /favicon.ico` so it stays on the site but isn't shown.
|
||||
|
||||
### Deleting listings
|
||||
|
||||
On backends which can't have empty directories (S3, B2 and other
|
||||
bucket-based storage) a directory only exists while it has files in it.
|
||||
Once the last file in a directory has been removed its listing is
|
||||
deleted, since the directory only existed because of it. Backends which
|
||||
can have empty directories keep a listing for each empty directory.
|
||||
|
||||
### Using with sync
|
||||
|
||||
A sync from a local build deletes files on the destination which aren't
|
||||
in the source, which includes the listings. Either index the local build
|
||||
before syncing:
|
||||
|
||||
rclone index ./public
|
||||
rclone sync ./public r2:bucket
|
||||
|
||||
or exclude the listings from the sync so it neither uploads nor deletes
|
||||
them, then index the remote:
|
||||
|
||||
rclone sync ./public r2:bucket --exclude index.html
|
||||
rclone index r2:bucket
|
||||
|
||||
### Examples
|
||||
|
||||
An S3 website bucket, using the upload times so no object needs a HEAD
|
||||
request:
|
||||
|
||||
rclone index s3:my-site --use-server-modtime
|
||||
|
||||
HTML and rclone JSON listings, setting `Cache-Control` on them:
|
||||
|
||||
rclone index s3:bucket --output index.html=html --output index.json=json --header-upload "Cache-Control: max-age=300"
|
||||
|
||||
Only list release files, and don't write listings under `/private`:
|
||||
|
||||
rclone index r2:bucket --include "*.{zip,deb,rpm}" --index-exclude "/private/**"
|
||||
|
||||
A B2 bucket served from its friendly URLs:
|
||||
|
||||
rclone index b2:bucket --link-index
|
||||
|
||||
See what would change:
|
||||
|
||||
rclone index r2:bucket --dry-run -v
|
||||
|
||||
Listings which haven't changed aren't normally rewritten, so changing
|
||||
the headers or metadata set on them needs `--index-rewrite`, which
|
||||
writes every listing a run would render whether it changed or not:
|
||||
|
||||
rclone index r2:bucket --index-rewrite --header-upload "Cache-Control: max-age=3600"
|
||||
Reference in new issue
Block a user