diff --git a/cmd/all/all.go b/cmd/all/all.go index 912cde629..d17601dfe 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -33,6 +33,7 @@ import ( _ "github.com/rclone/rclone/cmd/gitannex" _ "github.com/rclone/rclone/cmd/gui" _ "github.com/rclone/rclone/cmd/hashsum" + _ "github.com/rclone/rclone/cmd/index" _ "github.com/rclone/rclone/cmd/link" _ "github.com/rclone/rclone/cmd/listremotes" _ "github.com/rclone/rclone/cmd/ls" diff --git a/cmd/index/index.go b/cmd/index/index.go new file mode 100644 index 000000000..72cb07ede --- /dev/null +++ b/cmd/index/index.go @@ -0,0 +1,66 @@ +// Package index provides the index command. +package index + +import ( + "context" + _ "embed" + "fmt" + "strings" + + "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/fs/config/flags" + "github.com/rclone/rclone/fs/operations" + "github.com/spf13/cobra" +) + +var ( + opt = operations.IndexOptDefault + printTemplate string +) + +func init() { + cmd.Root.AddCommand(commandDefinition) + cmdFlags := commandDefinition.Flags() + flags.StringArrayVarP(cmdFlags, &opt.Outputs, "output", "", opt.Outputs, "Listing to write in each directory as NAME=FORMAT", "") + flags.StringArrayVarP(cmdFlags, &opt.Rules.FilterRule, "index-filter", "", nil, "Add a rule for which directories get listings", "") + flags.StringArrayVarP(cmdFlags, &opt.Rules.FilterFrom, "index-filter-from", "", nil, "Read directory rules from a file (use - to read from stdin)", "") + flags.StringArrayVarP(cmdFlags, &opt.Rules.ExcludeRule, "index-exclude", "", nil, "Don't write listings in directories matching pattern", "") + flags.StringArrayVarP(cmdFlags, &opt.Rules.ExcludeFrom, "index-exclude-from", "", nil, "Read directory exclude patterns from file (use - to read from stdin)", "") + flags.StringArrayVarP(cmdFlags, &opt.Rules.IncludeRule, "index-include", "", nil, "Only write listings in directories matching pattern", "") + flags.StringArrayVarP(cmdFlags, &opt.Rules.IncludeFrom, "index-include-from", "", nil, "Read directory include patterns from file (use - to read from stdin)", "") + flags.IntVarP(cmdFlags, &opt.MaxDepth, "index-max-depth", "", opt.MaxDepth, "Only write listings this many directories deep (-1 for no limit)", "") + flags.BoolVarP(cmdFlags, &opt.LinkIndex, "link-index", "", opt.LinkIndex, "Link to dir/NAME rather than dir/ for hosts without index documents", "") + flags.FVarP(cmdFlags, &opt.DirTime, "dir-time", "", "How to work out the time shown for a directory", "") + flags.BoolVarP(cmdFlags, &opt.NoModTime, "no-modtime", "", opt.NoModTime, "Don't show modification times in listings", "") + flags.BoolVarP(cmdFlags, &opt.Rewrite, "index-rewrite", "", opt.Rewrite, "Write every listing even if it is unchanged", "") + flags.StringVarP(cmdFlags, &printTemplate, "print-template", "", "", "Print the built-in template for FORMAT and exit", "") +} + +//go:embed index.md +var indexHelp string + +var commandDefinition = &cobra.Command{ + Use: "index remote:path", + Short: `Write static directory listings into a remote.`, + Long: strings.TrimSpace(indexHelp), + Annotations: map[string]string{ + "versionIntroduced": "v1.76", + "groups": "Copy,Filter,Listing,Important", + }, + RunE: func(command *cobra.Command, args []string) error { + if printTemplate != "" { + text, err := operations.IndexTemplate(printTemplate) + if err != nil { + return err + } + fmt.Print(text) + return nil + } + cmd.CheckArgs(1, 1, command, args) + fdst := cmd.NewFsDir(args) + cmd.Run(true, true, command, func() error { + return operations.Index(context.Background(), fdst, &opt) + }) + return nil + }, +} diff --git a/cmd/index/index.md b/cmd/index/index.md new file mode 100644 index 000000000..5bb61cf5b --- /dev/null +++ b/cmd/index/index.md @@ -0,0 +1,145 @@ +Write a directory listing, `index.html` by default, into every +directory of `remote:path` so that it can be browsed when it is served +as a static website, for example from an S3 website endpoint, a bucket +behind a CDN, or a plain web server. + +This works like `rclone sync`: each run makes the minimum changes needed +to bring the listings into line with the files. Listings which haven't +changed aren't rewritten, so a run with no changes makes no uploads. The +listings never show the listing files themselves. + +**Note** that `rclone index` treats every file with an output name +(`index.html` by default) in the directories it indexes as its own, and +will overwrite it or delete it without checking where it came from. To +protect a hand-written index, exclude its directory with +`--index-exclude`. + +### Outputs + +`--output NAME=FORMAT` says what to write in each directory and can be +repeated. NAME is the file name, and FORMAT is one of the built-in +formats below or the path of a Go template file. + +| Format | MIME type | Contents | +|:-------|:----------|:---------| +| `html` | `text/html; charset=utf-8` | The HTML listing from the `rclone serve http` template | +| `json` | `application/json` | JSON in the format of `rclone lsjson` for the directory | +| `caddy` | `application/json` | JSON in the format of Caddy's `file_server browse` | + +The default is `--output index.html=html`. The built-in formats are +templates, so they can be printed with `--print-template FORMAT`, copied +and changed. Outputs whose name ends in `.html` or `.htm` are rendered +with Go's `html/template` and everything else with `text/template`, which +has a `json` function for safe encoding. The template data is that of +`rclone serve http` with `.Static` set (see +[rclone serve http](/commands/rclone_serve_http/#template)). Outputs are +uploaded with the MIME type of the built-in format, or that of the +output name's extension for template outputs, since static hosts serve +the stored type. + +### Filters + +The normal filters (`--include`, `--exclude`, `--filter-from`, `--max-age`, ...) +control what appears in the listings. An excluded file isn't listed, and +an excluded directory isn't listed or indexed. + +The `--index-include`, `--index-exclude`, `--index-filter` flags and their +`-from` variants control which directories get listings. They take the same +patterns as the normal filters, applied to directory paths, and +`--index-include` implies excluding all other directories. A directory +excluded here still appears in its parent's listing, but nothing is written +or deleted inside it, so it can have its own hand-made index. +`--index-max-depth` limits how deep listings are written. + +### Modification times + +Listings show the modification time of each file. When using the s3, +oracleobjectstorage and swift backends rclone stores the modification time as +metadata which isn't returned by the bucket listing, so **every object needs +a HEAD request**, which can mean thousands of transactions per run. Avoid +this with one of: + +- `--use-server-modtime` uses the time the object was uploaded instead, + which is what the web server sends as `Last-Modified`. Recommended for s3 + unless the original modification times matter. +- `--no-modtime` shows no times at all, which is the cheapest option. + +`--dir-time` sets the time shown for directories: + +- `newest` (default): the newest modification time of anything below the + directory. Note that this means one new file changes the listing of + every directory above it. +- `dir`: the directory's own modification time. On most file systems + writing a listing into a directory changes its time, so this can take + more than one run to settle. +- `none`: no time. + +The listing files are given the newest modification time below their +directory, so the `Last-Modified` header of a listing is meaningful. + +### Links + +Directory links are `dir/` by default, which works on hosts that serve +index documents: S3 website endpoints, GCS and Azure static websites, +nginx and Apache, and R2 custom domains with a URL rewrite rule. For +hosts that don't (plain S3 REST URLs, B2 friendly URLs) use +`--link-index` to make directory links `dir/index.html` instead. + +### Site icon + +The listings carry no icon or branding of their own. To give a site an +icon, upload a `favicon.ico` to the root of the remote, which browsers +fetch from there for every page. Exclude it from the listings with +`--exclude /favicon.ico` so it stays on the site but isn't shown. + +### Deleting listings + +On backends which can't have empty directories (S3, B2 and other +bucket-based storage) a directory only exists while it has files in it. +Once the last file in a directory has been removed its listing is +deleted, since the directory only existed because of it. Backends which +can have empty directories keep a listing for each empty directory. + +### Using with sync + +A sync from a local build deletes files on the destination which aren't +in the source, which includes the listings. Either index the local build +before syncing: + + rclone index ./public + rclone sync ./public r2:bucket + +or exclude the listings from the sync so it neither uploads nor deletes +them, then index the remote: + + rclone sync ./public r2:bucket --exclude index.html + rclone index r2:bucket + +### Examples + +An S3 website bucket, using the upload times so no object needs a HEAD +request: + + rclone index s3:my-site --use-server-modtime + +HTML and rclone JSON listings, setting `Cache-Control` on them: + + rclone index s3:bucket --output index.html=html --output index.json=json --header-upload "Cache-Control: max-age=300" + +Only list release files, and don't write listings under `/private`: + + rclone index r2:bucket --include "*.{zip,deb,rpm}" --index-exclude "/private/**" + +A B2 bucket served from its friendly URLs: + + rclone index b2:bucket --link-index + +See what would change: + + rclone index r2:bucket --dry-run -v + +Listings which haven't changed aren't normally rewritten, so changing +the headers or metadata set on them needs `--index-rewrite`, which +writes every listing a run would render whether it changed or not: + + rclone index r2:bucket --index-rewrite --header-upload "Cache-Control: max-age=3600"