From 2e2a50cc441dfab989ec4f3a73448f40ceb873f3 Mon Sep 17 00:00:00 2001 From: Erol Ozcan Date: Fri, 3 Jul 2026 23:27:42 +0300 Subject: [PATCH] zoho: rate limit repeated listings of the same folder WorkDrive throttles its listing API (GET files/{id}/files) PER folder, independently of the overall request rate: at most ~19 listings of one folder are allowed in any rolling ~60s window and the 20th returns F7008 with a ~300s Retry-After (measured live - every observed trip landed exactly on the 20th listing inside a window). fstests re-lists the same working directory after almost every sub-operation, which is why the integration suite could not pass. Add a per-folder listing limiter with a true per-window cap: each window starts with --zoho-list-folder-burst listings passing back-to-back (the burst re-arms at every window boundary, so a sync re-listing one directory a few times never waits), the rest of the budget is spaced evenly across the window, and a sliding log of recent listings guarantees no rolling window ever exceeds --zoho-list-folder-limit (default 19) per --zoho-list-folder-window (default 60s) for any traffic pattern. The registry is process-wide and keyed by region+folder id so every Fs instance shares one budget per physical folder; idle entries are evicted after a window, which is lossless because Zoho's window has also cleared by then. Defaults were validated against the live service: bursts of 4-6 under the 19-per-60s cap ran clean while an over-cap probe tripped F7008 exactly at the 20th listing, and a full test_all -backends zoho run passes cleanly. Fixes #9570 --- backend/zoho/folderlist_test.go | 238 +++++++++++++++++++++++++ backend/zoho/zoho.go | 299 +++++++++++++++++++++++++++++++- 2 files changed, 531 insertions(+), 6 deletions(-) create mode 100644 backend/zoho/folderlist_test.go diff --git a/backend/zoho/folderlist_test.go b/backend/zoho/folderlist_test.go new file mode 100644 index 000000000..1ee5d71a5 --- /dev/null +++ b/backend/zoho/folderlist_test.go @@ -0,0 +1,238 @@ +package zoho + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/rclone/rclone/fs" + "github.com/stretchr/testify/assert" +) + +// maxRolling returns the largest number of grants falling inside any +// half-open (t-window, t] interval - the quantity Zoho's throttle caps. +func maxRolling(grants []time.Time, window time.Duration) int { + mx := 0 + for i := range grants { + c := 0 + for j := 0; j <= i; j++ { + if grants[i].Sub(grants[j]) < window { + c++ + } + } + mx = max(mx, c) + } + return mx +} + +func TestFolderListLimiter(t *testing.T) { + // Isolate from the process-wide registry and restore it afterwards, so this + // test neither observes nor leaves shared state for integration runs in the + // same process. + saved := folderListLimiterRegistry + folderListLimiterRegistry = &folderListLimiters{limiters: make(map[string]*folderListLimiterEntry)} + defer func() { folderListLimiterRegistry = saved }() + + window := time.Duration(defaultListFolderWindow) + newFs := func(region string) *Fs { + f := &Fs{} + f.opt.Region = region + f.opt.ListFolderWindow = defaultListFolderWindow + f.opt.ListFolderLimit = defaultListFolderLimit + f.opt.ListFolderBurst = defaultListFolderBurst + return f + } + f := newFs("eu") + + // The defaults must satisfy the limiter's own precondition (burst is carved + // out of the limit, so it has to leave at least one paced token). + assert.Less(t, defaultListFolderBurst, defaultListFolderLimit, "default burst must be below default limit") + + // Same region+folder shares one limiter; a different folder, or the same + // folder id in a different region, is keyed separately. + limA := f.folderListLimiter("A") + assert.Same(t, limA, f.folderListLimiter("A"), "same folder shares one limiter") + assert.NotSame(t, limA, f.folderListLimiter("B"), "different folder gets its own limiter") + assert.NotSame(t, limA, newFs("com").folderListLimiter("A"), "same folder id in another region is distinct") + + // Idle eviction: an entry not listed within the window is dropped on the + // next sweep. Age folder A and the last sweep by hand (no sleeping), then a + // listing of a different folder triggers the sweep. + reg := folderListLimiterRegistry + keyA := f.opt.Region + "\x00" + "A" + reg.mu.Lock() + reg.limiters[keyA].lastListed = time.Now().Add(-2 * window) + reg.lastSweep = time.Now().Add(-2 * window) + reg.mu.Unlock() + + f.folderListLimiter("C") // triggers the sweep + + reg.mu.Lock() + _, aKept := reg.limiters[keyA] + _, cKept := reg.limiters[f.opt.Region+"\x00"+"C"] + reg.mu.Unlock() + assert.False(t, aKept, "stale folder A is evicted") + assert.True(t, cKept, "freshly listed folder C is kept") +} + +// TestFolderWindowLimiterShape drives reserve with a synthetic clock and +// checks the promised flow: a burst at the window start, the rest of the +// budget paced across the window, and the burst RE-ARMING at the next window +// boundary (delayed only by the sliding safety margin). +func TestFolderWindowLimiterShape(t *testing.T) { + saved := folderListLimiterRegistry + folderListLimiterRegistry = &folderListLimiters{limiters: make(map[string]*folderListLimiterEntry)} + defer func() { folderListLimiterRegistry = saved }() + + f := &Fs{} + f.opt.Region = "eu" + f.opt.ListFolderWindow = defaultListFolderWindow + f.opt.ListFolderLimit = defaultListFolderLimit + f.opt.ListFolderBurst = defaultListFolderBurst + lim := f.folderListLimiter("shape") + + window := time.Duration(defaultListFolderWindow) + interval := window / time.Duration(defaultListFolderLimit-defaultListFolderBurst) + base := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + var grants []time.Time + reserve := func(now time.Time) time.Time { + g := lim.reserve(now) + grants = append(grants, g) + return g + } + + // Window 1: the first burst grants pass immediately... + for i := 0; i < defaultListFolderBurst; i++ { + assert.Equal(t, base, reserve(base), "burst grant %d passes immediately", i+1) + } + // ...then the paced phase spends the rest of the budget one interval apart + // (greedy demand: the caller comes back as soon as the last grant is due). + g := base + for k := 1; k <= defaultListFolderLimit-defaultListFolderBurst; k++ { + g = reserve(g) + assert.Equal(t, base.Add(time.Duration(k)*interval), g, "paced grant %d", k) + } + assert.Less(t, g.Sub(base), window, "the whole budget fits inside the window") + + // Budget exhausted: the next grant jumps to the window boundary, where the + // burst re-arms; the sliding safety log defers it by its margin so the + // oldest grant has left Zoho's window. + reburst := reserve(g) + assert.Equal(t, base.Add(window+folderListSafetyMargin), reburst, "re-burst at the next window boundary") + for i := 1; i < defaultListFolderBurst; i++ { + assert.Equal(t, reburst, reserve(reburst), "window 2 burst grant %d is immediate", i+1) + } + // And window 2's paced phase continues from there. + assert.Equal(t, reburst.Add(interval), reserve(reburst), "window 2 paced phase") + + assert.LessOrEqual(t, maxRolling(grants, window), defaultListFolderLimit, "never more than limit grants in any rolling window") +} + +// TestFolderWindowLimiterIdleResume checks the corner the sliding safety log +// exists for: consuming a full budget, going idle into the middle of a later +// window and resuming - the rolling cap must hold across the grid boundaries. +func TestFolderWindowLimiterIdleResume(t *testing.T) { + saved := folderListLimiterRegistry + folderListLimiterRegistry = &folderListLimiters{limiters: make(map[string]*folderListLimiterEntry)} + defer func() { folderListLimiterRegistry = saved }() + + f := &Fs{} + f.opt.Region = "eu" + f.opt.ListFolderWindow = defaultListFolderWindow + f.opt.ListFolderLimit = defaultListFolderLimit + f.opt.ListFolderBurst = defaultListFolderBurst + lim := f.folderListLimiter("resume") + + window := time.Duration(defaultListFolderWindow) + base := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + var grants []time.Time + g := base + // Spend window 1's full budget greedily. + for i := 0; i < defaultListFolderLimit; i++ { + g = lim.reserve(g) + grants = append(grants, g) + } + // Resume mid-window-2 after an idle gap and hammer across the 2->3 + // boundary: a fresh burst fires on resume and again at the boundary. + g = base.Add(window + window/2) + for i := 0; i < defaultListFolderLimit; i++ { + g = lim.reserve(g) + grants = append(grants, g) + } + assert.LessOrEqual(t, maxRolling(grants, window), defaultListFolderLimit, "rolling cap holds across idle resume and grid boundaries") +} + +// TestFolderWindowLimiterClamps checks the burst/limit clamps behaviourally. +func TestFolderWindowLimiterClamps(t *testing.T) { + saved := folderListLimiterRegistry + folderListLimiterRegistry = &folderListLimiters{limiters: make(map[string]*folderListLimiterEntry)} + defer func() { folderListLimiterRegistry = saved }() + + window := time.Duration(defaultListFolderWindow) + base := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + newFs := func(limit, burst int) *Fs { + f := &Fs{} + f.opt.Region = "eu" + f.opt.ListFolderWindow = defaultListFolderWindow + f.opt.ListFolderLimit = limit + f.opt.ListFolderBurst = burst + return f + } + + // burst 0 clamps to 1: pacing starts from the second listing. + lim := newFs(defaultListFolderLimit, 0).folderListLimiter("Z") + assert.Equal(t, base, lim.reserve(base)) + assert.Equal(t, base.Add(window/time.Duration(defaultListFolderLimit-1)), lim.reserve(base), "second listing is paced") + + // burst >= limit clamps to limit-1, leaving one paced token per window. + lim = newFs(5, 99).folderListLimiter("Y") + for i := 0; i < 4; i++ { + assert.Equal(t, base, lim.reserve(base), "clamped burst grant %d", i+1) + } + assert.Equal(t, base.Add(window), lim.reserve(base), "5th grant rolls into the next window") + + // limit 1 keeps a single burst token: one listing per window (+margin once + // the safety log is full). + lim = newFs(1, 3).folderListLimiter("X") + assert.Equal(t, base, lim.reserve(base)) + assert.Equal(t, base.Add(window+folderListSafetyMargin), lim.reserve(base), "limit 1 waits a whole window") +} + +// TestFolderListLimiterConcurrent hammers the process-wide, mutex-guarded +// registry and one shared limiter from many goroutines at once. Correctness is +// proven by `go test -race`; the rolling-cap invariant is asserted after the +// fact. reserve never sleeps, so this still runs in ~0s. +func TestFolderListLimiterConcurrent(t *testing.T) { + saved := folderListLimiterRegistry + folderListLimiterRegistry = &folderListLimiters{limiters: make(map[string]*folderListLimiterEntry)} + defer func() { folderListLimiterRegistry = saved }() + + f := &Fs{} + f.opt.Region = "eu" + f.opt.ListFolderWindow = fs.Duration(time.Hour) // large so nothing is evicted mid-test + f.opt.ListFolderLimit = 19 + f.opt.ListFolderBurst = 6 + + const workers, folders = 64, 5 + var wg sync.WaitGroup + for i := range workers { + wg.Add(1) + go func(i int) { + defer wg.Done() + f.folderListLimiter(fmt.Sprintf("dir-%d", i%folders)).reserve(time.Now()) + }(i) + } + wg.Wait() + + // Exactly one shared limiter per distinct folder, whatever the interleaving. + assert.Len(t, folderListLimiterRegistry.limiters, folders) + // And each limiter granted every reservation without breaching its own cap: + // grants are capped at limit entries and non-decreasing. + for _, e := range folderListLimiterRegistry.limiters { + assert.LessOrEqual(t, len(e.limiter.grants), e.limiter.limit) + for i := 1; i < len(e.limiter.grants); i++ { + assert.False(t, e.limiter.grants[i].Before(e.limiter.grants[i-1]), "grants are non-decreasing") + } + } +} diff --git a/backend/zoho/zoho.go b/backend/zoho/zoho.go index d35be9226..dc82965d0 100644 --- a/backend/zoho/zoho.go +++ b/backend/zoho/zoho.go @@ -12,6 +12,7 @@ import ( "path" "strconv" "strings" + "sync" "sync/atomic" "time" @@ -54,6 +55,28 @@ const ( // caused synchronized 429 clusters. defaultTPSLimitBurst = 1 + // defaultListFolderWindow / defaultListFolderLimit / defaultListFolderBurst + // configure the per-folder listing limiter; folderListLimiter documents the + // measured throttle model. Zoho allows ~19 listings of one folder in any + // rolling ~60s window and the 20th returns F7008 (measured live 2026-07-05: + // every trip landed exactly on the 20th listing inside 60s, every clean run + // stayed at <=19 - including a deliberate over-limit probe tripping at #20). + // The limiter guarantees at most limit listings per window: each window + // starts with burst listings passing immediately (the burst RE-ARMS at every + // window boundary) and the remaining limit-burst are paced window/(limit-burst) + // apart, while a sliding log of the last limit grants enforces the cap across + // window boundaries and idle resumes. burst never raises the per-window total - + // it only sets how many listings may go back-to-back. Bursts of 4, 5 and 6 + // under this cap all ran clean live; 6 is the largest validated, hence the + // default. Only repeated same-folder listings are paced; set limit to 0 to + // disable entirely. + defaultListFolderWindow = fs.Duration(60 * time.Second) + defaultListFolderLimit = 19 // 0 = disabled + defaultListFolderBurst = 6 + // folderListLimitersMaxEntries caps the per-folder limiter map against unbounded + // growth on very long-lived processes. + folderListLimitersMaxEntries = 100000 + // retryAfterMargin adds a small extra delay after Zoho's Retry-After value. // This gives rate-limit tokens time to refill and helps avoid another 429. retryAfterMargin = 1 * time.Second @@ -259,6 +282,63 @@ lets several calls fire at once after an idle gap, which can trigger synchronized clusters of 429 errors.`, Default: defaultTPSLimitBurst, Advanced: true, + }, { + Name: "list_folder_limit", + Help: `Max listings of the SAME folder allowed per --zoho-list-folder-window. + +Zoho WorkDrive rate limits its listing API (GET files/{id}/files) PER +folder, independently of --zoho-tpslimit: listing one folder too often in a +short time returns HTTP 429 (error F7008) with a multi-minute Retry-After +penalty, which a tight polling loop can hit even at a low overall rate. +Measured live, Zoho allows ~19 listings of one folder in any rolling ~60s +window and the 20th fails, which the defaults (19 per 60s) model exactly. + +This is a true per-window cap for any traffic pattern: each window starts +with --zoho-list-folder-burst listings passing back-to-back (the burst +re-arms at every window boundary) and the rest are spaced +--zoho-list-folder-window/(limit - burst) apart (the defaults give ~4.6s), +while a sliding log of recent listings enforces the cap across window +boundaries. 0 disables the limiter. Only REPEATED listings of one folder are +delayed; different folders, or a folder listed fewer than +--zoho-list-folder-burst times, never are. + +A HIGHER value means MORE listings per window, not more safety: raising it +above 19 trips F7008. Lower it for a wider margin at the cost of listing +responsiveness.`, + Default: defaultListFolderLimit, + Advanced: true, + }, { + Name: "list_folder_window", + Help: `The window for --zoho-list-folder-limit. + +The default of 60s (shown as 1m0s) matches Zoho's real sliding window: at +most --zoho-list-folder-limit listings of one folder are allowed in any +window of this length. A bare number is parsed as seconds ("60" = "60s"). + +Widen it (or lower the limit) for a bigger safety margin; the sustained +spacing between same-folder listings is window/(limit - burst).`, + Default: defaultListFolderWindow, + Advanced: true, + }, { + Name: "list_folder_burst", + Help: `Same-folder listings allowed back-to-back before --zoho-list-folder-limit paces them. + +The burst is carved out of --zoho-list-folder-limit, so raising it never +raises the per-window total: this many listings may fire immediately and the +remaining limit - burst are spaced window/(limit - burst) apart. The burst +RE-ARMS at every window boundary, so sustained re-listing gets a fresh burst +each window while a sliding log of recent listings still enforces the +per-window cap. A folder listed only a handful of times (the common case - a +sync re-listing one directory a few times then moving on) never waits. + +The default 6 is the largest burst validated live under the default 19-per-60s +cap (bursts of 4, 5 and 6 all ran clean; an over-cap probe tripped F7008 +exactly at the 20th listing in a window). Keep it well below ~15 - Zoho also +has an instantaneous back-to-back cap around 15-16 regardless of the window. +Set to 1 to pace from the second listing. Values >= the limit are clamped to +limit - 1.`, + Default: defaultListFolderBurst, + Advanced: true, }, { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, @@ -273,12 +353,15 @@ synchronized clusters of 429 errors.`, // Options defines the configuration for this backend type Options struct { - UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` - RootFolderID string `config:"root_folder_id"` - Region string `config:"region"` - TPSLimit float64 `config:"tpslimit"` - TPSLimitBurst int `config:"tpslimit_burst"` - Enc encoder.MultiEncoder `config:"encoding"` + UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` + RootFolderID string `config:"root_folder_id"` + Region string `config:"region"` + TPSLimit float64 `config:"tpslimit"` + TPSLimitBurst int `config:"tpslimit_burst"` + ListFolderLimit int `config:"list_folder_limit"` + ListFolderWindow fs.Duration `config:"list_folder_window"` + ListFolderBurst int `config:"list_folder_burst"` + Enc encoder.MultiEncoder `config:"encoding"` } // Fs represents a remote workdrive @@ -305,6 +388,131 @@ type throttleState struct { progress atomic.Bool // a call succeeded after the last penalty window } +// folderListLimiters holds the per-folder-id listing rate limiters and the time +// each folder was last listed. One instance is shared process-wide (folderListLimiterRegistry) +// rather than per-*Fs because Zoho keys its listing throttle to the folder id: +// a per-Fs limiter would let each new Fs re-list a shared ancestor with a fresh, +// full-burst limiter, draining the bucket -> F7008. +type folderListLimiters struct { + mu sync.Mutex + limiters map[string]*folderListLimiterEntry + lastSweep time.Time +} + +// folderListLimiterRegistry is the process-wide registry of per-folder listing +// limiters, shared by every *Fs and keyed by region+folder-id so entries never +// collide across accounts or regions. +var folderListLimiterRegistry = &folderListLimiters{limiters: make(map[string]*folderListLimiterEntry)} + +// folderListLimiterEntry is a per-folder listing rate limiter plus the time the folder +// was last listed, used for idle eviction. +type folderListLimiterEntry struct { + limiter *folderWindowLimiter + lastListed time.Time +} + +// folderWindowLimiter shapes same-folder listings to at most limit per window +// with a burst re-armed at each window boundary. Two layers: +// +// Shape (fixed window): a grid anchored at the first request advances in whole +// windows; each window's first burst grants pass immediately and the remaining +// limit-burst are spaced interval = window/(limit-burst) apart, so under +// continuous demand every window starts with a fast burst - the flow the +// options promise. +// +// Safety (sliding log): grants keeps the last limit grant times; a new grant +// is also forced to be >= grants[len-limit] + window + margin, so no rolling +// window ever sees more than limit grants even across grid boundaries or on +// resume-after-idle, where the fixed-window shape alone could cluster a +// boundary burst too close to earlier grants. +type folderWindowLimiter struct { + mu sync.Mutex + window time.Duration // Zoho's per-folder window (--zoho-list-folder-window) + limit int // window budget AND sliding cap (--zoho-list-folder-limit) + burst int // grants passed immediately at each window start + interval time.Duration // spacing of the paced phase: window/(limit-burst) + windowStart time.Time // fixed grid anchor; zero until the first grant + used int // grants handed out in the current window + lastGrant time.Time // time of the most recent grant + grants []time.Time // last <=limit grant times (sliding safety log) +} + +// folderListSafetyMargin is added on top of the sliding-log wait so a grant +// lands strictly after the matching old grant has left Zoho's window. +const folderListSafetyMargin = 500 * time.Millisecond + +// reserve commits the next grant at or after now and returns its time. +// It converges in at most a few passes: rolling the grid forward resets the +// window budget, and the sliding log can push the candidate at most once more. +func (l *folderWindowLimiter) reserve(now time.Time) time.Time { + l.mu.Lock() + defer l.mu.Unlock() + if l.windowStart.IsZero() { + l.windowStart = now + } + cand := now + for range 4 { + // Roll the fixed grid forward so cand lies in the current window, + // re-arming the burst for the new window. + if d := cand.Sub(l.windowStart); d >= l.window { + l.windowStart = l.windowStart.Add(l.window * (d / l.window)) + l.used = 0 + } + next := cand + switch { + case l.used < l.burst: + // window-start burst passes immediately + case l.used < l.limit: + if t := l.lastGrant.Add(l.interval); t.After(next) { + next = t + } + default: + // window budget exhausted - wait for the next window's burst + next = l.windowStart.Add(l.window) + } + // Sliding safety: never more than limit grants in any rolling window. + if len(l.grants) >= l.limit { + if t := l.grants[len(l.grants)-l.limit].Add(l.window + folderListSafetyMargin); t.After(next) { + next = t + } + } + if next.Equal(cand) { + break + } + cand = next + } + l.used++ + l.lastGrant = cand + if len(l.grants) < l.limit { + l.grants = append(l.grants, cand) + } else { + copy(l.grants, l.grants[1:]) + l.grants[len(l.grants)-1] = cand + } + return cand +} + +// Wait blocks until the limiter grants a listing or ctx is cancelled. A grant +// reserved by a cancelled Wait is not returned - erring on the safe side, it +// just leaves a little of the window budget unused. +func (l *folderWindowLimiter) Wait(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + d := time.Until(l.reserve(time.Now())) + if d <= 0 { + return nil + } + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + // Object describes a Zoho WorkDrive object // // Will definitely have info but maybe not meta @@ -684,10 +892,89 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // Should return true to finish processing type listAllFn func(*api.Item) bool +// folderListLimiter provides the shared per-folder listing rate limiter for dirID. +// +// Zoho WorkDrive throttles its listing API (GET files/{id}/files) PER folder, +// independently of --zoho-tpslimit. Measured live (2026-07-05), that per-folder +// throttle is a sliding window: ~19 listings of one folder within any rolling +// ~60s are allowed and the 20th returns F7008 with a ~300s Retry-After. Every +// observed trip landed exactly on the 20th listing inside 60s and every clean +// run stayed at <=19, including a deliberate over-limit probe (window total 22) +// tripping at #20; an earlier token-bucket refill model fitted the same data +// inconsistently. There is also a much smaller instantaneous cap (~15-16 +// back-to-back listings trip on their own), so burst must stay well below that. +// +// The limiter (folderWindowLimiter) guarantees at most limit listings per +// window for any traffic pattern: each window starts with burst listings +// passing back-to-back - the burst RE-ARMS at every window boundary - and the +// remaining limit-burst are paced window/(limit-burst) apart, while the +// sliding log of the last limit grants enforces the cap across window +// boundaries and idle resumes. Raising burst therefore never raises the +// per-window total - it only front-loads it, widening the sustained spacing +// to compensate. Different folders, or a folder listed fewer than burst +// times, are never delayed - only tps applies unless the SAME folder is +// re-listed enough to drain the burst. +// +// It paces repeated listings of the same folder; without it, listing one folder +// too often trips Zoho's per-folder throttle and returns HTTP 429. The limiter is +// created on demand in folderListLimiterRegistry, and limiters idle longer than +// the window are evicted (at most once per window): after a full window idle +// Zoho's sliding window is empty and the limiter is fully refilled, so dropping +// it is lossless. Only called when --zoho-list-folder-limit > 0, so limit is +// never zero. +// +// Returns the folder's *folderWindowLimiter, ready to Wait on before listing. +func (f *Fs) folderListLimiter(dirID string) *folderWindowLimiter { + flReg := folderListLimiterRegistry + window := time.Duration(f.opt.ListFolderWindow) + // Key by region+folder-id so one limiter is shared per physical folder + // across every *Fs without colliding across accounts or regions. + key := f.opt.Region + "\x00" + dirID + flReg.mu.Lock() + defer flReg.mu.Unlock() + now := time.Now() + if now.Sub(flReg.lastSweep) > window || len(flReg.limiters) > folderListLimitersMaxEntries { + before := len(flReg.limiters) + for id, e := range flReg.limiters { + if now.Sub(e.lastListed) > window { + delete(flReg.limiters, id) + } + } + flReg.lastSweep = now + fs.Debugf(f, "folder listing limiter sweep: evicted %d of %d idle limiters", before-len(flReg.limiters), before) + } + e := flReg.limiters[key] + if e == nil { + // The first Fs to list this folder fixes the shape; in the normal case + // every Fs for the same account shares the same limit/window/burst. + // limit > 0 is guaranteed by the caller; clamp burst into [1, limit-1] + // (at limit 1 the burst token itself is the whole window's budget). + burst := min(max(f.opt.ListFolderBurst, 1), max(f.opt.ListFolderLimit-1, 1)) + intervalTokens := max(f.opt.ListFolderLimit-burst, 1) + e = &folderListLimiterEntry{limiter: &folderWindowLimiter{ + window: window, + limit: f.opt.ListFolderLimit, + burst: burst, + interval: window / time.Duration(intervalTokens), + }} + flReg.limiters[key] = e + } + e.lastListed = now + return e.limiter +} + // Lists the directory required calling the user function on each item found // // If the user fn ever returns true then it early exits with found = true func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { + // Pace repeated listings of the SAME folder under Zoho's per-folder throttle + // (see --zoho-list-folder-limit). One token per listAll call, not per page, + // so multi-page listings of a single folder are not penalised. + if f.opt.ListFolderLimit > 0 { + if err = f.folderListLimiter(dirID).Wait(ctx); err != nil { + return false, err + } + } const listItemsLimit = 1000 opts := rest.Opts{ Method: "GET",