mirror of
https://github.com/rclone/rclone.git
synced 2026-07-09 23:35:06 -04:00
zoho: rate limit repeated listings of the same folder
WorkDrive throttles its listing API (GET files/{id}/files) PER
folder, independently of the overall request rate: at most ~19
listings of one folder are allowed in any rolling ~60s window and
the 20th returns F7008 with a ~300s Retry-After (measured live -
every observed trip landed exactly on the 20th listing inside a
window). fstests re-lists the same working directory after almost
every sub-operation, which is why the integration suite could not
pass.
Add a per-folder listing limiter with a true per-window cap: each
window starts with --zoho-list-folder-burst listings passing
back-to-back (the burst re-arms at every window boundary, so a
sync re-listing one directory a few times never waits), the rest
of the budget is spaced evenly across the window, and a sliding
log of recent listings guarantees no rolling window ever exceeds
--zoho-list-folder-limit (default 19) per --zoho-list-folder-window
(default 60s) for any traffic pattern. The registry is
process-wide and keyed by region+folder id so every Fs instance
shares one budget per physical folder; idle entries are evicted
after a window, which is lossless because Zoho's window has also
cleared by then.
Defaults were validated against the live service: bursts of 4-6
under the 19-per-60s cap ran clean while an over-cap probe tripped
F7008 exactly at the 20th listing, and a full test_all -backends
zoho run passes cleanly.
Fixes #9570
This commit is contained in:
committed by
Nick Craig-Wood
parent
1daa03f108
commit
2e2a50cc44
238
backend/zoho/folderlist_test.go
Normal file
238
backend/zoho/folderlist_test.go
Normal file
@@ -0,0 +1,238 @@
|
||||
package zoho
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// maxRolling returns the largest number of grants falling inside any
|
||||
// half-open (t-window, t] interval - the quantity Zoho's throttle caps.
|
||||
func maxRolling(grants []time.Time, window time.Duration) int {
|
||||
mx := 0
|
||||
for i := range grants {
|
||||
c := 0
|
||||
for j := 0; j <= i; j++ {
|
||||
if grants[i].Sub(grants[j]) < window {
|
||||
c++
|
||||
}
|
||||
}
|
||||
mx = max(mx, c)
|
||||
}
|
||||
return mx
|
||||
}
|
||||
|
||||
func TestFolderListLimiter(t *testing.T) {
|
||||
// Isolate from the process-wide registry and restore it afterwards, so this
|
||||
// test neither observes nor leaves shared state for integration runs in the
|
||||
// same process.
|
||||
saved := folderListLimiterRegistry
|
||||
folderListLimiterRegistry = &folderListLimiters{limiters: make(map[string]*folderListLimiterEntry)}
|
||||
defer func() { folderListLimiterRegistry = saved }()
|
||||
|
||||
window := time.Duration(defaultListFolderWindow)
|
||||
newFs := func(region string) *Fs {
|
||||
f := &Fs{}
|
||||
f.opt.Region = region
|
||||
f.opt.ListFolderWindow = defaultListFolderWindow
|
||||
f.opt.ListFolderLimit = defaultListFolderLimit
|
||||
f.opt.ListFolderBurst = defaultListFolderBurst
|
||||
return f
|
||||
}
|
||||
f := newFs("eu")
|
||||
|
||||
// The defaults must satisfy the limiter's own precondition (burst is carved
|
||||
// out of the limit, so it has to leave at least one paced token).
|
||||
assert.Less(t, defaultListFolderBurst, defaultListFolderLimit, "default burst must be below default limit")
|
||||
|
||||
// Same region+folder shares one limiter; a different folder, or the same
|
||||
// folder id in a different region, is keyed separately.
|
||||
limA := f.folderListLimiter("A")
|
||||
assert.Same(t, limA, f.folderListLimiter("A"), "same folder shares one limiter")
|
||||
assert.NotSame(t, limA, f.folderListLimiter("B"), "different folder gets its own limiter")
|
||||
assert.NotSame(t, limA, newFs("com").folderListLimiter("A"), "same folder id in another region is distinct")
|
||||
|
||||
// Idle eviction: an entry not listed within the window is dropped on the
|
||||
// next sweep. Age folder A and the last sweep by hand (no sleeping), then a
|
||||
// listing of a different folder triggers the sweep.
|
||||
reg := folderListLimiterRegistry
|
||||
keyA := f.opt.Region + "\x00" + "A"
|
||||
reg.mu.Lock()
|
||||
reg.limiters[keyA].lastListed = time.Now().Add(-2 * window)
|
||||
reg.lastSweep = time.Now().Add(-2 * window)
|
||||
reg.mu.Unlock()
|
||||
|
||||
f.folderListLimiter("C") // triggers the sweep
|
||||
|
||||
reg.mu.Lock()
|
||||
_, aKept := reg.limiters[keyA]
|
||||
_, cKept := reg.limiters[f.opt.Region+"\x00"+"C"]
|
||||
reg.mu.Unlock()
|
||||
assert.False(t, aKept, "stale folder A is evicted")
|
||||
assert.True(t, cKept, "freshly listed folder C is kept")
|
||||
}
|
||||
|
||||
// TestFolderWindowLimiterShape drives reserve with a synthetic clock and
|
||||
// checks the promised flow: a burst at the window start, the rest of the
|
||||
// budget paced across the window, and the burst RE-ARMING at the next window
|
||||
// boundary (delayed only by the sliding safety margin).
|
||||
func TestFolderWindowLimiterShape(t *testing.T) {
|
||||
saved := folderListLimiterRegistry
|
||||
folderListLimiterRegistry = &folderListLimiters{limiters: make(map[string]*folderListLimiterEntry)}
|
||||
defer func() { folderListLimiterRegistry = saved }()
|
||||
|
||||
f := &Fs{}
|
||||
f.opt.Region = "eu"
|
||||
f.opt.ListFolderWindow = defaultListFolderWindow
|
||||
f.opt.ListFolderLimit = defaultListFolderLimit
|
||||
f.opt.ListFolderBurst = defaultListFolderBurst
|
||||
lim := f.folderListLimiter("shape")
|
||||
|
||||
window := time.Duration(defaultListFolderWindow)
|
||||
interval := window / time.Duration(defaultListFolderLimit-defaultListFolderBurst)
|
||||
base := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
var grants []time.Time
|
||||
reserve := func(now time.Time) time.Time {
|
||||
g := lim.reserve(now)
|
||||
grants = append(grants, g)
|
||||
return g
|
||||
}
|
||||
|
||||
// Window 1: the first burst grants pass immediately...
|
||||
for i := 0; i < defaultListFolderBurst; i++ {
|
||||
assert.Equal(t, base, reserve(base), "burst grant %d passes immediately", i+1)
|
||||
}
|
||||
// ...then the paced phase spends the rest of the budget one interval apart
|
||||
// (greedy demand: the caller comes back as soon as the last grant is due).
|
||||
g := base
|
||||
for k := 1; k <= defaultListFolderLimit-defaultListFolderBurst; k++ {
|
||||
g = reserve(g)
|
||||
assert.Equal(t, base.Add(time.Duration(k)*interval), g, "paced grant %d", k)
|
||||
}
|
||||
assert.Less(t, g.Sub(base), window, "the whole budget fits inside the window")
|
||||
|
||||
// Budget exhausted: the next grant jumps to the window boundary, where the
|
||||
// burst re-arms; the sliding safety log defers it by its margin so the
|
||||
// oldest grant has left Zoho's window.
|
||||
reburst := reserve(g)
|
||||
assert.Equal(t, base.Add(window+folderListSafetyMargin), reburst, "re-burst at the next window boundary")
|
||||
for i := 1; i < defaultListFolderBurst; i++ {
|
||||
assert.Equal(t, reburst, reserve(reburst), "window 2 burst grant %d is immediate", i+1)
|
||||
}
|
||||
// And window 2's paced phase continues from there.
|
||||
assert.Equal(t, reburst.Add(interval), reserve(reburst), "window 2 paced phase")
|
||||
|
||||
assert.LessOrEqual(t, maxRolling(grants, window), defaultListFolderLimit, "never more than limit grants in any rolling window")
|
||||
}
|
||||
|
||||
// TestFolderWindowLimiterIdleResume checks the corner the sliding safety log
|
||||
// exists for: consuming a full budget, going idle into the middle of a later
|
||||
// window and resuming - the rolling cap must hold across the grid boundaries.
|
||||
func TestFolderWindowLimiterIdleResume(t *testing.T) {
|
||||
saved := folderListLimiterRegistry
|
||||
folderListLimiterRegistry = &folderListLimiters{limiters: make(map[string]*folderListLimiterEntry)}
|
||||
defer func() { folderListLimiterRegistry = saved }()
|
||||
|
||||
f := &Fs{}
|
||||
f.opt.Region = "eu"
|
||||
f.opt.ListFolderWindow = defaultListFolderWindow
|
||||
f.opt.ListFolderLimit = defaultListFolderLimit
|
||||
f.opt.ListFolderBurst = defaultListFolderBurst
|
||||
lim := f.folderListLimiter("resume")
|
||||
|
||||
window := time.Duration(defaultListFolderWindow)
|
||||
base := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
var grants []time.Time
|
||||
g := base
|
||||
// Spend window 1's full budget greedily.
|
||||
for i := 0; i < defaultListFolderLimit; i++ {
|
||||
g = lim.reserve(g)
|
||||
grants = append(grants, g)
|
||||
}
|
||||
// Resume mid-window-2 after an idle gap and hammer across the 2->3
|
||||
// boundary: a fresh burst fires on resume and again at the boundary.
|
||||
g = base.Add(window + window/2)
|
||||
for i := 0; i < defaultListFolderLimit; i++ {
|
||||
g = lim.reserve(g)
|
||||
grants = append(grants, g)
|
||||
}
|
||||
assert.LessOrEqual(t, maxRolling(grants, window), defaultListFolderLimit, "rolling cap holds across idle resume and grid boundaries")
|
||||
}
|
||||
|
||||
// TestFolderWindowLimiterClamps checks the burst/limit clamps behaviourally.
|
||||
func TestFolderWindowLimiterClamps(t *testing.T) {
|
||||
saved := folderListLimiterRegistry
|
||||
folderListLimiterRegistry = &folderListLimiters{limiters: make(map[string]*folderListLimiterEntry)}
|
||||
defer func() { folderListLimiterRegistry = saved }()
|
||||
|
||||
window := time.Duration(defaultListFolderWindow)
|
||||
base := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
newFs := func(limit, burst int) *Fs {
|
||||
f := &Fs{}
|
||||
f.opt.Region = "eu"
|
||||
f.opt.ListFolderWindow = defaultListFolderWindow
|
||||
f.opt.ListFolderLimit = limit
|
||||
f.opt.ListFolderBurst = burst
|
||||
return f
|
||||
}
|
||||
|
||||
// burst 0 clamps to 1: pacing starts from the second listing.
|
||||
lim := newFs(defaultListFolderLimit, 0).folderListLimiter("Z")
|
||||
assert.Equal(t, base, lim.reserve(base))
|
||||
assert.Equal(t, base.Add(window/time.Duration(defaultListFolderLimit-1)), lim.reserve(base), "second listing is paced")
|
||||
|
||||
// burst >= limit clamps to limit-1, leaving one paced token per window.
|
||||
lim = newFs(5, 99).folderListLimiter("Y")
|
||||
for i := 0; i < 4; i++ {
|
||||
assert.Equal(t, base, lim.reserve(base), "clamped burst grant %d", i+1)
|
||||
}
|
||||
assert.Equal(t, base.Add(window), lim.reserve(base), "5th grant rolls into the next window")
|
||||
|
||||
// limit 1 keeps a single burst token: one listing per window (+margin once
|
||||
// the safety log is full).
|
||||
lim = newFs(1, 3).folderListLimiter("X")
|
||||
assert.Equal(t, base, lim.reserve(base))
|
||||
assert.Equal(t, base.Add(window+folderListSafetyMargin), lim.reserve(base), "limit 1 waits a whole window")
|
||||
}
|
||||
|
||||
// TestFolderListLimiterConcurrent hammers the process-wide, mutex-guarded
|
||||
// registry and one shared limiter from many goroutines at once. Correctness is
|
||||
// proven by `go test -race`; the rolling-cap invariant is asserted after the
|
||||
// fact. reserve never sleeps, so this still runs in ~0s.
|
||||
func TestFolderListLimiterConcurrent(t *testing.T) {
|
||||
saved := folderListLimiterRegistry
|
||||
folderListLimiterRegistry = &folderListLimiters{limiters: make(map[string]*folderListLimiterEntry)}
|
||||
defer func() { folderListLimiterRegistry = saved }()
|
||||
|
||||
f := &Fs{}
|
||||
f.opt.Region = "eu"
|
||||
f.opt.ListFolderWindow = fs.Duration(time.Hour) // large so nothing is evicted mid-test
|
||||
f.opt.ListFolderLimit = 19
|
||||
f.opt.ListFolderBurst = 6
|
||||
|
||||
const workers, folders = 64, 5
|
||||
var wg sync.WaitGroup
|
||||
for i := range workers {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
f.folderListLimiter(fmt.Sprintf("dir-%d", i%folders)).reserve(time.Now())
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Exactly one shared limiter per distinct folder, whatever the interleaving.
|
||||
assert.Len(t, folderListLimiterRegistry.limiters, folders)
|
||||
// And each limiter granted every reservation without breaching its own cap:
|
||||
// grants are capped at limit entries and non-decreasing.
|
||||
for _, e := range folderListLimiterRegistry.limiters {
|
||||
assert.LessOrEqual(t, len(e.limiter.grants), e.limiter.limit)
|
||||
for i := 1; i < len(e.limiter.grants); i++ {
|
||||
assert.False(t, e.limiter.grants[i].Before(e.limiter.grants[i-1]), "grants are non-decreasing")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -54,6 +55,28 @@ const (
|
||||
// caused synchronized 429 clusters.
|
||||
defaultTPSLimitBurst = 1
|
||||
|
||||
// defaultListFolderWindow / defaultListFolderLimit / defaultListFolderBurst
|
||||
// configure the per-folder listing limiter; folderListLimiter documents the
|
||||
// measured throttle model. Zoho allows ~19 listings of one folder in any
|
||||
// rolling ~60s window and the 20th returns F7008 (measured live 2026-07-05:
|
||||
// every trip landed exactly on the 20th listing inside 60s, every clean run
|
||||
// stayed at <=19 - including a deliberate over-limit probe tripping at #20).
|
||||
// The limiter guarantees at most limit listings per window: each window
|
||||
// starts with burst listings passing immediately (the burst RE-ARMS at every
|
||||
// window boundary) and the remaining limit-burst are paced window/(limit-burst)
|
||||
// apart, while a sliding log of the last limit grants enforces the cap across
|
||||
// window boundaries and idle resumes. burst never raises the per-window total -
|
||||
// it only sets how many listings may go back-to-back. Bursts of 4, 5 and 6
|
||||
// under this cap all ran clean live; 6 is the largest validated, hence the
|
||||
// default. Only repeated same-folder listings are paced; set limit to 0 to
|
||||
// disable entirely.
|
||||
defaultListFolderWindow = fs.Duration(60 * time.Second)
|
||||
defaultListFolderLimit = 19 // 0 = disabled
|
||||
defaultListFolderBurst = 6
|
||||
// folderListLimitersMaxEntries caps the per-folder limiter map against unbounded
|
||||
// growth on very long-lived processes.
|
||||
folderListLimitersMaxEntries = 100000
|
||||
|
||||
// retryAfterMargin adds a small extra delay after Zoho's Retry-After value.
|
||||
// This gives rate-limit tokens time to refill and helps avoid another 429.
|
||||
retryAfterMargin = 1 * time.Second
|
||||
@@ -259,6 +282,63 @@ lets several calls fire at once after an idle gap, which can trigger
|
||||
synchronized clusters of 429 errors.`,
|
||||
Default: defaultTPSLimitBurst,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "list_folder_limit",
|
||||
Help: `Max listings of the SAME folder allowed per --zoho-list-folder-window.
|
||||
|
||||
Zoho WorkDrive rate limits its listing API (GET files/{id}/files) PER
|
||||
folder, independently of --zoho-tpslimit: listing one folder too often in a
|
||||
short time returns HTTP 429 (error F7008) with a multi-minute Retry-After
|
||||
penalty, which a tight polling loop can hit even at a low overall rate.
|
||||
Measured live, Zoho allows ~19 listings of one folder in any rolling ~60s
|
||||
window and the 20th fails, which the defaults (19 per 60s) model exactly.
|
||||
|
||||
This is a true per-window cap for any traffic pattern: each window starts
|
||||
with --zoho-list-folder-burst listings passing back-to-back (the burst
|
||||
re-arms at every window boundary) and the rest are spaced
|
||||
--zoho-list-folder-window/(limit - burst) apart (the defaults give ~4.6s),
|
||||
while a sliding log of recent listings enforces the cap across window
|
||||
boundaries. 0 disables the limiter. Only REPEATED listings of one folder are
|
||||
delayed; different folders, or a folder listed fewer than
|
||||
--zoho-list-folder-burst times, never are.
|
||||
|
||||
A HIGHER value means MORE listings per window, not more safety: raising it
|
||||
above 19 trips F7008. Lower it for a wider margin at the cost of listing
|
||||
responsiveness.`,
|
||||
Default: defaultListFolderLimit,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "list_folder_window",
|
||||
Help: `The window for --zoho-list-folder-limit.
|
||||
|
||||
The default of 60s (shown as 1m0s) matches Zoho's real sliding window: at
|
||||
most --zoho-list-folder-limit listings of one folder are allowed in any
|
||||
window of this length. A bare number is parsed as seconds ("60" = "60s").
|
||||
|
||||
Widen it (or lower the limit) for a bigger safety margin; the sustained
|
||||
spacing between same-folder listings is window/(limit - burst).`,
|
||||
Default: defaultListFolderWindow,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "list_folder_burst",
|
||||
Help: `Same-folder listings allowed back-to-back before --zoho-list-folder-limit paces them.
|
||||
|
||||
The burst is carved out of --zoho-list-folder-limit, so raising it never
|
||||
raises the per-window total: this many listings may fire immediately and the
|
||||
remaining limit - burst are spaced window/(limit - burst) apart. The burst
|
||||
RE-ARMS at every window boundary, so sustained re-listing gets a fresh burst
|
||||
each window while a sliding log of recent listings still enforces the
|
||||
per-window cap. A folder listed only a handful of times (the common case - a
|
||||
sync re-listing one directory a few times then moving on) never waits.
|
||||
|
||||
The default 6 is the largest burst validated live under the default 19-per-60s
|
||||
cap (bursts of 4, 5 and 6 all ran clean; an over-cap probe tripped F7008
|
||||
exactly at the 20th listing in a window). Keep it well below ~15 - Zoho also
|
||||
has an instantaneous back-to-back cap around 15-16 regardless of the window.
|
||||
Set to 1 to pace from the second listing. Values >= the limit are clamped to
|
||||
limit - 1.`,
|
||||
Default: defaultListFolderBurst,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -273,12 +353,15 @@ synchronized clusters of 429 errors.`,
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
RootFolderID string `config:"root_folder_id"`
|
||||
Region string `config:"region"`
|
||||
TPSLimit float64 `config:"tpslimit"`
|
||||
TPSLimitBurst int `config:"tpslimit_burst"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
RootFolderID string `config:"root_folder_id"`
|
||||
Region string `config:"region"`
|
||||
TPSLimit float64 `config:"tpslimit"`
|
||||
TPSLimitBurst int `config:"tpslimit_burst"`
|
||||
ListFolderLimit int `config:"list_folder_limit"`
|
||||
ListFolderWindow fs.Duration `config:"list_folder_window"`
|
||||
ListFolderBurst int `config:"list_folder_burst"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
// Fs represents a remote workdrive
|
||||
@@ -305,6 +388,131 @@ type throttleState struct {
|
||||
progress atomic.Bool // a call succeeded after the last penalty window
|
||||
}
|
||||
|
||||
// folderListLimiters holds the per-folder-id listing rate limiters and the time
|
||||
// each folder was last listed. One instance is shared process-wide (folderListLimiterRegistry)
|
||||
// rather than per-*Fs because Zoho keys its listing throttle to the folder id:
|
||||
// a per-Fs limiter would let each new Fs re-list a shared ancestor with a fresh,
|
||||
// full-burst limiter, draining the bucket -> F7008.
|
||||
type folderListLimiters struct {
|
||||
mu sync.Mutex
|
||||
limiters map[string]*folderListLimiterEntry
|
||||
lastSweep time.Time
|
||||
}
|
||||
|
||||
// folderListLimiterRegistry is the process-wide registry of per-folder listing
|
||||
// limiters, shared by every *Fs and keyed by region+folder-id so entries never
|
||||
// collide across accounts or regions.
|
||||
var folderListLimiterRegistry = &folderListLimiters{limiters: make(map[string]*folderListLimiterEntry)}
|
||||
|
||||
// folderListLimiterEntry is a per-folder listing rate limiter plus the time the folder
|
||||
// was last listed, used for idle eviction.
|
||||
type folderListLimiterEntry struct {
|
||||
limiter *folderWindowLimiter
|
||||
lastListed time.Time
|
||||
}
|
||||
|
||||
// folderWindowLimiter shapes same-folder listings to at most limit per window
|
||||
// with a burst re-armed at each window boundary. Two layers:
|
||||
//
|
||||
// Shape (fixed window): a grid anchored at the first request advances in whole
|
||||
// windows; each window's first burst grants pass immediately and the remaining
|
||||
// limit-burst are spaced interval = window/(limit-burst) apart, so under
|
||||
// continuous demand every window starts with a fast burst - the flow the
|
||||
// options promise.
|
||||
//
|
||||
// Safety (sliding log): grants keeps the last limit grant times; a new grant
|
||||
// is also forced to be >= grants[len-limit] + window + margin, so no rolling
|
||||
// window ever sees more than limit grants even across grid boundaries or on
|
||||
// resume-after-idle, where the fixed-window shape alone could cluster a
|
||||
// boundary burst too close to earlier grants.
|
||||
type folderWindowLimiter struct {
|
||||
mu sync.Mutex
|
||||
window time.Duration // Zoho's per-folder window (--zoho-list-folder-window)
|
||||
limit int // window budget AND sliding cap (--zoho-list-folder-limit)
|
||||
burst int // grants passed immediately at each window start
|
||||
interval time.Duration // spacing of the paced phase: window/(limit-burst)
|
||||
windowStart time.Time // fixed grid anchor; zero until the first grant
|
||||
used int // grants handed out in the current window
|
||||
lastGrant time.Time // time of the most recent grant
|
||||
grants []time.Time // last <=limit grant times (sliding safety log)
|
||||
}
|
||||
|
||||
// folderListSafetyMargin is added on top of the sliding-log wait so a grant
|
||||
// lands strictly after the matching old grant has left Zoho's window.
|
||||
const folderListSafetyMargin = 500 * time.Millisecond
|
||||
|
||||
// reserve commits the next grant at or after now and returns its time.
|
||||
// It converges in at most a few passes: rolling the grid forward resets the
|
||||
// window budget, and the sliding log can push the candidate at most once more.
|
||||
func (l *folderWindowLimiter) reserve(now time.Time) time.Time {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.windowStart.IsZero() {
|
||||
l.windowStart = now
|
||||
}
|
||||
cand := now
|
||||
for range 4 {
|
||||
// Roll the fixed grid forward so cand lies in the current window,
|
||||
// re-arming the burst for the new window.
|
||||
if d := cand.Sub(l.windowStart); d >= l.window {
|
||||
l.windowStart = l.windowStart.Add(l.window * (d / l.window))
|
||||
l.used = 0
|
||||
}
|
||||
next := cand
|
||||
switch {
|
||||
case l.used < l.burst:
|
||||
// window-start burst passes immediately
|
||||
case l.used < l.limit:
|
||||
if t := l.lastGrant.Add(l.interval); t.After(next) {
|
||||
next = t
|
||||
}
|
||||
default:
|
||||
// window budget exhausted - wait for the next window's burst
|
||||
next = l.windowStart.Add(l.window)
|
||||
}
|
||||
// Sliding safety: never more than limit grants in any rolling window.
|
||||
if len(l.grants) >= l.limit {
|
||||
if t := l.grants[len(l.grants)-l.limit].Add(l.window + folderListSafetyMargin); t.After(next) {
|
||||
next = t
|
||||
}
|
||||
}
|
||||
if next.Equal(cand) {
|
||||
break
|
||||
}
|
||||
cand = next
|
||||
}
|
||||
l.used++
|
||||
l.lastGrant = cand
|
||||
if len(l.grants) < l.limit {
|
||||
l.grants = append(l.grants, cand)
|
||||
} else {
|
||||
copy(l.grants, l.grants[1:])
|
||||
l.grants[len(l.grants)-1] = cand
|
||||
}
|
||||
return cand
|
||||
}
|
||||
|
||||
// Wait blocks until the limiter grants a listing or ctx is cancelled. A grant
|
||||
// reserved by a cancelled Wait is not returned - erring on the safe side, it
|
||||
// just leaves a little of the window budget unused.
|
||||
func (l *folderWindowLimiter) Wait(ctx context.Context) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
d := time.Until(l.reserve(time.Now()))
|
||||
if d <= 0 {
|
||||
return nil
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// Object describes a Zoho WorkDrive object
|
||||
//
|
||||
// Will definitely have info but maybe not meta
|
||||
@@ -684,10 +892,89 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
// Should return true to finish processing
|
||||
type listAllFn func(*api.Item) bool
|
||||
|
||||
// folderListLimiter provides the shared per-folder listing rate limiter for dirID.
|
||||
//
|
||||
// Zoho WorkDrive throttles its listing API (GET files/{id}/files) PER folder,
|
||||
// independently of --zoho-tpslimit. Measured live (2026-07-05), that per-folder
|
||||
// throttle is a sliding window: ~19 listings of one folder within any rolling
|
||||
// ~60s are allowed and the 20th returns F7008 with a ~300s Retry-After. Every
|
||||
// observed trip landed exactly on the 20th listing inside 60s and every clean
|
||||
// run stayed at <=19, including a deliberate over-limit probe (window total 22)
|
||||
// tripping at #20; an earlier token-bucket refill model fitted the same data
|
||||
// inconsistently. There is also a much smaller instantaneous cap (~15-16
|
||||
// back-to-back listings trip on their own), so burst must stay well below that.
|
||||
//
|
||||
// The limiter (folderWindowLimiter) guarantees at most limit listings per
|
||||
// window for any traffic pattern: each window starts with burst listings
|
||||
// passing back-to-back - the burst RE-ARMS at every window boundary - and the
|
||||
// remaining limit-burst are paced window/(limit-burst) apart, while the
|
||||
// sliding log of the last limit grants enforces the cap across window
|
||||
// boundaries and idle resumes. Raising burst therefore never raises the
|
||||
// per-window total - it only front-loads it, widening the sustained spacing
|
||||
// to compensate. Different folders, or a folder listed fewer than burst
|
||||
// times, are never delayed - only tps applies unless the SAME folder is
|
||||
// re-listed enough to drain the burst.
|
||||
//
|
||||
// It paces repeated listings of the same folder; without it, listing one folder
|
||||
// too often trips Zoho's per-folder throttle and returns HTTP 429. The limiter is
|
||||
// created on demand in folderListLimiterRegistry, and limiters idle longer than
|
||||
// the window are evicted (at most once per window): after a full window idle
|
||||
// Zoho's sliding window is empty and the limiter is fully refilled, so dropping
|
||||
// it is lossless. Only called when --zoho-list-folder-limit > 0, so limit is
|
||||
// never zero.
|
||||
//
|
||||
// Returns the folder's *folderWindowLimiter, ready to Wait on before listing.
|
||||
func (f *Fs) folderListLimiter(dirID string) *folderWindowLimiter {
|
||||
flReg := folderListLimiterRegistry
|
||||
window := time.Duration(f.opt.ListFolderWindow)
|
||||
// Key by region+folder-id so one limiter is shared per physical folder
|
||||
// across every *Fs without colliding across accounts or regions.
|
||||
key := f.opt.Region + "\x00" + dirID
|
||||
flReg.mu.Lock()
|
||||
defer flReg.mu.Unlock()
|
||||
now := time.Now()
|
||||
if now.Sub(flReg.lastSweep) > window || len(flReg.limiters) > folderListLimitersMaxEntries {
|
||||
before := len(flReg.limiters)
|
||||
for id, e := range flReg.limiters {
|
||||
if now.Sub(e.lastListed) > window {
|
||||
delete(flReg.limiters, id)
|
||||
}
|
||||
}
|
||||
flReg.lastSweep = now
|
||||
fs.Debugf(f, "folder listing limiter sweep: evicted %d of %d idle limiters", before-len(flReg.limiters), before)
|
||||
}
|
||||
e := flReg.limiters[key]
|
||||
if e == nil {
|
||||
// The first Fs to list this folder fixes the shape; in the normal case
|
||||
// every Fs for the same account shares the same limit/window/burst.
|
||||
// limit > 0 is guaranteed by the caller; clamp burst into [1, limit-1]
|
||||
// (at limit 1 the burst token itself is the whole window's budget).
|
||||
burst := min(max(f.opt.ListFolderBurst, 1), max(f.opt.ListFolderLimit-1, 1))
|
||||
intervalTokens := max(f.opt.ListFolderLimit-burst, 1)
|
||||
e = &folderListLimiterEntry{limiter: &folderWindowLimiter{
|
||||
window: window,
|
||||
limit: f.opt.ListFolderLimit,
|
||||
burst: burst,
|
||||
interval: window / time.Duration(intervalTokens),
|
||||
}}
|
||||
flReg.limiters[key] = e
|
||||
}
|
||||
e.lastListed = now
|
||||
return e.limiter
|
||||
}
|
||||
|
||||
// Lists the directory required calling the user function on each item found
|
||||
//
|
||||
// If the user fn ever returns true then it early exits with found = true
|
||||
func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
|
||||
// Pace repeated listings of the SAME folder under Zoho's per-folder throttle
|
||||
// (see --zoho-list-folder-limit). One token per listAll call, not per page,
|
||||
// so multi-page listings of a single folder are not penalised.
|
||||
if f.opt.ListFolderLimit > 0 {
|
||||
if err = f.folderListLimiter(dirID).Wait(ctx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
const listItemsLimit = 1000
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
|
||||
Reference in New Issue
Block a user