diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go
index 41d0462e3..8dc14f236 100644
--- a/fs/accounting/stats.go
+++ b/fs/accounting/stats.go
@@ -730,9 +730,15 @@ func (s *StatsInfo) ResetCounters() {
 	s.startedTransfers = nil
 	s.oldDuration = 0
 
+	// Only restart the average loop if it was running. Otherwise
+	// ResetCounters would spawn a goroutine that pins the StatsInfo,
+	// leaking memory when called on stats that never started the loop.
+	wasStarted := s.average.started
 	s._stopAverageLoop()
 	s.average = averageValues{}
-	s._startAverageLoop()
+	if wasStarted {
+		s._startAverageLoop()
+	}
 }
 
 // ResetErrors sets the errors count to 0 and resets lastError, fatalError and retryError