upload: scan the directory to be uploaded in parallel to estimate the amount of data to be uploaded (#622)

This allows better progress indicator in the CLI and UI.
The percentage completed is not displayed until estimate is available.

Quick demo: https://asciinema.org/a/O7ktcWSgaGUPfJwhzc65mMWM1

cli/cli_progress.go

@@ -47,8 +47,8 @@ type cliProgress struct {
 	spinPhase       int
 	uploadStartTime time.Time
 
-	previousFileCount int
-	previousTotalSize int64
+	estimatedFileCount  int
+	estimatedTotalBytes int64
 
 	// indicates shared instance that does not reset counters at the beginning of upload.
 	shared bool
@@ -152,13 +152,29 @@ func (p *cliProgress) output(col *color.Color, msg string) {
 		return
 	}
 
-	if p.previousTotalSize > 0 {
-		percent := (float64(hashedBytes+cachedBytes) * hundredPercent / float64(p.previousTotalSize))
-		if percent > hundredPercent {
-			percent = hundredPercent
+	if p.estimatedTotalBytes > 0 {
+		line += fmt.Sprintf(", estimated %v", units.BytesStringBase10(p.estimatedTotalBytes))
+
+		ratio := float64(hashedBytes+cachedBytes) / float64(p.estimatedTotalBytes)
+		if ratio > 1 {
+			ratio = 1
 		}
 
-		line += fmt.Sprintf(" %.1f%%", percent)
+		timeSoFarSeconds := clock.Since(p.uploadStartTime).Seconds()
+		estimatedTotalTime := time.Second * time.Duration(timeSoFarSeconds/ratio)
+		estimatedEndTime := p.uploadStartTime.Add(estimatedTotalTime)
+
+		remaining := clock.Until(estimatedEndTime)
+		if remaining < 0 {
+			remaining = 0
+		}
+
+		remaining = remaining.Round(time.Second)
+
+		line += fmt.Sprintf(" (%.1f%%)", ratio*hundredPercent)
+		line += fmt.Sprintf(" %v left", remaining)
+	} else {
+		line += ", estimating..."
 	}
 
 	var extraSpaces string
@@ -197,20 +213,31 @@ func (p *cliProgress) FinishShared() {
 	p.output(defaultColor, "")
 }
 
-func (p *cliProgress) UploadStarted(previousFileCount int, previousTotalSize int64) {
+func (p *cliProgress) UploadStarted() {
 	if p.shared {
 		// do nothing
 		return
 	}
 
 	*p = cliProgress{
-		uploading:         1,
-		uploadStartTime:   clock.Now(),
-		previousFileCount: previousFileCount,
-		previousTotalSize: previousTotalSize,
+		uploading:       1,
+		uploadStartTime: clock.Now(),
 	}
 }
 
+func (p *cliProgress) EstimatedDataSize(fileCount int, totalBytes int64) {
+	if p.shared {
+		// do nothing
+		return
+	}
+
+	p.outputMutex.Lock()
+	defer p.outputMutex.Unlock()
+
+	p.estimatedFileCount = fileCount
+	p.estimatedTotalBytes = totalBytes
+}
+
 func (p *cliProgress) UploadFinished() {
 	// do nothing here, we still want to report the files flushed after the Upload has completed.
 	// instead, Finish() will be called.