operations: fix silent truncation of streaming uploads whose source ends early

Uploads through RcatSize with a known size - used by rcat --size, the
rc operations/uploadfile and the serve backends, eg serve restic - did
not check the size of the uploaded object. If the source stream ended
before the declared size worth of data had been read, the truncated
object was reported as a successful upload. This could corrupt data
for callers which trust the result, eg a restic repository accessed
via serve restic (see #9722).

This adds the same size check operations.Copy performs after a copy,
respecting --ignore-size and backends which do not report sizes.

(cherry picked from commit 9b13247ba8)

fs/operations/operations.go

@@ -1838,6 +1838,15 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo
 
 			return nil, err
 		}
+
+		// Check transfer - the source may have ended before size
+		// bytes in which case the object will have been truncated
+		if sizeDiffers(ctx, info, obj) {
+			err = fmt.Errorf("corrupted on transfer: sizes differ src %d vs dst(%s) %d", info.Size(), fdst, obj.Size())
+			err = fs.CountError(ctx, err)
+			fs.Errorf(obj, "%v", err)
+			return obj, err
+		}
 	} else {
 		// Size unknown use Rcat
 		obj, err = Rcat(ctx, fdst, dstFileName, in, modTime, meta)

fs/operations/operations_test.go

@@ -1637,6 +1637,18 @@ func TestRcatSize(t *testing.T) {
 	r.CheckRemoteItems(t, file1, file2)
 }
 
+func TestRcatSizeShortEOF(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+
+	const body = "------------------------------------------------------------"
+
+	// Upload declaring twice as many bytes as the source supplies
+	bodyReader := io.NopCloser(strings.NewReader(body))
+	_, err := operations.RcatSize(ctx, r.Fremote, "potato1", bodyReader, 2*int64(len(body)), t1, nil)
+	require.Error(t, err, "uploading a source which ends before its declared size must not succeed")
+}
+
 func TestRcatSizeMetadata(t *testing.T) {
 	r := fstest.NewRun(t)