From 4096a35b86822e9cd09969404fd0bd453ba9fb75 Mon Sep 17 00:00:00 2001
From: Jakob Borg <jakob@kastelo.net>
Date: Wed, 2 Apr 2025 10:35:37 -0700
Subject: [PATCH] fix(db): handle large numbers of blocks in update (#10025)

Avoid failure when inserting file with very large block list
---
 internal/db/sqlite/db_test.go   | 33 +++++++++++++++++++++++++++++++++
 internal/db/sqlite/db_update.go | 17 ++++++++++++-----
 2 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/internal/db/sqlite/db_test.go b/internal/db/sqlite/db_test.go
index 82749d4a9..2fb7a6721 100644
--- a/internal/db/sqlite/db_test.go
+++ b/internal/db/sqlite/db_test.go
@@ -1055,6 +1055,39 @@ func TestBlocklistGarbageCollection(t *testing.T) {
 	}
 }
 
+func TestInsertLargeFile(t *testing.T) {
+	t.Parallel()
+
+	sdb, err := OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := sdb.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// Add a large file (many blocks)
+
+	files := []protocol.FileInfo{genFile("test1", 16000, 1)}
+	if err := sdb.Update(folderID, protocol.LocalDeviceID, files); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify all the blocks are here
+
+	for i, block := range files[0].Blocks {
+		bs, err := itererr.Collect(sdb.AllLocalBlocksWithHash(block.Hash))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(bs) == 0 {
+			t.Error("missing blocks for", i)
+		}
+	}
+}
+
 func TestErrorWrap(t *testing.T) {
 	if wrap(nil, "foo") != nil {
 		t.Fatal("nil should wrap to nil")
diff --git a/internal/db/sqlite/db_update.go b/internal/db/sqlite/db_update.go
index 1cfa652c5..d6db8b30a 100644
--- a/internal/db/sqlite/db_update.go
+++ b/internal/db/sqlite/db_update.go
@@ -310,11 +310,18 @@ func (*DB) insertBlocksLocked(tx *txPreparedStmts, blocklistHash []byte, blocks
 			"size":           b.Size,
 		}
 	}
-	_, err := tx.NamedExec(`
-		INSERT OR IGNORE INTO blocks (hash, blocklist_hash, idx, offset, size)
-		VALUES (:hash, :blocklist_hash, :idx, :offset, :size)
-	`, bs)
-	return wrap(err)
+
+	// Very large block lists (>8000 blocks) result in "too many variables"
+	// error. Chunk it to a reasonable size.
+	for chunk := range slices.Chunk(bs, 1000) {
+		if _, err := tx.NamedExec(`
+			INSERT OR IGNORE INTO blocks (hash, blocklist_hash, idx, offset, size)
+			VALUES (:hash, :blocklist_hash, :idx, :offset, :size)
+		`, chunk); err != nil {
+			return wrap(err)
+		}
+	}
+	return nil
 }
 
 func (s *DB) recalcGlobalForFolder(txp *txPreparedStmts, folderIdx int64) error {