wip

Signed-off-by: Jakob Borg <jakob@kastelo.net>

gui/default/syncthing/folder/editFolderModalView.html

@@ -342,6 +342,20 @@
             </div>
           </div>
 
+          <div class="row">
+            <div class="col-md-6 form-group">
+              <p>
+                <label translate>Block Index</label>
+              </p>
+              <label>
+                <input type="checkbox" ng-model="currentFolder.fullBlockIndex" /> <span translate>Full Block Index</span>
+              </label>
+              <p translate class="help-block">
+                Maintain an index of all blocks in the folder, enabling reuse of blocks from other files when syncing changes. Disable to reduce database size at the cost of not being able to reuse blocks across files.
+              </p>
+            </div>
+          </div>
+
           <div class="row" ng-if="currentFolder.syncXattrs || currentFolder.sendXattrs">
             <div class="col-md-12">
               <p>

internal/db/interface.go

@@ -27,13 +27,29 @@ type DBService interface {
 	LastMaintenanceTime() time.Time
 }
 
+// UpdateOption modifies the behavior of a DB Update call.
+type UpdateOption func(*UpdateOptions)
+
+// UpdateOptions holds options for a DB Update call.
+type UpdateOptions struct {
+	SkipBlockIndex bool
+}
+
+// WithSkipBlockIndex skips inserting individual blocks into the block
+// index (the "blocks" table). Blocklists are still stored.
+func WithSkipBlockIndex() UpdateOption {
+	return func(o *UpdateOptions) {
+		o.SkipBlockIndex = true
+	}
+}
+
 type DB interface {
 	// Create a service that performs database maintenance periodically (no
 	// more often than the requested interval)
 	Service(maintenanceInterval time.Duration) DBService
 
 	// Basics
-	Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo) error
+	Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo, opts ...UpdateOption) error
 	Close() error
 
 	// Single files

internal/db/metrics.go

@@ -198,10 +198,10 @@ func (m metricsDB) SetIndexID(folder string, device protocol.DeviceID, id protoc
 	return m.DB.SetIndexID(folder, device, id)
 }
 
-func (m metricsDB) Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo) error {
+func (m metricsDB) Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo, opts ...UpdateOption) error {
 	defer m.account(folder, "Update")()
 	defer metricTotalFilesUpdatedCount.WithLabelValues(folder).Add(float64(len(fs)))
-	return m.DB.Update(folder, device, fs)
+	return m.DB.Update(folder, device, fs, opts...)
 }
 
 func (m metricsDB) GetKV(key string) ([]byte, error) {

internal/db/sqlite/db_folderdb.go

@@ -92,12 +92,16 @@ func (s *DB) getFolderDB(folder string, create bool) (*folderDB, error) {
 	return fdb, nil
 }
 
-func (s *DB) Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo) error {
+func (s *DB) Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo, opts ...db.UpdateOption) error {
 	fdb, err := s.getFolderDB(folder, true)
 	if err != nil {
 		return err
 	}
-	return fdb.Update(device, fs)
+	var options db.UpdateOptions
+	for _, o := range opts {
+		o(&options)
+	}
+	return fdb.Update(device, fs, options)
 }
 
 func (s *DB) GetDeviceFile(folder string, device protocol.DeviceID, file string) (protocol.FileInfo, bool, error) {

internal/db/sqlite/folderdb_update.go

@@ -14,6 +14,7 @@ import (
 	"slices"
 
 	"github.com/jmoiron/sqlx"
+	"github.com/syncthing/syncthing/internal/db"
 	"github.com/syncthing/syncthing/internal/gen/dbproto"
 	"github.com/syncthing/syncthing/internal/itererr"
 	"github.com/syncthing/syncthing/internal/slogutil"
@@ -30,7 +31,7 @@ const (
 	updatePointsThreshold = 250_000
 )
 
-func (s *folderDB) Update(device protocol.DeviceID, fs []protocol.FileInfo) error {
+func (s *folderDB) Update(device protocol.DeviceID, fs []protocol.FileInfo, options db.UpdateOptions) error {
 	s.updateLock.Lock()
 	defer s.updateLock.Unlock()
 
@@ -151,7 +152,7 @@ func (s *folderDB) Update(device protocol.DeviceID, fs []protocol.FileInfo) erro
 			}
 			if _, err := insertBlockListStmt.Exec(f.BlocksHash, bs); err != nil {
 				return wrap(err, "insert blocklist")
-			} else if device == protocol.LocalDeviceID {
+			} else if device == protocol.LocalDeviceID && !options.SkipBlockIndex {
 				// Insert all blocks
 				if err := s.insertBlocksLocked(txp, f.BlocksHash, f.Blocks); err != nil {
 					return wrap(err, "insert blocks")

lib/config/config.go

@@ -33,7 +33,7 @@ import (
 
 const (
 	OldestHandledVersion = 10
-	CurrentVersion       = 52
+	CurrentVersion       = 53
 	MaxRescanIntervalS   = 365 * 24 * 60 * 60
 )
 

lib/config/folderconfiguration.go

@@ -86,6 +86,7 @@ type FolderConfiguration struct {
 	SendOwnership           bool                        `json:"sendOwnership" xml:"sendOwnership"`
 	SyncXattrs              bool                        `json:"syncXattrs" xml:"syncXattrs"`
 	SendXattrs              bool                        `json:"sendXattrs" xml:"sendXattrs"`
+	FullBlockIndex          bool                        `json:"fullBlockIndex" xml:"fullBlockIndex"`
 	XattrFilter             XattrFilter                 `json:"xattrFilter" xml:"xattrFilter"`
 	// Legacy deprecated
 	DeprecatedReadOnly       bool    `json:"-" xml:"ro,attr,omitempty"`        // Deprecated: Do not use.

lib/config/migrations.go

@@ -30,6 +30,7 @@ import (
 // put the newest on top for readability.
 var (
 	migrations = migrationSet{
+		{53, migrateToConfigV53},
 		{52, migrateToConfigV52},
 		{51, migrateToConfigV51},
 		{50, migrateToConfigV50},
@@ -102,6 +103,15 @@ func (m migration) apply(cfg *Configuration) {
 	cfg.Version = m.targetVersion
 }
 
+func migrateToConfigV53(cfg *Configuration) {
+	for i, f := range cfg.Folders {
+		switch f.Type {
+		case FolderTypeSendReceive, FolderTypeReceiveOnly:
+			cfg.Folders[i].FullBlockIndex = true
+		}
+	}
+}
+
 func migrateToConfigV52(cfg *Configuration) {
 	oldQuicInterval := max(cfg.Options.ReconnectIntervalS/3, 10)
 	cfg.Options.ReconnectIntervalS = min(cfg.Options.ReconnectIntervalS, oldQuicInterval)

lib/model/folder.go

@@ -1264,7 +1264,11 @@ func (f *folder) updateLocalsFromPulling(fs []protocol.FileInfo) error {
 }
 
 func (f *folder) updateLocals(fs []protocol.FileInfo) error {
-	if err := f.db.Update(f.folderID, protocol.LocalDeviceID, fs); err != nil {
+	var opts []db.UpdateOption
+	if !f.FullBlockIndex {
+		opts = append(opts, db.WithSkipBlockIndex())
+	}
+	if err := f.db.Update(f.folderID, protocol.LocalDeviceID, fs, opts...); err != nil {
 		return err
 	}
 

lib/model/folder_sendrecv.go

@@ -1166,6 +1166,7 @@ func (f *sendReceiveFolder) handleFile(ctx context.Context, file protocol.FileIn
 		blocks:            blocks,
 		have:              len(have),
 	}
+
 	copyChan <- cs
 	return nil
 }
@@ -1322,7 +1323,7 @@ func (f *sendReceiveFolder) shortcutFile(file protocol.FileInfo, dbUpdateChan ch
 func (f *sendReceiveFolder) copierRoutine(ctx context.Context, in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState) {
 	otherFolderFilesystems := make(map[string]fs.Filesystem)
 	for folder, cfg := range f.model.cfg.Folders() {
-		if folder == f.ID {
+		if folder == f.ID || !f.FullBlockIndex {
 			continue
 		}
 		otherFolderFilesystems[folder] = cfg.Filesystem()
@@ -1390,13 +1391,26 @@ func (f *sendReceiveFolder) copyBlock(ctx context.Context, block protocol.BlockI
 	buf := protocol.BufferPool.Get(block.Size)
 	defer protocol.BufferPool.Put(buf)
 
-	// Hope that it's usually in the same folder, so start with that
-	// one. Also possibly more efficient copy (same filesystem).
-	if f.copyBlockFromFolder(ctx, f.ID, block, state, f.mtimefs, buf) {
-		return true
+	// Check for the block in the current version of the file
+	if idx, ok := state.curFileBlocks[string(block.Hash)]; ok {
+		if f.copyBlockFromFile(ctx, state.file.Name, state.curFile.Blocks[idx].Offset, state, f.mtimefs, block, buf) {
+			state.copiedFromOrigin(block.Size)
+			return true
+		}
+		if state.failed() != nil {
+			return false
+		}
 	}
-	if state.failed() != nil {
-		return false
+
+	if f.folder.FullBlockIndex {
+		// Hope that it's usually in the same folder, so start with that
+		// one. Also possibly more efficient copy (same filesystem).
+		if f.copyBlockFromFolder(ctx, f.ID, block, state, f.mtimefs, buf) {
+			return true
+		}
+		if state.failed() != nil {
+			return false
+		}
 	}
 
 	for folderID, ffs := range otherFolderFilesystems {

lib/model/sharedpullerstate.go

@@ -25,18 +25,19 @@ import (
 // updated along the way.
 type sharedPullerState struct {
 	// Immutable, does not require locking
-	file        protocol.FileInfo // The new file (desired end state)
-	fs          fs.Filesystem
-	folder      string
-	tempName    string
-	realName    string
-	reused      int // Number of blocks reused from temporary file
-	ignorePerms bool
-	hasCurFile  bool              // Whether curFile is set
-	curFile     protocol.FileInfo // The file as it exists now in our database
-	sparse      bool
-	created     time.Time
-	fsync       bool
+	file          protocol.FileInfo // The new file (desired end state)
+	fs            fs.Filesystem
+	folder        string
+	tempName      string
+	realName      string
+	reused        int // Number of blocks reused from temporary file
+	ignorePerms   bool
+	hasCurFile    bool              // Whether curFile is set
+	curFile       protocol.FileInfo // The file as it exists now in our database
+	curFileBlocks map[string]int    // block hash to index in curFile
+	sparse        bool
+	created       time.Time
+	fsync         bool
 
 	// Mutable, must be locked for access
 	err              error           // The first error we hit
@@ -54,6 +55,12 @@ type sharedPullerState struct {
 }
 
 func newSharedPullerState(file protocol.FileInfo, fs fs.Filesystem, folderID, tempName string, blocks []protocol.BlockInfo, reused []int, ignorePerms, hasCurFile bool, curFile protocol.FileInfo, sparse bool, fsync bool) *sharedPullerState {
+	// Map the existing blocks by hash to block index in the current file
+	blocksMap := make(map[string]int, len(curFile.Blocks))
+	for idx, block := range curFile.Blocks {
+		blocksMap[string(block.Hash)] = idx
+	}
+
 	return &sharedPullerState{
 		file:             file,
 		fs:               fs,
@@ -69,6 +76,7 @@ func newSharedPullerState(file protocol.FileInfo, fs fs.Filesystem, folderID, te
 		ignorePerms:      ignorePerms,
 		hasCurFile:       hasCurFile,
 		curFile:          curFile,
+		curFileBlocks:    blocksMap,
 		sparse:           sparse,
 		fsync:            fsync,
 		created:          time.Now(),

lib/model/testutils_test.go

@@ -110,6 +110,7 @@ func newFolderConfig() config.FolderConfiguration {
 	cfg.FSWatcherEnabled = false
 	cfg.PullerDelayS = 0
 	cfg.Devices = append(cfg.Devices, config.FolderDeviceConfiguration{DeviceID: device1})
+	cfg.FullBlockIndex = true
 	return cfg
 }