From dd5d3fd496e5c6af94b76fbe6f1b2c5b528c27ec Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 16 Mar 2026 13:44:11 +0100 Subject: [PATCH] wip Signed-off-by: Jakob Borg --- .../syncthing/folder/editFolderModalView.html | 14 ++++++++ internal/db/interface.go | 18 ++++++++++- internal/db/metrics.go | 4 +-- internal/db/sqlite/db_folderdb.go | 8 +++-- internal/db/sqlite/folderdb_update.go | 5 +-- lib/config/config.go | 2 +- lib/config/folderconfiguration.go | 1 + lib/config/migrations.go | 10 ++++++ lib/model/folder.go | 6 +++- lib/model/folder_sendrecv.go | 28 ++++++++++++---- lib/model/sharedpullerstate.go | 32 ++++++++++++------- lib/model/testutils_test.go | 1 + 12 files changed, 101 insertions(+), 28 deletions(-) diff --git a/gui/default/syncthing/folder/editFolderModalView.html b/gui/default/syncthing/folder/editFolderModalView.html index 09643c887..24cfd4cda 100644 --- a/gui/default/syncthing/folder/editFolderModalView.html +++ b/gui/default/syncthing/folder/editFolderModalView.html @@ -342,6 +342,20 @@ +
+
+

+ +

+ +

+ Maintain an index of all blocks in the folder, enabling reuse of blocks from other files when syncing changes. Disable to reduce database size at the cost of not being able to reuse blocks across files. +

+
+
+

diff --git a/internal/db/interface.go b/internal/db/interface.go index 32c79cc49..53ad9769f 100644 --- a/internal/db/interface.go +++ b/internal/db/interface.go @@ -27,13 +27,29 @@ type DBService interface { LastMaintenanceTime() time.Time } +// UpdateOption modifies the behavior of a DB Update call. +type UpdateOption func(*UpdateOptions) + +// UpdateOptions holds options for a DB Update call. +type UpdateOptions struct { + SkipBlockIndex bool +} + +// WithSkipBlockIndex skips inserting individual blocks into the block +// index (the "blocks" table). Blocklists are still stored. +func WithSkipBlockIndex() UpdateOption { + return func(o *UpdateOptions) { + o.SkipBlockIndex = true + } +} + type DB interface { // Create a service that performs database maintenance periodically (no // more often than the requested interval) Service(maintenanceInterval time.Duration) DBService // Basics - Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo) error + Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo, opts ...UpdateOption) error Close() error // Single files diff --git a/internal/db/metrics.go b/internal/db/metrics.go index abb350af3..c9565eb20 100644 --- a/internal/db/metrics.go +++ b/internal/db/metrics.go @@ -198,10 +198,10 @@ func (m metricsDB) SetIndexID(folder string, device protocol.DeviceID, id protoc return m.DB.SetIndexID(folder, device, id) } -func (m metricsDB) Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo) error { +func (m metricsDB) Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo, opts ...UpdateOption) error { defer m.account(folder, "Update")() defer metricTotalFilesUpdatedCount.WithLabelValues(folder).Add(float64(len(fs))) - return m.DB.Update(folder, device, fs) + return m.DB.Update(folder, device, fs, opts...) } func (m metricsDB) GetKV(key string) ([]byte, error) { diff --git a/internal/db/sqlite/db_folderdb.go b/internal/db/sqlite/db_folderdb.go index 2431f7576..1abe7ff80 100644 --- a/internal/db/sqlite/db_folderdb.go +++ b/internal/db/sqlite/db_folderdb.go @@ -92,12 +92,16 @@ func (s *DB) getFolderDB(folder string, create bool) (*folderDB, error) { return fdb, nil } -func (s *DB) Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo) error { +func (s *DB) Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo, opts ...db.UpdateOption) error { fdb, err := s.getFolderDB(folder, true) if err != nil { return err } - return fdb.Update(device, fs) + var options db.UpdateOptions + for _, o := range opts { + o(&options) + } + return fdb.Update(device, fs, options) } func (s *DB) GetDeviceFile(folder string, device protocol.DeviceID, file string) (protocol.FileInfo, bool, error) { diff --git a/internal/db/sqlite/folderdb_update.go b/internal/db/sqlite/folderdb_update.go index 50366a802..4b8b7551f 100644 --- a/internal/db/sqlite/folderdb_update.go +++ b/internal/db/sqlite/folderdb_update.go @@ -14,6 +14,7 @@ import ( "slices" "github.com/jmoiron/sqlx" + "github.com/syncthing/syncthing/internal/db" "github.com/syncthing/syncthing/internal/gen/dbproto" "github.com/syncthing/syncthing/internal/itererr" "github.com/syncthing/syncthing/internal/slogutil" @@ -30,7 +31,7 @@ const ( updatePointsThreshold = 250_000 ) -func (s *folderDB) Update(device protocol.DeviceID, fs []protocol.FileInfo) error { +func (s *folderDB) Update(device protocol.DeviceID, fs []protocol.FileInfo, options db.UpdateOptions) error { s.updateLock.Lock() defer s.updateLock.Unlock() @@ -151,7 +152,7 @@ func (s *folderDB) Update(device protocol.DeviceID, fs []protocol.FileInfo) erro } if _, err := insertBlockListStmt.Exec(f.BlocksHash, bs); err != nil { return wrap(err, "insert blocklist") - } else if device == protocol.LocalDeviceID { + } else if device == protocol.LocalDeviceID && !options.SkipBlockIndex { // Insert all blocks if err := s.insertBlocksLocked(txp, f.BlocksHash, f.Blocks); err != nil { return wrap(err, "insert blocks") diff --git a/lib/config/config.go b/lib/config/config.go index 4407cf35b..0ebc7831c 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -33,7 +33,7 @@ import ( const ( OldestHandledVersion = 10 - CurrentVersion = 52 + CurrentVersion = 53 MaxRescanIntervalS = 365 * 24 * 60 * 60 ) diff --git a/lib/config/folderconfiguration.go b/lib/config/folderconfiguration.go index c6696eee3..567c6b43c 100644 --- a/lib/config/folderconfiguration.go +++ b/lib/config/folderconfiguration.go @@ -86,6 +86,7 @@ type FolderConfiguration struct { SendOwnership bool `json:"sendOwnership" xml:"sendOwnership"` SyncXattrs bool `json:"syncXattrs" xml:"syncXattrs"` SendXattrs bool `json:"sendXattrs" xml:"sendXattrs"` + FullBlockIndex bool `json:"fullBlockIndex" xml:"fullBlockIndex"` XattrFilter XattrFilter `json:"xattrFilter" xml:"xattrFilter"` // Legacy deprecated DeprecatedReadOnly bool `json:"-" xml:"ro,attr,omitempty"` // Deprecated: Do not use. diff --git a/lib/config/migrations.go b/lib/config/migrations.go index 382bccfad..cc9c17e5e 100644 --- a/lib/config/migrations.go +++ b/lib/config/migrations.go @@ -30,6 +30,7 @@ import ( // put the newest on top for readability. var ( migrations = migrationSet{ + {53, migrateToConfigV53}, {52, migrateToConfigV52}, {51, migrateToConfigV51}, {50, migrateToConfigV50}, @@ -102,6 +103,15 @@ func (m migration) apply(cfg *Configuration) { cfg.Version = m.targetVersion } +func migrateToConfigV53(cfg *Configuration) { + for i, f := range cfg.Folders { + switch f.Type { + case FolderTypeSendReceive, FolderTypeReceiveOnly: + cfg.Folders[i].FullBlockIndex = true + } + } +} + func migrateToConfigV52(cfg *Configuration) { oldQuicInterval := max(cfg.Options.ReconnectIntervalS/3, 10) cfg.Options.ReconnectIntervalS = min(cfg.Options.ReconnectIntervalS, oldQuicInterval) diff --git a/lib/model/folder.go b/lib/model/folder.go index 70fe941f2..071db2d5a 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -1264,7 +1264,11 @@ func (f *folder) updateLocalsFromPulling(fs []protocol.FileInfo) error { } func (f *folder) updateLocals(fs []protocol.FileInfo) error { - if err := f.db.Update(f.folderID, protocol.LocalDeviceID, fs); err != nil { + var opts []db.UpdateOption + if !f.FullBlockIndex { + opts = append(opts, db.WithSkipBlockIndex()) + } + if err := f.db.Update(f.folderID, protocol.LocalDeviceID, fs, opts...); err != nil { return err } diff --git a/lib/model/folder_sendrecv.go b/lib/model/folder_sendrecv.go index c63921177..80958edbd 100644 --- a/lib/model/folder_sendrecv.go +++ b/lib/model/folder_sendrecv.go @@ -1166,6 +1166,7 @@ func (f *sendReceiveFolder) handleFile(ctx context.Context, file protocol.FileIn blocks: blocks, have: len(have), } + copyChan <- cs return nil } @@ -1322,7 +1323,7 @@ func (f *sendReceiveFolder) shortcutFile(file protocol.FileInfo, dbUpdateChan ch func (f *sendReceiveFolder) copierRoutine(ctx context.Context, in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState) { otherFolderFilesystems := make(map[string]fs.Filesystem) for folder, cfg := range f.model.cfg.Folders() { - if folder == f.ID { + if folder == f.ID || !f.FullBlockIndex { continue } otherFolderFilesystems[folder] = cfg.Filesystem() @@ -1390,13 +1391,26 @@ func (f *sendReceiveFolder) copyBlock(ctx context.Context, block protocol.BlockI buf := protocol.BufferPool.Get(block.Size) defer protocol.BufferPool.Put(buf) - // Hope that it's usually in the same folder, so start with that - // one. Also possibly more efficient copy (same filesystem). - if f.copyBlockFromFolder(ctx, f.ID, block, state, f.mtimefs, buf) { - return true + // Check for the block in the current version of the file + if idx, ok := state.curFileBlocks[string(block.Hash)]; ok { + if f.copyBlockFromFile(ctx, state.file.Name, state.curFile.Blocks[idx].Offset, state, f.mtimefs, block, buf) { + state.copiedFromOrigin(block.Size) + return true + } + if state.failed() != nil { + return false + } } - if state.failed() != nil { - return false + + if f.folder.FullBlockIndex { + // Hope that it's usually in the same folder, so start with that + // one. Also possibly more efficient copy (same filesystem). + if f.copyBlockFromFolder(ctx, f.ID, block, state, f.mtimefs, buf) { + return true + } + if state.failed() != nil { + return false + } } for folderID, ffs := range otherFolderFilesystems { diff --git a/lib/model/sharedpullerstate.go b/lib/model/sharedpullerstate.go index dddbb4c36..4d4b309ca 100644 --- a/lib/model/sharedpullerstate.go +++ b/lib/model/sharedpullerstate.go @@ -25,18 +25,19 @@ import ( // updated along the way. type sharedPullerState struct { // Immutable, does not require locking - file protocol.FileInfo // The new file (desired end state) - fs fs.Filesystem - folder string - tempName string - realName string - reused int // Number of blocks reused from temporary file - ignorePerms bool - hasCurFile bool // Whether curFile is set - curFile protocol.FileInfo // The file as it exists now in our database - sparse bool - created time.Time - fsync bool + file protocol.FileInfo // The new file (desired end state) + fs fs.Filesystem + folder string + tempName string + realName string + reused int // Number of blocks reused from temporary file + ignorePerms bool + hasCurFile bool // Whether curFile is set + curFile protocol.FileInfo // The file as it exists now in our database + curFileBlocks map[string]int // block hash to index in curFile + sparse bool + created time.Time + fsync bool // Mutable, must be locked for access err error // The first error we hit @@ -54,6 +55,12 @@ type sharedPullerState struct { } func newSharedPullerState(file protocol.FileInfo, fs fs.Filesystem, folderID, tempName string, blocks []protocol.BlockInfo, reused []int, ignorePerms, hasCurFile bool, curFile protocol.FileInfo, sparse bool, fsync bool) *sharedPullerState { + // Map the existing blocks by hash to block index in the current file + blocksMap := make(map[string]int, len(curFile.Blocks)) + for idx, block := range curFile.Blocks { + blocksMap[string(block.Hash)] = idx + } + return &sharedPullerState{ file: file, fs: fs, @@ -69,6 +76,7 @@ func newSharedPullerState(file protocol.FileInfo, fs fs.Filesystem, folderID, te ignorePerms: ignorePerms, hasCurFile: hasCurFile, curFile: curFile, + curFileBlocks: blocksMap, sparse: sparse, fsync: fsync, created: time.Now(), diff --git a/lib/model/testutils_test.go b/lib/model/testutils_test.go index ad0a05cd8..564c5dccb 100644 --- a/lib/model/testutils_test.go +++ b/lib/model/testutils_test.go @@ -110,6 +110,7 @@ func newFolderConfig() config.FolderConfiguration { cfg.FSWatcherEnabled = false cfg.PullerDelayS = 0 cfg.Devices = append(cfg.Devices, config.FolderDeviceConfiguration{DeviceID: device1}) + cfg.FullBlockIndex = true return cfg }