diff --git a/internal/db/interface.go b/internal/db/interface.go index 53ad9769f..980e12879 100644 --- a/internal/db/interface.go +++ b/internal/db/interface.go @@ -73,6 +73,10 @@ type DB interface { AllNeededGlobalFiles(folder string, device protocol.DeviceID, order config.PullOrder, limit, offset int) (iter.Seq[protocol.FileInfo], func() error) AllLocalBlocksWithHash(folder string, hash []byte) (iter.Seq[BlockMapEntry], func() error) + // Block index management + DropBlockIndex(folder string) error + PopulateBlockIndex(folder string) error + // Cleanup DropAllFiles(folder string, device protocol.DeviceID) error DropDevice(device protocol.DeviceID) error diff --git a/internal/db/sqlite/db_folderdb.go b/internal/db/sqlite/db_folderdb.go index 1abe7ff80..c520580c5 100644 --- a/internal/db/sqlite/db_folderdb.go +++ b/internal/db/sqlite/db_folderdb.go @@ -104,6 +104,25 @@ func (s *DB) Update(folder string, device protocol.DeviceID, fs []protocol.FileI return fdb.Update(device, fs, options) } +func (s *DB) DropBlockIndex(folder string) error { + fdb, err := s.getFolderDB(folder, false) + if errors.Is(err, errNoSuchFolder) { + return nil + } + if err != nil { + return err + } + return fdb.DropBlockIndex() +} + +func (s *DB) PopulateBlockIndex(folder string) error { + fdb, err := s.getFolderDB(folder, true) + if err != nil { + return err + } + return fdb.PopulateBlockIndex() +} + func (s *DB) GetDeviceFile(folder string, device protocol.DeviceID, file string) (protocol.FileInfo, bool, error) { fdb, err := s.getFolderDB(folder, false) if errors.Is(err, errNoSuchFolder) { diff --git a/internal/db/sqlite/folderdb_update.go b/internal/db/sqlite/folderdb_update.go index 4b8b7551f..5c27e7b15 100644 --- a/internal/db/sqlite/folderdb_update.go +++ b/internal/db/sqlite/folderdb_update.go @@ -304,6 +304,83 @@ func (s *folderDB) DropFilesNamed(device protocol.DeviceID, names []string) erro return wrap(tx.Commit()) } +func (s *folderDB) blockIndexEmpty() (bool, error) { + var exists bool + err := s.sql.Get(&exists, `SELECT EXISTS (SELECT 1 FROM blocks LIMIT 1)`) + if err != nil { + return false, wrap(err) + } + return !exists, nil +} + +func (s *folderDB) DropBlockIndex() error { + empty, err := s.blockIndexEmpty() + if err != nil || empty { + return err + } + + s.updateLock.Lock() + defer s.updateLock.Unlock() + + _, err = s.sql.Exec(`DELETE FROM blocks`) + return wrap(err) +} + +func (s *folderDB) PopulateBlockIndex() error { + empty, err := s.blockIndexEmpty() + if err != nil || !empty { + return err + } + + s.updateLock.Lock() + defer s.updateLock.Unlock() + + tx, err := s.sql.BeginTxx(context.Background(), nil) + if err != nil { + return wrap(err) + } + defer tx.Rollback() + txp := &txPreparedStmts{Tx: tx} + + // Iterate all local files that have a blocklist + rows, err := tx.Queryx(` + SELECT f.blocklist_hash, bl.blprotobuf FROM files f + INNER JOIN blocklists bl ON bl.blocklist_hash = f.blocklist_hash + WHERE f.device_idx = ? AND f.blocklist_hash IS NOT NULL + `, s.localDeviceIdx) + if err != nil { + return wrap(err) + } + defer rows.Close() + + for rows.Next() { + var blocklistHash []byte + var blProtobuf []byte + if err := rows.Scan(&blocklistHash, &blProtobuf); err != nil { + return wrap(err) + } + + var bl dbproto.BlockList + if err := proto.Unmarshal(blProtobuf, &bl); err != nil { + return wrap(err, "unmarshal blocklist") + } + + blocks := make([]protocol.BlockInfo, len(bl.Blocks)) + for i, b := range bl.Blocks { + blocks[i] = protocol.BlockInfoFromWire(b) + } + + if err := s.insertBlocksLocked(txp, blocklistHash, blocks); err != nil { + return wrap(err, "insert blocks") + } + } + if err := rows.Err(); err != nil { + return wrap(err) + } + + return wrap(tx.Commit()) +} + func (*folderDB) insertBlocksLocked(tx *txPreparedStmts, blocklistHash []byte, blocks []protocol.BlockInfo) error { if len(blocks) == 0 { return nil diff --git a/lib/model/folder.go b/lib/model/folder.go index 071db2d5a..c0310330a 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -159,6 +159,10 @@ func (f *folder) Serve(ctx context.Context) error { f.setState(FolderIdle) }() + if err := f.reconcileBlockIndex(ctx); err != nil { + return err + } + if f.FSWatcherEnabled && f.getHealthErrorAndLoadIgnores() == nil { f.startWatch(ctx) } @@ -256,6 +260,15 @@ func (f *folder) Serve(ctx context.Context) error { } } +func (f *folder) reconcileBlockIndex(ctx context.Context) error { + if !f.FullBlockIndex { + f.sl.DebugContext(ctx, "Dropping block index (full block index disabled)") + return f.db.DropBlockIndex(f.folderID) + } + f.sl.DebugContext(ctx, "Populating block index if empty") + return f.db.PopulateBlockIndex(f.folderID) +} + func (*folder) BringToFront(string) {} func (*folder) Override() {}