wip

Signed-off-by: Jakob Borg <jakob@kastelo.net>

internal/db/interface.go

@@ -73,6 +73,10 @@ type DB interface {
 	AllNeededGlobalFiles(folder string, device protocol.DeviceID, order config.PullOrder, limit, offset int) (iter.Seq[protocol.FileInfo], func() error)
 	AllLocalBlocksWithHash(folder string, hash []byte) (iter.Seq[BlockMapEntry], func() error)
 
+	// Block index management
+	DropBlockIndex(folder string) error
+	PopulateBlockIndex(folder string) error
+
 	// Cleanup
 	DropAllFiles(folder string, device protocol.DeviceID) error
 	DropDevice(device protocol.DeviceID) error

internal/db/sqlite/db_folderdb.go

@@ -104,6 +104,25 @@ func (s *DB) Update(folder string, device protocol.DeviceID, fs []protocol.FileI
 	return fdb.Update(device, fs, options)
 }
 
+func (s *DB) DropBlockIndex(folder string) error {
+	fdb, err := s.getFolderDB(folder, false)
+	if errors.Is(err, errNoSuchFolder) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	return fdb.DropBlockIndex()
+}
+
+func (s *DB) PopulateBlockIndex(folder string) error {
+	fdb, err := s.getFolderDB(folder, true)
+	if err != nil {
+		return err
+	}
+	return fdb.PopulateBlockIndex()
+}
+
 func (s *DB) GetDeviceFile(folder string, device protocol.DeviceID, file string) (protocol.FileInfo, bool, error) {
 	fdb, err := s.getFolderDB(folder, false)
 	if errors.Is(err, errNoSuchFolder) {

internal/db/sqlite/folderdb_update.go

@@ -304,6 +304,83 @@ func (s *folderDB) DropFilesNamed(device protocol.DeviceID, names []string) erro
 	return wrap(tx.Commit())
 }
 
+func (s *folderDB) blockIndexEmpty() (bool, error) {
+	var exists bool
+	err := s.sql.Get(&exists, `SELECT EXISTS (SELECT 1 FROM blocks LIMIT 1)`)
+	if err != nil {
+		return false, wrap(err)
+	}
+	return !exists, nil
+}
+
+func (s *folderDB) DropBlockIndex() error {
+	empty, err := s.blockIndexEmpty()
+	if err != nil || empty {
+		return err
+	}
+
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+
+	_, err = s.sql.Exec(`DELETE FROM blocks`)
+	return wrap(err)
+}
+
+func (s *folderDB) PopulateBlockIndex() error {
+	empty, err := s.blockIndexEmpty()
+	if err != nil || !empty {
+		return err
+	}
+
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+
+	tx, err := s.sql.BeginTxx(context.Background(), nil)
+	if err != nil {
+		return wrap(err)
+	}
+	defer tx.Rollback()
+	txp := &txPreparedStmts{Tx: tx}
+
+	// Iterate all local files that have a blocklist
+	rows, err := tx.Queryx(`
+		SELECT f.blocklist_hash, bl.blprotobuf FROM files f
+		INNER JOIN blocklists bl ON bl.blocklist_hash = f.blocklist_hash
+		WHERE f.device_idx = ? AND f.blocklist_hash IS NOT NULL
+	`, s.localDeviceIdx)
+	if err != nil {
+		return wrap(err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var blocklistHash []byte
+		var blProtobuf []byte
+		if err := rows.Scan(&blocklistHash, &blProtobuf); err != nil {
+			return wrap(err)
+		}
+
+		var bl dbproto.BlockList
+		if err := proto.Unmarshal(blProtobuf, &bl); err != nil {
+			return wrap(err, "unmarshal blocklist")
+		}
+
+		blocks := make([]protocol.BlockInfo, len(bl.Blocks))
+		for i, b := range bl.Blocks {
+			blocks[i] = protocol.BlockInfoFromWire(b)
+		}
+
+		if err := s.insertBlocksLocked(txp, blocklistHash, blocks); err != nil {
+			return wrap(err, "insert blocks")
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return wrap(err)
+	}
+
+	return wrap(tx.Commit())
+}
+
 func (*folderDB) insertBlocksLocked(tx *txPreparedStmts, blocklistHash []byte, blocks []protocol.BlockInfo) error {
 	if len(blocks) == 0 {
 		return nil

lib/model/folder.go

@@ -159,6 +159,10 @@ func (f *folder) Serve(ctx context.Context) error {
 		f.setState(FolderIdle)
 	}()
 
+	if err := f.reconcileBlockIndex(ctx); err != nil {
+		return err
+	}
+
 	if f.FSWatcherEnabled && f.getHealthErrorAndLoadIgnores() == nil {
 		f.startWatch(ctx)
 	}
@@ -256,6 +260,15 @@ func (f *folder) Serve(ctx context.Context) error {
 	}
 }
 
+func (f *folder) reconcileBlockIndex(ctx context.Context) error {
+	if !f.FullBlockIndex {
+		f.sl.DebugContext(ctx, "Dropping block index (full block index disabled)")
+		return f.db.DropBlockIndex(f.folderID)
+	}
+	f.sl.DebugContext(ctx, "Populating block index if empty")
+	return f.db.PopulateBlockIndex(f.folderID)
+}
+
 func (*folder) BringToFront(string) {}
 
 func (*folder) Override() {}