mirror/syncthing
1
0
Fork 0
mirror of https://github.com/syncthing/syncthing.git synced 2026-02-07 12:44:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore: configurable delete retention interval (#10030)

Browse Source 
Command line flag, as it also needs to be able to take effect during
migration.
This commit is contained in:
Jakob Borg
2025-04-03 00:55:19 -07:00
committed by GitHub
parent b88aea34b6
commit 8a2d8ebf81
5 changed files with 62 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
cmd/syncthing/main.go
View File

@@ -161,24 +161,25 @@ func (c *CLI) AfterApply() error {
type serveCmd struct {
buildSpecificOptions
AllowNewerConfig bool `help:"Allow loading newer than current config version" env:"STALLOWNEWERCONFIG"`
Audit bool `help:"Write events to audit file" env:"STAUDIT"`
AuditFile string `name:"auditfile" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)" placeholder:"PATH" env:"STAUDITFILE"`
DBMaintenanceInterval time.Duration `help:"Database maintenance interval" default:"8h" env:"STDBMAINTINTERVAL"`
GUIAddress string `name:"gui-address" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")" placeholder:"URL" env:"STGUIADDRESS"`
GUIAPIKey string `name:"gui-apikey" help:"Override GUI API key" placeholder:"API-KEY" env:"STGUIAPIKEY"`
LogFile string `name:"logfile" help:"Log file name (see below)" default:"${logFile}" placeholder:"PATH" env:"STLOGFILE"`
LogFlags int `name:"logflags" help:"Select information in log line prefix (see below)" default:"${logFlags}" placeholder:"BITS" env:"STLOGFLAGS"`
LogMaxFiles int `name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)" default:"${logMaxFiles}" placeholder:"N" env:"STNUMLOGFILES"`
LogMaxSize int `help:"Maximum size of any file (zero to disable log rotation)" default:"${logMaxSize}" placeholder:"BYTES" env:"STLOGMAXSIZE"`
NoBrowser bool `help:"Do not start browser" env:"STNOBROWSER"`
NoDefaultFolder bool `help:"Don't create the \"default\" folder on first startup" env:"STNODEFAULTFOLDER"`
NoPortProbing bool `help:"Don't try to find free ports for GUI and listen addresses on first startup" env:"STNOPORTPROBING"`
NoRestart bool `help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash" env:"STNORESTART"`
NoUpgrade bool `help:"Disable automatic upgrades" env:"STNOUPGRADE"`
Paused bool `help:"Start with all devices and folders paused" env:"STPAUSED"`
Unpaused bool `help:"Start with all devices and folders unpaused" env:"STUNPAUSED"`
Verbose bool `help:"Print verbose log output" env:"STVERBOSE"`
AllowNewerConfig bool `help:"Allow loading newer than current config version" env:"STALLOWNEWERCONFIG"`
Audit bool `help:"Write events to audit file" env:"STAUDIT"`
AuditFile string `name:"auditfile" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)" placeholder:"PATH" env:"STAUDITFILE"`
DBMaintenanceInterval time.Duration `help:"Database maintenance interval" default:"8h" env:"STDBMAINTENANCEINTERVAL"`
DBDeleteRetentionInterval time.Duration `help:"Database deleted item retention interval" default:"4320h" env:"STDBDELETERETENTIONINTERVAL"`
GUIAddress string `name:"gui-address" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")" placeholder:"URL" env:"STGUIADDRESS"`
GUIAPIKey string `name:"gui-apikey" help:"Override GUI API key" placeholder:"API-KEY" env:"STGUIAPIKEY"`
LogFile string `name:"logfile" help:"Log file name (see below)" default:"${logFile}" placeholder:"PATH" env:"STLOGFILE"`
LogFlags int `name:"logflags" help:"Select information in log line prefix (see below)" default:"${logFlags}" placeholder:"BITS" env:"STLOGFLAGS"`
LogMaxFiles int `name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)" default:"${logMaxFiles}" placeholder:"N" env:"STLOGMAXOLDFILES"`
LogMaxSize int `help:"Maximum size of any file (zero to disable log rotation)" default:"${logMaxSize}" placeholder:"BYTES" env:"STLOGMAXSIZE"`
NoBrowser bool `help:"Do not start browser" env:"STNOBROWSER"`
NoDefaultFolder bool `help:"Don't create the \"default\" folder on first startup" env:"STNODEFAULTFOLDER"`
NoPortProbing bool `help:"Don't try to find free ports for GUI and listen addresses on first startup" env:"STNOPORTPROBING"`
NoRestart bool `help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash" env:"STNORESTART"`
NoUpgrade bool `help:"Disable automatic upgrades" env:"STNOUPGRADE"`
Paused bool `help:"Start with all devices and folders paused" env:"STPAUSED"`
Unpaused bool `help:"Start with all devices and folders unpaused" env:"STUNPAUSED"`
Verbose bool `help:"Print verbose log output" env:"STVERBOSE"`
// Debug options below
DebugGUIAssetsDir string `help:"Directory to load GUI assets from" placeholder:"PATH" env:"STGUIASSETS"`
@@ -487,12 +488,12 @@ func (c *serveCmd) syncthingMain() {
})
}
if err := syncthing.TryMigrateDatabase(); err != nil {
if err := syncthing.TryMigrateDatabase(c.DBDeleteRetentionInterval); err != nil {
l.Warnln("Failed to migrate old-style database:", err)
os.Exit(1)
}
sdb, err := syncthing.OpenDatabase(locations.Get(locations.Database))
sdb, err := syncthing.OpenDatabase(locations.Get(locations.Database), c.DBDeleteRetentionInterval)
if err != nil {
l.Warnln("Error opening database:", err)
os.Exit(1)

18
internal/db/sqlite/db.go
View File

@@ -17,10 +17,12 @@ import (
)
type DB struct {
sql *sqlx.DB
localDeviceIdx int64
updateLock sync.Mutex
updatePoints int
sql *sqlx.DB
localDeviceIdx int64
deleteRetention time.Duration
updateLock sync.Mutex
updatePoints int
statementsMut sync.RWMutex
statements map[string]*sqlx.Stmt
@@ -29,6 +31,14 @@ type DB struct {
var _ db.DB = (*DB)(nil)
type Option func(*DB)
func WithDeleteRetention(d time.Duration) Option {
return func(s *DB) {
s.deleteRetention = d
}
}
func (s *DB) Close() error {
s.updateLock.Lock()
s.statementsMut.Lock()

17
internal/db/sqlite/db_open.go
View File

@@ -21,7 +21,7 @@ import (
const maxDBConns = 128
func Open(path string) (*DB, error) {
func Open(path string, opts ...Option) (*DB, error) {
// Open the database with options to enable foreign keys and recursive
// triggers (needed for the delete+insert triggers on row replace).
sqlDB, err := sqlx.Open(dbDriver, "file:"+path+"?"+commonOptions)
@@ -36,7 +36,7 @@ func Open(path string) (*DB, error) {
// https://www.sqlite.org/pragma.html#pragma_optimize
return nil, wrap(err, "PRAGMA optimize")
}
return openCommon(sqlDB)
return openCommon(sqlDB, opts...)
}
// Open the database with options suitable for the migration inserts. This
@@ -73,7 +73,7 @@ func OpenTemp() (*DB, error) {
return Open(path)
}
func openCommon(sqlDB *sqlx.DB) (*DB, error) {
func openCommon(sqlDB *sqlx.DB, opts ...Option) (*DB, error) {
if _, err := sqlDB.Exec(`PRAGMA auto_vacuum = INCREMENTAL`); err != nil {
return nil, wrap(err, "PRAGMA auto_vacuum")
}
@@ -85,8 +85,15 @@ func openCommon(sqlDB *sqlx.DB) (*DB, error) {
}
db := &DB{
sql: sqlDB,
statements: make(map[string]*sqlx.Stmt),
sql: sqlDB,
deleteRetention: defaultDeleteRetention,
statements: make(map[string]*sqlx.Stmt),
}
for _, opt := range opts {
opt(db)
}
if db.deleteRetention > 0 && db.deleteRetention < minDeleteRetention {
db.deleteRetention = minDeleteRetention
}
if err := db.runScripts("sql/schema/*"); err != nil {

15
internal/db/sqlite/db_service.go
View File

@@ -15,9 +15,10 @@ import (
)
const (
internalMetaPrefix = "dbsvc"
lastMaintKey = "lastMaint"
MaxDeletedFileAge = 180 * 24 * time.Hour
internalMetaPrefix = "dbsvc"
lastMaintKey = "lastMaint"
defaultDeleteRetention = 180 * 24 * time.Hour
minDeleteRetention = 24 * time.Hour
)
type Service struct {
@@ -101,12 +102,18 @@ func (s *Service) periodic(ctx context.Context) error {
}
func (s *Service) garbageCollectOldDeletedLocked() error {
if s.sdb.deleteRetention <= 0 {
l.Debugln("Delete retention is infinite, skipping cleanup")
return nil
}
// Remove deleted files that are marked as not needed (we have processed
// them) and they were deleted more than MaxDeletedFileAge ago.
l.Debugln("Forgetting deleted files older than", s.sdb.deleteRetention)
res, err := s.sdb.stmt(`
DELETE FROM files
WHERE deleted AND modified < ? AND local_flags & {{.FlagLocalNeeded}} == 0
`).Exec(time.Now().Add(-MaxDeletedFileAge).UnixNano())
`).Exec(time.Now().Add(-s.sdb.deleteRetention).UnixNano())
if err != nil {
return wrap(err)
}

8
lib/syncthing/utils.go
View File

@@ -158,8 +158,8 @@ func copyFile(src, dst string) error {
}
// Opens a database
func OpenDatabase(path string) (newdb.DB, error) {
sql, err := sqlite.Open(path)
func OpenDatabase(path string, deleteRetention time.Duration) (newdb.DB, error) {
sql, err := sqlite.Open(path, sqlite.WithDeleteRetention(deleteRetention))
if err != nil {
return nil, err
}
@@ -170,7 +170,7 @@ func OpenDatabase(path string) (newdb.DB, error) {
}
// Attempts migration of the old (LevelDB-based) database type to the new (SQLite-based) type
func TryMigrateDatabase() error {
func TryMigrateDatabase(deleteRetention time.Duration) error {
oldDBDir := locations.Get(locations.LegacyDatabase)
if _, err := os.Lstat(oldDBDir); err != nil {
// No old database
@@ -251,7 +251,7 @@ func TryMigrateDatabase() error {
return err
}
_ = snap.WithHaveSequence(0, func(fi protocol.FileInfo) bool {
if fi.Deleted && time.Since(fi.ModTime()) > sqlite.MaxDeletedFileAge {
if deleteRetention > 0 && fi.Deleted && time.Since(fi.ModTime()) > deleteRetention {
// Skip deleted files that match the garbage collection
// criteria in the database
return true