diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index 1c2633a0c..172ca9cbe 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -161,24 +161,25 @@ func (c *CLI) AfterApply() error { type serveCmd struct { buildSpecificOptions - AllowNewerConfig bool `help:"Allow loading newer than current config version" env:"STALLOWNEWERCONFIG"` - Audit bool `help:"Write events to audit file" env:"STAUDIT"` - AuditFile string `name:"auditfile" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)" placeholder:"PATH" env:"STAUDITFILE"` - DBMaintenanceInterval time.Duration `help:"Database maintenance interval" default:"8h" env:"STDBMAINTINTERVAL"` - GUIAddress string `name:"gui-address" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")" placeholder:"URL" env:"STGUIADDRESS"` - GUIAPIKey string `name:"gui-apikey" help:"Override GUI API key" placeholder:"API-KEY" env:"STGUIAPIKEY"` - LogFile string `name:"logfile" help:"Log file name (see below)" default:"${logFile}" placeholder:"PATH" env:"STLOGFILE"` - LogFlags int `name:"logflags" help:"Select information in log line prefix (see below)" default:"${logFlags}" placeholder:"BITS" env:"STLOGFLAGS"` - LogMaxFiles int `name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)" default:"${logMaxFiles}" placeholder:"N" env:"STNUMLOGFILES"` - LogMaxSize int `help:"Maximum size of any file (zero to disable log rotation)" default:"${logMaxSize}" placeholder:"BYTES" env:"STLOGMAXSIZE"` - NoBrowser bool `help:"Do not start browser" env:"STNOBROWSER"` - NoDefaultFolder bool `help:"Don't create the \"default\" folder on first startup" env:"STNODEFAULTFOLDER"` - NoPortProbing bool `help:"Don't try to find free ports for GUI and listen addresses on first startup" env:"STNOPORTPROBING"` - NoRestart bool `help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash" env:"STNORESTART"` - NoUpgrade bool `help:"Disable automatic upgrades" env:"STNOUPGRADE"` - Paused bool `help:"Start with all devices and folders paused" env:"STPAUSED"` - Unpaused bool `help:"Start with all devices and folders unpaused" env:"STUNPAUSED"` - Verbose bool `help:"Print verbose log output" env:"STVERBOSE"` + AllowNewerConfig bool `help:"Allow loading newer than current config version" env:"STALLOWNEWERCONFIG"` + Audit bool `help:"Write events to audit file" env:"STAUDIT"` + AuditFile string `name:"auditfile" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)" placeholder:"PATH" env:"STAUDITFILE"` + DBMaintenanceInterval time.Duration `help:"Database maintenance interval" default:"8h" env:"STDBMAINTENANCEINTERVAL"` + DBDeleteRetentionInterval time.Duration `help:"Database deleted item retention interval" default:"4320h" env:"STDBDELETERETENTIONINTERVAL"` + GUIAddress string `name:"gui-address" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")" placeholder:"URL" env:"STGUIADDRESS"` + GUIAPIKey string `name:"gui-apikey" help:"Override GUI API key" placeholder:"API-KEY" env:"STGUIAPIKEY"` + LogFile string `name:"logfile" help:"Log file name (see below)" default:"${logFile}" placeholder:"PATH" env:"STLOGFILE"` + LogFlags int `name:"logflags" help:"Select information in log line prefix (see below)" default:"${logFlags}" placeholder:"BITS" env:"STLOGFLAGS"` + LogMaxFiles int `name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)" default:"${logMaxFiles}" placeholder:"N" env:"STLOGMAXOLDFILES"` + LogMaxSize int `help:"Maximum size of any file (zero to disable log rotation)" default:"${logMaxSize}" placeholder:"BYTES" env:"STLOGMAXSIZE"` + NoBrowser bool `help:"Do not start browser" env:"STNOBROWSER"` + NoDefaultFolder bool `help:"Don't create the \"default\" folder on first startup" env:"STNODEFAULTFOLDER"` + NoPortProbing bool `help:"Don't try to find free ports for GUI and listen addresses on first startup" env:"STNOPORTPROBING"` + NoRestart bool `help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash" env:"STNORESTART"` + NoUpgrade bool `help:"Disable automatic upgrades" env:"STNOUPGRADE"` + Paused bool `help:"Start with all devices and folders paused" env:"STPAUSED"` + Unpaused bool `help:"Start with all devices and folders unpaused" env:"STUNPAUSED"` + Verbose bool `help:"Print verbose log output" env:"STVERBOSE"` // Debug options below DebugGUIAssetsDir string `help:"Directory to load GUI assets from" placeholder:"PATH" env:"STGUIASSETS"` @@ -487,12 +488,12 @@ func (c *serveCmd) syncthingMain() { }) } - if err := syncthing.TryMigrateDatabase(); err != nil { + if err := syncthing.TryMigrateDatabase(c.DBDeleteRetentionInterval); err != nil { l.Warnln("Failed to migrate old-style database:", err) os.Exit(1) } - sdb, err := syncthing.OpenDatabase(locations.Get(locations.Database)) + sdb, err := syncthing.OpenDatabase(locations.Get(locations.Database), c.DBDeleteRetentionInterval) if err != nil { l.Warnln("Error opening database:", err) os.Exit(1) diff --git a/internal/db/sqlite/db.go b/internal/db/sqlite/db.go index 960218e1b..a0a6a5f1f 100644 --- a/internal/db/sqlite/db.go +++ b/internal/db/sqlite/db.go @@ -17,10 +17,12 @@ import ( ) type DB struct { - sql *sqlx.DB - localDeviceIdx int64 - updateLock sync.Mutex - updatePoints int + sql *sqlx.DB + localDeviceIdx int64 + deleteRetention time.Duration + + updateLock sync.Mutex + updatePoints int statementsMut sync.RWMutex statements map[string]*sqlx.Stmt @@ -29,6 +31,14 @@ type DB struct { var _ db.DB = (*DB)(nil) +type Option func(*DB) + +func WithDeleteRetention(d time.Duration) Option { + return func(s *DB) { + s.deleteRetention = d + } +} + func (s *DB) Close() error { s.updateLock.Lock() s.statementsMut.Lock() diff --git a/internal/db/sqlite/db_open.go b/internal/db/sqlite/db_open.go index 0a482f17c..b7d6346b7 100644 --- a/internal/db/sqlite/db_open.go +++ b/internal/db/sqlite/db_open.go @@ -21,7 +21,7 @@ import ( const maxDBConns = 128 -func Open(path string) (*DB, error) { +func Open(path string, opts ...Option) (*DB, error) { // Open the database with options to enable foreign keys and recursive // triggers (needed for the delete+insert triggers on row replace). sqlDB, err := sqlx.Open(dbDriver, "file:"+path+"?"+commonOptions) @@ -36,7 +36,7 @@ func Open(path string) (*DB, error) { // https://www.sqlite.org/pragma.html#pragma_optimize return nil, wrap(err, "PRAGMA optimize") } - return openCommon(sqlDB) + return openCommon(sqlDB, opts...) } // Open the database with options suitable for the migration inserts. This @@ -73,7 +73,7 @@ func OpenTemp() (*DB, error) { return Open(path) } -func openCommon(sqlDB *sqlx.DB) (*DB, error) { +func openCommon(sqlDB *sqlx.DB, opts ...Option) (*DB, error) { if _, err := sqlDB.Exec(`PRAGMA auto_vacuum = INCREMENTAL`); err != nil { return nil, wrap(err, "PRAGMA auto_vacuum") } @@ -85,8 +85,15 @@ func openCommon(sqlDB *sqlx.DB) (*DB, error) { } db := &DB{ - sql: sqlDB, - statements: make(map[string]*sqlx.Stmt), + sql: sqlDB, + deleteRetention: defaultDeleteRetention, + statements: make(map[string]*sqlx.Stmt), + } + for _, opt := range opts { + opt(db) + } + if db.deleteRetention > 0 && db.deleteRetention < minDeleteRetention { + db.deleteRetention = minDeleteRetention } if err := db.runScripts("sql/schema/*"); err != nil { diff --git a/internal/db/sqlite/db_service.go b/internal/db/sqlite/db_service.go index b4bd71a1b..c31acb19a 100644 --- a/internal/db/sqlite/db_service.go +++ b/internal/db/sqlite/db_service.go @@ -15,9 +15,10 @@ import ( ) const ( - internalMetaPrefix = "dbsvc" - lastMaintKey = "lastMaint" - MaxDeletedFileAge = 180 * 24 * time.Hour + internalMetaPrefix = "dbsvc" + lastMaintKey = "lastMaint" + defaultDeleteRetention = 180 * 24 * time.Hour + minDeleteRetention = 24 * time.Hour ) type Service struct { @@ -101,12 +102,18 @@ func (s *Service) periodic(ctx context.Context) error { } func (s *Service) garbageCollectOldDeletedLocked() error { + if s.sdb.deleteRetention <= 0 { + l.Debugln("Delete retention is infinite, skipping cleanup") + return nil + } + // Remove deleted files that are marked as not needed (we have processed // them) and they were deleted more than MaxDeletedFileAge ago. + l.Debugln("Forgetting deleted files older than", s.sdb.deleteRetention) res, err := s.sdb.stmt(` DELETE FROM files WHERE deleted AND modified < ? AND local_flags & {{.FlagLocalNeeded}} == 0 - `).Exec(time.Now().Add(-MaxDeletedFileAge).UnixNano()) + `).Exec(time.Now().Add(-s.sdb.deleteRetention).UnixNano()) if err != nil { return wrap(err) } diff --git a/lib/syncthing/utils.go b/lib/syncthing/utils.go index 019b59eaa..25ed23034 100644 --- a/lib/syncthing/utils.go +++ b/lib/syncthing/utils.go @@ -158,8 +158,8 @@ func copyFile(src, dst string) error { } // Opens a database -func OpenDatabase(path string) (newdb.DB, error) { - sql, err := sqlite.Open(path) +func OpenDatabase(path string, deleteRetention time.Duration) (newdb.DB, error) { + sql, err := sqlite.Open(path, sqlite.WithDeleteRetention(deleteRetention)) if err != nil { return nil, err } @@ -170,7 +170,7 @@ func OpenDatabase(path string) (newdb.DB, error) { } // Attempts migration of the old (LevelDB-based) database type to the new (SQLite-based) type -func TryMigrateDatabase() error { +func TryMigrateDatabase(deleteRetention time.Duration) error { oldDBDir := locations.Get(locations.LegacyDatabase) if _, err := os.Lstat(oldDBDir); err != nil { // No old database @@ -251,7 +251,7 @@ func TryMigrateDatabase() error { return err } _ = snap.WithHaveSequence(0, func(fi protocol.FileInfo) bool { - if fi.Deleted && time.Since(fi.ModTime()) > sqlite.MaxDeletedFileAge { + if deleteRetention > 0 && fi.Deleted && time.Since(fi.ModTime()) > deleteRetention { // Skip deleted files that match the garbage collection // criteria in the database return true