mirror/syncthing
1
0
Fork 0
mirror of https://github.com/syncthing/syncthing.git synced 2025-12-23 22:18:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: add syncthing debug database-statistics command (#10117)

Browse Source 
This adds a command that shows database statistics. Currently it
requires a fork of the sqlite package to add the dbstats virtual table;
the modernc variant already has it.

This also provides the canonical mapping between folder ID and database
file, for tinkerers...

```
% ./bin/syncthing debug database-statistics
DATABASE                 FOLDER ID    TABLE                                  SIZE     FILL
========                 ====== ==    =====                                  ====     ====
main.db                  -            folders                               4 KiB    8.4 %
main.db                  -            folders_database_name                 4 KiB    6.0 %
main.db                  -            kv                                    4 KiB   41.1 %
main.db                  -            schemamigrations                      4 KiB    3.9 %
main.db                  -            sqlite_autoindex_folders_1            4 KiB    3.7 %
...
folder.0007-txpxsvyd.db  w3ejt-fn4dm  indexids                              4 KiB    1.5 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  kv                                    4 KiB    0.8 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  mtimes                              608 KiB   81.5 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  schemamigrations                      4 KiB    3.9 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_autoindex_blocklists_1      4108 KiB   89.5 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_autoindex_blocks_1        700020 KiB   88.1 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_autoindex_devices_1            4 KiB    3.6 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_autoindex_kv_1                 4 KiB    0.6 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_schema                        12 KiB   45.9 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_sequence                       4 KiB    1.0 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_stat1                          4 KiB   12.2 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  sqlite_stat4                          4 KiB    0.2 %
folder.0007-txpxsvyd.db  w3ejt-fn4dm  (total)                         1906020 KiB   92.8 %
main.db + children       -            (total)                         2205888 KiB   92.0 %
```
This commit is contained in:
Jakob Borg
2025-05-20 14:27:08 +02:00
committed by GitHub
parent 72849690c9
commit 085455d72e
6 changed files with 121 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/build-syncthing.yaml vendored
View File

@@ -22,7 +22,7 @@ env:
BUILD_USER: builder
BUILD_HOST: github.syncthing.net
TAGS: "netgo osusergo sqlite_omit_load_extension"
TAGS: "netgo osusergo sqlite_omit_load_extension sqlite_dbstat"
# A note on actions and third party code... The actions under actions/ (like
# `uses: actions/checkout`) are maintained by GitHub, and we need to trust

64
cmd/syncthing/main.go
View File

@@ -8,6 +8,7 @@ package main
import (
"bytes"
"cmp"
"context"
"crypto/tls"
"errors"
@@ -26,6 +27,7 @@ import (
"sort"
"strconv"
"syscall"
"text/tabwriter"
"time"
"github.com/alecthomas/kong"
@@ -37,6 +39,7 @@ import (
"github.com/syncthing/syncthing/cmd/syncthing/decrypt"
"github.com/syncthing/syncthing/cmd/syncthing/generate"
"github.com/syncthing/syncthing/internal/db"
"github.com/syncthing/syncthing/internal/db/sqlite"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
@@ -825,24 +828,6 @@ func exitCodeForUpgrade(err error) int {
return svcutil.ExitError.AsInt()
}
// convertLegacyArgs returns the slice of arguments with single dash long
// flags converted to double dash long flags.
func convertLegacyArgs(args []string) []string {
// Legacy args begin with a single dash, followed by two or more characters.
legacyExp := regexp.MustCompile(`^-\w{2,}`)
res := make([]string, len(args))
for i, arg := range args {
if legacyExp.MatchString(arg) {
res[i] = "-" + arg
} else {
res[i] = arg
}
}
return res
}
type versionCmd struct{}
func (versionCmd) Run() error {
@@ -900,7 +885,8 @@ func (u upgradeCmd) Run() error {
release, err := checkUpgrade()
if err == nil {
lf := flock.New(locations.Get(locations.LockFile))
locked, err := lf.TryLock()
var locked bool
locked, err = lf.TryLock()
if err != nil {
l.Warnln("Upgrade:", err)
os.Exit(1)
@@ -930,7 +916,8 @@ func (browserCmd) Run() error {
}
type debugCmd struct {
ResetDatabase resetDatabaseCmd `cmd:"" help:"Reset the database, forcing a full rescan and resync"`
ResetDatabase resetDatabaseCmd `cmd:"" help:"Reset the database, forcing a full rescan and resync"`
DatabaseStatistics databaseStatsCmd `cmd:"" help:"Display database size statistics"`
}
type resetDatabaseCmd struct{}
@@ -945,6 +932,43 @@ func (resetDatabaseCmd) Run() error {
return nil
}
type databaseStatsCmd struct{}
func (c databaseStatsCmd) Run() error {
db, err := sqlite.Open(locations.Get(locations.Database))
if err != nil {
return err
}
ds, err := db.Statistics()
if err != nil {
return err
}
tw := tabwriter.NewWriter(os.Stdout, 2, 2, 2, ' ', 0)
hdr := fmt.Sprintf("%s\t%s\t%s\t%12s\t%7s\n", "DATABASE", "FOLDER ID", "TABLE", "SIZE", "FILL")
fmt.Fprint(tw, hdr)
fmt.Fprint(tw, regexp.MustCompile(`[A-Z]`).ReplaceAllString(hdr, "="))
c.printStat(tw, ds)
return tw.Flush()
}
func (c databaseStatsCmd) printStat(w io.Writer, s *sqlite.DatabaseStatistics) {
for _, table := range s.Tables {
fmt.Fprintf(w, "%s\t%s\t%s\t%8d KiB\t%5.01f %%\n", s.Name, cmp.Or(s.FolderID, "-"), table.Name, table.Size/1024, float64(table.Size-table.Unused)*100/float64(table.Size))
}
for _, next := range s.Children {
c.printStat(w, &next)
s.Total.Size += next.Total.Size
s.Total.Unused += next.Total.Unused
}
totalName := s.Name
if len(s.Children) > 0 {
totalName += " + children"
}
fmt.Fprintf(w, "%s\t%s\t%s\t%8d KiB\t%5.01f %%\n", totalName, cmp.Or(s.FolderID, "-"), "(total)", s.Total.Size/1024, float64(s.Total.Size-s.Total.Unused)*100/float64(s.Total.Size))
}
func setConfigDataLocationsFromFlags(homeDir, confDir, dataDir string) error {
homeSet := homeDir != ""
confSet := confDir != ""

5
go.mod
View File

@@ -22,7 +22,7 @@ require (
github.com/julienschmidt/httprouter v1.3.0
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/maruel/panicparse/v2 v2.5.0
github.com/mattn/go-sqlite3 v1.14.27
github.com/mattn/go-sqlite3 v1.14.28
github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2
github.com/maxmind/geoipupdate/v6 v6.1.0
github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75
@@ -108,3 +108,6 @@ require (
// https://github.com/gobwas/glob/pull/55
replace github.com/gobwas/glob v0.2.3 => github.com/calmh/glob v0.0.0-20220615080505-1d823af5017b
// https://github.com/mattn/go-sqlite3/pull/1338
replace github.com/mattn/go-sqlite3 v1.14.28 => github.com/calmh/go-sqlite3 v1.14.29-0.20250520105817-2e94cda3f7f8

4
go.sum
View File

@@ -31,6 +31,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/calmh/glob v0.0.0-20220615080505-1d823af5017b h1:Fjm4GuJ+TGMgqfGHN42IQArJb77CfD/mAwLbDUoJe6g=
github.com/calmh/glob v0.0.0-20220615080505-1d823af5017b/go.mod h1:91K7jfEsgJSyfSrX+gmrRfZMtntx6JsHolWubGXDopg=
github.com/calmh/go-sqlite3 v1.14.29-0.20250520105817-2e94cda3f7f8 h1:oNVrBJGXkD334ToEmxJz8G6LhzD1/sMA4twMHsMLzQo=
github.com/calmh/go-sqlite3 v1.14.29-0.20250520105817-2e94cda3f7f8/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/calmh/incontainer v1.0.0 h1:g2cTUtZuFGmMGX8GoykPkN1Judj2uw8/3/aEtq4Z/rg=
github.com/calmh/incontainer v1.0.0/go.mod h1:eOhqnw15c9X+4RNBe0W3HlUZFfX16O0EDsCOInTndHY=
github.com/calmh/xdr v1.2.0 h1:GaGSNH4ZDw9kNdYqle6+RcAENiaQ8/611Ok+jQbBEeU=
@@ -164,8 +166,6 @@ github.com/maruel/panicparse/v2 v2.5.0/go.mod h1:DA2fDiBk63bKfBf4CVZP9gb4fuvzdPb
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/go-sqlite3 v1.14.27 h1:drZCnuvf37yPfs95E5jd9s3XhdVWLal+6BOK6qrv6IU=
github.com/mattn/go-sqlite3 v1.14.27/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 h1:yVCLo4+ACVroOEr4iFU1iH46Ldlzz2rTuu18Ra7M8sU=
github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2/go.mod h1:VzB2VoMh1Y32/QqDfg9ZJYHj99oM4LiGtqPZydTiQSQ=
github.com/maxmind/geoipupdate/v6 v6.1.0 h1:sdtTHzzQNJlXF5+fd/EoPTucRHyMonYt/Cok8xzzfqA=

69
internal/db/sqlite/db_stats.go Normal file
View File

@@ -0,0 +1,69 @@
// Copyright (C) 2025 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package sqlite
type DatabaseStatistics struct {
Name string `json:"name"`
FolderID string `json:"folderID,omitempty"`
Tables []TableStatistics `json:"tables"`
Total TableStatistics `json:"total"`
Children []DatabaseStatistics `json:"children,omitempty"`
}
type TableStatistics struct {
Name string `json:"name,omitempty"`
Size int64 `json:"size"`
Unused int64 `json:"unused"`
}
func (s *DB) Statistics() (*DatabaseStatistics, error) {
ts, total, err := s.tableStats()
if err != nil {
return nil, wrap(err)
}
ds := DatabaseStatistics{
Name: s.baseName,
Tables: ts,
Total: total,
}
err = s.forEachFolder(func(fdb *folderDB) error {
tables, total, err := fdb.tableStats()
if err != nil {
return wrap(err)
}
ds.Children = append(ds.Children, DatabaseStatistics{
Name: fdb.baseName,
FolderID: fdb.folderID,
Tables: tables,
Total: total,
})
return nil
})
if err != nil {
return nil, wrap(err)
}
return &ds, nil
}
func (s *baseDB) tableStats() ([]TableStatistics, TableStatistics, error) {
var stats []TableStatistics
if err := s.stmt(`
SELECT name, pgsize AS size, unused FROM dbstat
WHERE aggregate=true
ORDER BY name
`).Select(&stats); err != nil {
return nil, TableStatistics{}, wrap(err)
}
var total TableStatistics
for _, s := range stats {
total.Size += s.Size
total.Unused += s.Unused
}
return stats, total, nil
}

1
lib/build/build.go
View File

@@ -48,6 +48,7 @@ var (
}
replaceTags = map[string]string{
"sqlite_omit_load_extension": "",
"sqlite_dbstat": "",
"osusergo": "",
"netgo": "",
}