Merge branch 'main' into v2

* main:
  fix(config): properly apply defaults when reading folder configuration (#10034)
  chore(model): add metric for total number of conflicts (#10037)
  build: replace underscore in Debian version (#10032)

.github/workflows/build-syncthing.yaml

@@ -13,8 +13,6 @@ env:
   GO_VERSION: "~1.24.0"
 
   # Optimize compatibility on the slow archictures.
-  GO386: softfloat
-  GOARM: "5"
   GOMIPS: softfloat
 
   # Avoid hilarious amounts of obscuring log output when running tests.
@@ -24,6 +22,8 @@ env:
   BUILD_USER: builder
   BUILD_HOST: github.syncthing.net
 
+  TAGS: "netgo osusergo sqlite_omit_load_extension"
+
 # A note on actions and third party code... The actions under actions/ (like
 # `uses: actions/checkout`) are maintained by GitHub, and we need to trust
 # GitHub to maintain their code and infrastructure or we're in deep shit in
@@ -85,6 +85,7 @@ jobs:
           LOKI_USER: ${{ secrets.LOKI_USER }}
           LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD }}
           LOKI_LABELS: "go=${{ matrix.go }},runner=${{ matrix.runner }},repo=${{ github.repository }},ref=${{ github.ref }}"
+          CGO_ENABLED: "1"
 
   #
   # Meta checks for formatting, copyright, etc
@@ -136,17 +137,8 @@ jobs:
 
   package-windows:
     name: Package for Windows
-    runs-on: windows-latest
+    runs-on: ubuntu-latest
     steps:
-      - name: Set git to use LF
-        # Without this, the checkout will happen with CRLF line endings,
-        # which is fine for the source code but messes up tests that depend
-        # on data on disk being as expected. Ideally, those tests should be
-        # fixed, but not today.
-        run: |
-          git config --global core.autocrlf false
-          git config --global core.eol lf
-
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
@@ -158,17 +150,14 @@ jobs:
           cache: false
           check-latest: true
 
-      - name: Get actual Go version
-        run: |
-          go version
-          echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
+      - uses: mlugg/setup-zig@v1
 
       - uses: actions/cache@v4
         with:
           path: |
-            ~\AppData\Local\go-build
-            ~\go\pkg\mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-windows-${{ hashFiles('**/go.sum') }}
 
       - name: Install dependencies
         run: |
@@ -176,15 +165,14 @@ jobs:
 
       - name: Create packages
         run: |
-          $targets = 'syncthing', 'stdiscosrv', 'strelaysrv'
-          $archs = 'amd64', 'arm', 'arm64', '386'
-          foreach ($arch in $archs) {
-            foreach ($tgt in $targets) {
-              go run build.go -goarch $arch zip $tgt
-            }
-          }
+          for tgt in syncthing stdiscosrv strelaysrv ; do
+            go run build.go -tags "${{env.TAGS}}" -goos windows -goarch amd64 -cc "zig cc -target x86_64-windows" zip $tgt
+            go run build.go -tags "${{env.TAGS}}" -goos windows -goarch 386 -cc "zig cc -target x86-windows" zip $tgt
+            go run build.go -tags "${{env.TAGS}}" -goos windows -goarch arm64 -cc "zig cc -target aarch64-windows" zip $tgt
+            # go run build.go -tags "${{env.TAGS}}" -goos windows -goarch arm -cc "zig cc -target thumb-windows" zip $tgt # failes with linker errors
+          done
         env:
-          CGO_ENABLED: "0"
+          CGO_ENABLED: "1"
 
       - name: Archive artifacts
         uses: actions/upload-artifact@v4
@@ -194,7 +182,7 @@ jobs:
 
   codesign-windows:
     name: Codesign for Windows
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
+    if: github.repository_owner == 'syncthing' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
     environment: release
     runs-on: windows-latest
     needs:
@@ -269,6 +257,8 @@ jobs:
           go version
           echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
 
+      - uses: mlugg/setup-zig@v1
+
       - uses: actions/cache@v4
         with:
           path: |
@@ -278,14 +268,25 @@ jobs:
 
       - name: Create packages
         run: |
-          archs=$(go tool dist list | grep linux | sed 's#linux/##')
-          for goarch in $archs ; do
-            for tgt in syncthing stdiscosrv strelaysrv ; do
-              go run build.go -goarch "$goarch" tar "$tgt"
-            done
+          sudo apt-get install -y gcc-mips64-linux-gnuabi64 gcc-mips64el-linux-gnuabi64
+          for tgt in syncthing stdiscosrv strelaysrv ; do
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch amd64 -cc "zig cc -target x86_64-linux-musl" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch 386 -cc "zig cc -target x86-linux-musl" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch arm -cc "zig cc -target arm-linux-musleabi" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch arm64 -cc "zig cc -target aarch64-linux-musl" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch mips -cc "zig cc -target mips-linux-musleabi" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch mipsle -cc "zig cc -target mipsel-linux-musleabi" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch mips64 -cc mips64-linux-gnuabi64-gcc tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch mips64le -cc mips64el-linux-gnuabi64-gcc tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch riscv64 -cc "zig cc -target riscv64-linux-musl" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch s390x -cc "zig cc -target s390x-linux-musl" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch loong64 -cc "zig cc -target loongarch64-linux-musl" tar "$tgt"
+            # go run build.go -tags "${{env.TAGS}}" -goos linux -goarch ppc64 -cc "zig cc -target powerpc64-linux-musl" tar "$tgt" # fails with linkmode not supported
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch ppc64le -cc "zig cc -target powerpc64le-linux-musl" tar "$tgt"
           done
         env:
-          CGO_ENABLED: "0"
+          CGO_ENABLED: "1"
+          EXTRA_LDFLAGS: "-linkmode=external -extldflags=-static"
 
       - name: Archive artifacts
         uses: actions/upload-artifact@v4
@@ -303,6 +304,8 @@ jobs:
     name: Package for macOS
     if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
     environment: release
+    env:
+      CODESIGN_IDENTITY: ${{ secrets.CODESIGN_IDENTITY }}
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4
@@ -329,6 +332,7 @@ jobs:
           key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
 
       - name: Import signing certificate
+        if: env.CODESIGN_IDENTITY != ''
         run: |
           # Set up a run-specific keychain, making it available for the
           # `codesign` tool.
@@ -356,7 +360,7 @@ jobs:
       - name: Create package (amd64)
         run: |
           for tgt in syncthing stdiscosrv strelaysrv ; do
-            go run build.go -goarch amd64 zip "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goarch amd64 zip "$tgt"
           done
         env:
           CGO_ENABLED: "1"
@@ -372,7 +376,7 @@ jobs:
           EOT
           chmod 755 xgo.sh
           for tgt in syncthing stdiscosrv strelaysrv ; do
-            go run build.go -gocmd ./xgo.sh -goarch arm64 zip "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -gocmd ./xgo.sh -goarch arm64 zip "$tgt"
           done
         env:
           CGO_ENABLED: "1"
@@ -401,7 +405,7 @@ jobs:
 
   notarize-macos:
     name: Notarize for macOS
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
+    if: github.repository_owner == 'syncthing' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
     environment: release
     needs:
       - package-macos
@@ -483,7 +487,7 @@ jobs:
             goarch="${plat#*/}"
             echo "::group ::$plat"
             for tgt in syncthing stdiscosrv strelaysrv ; do
-              if ! go run build.go -goos "$goos" -goarch "$goarch" tar "$tgt" 2>/dev/null; then
+              if ! go run build.go -goos "$goos" -goarch "$goarch" tar "$tgt" ; then
                 echo "::warning ::Failed to build $tgt for $plat"
               fi
             done
@@ -545,7 +549,7 @@ jobs:
 
   sign-for-upgrade:
     name: Sign for upgrade
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-')  || startsWith(github.ref, 'refs/tags/v'))
+    if: github.repository_owner == 'syncthing' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-')  || startsWith(github.ref, 'refs/tags/v'))
     environment: release
     needs:
       - codesign-windows
@@ -663,6 +667,8 @@ jobs:
         run: |
           gem install fpm
 
+      - uses: mlugg/setup-zig@v1
+
       - uses: actions/cache@v4
         with:
           path: |
@@ -670,15 +676,17 @@ jobs:
             ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-debian-${{ hashFiles('**/go.sum') }}
 
-      - name: Package for Debian
+      - name: Package for Debian (CGO)
         run: |
-          for arch in amd64 i386 armhf armel arm64 ; do
-            for tgt in syncthing stdiscosrv strelaysrv ; do
-              go run build.go -no-upgrade -installsuffix=no-upgrade -goarch "$arch" deb "$tgt"
-            done
+          for tgt in syncthing stdiscosrv strelaysrv ; do
+            go run build.go -no-upgrade -installsuffix=no-upgrade -tags "${{env.TAGS}}" -goos linux -goarch amd64 -cc "zig cc -target x86_64-linux-musl" deb "$tgt"
+            go run build.go -no-upgrade -installsuffix=no-upgrade -tags "${{env.TAGS}}" -goos linux -goarch arm -cc "zig cc -target arm-linux-musleabi" deb "$tgt"
+            go run build.go -no-upgrade -installsuffix=no-upgrade -tags "${{env.TAGS}}" -goos linux -goarch arm64 -cc "zig cc -target aarch64-linux-musl" deb "$tgt"
           done
         env:
           BUILD_USER: debian
+          CGO_ENABLED: "1"
+          EXTRA_LDFLAGS: "-linkmode=external -extldflags=-static"
 
       - name: Archive artifacts
         uses: actions/upload-artifact@v4
@@ -692,7 +700,7 @@ jobs:
 
   publish-nightly:
     name: Publish nightly build
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && startsWith(github.ref, 'refs/heads/release-nightly')
+    if: github.repository_owner == 'syncthing' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && startsWith(github.ref, 'refs/heads/release-nightly')
     environment: release
     needs:
       - sign-for-upgrade
@@ -742,7 +750,7 @@ jobs:
 
   publish-release-files:
     name: Publish release files
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v'))
+    if: github.repository_owner == 'syncthing' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v'))
     environment: release
     needs:
       - sign-for-upgrade
@@ -809,7 +817,7 @@ jobs:
 
   publish-apt:
     name: Publish APT
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
+    if: github.repository_owner == 'syncthing' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
     environment: release
     needs:
       - package-debian
@@ -836,7 +844,9 @@ jobs:
       - name: Prepare packages
         run: |
           kind=stable
-          if [[ $VERSION == *-rc.[0-9] ]] ; then
+          if [[ $VERSION == v2* ]] ; then
+            kind=v2
+          elif [[ $VERSION == *-rc.[0-9] ]] ; then
             kind=candidate
           elif [[ $VERSION == *-* ]] ; then
             kind=nightly
@@ -888,8 +898,10 @@ jobs:
   docker-syncthing:
     name: Build and push Docker images
     runs-on: ubuntu-latest
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release' || github.ref == 'refs/heads/infrastructure' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     environment: docker
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
     permissions:
       contents: read
       packages: write
@@ -902,13 +914,13 @@ jobs:
         include:
           - pkg: syncthing
             dockerfile: Dockerfile
-            image: syncthing/syncthing
+            image: syncthing
           - pkg: strelaysrv
             dockerfile: Dockerfile.strelaysrv
-            image: syncthing/relaysrv
+            image: relaysrv
           - pkg: stdiscosrv
             dockerfile: Dockerfile.stdiscosrv
-            image: syncthing/discosrv
+            image: discosrv
     steps:
       - uses: actions/checkout@v4
         with:
@@ -926,6 +938,8 @@ jobs:
           go version
           echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
 
+      - uses: mlugg/setup-zig@v1
+
       - uses: actions/cache@v4
         with:
           path: |
@@ -933,33 +947,34 @@ jobs:
             ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-docker-${{ matrix.pkg }}-${{ hashFiles('**/go.sum') }}
 
-      - name: Build binaries
+      - name: Build binaries (CGO)
         run: |
-          for arch in amd64 arm64 arm; do
-            go run build.go -goos linux -goarch "$arch" -no-upgrade build ${{ matrix.pkg }}
-            mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-"$arch"
-          done
-        env:
-          CGO_ENABLED: "0"
-          BUILD_USER: docker
+          # amd64
+          go run build.go -goos linux -goarch amd64 -tags "${{env.TAGS}}" -cc "zig cc -target x86_64-linux-musl" -no-upgrade build ${{ matrix.pkg }}
+          mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-amd64
 
-      - name: Check if we will be able to push images
-        run: |
-          if [[ "${{ secrets.DOCKERHUB_TOKEN }}" != "" ]]; then
-            echo "DOCKER_PUSH=true" >> $GITHUB_ENV;
-          fi
+          # arm64
+          go run build.go -goos linux -goarch arm64 -tags "${{env.TAGS}}" -cc "zig cc -target aarch64-linux-musl" -no-upgrade build ${{ matrix.pkg }}
+          mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-arm64
+
+          # arm
+          go run build.go -goos linux -goarch arm -tags "${{env.TAGS}}" -cc "zig cc -target arm-linux-musleabi" -no-upgrade build ${{ matrix.pkg }}
+          mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-arm
+        env:
+          CGO_ENABLED: "1"
+          BUILD_USER: docker
+          EXTRA_LDFLAGS: "-linkmode=external -extldflags=-static"
 
       - name: Login to Docker Hub
         uses: docker/login-action@v3
-        if: env.DOCKER_PUSH == 'true'
+        if: env.DOCKERHUB_USERNAME != ''
         with:
           registry: docker.io
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          username: ${{ env.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Login to GHCR
         uses: docker/login-action@v3
-        if: env.DOCKER_PUSH == 'true'
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -972,18 +987,31 @@ jobs:
         run: |
           version=$(go run build.go version)
           version=${version#v}
+          repo=ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}
+          ref="${{github.ref_name}}"
+          ref=${ref//\//-} # slashes to dashes
+
+          # List of tags for ghcr.io
           if [[ $version == @([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]) ]] ; then
-            echo Release version, pushing to :latest and version tags
             major=${version%.*.*}
             minor=${version%.*}
-            tags=docker.io/${{ matrix.image }}:$version,ghcr.io/${{ matrix.image }}:$version,docker.io/${{ matrix.image }}:$major,ghcr.io/${{ matrix.image }}:$major,docker.io/${{ matrix.image }}:$minor,ghcr.io/${{ matrix.image }}:$minor,docker.io/${{ matrix.image }}:latest,ghcr.io/${{ matrix.image }}:latest
+            tags=$repo:$version,$repo:$major,$repo:$minor,$repo:latest
           elif [[ $version == *-rc.@([0-9]|[0-9][0-9]) ]] ; then
-            echo Release candidate, pushing to :rc and version tags
-            tags=docker.io/${{ matrix.image }}:$version,ghcr.io/${{ matrix.image }}:$version,docker.io/${{ matrix.image }}:rc,ghcr.io/${{ matrix.image }}:rc
+            tags=$repo:$version,$repo:rc
+          elif [[ $ref == "main" ]] ; then
+            tags=$repo:edge
           else
-            echo Development version, pushing to :edge
-            tags=docker.io/${{ matrix.image }}:edge,ghcr.io/${{ matrix.image }}:edge
+            tags=$repo:$ref
           fi
+
+          # If we have a Docker Hub secret, also push to there.
+          if [[ $DOCKERHUB_USERNAME != "" ]] ; then
+            dockerhubtags="${tags//ghcr.io\/syncthing/docker.io\/syncthing}"
+            tags="$tags,$dockerhubtags"
+          fi
+
+          echo Pushing to $tags
+
           echo "DOCKER_TAGS=$tags" >> $GITHUB_ENV
           echo "VERSION=$version" >> $GITHUB_ENV
 
@@ -993,8 +1021,8 @@ jobs:
           context: .
           file: ${{ matrix.dockerfile }}
           platforms: linux/amd64,linux/arm64,linux/arm/7
-          push: ${{ env.DOCKER_PUSH == 'true' }}
           tags: ${{ env.DOCKER_TAGS }}
+          push: true
           labels: |
             org.opencontainers.image.version=${{ env.VERSION }}
             org.opencontainers.image.revision=${{ github.sha }}

.gitignore

@@ -17,5 +17,4 @@ deb
 *.bz2
 /repos
 /proto/scripts/protoc-gen-gosyncthing
-/gui/next-gen-gui
 /compat.json

.golangci.yml

@@ -3,6 +3,7 @@ linters:
   disable:
     - cyclop
     - depguard
+    - err113
     - exhaustive
     - exhaustruct
     - funlen
@@ -12,6 +13,7 @@ linters:
     - gocognit
     - goconst
     - gocyclo
+    - godot
     - godox
     - gofmt
     - goimports
@@ -21,15 +23,19 @@ linters:
     - ireturn
     - lll
     - maintidx
+    - musttag
     - nestif
+    - nlreturn
     - nonamedreturns
     - paralleltest
+    - prealloc
     - protogetter
     - scopelint
     - tagalign
     - tagliatelle
     - testpackage
     - varnamelen
+    - wrapcheck
     - wsl
 
 issues:

build.go

@@ -38,27 +38,26 @@ import (
 )
 
 var (
-	goarch         string
-	goos           string
-	noupgrade      bool
-	version        string
-	goCmd          string
-	race           bool
-	debug          = os.Getenv("BUILDDEBUG") != ""
-	extraTags      string
-	installSuffix  string
-	pkgdir         string
-	cc             string
-	run            string
-	benchRun       string
-	buildOut       string
-	debugBinary    bool
-	coverage       bool
-	long           bool
-	timeout        = "120s"
-	longTimeout    = "600s"
-	numVersions    = 5
-	withNextGenGUI = os.Getenv("BUILD_NEXT_GEN_GUI") != ""
+	goarch        string
+	goos          string
+	noupgrade     bool
+	version       string
+	goCmd         string
+	race          bool
+	debug         = os.Getenv("BUILDDEBUG") != ""
+	extraTags     string
+	installSuffix string
+	pkgdir        string
+	cc            string
+	run           string
+	benchRun      string
+	buildOut      string
+	debugBinary   bool
+	coverage      bool
+	long          bool
+	timeout       = "120s"
+	longTimeout   = "600s"
+	numVersions   = 5
 )
 
 type target struct {
@@ -289,10 +288,10 @@ func runCommand(cmd string, target target) {
 		build(target, tags)
 
 	case "test":
-		test(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
+		test(strings.Fields(extraTags), "github.com/syncthing/syncthing/internal/...", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
 
 	case "bench":
-		bench(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
+		bench(strings.Fields(extraTags), "github.com/syncthing/syncthing/internal/...", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
 
 	case "integration":
 		integration(false)
@@ -380,7 +379,6 @@ func parseFlags() {
 	flag.IntVar(&numVersions, "num-versions", numVersions, "Number of versions for changelog command")
 	flag.StringVar(&run, "run", "", "Specify which tests to run")
 	flag.StringVar(&benchRun, "bench", "", "Specify which benchmarks to run")
-	flag.BoolVar(&withNextGenGUI, "with-next-gen-gui", withNextGenGUI, "Also build 'newgui'")
 	flag.StringVar(&buildOut, "build-out", "", "Set the '-o' value for 'go build'")
 	flag.Parse()
 }
@@ -453,10 +451,6 @@ func benchArgs() []string {
 }
 
 func install(target target, tags []string) {
-	if (target.name == "syncthing" || target.name == "") && !withNextGenGUI {
-		log.Println("Notice: Next generation GUI will not be built; see --with-next-gen-gui.")
-	}
-
 	lazyRebuildAssets()
 
 	tags = append(target.tags, tags...)
@@ -480,16 +474,12 @@ func install(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	args := []string{"install", "-v"}
+	args := []string{"install"}
 	args = appendParameters(args, tags, target.buildPkgs...)
 	runPrint(goCmd, args...)
 }
 
 func build(target target, tags []string) {
-	if (target.name == "syncthing" || target.name == "") && !withNextGenGUI {
-		log.Println("Notice: Next generation GUI will not be built; see --with-next-gen-gui.")
-	}
-
 	lazyRebuildAssets()
 	tags = append(target.tags, tags...)
 
@@ -512,7 +502,7 @@ func build(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	args := []string{"build", "-v"}
+	args := []string{"build"}
 	if buildOut != "" {
 		args = append(args, "-o", buildOut)
 	}
@@ -524,13 +514,6 @@ func setBuildEnvVars() {
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
 	os.Setenv("CC", cc)
-	if os.Getenv("CGO_ENABLED") == "" {
-		switch goos {
-		case "darwin", "solaris":
-		default:
-			os.Setenv("CGO_ENABLED", "0")
-		}
-	}
 }
 
 func appendParameters(args []string, tags []string, pkgs ...string) []string {
@@ -749,12 +732,9 @@ func shouldBuildSyso(dir string) (string, error) {
 	sysoPath := filepath.Join(dir, "cmd", "syncthing", "resource.syso")
 
 	// See https://github.com/josephspurrier/goversioninfo#command-line-flags
-	armOption := ""
-	if strings.Contains(goarch, "arm") {
-		armOption = "-arm=true"
-	}
-
-	if _, err := runError("goversioninfo", "-o", sysoPath, armOption); err != nil {
+	arm := strings.HasPrefix(goarch, "arm")
+	a64 := strings.Contains(goarch, "64")
+	if _, err := runError("goversioninfo", "-o", sysoPath, fmt.Sprintf("-arm=%v", arm), fmt.Sprintf("-64=%v", a64)); err != nil {
 		return "", errors.New("failed to create " + sysoPath + ": " + err.Error())
 	}
 
@@ -826,43 +806,11 @@ func lazyRebuildAssets() {
 	shouldRebuild := shouldRebuildAssets("lib/api/auto/gui.files.go", "gui") ||
 		shouldRebuildAssets("cmd/infra/strelaypoolsrv/auto/gui.files.go", "cmd/infra/strelaypoolsrv/gui")
 
-	if withNextGenGUI {
-		shouldRebuild = buildNextGenGUI() || shouldRebuild
-	}
-
 	if shouldRebuild {
 		rebuildAssets()
 	}
 }
 
-func buildNextGenGUI() bool {
-	// Check if we need to run the npm process, and if so also set the flag
-	// to rebuild Go assets afterwards. The index.html is regenerated every
-	// time by the build process. This assumes the new GUI ends up in
-	// next-gen-gui/dist/next-gen-gui.
-
-	if !shouldRebuildAssets("gui/next-gen-gui/index.html", "next-gen-gui") {
-		// The GUI is up to date.
-		return false
-	}
-
-	runPrintInDir("next-gen-gui", "npm", "install")
-	runPrintInDir("next-gen-gui", "npm", "run", "build", "--", "--prod", "--subresource-integrity")
-
-	rmr("gui/tech-ui")
-
-	for _, src := range listFiles("next-gen-gui/dist") {
-		rel, _ := filepath.Rel("next-gen-gui/dist", src)
-		dst := filepath.Join("gui", rel)
-		if err := copyFile(src, dst, 0o644); err != nil {
-			fmt.Println("copy:", err)
-			os.Exit(1)
-		}
-	}
-
-	return true
-}
-
 func shouldRebuildAssets(target, srcdir string) bool {
 	info, err := os.Stat(target)
 	if err != nil {

cmd/dev/stfileinfo/main.go

@@ -72,7 +72,7 @@ func main() {
 		if *standardBlocks || blockSize < protocol.MinBlockSize {
 			blockSize = protocol.BlockSize(fi.Size())
 		}
-		bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
+		bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/syncthing/cli/debug.go

@@ -41,5 +41,4 @@ func (p *profileCommand) Run(ctx Context) error {
 type debugCommand struct {
 	File    fileCommand    `cmd:"" help:"Show information about a file (or directory/symlink)"`
 	Profile profileCommand `cmd:"" help:"Save a profile to help figuring out what Syncthing does"`
-	Index   indexCommand   `cmd:"" help:"Show information about the index (database)"`
 }

cmd/syncthing/cli/index.go

@@ -1,32 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cli
-
-import (
-	"github.com/alecthomas/kong"
-)
-
-type indexCommand struct {
-	Dump     struct{} `cmd:"" help:"Print the entire db"`
-	DumpSize struct{} `cmd:"" help:"Print the db size of different categories of information"`
-	Check    struct{} `cmd:"" help:"Check the database for inconsistencies"`
-	Account  struct{} `cmd:"" help:"Print key and value size statistics per key type"`
-}
-
-func (*indexCommand) Run(kongCtx *kong.Context) error {
-	switch kongCtx.Selected().Name {
-	case "dump":
-		return indexDump()
-	case "dump-size":
-		return indexDumpSize()
-	case "check":
-		return indexCheck()
-	case "account":
-		return indexAccount()
-	}
-	return nil
-}

cmd/syncthing/cli/index_accounting.go

@@ -1,62 +0,0 @@
-// Copyright (C) 2020 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cli
-
-import (
-	"fmt"
-	"os"
-	"text/tabwriter"
-)
-
-// indexAccount prints key and data size statistics per class
-func indexAccount() error {
-	ldb, err := getDB()
-	if err != nil {
-		return err
-	}
-
-	it, err := ldb.NewPrefixIterator(nil)
-	if err != nil {
-		return err
-	}
-
-	var ksizes [256]int
-	var dsizes [256]int
-	var counts [256]int
-	var max [256]int
-
-	for it.Next() {
-		key := it.Key()
-		t := key[0]
-		ds := len(it.Value())
-		ks := len(key)
-		s := ks + ds
-
-		counts[t]++
-		ksizes[t] += ks
-		dsizes[t] += ds
-		if s > max[t] {
-			max[t] = s
-		}
-	}
-
-	tw := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', tabwriter.AlignRight)
-	toti, totds, totks := 0, 0, 0
-	for t := range ksizes {
-		if ksizes[t] > 0 {
-			// yes metric kilobytes 🤘
-			fmt.Fprintf(tw, "0x%02x:\t%d items,\t%d KB keys +\t%d KB data,\t%d B +\t%d B avg,\t%d B max\t\n", t, counts[t], ksizes[t]/1000, dsizes[t]/1000, ksizes[t]/counts[t], dsizes[t]/counts[t], max[t])
-			toti += counts[t]
-			totds += dsizes[t]
-			totks += ksizes[t]
-		}
-	}
-	fmt.Fprintf(tw, "Total\t%d items,\t%d KB keys +\t%d KB data.\t\n", toti, totks/1000, totds/1000)
-	tw.Flush()
-
-	return nil
-}

cmd/syncthing/cli/index_dump.go

@@ -1,162 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cli
-
-import (
-	"encoding/binary"
-	"fmt"
-	"time"
-
-	"google.golang.org/protobuf/proto"
-
-	"github.com/syncthing/syncthing/internal/gen/bep"
-	"github.com/syncthing/syncthing/internal/gen/dbproto"
-	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-func indexDump() error {
-	ldb, err := getDB()
-	if err != nil {
-		return err
-	}
-	it, err := ldb.NewPrefixIterator(nil)
-	if err != nil {
-		return err
-	}
-	for it.Next() {
-		key := it.Key()
-		switch key[0] {
-		case db.KeyTypeDevice:
-			folder := binary.BigEndian.Uint32(key[1:])
-			device := binary.BigEndian.Uint32(key[1+4:])
-			name := nulString(key[1+4+4:])
-			fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
-
-			var f bep.FileInfo
-			err := proto.Unmarshal(it.Value(), &f)
-			if err != nil {
-				return err
-			}
-			fmt.Printf(" V:%v\n", &f)
-
-		case db.KeyTypeGlobal:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			var flv dbproto.VersionList
-			proto.Unmarshal(it.Value(), &flv)
-			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, &flv)
-
-		case db.KeyTypeBlock:
-			folder := binary.BigEndian.Uint32(key[1:])
-			hash := key[1+4 : 1+4+32]
-			name := nulString(key[1+4+32:])
-			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
-
-		case db.KeyTypeDeviceStatistic:
-			fmt.Printf("[dstat] K:%x V:%x\n", key, it.Value())
-
-		case db.KeyTypeFolderStatistic:
-			fmt.Printf("[fstat] K:%x V:%x\n", key, it.Value())
-
-		case db.KeyTypeVirtualMtime:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			val := it.Value()
-			var realTime, virtualTime time.Time
-			realTime.UnmarshalBinary(val[:len(val)/2])
-			virtualTime.UnmarshalBinary(val[len(val)/2:])
-			fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, realTime, virtualTime)
-
-		case db.KeyTypeFolderIdx:
-			key := binary.BigEndian.Uint32(key[1:])
-			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
-
-		case db.KeyTypeDeviceIdx:
-			key := binary.BigEndian.Uint32(key[1:])
-			val := it.Value()
-			device := "<nil>"
-			if len(val) > 0 {
-				dev, err := protocol.DeviceIDFromBytes(val)
-				if err != nil {
-					device = fmt.Sprintf("<invalid %d bytes>", len(val))
-				} else {
-					device = dev.String()
-				}
-			}
-			fmt.Printf("[deviceidx] K:%d V:%s\n", key, device)
-
-		case db.KeyTypeIndexID:
-			device := binary.BigEndian.Uint32(key[1:])
-			folder := binary.BigEndian.Uint32(key[5:])
-			fmt.Printf("[indexid] D:%d F:%d I:%x\n", device, folder, it.Value())
-
-		case db.KeyTypeFolderMeta:
-			folder := binary.BigEndian.Uint32(key[1:])
-			fmt.Printf("[foldermeta] F:%d", folder)
-			var cs dbproto.CountsSet
-			if err := proto.Unmarshal(it.Value(), &cs); err != nil {
-				fmt.Printf(" (invalid)\n")
-			} else {
-				fmt.Printf(" V:%v\n", &cs)
-			}
-
-		case db.KeyTypeMiscData:
-			fmt.Printf("[miscdata] K:%q V:%q\n", key[1:], it.Value())
-
-		case db.KeyTypeSequence:
-			folder := binary.BigEndian.Uint32(key[1:])
-			seq := binary.BigEndian.Uint64(key[5:])
-			fmt.Printf("[sequence] F:%d S:%d V:%q\n", folder, seq, it.Value())
-
-		case db.KeyTypeNeed:
-			folder := binary.BigEndian.Uint32(key[1:])
-			file := string(key[5:])
-			fmt.Printf("[need] F:%d V:%q\n", folder, file)
-
-		case db.KeyTypeBlockList:
-			fmt.Printf("[blocklist] H:%x\n", key[1:])
-
-		case db.KeyTypeBlockListMap:
-			folder := binary.BigEndian.Uint32(key[1:])
-			hash := key[5:37]
-			fileName := string(key[37:])
-			fmt.Printf("[blocklistmap] F:%d H:%x N:%s\n", folder, hash, fileName)
-
-		case db.KeyTypeVersion:
-			fmt.Printf("[version] H:%x", key[1:])
-			var v bep.Vector
-			err := proto.Unmarshal(it.Value(), &v)
-			if err != nil {
-				fmt.Printf(" (invalid)\n")
-			} else {
-				fmt.Printf(" V:%v\n", &v)
-			}
-
-		case db.KeyTypePendingFolder:
-			device := binary.BigEndian.Uint32(key[1:])
-			folder := string(key[5:])
-			var of dbproto.ObservedFolder
-			proto.Unmarshal(it.Value(), &of)
-			fmt.Printf("[pendingFolder] D:%d F:%s V:%v\n", device, folder, &of)
-
-		case db.KeyTypePendingDevice:
-			device := "<invalid>"
-			dev, err := protocol.DeviceIDFromBytes(key[1:])
-			if err == nil {
-				device = dev.String()
-			}
-			var od dbproto.ObservedDevice
-			proto.Unmarshal(it.Value(), &od)
-			fmt.Printf("[pendingDevice] D:%v V:%v\n", device, &od)
-
-		default:
-			fmt.Printf("[??? %d]\n  %x\n  %x\n", key[0], key, it.Value())
-		}
-	}
-	return nil
-}

cmd/syncthing/cli/index_dumpsize.go

@@ -1,88 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cli
-
-import (
-	"encoding/binary"
-	"fmt"
-	"sort"
-
-	"github.com/syncthing/syncthing/lib/db"
-)
-
-func indexDumpSize() error {
-	type sizedElement struct {
-		key  string
-		size int
-	}
-
-	ldb, err := getDB()
-	if err != nil {
-		return err
-	}
-
-	it, err := ldb.NewPrefixIterator(nil)
-	if err != nil {
-		return err
-	}
-
-	var elems []sizedElement
-	for it.Next() {
-		var ele sizedElement
-
-		key := it.Key()
-		switch key[0] {
-		case db.KeyTypeDevice:
-			folder := binary.BigEndian.Uint32(key[1:])
-			device := binary.BigEndian.Uint32(key[1+4:])
-			name := nulString(key[1+4+4:])
-			ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)
-
-		case db.KeyTypeGlobal:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)
-
-		case db.KeyTypeBlock:
-			folder := binary.BigEndian.Uint32(key[1:])
-			hash := key[1+4 : 1+4+32]
-			name := nulString(key[1+4+32:])
-			ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)
-
-		case db.KeyTypeDeviceStatistic:
-			ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])
-
-		case db.KeyTypeFolderStatistic:
-			ele.key = fmt.Sprintf("FOLDERSTATS:%s", key[1:])
-
-		case db.KeyTypeVirtualMtime:
-			ele.key = fmt.Sprintf("MTIME:%s", key[1:])
-
-		case db.KeyTypeFolderIdx:
-			id := binary.BigEndian.Uint32(key[1:])
-			ele.key = fmt.Sprintf("FOLDERIDX:%d", id)
-
-		case db.KeyTypeDeviceIdx:
-			id := binary.BigEndian.Uint32(key[1:])
-			ele.key = fmt.Sprintf("DEVICEIDX:%d", id)
-
-		default:
-			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
-		}
-		ele.size = len(it.Value())
-		elems = append(elems, ele)
-	}
-
-	sort.Slice(elems, func(i, j int) bool {
-		return elems[i].size > elems[j].size
-	})
-	for _, ele := range elems {
-		fmt.Println(ele.key, ele.size)
-	}
-
-	return nil
-}

cmd/syncthing/cli/index_idxck.go

@@ -1,434 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cli
-
-import (
-	"bytes"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"sort"
-
-	"google.golang.org/protobuf/proto"
-
-	"github.com/syncthing/syncthing/internal/gen/bep"
-	"github.com/syncthing/syncthing/internal/gen/dbproto"
-	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-type fileInfoKey struct {
-	folder uint32
-	device uint32
-	name   string
-}
-
-type globalKey struct {
-	folder uint32
-	name   string
-}
-
-type sequenceKey struct {
-	folder   uint32
-	sequence uint64
-}
-
-func indexCheck() (err error) {
-	ldb, err := getDB()
-	if err != nil {
-		return err
-	}
-
-	folders := make(map[uint32]string)
-	devices := make(map[uint32]string)
-	deviceToIDs := make(map[string]uint32)
-	fileInfos := make(map[fileInfoKey]*bep.FileInfo)
-	globals := make(map[globalKey]*dbproto.VersionList)
-	sequences := make(map[sequenceKey]string)
-	needs := make(map[globalKey]struct{})
-	blocklists := make(map[string]struct{})
-	versions := make(map[string]*bep.Vector)
-	usedBlocklists := make(map[string]struct{})
-	usedVersions := make(map[string]struct{})
-	var localDeviceKey uint32
-	success := true
-	defer func() {
-		if err == nil {
-			if success {
-				fmt.Println("Index check completed successfully.")
-			} else {
-				err = errors.New("Inconsistencies found in the index")
-			}
-		}
-	}()
-
-	it, err := ldb.NewPrefixIterator(nil)
-	if err != nil {
-		return err
-	}
-	for it.Next() {
-		key := it.Key()
-		switch key[0] {
-		case db.KeyTypeDevice:
-			folder := binary.BigEndian.Uint32(key[1:])
-			device := binary.BigEndian.Uint32(key[1+4:])
-			name := nulString(key[1+4+4:])
-
-			var f bep.FileInfo
-			err := proto.Unmarshal(it.Value(), &f)
-			if err != nil {
-				fmt.Println("Unable to unmarshal FileInfo:", err)
-				success = false
-				continue
-			}
-
-			fileInfos[fileInfoKey{folder, device, name}] = &f
-
-		case db.KeyTypeGlobal:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			var flv dbproto.VersionList
-			if err := proto.Unmarshal(it.Value(), &flv); err != nil {
-				fmt.Println("Unable to unmarshal VersionList:", err)
-				success = false
-				continue
-			}
-			globals[globalKey{folder, name}] = &flv
-
-		case db.KeyTypeFolderIdx:
-			key := binary.BigEndian.Uint32(it.Key()[1:])
-			folders[key] = string(it.Value())
-
-		case db.KeyTypeDeviceIdx:
-			key := binary.BigEndian.Uint32(it.Key()[1:])
-			devices[key] = string(it.Value())
-			deviceToIDs[string(it.Value())] = key
-			if bytes.Equal(it.Value(), protocol.LocalDeviceID[:]) {
-				localDeviceKey = key
-			}
-
-		case db.KeyTypeSequence:
-			folder := binary.BigEndian.Uint32(key[1:])
-			seq := binary.BigEndian.Uint64(key[5:])
-			val := it.Value()
-			sequences[sequenceKey{folder, seq}] = string(val[9:])
-
-		case db.KeyTypeNeed:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			needs[globalKey{folder, name}] = struct{}{}
-
-		case db.KeyTypeBlockList:
-			hash := string(key[1:])
-			blocklists[hash] = struct{}{}
-
-		case db.KeyTypeVersion:
-			hash := string(key[1:])
-			var v bep.Vector
-			if err := proto.Unmarshal(it.Value(), &v); err != nil {
-				fmt.Println("Unable to unmarshal Vector:", err)
-				success = false
-				continue
-			}
-			versions[hash] = &v
-		}
-	}
-
-	if localDeviceKey == 0 {
-		fmt.Println("Missing key for local device in device index (bailing out)")
-		success = false
-		return
-	}
-
-	var missingSeq []sequenceKey
-	for fk, fi := range fileInfos {
-		if fk.name != fi.Name {
-			fmt.Printf("Mismatching FileInfo name, %q (key) != %q (actual)\n", fk.name, fi.Name)
-			success = false
-		}
-
-		folder := folders[fk.folder]
-		if folder == "" {
-			fmt.Printf("Unknown folder ID %d for FileInfo %q\n", fk.folder, fk.name)
-			success = false
-			continue
-		}
-		if devices[fk.device] == "" {
-			fmt.Printf("Unknown device ID %d for FileInfo %q, folder %q\n", fk.folder, fk.name, folder)
-			success = false
-		}
-
-		if fk.device == localDeviceKey {
-			sk := sequenceKey{fk.folder, uint64(fi.Sequence)}
-			name, ok := sequences[sk]
-			if !ok {
-				fmt.Printf("Sequence entry missing for FileInfo %q, folder %q, seq %d\n", fi.Name, folder, fi.Sequence)
-				missingSeq = append(missingSeq, sk)
-				success = false
-				continue
-			}
-			if name != fi.Name {
-				fmt.Printf("Sequence entry refers to wrong name, %q (seq) != %q (FileInfo), folder %q, seq %d\n", name, fi.Name, folder, fi.Sequence)
-				success = false
-			}
-		}
-
-		if len(fi.Blocks) == 0 && len(fi.BlocksHash) != 0 {
-			key := string(fi.BlocksHash)
-			if _, ok := blocklists[key]; !ok {
-				fmt.Printf("Missing block list for file %q, block list hash %x\n", fi.Name, fi.BlocksHash)
-				success = false
-			} else {
-				usedBlocklists[key] = struct{}{}
-			}
-		}
-
-		if fi.VersionHash != nil {
-			key := string(fi.VersionHash)
-			if _, ok := versions[key]; !ok {
-				fmt.Printf("Missing version vector for file %q, version hash %x\n", fi.Name, fi.VersionHash)
-				success = false
-			} else {
-				usedVersions[key] = struct{}{}
-			}
-		}
-
-		_, ok := globals[globalKey{fk.folder, fk.name}]
-		if !ok {
-			fmt.Printf("Missing global for file %q\n", fi.Name)
-			success = false
-			continue
-		}
-	}
-
-	// Aggregate the ranges of missing sequence entries, print them
-
-	sort.Slice(missingSeq, func(a, b int) bool {
-		if missingSeq[a].folder != missingSeq[b].folder {
-			return missingSeq[a].folder < missingSeq[b].folder
-		}
-		return missingSeq[a].sequence < missingSeq[b].sequence
-	})
-
-	var folder uint32
-	var startSeq, prevSeq uint64
-	for _, sk := range missingSeq {
-		if folder != sk.folder || sk.sequence != prevSeq+1 {
-			if folder != 0 {
-				fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
-			}
-			startSeq = sk.sequence
-			folder = sk.folder
-		}
-		prevSeq = sk.sequence
-	}
-	if folder != 0 {
-		fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
-	}
-
-	for gk, vl := range globals {
-		folder := folders[gk.folder]
-		if folder == "" {
-			fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
-			success = false
-		}
-		checkGlobal := func(i int, device []byte, version protocol.Vector, invalid, deleted bool) {
-			dev, ok := deviceToIDs[string(device)]
-			if !ok {
-				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, device)
-				success = false
-			}
-			fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
-			if !ok {
-				fmt.Printf("VersionList %q, folder %q, entry %d refers to unknown FileInfo\n", gk.name, folder, i)
-				success = false
-			}
-
-			fiv := fi.Version
-			if fi.VersionHash != nil {
-				fiv = versions[string(fi.VersionHash)]
-			}
-			if !protocol.VectorFromWire(fiv).Equal(version) {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, version, fi.Version)
-				success = false
-			}
-			ffi := protocol.FileInfoFromDB(fi)
-			if ffi.IsInvalid() != invalid {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, ffi.IsInvalid())
-				success = false
-			}
-			if ffi.IsDeleted() != deleted {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, ffi.IsDeleted())
-				success = false
-			}
-		}
-		for i, fv := range vl.Versions {
-			ver := protocol.VectorFromWire(fv.Version)
-			for _, device := range fv.Devices {
-				checkGlobal(i, device, ver, false, fv.Deleted)
-			}
-			for _, device := range fv.InvalidDevices {
-				checkGlobal(i, device, ver, true, fv.Deleted)
-			}
-		}
-
-		// If we need this file we should have a need entry for it. False
-		// positives from needsLocally for deleted files, where we might
-		// legitimately lack an entry if we never had it, and ignored files.
-		if needsLocally(vl) {
-			_, ok := needs[gk]
-			if !ok {
-				fv, _ := vlGetGlobal(vl)
-				devB, _ := fvFirstDevice(fv)
-				dev := deviceToIDs[string(devB)]
-				fi := protocol.FileInfoFromDB(fileInfos[fileInfoKey{gk.folder, dev, gk.name}])
-				if !fi.IsDeleted() && !fi.IsIgnored() {
-					fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
-				}
-			}
-		}
-	}
-
-	seenSeq := make(map[fileInfoKey]uint64)
-	for sk, name := range sequences {
-		folder := folders[sk.folder]
-		if folder == "" {
-			fmt.Printf("Unknown folder ID %d for sequence entry %d, %q\n", sk.folder, sk.sequence, name)
-			success = false
-			continue
-		}
-
-		if prev, ok := seenSeq[fileInfoKey{folder: sk.folder, name: name}]; ok {
-			fmt.Printf("Duplicate sequence entry for %q, folder %q, seq %d (prev %d)\n", name, folder, sk.sequence, prev)
-			success = false
-		}
-		seenSeq[fileInfoKey{folder: sk.folder, name: name}] = sk.sequence
-
-		fi, ok := fileInfos[fileInfoKey{sk.folder, localDeviceKey, name}]
-		if !ok {
-			fmt.Printf("Missing FileInfo for sequence entry %d, folder %q, %q\n", sk.sequence, folder, name)
-			success = false
-			continue
-		}
-		if fi.Sequence != int64(sk.sequence) {
-			fmt.Printf("Sequence mismatch for %q, folder %q, %d (key) != %d (FileInfo)\n", name, folder, sk.sequence, fi.Sequence)
-			success = false
-		}
-	}
-
-	for nk := range needs {
-		folder := folders[nk.folder]
-		if folder == "" {
-			fmt.Printf("Unknown folder ID %d for need entry %q\n", nk.folder, nk.name)
-			success = false
-			continue
-		}
-
-		vl, ok := globals[nk]
-		if !ok {
-			fmt.Printf("Missing global for need entry %q, folder %q\n", nk.name, folder)
-			success = false
-			continue
-		}
-
-		if !needsLocally(vl) {
-			fmt.Printf("Need entry for file we don't need, %q, folder %q\n", nk.name, folder)
-			success = false
-		}
-	}
-
-	if d := len(blocklists) - len(usedBlocklists); d > 0 {
-		fmt.Printf("%d block list entries out of %d needs GC\n", d, len(blocklists))
-	}
-	if d := len(versions) - len(usedVersions); d > 0 {
-		fmt.Printf("%d version entries out of %d needs GC\n", d, len(versions))
-	}
-
-	return nil
-}
-
-func needsLocally(vl *dbproto.VersionList) bool {
-	gfv, gok := vlGetGlobal(vl)
-	if !gok { // That's weird, but we hardly need something non-existent
-		return false
-	}
-	fv, ok := vlGet(vl, protocol.LocalDeviceID[:])
-	return db.Need(gfv, ok, protocol.VectorFromWire(fv.Version))
-}
-
-// Get returns a FileVersion that contains the given device and whether it has
-// been found at all.
-func vlGet(vl *dbproto.VersionList, device []byte) (*dbproto.FileVersion, bool) {
-	_, i, _, ok := vlFindDevice(vl, device)
-	if !ok {
-		return &dbproto.FileVersion{}, false
-	}
-	return vl.Versions[i], true
-}
-
-// GetGlobal returns the current global FileVersion. The returned FileVersion
-// may be invalid, if all FileVersions are invalid. Returns false only if
-// VersionList is empty.
-func vlGetGlobal(vl *dbproto.VersionList) (*dbproto.FileVersion, bool) {
-	i := vlFindGlobal(vl)
-	if i == -1 {
-		return nil, false
-	}
-	return vl.Versions[i], true
-}
-
-// findGlobal returns the first version that isn't invalid, or if all versions are
-// invalid just the first version (i.e. 0) or -1, if there's no versions at all.
-func vlFindGlobal(vl *dbproto.VersionList) int {
-	for i := range vl.Versions {
-		if !fvIsInvalid(vl.Versions[i]) {
-			return i
-		}
-	}
-	if len(vl.Versions) == 0 {
-		return -1
-	}
-	return 0
-}
-
-// findDevice returns whether the device is in InvalidVersions or Versions and
-// in InvalidDevices or Devices (true for invalid), the positions in the version
-// and device slices and whether it has been found at all.
-func vlFindDevice(vl *dbproto.VersionList, device []byte) (bool, int, int, bool) {
-	for i, v := range vl.Versions {
-		if j := deviceIndex(v.Devices, device); j != -1 {
-			return false, i, j, true
-		}
-		if j := deviceIndex(v.InvalidDevices, device); j != -1 {
-			return true, i, j, true
-		}
-	}
-	return false, -1, -1, false
-}
-
-func deviceIndex(devices [][]byte, device []byte) int {
-	for i, dev := range devices {
-		if bytes.Equal(device, dev) {
-			return i
-		}
-	}
-	return -1
-}
-
-func fvFirstDevice(fv *dbproto.FileVersion) ([]byte, bool) {
-	if len(fv.Devices) != 0 {
-		return fv.Devices[0], true
-	}
-	if len(fv.InvalidDevices) != 0 {
-		return fv.InvalidDevices[0], true
-	}
-	return nil, false
-}
-
-func fvIsInvalid(fv *dbproto.FileVersion) bool {
-	return fv == nil || len(fv.Devices) == 0
-}

cmd/syncthing/cli/main.go

@@ -14,15 +14,12 @@ import (
 	"github.com/alecthomas/kong"
 	"github.com/kballard/go-shellquote"
 
-	"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
 	"github.com/syncthing/syncthing/lib/config"
 )
 
 type CLI struct {
-	cmdutil.CommonOptions
-	DataDir    string `name:"data" placeholder:"PATH" env:"STDATADIR" help:"Set data directory (database and logs)"`
-	GUIAddress string `name:"gui-address"`
-	GUIAPIKey  string `name:"gui-apikey"`
+	GUIAddress string `name:"gui-address" env:"STGUIADDRESS"`
+	GUIAPIKey  string `name:"gui-apikey" env:"STGUIAPIKEY"`
 
 	Show       showCommand      `cmd:"" help:"Show command group"`
 	Debug      debugCommand     `cmd:"" help:"Debug command group"`
@@ -37,11 +34,6 @@ type Context struct {
 }
 
 func (cli CLI) AfterApply(kongCtx *kong.Context) error {
-	err := cmdutil.SetConfigDataLocationsFromFlags(cli.HomeDir, cli.ConfDir, cli.DataDir)
-	if err != nil {
-		return fmt.Errorf("command line options: %w", err)
-	}
-
 	clientFactory := &apiClientFactory{
 		cfg: config.GUIConfiguration{
 			RawAddress: cli.GUIAddress,

cmd/syncthing/cli/utils.go

@@ -17,8 +17,6 @@ import (
 	"path/filepath"
 
 	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/db/backend"
-	"github.com/syncthing/syncthing/lib/locations"
 )
 
 func responseToBArray(response *http.Response) ([]byte, error) {
@@ -133,10 +131,6 @@ func prettyPrintResponse(response *http.Response) error {
 	return prettyPrintJSON(data)
 }
 
-func getDB() (backend.Backend, error) {
-	return backend.OpenLevelDBRO(locations.Get(locations.Database))
-}
-
 func nulString(bs []byte) string {
 	for i := range bs {
 		if bs[i] == 0 {

cmd/syncthing/cmdutil/options_common.go

@@ -1,16 +0,0 @@
-// Copyright (C) 2021 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cmdutil
-
-// CommonOptions are reused among several subcommands
-type CommonOptions struct {
-	buildCommonOptions
-	ConfDir         string `name:"config" placeholder:"PATH" env:"STCONFDIR" help:"Set configuration directory (config and keys)"`
-	HomeDir         string `name:"home" placeholder:"PATH" env:"STHOMEDIR" help:"Set configuration and data directory"`
-	NoDefaultFolder bool   `env:"STNODEFAULTFOLDER" help:"Don't create the \"default\" folder on first startup"`
-	SkipPortProbing bool   `help:"Don't try to find free ports for GUI and listen addresses on first startup"`
-}

cmd/syncthing/cmdutil/util.go

@@ -1,35 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cmdutil
-
-import (
-	"errors"
-
-	"github.com/syncthing/syncthing/lib/locations"
-)
-
-func SetConfigDataLocationsFromFlags(homeDir, confDir, dataDir string) error {
-	homeSet := homeDir != ""
-	confSet := confDir != ""
-	dataSet := dataDir != ""
-	switch {
-	case dataSet != confSet:
-		return errors.New("either both or none of --config and --data must be given, use --home to set both at once")
-	case homeSet && dataSet:
-		return errors.New("--home must not be used together with --config and --data")
-	case homeSet:
-		confDir = homeDir
-		dataDir = homeDir
-		fallthrough
-	case dataSet:
-		if err := locations.SetBaseDir(locations.ConfigBaseDir, confDir); err != nil {
-			return err
-		}
-		return locations.SetBaseDir(locations.DataBaseDir, dataDir)
-	}
-	return nil
-}

cmd/syncthing/decrypt/decrypt.go

@@ -238,7 +238,7 @@ func (c *CLI) decryptFile(encFi *protocol.FileInfo, plainFi *protocol.FileInfo,
 		}
 
 		// Verify the hash against the plaintext block info
-		if !scanner.Validate(dec, plainBlock.Hash, 0) {
+		if !scanner.Validate(dec, plainBlock.Hash) {
 			// The block decrypted correctly but fails the hash check. This
 			// is odd and unexpected, but it it's still a valid block from
 			// the source. The file might have changed while we pulled it?

cmd/syncthing/generate/generate.go

@@ -11,42 +11,26 @@ import (
 	"bufio"
 	"context"
 	"crypto/tls"
-	"errors"
 	"fmt"
 	"os"
 
-	"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/logger"
-	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/syncthing"
 )
 
 type CLI struct {
-	cmdutil.CommonOptions
-	GUIUser     string `placeholder:"STRING" help:"Specify new GUI authentication user name"`
-	GUIPassword string `placeholder:"STRING" help:"Specify new GUI authentication password (use - to read from standard input)"`
+	GUIUser         string `placeholder:"STRING" help:"Specify new GUI authentication user name"`
+	GUIPassword     string `placeholder:"STRING" help:"Specify new GUI authentication password (use - to read from standard input)"`
+	NoDefaultFolder bool   `help:"Don't create the \"default\" folder on first startup" env:"STNODEFAULTFOLDER"`
+	NoPortProbing   bool   `help:"Don't try to find free ports for GUI and listen addresses on first startup" env:"STNOPORTPROBING"`
 }
 
 func (c *CLI) Run(l logger.Logger) error {
-	if c.HideConsole {
-		osutil.HideConsole()
-	}
-
-	if c.HomeDir != "" {
-		if c.ConfDir != "" {
-			return errors.New("--home must not be used together with --config")
-		}
-		c.ConfDir = c.HomeDir
-	}
-	if c.ConfDir == "" {
-		c.ConfDir = locations.GetBaseDir(locations.ConfigBaseDir)
-	}
-
 	// Support reading the password from a pipe or similar
 	if c.GUIPassword == "-" {
 		reader := bufio.NewReader(os.Stdin)
@@ -57,7 +41,7 @@ func (c *CLI) Run(l logger.Logger) error {
 		c.GUIPassword = string(password)
 	}
 
-	if err := Generate(l, c.ConfDir, c.GUIUser, c.GUIPassword, c.NoDefaultFolder, c.SkipPortProbing); err != nil {
+	if err := Generate(l, locations.GetBaseDir(locations.ConfigBaseDir), c.GUIUser, c.GUIPassword, c.NoDefaultFolder, c.NoPortProbing); err != nil {
 		return fmt.Errorf("failed to generate config and keys: %w", err)
 	}
 	return nil

cmd/syncthing/hideconsole_others.go

@@ -7,8 +7,8 @@
 //go:build !windows
 // +build !windows
 
-package cmdutil
+package main
 
-type buildCommonOptions struct {
+type buildSpecificOptions struct {
 	HideConsole bool `hidden:""`
 }

cmd/syncthing/hideconsole_windows.go

@@ -4,8 +4,8 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package cmdutil
+package main
 
-type buildCommonOptions struct {
-	HideConsole bool `name:"no-console" help:"Hide console window"`
+type buildSpecificOptions struct {
+	HideConsole bool `name:"no-console" help:"Hide console window" env:"STHIDECONSOLE"`
 }

cmd/syncthing/main.go

@@ -22,10 +22,10 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"runtime/pprof"
 	"sort"
 	"strconv"
-	"strings"
 	"syscall"
 	"time"
 
@@ -35,13 +35,12 @@ import (
 	"github.com/willabides/kongplete"
 
 	"github.com/syncthing/syncthing/cmd/syncthing/cli"
-	"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
 	"github.com/syncthing/syncthing/cmd/syncthing/decrypt"
 	"github.com/syncthing/syncthing/cmd/syncthing/generate"
+	"github.com/syncthing/syncthing/internal/db"
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/dialer"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
@@ -128,53 +127,68 @@ var (
 // The entrypoint struct is the main entry point for the command line parser. The
 // commands and options here are top level commands to syncthing.
 // Cli is just a placeholder for the help text (see main).
-var entrypoint struct {
-	Serve              serveOptions                 `cmd:"" help:"Run Syncthing"`
-	Generate           generate.CLI                 `cmd:"" help:"Generate key and config, then exit"`
-	Decrypt            decrypt.CLI                  `cmd:"" help:"Decrypt or verify an encrypted folder"`
-	Cli                cli.CLI                      `cmd:"" help:"Command line interface for Syncthing"`
+type CLI struct {
+	// The directory options are defined at top level and available for all
+	// subcommands. Their settings take effect on the `locations` package by
+	// way of the command line parser, so anything using `locations.Get` etc
+	// will be doing the right thing.
+	ConfDir string `name:"config" short:"C" placeholder:"PATH" env:"STCONFDIR" help:"Set configuration directory (config and keys)"`
+	DataDir string `name:"data" short:"D" placeholder:"PATH" env:"STDATADIR" help:"Set data directory (database and logs)"`
+	HomeDir string `name:"home" short:"H" placeholder:"PATH" env:"STHOMEDIR" help:"Set configuration and data directory"`
+
+	Serve serveCmd `cmd:"" help:"Run Syncthing (default)" default:"withargs"`
+	CLI   cli.CLI  `cmd:"" help:"Command line interface for Syncthing"`
+
+	Browser  browserCmd   `cmd:"" help:"Open GUI in browser, then exit"`
+	Decrypt  decrypt.CLI  `cmd:"" help:"Decrypt or verify an encrypted folder"`
+	DeviceID deviceIDCmd  `cmd:"" help:"Show device ID, then exit"`
+	Generate generate.CLI `cmd:"" help:"Generate key and config, then exit"`
+	Paths    pathsCmd     `cmd:"" help:"Show configuration paths, then exit"`
+	Upgrade  upgradeCmd   `cmd:"" help:"Perform or check for upgrade, then exit"`
+	Version  versionCmd   `cmd:"" help:"Show current version, then exit"`
+	Debug    debugCmd     `cmd:"" help:"Various debugging commands"`
+
 	InstallCompletions kongplete.InstallCompletions `cmd:"" help:"Print commands to install shell completions"`
 }
 
-// serveOptions are the options for the `syncthing serve` command.
-type serveOptions struct {
-	cmdutil.CommonOptions
-	AllowNewerConfig bool   `help:"Allow loading newer than current config version"`
-	Audit            bool   `help:"Write events to audit file"`
-	AuditFile        string `name:"auditfile" placeholder:"PATH" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)"`
-	BrowserOnly      bool   `help:"Open GUI in browser"`
-	DataDir          string `name:"data" placeholder:"PATH" env:"STDATADIR" help:"Set data directory (database and logs)"`
-	DeviceID         bool   `help:"Show the device ID"`
-	GenerateDir      string `name:"generate" placeholder:"PATH" help:"Generate key and config in specified dir, then exit"` // DEPRECATED: replaced by subcommand!
-	GUIAddress       string `name:"gui-address" placeholder:"URL" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")"`
-	GUIAPIKey        string `name:"gui-apikey" placeholder:"API-KEY" help:"Override GUI API key"`
-	LogFile          string `name:"logfile" default:"${logFile}" placeholder:"PATH" help:"Log file name (see below)"`
-	LogFlags         int    `name:"logflags" default:"${logFlags}" placeholder:"BITS" help:"Select information in log line prefix (see below)"`
-	LogMaxFiles      int    `placeholder:"N" default:"${logMaxFiles}" name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)"`
-	LogMaxSize       int    `placeholder:"BYTES" default:"${logMaxSize}" help:"Maximum size of any file (zero to disable log rotation)"`
-	NoBrowser        bool   `help:"Do not start browser"`
-	NoRestart        bool   `env:"STNORESTART" help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash"`
-	NoUpgrade        bool   `env:"STNOUPGRADE" help:"Disable automatic upgrades"`
-	Paths            bool   `help:"Show configuration paths"`
-	Paused           bool   `help:"Start with all devices and folders paused"`
-	Unpaused         bool   `help:"Start with all devices and folders unpaused"`
-	Upgrade          bool   `help:"Perform upgrade"`
-	UpgradeCheck     bool   `help:"Check for available upgrade"`
-	UpgradeTo        string `placeholder:"URL" help:"Force upgrade directly from specified URL"`
-	Verbose          bool   `help:"Print verbose log output"`
-	Version          bool   `help:"Show version"`
+func (c *CLI) AfterApply() error {
+	// Executed after parsing command line options but before running actual
+	// subcommands
+	return setConfigDataLocationsFromFlags(c.HomeDir, c.ConfDir, c.DataDir)
+}
+
+// serveCmd are the options for the `syncthing serve` command.
+type serveCmd struct {
+	buildSpecificOptions
+
+	AllowNewerConfig          bool          `help:"Allow loading newer than current config version" env:"STALLOWNEWERCONFIG"`
+	Audit                     bool          `help:"Write events to audit file" env:"STAUDIT"`
+	AuditFile                 string        `name:"auditfile" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)" placeholder:"PATH" env:"STAUDITFILE"`
+	DBMaintenanceInterval     time.Duration `help:"Database maintenance interval" default:"8h" env:"STDBMAINTENANCEINTERVAL"`
+	DBDeleteRetentionInterval time.Duration `help:"Database deleted item retention interval" default:"4320h" env:"STDBDELETERETENTIONINTERVAL"`
+	GUIAddress                string        `name:"gui-address" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")" placeholder:"URL" env:"STGUIADDRESS"`
+	GUIAPIKey                 string        `name:"gui-apikey" help:"Override GUI API key" placeholder:"API-KEY" env:"STGUIAPIKEY"`
+	LogFile                   string        `name:"logfile" help:"Log file name (see below)" default:"${logFile}" placeholder:"PATH" env:"STLOGFILE"`
+	LogFlags                  int           `name:"logflags" help:"Select information in log line prefix (see below)" default:"${logFlags}" placeholder:"BITS" env:"STLOGFLAGS"`
+	LogMaxFiles               int           `name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)" default:"${logMaxFiles}" placeholder:"N" env:"STLOGMAXOLDFILES"`
+	LogMaxSize                int           `help:"Maximum size of any file (zero to disable log rotation)" default:"${logMaxSize}" placeholder:"BYTES" env:"STLOGMAXSIZE"`
+	NoBrowser                 bool          `help:"Do not start browser" env:"STNOBROWSER"`
+	NoDefaultFolder           bool          `help:"Don't create the \"default\" folder on first startup" env:"STNODEFAULTFOLDER"`
+	NoPortProbing             bool          `help:"Don't try to find free ports for GUI and listen addresses on first startup" env:"STNOPORTPROBING"`
+	NoRestart                 bool          `help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash" env:"STNORESTART"`
+	NoUpgrade                 bool          `help:"Disable automatic upgrades" env:"STNOUPGRADE"`
+	Paused                    bool          `help:"Start with all devices and folders paused" env:"STPAUSED"`
+	Unpaused                  bool          `help:"Start with all devices and folders unpaused" env:"STUNPAUSED"`
+	Verbose                   bool          `help:"Print verbose log output" env:"STVERBOSE"`
 
 	// Debug options below
-	DebugDBIndirectGCInterval time.Duration `env:"STGCINDIRECTEVERY" help:"Database indirection GC interval"`
-	DebugDBRecheckInterval    time.Duration `env:"STRECHECKDBEVERY" help:"Database metadata recalculation interval"`
-	DebugGUIAssetsDir         string        `placeholder:"PATH" help:"Directory to load GUI assets from" env:"STGUIASSETS"`
-	DebugPerfStats            bool          `env:"STPERFSTATS" help:"Write running performance statistics to perf-$pid.csv (Unix only)"`
-	DebugProfileBlock         bool          `env:"STBLOCKPROFILE" help:"Write block profiles to block-$pid-$timestamp.pprof every 20 seconds"`
-	DebugProfileCPU           bool          `help:"Write a CPU profile to cpu-$pid.pprof on exit" env:"STCPUPROFILE"`
-	DebugProfileHeap          bool          `env:"STHEAPPROFILE" help:"Write heap profiles to heap-$pid-$timestamp.pprof each time heap usage increases"`
-	DebugProfilerListen       string        `placeholder:"ADDR" env:"STPROFILER" help:"Network profiler listen address"`
-	DebugResetDatabase        bool          `name:"reset-database" help:"Reset the database, forcing a full rescan and resync"`
-	DebugResetDeltaIdxs       bool          `name:"reset-deltas" help:"Reset delta index IDs, forcing a full index exchange"`
+	DebugGUIAssetsDir   string `help:"Directory to load GUI assets from" placeholder:"PATH" env:"STGUIASSETS"`
+	DebugPerfStats      bool   `help:"Write running performance statistics to perf-$pid.csv (Unix only)" env:"STPERFSTATS"`
+	DebugProfileBlock   bool   `help:"Write block profiles to block-$pid-$timestamp.pprof every 20 seconds" env:"STBLOCKPROFILE"`
+	DebugProfileCPU     bool   `help:"Write a CPU profile to cpu-$pid.pprof on exit" env:"STCPUPROFILE"`
+	DebugProfileHeap    bool   `help:"Write heap profiles to heap-$pid-$timestamp.pprof each time heap usage increases" env:"STHEAPPROFILE"`
+	DebugProfilerListen string `help:"Network profiler listen address" placeholder:"ADDR" env:"STPROFILER" `
+	DebugResetDeltaIdxs bool   `help:"Reset delta index IDs, forcing a full index exchange"`
 
 	// Internal options, not shown to users
 	InternalRestarting   bool `env:"STRESTART" hidden:"1"`
@@ -206,31 +220,9 @@ func defaultVars() kong.Vars {
 }
 
 func main() {
-	// First some massaging of the raw command line to fit the new model.
-	// Basically this means adding the default command at the front, and
-	// converting -options to --options.
-
-	args := os.Args[1:]
-	switch {
-	case len(args) == 0:
-		// Empty command line is equivalent to just calling serve
-		args = []string{"serve"}
-	case args[0] == "-help":
-		// For consistency, we consider this equivalent with --help even
-		// though kong would otherwise consider it a bad flag.
-		args[0] = "--help"
-	case args[0] == "-h", args[0] == "--help":
-		// Top level request for help, let it pass as-is to be handled by
-		// kong to list commands.
-	case strings.HasPrefix(args[0], "-"):
-		// There are flags not preceded by a command, so we tack on the
-		// "serve" command and convert the old style arguments (single dash)
-		// to new style (double dash).
-		args = append([]string{"serve"}, convertLegacyArgs(args)...)
-	}
-
 	// Create a parser with an overridden help function to print our extra
 	// help info.
+	var entrypoint CLI
 	parser, err := kong.New(
 		&entrypoint,
 		kong.ConfigureHelp(kong.HelpOptions{
@@ -245,7 +237,7 @@ func main() {
 	}
 
 	kongplete.Complete(parser)
-	ctx, err := parser.Parse(args)
+	ctx, err := parser.Parse(os.Args[1:])
 	parser.FatalIfErrorf(err)
 	ctx.BindTo(l, (*logger.Logger)(nil)) // main logger available to subcommands
 	err = ctx.Run()
@@ -264,150 +256,54 @@ func helpHandler(options kong.HelpOptions, ctx *kong.Context) error {
 	return nil
 }
 
-// serveOptions.Run() is the entrypoint for `syncthing serve`
-func (options serveOptions) Run() error {
-	l.SetFlags(options.LogFlags)
+// serveCmd.Run() is the entrypoint for `syncthing serve`
+func (c *serveCmd) Run() error {
+	l.SetFlags(c.LogFlags)
 
-	if options.GUIAddress != "" {
+	if c.GUIAddress != "" {
 		// The config picks this up from the environment.
-		os.Setenv("STGUIADDRESS", options.GUIAddress)
+		os.Setenv("STGUIADDRESS", c.GUIAddress)
 	}
-	if options.GUIAPIKey != "" {
+	if c.GUIAPIKey != "" {
 		// The config picks this up from the environment.
-		os.Setenv("STGUIAPIKEY", options.GUIAPIKey)
+		os.Setenv("STGUIAPIKEY", c.GUIAPIKey)
 	}
 
-	if options.HideConsole {
+	if c.HideConsole {
 		osutil.HideConsole()
 	}
 
-	// Not set as default above because the strings can be really long.
-	err := cmdutil.SetConfigDataLocationsFromFlags(options.HomeDir, options.ConfDir, options.DataDir)
-	if err != nil {
-		l.Warnln("Command line options:", err)
-		os.Exit(svcutil.ExitError.AsInt())
-	}
-
 	// Treat an explicitly empty log file name as no log file
-	if options.LogFile == "" {
-		options.LogFile = "-"
+	if c.LogFile == "" {
+		c.LogFile = "-"
 	}
-	if options.LogFile != "default" {
+	if c.LogFile != "default" {
 		// We must set this *after* expandLocations above.
-		if err := locations.Set(locations.LogFile, options.LogFile); err != nil {
+		if err := locations.Set(locations.LogFile, c.LogFile); err != nil {
 			l.Warnln("Setting log file path:", err)
 			os.Exit(svcutil.ExitError.AsInt())
 		}
 	}
 
-	if options.DebugGUIAssetsDir != "" {
+	if c.DebugGUIAssetsDir != "" {
 		// The asset dir is blank if STGUIASSETS wasn't set, in which case we
 		// should look for extra assets in the default place.
-		if err := locations.Set(locations.GUIAssets, options.DebugGUIAssetsDir); err != nil {
+		if err := locations.Set(locations.GUIAssets, c.DebugGUIAssetsDir); err != nil {
 			l.Warnln("Setting GUI assets path:", err)
 			os.Exit(svcutil.ExitError.AsInt())
 		}
 	}
 
-	if options.Version {
-		fmt.Println(build.LongVersion)
-		return nil
-	}
-
-	if options.Paths {
-		fmt.Print(locations.PrettyPaths())
-		return nil
-	}
-
-	if options.DeviceID {
-		cert, err := tls.LoadX509KeyPair(
-			locations.Get(locations.CertFile),
-			locations.Get(locations.KeyFile),
-		)
-		if err != nil {
-			l.Warnln("Error reading device ID:", err)
-			os.Exit(svcutil.ExitError.AsInt())
-		}
-
-		fmt.Println(protocol.NewDeviceID(cert.Certificate[0]))
-		return nil
-	}
-
-	if options.BrowserOnly {
-		if err := openGUI(); err != nil {
-			l.Warnln("Failed to open web UI:", err)
-			os.Exit(svcutil.ExitError.AsInt())
-		}
-		return nil
-	}
-
-	if options.GenerateDir != "" {
-		if err := generate.Generate(l, options.GenerateDir, "", "", options.NoDefaultFolder, options.SkipPortProbing); err != nil {
-			l.Warnln("Failed to generate config and keys:", err)
-			os.Exit(svcutil.ExitError.AsInt())
-		}
-		return nil
-	}
-
 	// Ensure that our home directory exists.
 	if err := syncthing.EnsureDir(locations.GetBaseDir(locations.ConfigBaseDir), 0o700); err != nil {
 		l.Warnln("Failure on home directory:", err)
 		os.Exit(svcutil.ExitError.AsInt())
 	}
 
-	if options.UpgradeTo != "" {
-		err := upgrade.ToURL(options.UpgradeTo)
-		if err != nil {
-			l.Warnln("Error while Upgrading:", err)
-			os.Exit(svcutil.ExitError.AsInt())
-		}
-		l.Infoln("Upgraded from", options.UpgradeTo)
-		return nil
-	}
-
-	if options.UpgradeCheck {
-		if _, err := checkUpgrade(); err != nil {
-			l.Warnln("Checking for upgrade:", err)
-			os.Exit(exitCodeForUpgrade(err))
-		}
-		return nil
-	}
-
-	if options.Upgrade {
-		release, err := checkUpgrade()
-		if err == nil {
-			lf := flock.New(locations.Get(locations.CertFile))
-			locked, err := lf.TryLock()
-			if err != nil {
-				l.Warnln("Upgrade:", err)
-				os.Exit(1)
-			} else if locked {
-				err = upgradeViaRest()
-			} else {
-				err = upgrade.To(release)
-			}
-		}
-		if err != nil {
-			l.Warnln("Upgrade:", err)
-			os.Exit(exitCodeForUpgrade(err))
-		}
-		l.Infof("Upgraded to %q", release.Tag)
-		os.Exit(svcutil.ExitUpgrade.AsInt())
-	}
-
-	if options.DebugResetDatabase {
-		if err := resetDB(); err != nil {
-			l.Warnln("Resetting database:", err)
-			os.Exit(svcutil.ExitError.AsInt())
-		}
-		l.Infoln("Successfully reset database - it will be rebuilt after next start.")
-		return nil
-	}
-
-	if options.InternalInnerProcess {
-		syncthingMain(options)
+	if c.InternalInnerProcess {
+		c.syncthingMain()
 	} else {
-		monitorMain(options)
+		c.monitorMain()
 	}
 	return nil
 }
@@ -516,14 +412,14 @@ func upgradeViaRest() error {
 	return err
 }
 
-func syncthingMain(options serveOptions) {
-	if options.DebugProfileBlock {
+func (c *serveCmd) syncthingMain() {
+	if c.DebugProfileBlock {
 		startBlockProfiler()
 	}
-	if options.DebugProfileHeap {
+	if c.DebugProfileHeap {
 		startHeapProfiler()
 	}
-	if options.DebugPerfStats {
+	if c.DebugPerfStats {
 		startPerfStats()
 	}
 
@@ -568,7 +464,7 @@ func syncthingMain(options serveOptions) {
 	evLogger := events.NewLogger()
 	earlyService.Add(evLogger)
 
-	cfgWrapper, err := syncthing.LoadConfigAtStartup(locations.Get(locations.ConfigFile), cert, evLogger, options.AllowNewerConfig, options.NoDefaultFolder, options.SkipPortProbing)
+	cfgWrapper, err := syncthing.LoadConfigAtStartup(locations.Get(locations.ConfigFile), cert, evLogger, c.AllowNewerConfig, c.NoDefaultFolder, c.NoPortProbing)
 	if err != nil {
 		l.Warnln("Failed to initialize config:", err)
 		os.Exit(svcutil.ExitError.AsInt())
@@ -579,7 +475,7 @@ func syncthingMain(options serveOptions) {
 	// unless we are in a build where it's disabled or the STNOUPGRADE
 	// environment variable is set.
 
-	if build.IsCandidate && !upgrade.DisabledByCompilation && !options.NoUpgrade {
+	if build.IsCandidate && !upgrade.DisabledByCompilation && !c.NoUpgrade {
 		cfgWrapper.Modify(func(cfg *config.Configuration) {
 			l.Infoln("Automatic upgrade is always enabled for candidate releases.")
 			if cfg.Options.AutoUpgradeIntervalH == 0 || cfg.Options.AutoUpgradeIntervalH > 24 {
@@ -592,8 +488,12 @@ func syncthingMain(options serveOptions) {
 		})
 	}
 
-	dbFile := locations.Get(locations.Database)
-	ldb, err := syncthing.OpenDBBackend(dbFile, cfgWrapper.Options().DatabaseTuning)
+	if err := syncthing.TryMigrateDatabase(c.DBDeleteRetentionInterval); err != nil {
+		l.Warnln("Failed to migrate old-style database:", err)
+		os.Exit(1)
+	}
+
+	sdb, err := syncthing.OpenDatabase(locations.Get(locations.Database), c.DBDeleteRetentionInterval)
 	if err != nil {
 		l.Warnln("Error opening database:", err)
 		os.Exit(1)
@@ -602,11 +502,11 @@ func syncthingMain(options serveOptions) {
 	// Check if auto-upgrades is possible, and if yes, and it's enabled do an initial
 	// upgrade immediately. The auto-upgrade routine can only be started
 	// later after App is initialised.
-
-	autoUpgradePossible := autoUpgradePossible(options)
+	autoUpgradePossible := c.autoUpgradePossible()
 	if autoUpgradePossible && cfgWrapper.Options().AutoUpgradeEnabled() {
 		// try to do upgrade directly and log the error if relevant.
-		release, err := initialAutoUpgradeCheck(db.NewMiscDataNamespace(ldb))
+		miscDB := db.NewMiscDB(sdb)
+		release, err := initialAutoUpgradeCheck(miscDB)
 		if err == nil {
 			err = upgrade.To(release)
 		}
@@ -617,36 +517,29 @@ func syncthingMain(options serveOptions) {
 				l.Infoln("Initial automatic upgrade:", err)
 			}
 		} else {
-			l.Infof("Upgraded to %q, exiting now.", release.Tag)
+			l.Infof("Upgraded to %q, should exit now.", release.Tag)
 			os.Exit(svcutil.ExitUpgrade.AsInt())
 		}
 	}
 
-	if options.Unpaused {
+	if c.Unpaused {
 		setPauseState(cfgWrapper, false)
-	} else if options.Paused {
+	} else if c.Paused {
 		setPauseState(cfgWrapper, true)
 	}
 
 	appOpts := syncthing.Options{
-		NoUpgrade:            options.NoUpgrade,
-		ProfilerAddr:         options.DebugProfilerListen,
-		ResetDeltaIdxs:       options.DebugResetDeltaIdxs,
-		Verbose:              options.Verbose,
-		DBRecheckInterval:    options.DebugDBRecheckInterval,
-		DBIndirectGCInterval: options.DebugDBIndirectGCInterval,
+		NoUpgrade:             c.NoUpgrade,
+		ProfilerAddr:          c.DebugProfilerListen,
+		ResetDeltaIdxs:        c.DebugResetDeltaIdxs,
+		Verbose:               c.Verbose,
+		DBMaintenanceInterval: c.DBMaintenanceInterval,
 	}
-	if options.Audit {
-		appOpts.AuditWriter = auditWriter(options.AuditFile)
-	}
-	if dur, err := time.ParseDuration(os.Getenv("STRECHECKDBEVERY")); err == nil {
-		appOpts.DBRecheckInterval = dur
-	}
-	if dur, err := time.ParseDuration(os.Getenv("STGCINDIRECTEVERY")); err == nil {
-		appOpts.DBIndirectGCInterval = dur
+	if c.Audit {
+		appOpts.AuditWriter = auditWriter(c.AuditFile)
 	}
 
-	app, err := syncthing.New(cfgWrapper, ldb, evLogger, cert, appOpts)
+	app, err := syncthing.New(cfgWrapper, sdb, evLogger, cert, appOpts)
 	if err != nil {
 		l.Warnln("Failed to start Syncthing:", err)
 		os.Exit(svcutil.ExitError.AsInt())
@@ -658,7 +551,7 @@ func syncthingMain(options serveOptions) {
 
 	setupSignalHandling(app)
 
-	if options.DebugProfileCPU {
+	if c.DebugProfileCPU {
 		f, err := os.Create(fmt.Sprintf("cpu-%d.pprof", os.Getpid()))
 		if err != nil {
 			l.Warnln("Creating profile:", err)
@@ -676,7 +569,7 @@ func syncthingMain(options serveOptions) {
 
 	cleanConfigDirectory()
 
-	if cfgWrapper.Options().StartBrowser && !options.NoBrowser && !options.InternalRestarting {
+	if cfgWrapper.Options().StartBrowser && !c.NoBrowser && !c.InternalRestarting {
 		// Can potentially block if the utility we are invoking doesn't
 		// fork, and just execs, hence keep it in its own routine.
 		go func() { _ = openURL(cfgWrapper.GUI().URL()) }()
@@ -688,10 +581,11 @@ func syncthingMain(options serveOptions) {
 		l.Warnln("Syncthing stopped with error:", app.Error())
 	}
 
-	if options.DebugProfileCPU {
+	if c.DebugProfileCPU {
 		pprof.StopCPUProfile()
 	}
 
+	runtime.KeepAlive(lf) // ensure lock is still held to this point
 	os.Exit(int(status))
 }
 
@@ -761,15 +655,11 @@ func auditWriter(auditFile string) io.Writer {
 	return fd
 }
 
-func resetDB() error {
-	return os.RemoveAll(locations.Get(locations.Database))
-}
-
-func autoUpgradePossible(options serveOptions) bool {
+func (c *serveCmd) autoUpgradePossible() bool {
 	if upgrade.DisabledByCompilation {
 		return false
 	}
-	if options.NoUpgrade {
+	if c.NoUpgrade {
 		l.Infof("No automatic upgrades; STNOUPGRADE environment variable defined.")
 		return false
 	}
@@ -833,7 +723,7 @@ func autoUpgrade(cfg config.Wrapper, app *syncthing.App, evLogger events.Logger)
 	}
 }
 
-func initialAutoUpgradeCheck(misc *db.NamespacedKV) (upgrade.Release, error) {
+func initialAutoUpgradeCheck(misc *db.Typed) (upgrade.Release, error) {
 	if last, ok, err := misc.Time(upgradeCheckKey); err == nil && ok && time.Since(last) < upgradeCheckInterval {
 		return upgrade.Release{}, errTooEarlyUpgradeCheck
 	}
@@ -940,3 +830,127 @@ func convertLegacyArgs(args []string) []string {
 
 	return res
 }
+
+type versionCmd struct{}
+
+func (versionCmd) Run() error {
+	fmt.Println(build.LongVersion)
+	return nil
+}
+
+type deviceIDCmd struct{}
+
+func (deviceIDCmd) Run() error {
+	cert, err := tls.LoadX509KeyPair(
+		locations.Get(locations.CertFile),
+		locations.Get(locations.KeyFile),
+	)
+	if err != nil {
+		l.Warnln("Error reading device ID:", err)
+		os.Exit(svcutil.ExitError.AsInt())
+	}
+
+	fmt.Println(protocol.NewDeviceID(cert.Certificate[0]))
+	return nil
+}
+
+type pathsCmd struct{}
+
+func (pathsCmd) Run() error {
+	fmt.Print(locations.PrettyPaths())
+	return nil
+}
+
+type upgradeCmd struct {
+	CheckOnly bool   `short:"c" help:"Check for available upgrade, then exit"`
+	From      string `short:"u" placeholder:"URL" help:"Force upgrade directly from specified URL"`
+}
+
+func (u upgradeCmd) Run() error {
+	if u.CheckOnly {
+		if _, err := checkUpgrade(); err != nil {
+			l.Warnln("Checking for upgrade:", err)
+			os.Exit(exitCodeForUpgrade(err))
+		}
+		return nil
+	}
+
+	if u.From != "" {
+		err := upgrade.ToURL(u.From)
+		if err != nil {
+			l.Warnln("Error while Upgrading:", err)
+			os.Exit(svcutil.ExitError.AsInt())
+		}
+		l.Infoln("Upgraded from", u.From)
+		return nil
+	}
+
+	release, err := checkUpgrade()
+	if err == nil {
+		lf := flock.New(locations.Get(locations.CertFile))
+		locked, err := lf.TryLock()
+		if err != nil {
+			l.Warnln("Upgrade:", err)
+			os.Exit(1)
+		} else if locked {
+			err = upgradeViaRest()
+		} else {
+			err = upgrade.To(release)
+		}
+	}
+	if err != nil {
+		l.Warnln("Upgrade:", err)
+		os.Exit(exitCodeForUpgrade(err))
+	}
+	l.Infof("Upgraded to %q", release.Tag)
+	os.Exit(svcutil.ExitUpgrade.AsInt())
+	return nil
+}
+
+type browserCmd struct{}
+
+func (browserCmd) Run() error {
+	if err := openGUI(); err != nil {
+		l.Warnln("Failed to open web UI:", err)
+		os.Exit(svcutil.ExitError.AsInt())
+	}
+	return nil
+}
+
+type debugCmd struct {
+	ResetDatabase resetDatabaseCmd `cmd:"" help:"Reset the database, forcing a full rescan and resync"`
+}
+
+type resetDatabaseCmd struct{}
+
+func (resetDatabaseCmd) Run() error {
+	l.Infoln("Removing database in", locations.Get(locations.Database))
+	if err := os.RemoveAll(locations.Get(locations.Database)); err != nil {
+		l.Warnln("Resetting database:", err)
+		os.Exit(svcutil.ExitError.AsInt())
+	}
+	l.Infoln("Successfully reset database - it will be rebuilt after next start.")
+	return nil
+}
+
+func setConfigDataLocationsFromFlags(homeDir, confDir, dataDir string) error {
+	homeSet := homeDir != ""
+	confSet := confDir != ""
+	dataSet := dataDir != ""
+	switch {
+	case dataSet != confSet:
+		return errors.New("either both or none of --config and --data must be given, use --home to set both at once")
+	case homeSet && dataSet:
+		return errors.New("--home must not be used together with --config and --data")
+	case homeSet:
+		confDir = homeDir
+		dataDir = homeDir
+		fallthrough
+	case dataSet:
+		if err := locations.SetBaseDir(locations.ConfigBaseDir, confDir); err != nil {
+			return err
+		}
+		return locations.SetBaseDir(locations.DataBaseDir, dataDir)
+	}
+	return nil
+}

cmd/syncthing/monitor.go

@@ -43,7 +43,7 @@ const (
 	panicUploadNoticeWait = 10 * time.Second
 )
 
-func monitorMain(options serveOptions) {
+func (c *serveCmd) monitorMain() {
 	l.SetPrefix("[monitor] ")
 
 	var dst io.Writer = os.Stdout
@@ -58,8 +58,8 @@ func monitorMain(options serveOptions) {
 		open := func(name string) (io.WriteCloser, error) {
 			return newAutoclosedFile(name, logFileAutoCloseDelay, logFileMaxOpenTime)
 		}
-		if options.LogMaxSize > 0 {
-			fileDst, err = newRotatedFile(logFile, open, int64(options.LogMaxSize), options.LogMaxFiles)
+		if c.LogMaxSize > 0 {
+			fileDst, err = newRotatedFile(logFile, open, int64(c.LogMaxSize), c.LogMaxFiles)
 		} else {
 			fileDst, err = open(logFile)
 		}
@@ -178,7 +178,7 @@ func monitorMain(options serveOptions) {
 
 		if exiterr, ok := err.(*exec.ExitError); ok {
 			exitCode := exiterr.ExitCode()
-			if stopped || options.NoRestart {
+			if stopped || c.NoRestart {
 				os.Exit(exitCode)
 			}
 			if exitCode == svcutil.ExitUpgrade.AsInt() {
@@ -192,7 +192,7 @@ func monitorMain(options serveOptions) {
 			}
 		}
 
-		if options.NoRestart {
+		if c.NoRestart {
 			os.Exit(svcutil.ExitError.AsInt())
 		}
 

go.mod

@@ -9,19 +9,19 @@ require (
 	github.com/calmh/incontainer v1.0.0
 	github.com/calmh/xdr v1.2.0
 	github.com/ccding/go-stun v0.1.5
-	github.com/chmduquesne/rollinghash v4.0.0+incompatible
 	github.com/d4l3k/messagediff v1.2.1
 	github.com/getsentry/raven-go v0.2.0
 	github.com/go-ldap/ldap/v3 v3.4.10
 	github.com/gobwas/glob v0.2.3
 	github.com/gofrs/flock v0.12.1
-	github.com/greatroar/blobloom v0.8.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/jackpal/gateway v1.0.16
 	github.com/jackpal/go-nat-pmp v1.0.2
+	github.com/jmoiron/sqlx v1.4.0
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/maruel/panicparse/v2 v2.5.0
+	github.com/mattn/go-sqlite3 v1.14.27
 	github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2
 	github.com/maxmind/geoipupdate/v6 v6.1.0
 	github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75
@@ -47,6 +47,7 @@ require (
 	golang.org/x/time v0.11.0
 	golang.org/x/tools v0.31.0
 	google.golang.org/protobuf v1.36.6
+	modernc.org/sqlite v1.37.0
 	sigs.k8s.io/yaml v1.4.0
 )
 
@@ -58,20 +59,23 @@ require (
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/ebitengine/purego v0.8.2 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/pprof v0.0.0-20241009165004-a3522334989c // indirect
+	github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/nxadm/tail v1.4.11 // indirect
 	github.com/onsi/ginkgo/v2 v2.20.2 // indirect
 	github.com/oschwald/maxminddb-golang v1.13.1 // indirect
@@ -82,6 +86,7 @@ require (
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
@@ -90,10 +95,13 @@ require (
 	github.com/tklauser/numcpus v0.9.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.uber.org/mock v0.5.0 // indirect
-	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect
+	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
 	golang.org/x/mod v0.24.0 // indirect
 	golang.org/x/sync v0.12.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/libc v1.62.1 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.9.1 // indirect
 )
 
 // https://github.com/gobwas/glob/pull/55

go.sum

@@ -1,3 +1,5 @@
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/AudriusButkevicius/recli v0.0.7-0.20220911121932-d000ce8fbf0f h1:GmH5lT+moM7PbAJFBq57nH9WJ+wRnBXr/tyaYWbSAx8=
 github.com/AudriusButkevicius/recli v0.0.7-0.20220911121932-d000ce8fbf0f/go.mod h1:Nhfib1j/VFnLrXL9cHgA+/n2O6P5THuWelOnbfPNd78=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
@@ -29,8 +31,6 @@ github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV
 github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chmduquesne/rollinghash v4.0.0+incompatible h1:hnREQO+DXjqIw3rUTzWN7/+Dpw+N5Um8zpKV0JOEgbo=
-github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -41,6 +41,8 @@ github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkE
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I=
 github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -60,6 +62,8 @@ github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ4
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
 github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
@@ -85,14 +89,12 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20241009165004-a3522334989c h1:NDovD0SMpBYXlE1zJmS1q55vWB/fUQBcPAqAboZSccA=
-github.com/google/pprof v0.0.0-20241009165004-a3522334989c/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
-github.com/greatroar/blobloom v0.8.0 h1:I9RlEkfqK9/6f1v9mFmDYegDQ/x0mISCpiNpAm23Pt4=
-github.com/greatroar/blobloom v0.8.0/go.mod h1:mjMJ1hh1wjGVfr93QIHJ6FfDNVrA0IELv8OvMHJxHKs=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -128,6 +130,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
+github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
@@ -140,10 +144,17 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0=
 github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
 github.com/maruel/panicparse/v2 v2.5.0 h1:yCtuS0FWjfd0RTYMXGpDvWcb0kINm8xJGu18/xMUh00=
 github.com/maruel/panicparse/v2 v2.5.0/go.mod h1:DA2fDiBk63bKfBf4CVZP9gb4fuvzdPbLDsSI873hweQ=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mattn/go-sqlite3 v1.14.27 h1:drZCnuvf37yPfs95E5jd9s3XhdVWLal+6BOK6qrv6IU=
+github.com/mattn/go-sqlite3 v1.14.27/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 h1:yVCLo4+ACVroOEr4iFU1iH46Ldlzz2rTuu18Ra7M8sU=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2/go.mod h1:VzB2VoMh1Y32/QqDfg9ZJYHj99oM4LiGtqPZydTiQSQ=
 github.com/maxmind/geoipupdate/v6 v6.1.0 h1:sdtTHzzQNJlXF5+fd/EoPTucRHyMonYt/Cok8xzzfqA=
@@ -152,6 +163,8 @@ github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75 h1:cUVxyR+U
 github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75/go.mod h1:pBbZyGwC5i16IBkjVKoy/sznA8jPD/K9iedwe1ESE6w=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
@@ -203,6 +216,8 @@ github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzuk
 github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
 github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg=
 github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab h1:ZjX6I48eZSFetPb41dHudEyVr5v953N15TsNZXlkcWY=
 github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab/go.mod h1:/PfPXh0EntGc3QAAyUaviy4S9tzy4Zp0e2ilq4voC6E=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
@@ -266,8 +281,8 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
 golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
-golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY=
-golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8=
+golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw=
+golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -330,6 +345,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
@@ -399,5 +415,29 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/cc/v4 v4.25.2 h1:T2oH7sZdGvTaie0BRNFbIYsabzCxUQg8nLqCdQ2i0ic=
+modernc.org/cc/v4 v4.25.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
+modernc.org/ccgo/v4 v4.25.1 h1:TFSzPrAGmDsdnhT9X2UrcPMI3N/mJ9/X9ykKXwLhDsU=
+modernc.org/ccgo/v4 v4.25.1/go.mod h1:njjuAYiPflywOOrm3B7kCB444ONP5pAVr8PIEoE0uDw=
+modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
+modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/libc v1.62.1 h1:s0+fv5E3FymN8eJVmnk0llBe6rOxCu/DEU+XygRbS8s=
+modernc.org/libc v1.62.1/go.mod h1:iXhATfJQLjG3NWy56a6WVU73lWOcdYVxsvwCgoPljuo=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.9.1 h1:V/Z1solwAVmMW1yttq3nDdZPJqV1rM05Ccq6KMSZ34g=
+modernc.org/memory v1.9.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
+modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.37.0 h1:s1TMe7T3Q3ovQiK2Ouz4Jwh7dw4ZDqbebSDTlSJdfjI=
+modernc.org/sqlite v1.37.0/go.mod h1:5YiWv+YviqGMuGw4V+PNplcyaJ5v+vQd7TQOgkACoJM=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

gui/default/syncthing/core/aboutModalView.html

@@ -59,9 +59,12 @@ Jakob Borg, Audrius Butkevicius, Jesse Lucas, Simon Frei, Tomasz Wilczyński, Al
           <li><a href="https://github.com/golang/protobuf">golang/protobuf</a>, Copyright &copy; 2010 The Go Authors.</li>
           <li><a href="https://github.com/golang/snappy">golang/snappy</a>, Copyright &copy; 2011 The Snappy-Go Authors.</li>
           <li><a href="https://github.com/jackpal/gateway">jackpal/gateway</a>, Copyright &copy; 2010 Jack Palevich.</li>
+          <li><a href="https://github.com/jmoiron/sqlx">jmoiron/sqlx</a>, Copyright &copy; 2013 Jason Moiron.</li>
           <li><a href="https://github.com/kballard/go-shellquote">kballard/go-shellquote</a>, Copyright &copy; 2014 Kevin Ballard.</li>
           <li><a href="https://github.com/mattn/go-isatty">mattn/go-isatty</a>, Copyright &copy; Yasuhiro MATSUMOTO.</li>
+          <li><a href="https://github.com/mattn/go-sqlite3">mattn/go-sqlite3</a>, Copyright &copy; 2014 Yasuhiro Matsumoto</li>
           <li><a href="https://github.com/matttproud/golang_protobuf_extensions">matttproud/golang_protobuf_extensions</a>, Copyright &copy; 2012 Matt T. Proud.</li>
+          <li><a href="https://modernc.org/sqlite">modernc.org/sqlite</a>, Copyright &copy; 2017 The Sqlite Authors</li>
           <li><a href="https://github.com/oschwald/geoip2-golang">oschwald/geoip2-golang</a>, Copyright &copy; 2015, Gregory J. Oschwald.</li>
           <li><a href="https://github.com/oschwald/maxminddb-golang">oschwald/maxminddb-golang</a>, Copyright &copy; 2015, Gregory J. Oschwald.</li>
           <li><a href="https://github.com/petermattis/goid">petermattis/goid</a>, Copyright &copy; 2015-2016 Peter Mattis.</li>

internal/db/counts.go

@@ -0,0 +1,73 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type Counts struct {
+	Files       int
+	Directories int
+	Symlinks    int
+	Deleted     int
+	Bytes       int64
+	Sequence    int64             // zero for the global state
+	DeviceID    protocol.DeviceID // device ID for remote devices, or special values for local/global
+	LocalFlags  uint32            // the local flag for this count bucket
+}
+
+func (c Counts) Add(other Counts) Counts {
+	return Counts{
+		Files:       c.Files + other.Files,
+		Directories: c.Directories + other.Directories,
+		Symlinks:    c.Symlinks + other.Symlinks,
+		Deleted:     c.Deleted + other.Deleted,
+		Bytes:       c.Bytes + other.Bytes,
+		Sequence:    c.Sequence + other.Sequence,
+		DeviceID:    protocol.EmptyDeviceID,
+		LocalFlags:  c.LocalFlags | other.LocalFlags,
+	}
+}
+
+func (c Counts) TotalItems() int {
+	return c.Files + c.Directories + c.Symlinks + c.Deleted
+}
+
+func (c Counts) String() string {
+	var flags strings.Builder
+	if c.LocalFlags&protocol.FlagLocalNeeded != 0 {
+		flags.WriteString("Need")
+	}
+	if c.LocalFlags&protocol.FlagLocalIgnored != 0 {
+		flags.WriteString("Ignored")
+	}
+	if c.LocalFlags&protocol.FlagLocalMustRescan != 0 {
+		flags.WriteString("Rescan")
+	}
+	if c.LocalFlags&protocol.FlagLocalReceiveOnly != 0 {
+		flags.WriteString("Recvonly")
+	}
+	if c.LocalFlags&protocol.FlagLocalUnsupported != 0 {
+		flags.WriteString("Unsupported")
+	}
+	if c.LocalFlags != 0 {
+		flags.WriteString(fmt.Sprintf("(%x)", c.LocalFlags))
+	}
+	if flags.Len() == 0 {
+		flags.WriteString("---")
+	}
+	return fmt.Sprintf("{Device:%v, Files:%d, Dirs:%d, Symlinks:%d, Del:%d, Bytes:%d, Seq:%d, Flags:%s}", c.DeviceID, c.Files, c.Directories, c.Symlinks, c.Deleted, c.Bytes, c.Sequence, flags.String())
+}
+
+// Equal compares the numbers only, not sequence/dev/flags.
+func (c Counts) Equal(o Counts) bool {
+	return c.Files == o.Files && c.Directories == o.Directories && c.Symlinks == o.Symlinks && c.Deleted == o.Deleted && c.Bytes == o.Bytes
+}

internal/db/interface.go

@@ -0,0 +1,123 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db // import "github.com/syncthing/syncthing/internal/db/sqlite"
+
+import (
+	"iter"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/thejerf/suture/v4"
+)
+
+type DB interface {
+	Service(maintenanceInterval time.Duration) suture.Service
+
+	// Basics
+	Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo) error
+	Close() error
+
+	// Single files
+	GetDeviceFile(folder string, device protocol.DeviceID, file string) (protocol.FileInfo, bool, error)
+	GetGlobalAvailability(folder, file string) ([]protocol.DeviceID, error)
+	GetGlobalFile(folder string, file string) (protocol.FileInfo, bool, error)
+
+	// File iterators
+	//
+	// n.b. there is a slight inconsistency in the return types where some
+	// return a FileInfo iterator and some a FileMetadata iterator. The
+	// latter is more lightweight, and the discrepancy depends on how the
+	// functions tend to be used. We can introduce more variations as
+	// required.
+	AllGlobalFiles(folder string) (iter.Seq[FileMetadata], func() error)
+	AllGlobalFilesPrefix(folder string, prefix string) (iter.Seq[FileMetadata], func() error)
+	AllLocalBlocksWithHash(hash []byte) (iter.Seq[BlockMapEntry], func() error)
+	AllLocalFiles(folder string, device protocol.DeviceID) (iter.Seq[protocol.FileInfo], func() error)
+	AllLocalFilesBySequence(folder string, device protocol.DeviceID, startSeq int64, limit int) (iter.Seq[protocol.FileInfo], func() error)
+	AllLocalFilesWithPrefix(folder string, device protocol.DeviceID, prefix string) (iter.Seq[protocol.FileInfo], func() error)
+	AllLocalFilesWithBlocksHash(folder string, h []byte) (iter.Seq[FileMetadata], func() error)
+	AllLocalFilesWithBlocksHashAnyFolder(h []byte) (iter.Seq2[string, FileMetadata], func() error)
+	AllNeededGlobalFiles(folder string, device protocol.DeviceID, order config.PullOrder, limit, offset int) (iter.Seq[protocol.FileInfo], func() error)
+
+	// Cleanup
+	DropAllFiles(folder string, device protocol.DeviceID) error
+	DropDevice(device protocol.DeviceID) error
+	DropFilesNamed(folder string, device protocol.DeviceID, names []string) error
+	DropFolder(folder string) error
+
+	// Various metadata
+	GetDeviceSequence(folder string, device protocol.DeviceID) (int64, error)
+	ListFolders() ([]string, error)
+	ListDevicesForFolder(folder string) ([]protocol.DeviceID, error)
+	RemoteSequences(folder string) (map[protocol.DeviceID]int64, error)
+
+	// Counts
+	CountGlobal(folder string) (Counts, error)
+	CountLocal(folder string, device protocol.DeviceID) (Counts, error)
+	CountNeed(folder string, device protocol.DeviceID) (Counts, error)
+	CountReceiveOnlyChanged(folder string) (Counts, error)
+
+	// Index IDs
+	DropAllIndexIDs() error
+	GetIndexID(folder string, device protocol.DeviceID) (protocol.IndexID, error)
+	SetIndexID(folder string, device protocol.DeviceID, id protocol.IndexID) error
+
+	// MtimeFS
+	DeleteMtime(folder, name string) error
+	GetMtime(folder, name string) (ondisk, virtual time.Time)
+	PutMtime(folder, name string, ondisk, virtual time.Time) error
+
+	KV
+}
+
+// Generic KV store
+type KV interface {
+	GetKV(key string) ([]byte, error)
+	PutKV(key string, val []byte) error
+	DeleteKV(key string) error
+	PrefixKV(prefix string) (iter.Seq[KeyValue], func() error)
+}
+
+type BlockMapEntry struct {
+	BlocklistHash []byte
+	Offset        int64
+	BlockIndex    int
+	Size          int
+}
+
+type KeyValue struct {
+	Key   string
+	Value []byte
+}
+
+type FileMetadata struct {
+	Name       string
+	Sequence   int64
+	ModNanos   int64
+	Size       int64
+	LocalFlags int64
+	Type       protocol.FileInfoType
+	Deleted    bool
+	Invalid    bool
+}
+
+func (f *FileMetadata) ModTime() time.Time {
+	return time.Unix(0, f.ModNanos)
+}
+
+func (f *FileMetadata) IsReceiveOnlyChanged() bool {
+	return f.LocalFlags&protocol.FlagLocalReceiveOnly != 0
+}
+
+func (f *FileMetadata) IsDirectory() bool {
+	return f.Type == protocol.FileInfoTypeDirectory
+}
+
+func (f *FileMetadata) ShouldConflict() bool {
+	return f.LocalFlags&protocol.LocalConflictFlags != 0
+}

internal/db/metrics.go

@@ -0,0 +1,229 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"iter"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+var (
+	metricCurrentOperations = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "syncthing",
+		Subsystem: "db",
+		Name:      "operations_current",
+	}, []string{"folder", "operation"})
+	metricTotalOperationSeconds = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "db",
+		Name:      "operation_seconds_total",
+		Help:      "Total time spent in database operations, per folder and operation",
+	}, []string{"folder", "operation"})
+	metricTotalOperationsCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "db",
+		Name:      "operations_total",
+		Help:      "Total number of database operations, per folder and operation",
+	}, []string{"folder", "operation"})
+	metricTotalFilesUpdatedCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "db",
+		Name:      "files_updated_total",
+		Help:      "Total number of files updated",
+	}, []string{"folder"})
+)
+
+func MetricsWrap(db DB) DB {
+	return metricsDB{db}
+}
+
+type metricsDB struct {
+	DB
+}
+
+func (m metricsDB) account(folder, op string) func() {
+	t0 := time.Now()
+	metricCurrentOperations.WithLabelValues(folder, op).Inc()
+	return func() {
+		if dur := time.Since(t0).Seconds(); dur > 0 {
+			metricTotalOperationSeconds.WithLabelValues(folder, op).Add(dur)
+		}
+		metricTotalOperationsCount.WithLabelValues(folder, op).Inc()
+		metricCurrentOperations.WithLabelValues(folder, op).Dec()
+	}
+}
+
+func (m metricsDB) AllLocalFilesWithBlocksHash(folder string, h []byte) (iter.Seq[FileMetadata], func() error) {
+	defer m.account(folder, "AllLocalFilesWithBlocksHash")()
+	return m.DB.AllLocalFilesWithBlocksHash(folder, h)
+}
+
+func (m metricsDB) AllLocalFilesWithBlocksHashAnyFolder(h []byte) (iter.Seq2[string, FileMetadata], func() error) {
+	defer m.account("-", "AllLocalFilesWithBlocksHashAnyFolder")()
+	return m.DB.AllLocalFilesWithBlocksHashAnyFolder(h)
+}
+
+func (m metricsDB) AllGlobalFiles(folder string) (iter.Seq[FileMetadata], func() error) {
+	defer m.account(folder, "AllGlobalFiles")()
+	return m.DB.AllGlobalFiles(folder)
+}
+
+func (m metricsDB) AllGlobalFilesPrefix(folder string, prefix string) (iter.Seq[FileMetadata], func() error) {
+	defer m.account(folder, "AllGlobalFilesPrefix")()
+	return m.DB.AllGlobalFilesPrefix(folder, prefix)
+}
+
+func (m metricsDB) AllLocalFiles(folder string, device protocol.DeviceID) (iter.Seq[protocol.FileInfo], func() error) {
+	defer m.account(folder, "AllLocalFiles")()
+	return m.DB.AllLocalFiles(folder, device)
+}
+
+func (m metricsDB) AllLocalFilesWithPrefix(folder string, device protocol.DeviceID, prefix string) (iter.Seq[protocol.FileInfo], func() error) {
+	defer m.account(folder, "AllLocalFilesPrefix")()
+	return m.DB.AllLocalFilesWithPrefix(folder, device, prefix)
+}
+
+func (m metricsDB) AllLocalFilesBySequence(folder string, device protocol.DeviceID, startSeq int64, limit int) (iter.Seq[protocol.FileInfo], func() error) {
+	defer m.account(folder, "AllLocalFilesBySequence")()
+	return m.DB.AllLocalFilesBySequence(folder, device, startSeq, limit)
+}
+
+func (m metricsDB) AllNeededGlobalFiles(folder string, device protocol.DeviceID, order config.PullOrder, limit, offset int) (iter.Seq[protocol.FileInfo], func() error) {
+	defer m.account(folder, "AllNeededGlobalFiles")()
+	return m.DB.AllNeededGlobalFiles(folder, device, order, limit, offset)
+}
+
+func (m metricsDB) GetGlobalAvailability(folder, file string) ([]protocol.DeviceID, error) {
+	defer m.account(folder, "GetGlobalAvailability")()
+	return m.DB.GetGlobalAvailability(folder, file)
+}
+
+func (m metricsDB) AllLocalBlocksWithHash(hash []byte) (iter.Seq[BlockMapEntry], func() error) {
+	defer m.account("-", "AllLocalBlocksWithHash")()
+	return m.DB.AllLocalBlocksWithHash(hash)
+}
+
+func (m metricsDB) Close() error {
+	defer m.account("-", "Close")()
+	return m.DB.Close()
+}
+
+func (m metricsDB) ListDevicesForFolder(folder string) ([]protocol.DeviceID, error) {
+	defer m.account(folder, "ListDevicesForFolder")()
+	return m.DB.ListDevicesForFolder(folder)
+}
+
+func (m metricsDB) RemoteSequences(folder string) (map[protocol.DeviceID]int64, error) {
+	defer m.account(folder, "RemoteSequences")()
+	return m.DB.RemoteSequences(folder)
+}
+
+func (m metricsDB) DropAllFiles(folder string, device protocol.DeviceID) error {
+	defer m.account(folder, "DropAllFiles")()
+	return m.DB.DropAllFiles(folder, device)
+}
+
+func (m metricsDB) DropDevice(device protocol.DeviceID) error {
+	defer m.account("-", "DropDevice")()
+	return m.DB.DropDevice(device)
+}
+
+func (m metricsDB) DropFilesNamed(folder string, device protocol.DeviceID, names []string) error {
+	defer m.account(folder, "DropFilesNamed")()
+	return m.DB.DropFilesNamed(folder, device, names)
+}
+
+func (m metricsDB) DropFolder(folder string) error {
+	defer m.account(folder, "DropFolder")()
+	return m.DB.DropFolder(folder)
+}
+
+func (m metricsDB) DropAllIndexIDs() error {
+	defer m.account("-", "IndexIDDropAll")()
+	return m.DB.DropAllIndexIDs()
+}
+
+func (m metricsDB) ListFolders() ([]string, error) {
+	defer m.account("-", "ListFolders")()
+	return m.DB.ListFolders()
+}
+
+func (m metricsDB) GetGlobalFile(folder string, file string) (protocol.FileInfo, bool, error) {
+	defer m.account(folder, "GetGlobalFile")()
+	return m.DB.GetGlobalFile(folder, file)
+}
+
+func (m metricsDB) CountGlobal(folder string) (Counts, error) {
+	defer m.account(folder, "CountGlobal")()
+	return m.DB.CountGlobal(folder)
+}
+
+func (m metricsDB) GetIndexID(folder string, device protocol.DeviceID) (protocol.IndexID, error) {
+	defer m.account(folder, "IndexIDGet")()
+	return m.DB.GetIndexID(folder, device)
+}
+
+func (m metricsDB) GetDeviceFile(folder string, device protocol.DeviceID, file string) (protocol.FileInfo, bool, error) {
+	defer m.account(folder, "GetDeviceFile")()
+	return m.DB.GetDeviceFile(folder, device, file)
+}
+
+func (m metricsDB) CountLocal(folder string, device protocol.DeviceID) (Counts, error) {
+	defer m.account(folder, "CountLocal")()
+	return m.DB.CountLocal(folder, device)
+}
+
+func (m metricsDB) CountNeed(folder string, device protocol.DeviceID) (Counts, error) {
+	defer m.account(folder, "CountNeed")()
+	return m.DB.CountNeed(folder, device)
+}
+
+func (m metricsDB) CountReceiveOnlyChanged(folder string) (Counts, error) {
+	defer m.account(folder, "CountReceiveOnlyChanged")()
+	return m.DB.CountReceiveOnlyChanged(folder)
+}
+
+func (m metricsDB) GetDeviceSequence(folder string, device protocol.DeviceID) (int64, error) {
+	defer m.account(folder, "GetDeviceSequence")()
+	return m.DB.GetDeviceSequence(folder, device)
+}
+
+func (m metricsDB) SetIndexID(folder string, device protocol.DeviceID, id protocol.IndexID) error {
+	defer m.account(folder, "IndexIDSet")()
+	return m.DB.SetIndexID(folder, device, id)
+}
+
+func (m metricsDB) Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo) error {
+	defer m.account(folder, "Update")()
+	defer metricTotalFilesUpdatedCount.WithLabelValues(folder).Add(float64(len(fs)))
+	return m.DB.Update(folder, device, fs)
+}
+
+func (m metricsDB) GetKV(key string) ([]byte, error) {
+	defer m.account("-", "GetKV")()
+	return m.DB.GetKV(key)
+}
+
+func (m metricsDB) PutKV(key string, val []byte) error {
+	defer m.account("-", "PutKV")()
+	return m.DB.PutKV(key, val)
+}
+
+func (m metricsDB) DeleteKV(key string) error {
+	defer m.account("-", "DeleteKV")()
+	return m.DB.DeleteKV(key)
+}
+
+func (m metricsDB) PrefixKV(prefix string) (iter.Seq[KeyValue], func() error) {
+	defer m.account("-", "PrefixKV")()
+	return m.DB.PrefixKV(prefix)
+}

internal/db/observed.go

@@ -8,6 +8,7 @@ package db
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	"google.golang.org/protobuf/proto"
@@ -17,6 +18,14 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
+type ObservedDB struct {
+	kv KV
+}
+
+func NewObservedDB(kv KV) *ObservedDB {
+	return &ObservedDB{kv: kv}
+}
+
 type ObservedFolder struct {
 	Time             time.Time `json:"time"`
 	Label            string    `json:"label"`
@@ -52,39 +61,42 @@ func (o *ObservedDevice) fromWire(w *dbproto.ObservedDevice) {
 	o.Address = w.GetAddress()
 }
 
-func (db *Lowlevel) AddOrUpdatePendingDevice(device protocol.DeviceID, name, address string) error {
-	key := db.keyer.GeneratePendingDeviceKey(nil, device[:])
+func (db *ObservedDB) AddOrUpdatePendingDevice(device protocol.DeviceID, name, address string) error {
+	key := "device/" + device.String()
 	od := &dbproto.ObservedDevice{
 		Time:    timestamppb.New(time.Now().Truncate(time.Second)),
 		Name:    name,
 		Address: address,
 	}
-	return db.Put(key, mustMarshal(od))
+	return db.kv.PutKV(key, mustMarshal(od))
 }
 
-func (db *Lowlevel) RemovePendingDevice(device protocol.DeviceID) error {
-	key := db.keyer.GeneratePendingDeviceKey(nil, device[:])
-	return db.Delete(key)
+func (db *ObservedDB) RemovePendingDevice(device protocol.DeviceID) error {
+	key := "device/" + device.String()
+	return db.kv.DeleteKV(key)
 }
 
 // PendingDevices enumerates all entries.  Invalid ones are dropped from the database
 // after a warning log message, as a side-effect.
-func (db *Lowlevel) PendingDevices() (map[protocol.DeviceID]ObservedDevice, error) {
-	iter, err := db.NewPrefixIterator([]byte{KeyTypePendingDevice})
-	if err != nil {
-		return nil, err
-	}
-	defer iter.Release()
+func (db *ObservedDB) PendingDevices() (map[protocol.DeviceID]ObservedDevice, error) {
 	res := make(map[protocol.DeviceID]ObservedDevice)
-	for iter.Next() {
-		keyDev := db.keyer.DeviceFromPendingDeviceKey(iter.Key())
-		deviceID, err := protocol.DeviceIDFromBytes(keyDev)
+	it, errFn := db.kv.PrefixKV("device/")
+	for kv := range it {
+		_, keyDev, ok := strings.Cut(kv.Key, "/")
+		if !ok {
+			if err := db.kv.DeleteKV(kv.Key); err != nil {
+				return nil, fmt.Errorf("delete invalid pending device: %w", err)
+			}
+			continue
+		}
+
+		deviceID, err := protocol.DeviceIDFromString(keyDev)
 		var protoD dbproto.ObservedDevice
 		var od ObservedDevice
 		if err != nil {
 			goto deleteKey
 		}
-		if err = proto.Unmarshal(iter.Value(), &protoD); err != nil {
+		if err = proto.Unmarshal(kv.Value, &protoD); err != nil {
 			goto deleteKey
 		}
 		od.fromWire(&protoD)
@@ -94,52 +106,37 @@ func (db *Lowlevel) PendingDevices() (map[protocol.DeviceID]ObservedDevice, erro
 		// Deleting invalid entries is the only possible "repair" measure and
 		// appropriate for the importance of pending entries.  They will come back
 		// soon if still relevant.
-		l.Infof("Invalid pending device entry, deleting from database: %x", iter.Key())
-		if err := db.Delete(iter.Key()); err != nil {
-			return nil, err
+		if err := db.kv.DeleteKV(kv.Key); err != nil {
+			return nil, fmt.Errorf("delete invalid pending device: %w", err)
 		}
 	}
-	return res, nil
+	return res, errFn()
 }
 
-func (db *Lowlevel) AddOrUpdatePendingFolder(id string, of ObservedFolder, device protocol.DeviceID) error {
-	key, err := db.keyer.GeneratePendingFolderKey(nil, device[:], []byte(id))
-	if err != nil {
-		return err
-	}
-	return db.Put(key, mustMarshal(of.toWire()))
+func (db *ObservedDB) AddOrUpdatePendingFolder(id string, of ObservedFolder, device protocol.DeviceID) error {
+	key := "folder/" + device.String() + "/" + id
+	return db.kv.PutKV(key, mustMarshal(of.toWire()))
 }
 
 // RemovePendingFolderForDevice removes entries for specific folder / device combinations.
-func (db *Lowlevel) RemovePendingFolderForDevice(id string, device protocol.DeviceID) error {
-	key, err := db.keyer.GeneratePendingFolderKey(nil, device[:], []byte(id))
-	if err != nil {
-		return err
-	}
-	return db.Delete(key)
+func (db *ObservedDB) RemovePendingFolderForDevice(id string, device protocol.DeviceID) error {
+	key := "folder/" + device.String() + "/" + id
+	return db.kv.DeleteKV(key)
 }
 
 // RemovePendingFolder removes all entries matching a specific folder ID.
-func (db *Lowlevel) RemovePendingFolder(id string) error {
-	iter, err := db.NewPrefixIterator([]byte{KeyTypePendingFolder})
-	if err != nil {
-		return fmt.Errorf("creating iterator: %w", err)
-	}
-	defer iter.Release()
-	var iterErr error
-	for iter.Next() {
-		if id != string(db.keyer.FolderFromPendingFolderKey(iter.Key())) {
+func (db *ObservedDB) RemovePendingFolder(id string) error {
+	it, errFn := db.kv.PrefixKV("folder/")
+	for kv := range it {
+		parts := strings.Split(kv.Key, "/")
+		if len(parts) != 3 || parts[2] != id {
 			continue
 		}
-		if err = db.Delete(iter.Key()); err != nil {
-			if iterErr != nil {
-				l.Debugf("Repeat error removing pending folder: %v", err)
-			} else {
-				iterErr = err
-			}
+		if err := db.kv.DeleteKV(kv.Key); err != nil {
+			return fmt.Errorf("delete pending folder: %w", err)
 		}
 	}
-	return iterErr
+	return errFn()
 }
 
 // Consolidated information about a pending folder
@@ -147,41 +144,37 @@ type PendingFolder struct {
 	OfferedBy map[protocol.DeviceID]ObservedFolder `json:"offeredBy"`
 }
 
-func (db *Lowlevel) PendingFolders() (map[string]PendingFolder, error) {
+func (db *ObservedDB) PendingFolders() (map[string]PendingFolder, error) {
 	return db.PendingFoldersForDevice(protocol.EmptyDeviceID)
 }
 
 // PendingFoldersForDevice enumerates only entries matching the given device ID, unless it
 // is EmptyDeviceID.  Invalid ones are dropped from the database after a info log
 // message, as a side-effect.
-func (db *Lowlevel) PendingFoldersForDevice(device protocol.DeviceID) (map[string]PendingFolder, error) {
-	var err error
-	prefixKey := []byte{KeyTypePendingFolder}
+func (db *ObservedDB) PendingFoldersForDevice(device protocol.DeviceID) (map[string]PendingFolder, error) {
+	prefix := "folder/"
 	if device != protocol.EmptyDeviceID {
-		prefixKey, err = db.keyer.GeneratePendingFolderKey(nil, device[:], nil)
-		if err != nil {
-			return nil, err
-		}
+		prefix += device.String() + "/"
 	}
-	iter, err := db.NewPrefixIterator(prefixKey)
-	if err != nil {
-		return nil, err
-	}
-	defer iter.Release()
 	res := make(map[string]PendingFolder)
-	for iter.Next() {
-		keyDev, ok := db.keyer.DeviceFromPendingFolderKey(iter.Key())
-		deviceID, err := protocol.DeviceIDFromBytes(keyDev)
+	it, errFn := db.kv.PrefixKV(prefix)
+	for kv := range it {
+		parts := strings.Split(kv.Key, "/")
+		if len(parts) != 3 {
+			continue
+		}
+		keyDev := parts[1]
+		deviceID, err := protocol.DeviceIDFromString(keyDev)
 		var protoF dbproto.ObservedFolder
 		var of ObservedFolder
 		var folderID string
-		if !ok || err != nil {
+		if err != nil {
 			goto deleteKey
 		}
-		if folderID = string(db.keyer.FolderFromPendingFolderKey(iter.Key())); len(folderID) < 1 {
+		if folderID = parts[2]; len(folderID) < 1 {
 			goto deleteKey
 		}
-		if err = proto.Unmarshal(iter.Value(), &protoF); err != nil {
+		if err = proto.Unmarshal(kv.Value, &protoF); err != nil {
 			goto deleteKey
 		}
 		if _, ok := res[folderID]; !ok {
@@ -196,10 +189,17 @@ func (db *Lowlevel) PendingFoldersForDevice(device protocol.DeviceID) (map[strin
 		// Deleting invalid entries is the only possible "repair" measure and
 		// appropriate for the importance of pending entries.  They will come back
 		// soon if still relevant.
-		l.Infof("Invalid pending folder entry, deleting from database: %x", iter.Key())
-		if err := db.Delete(iter.Key()); err != nil {
-			return nil, err
+		if err := db.kv.DeleteKV(kv.Key); err != nil {
+			return nil, fmt.Errorf("delete invalid pending folder: %w", err)
 		}
 	}
-	return res, nil
+	return res, errFn()
+}
+
+func mustMarshal(m proto.Message) []byte {
+	bs, err := proto.Marshal(m)
+	if err != nil {
+		panic(err)
+	}
+	return bs
 }

internal/db/olddb/backend/backend.go

@@ -108,29 +108,8 @@ type Iterator interface {
 // is empty for a db in memory.
 type Backend interface {
 	Reader
-	Writer
 	NewReadTransaction() (ReadTransaction, error)
-	NewWriteTransaction(hooks ...CommitHook) (WriteTransaction, error)
 	Close() error
-	Compact() error
-	Location() string
-}
-
-type Tuning int
-
-const (
-	// N.b. these constants must match those in lib/config.Tuning!
-	TuningAuto Tuning = iota
-	TuningSmall
-	TuningLarge
-)
-
-func Open(path string, tuning Tuning) (Backend, error) {
-	return OpenLevelDB(path, tuning)
-}
-
-func OpenMemory() Backend {
-	return OpenLevelDBMemory()
 }
 
 var (

internal/db/olddb/backend/leveldb_backend.go

@@ -0,0 +1,113 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// leveldbBackend implements Backend on top of a leveldb
+type leveldbBackend struct {
+	ldb      *leveldb.DB
+	closeWG  *closeWaitGroup
+	location string
+}
+
+func newLeveldbBackend(ldb *leveldb.DB, location string) *leveldbBackend {
+	return &leveldbBackend{
+		ldb:      ldb,
+		closeWG:  &closeWaitGroup{},
+		location: location,
+	}
+}
+
+func (b *leveldbBackend) NewReadTransaction() (ReadTransaction, error) {
+	return b.newSnapshot()
+}
+
+func (b *leveldbBackend) newSnapshot() (leveldbSnapshot, error) {
+	rel, err := newReleaser(b.closeWG)
+	if err != nil {
+		return leveldbSnapshot{}, err
+	}
+	snap, err := b.ldb.GetSnapshot()
+	if err != nil {
+		rel.Release()
+		return leveldbSnapshot{}, wrapLeveldbErr(err)
+	}
+	return leveldbSnapshot{
+		snap: snap,
+		rel:  rel,
+	}, nil
+}
+
+func (b *leveldbBackend) Close() error {
+	b.closeWG.CloseWait()
+	return wrapLeveldbErr(b.ldb.Close())
+}
+
+func (b *leveldbBackend) Get(key []byte) ([]byte, error) {
+	val, err := b.ldb.Get(key, nil)
+	return val, wrapLeveldbErr(err)
+}
+
+func (b *leveldbBackend) NewPrefixIterator(prefix []byte) (Iterator, error) {
+	return &leveldbIterator{b.ldb.NewIterator(util.BytesPrefix(prefix), nil)}, nil
+}
+
+func (b *leveldbBackend) NewRangeIterator(first, last []byte) (Iterator, error) {
+	return &leveldbIterator{b.ldb.NewIterator(&util.Range{Start: first, Limit: last}, nil)}, nil
+}
+
+func (b *leveldbBackend) Location() string {
+	return b.location
+}
+
+// leveldbSnapshot implements backend.ReadTransaction
+type leveldbSnapshot struct {
+	snap *leveldb.Snapshot
+	rel  *releaser
+}
+
+func (l leveldbSnapshot) Get(key []byte) ([]byte, error) {
+	val, err := l.snap.Get(key, nil)
+	return val, wrapLeveldbErr(err)
+}
+
+func (l leveldbSnapshot) NewPrefixIterator(prefix []byte) (Iterator, error) {
+	return l.snap.NewIterator(util.BytesPrefix(prefix), nil), nil
+}
+
+func (l leveldbSnapshot) NewRangeIterator(first, last []byte) (Iterator, error) {
+	return l.snap.NewIterator(&util.Range{Start: first, Limit: last}, nil), nil
+}
+
+func (l leveldbSnapshot) Release() {
+	l.snap.Release()
+	l.rel.Release()
+}
+
+type leveldbIterator struct {
+	iterator.Iterator
+}
+
+func (it *leveldbIterator) Error() error {
+	return wrapLeveldbErr(it.Iterator.Error())
+}
+
+// wrapLeveldbErr wraps errors so that the backend package can recognize them
+func wrapLeveldbErr(err error) error {
+	switch err {
+	case leveldb.ErrClosed:
+		return errClosed
+	case leveldb.ErrNotFound:
+		return errNotFound
+	}
+	return err
+}

internal/db/olddb/backend/leveldb_open.go

@@ -0,0 +1,32 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+const dbMaxOpenFiles = 100
+
+// OpenLevelDBRO attempts to open the database at the given location, read
+// only.
+func OpenLevelDBRO(location string) (Backend, error) {
+	opts := &opt.Options{
+		OpenFilesCacheCapacity: dbMaxOpenFiles,
+		ReadOnly:               true,
+	}
+	ldb, err := open(location, opts)
+	if err != nil {
+		return nil, err
+	}
+	return newLeveldbBackend(ldb, location), nil
+}
+
+func open(location string, opts *opt.Options) (*leveldb.DB, error) {
+	return leveldb.OpenFile(location, opts)
+}

internal/db/olddb/keyer.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package db
+package olddb
 
 import (
 	"encoding/binary"

internal/db/olddb/lowlevel.go

@@ -0,0 +1,70 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package olddb
+
+import (
+	"encoding/binary"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/db/olddb/backend"
+)
+
+// deprecatedLowlevel is the lowest level database interface. It has a very simple
+// purpose: hold the actual backend database, and the in-memory state
+// that belong to that database. In the same way that a single on disk
+// database can only be opened once, there should be only one deprecatedLowlevel for
+// any given backend.
+type deprecatedLowlevel struct {
+	backend.Backend
+	folderIdx *smallIndex
+	deviceIdx *smallIndex
+	keyer     keyer
+}
+
+func NewLowlevel(backend backend.Backend) (*deprecatedLowlevel, error) {
+	// Only log restarts in debug mode.
+	db := &deprecatedLowlevel{
+		Backend:   backend,
+		folderIdx: newSmallIndex(backend, []byte{KeyTypeFolderIdx}),
+		deviceIdx: newSmallIndex(backend, []byte{KeyTypeDeviceIdx}),
+	}
+	db.keyer = newDefaultKeyer(db.folderIdx, db.deviceIdx)
+	return db, nil
+}
+
+// ListFolders returns the list of folders currently in the database
+func (db *deprecatedLowlevel) ListFolders() []string {
+	return db.folderIdx.Values()
+}
+
+func (db *deprecatedLowlevel) IterateMtimes(fn func(folder, name string, ondisk, virtual time.Time) error) error {
+	it, err := db.NewPrefixIterator([]byte{KeyTypeVirtualMtime})
+	if err != nil {
+		return err
+	}
+	defer it.Release()
+	for it.Next() {
+		key := it.Key()[1:]
+		folderID, ok := db.folderIdx.Val(binary.BigEndian.Uint32(key))
+		if !ok {
+			continue
+		}
+		name := key[4:]
+		val := it.Value()
+		var ondisk, virtual time.Time
+		if err := ondisk.UnmarshalBinary(val[:len(val)/2]); err != nil {
+			continue
+		}
+		if err := virtual.UnmarshalBinary(val[len(val)/2:]); err != nil {
+			continue
+		}
+		if err := fn(string(folderID), string(name), ondisk, virtual); err != nil {
+			return err
+		}
+	}
+	return it.Error()
+}

internal/db/olddb/set.go

@@ -0,0 +1,67 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Package db provides a set type to track local/remote files with newness
+// checks. We must do a certain amount of normalization in here. We will get
+// fed paths with either native or wire-format separators and encodings
+// depending on who calls us. We transform paths to wire-format (NFC and
+// slashes) on the way to the database, and transform to native format
+// (varying separator and encoding) on the way back out.
+package olddb
+
+import (
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type deprecatedFileSet struct {
+	folder string
+	db     *deprecatedLowlevel
+}
+
+// The Iterator is called with either a protocol.FileInfo or a
+// FileInfoTruncated (depending on the method) and returns true to
+// continue iteration, false to stop.
+type Iterator func(f protocol.FileInfo) bool
+
+func NewFileSet(folder string, db *deprecatedLowlevel) (*deprecatedFileSet, error) {
+	s := &deprecatedFileSet{
+		folder: folder,
+		db:     db,
+	}
+	return s, nil
+}
+
+type Snapshot struct {
+	folder string
+	t      readOnlyTransaction
+}
+
+func (s *deprecatedFileSet) Snapshot() (*Snapshot, error) {
+	t, err := s.db.newReadOnlyTransaction()
+	if err != nil {
+		return nil, err
+	}
+	return &Snapshot{
+		folder: s.folder,
+		t:      t,
+	}, nil
+}
+
+func (s *Snapshot) Release() {
+	s.t.close()
+}
+
+func (s *Snapshot) WithHaveSequence(startSeq int64, fn Iterator) error {
+	return s.t.withHaveSequence([]byte(s.folder), startSeq, nativeFileIterator(fn))
+}
+
+func nativeFileIterator(fn Iterator) Iterator {
+	return func(fi protocol.FileInfo) bool {
+		fi.Name = osutil.NativeFilename(fi.Name)
+		return fn(fi)
+	}
+}

internal/db/olddb/smallindex.go

@@ -4,13 +4,13 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package db
+package olddb
 
 import (
 	"encoding/binary"
 	"sort"
 
-	"github.com/syncthing/syncthing/lib/db/backend"
+	"github.com/syncthing/syncthing/internal/db/olddb/backend"
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
@@ -74,23 +74,7 @@ func (i *smallIndex) ID(val []byte) (uint32, error) {
 		return id, nil
 	}
 
-	id := i.nextID
-	i.nextID++
-
-	valStr := string(val)
-	i.val2id[valStr] = id
-	i.id2val[id] = valStr
-
-	key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
-	copy(key, i.prefix)
-	binary.BigEndian.PutUint32(key[len(i.prefix):], id)
-	if err := i.db.Put(key, val); err != nil {
-		i.mut.Unlock()
-		return 0, err
-	}
-
-	i.mut.Unlock()
-	return id, nil
+	panic("missing ID")
 }
 
 // Val returns the value for the given index number, or (nil, false) if there
@@ -106,33 +90,6 @@ func (i *smallIndex) Val(id uint32) ([]byte, bool) {
 	return []byte(val), true
 }
 
-func (i *smallIndex) Delete(val []byte) error {
-	i.mut.Lock()
-	defer i.mut.Unlock()
-
-	// Check the reverse mapping to get the ID for the value.
-	if id, ok := i.val2id[string(val)]; ok {
-		// Generate the corresponding database key.
-		key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
-		copy(key, i.prefix)
-		binary.BigEndian.PutUint32(key[len(i.prefix):], id)
-
-		// Put an empty value into the database. This indicates that the
-		// entry does not exist any more and prevents the ID from being
-		// reused in the future.
-		if err := i.db.Put(key, []byte{}); err != nil {
-			return err
-		}
-
-		// Delete reverse mapping.
-		delete(i.id2val, id)
-	}
-
-	// Delete forward mapping.
-	delete(i.val2id, string(val))
-	return nil
-}
-
 // Values returns the set of values in the index
 func (i *smallIndex) Values() []string {
 	// In principle this method should return [][]byte because all the other

internal/db/olddb/transactions.go

@@ -0,0 +1,193 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package olddb
+
+import (
+	"fmt"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/syncthing/syncthing/internal/db/olddb/backend"
+	"github.com/syncthing/syncthing/internal/gen/bep"
+	"github.com/syncthing/syncthing/internal/gen/dbproto"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+// A readOnlyTransaction represents a database snapshot.
+type readOnlyTransaction struct {
+	backend.ReadTransaction
+	keyer keyer
+}
+
+func (db *deprecatedLowlevel) newReadOnlyTransaction() (readOnlyTransaction, error) {
+	tran, err := db.NewReadTransaction()
+	if err != nil {
+		return readOnlyTransaction{}, err
+	}
+	return db.readOnlyTransactionFromBackendTransaction(tran), nil
+}
+
+func (db *deprecatedLowlevel) readOnlyTransactionFromBackendTransaction(tran backend.ReadTransaction) readOnlyTransaction {
+	return readOnlyTransaction{
+		ReadTransaction: tran,
+		keyer:           db.keyer,
+	}
+}
+
+func (t readOnlyTransaction) close() {
+	t.Release()
+}
+
+func (t readOnlyTransaction) getFileByKey(key []byte) (protocol.FileInfo, bool, error) {
+	f, ok, err := t.getFileTrunc(key, false)
+	if err != nil || !ok {
+		return protocol.FileInfo{}, false, err
+	}
+	return f, true, nil
+}
+
+func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (protocol.FileInfo, bool, error) {
+	bs, err := t.Get(key)
+	if backend.IsNotFound(err) {
+		return protocol.FileInfo{}, false, nil
+	}
+	if err != nil {
+		return protocol.FileInfo{}, false, err
+	}
+	f, err := t.unmarshalTrunc(bs, trunc)
+	if backend.IsNotFound(err) {
+		return protocol.FileInfo{}, false, nil
+	}
+	if err != nil {
+		return protocol.FileInfo{}, false, err
+	}
+	return f, true, nil
+}
+
+func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (protocol.FileInfo, error) {
+	if trunc {
+		var bfi dbproto.FileInfoTruncated
+		err := proto.Unmarshal(bs, &bfi)
+		if err != nil {
+			return protocol.FileInfo{}, err
+		}
+		if err := t.fillTruncated(&bfi); err != nil {
+			return protocol.FileInfo{}, err
+		}
+		return protocol.FileInfoFromDBTruncated(&bfi), nil
+	}
+
+	var bfi bep.FileInfo
+	err := proto.Unmarshal(bs, &bfi)
+	if err != nil {
+		return protocol.FileInfo{}, err
+	}
+	if err := t.fillFileInfo(&bfi); err != nil {
+		return protocol.FileInfo{}, err
+	}
+	return protocol.FileInfoFromDB(&bfi), nil
+}
+
+type blocksIndirectionError struct {
+	err error
+}
+
+func (e *blocksIndirectionError) Error() string {
+	return fmt.Sprintf("filling Blocks: %v", e.err)
+}
+
+func (e *blocksIndirectionError) Unwrap() error {
+	return e.err
+}
+
+// fillFileInfo follows the (possible) indirection of blocks and version
+// vector and fills it out.
+func (t readOnlyTransaction) fillFileInfo(fi *bep.FileInfo) error {
+	var key []byte
+
+	if len(fi.Blocks) == 0 && len(fi.BlocksHash) != 0 {
+		// The blocks list is indirected and we need to load it.
+		key = t.keyer.GenerateBlockListKey(key, fi.BlocksHash)
+		bs, err := t.Get(key)
+		if err != nil {
+			return &blocksIndirectionError{err}
+		}
+		var bl dbproto.BlockList
+		if err := proto.Unmarshal(bs, &bl); err != nil {
+			return err
+		}
+		fi.Blocks = bl.Blocks
+	}
+
+	if len(fi.VersionHash) != 0 {
+		key = t.keyer.GenerateVersionKey(key, fi.VersionHash)
+		bs, err := t.Get(key)
+		if err != nil {
+			return fmt.Errorf("filling Version: %w", err)
+		}
+		var v bep.Vector
+		if err := proto.Unmarshal(bs, &v); err != nil {
+			return err
+		}
+		fi.Version = &v
+	}
+
+	return nil
+}
+
+// fillTruncated follows the (possible) indirection of version vector and
+// fills it.
+func (t readOnlyTransaction) fillTruncated(fi *dbproto.FileInfoTruncated) error {
+	var key []byte
+
+	if len(fi.VersionHash) == 0 {
+		return nil
+	}
+
+	key = t.keyer.GenerateVersionKey(key, fi.VersionHash)
+	bs, err := t.Get(key)
+	if err != nil {
+		return err
+	}
+	var v bep.Vector
+	if err := proto.Unmarshal(bs, &v); err != nil {
+		return err
+	}
+	fi.Version = &v
+	return nil
+}
+
+func (t *readOnlyTransaction) withHaveSequence(folder []byte, startSeq int64, fn Iterator) error {
+	first, err := t.keyer.GenerateSequenceKey(nil, folder, startSeq)
+	if err != nil {
+		return err
+	}
+	last, err := t.keyer.GenerateSequenceKey(nil, folder, maxInt64)
+	if err != nil {
+		return err
+	}
+	dbi, err := t.NewRangeIterator(first, last)
+	if err != nil {
+		return err
+	}
+	defer dbi.Release()
+
+	for dbi.Next() {
+		f, ok, err := t.getFileByKey(dbi.Value())
+		if err != nil {
+			return err
+		}
+		if !ok {
+			continue
+		}
+
+		if !fn(f) {
+			return nil
+		}
+	}
+	return dbi.Error()
+}

internal/db/sqlite/db.go

@@ -0,0 +1,88 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"sync"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/syncthing/syncthing/internal/db"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/thejerf/suture/v4"
+)
+
+type DB struct {
+	sql             *sqlx.DB
+	localDeviceIdx  int64
+	deleteRetention time.Duration
+
+	updateLock   sync.Mutex
+	updatePoints int
+
+	statementsMut sync.RWMutex
+	statements    map[string]*sqlx.Stmt
+	tplInput      map[string]any
+}
+
+var _ db.DB = (*DB)(nil)
+
+type Option func(*DB)
+
+func WithDeleteRetention(d time.Duration) Option {
+	return func(s *DB) {
+		s.deleteRetention = d
+	}
+}
+
+func (s *DB) Close() error {
+	s.updateLock.Lock()
+	s.statementsMut.Lock()
+	defer s.updateLock.Unlock()
+	defer s.statementsMut.Unlock()
+	for _, stmt := range s.statements {
+		stmt.Close()
+	}
+	return wrap(s.sql.Close())
+}
+
+func (s *DB) Service(maintenanceInterval time.Duration) suture.Service {
+	return newService(s, maintenanceInterval)
+}
+
+func (s *DB) ListFolders() ([]string, error) {
+	var res []string
+	err := s.stmt(`
+		SELECT folder_id FROM folders
+		ORDER BY folder_id
+	`).Select(&res)
+	return res, wrap(err)
+}
+
+func (s *DB) ListDevicesForFolder(folder string) ([]protocol.DeviceID, error) {
+	var res []string
+	err := s.stmt(`
+		SELECT d.device_id FROM counts s
+		INNER JOIN folders o ON o.idx = s.folder_idx
+		INNER JOIN devices d ON d.idx = s.device_idx
+		WHERE o.folder_id = ? AND s.count > 0 AND s.device_idx != {{.LocalDeviceIdx}}
+		GROUP BY d.device_id
+		ORDER BY d.device_id
+	`).Select(&res, folder)
+	if err != nil {
+		return nil, wrap(err)
+	}
+
+	devs := make([]protocol.DeviceID, len(res))
+	for i, s := range res {
+		devs[i], err = protocol.DeviceIDFromString(s)
+		if err != nil {
+			return nil, wrap(err)
+		}
+	}
+	return devs, nil
+}

internal/db/sqlite/db_bench_test.go

@@ -0,0 +1,243 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/timeutil"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/rand"
+)
+
+var globalFi protocol.FileInfo
+
+func BenchmarkUpdate(b *testing.B) {
+	db, err := OpenTemp()
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			b.Fatal(err)
+		}
+	})
+	svc := db.Service(time.Hour).(*Service)
+
+	fs := make([]protocol.FileInfo, 100)
+	seed := 0
+
+	size := 10000
+	for size < 200_000 {
+		t0 := time.Now()
+		if err := svc.periodic(context.Background()); err != nil {
+			b.Fatal(err)
+		}
+		b.Log("garbage collect in", time.Since(t0))
+
+		for {
+			local, err := db.CountLocal(folderID, protocol.LocalDeviceID)
+			if err != nil {
+				b.Fatal(err)
+			}
+			if local.Files >= size {
+				break
+			}
+			fs := make([]protocol.FileInfo, 1000)
+			for i := range fs {
+				fs[i] = genFile(rand.String(24), 64, 0)
+			}
+			if err := db.Update(folderID, protocol.LocalDeviceID, fs); err != nil {
+				b.Fatal(err)
+			}
+		}
+
+		b.Run(fmt.Sprintf("Insert100Loc@%d", size), func(b *testing.B) {
+			for range b.N {
+				for i := range fs {
+					fs[i] = genFile(rand.String(24), 64, 0)
+				}
+				if err := db.Update(folderID, protocol.LocalDeviceID, fs); err != nil {
+					b.Fatal(err)
+				}
+			}
+			b.ReportMetric(float64(b.N)*100.0/b.Elapsed().Seconds(), "files/s")
+		})
+
+		b.Run(fmt.Sprintf("RepBlocks100@%d", size), func(b *testing.B) {
+			for range b.N {
+				for i := range fs {
+					fs[i].Blocks = genBlocks(fs[i].Name, seed, 64)
+					fs[i].Version = fs[i].Version.Update(42)
+				}
+				seed++
+				if err := db.Update(folderID, protocol.LocalDeviceID, fs); err != nil {
+					b.Fatal(err)
+				}
+			}
+			b.ReportMetric(float64(b.N)*100.0/b.Elapsed().Seconds(), "files/s")
+		})
+
+		b.Run(fmt.Sprintf("RepSame100@%d", size), func(b *testing.B) {
+			for range b.N {
+				for i := range fs {
+					fs[i].Version = fs[i].Version.Update(42)
+				}
+				if err := db.Update(folderID, protocol.LocalDeviceID, fs); err != nil {
+					b.Fatal(err)
+				}
+			}
+			b.ReportMetric(float64(b.N)*100.0/b.Elapsed().Seconds(), "files/s")
+		})
+
+		b.Run(fmt.Sprintf("Insert100Rem@%d", size), func(b *testing.B) {
+			for range b.N {
+				for i := range fs {
+					fs[i].Blocks = genBlocks(fs[i].Name, seed, 64)
+					fs[i].Version = fs[i].Version.Update(42)
+					fs[i].Sequence = timeutil.StrictlyMonotonicNanos()
+				}
+				if err := db.Update(folderID, protocol.DeviceID{42}, fs); err != nil {
+					b.Fatal(err)
+				}
+			}
+			b.ReportMetric(float64(b.N)*100.0/b.Elapsed().Seconds(), "files/s")
+		})
+
+		b.Run(fmt.Sprintf("GetGlobal100@%d", size), func(b *testing.B) {
+			for range b.N {
+				for i := range fs {
+					_, ok, err := db.GetGlobalFile(folderID, fs[i].Name)
+					if err != nil {
+						b.Fatal(err)
+					}
+					if !ok {
+						b.Fatal("should exist")
+					}
+				}
+			}
+			b.ReportMetric(float64(b.N)*100.0/b.Elapsed().Seconds(), "files/s")
+		})
+
+		b.Run(fmt.Sprintf("LocalSequenced@%d", size), func(b *testing.B) {
+			count := 0
+			for range b.N {
+				cur, err := db.GetDeviceSequence(folderID, protocol.LocalDeviceID)
+				if err != nil {
+					b.Fatal(err)
+				}
+				it, errFn := db.AllLocalFilesBySequence(folderID, protocol.LocalDeviceID, cur-100, 0)
+				for f := range it {
+					count++
+					globalFi = f
+				}
+				if err := errFn(); err != nil {
+					b.Fatal(err)
+				}
+			}
+			b.ReportMetric(float64(count)/b.Elapsed().Seconds(), "files/s")
+		})
+
+		b.Run(fmt.Sprintf("GetDeviceSequenceLoc@%d", size), func(b *testing.B) {
+			for range b.N {
+				_, err := db.GetDeviceSequence(folderID, protocol.LocalDeviceID)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+		b.Run(fmt.Sprintf("GetDeviceSequenceRem@%d", size), func(b *testing.B) {
+			for range b.N {
+				_, err := db.GetDeviceSequence(folderID, protocol.DeviceID{42})
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+
+		b.Run(fmt.Sprintf("RemoteNeed@%d", size), func(b *testing.B) {
+			count := 0
+			for range b.N {
+				it, errFn := db.AllNeededGlobalFiles(folderID, protocol.DeviceID{42}, config.PullOrderAlphabetic, 0, 0)
+				for f := range it {
+					count++
+					globalFi = f
+				}
+				if err := errFn(); err != nil {
+					b.Fatal(err)
+				}
+			}
+			b.ReportMetric(float64(count)/b.Elapsed().Seconds(), "files/s")
+		})
+
+		b.Run(fmt.Sprintf("LocalNeed100Largest@%d", size), func(b *testing.B) {
+			count := 0
+			for range b.N {
+				it, errFn := db.AllNeededGlobalFiles(folderID, protocol.LocalDeviceID, config.PullOrderLargestFirst, 100, 0)
+				for f := range it {
+					globalFi = f
+					count++
+				}
+				if err := errFn(); err != nil {
+					b.Fatal(err)
+				}
+			}
+			b.ReportMetric(float64(count)/b.Elapsed().Seconds(), "files/s")
+		})
+
+		size <<= 1
+	}
+}
+
+func TestBenchmarkDropAllRemote(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow test")
+	}
+
+	db, err := OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	fs := make([]protocol.FileInfo, 1000)
+	seq := 0
+	for {
+		local, err := db.CountLocal(folderID, protocol.LocalDeviceID)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if local.Files >= 15_000 {
+			break
+		}
+		for i := range fs {
+			seq++
+			fs[i] = genFile(rand.String(24), 64, seq)
+		}
+		if err := db.Update(folderID, protocol.DeviceID{42}, fs); err != nil {
+			t.Fatal(err)
+		}
+		if err := db.Update(folderID, protocol.LocalDeviceID, fs); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	t0 := time.Now()
+	if err := db.DropAllFiles(folderID, protocol.DeviceID{42}); err != nil {
+		t.Fatal(err)
+	}
+	d := time.Since(t0)
+	t.Log("drop all took", d)
+}

internal/db/sqlite/db_counts.go

@@ -0,0 +1,137 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"github.com/syncthing/syncthing/internal/db"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type countsRow struct {
+	Type       protocol.FileInfoType
+	Count      int
+	Size       int64
+	Deleted    bool
+	LocalFlags int64 `db:"local_flags"`
+}
+
+func (s *DB) CountLocal(folder string, device protocol.DeviceID) (db.Counts, error) {
+	var res []countsRow
+	if err := s.stmt(`
+		SELECT s.type, s.count, s.size, s.local_flags, s.deleted FROM counts s
+		INNER JOIN folders o ON o.idx = s.folder_idx
+		INNER JOIN devices d ON d.idx = s.device_idx
+		WHERE o.folder_id = ? AND d.device_id = ? AND s.local_flags & {{.FlagLocalIgnored}} = 0
+	`).Select(&res, folder, device.String()); err != nil {
+		return db.Counts{}, wrap(err)
+	}
+	return summarizeCounts(res), nil
+}
+
+func (s *DB) CountNeed(folder string, device protocol.DeviceID) (db.Counts, error) {
+	if device == protocol.LocalDeviceID {
+		return s.needSizeLocal(folder)
+	}
+	return s.needSizeRemote(folder, device)
+}
+
+func (s *DB) CountGlobal(folder string) (db.Counts, error) {
+	// Exclude ignored and receive-only changed files from the global count
+	// (legacy expectation? it's a bit weird since those files can in fact
+	// be global and you can get them with GetGlobal etc.)
+	var res []countsRow
+	err := s.stmt(`
+		SELECT s.type, s.count, s.size, s.local_flags, s.deleted FROM counts s
+		INNER JOIN folders o ON o.idx = s.folder_idx
+		WHERE o.folder_id = ? AND s.local_flags & {{.FlagLocalGlobal}} != 0 AND s.local_flags & {{or .FlagLocalReceiveOnly .FlagLocalIgnored}} = 0
+	`).Select(&res, folder)
+	if err != nil {
+		return db.Counts{}, wrap(err)
+	}
+	return summarizeCounts(res), nil
+}
+
+func (s *DB) CountReceiveOnlyChanged(folder string) (db.Counts, error) {
+	var res []countsRow
+	err := s.stmt(`
+		SELECT s.type, s.count, s.size, s.local_flags, s.deleted FROM counts s
+		INNER JOIN folders o ON o.idx = s.folder_idx
+		WHERE o.folder_id = ? AND local_flags & {{.FlagLocalReceiveOnly}} != 0
+	`).Select(&res, folder)
+	if err != nil {
+		return db.Counts{}, wrap(err)
+	}
+	return summarizeCounts(res), nil
+}
+
+func (s *DB) needSizeLocal(folder string) (db.Counts, error) {
+	// The need size for the local device is the sum of entries with the
+	// need bit set.
+	var res []countsRow
+	err := s.stmt(`
+		SELECT s.type, s.count, s.size, s.local_flags, s.deleted FROM counts s
+		INNER JOIN folders o ON o.idx = s.folder_idx
+		WHERE o.folder_id = ? AND s.local_flags & {{.FlagLocalNeeded}} != 0
+	`).Select(&res, folder)
+	if err != nil {
+		return db.Counts{}, wrap(err)
+	}
+	return summarizeCounts(res), nil
+}
+
+func (s *DB) needSizeRemote(folder string, device protocol.DeviceID) (db.Counts, error) {
+	var res []countsRow
+	// See neededGlobalFilesRemote for commentary as that is the same query without summing
+	if err := s.stmt(`
+		SELECT g.type, count(*) as count, sum(g.size) as size, g.local_flags, g.deleted FROM files g
+		INNER JOIN folders o ON o.idx = g.folder_idx
+		WHERE o.folder_id = ? AND g.local_flags & {{.FlagLocalGlobal}} != 0 AND NOT g.deleted AND NOT g.invalid AND NOT EXISTS (
+			SELECT 1 FROM FILES f
+			INNER JOIN devices d ON d.idx = f.device_idx
+			WHERE f.name = g.name AND f.version = g.version AND f.folder_idx = g.folder_idx AND d.device_id = ?
+		)
+		GROUP BY g.type, g.local_flags, g.deleted
+
+		UNION ALL
+
+		SELECT g.type, count(*) as count, sum(g.size) as size, g.local_flags, g.deleted FROM files g
+		INNER JOIN folders o ON o.idx = g.folder_idx
+		WHERE o.folder_id = ? AND g.local_flags & {{.FlagLocalGlobal}} != 0 AND g.deleted AND NOT g.invalid AND EXISTS (
+			SELECT 1 FROM FILES f
+			INNER JOIN devices d ON d.idx = f.device_idx
+			WHERE f.name = g.name AND f.folder_idx = g.folder_idx AND d.device_id = ? AND NOT f.deleted
+		)
+		GROUP BY g.type, g.local_flags, g.deleted
+	`).Select(&res, folder, device.String(),
+		folder, device.String()); err != nil {
+		return db.Counts{}, wrap(err)
+	}
+
+	return summarizeCounts(res), nil
+}
+
+func summarizeCounts(res []countsRow) db.Counts {
+	c := db.Counts{
+		DeviceID: protocol.LocalDeviceID,
+	}
+	for _, r := range res {
+		switch {
+		case r.Deleted:
+			c.Deleted += r.Count
+		case r.Type == protocol.FileInfoTypeFile:
+			c.Files += r.Count
+			c.Bytes += r.Size
+		case r.Type == protocol.FileInfoTypeDirectory:
+			c.Directories += r.Count
+			c.Bytes += r.Size
+		case r.Type == protocol.FileInfoTypeSymlink:
+			c.Symlinks += r.Count
+			c.Bytes += r.Size
+		}
+	}
+	return c
+}

internal/db/sqlite/db_global.go

@@ -0,0 +1,189 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"iter"
+
+	"github.com/syncthing/syncthing/internal/db"
+	"github.com/syncthing/syncthing/internal/itererr"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func (s *DB) GetGlobalFile(folder string, file string) (protocol.FileInfo, bool, error) {
+	file = osutil.NormalizedFilename(file)
+
+	var ind indirectFI
+	err := s.stmt(`
+		SELECT fi.fiprotobuf, bl.blprotobuf FROM fileinfos fi
+		INNER JOIN files f on fi.sequence = f.sequence
+		LEFT JOIN blocklists bl ON bl.blocklist_hash = f.blocklist_hash
+		INNER JOIN folders o ON o.idx = f.folder_idx
+		WHERE o.folder_id = ? AND f.name = ? AND f.local_flags & {{.FlagLocalGlobal}} != 0
+	`).Get(&ind, folder, file)
+	if errors.Is(err, sql.ErrNoRows) {
+		return protocol.FileInfo{}, false, nil
+	}
+	if err != nil {
+		return protocol.FileInfo{}, false, wrap(err)
+	}
+	fi, err := ind.FileInfo()
+	if err != nil {
+		return protocol.FileInfo{}, false, wrap(err)
+	}
+	return fi, true, nil
+}
+
+func (s *DB) GetGlobalAvailability(folder, file string) ([]protocol.DeviceID, error) {
+	file = osutil.NormalizedFilename(file)
+
+	var devStrs []string
+	err := s.stmt(`
+		SELECT d.device_id FROM files f
+		INNER JOIN devices d ON d.idx = f.device_idx
+		INNER JOIN folders o ON o.idx = f.folder_idx
+		INNER JOIN files g ON f.folder_idx = g.folder_idx AND g.version = f.version AND g.name = f.name
+		WHERE o.folder_id = ? AND g.name = ? AND g.local_flags & {{.FlagLocalGlobal}} != 0 AND f.device_idx != {{.LocalDeviceIdx}}
+		ORDER BY d.device_id
+	`).Select(&devStrs, folder, file)
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, wrap(err)
+	}
+
+	devs := make([]protocol.DeviceID, 0, len(devStrs))
+	for _, s := range devStrs {
+		d, err := protocol.DeviceIDFromString(s)
+		if err != nil {
+			return nil, wrap(err)
+		}
+		devs = append(devs, d)
+	}
+
+	return devs, nil
+}
+
+func (s *DB) AllGlobalFiles(folder string) (iter.Seq[db.FileMetadata], func() error) {
+	it, errFn := iterStructs[db.FileMetadata](s.stmt(`
+		SELECT f.sequence, f.name, f.type, f.modified as modnanos, f.size, f.deleted, f.invalid, f.local_flags as localflags FROM files f
+		INNER JOIN folders o ON o.idx = f.folder_idx
+		WHERE o.folder_id = ? AND f.local_flags & {{.FlagLocalGlobal}} != 0
+		ORDER BY f.name
+	`).Queryx(folder))
+	return itererr.Map(it, errFn, func(m db.FileMetadata) (db.FileMetadata, error) {
+		m.Name = osutil.NativeFilename(m.Name)
+		return m, nil
+	})
+}
+
+func (s *DB) AllGlobalFilesPrefix(folder string, prefix string) (iter.Seq[db.FileMetadata], func() error) {
+	if prefix == "" {
+		return s.AllGlobalFiles(folder)
+	}
+
+	prefix = osutil.NormalizedFilename(prefix)
+	end := prefixEnd(prefix)
+
+	it, errFn := iterStructs[db.FileMetadata](s.stmt(`
+		SELECT f.sequence, f.name, f.type, f.modified as modnanos, f.size, f.deleted, f.invalid, f.local_flags as localflags FROM files f
+		INNER JOIN folders o ON o.idx = f.folder_idx
+		WHERE o.folder_id = ? AND f.name >= ? AND f.name < ? AND f.local_flags & {{.FlagLocalGlobal}} != 0
+		ORDER BY f.name
+	`).Queryx(folder, prefix, end))
+	return itererr.Map(it, errFn, func(m db.FileMetadata) (db.FileMetadata, error) {
+		m.Name = osutil.NativeFilename(m.Name)
+		return m, nil
+	})
+}
+
+func (s *DB) AllNeededGlobalFiles(folder string, device protocol.DeviceID, order config.PullOrder, limit, offset int) (iter.Seq[protocol.FileInfo], func() error) {
+	var selectOpts string
+	switch order {
+	case config.PullOrderRandom:
+		selectOpts = "ORDER BY RANDOM()"
+	case config.PullOrderAlphabetic:
+		selectOpts = "ORDER BY g.name ASC"
+	case config.PullOrderSmallestFirst:
+		selectOpts = "ORDER BY g.size ASC"
+	case config.PullOrderLargestFirst:
+		selectOpts = "ORDER BY g.size DESC"
+	case config.PullOrderOldestFirst:
+		selectOpts = "ORDER BY g.modified ASC"
+	case config.PullOrderNewestFirst:
+		selectOpts = "ORDER BY g.modified DESC"
+	}
+
+	if limit > 0 {
+		selectOpts += fmt.Sprintf(" LIMIT %d", limit)
+	}
+	if offset > 0 {
+		selectOpts += fmt.Sprintf(" OFFSET %d", offset)
+	}
+
+	if device == protocol.LocalDeviceID {
+		return s.neededGlobalFilesLocal(folder, selectOpts)
+	}
+
+	return s.neededGlobalFilesRemote(folder, device, selectOpts)
+}
+
+func (s *DB) neededGlobalFilesLocal(folder, selectOpts string) (iter.Seq[protocol.FileInfo], func() error) {
+	// Select all the non-ignored files with the need bit set.
+	it, errFn := iterStructs[indirectFI](s.stmt(`
+		SELECT fi.fiprotobuf, bl.blprotobuf, g.name, g.size, g.modified FROM fileinfos fi
+		INNER JOIN files g on fi.sequence = g.sequence
+		LEFT JOIN blocklists bl ON bl.blocklist_hash = g.blocklist_hash
+		INNER JOIN folders o ON o.idx = g.folder_idx
+		WHERE o.folder_id = ? AND g.local_flags & {{.FlagLocalIgnored}} = 0 AND g.local_flags & {{.FlagLocalNeeded}} != 0
+	` + selectOpts).Queryx(folder))
+	return itererr.Map(it, errFn, indirectFI.FileInfo)
+}
+
+func (s *DB) neededGlobalFilesRemote(folder string, device protocol.DeviceID, selectOpts string) (iter.Seq[protocol.FileInfo], func() error) {
+	// Select:
+	//
+	// - all the valid, non-deleted global files that don't have a corresponding
+	//   remote file with the same version.
+	//
+	// - all the valid, deleted global files that have a corresponding non-deleted
+	//   remote file (of any version)
+
+	it, errFn := iterStructs[indirectFI](s.stmt(`
+		SELECT fi.fiprotobuf, bl.blprotobuf, g.name, g.size, g.modified FROM fileinfos fi
+		INNER JOIN files g on fi.sequence = g.sequence
+		LEFT JOIN blocklists bl ON bl.blocklist_hash = g.blocklist_hash
+		INNER JOIN folders o ON o.idx = g.folder_idx
+		WHERE o.folder_id = ? AND g.local_flags & {{.FlagLocalGlobal}} != 0 AND NOT g.deleted AND NOT g.invalid AND NOT EXISTS (
+			SELECT 1 FROM FILES f
+			INNER JOIN devices d ON d.idx = f.device_idx
+			WHERE f.name = g.name AND f.version = g.version AND f.folder_idx = g.folder_idx AND d.device_id = ?
+		)
+
+		UNION ALL
+
+		SELECT fi.fiprotobuf, bl.blprotobuf, g.name, g.size, g.modified FROM fileinfos fi
+		INNER JOIN files g on fi.sequence = g.sequence
+		LEFT JOIN blocklists bl ON bl.blocklist_hash = g.blocklist_hash
+		INNER JOIN folders o ON o.idx = g.folder_idx
+		WHERE o.folder_id = ? AND g.local_flags & {{.FlagLocalGlobal}} != 0 AND g.deleted AND NOT g.invalid AND EXISTS (
+			SELECT 1 FROM FILES f
+			INNER JOIN devices d ON d.idx = f.device_idx
+			WHERE f.name = g.name AND f.folder_idx = g.folder_idx AND d.device_id = ? AND NOT f.deleted
+		)
+	`+selectOpts).Queryx(
+		folder, device.String(),
+		folder, device.String(),
+	))
+	return itererr.Map(it, errFn, indirectFI.FileInfo)
+}

internal/db/sqlite/db_global_test.go

@@ -0,0 +1,493 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"slices"
+	"testing"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func TestNeed(t *testing.T) {
+	t.Helper()
+
+	db, err := OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// Some local files
+	var v protocol.Vector
+	baseV := v.Update(1)
+	newerV := baseV.Update(42)
+	files := []protocol.FileInfo{
+		genFile("test1", 1, 0), // remote need
+		genFile("test2", 2, 0), // local need
+		genFile("test3", 3, 0), // global
+	}
+	files[0].Version = baseV
+	files[1].Version = baseV
+	files[2].Version = newerV
+	err = db.Update(folderID, protocol.LocalDeviceID, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Some remote files
+	remote := []protocol.FileInfo{
+		genFile("test2", 2, 100), // global
+		genFile("test3", 3, 101), // remote need
+		genFile("test4", 4, 102), // local need
+	}
+	remote[0].Version = newerV
+	remote[1].Version = baseV
+	remote[2].Version = newerV
+	err = db.Update(folderID, protocol.DeviceID{42}, remote)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// A couple are needed locally
+	localNeed := fiNames(mustCollect[protocol.FileInfo](t)(db.AllNeededGlobalFiles(folderID, protocol.LocalDeviceID, config.PullOrderAlphabetic, 0, 0)))
+	if !slices.Equal(localNeed, []string{"test2", "test4"}) {
+		t.Log(localNeed)
+		t.Fatal("bad local need")
+	}
+
+	// Another couple are needed remotely
+	remoteNeed := fiNames(mustCollect[protocol.FileInfo](t)(db.AllNeededGlobalFiles(folderID, protocol.DeviceID{42}, config.PullOrderAlphabetic, 0, 0)))
+	if !slices.Equal(remoteNeed, []string{"test1", "test3"}) {
+		t.Log(remoteNeed)
+		t.Fatal("bad remote need")
+	}
+}
+
+func TestDropRecalcsGlobal(t *testing.T) {
+	// When we drop a device we may get a new global
+
+	t.Parallel()
+
+	t.Run("DropAllFiles", func(t *testing.T) {
+		t.Parallel()
+
+		testDropWithDropper(t, func(t *testing.T, db *DB) {
+			t.Helper()
+			if err := db.DropAllFiles(folderID, protocol.DeviceID{42}); err != nil {
+				t.Fatal(err)
+			}
+		})
+	})
+
+	t.Run("DropDevice", func(t *testing.T) {
+		t.Parallel()
+
+		testDropWithDropper(t, func(t *testing.T, db *DB) {
+			t.Helper()
+			if err := db.DropDevice(protocol.DeviceID{42}); err != nil {
+				t.Fatal(err)
+			}
+		})
+	})
+
+	t.Run("DropFilesNamed", func(t *testing.T) {
+		t.Parallel()
+
+		testDropWithDropper(t, func(t *testing.T, db *DB) {
+			t.Helper()
+			if err := db.DropFilesNamed(folderID, protocol.DeviceID{42}, []string{"test1", "test42"}); err != nil {
+				t.Fatal(err)
+			}
+		})
+	})
+}
+
+func testDropWithDropper(t *testing.T, dropper func(t *testing.T, db *DB)) {
+	t.Helper()
+
+	db, err := OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// Some local files
+	err = db.Update(folderID, protocol.LocalDeviceID, []protocol.FileInfo{
+		genFile("test1", 1, 0),
+		genFile("test2", 2, 0),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Some remote files
+	remote := []protocol.FileInfo{
+		genFile("test1", 3, 0),
+	}
+	remote[0].Version = remote[0].Version.Update(42)
+	err = db.Update(folderID, protocol.DeviceID{42}, remote)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Remote test1 wins as the global, verify.
+	count, err := db.CountGlobal(folderID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count.Bytes != (2+3)*128<<10 {
+		t.Log(count)
+		t.Fatal("bad global size to begin with")
+	}
+	if g, ok, err := db.GetGlobalFile(folderID, "test1"); err != nil || !ok {
+		t.Fatal("missing global to begin with")
+	} else if g.Size != 3*128<<10 {
+		t.Fatal("remote test1 should be the global")
+	}
+
+	// Now remove that remote device
+	dropper(t, db)
+
+	// Our test1 should now be the global
+	count, err = db.CountGlobal(folderID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count.Bytes != (1+2)*128<<10 {
+		t.Log(count)
+		t.Fatal("bad global size after drop")
+	}
+	if g, ok, err := db.GetGlobalFile(folderID, "test1"); err != nil || !ok {
+		t.Fatal("missing global after drop")
+	} else if g.Size != 1*128<<10 {
+		t.Fatal("local test1 should be the global")
+	}
+}
+
+func TestNeedDeleted(t *testing.T) {
+	t.Parallel()
+
+	db, err := OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// Some local files
+	err = db.Update(folderID, protocol.LocalDeviceID, []protocol.FileInfo{
+		genFile("test1", 1, 0),
+		genFile("test2", 2, 0),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// A remote deleted file
+	remote := []protocol.FileInfo{
+		genFile("test1", 1, 101),
+	}
+	remote[0].SetDeleted(42)
+	err = db.Update(folderID, protocol.DeviceID{42}, remote)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// We need the one deleted file
+	s, err := db.CountNeed(folderID, protocol.LocalDeviceID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.Bytes != 0 || s.Deleted != 1 {
+		t.Log(s)
+		t.Error("bad need")
+	}
+}
+
+func TestDontNeedIgnored(t *testing.T) {
+	t.Parallel()
+
+	db, err := OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// A remote file
+	files := []protocol.FileInfo{
+		genFile("test1", 1, 103),
+	}
+	err = db.Update(folderID, protocol.DeviceID{42}, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Which we've ignored locally
+	files[0].SetIgnored()
+	err = db.Update(folderID, protocol.LocalDeviceID, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// We don't need it
+	s, err := db.CountNeed(folderID, protocol.LocalDeviceID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.Bytes != 0 || s.Files != 0 {
+		t.Log(s)
+		t.Error("bad need")
+	}
+
+	// It shouldn't show up in the need list
+	names := mustCollect[protocol.FileInfo](t)(db.AllNeededGlobalFiles(folderID, protocol.LocalDeviceID, config.PullOrderAlphabetic, 0, 0))
+	if len(names) != 0 {
+		t.Log(names)
+		t.Error("need no files")
+	}
+}
+
+func TestRemoveDontNeedLocalIgnored(t *testing.T) {
+	t.Parallel()
+
+	db, err := OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// A local ignored file
+	file := genFile("test1", 1, 103)
+	file.SetIgnored()
+	files := []protocol.FileInfo{file}
+	err = db.Update(folderID, protocol.LocalDeviceID, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Which the remote doesn't have (no update)
+
+	// They don't need it
+	s, err := db.CountNeed(folderID, protocol.DeviceID{42})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.Bytes != 0 || s.Files != 0 {
+		t.Log(s)
+		t.Error("bad need")
+	}
+
+	// It shouldn't show up in their need list
+	names := mustCollect[protocol.FileInfo](t)(db.AllNeededGlobalFiles(folderID, protocol.DeviceID{42}, config.PullOrderAlphabetic, 0, 0))
+	if len(names) != 0 {
+		t.Log(names)
+		t.Error("need no files")
+	}
+}
+
+func TestLocalDontNeedDeletedMissing(t *testing.T) {
+	t.Parallel()
+
+	db, err := OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// A remote deleted file
+	file := genFile("test1", 1, 103)
+	file.SetDeleted(42)
+	files := []protocol.FileInfo{file}
+	err = db.Update(folderID, protocol.DeviceID{42}, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Which we don't have (no local update)
+
+	// We don't need it
+	s, err := db.CountNeed(folderID, protocol.LocalDeviceID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.Bytes != 0 || s.Files != 0 || s.Deleted != 0 {
+		t.Log(s)
+		t.Error("bad need")
+	}
+
+	// It shouldn't show up in the need list
+	names := mustCollect[protocol.FileInfo](t)(db.AllNeededGlobalFiles(folderID, protocol.LocalDeviceID, config.PullOrderAlphabetic, 0, 0))
+	if len(names) != 0 {
+		t.Log(names)
+		t.Error("need no files")
+	}
+}
+
+func TestRemoteDontNeedDeletedMissing(t *testing.T) {
+	t.Parallel()
+
+	db, err := OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// A local deleted file
+	file := genFile("test1", 1, 103)
+	file.SetDeleted(42)
+	files := []protocol.FileInfo{file}
+	err = db.Update(folderID, protocol.LocalDeviceID, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Which the remote doesn't have (no local update)
+
+	// They don't need it
+	s, err := db.CountNeed(folderID, protocol.DeviceID{42})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.Bytes != 0 || s.Files != 0 || s.Deleted != 0 {
+		t.Log(s)
+		t.Error("bad need")
+	}
+
+	// It shouldn't show up in their need list
+	names := mustCollect[protocol.FileInfo](t)(db.AllNeededGlobalFiles(folderID, protocol.DeviceID{42}, config.PullOrderAlphabetic, 0, 0))
+	if len(names) != 0 {
+		t.Log(names)
+		t.Error("need no files")
+	}
+}
+
+func TestNeedRemoteSymlinkAndDir(t *testing.T) {
+	t.Parallel()
+
+	db, err := OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// Two remote "specials", a symlink and a directory
+	var v protocol.Vector
+	v.Update(1)
+	files := []protocol.FileInfo{
+		{Name: "sym", Type: protocol.FileInfoTypeSymlink, Sequence: 100, Version: v, Blocks: genBlocks("symlink", 0, 1)},
+		{Name: "dir", Type: protocol.FileInfoTypeDirectory, Sequence: 101, Version: v},
+	}
+	err = db.Update(folderID, protocol.DeviceID{42}, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// We need them
+	s, err := db.CountNeed(folderID, protocol.LocalDeviceID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.Directories != 1 || s.Symlinks != 1 {
+		t.Log(s)
+		t.Error("bad need")
+	}
+
+	// They should be in the need list
+	names := mustCollect[protocol.FileInfo](t)(db.AllNeededGlobalFiles(folderID, protocol.LocalDeviceID, config.PullOrderAlphabetic, 0, 0))
+	if len(names) != 2 {
+		t.Log(names)
+		t.Error("bad need")
+	}
+}
+
+func TestNeedPagination(t *testing.T) {
+	t.Parallel()
+
+	db, err := OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// Several remote files
+	var v protocol.Vector
+	v.Update(1)
+	files := []protocol.FileInfo{
+		genFile("test0", 1, 100),
+		genFile("test1", 1, 101),
+		genFile("test2", 1, 102),
+		genFile("test3", 1, 103),
+		genFile("test4", 1, 104),
+		genFile("test5", 1, 105),
+		genFile("test6", 1, 106),
+		genFile("test7", 1, 107),
+		genFile("test8", 1, 108),
+		genFile("test9", 1, 109),
+	}
+	err = db.Update(folderID, protocol.DeviceID{42}, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// We should get the first two
+	names := fiNames(mustCollect[protocol.FileInfo](t)(db.AllNeededGlobalFiles(folderID, protocol.LocalDeviceID, config.PullOrderAlphabetic, 2, 0)))
+	if !slices.Equal(names, []string{"test0", "test1"}) {
+		t.Log(names)
+		t.Error("bad need")
+	}
+
+	// We should get the next three
+	names = fiNames(mustCollect[protocol.FileInfo](t)(db.AllNeededGlobalFiles(folderID, protocol.LocalDeviceID, config.PullOrderAlphabetic, 3, 2)))
+	if !slices.Equal(names, []string{"test2", "test3", "test4"}) {
+		t.Log(names)
+		t.Error("bad need")
+	}
+
+	// We should get the last five
+	names = fiNames(mustCollect[protocol.FileInfo](t)(db.AllNeededGlobalFiles(folderID, protocol.LocalDeviceID, config.PullOrderAlphabetic, 5, 5)))
+	if !slices.Equal(names, []string{"test5", "test6", "test7", "test8", "test9"}) {
+		t.Log(names)
+		t.Error("bad need")
+	}
+}

internal/db/sqlite/db_indexid.go

@@ -0,0 +1,163 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"database/sql"
+	"encoding/hex"
+	"errors"
+	"fmt"
+
+	"github.com/syncthing/syncthing/internal/itererr"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func (s *DB) GetIndexID(folder string, device protocol.DeviceID) (protocol.IndexID, error) {
+	// Try a fast read-only query to begin with. If it does not find the ID
+	// we'll do the full thing under a lock.
+	var indexID string
+	if err := s.stmt(`
+		SELECT i.index_id FROM indexids i
+		INNER JOIN folders o ON o.idx  = i.folder_idx
+		INNER JOIN devices d ON d.idx  = i.device_idx
+		WHERE o.folder_id = ? AND d.device_id = ?
+	`).Get(&indexID, folder, device.String()); err == nil && indexID != "" {
+		idx, err := indexIDFromHex(indexID)
+		return idx, wrap(err, "select")
+	}
+	if device != protocol.LocalDeviceID {
+		// For non-local devices we do not create the index ID, so return
+		// zero anyway if we don't have one.
+		return 0, nil
+	}
+
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+
+	// We are now operating only for the local device ID
+
+	folderIdx, err := s.folderIdxLocked(folder)
+	if err != nil {
+		return 0, wrap(err)
+	}
+
+	if err := s.stmt(`
+		SELECT index_id FROM indexids WHERE folder_idx = ? AND device_idx = {{.LocalDeviceIdx}}
+	`).Get(&indexID, folderIdx); err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return 0, wrap(err, "select local")
+	}
+
+	if indexID == "" {
+		// Generate a new index ID. Some trickiness in the query as we need
+		// to find the max sequence of local files if there already exist
+		// any.
+		id := protocol.NewIndexID()
+		if _, err := s.stmt(`
+			INSERT INTO indexids (folder_idx, device_idx, index_id, sequence)
+				SELECT ?, {{.LocalDeviceIdx}}, ?, COALESCE(MAX(sequence), 0) FROM files
+				WHERE folder_idx = ? AND device_idx = {{.LocalDeviceIdx}}
+			ON CONFLICT DO UPDATE SET index_id = ?
+		`).Exec(folderIdx, indexIDToHex(id), folderIdx, indexIDToHex(id)); err != nil {
+			return 0, wrap(err, "insert")
+		}
+		return id, nil
+	}
+
+	return indexIDFromHex(indexID)
+}
+
+func (s *DB) SetIndexID(folder string, device protocol.DeviceID, id protocol.IndexID) error {
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+
+	folderIdx, err := s.folderIdxLocked(folder)
+	if err != nil {
+		return wrap(err, "folder idx")
+	}
+	deviceIdx, err := s.deviceIdxLocked(device)
+	if err != nil {
+		return wrap(err, "device idx")
+	}
+
+	if _, err := s.stmt(`
+		INSERT OR REPLACE INTO indexids (folder_idx, device_idx, index_id, sequence) values (?, ?, ?, 0)
+	`).Exec(folderIdx, deviceIdx, indexIDToHex(id)); err != nil {
+		return wrap(err, "insert")
+	}
+	return nil
+}
+
+func (s *DB) DropAllIndexIDs() error {
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+	_, err := s.stmt(`DELETE FROM indexids`).Exec()
+	return wrap(err)
+}
+
+func (s *DB) GetDeviceSequence(folder string, device protocol.DeviceID) (int64, error) {
+	var res sql.NullInt64
+	err := s.stmt(`
+		SELECT sequence FROM indexids i
+		INNER JOIN folders o ON o.idx = i.folder_idx
+		INNER JOIN devices d ON d.idx = i.device_idx
+		WHERE o.folder_id = ? AND d.device_id = ?
+	`).Get(&res, folder, device.String())
+	if errors.Is(err, sql.ErrNoRows) {
+		return 0, nil
+	}
+	if err != nil {
+		return 0, wrap(err)
+	}
+	if !res.Valid {
+		return 0, nil
+	}
+	return res.Int64, nil
+}
+
+func (s *DB) RemoteSequences(folder string) (map[protocol.DeviceID]int64, error) {
+	type row struct {
+		Device string
+		Seq    int64
+	}
+
+	it, errFn := iterStructs[row](s.stmt(`
+		SELECT d.device_id AS device, i.sequence AS seq FROM indexids i
+		INNER JOIN folders o ON o.idx = i.folder_idx
+		INNER JOIN devices d ON d.idx = i.device_idx
+		WHERE o.folder_id = ? AND i.device_idx != {{.LocalDeviceIdx}}
+	`).Queryx(folder))
+
+	res := make(map[protocol.DeviceID]int64)
+	for row, err := range itererr.Zip(it, errFn) {
+		if err != nil {
+			return nil, wrap(err)
+		}
+		dev, err := protocol.DeviceIDFromString(row.Device)
+		if err != nil {
+			return nil, wrap(err, "device ID")
+		}
+		res[dev] = row.Seq
+	}
+	return res, nil
+}
+
+func indexIDFromHex(s string) (protocol.IndexID, error) {
+	bs, err := hex.DecodeString(s)
+	if err != nil {
+		return 0, fmt.Errorf("indexIDFromHex: %q: %w", s, err)
+	}
+	var id protocol.IndexID
+	if err := id.Unmarshal(bs); err != nil {
+		return 0, fmt.Errorf("indexIDFromHex: %q: %w", s, err)
+	}
+	return id, nil
+}
+
+func indexIDToHex(i protocol.IndexID) string {
+	bs, _ := i.Marshal()
+	return hex.EncodeToString(bs)
+}

internal/db/sqlite/db_indexid_test.go

@@ -0,0 +1,81 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"testing"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func TestIndexIDs(t *testing.T) {
+	t.Parallel()
+
+	db, err := OpenTemp()
+	if err != nil {
+		t.Fatal()
+	}
+	t.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("LocalDeviceID", func(t *testing.T) {
+		t.Parallel()
+
+		localID, err := db.GetIndexID("foo", protocol.LocalDeviceID)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if localID == 0 {
+			t.Fatal("should have been generated")
+		}
+
+		again, err := db.GetIndexID("foo", protocol.LocalDeviceID)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if again != localID {
+			t.Fatal("should get same again")
+		}
+
+		other, err := db.GetIndexID("bar", protocol.LocalDeviceID)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if other == localID {
+			t.Fatal("should not get same for other folder")
+		}
+	})
+
+	t.Run("OtherDeviceID", func(t *testing.T) {
+		t.Parallel()
+
+		localID, err := db.GetIndexID("foo", protocol.DeviceID{42})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if localID != 0 {
+			t.Fatal("should have been zero")
+		}
+
+		newID := protocol.NewIndexID()
+		if err := db.SetIndexID("foo", protocol.DeviceID{42}, newID); err != nil {
+			t.Fatal(err)
+		}
+
+		again, err := db.GetIndexID("foo", protocol.DeviceID{42})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if again != newID {
+			t.Log(again, newID)
+			t.Fatal("should get the ID we set")
+		}
+	})
+}

internal/db/sqlite/db_kv.go

@@ -0,0 +1,78 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"iter"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/syncthing/syncthing/internal/db"
+)
+
+func (s *DB) GetKV(key string) ([]byte, error) {
+	var val []byte
+	if err := s.stmt(`
+		SELECT value FROM kv
+		WHERE key = ?
+	`).Get(&val, key); err != nil {
+		return nil, wrap(err)
+	}
+	return val, nil
+}
+
+func (s *DB) PutKV(key string, val []byte) error {
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+	_, err := s.stmt(`
+		INSERT OR REPLACE INTO kv (key, value)
+		VALUES (?, ?)
+	`).Exec(key, val)
+	return wrap(err)
+}
+
+func (s *DB) DeleteKV(key string) error {
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+	_, err := s.stmt(`
+		DELETE FROM kv WHERE key = ?
+	`).Exec(key)
+	return wrap(err)
+}
+
+func (s *DB) PrefixKV(prefix string) (iter.Seq[db.KeyValue], func() error) {
+	var rows *sqlx.Rows
+	var err error
+	if prefix == "" {
+		rows, err = s.stmt(`SELECT key, value FROM kv`).Queryx()
+	} else {
+		end := prefixEnd(prefix)
+		rows, err = s.stmt(`
+			SELECT key, value FROM kv
+			WHERE key >= ? AND key < ?
+		`).Queryx(prefix, end)
+	}
+	if err != nil {
+		return func(_ func(db.KeyValue) bool) {}, func() error { return err }
+	}
+
+	return func(yield func(db.KeyValue) bool) {
+			defer rows.Close()
+			for rows.Next() {
+				var key string
+				var val []byte
+				if err = rows.Scan(&key, &val); err != nil {
+					return
+				}
+				if !yield(db.KeyValue{Key: key, Value: val}) {
+					return
+				}
+			}
+			err = rows.Err()
+		}, func() error {
+			return err
+		}
+}

internal/db/sqlite/db_local.go

@@ -0,0 +1,126 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"iter"
+
+	"github.com/syncthing/syncthing/internal/db"
+	"github.com/syncthing/syncthing/internal/itererr"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func (s *DB) GetDeviceFile(folder string, device protocol.DeviceID, file string) (protocol.FileInfo, bool, error) {
+	file = osutil.NormalizedFilename(file)
+
+	var ind indirectFI
+	err := s.stmt(`
+		SELECT fi.fiprotobuf, bl.blprotobuf FROM fileinfos fi
+		INNER JOIN files f on fi.sequence = f.sequence
+		LEFT JOIN blocklists bl ON bl.blocklist_hash = f.blocklist_hash
+		INNER JOIN devices d ON f.device_idx = d.idx
+		INNER JOIN folders o ON f.folder_idx = o.idx
+		WHERE o.folder_id = ? AND d.device_id = ? AND f.name = ?
+	`).Get(&ind, folder, device.String(), file)
+	if errors.Is(err, sql.ErrNoRows) {
+		return protocol.FileInfo{}, false, nil
+	}
+	if err != nil {
+		return protocol.FileInfo{}, false, wrap(err)
+	}
+	fi, err := ind.FileInfo()
+	if err != nil {
+		return protocol.FileInfo{}, false, wrap(err, "indirect")
+	}
+	return fi, true, nil
+}
+
+func (s *DB) AllLocalFiles(folder string, device protocol.DeviceID) (iter.Seq[protocol.FileInfo], func() error) {
+	it, errFn := iterStructs[indirectFI](s.stmt(`
+		SELECT fi.fiprotobuf, bl.blprotobuf FROM fileinfos fi
+		INNER JOIN files f on fi.sequence = f.sequence
+		LEFT JOIN blocklists bl ON bl.blocklist_hash = f.blocklist_hash
+		INNER JOIN folders o ON o.idx = f.folder_idx
+		INNER JOIN devices d ON d.idx = f.device_idx
+		WHERE o.folder_id = ? AND d.device_id = ?
+	`).Queryx(folder, device.String()))
+	return itererr.Map(it, errFn, indirectFI.FileInfo)
+}
+
+func (s *DB) AllLocalFilesBySequence(folder string, device protocol.DeviceID, startSeq int64, limit int) (iter.Seq[protocol.FileInfo], func() error) {
+	var limitStr string
+	if limit > 0 {
+		limitStr = fmt.Sprintf(" LIMIT %d", limit)
+	}
+	it, errFn := iterStructs[indirectFI](s.stmt(`
+		SELECT fi.fiprotobuf, bl.blprotobuf FROM fileinfos fi
+		INNER JOIN files f on fi.sequence = f.sequence
+		LEFT JOIN blocklists bl ON bl.blocklist_hash = f.blocklist_hash
+		INNER JOIN folders o ON o.idx = f.folder_idx
+		INNER JOIN devices d ON d.idx = f.device_idx
+		WHERE o.folder_id = ? AND d.device_id = ? AND f.sequence >= ?
+		ORDER BY f.sequence`+limitStr).Queryx(
+		folder, device.String(), startSeq))
+	return itererr.Map(it, errFn, indirectFI.FileInfo)
+}
+
+func (s *DB) AllLocalFilesWithPrefix(folder string, device protocol.DeviceID, prefix string) (iter.Seq[protocol.FileInfo], func() error) {
+	if prefix == "" {
+		return s.AllLocalFiles(folder, device)
+	}
+
+	prefix = osutil.NormalizedFilename(prefix)
+	end := prefixEnd(prefix)
+
+	it, errFn := iterStructs[indirectFI](s.sql.Queryx(`
+		SELECT fi.fiprotobuf, bl.blprotobuf FROM fileinfos fi
+		INNER JOIN files f on fi.sequence = f.sequence
+		LEFT JOIN blocklists bl ON bl.blocklist_hash = f.blocklist_hash
+		INNER JOIN folders o ON o.idx = f.folder_idx
+		INNER JOIN devices d ON d.idx = f.device_idx
+		WHERE o.folder_id = ? AND d.device_id = ? AND f.name >= ? AND f.name < ?
+	`, folder, device.String(), prefix, end))
+	return itererr.Map(it, errFn, indirectFI.FileInfo)
+}
+
+func (s *DB) AllLocalFilesWithBlocksHash(folder string, h []byte) (iter.Seq[db.FileMetadata], func() error) {
+	return iterStructs[db.FileMetadata](s.stmt(`
+		SELECT f.sequence, f.name, f.type, f.modified as modnanos, f.size, f.deleted, f.invalid, f.local_flags as localflags FROM files f
+		INNER JOIN folders o ON o.idx = f.folder_idx
+		WHERE o.folder_id = ? AND f.device_idx = {{.LocalDeviceIdx}} AND f.blocklist_hash = ?
+	`).Queryx(folder, h))
+}
+
+func (s *DB) AllLocalFilesWithBlocksHashAnyFolder(h []byte) (iter.Seq2[string, db.FileMetadata], func() error) {
+	type row struct {
+		FolderID string `db:"folder_id"`
+		db.FileMetadata
+	}
+	it, errFn := iterStructs[row](s.stmt(`
+		SELECT o.folder_id, f.sequence, f.name, f.type, f.modified as modnanos, f.size, f.deleted, f.invalid, f.local_flags as localflags FROM files f
+		INNER JOIN folders o ON o.idx = f.folder_idx
+		WHERE f.device_idx = {{.LocalDeviceIdx}} AND f.blocklist_hash = ?
+	`).Queryx(h))
+	return itererr.Map2(it, errFn, func(r row) (string, db.FileMetadata, error) {
+		return r.FolderID, r.FileMetadata, nil
+	})
+}
+
+func (s *DB) AllLocalBlocksWithHash(hash []byte) (iter.Seq[db.BlockMapEntry], func() error) {
+	// We involve the files table in this select because deletion of blocks
+	// & blocklists is deferred (garbage collected) while the files list is
+	// not. This filters out blocks that are in fact deleted.
+	return iterStructs[db.BlockMapEntry](s.stmt(`
+		SELECT f.blocklist_hash as blocklisthash, b.idx as blockindex, b.offset, b.size FROM files f
+		LEFT JOIN blocks b ON f.blocklist_hash = b.blocklist_hash
+		WHERE f.device_idx = {{.LocalDeviceIdx}} AND b.hash = ?
+	`).Queryx(hash))
+}

internal/db/sqlite/db_local_test.go

@@ -0,0 +1,202 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"testing"
+
+	"github.com/syncthing/syncthing/internal/db"
+	"github.com/syncthing/syncthing/internal/itererr"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func TestBlocks(t *testing.T) {
+	t.Parallel()
+
+	db, err := OpenTemp()
+	if err != nil {
+		t.Fatal()
+	}
+	t.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	files := []protocol.FileInfo{
+		{
+			Name: "file1",
+			Blocks: []protocol.BlockInfo{
+				{Hash: []byte{1, 2, 3}, Offset: 0, Size: 42},
+				{Hash: []byte{2, 3, 4}, Offset: 42, Size: 42},
+				{Hash: []byte{3, 4, 5}, Offset: 84, Size: 42},
+			},
+		},
+		{
+			Name: "file2",
+			Blocks: []protocol.BlockInfo{
+				{Hash: []byte{2, 3, 4}, Offset: 0, Size: 42},
+				{Hash: []byte{3, 4, 5}, Offset: 42, Size: 42},
+				{Hash: []byte{4, 5, 6}, Offset: 84, Size: 42},
+			},
+		},
+	}
+
+	if err := db.Update("test", protocol.LocalDeviceID, files); err != nil {
+		t.Fatal(err)
+	}
+
+	// Search for blocks
+
+	vals, err := itererr.Collect(db.AllLocalBlocksWithHash([]byte{1, 2, 3}))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(vals) != 1 {
+		t.Log(vals)
+		t.Fatal("expected one hit")
+	} else if vals[0].BlockIndex != 0 || vals[0].Offset != 0 || vals[0].Size != 42 {
+		t.Log(vals[0])
+		t.Fatal("bad entry")
+	}
+
+	// Get FileInfos for those blocks
+
+	found := 0
+	it, errFn := db.AllLocalFilesWithBlocksHashAnyFolder(vals[0].BlocklistHash)
+	for folder, fileInfo := range it {
+		if folder != folderID {
+			t.Fatal("should be same folder")
+		}
+		if fileInfo.Name != "file1" {
+			t.Fatal("should be file1")
+		}
+		found++
+	}
+	if err := errFn(); err != nil {
+		t.Fatal(err)
+	}
+	if found != 1 {
+		t.Fatal("should find one file")
+	}
+
+	// Get the other blocks
+
+	vals, err = itererr.Collect(db.AllLocalBlocksWithHash([]byte{3, 4, 5}))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(vals) != 2 {
+		t.Log(vals)
+		t.Fatal("expected two hits")
+	}
+	// if vals[0].Index != 2 || vals[0].Offset != 84 || vals[0].Size != 42 {
+	// 	t.Log(vals[0])
+	// 	t.Fatal("bad entry 1")
+	// }
+	// if vals[1].Index != 1 || vals[1].Offset != 42 || vals[1].Size != 42 {
+	// 	t.Log(vals[1])
+	// 	t.Fatal("bad entry 2")
+	// }
+}
+
+func TestBlocksDeleted(t *testing.T) {
+	t.Parallel()
+
+	sdb, err := OpenTemp()
+	if err != nil {
+		t.Fatal()
+	}
+	t.Cleanup(func() {
+		if err := sdb.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// Insert a file
+	file := genFile("foo", 1, 0)
+	if err := sdb.Update(folderID, protocol.LocalDeviceID, []protocol.FileInfo{file}); err != nil {
+		t.Fatal()
+	}
+
+	// We should find one entry for the block hash
+	search := file.Blocks[0].Hash
+	es := mustCollect[db.BlockMapEntry](t)(sdb.AllLocalBlocksWithHash(search))
+	if len(es) != 1 {
+		t.Fatal("expected one hit")
+	}
+
+	// Update the file with a new block hash
+	file.Blocks = genBlocks("foo", 42, 1)
+	if err := sdb.Update(folderID, protocol.LocalDeviceID, []protocol.FileInfo{file}); err != nil {
+		t.Fatal()
+	}
+
+	// Searching for the old hash should yield no hits
+	if hits := mustCollect[db.BlockMapEntry](t)(sdb.AllLocalBlocksWithHash(search)); len(hits) != 0 {
+		t.Log(hits)
+		t.Error("expected no hits")
+	}
+
+	// Searching for the new hash should yield one hits
+	if hits := mustCollect[db.BlockMapEntry](t)(sdb.AllLocalBlocksWithHash(file.Blocks[0].Hash)); len(hits) != 1 {
+		t.Log(hits)
+		t.Error("expected one hit")
+	}
+}
+
+func TestRemoteSequence(t *testing.T) {
+	t.Parallel()
+
+	sdb, err := OpenTemp()
+	if err != nil {
+		t.Fatal()
+	}
+	t.Cleanup(func() {
+		if err := sdb.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	// Insert a local file
+	file := genFile("foo", 1, 0)
+	if err := sdb.Update(folderID, protocol.LocalDeviceID, []protocol.FileInfo{file}); err != nil {
+		t.Fatal()
+	}
+
+	// Insert several remote files
+	file = genFile("foo1", 1, 42)
+	if err := sdb.Update(folderID, protocol.DeviceID{42}, []protocol.FileInfo{file}); err != nil {
+		t.Fatal()
+	}
+	if err := sdb.Update(folderID, protocol.DeviceID{43}, []protocol.FileInfo{file}); err != nil {
+		t.Fatal()
+	}
+	file = genFile("foo2", 1, 43)
+	if err := sdb.Update(folderID, protocol.DeviceID{43}, []protocol.FileInfo{file}); err != nil {
+		t.Fatal()
+	}
+	if err := sdb.Update(folderID, protocol.DeviceID{44}, []protocol.FileInfo{file}); err != nil {
+		t.Fatal()
+	}
+	file = genFile("foo3", 1, 44)
+	if err := sdb.Update(folderID, protocol.DeviceID{44}, []protocol.FileInfo{file}); err != nil {
+		t.Fatal()
+	}
+
+	// Verify remote sequences
+	seqs, err := sdb.RemoteSequences(folderID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(seqs) != 3 || seqs[protocol.DeviceID{42}] != 42 ||
+		seqs[protocol.DeviceID{43}] != 43 ||
+		seqs[protocol.DeviceID{44}] != 44 {
+		t.Log(seqs)
+		t.Error("bad seqs")
+	}
+}

internal/db/sqlite/db_mtimes.go

@@ -0,0 +1,54 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"time"
+)
+
+func (s *DB) GetMtime(folder, name string) (ondisk, virtual time.Time) {
+	var res struct {
+		Ondisk  int64
+		Virtual int64
+	}
+	if err := s.stmt(`
+		SELECT m.ondisk, m.virtual FROM mtimes m
+		INNER JOIN folders o ON o.idx = m.folder_idx
+		WHERE o.folder_id = ? AND m.name = ?
+	`).Get(&res, folder, name); err != nil {
+		return time.Time{}, time.Time{}
+	}
+	return time.Unix(0, res.Ondisk), time.Unix(0, res.Virtual)
+}
+
+func (s *DB) PutMtime(folder, name string, ondisk, virtual time.Time) error {
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+	folderIdx, err := s.folderIdxLocked(folder)
+	if err != nil {
+		return wrap(err)
+	}
+	_, err = s.stmt(`
+		INSERT OR REPLACE INTO mtimes (folder_idx, name, ondisk, virtual)
+		VALUES (?, ?, ?, ?)
+	`).Exec(folderIdx, name, ondisk.UnixNano(), virtual.UnixNano())
+	return wrap(err)
+}
+
+func (s *DB) DeleteMtime(folder, name string) error {
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+	folderIdx, err := s.folderIdxLocked(folder)
+	if err != nil {
+		return wrap(err)
+	}
+	_, err = s.stmt(`
+		DELETE FROM mtimes
+		WHERE folder_idx = ? AND name = ?
+	`).Exec(folderIdx, name)
+	return wrap(err)
+}

internal/db/sqlite/db_mtimes_test.go

@@ -0,0 +1,54 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"testing"
+	"time"
+)
+
+func TestMtimePairs(t *testing.T) {
+	t.Parallel()
+
+	db, err := OpenTemp()
+	if err != nil {
+		t.Fatal()
+	}
+	t.Cleanup(func() {
+		if err := db.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t0 := time.Now().Truncate(time.Second)
+	t1 := t0.Add(1234567890)
+
+	// Set a pair
+	if err := db.PutMtime("foo", "bar", t0, t1); err != nil {
+		t.Fatal(err)
+	}
+
+	// Check it
+	gt0, gt1 := db.GetMtime("foo", "bar")
+	if !gt0.Equal(t0) || !gt1.Equal(t1) {
+		t.Log(t0, gt0)
+		t.Log(t1, gt1)
+		t.Log("bad times")
+	}
+
+	// Delete it
+	if err := db.DeleteMtime("foo", "bar"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Check it
+	gt0, gt1 = db.GetMtime("foo", "bar")
+	if !gt0.IsZero() || !gt1.IsZero() {
+		t.Log(gt0, gt1)
+		t.Log("bad times")
+	}
+}

internal/db/sqlite/db_open.go

@@ -0,0 +1,206 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"database/sql"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"text/template"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+const maxDBConns = 128
+
+func Open(path string, opts ...Option) (*DB, error) {
+	// Open the database with options to enable foreign keys and recursive
+	// triggers (needed for the delete+insert triggers on row replace).
+	sqlDB, err := sqlx.Open(dbDriver, "file:"+path+"?"+commonOptions)
+	if err != nil {
+		return nil, wrap(err)
+	}
+	sqlDB.SetMaxOpenConns(maxDBConns)
+	if _, err := sqlDB.Exec(`PRAGMA journal_mode = WAL`); err != nil {
+		return nil, wrap(err, "PRAGMA journal_mode")
+	}
+	if _, err := sqlDB.Exec(`PRAGMA optimize = 0x10002`); err != nil {
+		// https://www.sqlite.org/pragma.html#pragma_optimize
+		return nil, wrap(err, "PRAGMA optimize")
+	}
+	return openCommon(sqlDB, opts...)
+}
+
+// Open the database with options suitable for the migration inserts. This
+// is not a safe mode of operation for normal processing, use only for bulk
+// inserts with a close afterwards.
+func OpenForMigration(path string) (*DB, error) {
+	sqlDB, err := sqlx.Open(dbDriver, "file:"+path+"?"+commonOptions)
+	if err != nil {
+		return nil, wrap(err, "open")
+	}
+	sqlDB.SetMaxOpenConns(1)
+	if _, err := sqlDB.Exec(`PRAGMA foreign_keys = 0`); err != nil {
+		return nil, wrap(err, "PRAGMA foreign_keys")
+	}
+	if _, err := sqlDB.Exec(`PRAGMA journal_mode = OFF`); err != nil {
+		return nil, wrap(err, "PRAGMA journal_mode")
+	}
+	if _, err := sqlDB.Exec(`PRAGMA synchronous = 0`); err != nil {
+		return nil, wrap(err, "PRAGMA synchronous")
+	}
+	return openCommon(sqlDB)
+}
+
+func OpenTemp() (*DB, error) {
+	// SQLite has a memory mode, but it works differently with concurrency
+	// compared to what we need with the WAL mode. So, no memory databases
+	// for now.
+	dir, err := os.MkdirTemp("", "syncthing-db")
+	if err != nil {
+		return nil, wrap(err)
+	}
+	path := filepath.Join(dir, "db")
+	l.Debugln("Test DB in", path)
+	return Open(path)
+}
+
+func openCommon(sqlDB *sqlx.DB, opts ...Option) (*DB, error) {
+	if _, err := sqlDB.Exec(`PRAGMA auto_vacuum = INCREMENTAL`); err != nil {
+		return nil, wrap(err, "PRAGMA auto_vacuum")
+	}
+	if _, err := sqlDB.Exec(`PRAGMA default_temp_store = MEMORY`); err != nil {
+		return nil, wrap(err, "PRAGMA default_temp_store")
+	}
+	if _, err := sqlDB.Exec(`PRAGMA temp_store = MEMORY`); err != nil {
+		return nil, wrap(err, "PRAGMA temp_store")
+	}
+
+	db := &DB{
+		sql:             sqlDB,
+		deleteRetention: defaultDeleteRetention,
+		statements:      make(map[string]*sqlx.Stmt),
+	}
+	for _, opt := range opts {
+		opt(db)
+	}
+	if db.deleteRetention > 0 && db.deleteRetention < minDeleteRetention {
+		db.deleteRetention = minDeleteRetention
+	}
+
+	if err := db.runScripts("sql/schema/*"); err != nil {
+		return nil, wrap(err)
+	}
+
+	ver, _ := db.getAppliedSchemaVersion()
+	if ver.SchemaVersion > 0 {
+		filter := func(scr string) bool {
+			scr = filepath.Base(scr)
+			nstr, _, ok := strings.Cut(scr, "-")
+			if !ok {
+				return false
+			}
+			n, err := strconv.ParseInt(nstr, 10, 32)
+			if err != nil {
+				return false
+			}
+			return int(n) > ver.SchemaVersion
+		}
+		if err := db.runScripts("sql/migrations/*", filter); err != nil {
+			return nil, wrap(err)
+		}
+	}
+
+	// Touch device IDs that should always exist and have a low index
+	// numbers, and will never change
+	db.localDeviceIdx, _ = db.deviceIdxLocked(protocol.LocalDeviceID)
+
+	// Set the current schema version, if not already set
+	if err := db.setAppliedSchemaVersion(currentSchemaVersion); err != nil {
+		return nil, wrap(err)
+	}
+
+	db.tplInput = map[string]any{
+		"FlagLocalUnsupported": protocol.FlagLocalUnsupported,
+		"FlagLocalIgnored":     protocol.FlagLocalIgnored,
+		"FlagLocalMustRescan":  protocol.FlagLocalMustRescan,
+		"FlagLocalReceiveOnly": protocol.FlagLocalReceiveOnly,
+		"FlagLocalGlobal":      protocol.FlagLocalGlobal,
+		"FlagLocalNeeded":      protocol.FlagLocalNeeded,
+		"LocalDeviceIdx":       db.localDeviceIdx,
+		"SyncthingVersion":     build.LongVersion,
+	}
+
+	return db, nil
+}
+
+var tplFuncs = template.FuncMap{
+	"or": func(vs ...int) int {
+		v := vs[0]
+		for _, ov := range vs[1:] {
+			v |= ov
+		}
+		return v
+	},
+}
+
+// stmt returns a prepared statement for the given SQL string, after
+// applying local template expansions. The statement is cached.
+func (s *DB) stmt(tpl string) stmt {
+	tpl = strings.TrimSpace(tpl)
+
+	// Fast concurrent lookup of cached statement
+	s.statementsMut.RLock()
+	stmt, ok := s.statements[tpl]
+	s.statementsMut.RUnlock()
+	if ok {
+		return stmt
+	}
+
+	// On miss, take the full lock, check again
+	s.statementsMut.Lock()
+	defer s.statementsMut.Unlock()
+	stmt, ok = s.statements[tpl]
+	if ok {
+		return stmt
+	}
+
+	// Apply template expansions
+	var sb strings.Builder
+	compTpl := template.Must(template.New("tpl").Funcs(tplFuncs).Parse(tpl))
+	if err := compTpl.Execute(&sb, s.tplInput); err != nil {
+		panic("bug: bad template: " + err.Error())
+	}
+
+	// Prepare and cache
+	stmt, err := s.sql.Preparex(sb.String())
+	if err != nil {
+		return failedStmt{err}
+	}
+	s.statements[tpl] = stmt
+	return stmt
+}
+
+type stmt interface {
+	Exec(args ...any) (sql.Result, error)
+	Get(dest any, args ...any) error
+	Queryx(args ...any) (*sqlx.Rows, error)
+	Select(dest any, args ...any) error
+}
+
+type failedStmt struct {
+	err error
+}
+
+func (f failedStmt) Exec(_ ...any) (sql.Result, error)   { return nil, f.err }
+func (f failedStmt) Get(_ any, _ ...any) error           { return f.err }
+func (f failedStmt) Queryx(_ ...any) (*sqlx.Rows, error) { return nil, f.err }
+func (f failedStmt) Select(_ any, _ ...any) error        { return f.err }

internal/db/sqlite/db_open_cgo.go

@@ -0,0 +1,18 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//go:build cgo
+
+package sqlite
+
+import (
+	_ "github.com/mattn/go-sqlite3" // register sqlite3 database driver
+)
+
+const (
+	dbDriver      = "sqlite3"
+	commonOptions = "_fk=true&_rt=true&_cache_size=-65536&_sync=1&_txlock=immediate"
+)

internal/db/sqlite/db_open_nocgo.go

@@ -0,0 +1,23 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//go:build !cgo && !wazero
+
+package sqlite
+
+import (
+	"github.com/syncthing/syncthing/lib/build"
+	_ "modernc.org/sqlite" // register sqlite database driver
+)
+
+const (
+	dbDriver      = "sqlite"
+	commonOptions = "_pragma=foreign_keys(1)&_pragma=recursive_triggers(1)&_pragma=cache_size(-65536)&_pragma=synchronous(1)"
+)
+
+func init() {
+	build.AddTag("modernc-sqlite")
+}

internal/db/sqlite/db_prepared.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import "github.com/jmoiron/sqlx"
+
+type txPreparedStmts struct {
+	*sqlx.Tx
+	stmts map[string]*sqlx.Stmt
+}
+
+func (p *txPreparedStmts) Preparex(query string) (*sqlx.Stmt, error) {
+	if p.stmts == nil {
+		p.stmts = make(map[string]*sqlx.Stmt)
+	}
+	stmt, ok := p.stmts[query]
+	if ok {
+		return stmt, nil
+	}
+	stmt, err := p.Tx.Preparex(query)
+	if err != nil {
+		return nil, wrap(err)
+	}
+	p.stmts[query] = stmt
+	return stmt, nil
+}
+
+func (p *txPreparedStmts) Commit() error {
+	for _, s := range p.stmts {
+		s.Close()
+	}
+	return p.Tx.Commit()
+}
+
+func (p *txPreparedStmts) Rollback() error {
+	for _, s := range p.stmts {
+		s.Close()
+	}
+	return p.Tx.Rollback()
+}

internal/db/sqlite/db_schema.go

@@ -0,0 +1,88 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"embed"
+	"io/fs"
+	"strings"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/build"
+)
+
+const currentSchemaVersion = 1
+
+//go:embed sql/**
+var embedded embed.FS
+
+func (s *DB) runScripts(glob string, filter ...func(s string) bool) error {
+	scripts, err := fs.Glob(embedded, glob)
+	if err != nil {
+		return wrap(err)
+	}
+
+	tx, err := s.sql.Begin()
+	if err != nil {
+		return wrap(err)
+	}
+	defer tx.Rollback() //nolint:errcheck
+
+nextScript:
+	for _, scr := range scripts {
+		for _, fn := range filter {
+			if !fn(scr) {
+				l.Debugln("Skipping script", scr)
+				continue nextScript
+			}
+		}
+		l.Debugln("Executing script", scr)
+		bs, err := fs.ReadFile(embedded, scr)
+		if err != nil {
+			return wrap(err, scr)
+		}
+		// SQLite requires one statement per exec, so we split the init
+		// files on lines containing only a semicolon and execute them
+		// separately. We require it on a separate line because there are
+		// also statement-internal semicolons in the triggers.
+		for _, stmt := range strings.Split(string(bs), "\n;") {
+			if _, err := tx.Exec(stmt); err != nil {
+				return wrap(err, stmt)
+			}
+		}
+	}
+
+	return wrap(tx.Commit())
+}
+
+type schemaVersion struct {
+	SchemaVersion    int
+	AppliedAt        int64
+	SyncthingVersion string
+}
+
+func (s *schemaVersion) AppliedTime() time.Time {
+	return time.Unix(0, s.AppliedAt)
+}
+
+func (s *DB) setAppliedSchemaVersion(ver int) error {
+	_, err := s.stmt(`
+		INSERT OR IGNORE INTO schemamigrations (schema_version, applied_at, syncthing_version)
+		VALUES (?, ?, ?)
+	`).Exec(ver, time.Now().UnixNano(), build.LongVersion)
+	return wrap(err)
+}
+
+func (s *DB) getAppliedSchemaVersion() (schemaVersion, error) {
+	var v schemaVersion
+	err := s.stmt(`
+		SELECT schema_version as schemaversion, applied_at as appliedat, syncthing_version as syncthingversion FROM schemamigrations
+		ORDER BY schema_version DESC
+		LIMIT 1
+	`).Get(&v)
+	return v, wrap(err)
+}

internal/db/sqlite/db_service.go

@@ -0,0 +1,179 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/db"
+)
+
+const (
+	internalMetaPrefix     = "dbsvc"
+	lastMaintKey           = "lastMaint"
+	defaultDeleteRetention = 180 * 24 * time.Hour
+	minDeleteRetention     = 24 * time.Hour
+)
+
+type Service struct {
+	sdb                 *DB
+	maintenanceInterval time.Duration
+	internalMeta        *db.Typed
+}
+
+func (s *Service) String() string {
+	return fmt.Sprintf("sqlite.service@%p", s)
+}
+
+func newService(sdb *DB, maintenanceInterval time.Duration) *Service {
+	return &Service{
+		sdb:                 sdb,
+		maintenanceInterval: maintenanceInterval,
+		internalMeta:        db.NewTyped(sdb, internalMetaPrefix),
+	}
+}
+
+func (s *Service) Serve(ctx context.Context) error {
+	// Run periodic maintenance
+
+	// Figure out when we last ran maintenance and schedule accordingly. If
+	// it was never, do it now.
+	lastMaint, _, _ := s.internalMeta.Time(lastMaintKey)
+	nextMaint := lastMaint.Add(s.maintenanceInterval)
+	wait := time.Until(nextMaint)
+	if wait < 0 {
+		wait = time.Minute
+	}
+	l.Debugln("Next periodic run in", wait)
+
+	timer := time.NewTimer(wait)
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-timer.C:
+		}
+
+		if err := s.periodic(ctx); err != nil {
+			return wrap(err)
+		}
+
+		timer.Reset(s.maintenanceInterval)
+		l.Debugln("Next periodic run in", s.maintenanceInterval)
+		_ = s.internalMeta.PutTime(lastMaintKey, time.Now())
+	}
+}
+
+func (s *Service) periodic(ctx context.Context) error {
+	t0 := time.Now()
+	l.Debugln("Periodic start")
+
+	s.sdb.updateLock.Lock()
+	defer s.sdb.updateLock.Unlock()
+
+	t1 := time.Now()
+	defer func() { l.Debugln("Periodic done in", time.Since(t1), "+", t1.Sub(t0)) }()
+
+	if err := s.garbageCollectOldDeletedLocked(); err != nil {
+		return wrap(err)
+	}
+	if err := s.garbageCollectBlocklistsAndBlocksLocked(ctx); err != nil {
+		return wrap(err)
+	}
+
+	conn, err := s.sdb.sql.Conn(ctx)
+	if err != nil {
+		return wrap(err)
+	}
+	defer conn.Close()
+	_, _ = conn.ExecContext(ctx, `ANALYZE`)
+	_, _ = conn.ExecContext(ctx, `PRAGMA optimize`)
+	_, _ = conn.ExecContext(ctx, `PRAGMA incremental_vacuum`)
+	_, _ = conn.ExecContext(ctx, `PRAGMA journal_size_limit = 67108864`)
+	_, _ = conn.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`)
+
+	return nil
+}
+
+func (s *Service) garbageCollectOldDeletedLocked() error {
+	if s.sdb.deleteRetention <= 0 {
+		l.Debugln("Delete retention is infinite, skipping cleanup")
+		return nil
+	}
+
+	// Remove deleted files that are marked as not needed (we have processed
+	// them) and they were deleted more than MaxDeletedFileAge ago.
+	l.Debugln("Forgetting deleted files older than", s.sdb.deleteRetention)
+	res, err := s.sdb.stmt(`
+		DELETE FROM files
+		WHERE deleted AND modified < ? AND local_flags & {{.FlagLocalNeeded}} == 0
+	`).Exec(time.Now().Add(-s.sdb.deleteRetention).UnixNano())
+	if err != nil {
+		return wrap(err)
+	}
+	if aff, err := res.RowsAffected(); err == nil {
+		l.Debugln("Removed old deleted file records:", aff)
+	}
+	return nil
+}
+
+func (s *Service) garbageCollectBlocklistsAndBlocksLocked(ctx context.Context) error {
+	// Remove all blocklists not referred to by any files and, by extension,
+	// any blocks not referred to by a blocklist. This is an expensive
+	// operation when run normally, especially if there are a lot of blocks
+	// to collect.
+	//
+	// We make this orders of magnitude faster by disabling foreign keys for
+	// the transaction and doing the cleanup manually. This requires using
+	// an explicit connection and disabling foreign keys before starting the
+	// transaction. We make sure to clean up on the way out.
+
+	conn, err := s.sdb.sql.Connx(ctx)
+	if err != nil {
+		return wrap(err)
+	}
+	defer conn.Close()
+
+	if _, err := conn.ExecContext(ctx, `PRAGMA foreign_keys = 0`); err != nil {
+		return wrap(err)
+	}
+	defer func() { //nolint:contextcheck
+		_, _ = conn.ExecContext(context.Background(), `PRAGMA foreign_keys = 1`)
+	}()
+
+	tx, err := conn.BeginTxx(ctx, nil)
+	if err != nil {
+		return wrap(err)
+	}
+	defer tx.Rollback() //nolint:errcheck
+
+	if res, err := tx.ExecContext(ctx, `
+		DELETE FROM blocklists
+		WHERE NOT EXISTS (
+			SELECT 1 FROM files WHERE files.blocklist_hash = blocklists.blocklist_hash
+		)`); err != nil {
+		return wrap(err, "delete blocklists")
+	} else if shouldDebug() {
+		rows, err := res.RowsAffected()
+		l.Debugln("Blocklist GC:", rows, err)
+	}
+
+	if res, err := tx.ExecContext(ctx, `
+		DELETE FROM blocks
+		WHERE NOT EXISTS (
+			SELECT 1 FROM blocklists WHERE blocklists.blocklist_hash = blocks.blocklist_hash
+		)`); err != nil {
+		return wrap(err, "delete blocks")
+	} else if shouldDebug() {
+		rows, err := res.RowsAffected()
+		l.Debugln("Blocks GC:", rows, err)
+	}
+
+	return wrap(tx.Commit())
+}

internal/db/sqlite/db_update.go

@@ -0,0 +1,600 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"cmp"
+	"context"
+	"fmt"
+	"runtime"
+	"slices"
+	"strings"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/syncthing/syncthing/internal/gen/dbproto"
+	"github.com/syncthing/syncthing/internal/itererr"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/sliceutil"
+	"google.golang.org/protobuf/proto"
+)
+
+const (
+	// Arbitrarily chosen values for checkpoint frequency....
+	updatePointsPerFile   = 100
+	updatePointsPerBlock  = 1
+	updatePointsThreshold = 250_000
+)
+
+func (s *DB) Update(folder string, device protocol.DeviceID, fs []protocol.FileInfo) error {
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+
+	folderIdx, err := s.folderIdxLocked(folder)
+	if err != nil {
+		return wrap(err)
+	}
+	deviceIdx, err := s.deviceIdxLocked(device)
+	if err != nil {
+		return wrap(err)
+	}
+
+	tx, err := s.sql.BeginTxx(context.Background(), nil)
+	if err != nil {
+		return wrap(err)
+	}
+	defer tx.Rollback() //nolint:errcheck
+	txp := &txPreparedStmts{Tx: tx}
+
+	//nolint:sqlclosecheck
+	insertFileStmt, err := txp.Preparex(`
+		INSERT OR REPLACE INTO files (folder_idx, device_idx, remote_sequence, name, type, modified, size, version, deleted, invalid, local_flags, blocklist_hash)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+		RETURNING sequence
+	`)
+	if err != nil {
+		return wrap(err, "prepare insert file")
+	}
+
+	//nolint:sqlclosecheck
+	insertFileInfoStmt, err := txp.Preparex(`
+		INSERT INTO fileinfos (sequence, fiprotobuf)
+		VALUES (?, ?)
+	`)
+	if err != nil {
+		return wrap(err, "prepare insert fileinfo")
+	}
+
+	//nolint:sqlclosecheck
+	insertBlockListStmt, err := txp.Preparex(`
+		INSERT OR IGNORE INTO blocklists (blocklist_hash, blprotobuf)
+		VALUES (?, ?)
+	`)
+	if err != nil {
+		return wrap(err, "prepare insert blocklist")
+	}
+
+	var prevRemoteSeq int64
+	for i, f := range fs {
+		f.Name = osutil.NormalizedFilename(f.Name)
+
+		var blockshash *[]byte
+		if len(f.Blocks) > 0 {
+			f.BlocksHash = protocol.BlocksHash(f.Blocks)
+			blockshash = &f.BlocksHash
+		} else {
+			f.BlocksHash = nil
+		}
+
+		if f.Type == protocol.FileInfoTypeDirectory {
+			f.Size = 128 // synthetic directory size
+		}
+
+		// Insert the file.
+		//
+		// If it is a remote file, set remote_sequence otherwise leave it at
+		// null. Returns the new local sequence.
+		var remoteSeq *int64
+		if device != protocol.LocalDeviceID {
+			if i > 0 && f.Sequence == prevRemoteSeq {
+				return fmt.Errorf("duplicate remote sequence number %d", prevRemoteSeq)
+			}
+			prevRemoteSeq = f.Sequence
+			remoteSeq = &f.Sequence
+		}
+		var localSeq int64
+		if err := insertFileStmt.Get(&localSeq, folderIdx, deviceIdx, remoteSeq, f.Name, f.Type, f.ModTime().UnixNano(), f.Size, f.Version.String(), f.IsDeleted(), f.IsInvalid(), f.LocalFlags, blockshash); err != nil {
+			return wrap(err, "insert file")
+		}
+
+		if len(f.Blocks) > 0 {
+			// Indirect the block list
+			blocks := sliceutil.Map(f.Blocks, protocol.BlockInfo.ToWire)
+			bs, err := proto.Marshal(&dbproto.BlockList{Blocks: blocks})
+			if err != nil {
+				return wrap(err, "marshal blocklist")
+			}
+			if _, err := insertBlockListStmt.Exec(f.BlocksHash, bs); err != nil {
+				return wrap(err, "insert blocklist")
+			}
+
+			if device == protocol.LocalDeviceID {
+				// Insert all blocks
+				if err := s.insertBlocksLocked(txp, f.BlocksHash, f.Blocks); err != nil {
+					return wrap(err, "insert blocks")
+				}
+			}
+
+			f.Blocks = nil
+		}
+
+		// Insert the fileinfo
+		if device == protocol.LocalDeviceID {
+			f.Sequence = localSeq
+		}
+		bs, err := proto.Marshal(f.ToWire(true))
+		if err != nil {
+			return wrap(err, "marshal fileinfo")
+		}
+		if _, err := insertFileInfoStmt.Exec(localSeq, bs); err != nil {
+			return wrap(err, "insert fileinfo")
+		}
+
+		// Update global and need
+		if err := s.recalcGlobalForFile(txp, folderIdx, f.Name); err != nil {
+			return wrap(err)
+		}
+	}
+
+	if err := tx.Commit(); err != nil {
+		return wrap(err)
+	}
+
+	s.periodicCheckpointLocked(fs)
+	return nil
+}
+
+func (s *DB) DropFolder(folder string) error {
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+	_, err := s.stmt(`
+		DELETE FROM folders
+		WHERE folder_id = ?
+	`).Exec(folder)
+	return wrap(err)
+}
+
+func (s *DB) DropDevice(device protocol.DeviceID) error {
+	if device == protocol.LocalDeviceID {
+		panic("bug: cannot drop local device")
+	}
+
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+
+	deviceIdx, err := s.deviceIdxLocked(device)
+	if err != nil {
+		return wrap(err)
+	}
+
+	tx, err := s.sql.BeginTxx(context.Background(), nil)
+	if err != nil {
+		return wrap(err)
+	}
+	defer tx.Rollback() //nolint:errcheck
+	txp := &txPreparedStmts{Tx: tx}
+
+	// Find all folders where the device is involved
+	var folderIdxs []int64
+	if err := tx.Select(&folderIdxs, `
+		SELECT folder_idx
+		FROM counts
+		WHERE device_idx = ? AND count > 0
+		GROUP BY folder_idx
+	`, deviceIdx); err != nil {
+		return wrap(err)
+	}
+
+	// Drop the device, which cascades to delete all files etc for it
+	if _, err := tx.Exec(`DELETE FROM devices WHERE device_id = ?`, device.String()); err != nil {
+		return wrap(err)
+	}
+
+	// Recalc the globals for all affected folders
+	for _, idx := range folderIdxs {
+		if err := s.recalcGlobalForFolder(txp, idx); err != nil {
+			return wrap(err)
+		}
+	}
+
+	return wrap(tx.Commit())
+}
+
+func (s *DB) DropAllFiles(folder string, device protocol.DeviceID) error {
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+
+	// This is a two part operation, first dropping all the files and then
+	// recalculating the global state for the entire folder.
+
+	folderIdx, err := s.folderIdxLocked(folder)
+	if err != nil {
+		return wrap(err)
+	}
+	deviceIdx, err := s.deviceIdxLocked(device)
+	if err != nil {
+		return wrap(err)
+	}
+
+	tx, err := s.sql.BeginTxx(context.Background(), nil)
+	if err != nil {
+		return wrap(err)
+	}
+	defer tx.Rollback() //nolint:errcheck
+	txp := &txPreparedStmts{Tx: tx}
+
+	// Drop all the file entries
+
+	result, err := tx.Exec(`
+		DELETE FROM files
+		WHERE folder_idx = ? AND device_idx = ?
+	`, folderIdx, deviceIdx)
+	if err != nil {
+		return wrap(err)
+	}
+	if n, err := result.RowsAffected(); err == nil && n == 0 {
+		// The delete affected no rows, so we don't need to redo the entire
+		// global/need calculation.
+		return wrap(tx.Commit())
+	}
+
+	// Recalc global for the entire folder
+
+	if err := s.recalcGlobalForFolder(txp, folderIdx); err != nil {
+		return wrap(err)
+	}
+	return wrap(tx.Commit())
+}
+
+func (s *DB) DropFilesNamed(folder string, device protocol.DeviceID, names []string) error {
+	for i := range names {
+		names[i] = osutil.NormalizedFilename(names[i])
+	}
+
+	s.updateLock.Lock()
+	defer s.updateLock.Unlock()
+
+	folderIdx, err := s.folderIdxLocked(folder)
+	if err != nil {
+		return wrap(err)
+	}
+	deviceIdx, err := s.deviceIdxLocked(device)
+	if err != nil {
+		return wrap(err)
+	}
+
+	tx, err := s.sql.BeginTxx(context.Background(), nil)
+	if err != nil {
+		return wrap(err)
+	}
+	defer tx.Rollback() //nolint:errcheck
+	txp := &txPreparedStmts{Tx: tx}
+
+	// Drop the named files
+
+	query, args, err := sqlx.In(`
+		DELETE FROM files
+		WHERE folder_idx = ? AND device_idx = ? AND name IN (?)
+	`, folderIdx, deviceIdx, names)
+	if err != nil {
+		return wrap(err)
+	}
+	if _, err := tx.Exec(query, args...); err != nil {
+		return wrap(err)
+	}
+
+	// Recalc globals for the named files
+
+	for _, name := range names {
+		if err := s.recalcGlobalForFile(txp, folderIdx, name); err != nil {
+			return wrap(err)
+		}
+	}
+
+	return wrap(tx.Commit())
+}
+
+func (*DB) insertBlocksLocked(tx *txPreparedStmts, blocklistHash []byte, blocks []protocol.BlockInfo) error {
+	if len(blocks) == 0 {
+		return nil
+	}
+	bs := make([]map[string]any, len(blocks))
+	for i, b := range blocks {
+		bs[i] = map[string]any{
+			"hash":           b.Hash,
+			"blocklist_hash": blocklistHash,
+			"idx":            i,
+			"offset":         b.Offset,
+			"size":           b.Size,
+		}
+	}
+
+	// Very large block lists (>8000 blocks) result in "too many variables"
+	// error. Chunk it to a reasonable size.
+	for chunk := range slices.Chunk(bs, 1000) {
+		if _, err := tx.NamedExec(`
+			INSERT OR IGNORE INTO blocks (hash, blocklist_hash, idx, offset, size)
+			VALUES (:hash, :blocklist_hash, :idx, :offset, :size)
+		`, chunk); err != nil {
+			return wrap(err)
+		}
+	}
+	return nil
+}
+
+func (s *DB) recalcGlobalForFolder(txp *txPreparedStmts, folderIdx int64) error {
+	// Select files where there is no global, those are the ones we need to
+	// recalculate.
+	//nolint:sqlclosecheck
+	namesStmt, err := txp.Preparex(`
+		SELECT f.name FROM files f
+		WHERE f.folder_idx = ? AND NOT EXISTS (
+			SELECT 1 FROM files g
+			WHERE g.folder_idx = ? AND g.name = f.name AND g.local_flags & ? != 0
+		)
+		GROUP BY name
+	`)
+	if err != nil {
+		return wrap(err)
+	}
+	rows, err := namesStmt.Queryx(folderIdx, folderIdx, protocol.FlagLocalGlobal)
+	if err != nil {
+		return wrap(err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var name string
+		if err := rows.Scan(&name); err != nil {
+			return wrap(err)
+		}
+		if err := s.recalcGlobalForFile(txp, folderIdx, name); err != nil {
+			return wrap(err)
+		}
+	}
+	return wrap(rows.Err())
+}
+
+func (s *DB) recalcGlobalForFile(txp *txPreparedStmts, folderIdx int64, file string) error {
+	//nolint:sqlclosecheck
+	selStmt, err := txp.Preparex(`
+		SELECT name, folder_idx, device_idx, sequence, modified, version, deleted, invalid, local_flags FROM files
+		WHERE folder_idx = ? AND name = ?
+	`)
+	if err != nil {
+		return wrap(err)
+	}
+	es, err := itererr.Collect(iterStructs[fileRow](selStmt.Queryx(folderIdx, file)))
+	if err != nil {
+		return wrap(err)
+	}
+	if len(es) == 0 {
+		// shouldn't happen
+		return nil
+	}
+
+	// Sort the entries; the global entry is at the head of the list
+	slices.SortFunc(es, fileRow.Compare)
+
+	// The global version is the first one in the list that is not invalid,
+	// or just the first one in the list if all are invalid.
+	var global fileRow
+	globIdx := slices.IndexFunc(es, func(e fileRow) bool { return !e.Invalid })
+	if globIdx < 0 {
+		globIdx = 0
+	}
+	global = es[globIdx]
+
+	// We "have" the file if the position in the list of versions is at the
+	// global version or better, or if the version is the same as the global
+	// file (we might be further down the list due to invalid flags), or if
+	// the global is deleted and we don't have it at all...
+	localIdx := slices.IndexFunc(es, func(e fileRow) bool { return e.DeviceIdx == s.localDeviceIdx })
+	hasLocal := localIdx >= 0 && localIdx <= globIdx || // have a better or equal version
+		localIdx >= 0 && es[localIdx].Version.Equal(global.Version.Vector) || // have an equal version but invalid/ignored
+		localIdx < 0 && global.Deleted // missing it, but the global is also deleted
+
+	// Set the global flag on the global entry. Set the need flag if the
+	// local device needs this file, unless it's invalid.
+	global.LocalFlags |= protocol.FlagLocalGlobal
+	if hasLocal || global.Invalid {
+		global.LocalFlags &= ^protocol.FlagLocalNeeded
+	} else {
+		global.LocalFlags |= protocol.FlagLocalNeeded
+	}
+	//nolint:sqlclosecheck
+	upStmt, err := txp.Prepare(`
+		UPDATE files SET local_flags = ?
+		WHERE folder_idx = ? AND device_idx = ? AND sequence = ?
+	`)
+	if err != nil {
+		return wrap(err)
+	}
+	if _, err := upStmt.Exec(global.LocalFlags, global.FolderIdx, global.DeviceIdx, global.Sequence); err != nil {
+		return wrap(err)
+	}
+
+	// Clear the need and global flags on all other entries
+	//nolint:sqlclosecheck
+	upStmt, err = txp.Prepare(`
+		UPDATE files SET local_flags = local_flags & ?
+		WHERE folder_idx = ? AND name = ? AND sequence != ? AND local_flags & ? != 0
+	`)
+	if err != nil {
+		return wrap(err)
+	}
+	if _, err := upStmt.Exec(^(protocol.FlagLocalNeeded | protocol.FlagLocalGlobal), folderIdx, global.Name, global.Sequence, protocol.FlagLocalNeeded|protocol.FlagLocalGlobal); err != nil {
+		return wrap(err)
+	}
+
+	return nil
+}
+
+func (s *DB) folderIdxLocked(folderID string) (int64, error) {
+	if _, err := s.stmt(`
+		INSERT OR IGNORE INTO folders(folder_id)
+		VALUES (?)
+	`).Exec(folderID); err != nil {
+		return 0, wrap(err)
+	}
+	var idx int64
+	if err := s.stmt(`
+		SELECT idx FROM folders
+		WHERE folder_id = ?
+	`).Get(&idx, folderID); err != nil {
+		return 0, wrap(err)
+	}
+
+	return idx, nil
+}
+
+func (s *DB) deviceIdxLocked(deviceID protocol.DeviceID) (int64, error) {
+	devStr := deviceID.String()
+	if _, err := s.stmt(`
+		INSERT OR IGNORE INTO devices(device_id)
+		VALUES (?)
+	`).Exec(devStr); err != nil {
+		return 0, wrap(err)
+	}
+	var idx int64
+	if err := s.stmt(`
+		SELECT idx FROM devices
+		WHERE device_id = ?
+	`).Get(&idx, devStr); err != nil {
+		return 0, wrap(err)
+	}
+
+	return idx, nil
+}
+
+// wrap returns the error wrapped with the calling function name and
+// optional extra context strings as prefix. A nil error wraps to nil.
+func wrap(err error, context ...string) error {
+	if err == nil {
+		return nil
+	}
+
+	prefix := "error"
+	pc, _, _, ok := runtime.Caller(1)
+	details := runtime.FuncForPC(pc)
+	if ok && details != nil {
+		prefix = strings.ToLower(details.Name())
+		if dotIdx := strings.LastIndex(prefix, "."); dotIdx > 0 {
+			prefix = prefix[dotIdx+1:]
+		}
+	}
+
+	if len(context) > 0 {
+		for i := range context {
+			context[i] = strings.TrimSpace(context[i])
+		}
+		extra := strings.Join(context, ", ")
+		return fmt.Errorf("%s (%s): %w", prefix, extra, err)
+	}
+
+	return fmt.Errorf("%s: %w", prefix, err)
+}
+
+type fileRow struct {
+	Name       string
+	Version    dbVector
+	FolderIdx  int64 `db:"folder_idx"`
+	DeviceIdx  int64 `db:"device_idx"`
+	Sequence   int64
+	Modified   int64
+	Size       int64
+	LocalFlags int64 `db:"local_flags"`
+	Deleted    bool
+	Invalid    bool
+}
+
+func (e fileRow) Compare(other fileRow) int {
+	// From FileInfo.WinsConflict
+	vc := e.Version.Vector.Compare(other.Version.Vector)
+	switch vc {
+	case protocol.Equal:
+		if e.Invalid != other.Invalid {
+			if e.Invalid {
+				return 1
+			}
+			return -1
+		}
+
+		// Compare the device ID index, lower is better. This is only
+		// deterministic to the extent that LocalDeviceID will always be the
+		// lowest one, order between remote devices is random (and
+		// irrelevant).
+		return cmp.Compare(e.DeviceIdx, other.DeviceIdx)
+	case protocol.Greater: // we are newer
+		return -1
+	case protocol.Lesser: // we are older
+		return 1
+	case protocol.ConcurrentGreater, protocol.ConcurrentLesser: // there is a conflict
+		if e.Invalid != other.Invalid {
+			if e.Invalid { // we are invalid, we lose
+				return 1
+			}
+			return -1 // they are invalid, we win
+		}
+		if e.Deleted != other.Deleted {
+			if e.Deleted { // we are deleted, we lose
+				return 1
+			}
+			return -1 // they are deleted, we win
+		}
+		if d := cmp.Compare(e.Modified, other.Modified); d != 0 {
+			return -d // positive d means we were newer, so we win (negative return)
+		}
+		if vc == protocol.ConcurrentGreater {
+			return -1 // we have a better device ID, we win
+		}
+		return 1 // they win
+	default:
+		return 0
+	}
+}
+
+func (s *DB) periodicCheckpointLocked(fs []protocol.FileInfo) {
+	// Induce periodic checkpoints. We add points for each file and block,
+	// and checkpoint when we've written more than a threshold of points.
+	// This ensures we do not go too long without a checkpoint, while also
+	// not doing it incessantly for every update.
+	s.updatePoints += updatePointsPerFile * len(fs)
+	for _, f := range fs {
+		s.updatePoints += len(f.Blocks) * updatePointsPerBlock
+	}
+	if s.updatePoints > updatePointsThreshold {
+		conn, err := s.sql.Conn(context.Background())
+		if err != nil {
+			l.Debugln("conn:", err)
+			return
+		}
+		defer conn.Close()
+		if _, err := conn.ExecContext(context.Background(), `PRAGMA journal_size_limit = 67108864`); err != nil {
+			l.Debugln("PRAGMA journal_size_limit:", err)
+		}
+		row := conn.QueryRowContext(context.Background(), `PRAGMA wal_checkpoint(RESTART)`)
+		var res, modified, moved int
+		if row.Err() != nil {
+			l.Debugln("PRAGMA wal_checkpoint(RESTART):", err)
+		} else if err := row.Scan(&res, &modified, &moved); err != nil {
+			l.Debugln("PRAGMA wal_checkpoint(RESTART) (scan):", err)
+		} else {
+			l.Debugln("checkpoint at", s.updatePoints, "returned", res, modified, moved)
+		}
+		s.updatePoints = 0
+	}
+}

internal/db/sqlite/debug.go

@@ -1,17 +1,15 @@
-// Copyright (C) 2014 The Syncthing Authors.
+// Copyright (C) 2025 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package db
+package sqlite
 
 import (
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
-var l = logger.DefaultLogger.NewFacility("db", "The database layer")
+var l = logger.DefaultLogger.NewFacility("sqlite", "SQLite database")
 
-func shouldDebug() bool {
-	return l.ShouldDebug("db")
-}
+func shouldDebug() bool { return l.ShouldDebug("sqlite") }

internal/db/sqlite/sql/README.md

@@ -0,0 +1,8 @@
+These SQL scripts are embedded in the binary.
+
+Scripts in `schema/` are run at every startup, in alphanumerical order.
+
+Scripts in `migrations/` are run when a migration is needed; the must begin
+with a number that equals the schema version that results from that
+migration. Migrations are not run on initial database creation, so the
+scripts in `schema/` should create the latest version.

internal/db/sqlite/sql/migrations/01-placeholder.sql

@@ -0,0 +1,7 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- The next migration should be number two.

internal/db/sqlite/sql/schema/00-indexes.sql

@@ -0,0 +1,19 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- folders map folder IDs as used by Syncthing to database folder indexes
+CREATE TABLE IF NOT EXISTS folders (
+    idx INTEGER NOT NULL PRIMARY KEY,
+    folder_id TEXT NOT NULL UNIQUE COLLATE BINARY
+) STRICT
+;
+
+-- devices map device IDs as used by Syncthing to database device indexes
+CREATE TABLE IF NOT EXISTS devices (
+    idx INTEGER NOT NULL PRIMARY KEY,
+    device_id TEXT NOT NULL UNIQUE COLLATE BINARY
+) STRICT
+;

internal/db/sqlite/sql/schema/10-schema.sql

@@ -0,0 +1,14 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- Schema migrations hold the list of historical migrations applied
+CREATE TABLE IF NOT EXISTS schemamigrations (
+    schema_version INTEGER NOT NULL,
+    applied_at INTEGER NOT NULL, -- unix nanos
+    syncthing_version TEXT NOT NULL COLLATE BINARY,
+    PRIMARY KEY(schema_version)
+) STRICT
+;

internal/db/sqlite/sql/schema/20-files.sql

@@ -0,0 +1,62 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- Files
+--
+-- The files table contains all files announced by any device. Files present
+-- on this device are filed under the LocalDeviceID, not the actual current
+-- device ID, for simplicity, consistency and portability. One announced
+-- version of each file is considered the "global" version - the latest one,
+-- that all other devices strive to replicate. This instance gets the Global
+-- flag bit set. There may be other identical instances of this file
+-- announced by other devices, but only one onstance gets the Global flag;
+-- this simplifies accounting. If the current device has the Global version,
+-- the LocalDeviceID instance of the file is the one that has the Global
+-- bit.
+--
+-- If the current device does not have that version of the file it gets the
+-- Need bit set. Only Global files announced by another device can have the
+-- Need bit. This allows for very efficient lookup of files needing handling
+-- on this device, which is a common query.
+CREATE TABLE IF NOT EXISTS files (
+    folder_idx INTEGER NOT NULL,
+    device_idx INTEGER NOT NULL, -- actual device ID or LocalDeviceID
+    sequence INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, -- our local database sequence, for each and every entry
+    remote_sequence INTEGER, -- remote device's sequence number, null for local or synthetic entries
+    name TEXT NOT NULL COLLATE BINARY,
+    type INTEGER NOT NULL, -- protocol.FileInfoType
+    modified INTEGER NOT NULL, -- Unix nanos
+    size INTEGER NOT NULL,
+    version TEXT NOT NULL COLLATE BINARY,
+    deleted INTEGER NOT NULL, -- boolean
+    invalid INTEGER NOT NULL, -- boolean
+    local_flags INTEGER NOT NULL,
+    blocklist_hash BLOB, -- null when there are no blocks
+    FOREIGN KEY(device_idx) REFERENCES devices(idx) ON DELETE CASCADE,
+    FOREIGN KEY(folder_idx) REFERENCES folders(idx) ON DELETE CASCADE
+) STRICT
+;
+-- FileInfos store the actual protobuf object. We do this separately to keep
+-- the files rows smaller and more efficient.
+CREATE TABLE IF NOT EXISTS fileinfos (
+    sequence INTEGER NOT NULL PRIMARY KEY, -- our local database sequence from the files table
+    fiprotobuf BLOB NOT NULL,
+    FOREIGN KEY(sequence) REFERENCES files(sequence) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED
+) STRICT
+;
+-- There can be only one file per folder, device, and remote sequence number
+CREATE UNIQUE INDEX IF NOT EXISTS files_remote_sequence ON files (folder_idx, device_idx, remote_sequence)
+    WHERE remote_sequence IS NOT NULL
+;
+-- There can be only one file per folder, device, and name
+CREATE UNIQUE INDEX IF NOT EXISTS files_device_name ON files (folder_idx, device_idx, name)
+;
+-- We want to be able to look up & iterate files based on just folder and name
+CREATE INDEX IF NOT EXISTS files_name_only ON files (folder_idx, name)
+;
+-- We want to be able to look up & iterate files based on blocks hash
+CREATE INDEX IF NOT EXISTS files_blocklist_hash_only ON files (blocklist_hash, device_idx, folder_idx) WHERE blocklist_hash IS NOT NULL
+;

internal/db/sqlite/sql/schema/30-indexids.sql

@@ -0,0 +1,24 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- indexids holds the index ID and maximum sequence for a given device and folder
+CREATE TABLE IF NOT EXISTS indexids (
+    device_idx INTEGER NOT NULL,
+    folder_idx INTEGER NOT NULL,
+    index_id TEXT NOT NULL COLLATE BINARY,
+    sequence INTEGER NOT NULL DEFAULT 0,
+    PRIMARY KEY(device_idx, folder_idx),
+    FOREIGN KEY(folder_idx) REFERENCES folders(idx) ON DELETE CASCADE,
+    FOREIGN KEY(device_idx) REFERENCES devices(idx) ON DELETE CASCADE
+) STRICT, WITHOUT ROWID
+;
+CREATE TRIGGER IF NOT EXISTS indexids_seq AFTER INSERT ON files
+BEGIN
+    INSERT INTO indexids (folder_idx, device_idx, index_id, sequence)
+        VALUES (NEW.folder_idx, NEW.device_idx, "", COALESCE(NEW.remote_sequence, NEW.sequence))
+        ON CONFLICT DO UPDATE SET sequence = COALESCE(NEW.remote_sequence, NEW.sequence);
+END
+;

internal/db/sqlite/sql/schema/40-counts.sql

@@ -0,0 +1,53 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- Counts
+--
+-- Counts and sizes are maintained for each device, folder, type, flag bits
+-- combination.
+CREATE TABLE IF NOT EXISTS counts (
+    folder_idx INTEGER NOT NULL,
+    device_idx INTEGER NOT NULL,
+    type INTEGER NOT NULL,
+    local_flags INTEGER NOT NULL,
+    count INTEGER NOT NULL,
+    size INTEGER NOT NULL,
+    deleted INTEGER NOT NULL, -- boolean
+    PRIMARY KEY(folder_idx, device_idx, type, local_flags, deleted),
+    FOREIGN KEY(device_idx) REFERENCES devices(idx) ON DELETE CASCADE,
+    FOREIGN KEY(folder_idx) REFERENCES folders(idx) ON DELETE CASCADE
+) STRICT, WITHOUT ROWID
+;
+
+--- Maintain counts when files are added and removed using triggers
+
+CREATE TRIGGER IF NOT EXISTS counts_insert AFTER INSERT ON files
+BEGIN
+    INSERT INTO counts (folder_idx, device_idx, type, local_flags, count, size, deleted)
+        VALUES (NEW.folder_idx, NEW.device_idx, NEW.type, NEW.local_flags, 1, NEW.size, NEW.deleted)
+        ON CONFLICT DO UPDATE SET count = count + 1, size = size + NEW.size;
+END
+;
+CREATE TRIGGER IF NOT EXISTS counts_delete AFTER DELETE ON files
+BEGIN
+    UPDATE counts SET count = count - 1, size = size - OLD.size
+        WHERE folder_idx = OLD.folder_idx AND device_idx = OLD.device_idx AND type = OLD.type AND local_flags = OLD.local_flags AND deleted = OLD.deleted;
+END
+;
+CREATE TRIGGER IF NOT EXISTS counts_update AFTER UPDATE OF local_flags ON files
+WHEN NEW.local_flags != OLD.local_flags
+BEGIN
+    INSERT INTO counts (folder_idx, device_idx, type, local_flags, count, size, deleted)
+        VALUES (NEW.folder_idx, NEW.device_idx, NEW.type, NEW.local_flags, 1, NEW.size, NEW.deleted)
+        ON CONFLICT DO UPDATE SET count = count + 1, size = size + NEW.size;
+    UPDATE counts SET count = count - 1, size = size - OLD.size
+        WHERE folder_idx = OLD.folder_idx AND device_idx = OLD.device_idx AND type = OLD.type AND local_flags = OLD.local_flags AND deleted = OLD.deleted;
+END
+;
+DROP TRIGGER IF EXISTS counts_update_add -- tmp migration
+;
+DROP TRIGGER IF EXISTS counts_update_del -- tmp migration
+;

internal/db/sqlite/sql/schema/50-blocks.sql

@@ -0,0 +1,34 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+-- Block lists
+--
+-- The block lists are extracted from FileInfos and stored separately. This
+-- reduces the database size by reusing the same block list entry for all
+-- devices announcing the same file. Doing it for all block lists instead of
+-- using a size cutoff simplifies queries. Block lists are garbage collected
+-- "manually", not using a trigger as that was too performance impacting.
+CREATE TABLE IF NOT EXISTS blocklists (
+    blocklist_hash BLOB NOT NULL PRIMARY KEY,
+    blprotobuf BLOB NOT NULL
+) STRICT
+;
+
+-- Blocks
+--
+-- For all local files we store the blocks individually for quick lookup. A
+-- given block can exist in multiple blocklists and at multiple offsets in a
+-- blocklist.
+CREATE TABLE IF NOT EXISTS blocks (
+    hash BLOB NOT NULL,
+    blocklist_hash BLOB NOT NULL,
+    idx INTEGER NOT NULL,
+    offset INTEGER NOT NULL,
+    size INTEGER NOT NULL,
+    PRIMARY KEY (hash, blocklist_hash, idx),
+    FOREIGN KEY(blocklist_hash) REFERENCES blocklists(blocklist_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED
+) STRICT
+;

internal/db/sqlite/sql/schema/50-mtimes.sql

@@ -0,0 +1,16 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+--- Backing for the MtimeFS
+CREATE TABLE IF NOT EXISTS mtimes (
+    folder_idx INTEGER NOT NULL,
+    name TEXT NOT NULL,
+    ondisk INTEGER NOT NULL, -- unix nanos
+    virtual INTEGER NOT NULL, -- unix nanos
+    PRIMARY KEY(folder_idx, name),
+    FOREIGN KEY(folder_idx) REFERENCES folders(idx) ON DELETE CASCADE
+) STRICT, WITHOUT ROWID
+;

internal/db/sqlite/sql/schema/70-kv.sql

@@ -0,0 +1,13 @@
+-- Copyright (C) 2025 The Syncthing Authors.
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
+-- You can obtain one at https://mozilla.org/MPL/2.0/.
+
+--- Simple KV store. This backs the "miscDB" we use for certain minor pieces
+--  of data.
+CREATE TABLE IF NOT EXISTS kv (
+    key TEXT NOT NULL PRIMARY KEY COLLATE BINARY,
+    value BLOB NOT NULL
+) STRICT
+;

internal/db/sqlite/util.go

@@ -0,0 +1,117 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package sqlite
+
+import (
+	"database/sql/driver"
+	"errors"
+	"iter"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/syncthing/syncthing/internal/gen/bep"
+	"github.com/syncthing/syncthing/internal/gen/dbproto"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"google.golang.org/protobuf/proto"
+)
+
+// iterStructs returns an iterator over the given struct type by scanning
+// the SQL rows. `rows` is closed when the iterator exits.
+func iterStructs[T any](rows *sqlx.Rows, err error) (iter.Seq[T], func() error) {
+	if err != nil {
+		return func(_ func(T) bool) {}, func() error { return err }
+	}
+
+	var retErr error
+	return func(yield func(T) bool) {
+		defer rows.Close()
+		for rows.Next() {
+			v := new(T)
+			if err := rows.StructScan(v); err != nil {
+				retErr = err
+				break
+			}
+			if cleanuper, ok := any(v).(interface{ cleanup() }); ok {
+				cleanuper.cleanup()
+			}
+			if !yield(*v) {
+				return
+			}
+		}
+		if err := rows.Err(); err != nil && retErr == nil {
+			retErr = err
+		}
+	}, func() error { return retErr }
+}
+
+// dbVector is a wrapper that allows protocol.Vector values to be serialized
+// to and from the database.
+type dbVector struct { //nolint:recvcheck
+	protocol.Vector
+}
+
+func (v dbVector) Value() (driver.Value, error) {
+	return v.String(), nil
+}
+
+func (v *dbVector) Scan(value any) error {
+	str, ok := value.(string)
+	if !ok {
+		return errors.New("not a string")
+	}
+	if str == "" {
+		v.Vector = protocol.Vector{}
+		return nil
+	}
+	vec, err := protocol.VectorFromString(str)
+	if err != nil {
+		return wrap(err)
+	}
+	v.Vector = vec
+
+	return nil
+}
+
+// indirectFI constructs a FileInfo from separate marshalled FileInfo and
+// BlockList bytes.
+type indirectFI struct {
+	Name       string // not used, must be present as dest for Need iterator
+	FiProtobuf []byte
+	BlProtobuf []byte
+	Size       int64 // not used
+	Modified   int64 // not used
+}
+
+func (i indirectFI) FileInfo() (protocol.FileInfo, error) {
+	var fi bep.FileInfo
+	if err := proto.Unmarshal(i.FiProtobuf, &fi); err != nil {
+		return protocol.FileInfo{}, wrap(err, "unmarshal fileinfo")
+	}
+	if len(i.BlProtobuf) > 0 {
+		var bl dbproto.BlockList
+		if err := proto.Unmarshal(i.BlProtobuf, &bl); err != nil {
+			return protocol.FileInfo{}, wrap(err, "unmarshal blocklist")
+		}
+		fi.Blocks = bl.Blocks
+	}
+	fi.Name = osutil.NativeFilename(fi.Name)
+	return protocol.FileInfoFromDB(&fi), nil
+}
+
+func prefixEnd(s string) string {
+	if s == "" {
+		panic("bug: cannot represent end prefix for empty string")
+	}
+	bs := []byte(s)
+	for i := len(bs) - 1; i >= 0; i-- {
+		if bs[i] < 0xff {
+			bs[i]++
+			break
+		}
+	}
+	return string(bs)
+}

internal/db/typed.go

@@ -0,0 +1,140 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"database/sql"
+	"encoding/binary"
+	"errors"
+	"time"
+)
+
+// Typed is a simple key-value store using a specific namespace within a
+// lower level KV.
+type Typed struct {
+	db     KV
+	prefix string
+}
+
+func NewMiscDB(db KV) *Typed {
+	return NewTyped(db, "misc")
+}
+
+// NewTyped returns a new typed key-value store that lives in the namespace
+// specified by the prefix.
+func NewTyped(db KV, prefix string) *Typed {
+	return &Typed{
+		db:     db,
+		prefix: prefix,
+	}
+}
+
+// PutInt64 stores a new int64. Any existing value (even if of another type)
+// is overwritten.
+func (n *Typed) PutInt64(key string, val int64) error {
+	var valBs [8]byte
+	binary.BigEndian.PutUint64(valBs[:], uint64(val))
+	return n.db.PutKV(n.prefixedKey(key), valBs[:])
+}
+
+// Int64 returns the stored value interpreted as an int64 and a boolean that
+// is false if no value was stored at the key.
+func (n *Typed) Int64(key string) (int64, bool, error) {
+	valBs, err := n.db.GetKV(n.prefixedKey(key))
+	if err != nil {
+		return 0, false, filterNotFound(err)
+	}
+	val := binary.BigEndian.Uint64(valBs)
+	return int64(val), true, nil
+}
+
+// PutTime stores a new time.Time. Any existing value (even if of another
+// type) is overwritten.
+func (n *Typed) PutTime(key string, val time.Time) error {
+	valBs, _ := val.MarshalBinary() // never returns an error
+	return n.db.PutKV(n.prefixedKey(key), valBs)
+}
+
+// Time returns the stored value interpreted as a time.Time and a boolean
+// that is false if no value was stored at the key.
+func (n *Typed) Time(key string) (time.Time, bool, error) {
+	var t time.Time
+	valBs, err := n.db.GetKV(n.prefixedKey(key))
+	if err != nil {
+		return t, false, filterNotFound(err)
+	}
+	err = t.UnmarshalBinary(valBs)
+	return t, err == nil, err
+}
+
+// PutString stores a new string. Any existing value (even if of another type)
+// is overwritten.
+func (n *Typed) PutString(key, val string) error {
+	return n.db.PutKV(n.prefixedKey(key), []byte(val))
+}
+
+// String returns the stored value interpreted as a string and a boolean that
+// is false if no value was stored at the key.
+func (n *Typed) String(key string) (string, bool, error) {
+	valBs, err := n.db.GetKV(n.prefixedKey(key))
+	if err != nil {
+		return "", false, filterNotFound(err)
+	}
+	return string(valBs), true, nil
+}
+
+// PutBytes stores a new byte slice. Any existing value (even if of another type)
+// is overwritten.
+func (n *Typed) PutBytes(key string, val []byte) error {
+	return n.db.PutKV(n.prefixedKey(key), val)
+}
+
+// Bytes returns the stored value as a raw byte slice and a boolean that
+// is false if no value was stored at the key.
+func (n *Typed) Bytes(key string) ([]byte, bool, error) {
+	valBs, err := n.db.GetKV(n.prefixedKey(key))
+	if err != nil {
+		return nil, false, filterNotFound(err)
+	}
+	return valBs, true, nil
+}
+
+// PutBool stores a new boolean. Any existing value (even if of another type)
+// is overwritten.
+func (n *Typed) PutBool(key string, val bool) error {
+	if val {
+		return n.db.PutKV(n.prefixedKey(key), []byte{0x0})
+	}
+	return n.db.PutKV(n.prefixedKey(key), []byte{0x1})
+}
+
+// Bool returns the stored value as a boolean and a boolean that
+// is false if no value was stored at the key.
+func (n *Typed) Bool(key string) (bool, bool, error) {
+	valBs, err := n.db.GetKV(n.prefixedKey(key))
+	if err != nil {
+		return false, false, filterNotFound(err)
+	}
+	return valBs[0] == 0x0, true, nil
+}
+
+// Delete deletes the specified key. It is allowed to delete a nonexistent
+// key.
+func (n *Typed) Delete(key string) error {
+	return n.db.DeleteKV(n.prefixedKey(key))
+}
+
+func (n *Typed) prefixedKey(key string) string {
+	return n.prefix + "/" + key
+}
+
+func filterNotFound(err error) error {
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil
+	}
+	return err
+}

internal/db/typed_test.go

@@ -0,0 +1,115 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/db"
+	"github.com/syncthing/syncthing/internal/db/sqlite"
+)
+
+func TestNamespacedInt(t *testing.T) {
+	t.Parallel()
+
+	ldb, err := sqlite.OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		ldb.Close()
+	})
+
+	n1 := db.NewTyped(ldb, "foo")
+	n2 := db.NewTyped(ldb, "bar")
+
+	t.Run("Int", func(t *testing.T) {
+		t.Parallel()
+
+		// Key is missing to start with
+
+		if v, ok, err := n1.Int64("testint"); err != nil {
+			t.Error("Unexpected error:", err)
+		} else if v != 0 || ok {
+			t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
+		}
+
+		if err := n1.PutInt64("testint", 42); err != nil {
+			t.Fatal(err)
+		}
+
+		// It should now exist in n1
+
+		if v, ok, err := n1.Int64("testint"); err != nil {
+			t.Error("Unexpected error:", err)
+		} else if v != 42 || !ok {
+			t.Errorf("Incorrect return v %v != 42 || ok %v != true", v, ok)
+		}
+
+		// ... but not in n2, which is in a different namespace
+
+		if v, ok, err := n2.Int64("testint"); err != nil {
+			t.Error("Unexpected error:", err)
+		} else if v != 0 || ok {
+			t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
+		}
+
+		if err := n1.Delete("testint"); err != nil {
+			t.Fatal(err)
+		}
+
+		// It should no longer exist
+
+		if v, ok, err := n1.Int64("testint"); err != nil {
+			t.Error("Unexpected error:", err)
+		} else if v != 0 || ok {
+			t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
+		}
+	})
+
+	t.Run("Time", func(t *testing.T) {
+		t.Parallel()
+
+		if v, ok, err := n1.Time("testtime"); err != nil {
+			t.Error("Unexpected error:", err)
+		} else if !v.IsZero() || ok {
+			t.Errorf("Incorrect return v %v != %v || ok %v != false", v, time.Time{}, ok)
+		}
+
+		now := time.Now()
+		if err := n1.PutTime("testtime", now); err != nil {
+			t.Fatal(err)
+		}
+
+		if v, ok, err := n1.Time("testtime"); err != nil {
+			t.Error("Unexpected error:", err)
+		} else if !v.Equal(now) || !ok {
+			t.Errorf("Incorrect return v %v != %v || ok %v != true", v, now, ok)
+		}
+	})
+
+	t.Run("String", func(t *testing.T) {
+		t.Parallel()
+
+		if v, ok, err := n1.String("teststring"); err != nil {
+			t.Error("Unexpected error:", err)
+		} else if v != "" || ok {
+			t.Errorf("Incorrect return v %q != \"\" || ok %v != false", v, ok)
+		}
+
+		if err := n1.PutString("teststring", "yo"); err != nil {
+			t.Fatal(err)
+		}
+
+		if v, ok, err := n1.String("teststring"); err != nil {
+			t.Error("Unexpected error:", err)
+		} else if v != "yo" || !ok {
+			t.Errorf("Incorrect return v %q != \"yo\" || ok %v != true", v, ok)
+		}
+	})
+}

internal/gen/bep/bep.pb.go

@@ -1093,10 +1093,9 @@ type BlockInfo struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Hash     []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"`
-	Offset   int64  `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"`
-	Size     int32  `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
-	WeakHash uint32 `protobuf:"varint,4,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"`
+	Hash   []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"`
+	Offset int64  `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"`
+	Size   int32  `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
 }
 
 func (x *BlockInfo) Reset() {
@@ -1150,13 +1149,6 @@ func (x *BlockInfo) GetSize() int32 {
 	return 0
 }
 
-func (x *BlockInfo) GetWeakHash() uint32 {
-	if x != nil {
-		return x.WeakHash
-	}
-	return 0
-}
-
 type Vector struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -1578,7 +1570,6 @@ type Request struct {
 	Size          int32  `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"`
 	Hash          []byte `protobuf:"bytes,6,opt,name=hash,proto3" json:"hash,omitempty"`
 	FromTemporary bool   `protobuf:"varint,7,opt,name=from_temporary,json=fromTemporary,proto3" json:"from_temporary,omitempty"`
-	WeakHash      uint32 `protobuf:"varint,8,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"`
 	BlockNo       int32  `protobuf:"varint,9,opt,name=block_no,json=blockNo,proto3" json:"block_no,omitempty"`
 }
 
@@ -1661,13 +1652,6 @@ func (x *Request) GetFromTemporary() bool {
 	return false
 }
 
-func (x *Request) GetWeakHash() uint32 {
-	if x != nil {
-		return x.WeakHash
-	}
-	return 0
-}
-
 func (x *Request) GetBlockNo() int32 {
 	if x != nil {
 		return x.BlockNo
@@ -2079,158 +2063,155 @@ var file_bep_bep_proto_rawDesc = []byte{
 	0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x6e, 0x76,
 	0x61, 0x6c, 0x69, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x6e, 0x6f, 0x5f, 0x70, 0x65, 0x72, 0x6d, 0x69,
 	0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e, 0x6f,
-	0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x68, 0x0a, 0x09, 0x42,
+	0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x51, 0x0a, 0x09, 0x42,
 	0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68,
 	0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x16, 0x0a, 0x06,
 	0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66,
 	0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01,
-	0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x65, 0x61, 0x6b,
-	0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x77, 0x65, 0x61,
-	0x6b, 0x48, 0x61, 0x73, 0x68, 0x22, 0x32, 0x0a, 0x06, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12,
-	0x28, 0x0a, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28,
-	0x0b, 0x32, 0x0c, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x52,
-	0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x22, 0x2f, 0x0a, 0x07, 0x43, 0x6f, 0x75,
-	0x6e, 0x74, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04,
-	0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xfd, 0x01, 0x0a, 0x0c, 0x50,
-	0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x04, 0x75,
-	0x6e, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x62, 0x65, 0x70, 0x2e,
-	0x55, 0x6e, 0x69, 0x78, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x75, 0x6e, 0x69, 0x78, 0x12, 0x2a,
-	0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x10, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x44, 0x61, 0x74,
-	0x61, 0x52, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x12, 0x24, 0x0a, 0x05, 0x6c, 0x69,
-	0x6e, 0x75, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e,
-	0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x75, 0x78,
-	0x12, 0x26, 0x0a, 0x06, 0x64, 0x61, 0x72, 0x77, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b,
-	0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61,
-	0x52, 0x06, 0x64, 0x61, 0x72, 0x77, 0x69, 0x6e, 0x12, 0x28, 0x0a, 0x07, 0x66, 0x72, 0x65, 0x65,
-	0x62, 0x73, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e,
-	0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x07, 0x66, 0x72, 0x65, 0x65, 0x62,
-	0x73, 0x64, 0x12, 0x26, 0x0a, 0x06, 0x6e, 0x65, 0x74, 0x62, 0x73, 0x64, 0x18, 0x06, 0x20, 0x01,
+	0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x32,
+	0x0a, 0x06, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x28, 0x0a, 0x08, 0x63, 0x6f, 0x75, 0x6e,
+	0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x62, 0x65, 0x70,
+	0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x52, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65,
+	0x72, 0x73, 0x22, 0x2f, 0x0a, 0x07, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x0e, 0x0a,
+	0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a,
+	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61,
+	0x6c, 0x75, 0x65, 0x22, 0xfd, 0x01, 0x0a, 0x0c, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d,
+	0x44, 0x61, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x04, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x55, 0x6e, 0x69, 0x78, 0x44, 0x61, 0x74,
+	0x61, 0x52, 0x04, 0x75, 0x6e, 0x69, 0x78, 0x12, 0x2a, 0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f,
+	0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x57,
+	0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x44, 0x61, 0x74, 0x61, 0x52, 0x07, 0x77, 0x69, 0x6e, 0x64,
+	0x6f, 0x77, 0x73, 0x12, 0x24, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x75, 0x78, 0x18, 0x03, 0x20, 0x01,
 	0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61,
-	0x74, 0x61, 0x52, 0x06, 0x6e, 0x65, 0x74, 0x62, 0x73, 0x64, 0x22, 0x6c, 0x0a, 0x08, 0x55, 0x6e,
-	0x69, 0x78, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f,
-	0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65,
-	0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e,
-	0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70,
-	0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28,
-	0x05, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x04, 0x20,
-	0x01, 0x28, 0x05, 0x52, 0x03, 0x67, 0x69, 0x64, 0x22, 0x52, 0x0a, 0x0b, 0x57, 0x69, 0x6e, 0x64,
-	0x6f, 0x77, 0x73, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72,
-	0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e,
-	0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f,
-	0x69, 0x73, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c,
-	0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x2f, 0x0a, 0x09,
-	0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x12, 0x22, 0x0a, 0x06, 0x78, 0x61, 0x74,
-	0x74, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x62, 0x65, 0x70, 0x2e,
-	0x58, 0x61, 0x74, 0x74, 0x72, 0x52, 0x06, 0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x22, 0x31, 0x0a,
-	0x05, 0x58, 0x61, 0x74, 0x74, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61,
-	0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
-	0x22, 0xe4, 0x01, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02,
-	0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06,
-	0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f,
-	0x6c, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73,
-	0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74,
-	0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04,
-	0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x06, 0x20, 0x01,
-	0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x72, 0x6f, 0x6d,
-	0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x72, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08,
-	0x52, 0x0d, 0x66, 0x72, 0x6f, 0x6d, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x72, 0x79, 0x12,
-	0x1b, 0x0a, 0x09, 0x77, 0x65, 0x61, 0x6b, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x08, 0x20, 0x01,
-	0x28, 0x0d, 0x52, 0x08, 0x77, 0x65, 0x61, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x12, 0x19, 0x0a, 0x08,
-	0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x6f, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07,
-	0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x6f, 0x22, 0x52, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52,
-	0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18,
-	0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x45, 0x72, 0x72, 0x6f,
-	0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x65, 0x0a, 0x10, 0x44,
-	0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12,
-	0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x39, 0x0a, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74,
-	0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x46,
+	0x74, 0x61, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x75, 0x78, 0x12, 0x26, 0x0a, 0x06, 0x64, 0x61, 0x72,
+	0x77, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e,
+	0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x06, 0x64, 0x61, 0x72, 0x77, 0x69,
+	0x6e, 0x12, 0x28, 0x0a, 0x07, 0x66, 0x72, 0x65, 0x65, 0x62, 0x73, 0x64, 0x18, 0x05, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61,
+	0x74, 0x61, 0x52, 0x07, 0x66, 0x72, 0x65, 0x65, 0x62, 0x73, 0x64, 0x12, 0x26, 0x0a, 0x06, 0x6e,
+	0x65, 0x74, 0x62, 0x73, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65,
+	0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x06, 0x6e, 0x65, 0x74,
+	0x62, 0x73, 0x64, 0x22, 0x6c, 0x0a, 0x08, 0x55, 0x6e, 0x69, 0x78, 0x44, 0x61, 0x74, 0x61, 0x12,
+	0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d,
+	0x0a, 0x0a, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a,
+	0x03, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12,
+	0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x67, 0x69,
+	0x64, 0x22, 0x52, 0x0a, 0x0b, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x44, 0x61, 0x74, 0x61,
+	0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12,
+	0x24, 0x0a, 0x0e, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x73, 0x5f, 0x67, 0x72, 0x6f, 0x75,
+	0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x73,
+	0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x2f, 0x0a, 0x09, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61,
+	0x74, 0x61, 0x12, 0x22, 0x0a, 0x06, 0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x52, 0x06,
+	0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x22, 0x31, 0x0a, 0x05, 0x58, 0x61, 0x74, 0x74, 0x72, 0x12,
+	0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e,
+	0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x07, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a,
+	0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d,
+	0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28,
+	0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a,
+	0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a,
+	0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73,
+	0x68, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72,
+	0x61, 0x72, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x66, 0x72, 0x6f, 0x6d, 0x54,
+	0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x62, 0x6c, 0x6f, 0x63,
+	0x6b, 0x5f, 0x6e, 0x6f, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x62, 0x6c, 0x6f, 0x63,
+	0x6b, 0x4e, 0x6f, 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22, 0x52, 0x0a, 0x08, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x0a, 0x04, 0x63, 0x6f, 0x64,
+	0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x45, 0x72,
+	0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x65, 0x0a,
+	0x10, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73,
+	0x73, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x39, 0x0a, 0x07, 0x75, 0x70, 0x64,
+	0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x62, 0x65, 0x70,
+	0x2e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f,
+	0x67, 0x72, 0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x07, 0x75, 0x70, 0x64,
+	0x61, 0x74, 0x65, 0x73, 0x22, 0xe5, 0x01, 0x0a, 0x1a, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77,
+	0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x55, 0x70, 0x64,
+	0x61, 0x74, 0x65, 0x12, 0x44, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x79,
+	0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x46,
 	0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72,
-	0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74,
-	0x65, 0x73, 0x22, 0xe5, 0x01, 0x0a, 0x1a, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c,
-	0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74,
-	0x65, 0x12, 0x44, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x46, 0x69, 0x6c,
-	0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73,
-	0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x75, 0x70, 0x64,
-	0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
-	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x07, 0x76,
-	0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x62,
-	0x65, 0x70, 0x2e, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69,
-	0x6f, 0x6e, 0x12, 0x27, 0x0a, 0x0d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x69, 0x6e, 0x64, 0x65,
-	0x78, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, 0x10, 0x00, 0x52, 0x0c, 0x62,
-	0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x62,
-	0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52,
-	0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x06, 0x0a, 0x04, 0x50, 0x69,
-	0x6e, 0x67, 0x22, 0x1f, 0x0a, 0x05, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72,
-	0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x61,
-	0x73, 0x6f, 0x6e, 0x2a, 0xed, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54,
-	0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x1b, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x43, 0x4f, 0x4e, 0x46,
-	0x49, 0x47, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x44, 0x45, 0x58, 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19,
-	0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x44,
-	0x45, 0x58, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x4d,
-	0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x51, 0x55,
-	0x45, 0x53, 0x54, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x10, 0x04,
-	0x12, 0x22, 0x0a, 0x1e, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45,
-	0x53, 0x53, 0x10, 0x05, 0x12, 0x15, 0x0a, 0x11, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x4d,
-	0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x53,
-	0x45, 0x10, 0x07, 0x2a, 0x4f, 0x0a, 0x12, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f,
-	0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x18, 0x4d, 0x45, 0x53,
+	0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x75,
+	0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d,
+	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a,
+	0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b,
+	0x2e, 0x62, 0x65, 0x70, 0x2e, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x07, 0x76, 0x65, 0x72,
+	0x73, 0x69, 0x6f, 0x6e, 0x12, 0x27, 0x0a, 0x0d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x69, 0x6e,
+	0x64, 0x65, 0x78, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, 0x10, 0x00, 0x52,
+	0x0c, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x12, 0x1d, 0x0a,
+	0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28,
+	0x05, 0x52, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x06, 0x0a, 0x04,
+	0x50, 0x69, 0x6e, 0x67, 0x22, 0x1f, 0x0a, 0x05, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x12, 0x16, 0x0a,
+	0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72,
+	0x65, 0x61, 0x73, 0x6f, 0x6e, 0x2a, 0xed, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67,
+	0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x1b, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x43, 0x4f,
+	0x4e, 0x46, 0x49, 0x47, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x44, 0x45, 0x58, 0x10, 0x01, 0x12, 0x1d,
+	0x0a, 0x19, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49,
+	0x4e, 0x44, 0x45, 0x58, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x18, 0x0a,
+	0x14, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45,
+	0x51, 0x55, 0x45, 0x53, 0x54, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4d, 0x45, 0x53, 0x53, 0x41,
+	0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45,
+	0x10, 0x04, 0x12, 0x22, 0x0a, 0x1e, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47,
+	0x52, 0x45, 0x53, 0x53, 0x10, 0x05, 0x12, 0x15, 0x0a, 0x11, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x16, 0x0a,
+	0x12, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c,
+	0x4f, 0x53, 0x45, 0x10, 0x07, 0x2a, 0x4f, 0x0a, 0x12, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
+	0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x18, 0x4d,
+	0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49,
+	0x4f, 0x4e, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x4d, 0x45, 0x53,
 	0x53, 0x41, 0x47, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e,
-	0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x4d, 0x45, 0x53, 0x53, 0x41,
-	0x47, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4c,
-	0x5a, 0x34, 0x10, 0x01, 0x2a, 0x56, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73,
-	0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49,
-	0x4f, 0x4e, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x44, 0x41, 0x54, 0x41, 0x10, 0x00, 0x12, 0x15, 0x0a,
-	0x11, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4e, 0x45, 0x56,
-	0x45, 0x52, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53,
-	0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x4c, 0x57, 0x41, 0x59, 0x53, 0x10, 0x02, 0x2a, 0xb0, 0x01, 0x0a,
-	0x0c, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a,
-	0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x46, 0x49, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49,
-	0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x4f,
-	0x52, 0x59, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1b, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46,
-	0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x5f, 0x46,
-	0x49, 0x4c, 0x45, 0x10, 0x02, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x28, 0x0a, 0x20, 0x46, 0x49, 0x4c,
-	0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c,
-	0x49, 0x4e, 0x4b, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x03, 0x1a,
-	0x02, 0x08, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x10, 0x04, 0x2a,
-	0x76, 0x0a, 0x09, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x17, 0x0a, 0x13,
-	0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x5f, 0x45, 0x52,
-	0x52, 0x4f, 0x52, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43,
-	0x4f, 0x44, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, 0x01, 0x12, 0x1b, 0x0a,
-	0x17, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x5f, 0x53,
-	0x55, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x1b, 0x0a, 0x17, 0x45, 0x52,
-	0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44,
-	0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x03, 0x2a, 0x7e, 0x0a, 0x1e, 0x46, 0x69, 0x6c, 0x65, 0x44,
-	0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x55,
-	0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2d, 0x0a, 0x29, 0x46, 0x49, 0x4c,
-	0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52,
-	0x45, 0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x00, 0x12, 0x2d, 0x0a, 0x29, 0x46, 0x49, 0x4c, 0x45,
-	0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45,
-	0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46,
-	0x4f, 0x52, 0x47, 0x45, 0x54, 0x10, 0x01, 0x42, 0x70, 0x0a, 0x07, 0x63, 0x6f, 0x6d, 0x2e, 0x62,
-	0x65, 0x70, 0x42, 0x08, 0x42, 0x65, 0x70, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f,
-	0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x79, 0x6e, 0x63, 0x74,
-	0x68, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x79, 0x6e, 0x63, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x2f, 0x69,
-	0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x62, 0x65, 0x70, 0xa2,
-	0x02, 0x03, 0x42, 0x58, 0x58, 0xaa, 0x02, 0x03, 0x42, 0x65, 0x70, 0xca, 0x02, 0x03, 0x42, 0x65,
-	0x70, 0xe2, 0x02, 0x0f, 0x42, 0x65, 0x70, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64,
-	0x61, 0x74, 0x61, 0xea, 0x02, 0x03, 0x42, 0x65, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x33,
+	0x5f, 0x4c, 0x5a, 0x34, 0x10, 0x01, 0x2a, 0x56, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65,
+	0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53,
+	0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x44, 0x41, 0x54, 0x41, 0x10, 0x00, 0x12,
+	0x15, 0x0a, 0x11, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4e,
+	0x45, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45,
+	0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x4c, 0x57, 0x41, 0x59, 0x53, 0x10, 0x02, 0x2a, 0xb0,
+	0x01, 0x0a, 0x0c, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x54, 0x79, 0x70, 0x65, 0x12,
+	0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x49, 0x4c, 0x45,
+	0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43,
+	0x54, 0x4f, 0x52, 0x59, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1b, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49,
+	0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b,
+	0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x02, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x28, 0x0a, 0x20, 0x46,
+	0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59,
+	0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10,
+	0x03, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e,
+	0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x10,
+	0x04, 0x2a, 0x76, 0x0a, 0x09, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x17,
+	0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x5f,
+	0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x52, 0x52, 0x4f, 0x52,
+	0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, 0x01, 0x12,
+	0x1b, 0x0a, 0x17, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f,
+	0x5f, 0x53, 0x55, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x1b, 0x0a, 0x17,
+	0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c,
+	0x49, 0x44, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x03, 0x2a, 0x7e, 0x0a, 0x1e, 0x46, 0x69, 0x6c,
+	0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73,
+	0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2d, 0x0a, 0x29, 0x46,
+	0x49, 0x4c, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f,
+	0x47, 0x52, 0x45, 0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x00, 0x12, 0x2d, 0x0a, 0x29, 0x46, 0x49,
+	0x4c, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47,
+	0x52, 0x45, 0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
+	0x5f, 0x46, 0x4f, 0x52, 0x47, 0x45, 0x54, 0x10, 0x01, 0x42, 0x70, 0x0a, 0x07, 0x63, 0x6f, 0x6d,
+	0x2e, 0x62, 0x65, 0x70, 0x42, 0x08, 0x42, 0x65, 0x70, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01,
+	0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x79, 0x6e,
+	0x63, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x79, 0x6e, 0x63, 0x74, 0x68, 0x69, 0x6e, 0x67,
+	0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x62, 0x65,
+	0x70, 0xa2, 0x02, 0x03, 0x42, 0x58, 0x58, 0xaa, 0x02, 0x03, 0x42, 0x65, 0x70, 0xca, 0x02, 0x03,
+	0x42, 0x65, 0x70, 0xe2, 0x02, 0x0f, 0x42, 0x65, 0x70, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74,
+	0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x03, 0x42, 0x65, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x33,
 }
 
 var (

internal/itererr/itererr.go

@@ -0,0 +1,83 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package itererr
+
+import "iter"
+
+// Collect returns a slice of the items from the iterator, plus the error if
+// any.
+func Collect[T any](it iter.Seq[T], errFn func() error) ([]T, error) {
+	var s []T
+	for v := range it {
+		s = append(s, v)
+	}
+	return s, errFn()
+}
+
+// Zip interleaves the iterator value with the error. The iteration ends
+// after a non-nil error.
+func Zip[T any](it iter.Seq[T], errFn func() error) iter.Seq2[T, error] {
+	return func(yield func(T, error) bool) {
+		for v := range it {
+			if !yield(v, nil) {
+				break
+			}
+		}
+		if err := errFn(); err != nil {
+			var zero T
+			yield(zero, err)
+		}
+	}
+}
+
+// Map returns a new iterator by applying the map function, while respecting
+// the error function. Additionally, the map function can return an error if
+// its own.
+func Map[A, B any](i iter.Seq[A], errFn func() error, mapFn func(A) (B, error)) (iter.Seq[B], func() error) {
+	var retErr error
+	return func(yield func(B) bool) {
+			for v := range i {
+				mapped, err := mapFn(v)
+				if err != nil {
+					retErr = err
+					return
+				}
+				if !yield(mapped) {
+					return
+				}
+			}
+		}, func() error {
+			if prevErr := errFn(); prevErr != nil {
+				return prevErr
+			}
+			return retErr
+		}
+}
+
+// Map returns a new iterator by applying the map function, while respecting
+// the error function. Additionally, the map function can return an error if
+// its own.
+func Map2[A, B, C any](i iter.Seq[A], errFn func() error, mapFn func(A) (B, C, error)) (iter.Seq2[B, C], func() error) {
+	var retErr error
+	return func(yield func(B, C) bool) {
+			for v := range i {
+				ma, mb, err := mapFn(v)
+				if err != nil {
+					retErr = err
+					return
+				}
+				if !yield(ma, mb) {
+					return
+				}
+			}
+		}, func() error {
+			if prevErr := errFn(); prevErr != nil {
+				return prevErr
+			}
+			return retErr
+		}
+}

internal/protoutil/protoutil.go

@@ -1,3 +1,9 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
 package protoutil
 
 import (

internal/timeutil/timeutil.go

@@ -0,0 +1,27 @@
+// Copyright (C) 2025 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package timeutil
+
+import (
+	"sync/atomic"
+	"time"
+)
+
+var prevNanos atomic.Int64
+
+// StrictlyMonotonicNanos returns the current time in Unix nanoseconds.
+// Guaranteed to strictly increase for each call, regardless of the
+// underlying OS timer resolution or clock jumps.
+func StrictlyMonotonicNanos() int64 {
+	for {
+		old := prevNanos.Load()
+		now := max(time.Now().UnixNano(), old+1)
+		if prevNanos.CompareAndSwap(old, now) {
+			return now
+		}
+	}
+}

lib/api/api.go

@@ -40,10 +40,10 @@ import (
 	"golang.org/x/text/transform"
 	"golang.org/x/text/unicode/norm"
 
+	"github.com/syncthing/syncthing/internal/db"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/connections"
-	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/discover"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
@@ -91,7 +91,7 @@ type service struct {
 	startupErr           error
 	listenerAddr         net.Addr
 	exitChan             chan *svcutil.FatalErr
-	miscDB               *db.NamespacedKV
+	miscDB               *db.Typed
 	shutdownTimeout      time.Duration
 
 	guiErrors logger.Recorder
@@ -106,7 +106,7 @@ type Service interface {
 	WaitForStart() error
 }
 
-func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonName string, m model.Model, defaultSub, diskSub events.BufferedSubscription, evLogger events.Logger, discoverer discover.Manager, connectionsService connections.Service, urService *ur.Service, fss model.FolderSummaryService, errors, systemLog logger.Recorder, noUpgrade bool, miscDB *db.NamespacedKV) Service {
+func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonName string, m model.Model, defaultSub, diskSub events.BufferedSubscription, evLogger events.Logger, discoverer discover.Manager, connectionsService connections.Service, urService *ur.Service, fss model.FolderSummaryService, errors, systemLog logger.Recorder, noUpgrade bool, miscDB *db.Typed) Service {
 	return &service{
 		id:      id,
 		cfg:     cfg,
@@ -984,16 +984,11 @@ func (s *service) getDBFile(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	mtimeMapping, mtimeErr := s.model.GetMtimeMapping(folder, file)
 
 	sendJSON(w, map[string]interface{}{
 		"global":       jsonFileInfo(gf),
 		"local":        jsonFileInfo(lf),
 		"availability": av,
-		"mtime": map[string]interface{}{
-			"err":   mtimeErr,
-			"value": mtimeMapping,
-		},
 	})
 }
 
@@ -1002,28 +997,14 @@ func (s *service) getDebugFile(w http.ResponseWriter, r *http.Request) {
 	folder := qs.Get("folder")
 	file := qs.Get("file")
 
-	snap, err := s.model.DBSnapshot(folder)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusNotFound)
-		return
-	}
-
-	mtimeMapping, mtimeErr := s.model.GetMtimeMapping(folder, file)
-
-	lf, _ := snap.Get(protocol.LocalDeviceID, file)
-	gf, _ := snap.GetGlobal(file)
-	av := snap.Availability(file)
-	vl := snap.DebugGlobalVersions(file)
+	lf, _, _ := s.model.CurrentFolderFile(folder, file)
+	gf, _, _ := s.model.CurrentGlobalFile(folder, file)
+	av, _ := s.model.Availability(folder, protocol.FileInfo{Name: file}, protocol.BlockInfo{})
 
 	sendJSON(w, map[string]interface{}{
-		"global":         jsonFileInfo(gf),
-		"local":          jsonFileInfo(lf),
-		"availability":   av,
-		"globalVersions": vl.String(),
-		"mtime": map[string]interface{}{
-			"err":   mtimeErr,
-			"value": mtimeMapping,
-		},
+		"global":       jsonFileInfo(gf),
+		"local":        jsonFileInfo(lf),
+		"availability": av,
 	})
 }
 

lib/api/api_auth_test.go

@@ -10,10 +10,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/syncthing/syncthing/internal/db"
+	"github.com/syncthing/syncthing/internal/db/sqlite"
 	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/db/backend"
-	"github.com/syncthing/syncthing/lib/events"
 )
 
 var guiCfg config.GUIConfiguration
@@ -131,8 +130,14 @@ func (c *mockClock) wind(t time.Duration) {
 func TestTokenManager(t *testing.T) {
 	t.Parallel()
 
-	mdb, _ := db.NewLowlevel(backend.OpenMemory(), events.NoopLogger)
-	kdb := db.NewNamespacedKV(mdb, "test")
+	mdb, err := sqlite.OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		mdb.Close()
+	})
+	kdb := db.NewMiscDB(mdb)
 	clock := &mockClock{now: time.Now()}
 
 	// Token manager keeps up to three tokens with a validity time of 24 hours.

lib/api/api_csrf.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/internal/db"
 )
 
 const (
@@ -34,7 +34,7 @@ type apiKeyValidator interface {
 // Check for CSRF token on /rest/ URLs. If a correct one is not given, reject
 // the request with 403. For / and /index.html, set a new CSRF cookie if none
 // is currently set.
-func newCsrfManager(unique string, prefix string, apiKeyValidator apiKeyValidator, next http.Handler, miscDB *db.NamespacedKV) *csrfManager {
+func newCsrfManager(unique string, prefix string, apiKeyValidator apiKeyValidator, next http.Handler, miscDB *db.Typed) *csrfManager {
 	m := &csrfManager{
 		unique:          unique,
 		prefix:          prefix,

lib/api/api_test.go

@@ -27,12 +27,12 @@ import (
 	"github.com/d4l3k/messagediff"
 	"github.com/thejerf/suture/v4"
 
+	"github.com/syncthing/syncthing/internal/db"
+	"github.com/syncthing/syncthing/internal/db/sqlite"
 	"github.com/syncthing/syncthing/lib/assets"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
 	connmocks "github.com/syncthing/syncthing/lib/connections/mocks"
-	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/db/backend"
 	discovermocks "github.com/syncthing/syncthing/lib/discover/mocks"
 	"github.com/syncthing/syncthing/lib/events"
 	eventmocks "github.com/syncthing/syncthing/lib/events/mocks"
@@ -84,8 +84,14 @@ func TestStopAfterBrokenConfig(t *testing.T) {
 	}
 	w := config.Wrap("/dev/null", cfg, protocol.LocalDeviceID, events.NoopLogger)
 
-	mdb, _ := db.NewLowlevel(backend.OpenMemory(), events.NoopLogger)
-	kdb := db.NewMiscDataNamespace(mdb)
+	mdb, err := sqlite.OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		mdb.Close()
+	})
+	kdb := db.NewMiscDB(mdb)
 	srv := New(protocol.LocalDeviceID, w, "", "syncthing", nil, nil, nil, events.NoopLogger, nil, nil, nil, nil, nil, nil, false, kdb).(*service)
 
 	srv.started = make(chan string)
@@ -217,11 +223,7 @@ type httpTestCase struct {
 func TestAPIServiceRequests(t *testing.T) {
 	t.Parallel()
 
-	baseURL, cancel, err := startHTTP(apiCfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Cleanup(cancel)
+	baseURL := startHTTP(t, apiCfg)
 
 	cases := []httpTestCase{
 		// /rest/db
@@ -598,11 +600,7 @@ func TestHTTPLogin(t *testing.T) {
 			APIKey:              testAPIKey,
 			SendBasicAuthPrompt: sendBasicAuthPrompt,
 		})
-		baseURL, cancel, err := startHTTP(cfg)
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Cleanup(cancel)
+		baseURL := startHTTP(t, cfg)
 		url := baseURL + path
 
 		t.Run(fmt.Sprintf("%d path", expectedOkStatus), func(t *testing.T) {
@@ -795,13 +793,9 @@ func TestHTTPLogin(t *testing.T) {
 
 			w := initConfig(initialPassword, t)
 			{
-				baseURL, cancel, err := startHTTPWithShutdownTimeout(w, shutdownTimeout)
+				baseURL := startHTTPWithShutdownTimeout(t, w, shutdownTimeout)
 				cfgPath := baseURL + "/rest/config"
 				path := baseURL + "/meta.js"
-				t.Cleanup(cancel)
-				if err != nil {
-					t.Fatal(err)
-				}
 
 				resp := httpGetBasicAuth(path, "user", initialPassword)
 				if resp.StatusCode != http.StatusOK {
@@ -813,12 +807,8 @@ func TestHTTPLogin(t *testing.T) {
 				httpRequest(http.MethodPut, cfgPath, cfg, "", "", testAPIKey, "", "", "", nil, t)
 			}
 			{
-				baseURL, cancel, err := startHTTP(w)
+				baseURL := startHTTP(t, w)
 				path := baseURL + "/meta.js"
-				t.Cleanup(cancel)
-				if err != nil {
-					t.Fatal(err)
-				}
 
 				resp := httpGetBasicAuth(path, "user", initialPassword)
 				if resp.StatusCode != http.StatusForbidden {
@@ -837,13 +827,9 @@ func TestHTTPLogin(t *testing.T) {
 
 			w := initConfig(initialPassword, t)
 			{
-				baseURL, cancel, err := startHTTPWithShutdownTimeout(w, shutdownTimeout)
+				baseURL := startHTTPWithShutdownTimeout(t, w, shutdownTimeout)
 				cfgPath := baseURL + "/rest/config/gui"
 				path := baseURL + "/meta.js"
-				t.Cleanup(cancel)
-				if err != nil {
-					t.Fatal(err)
-				}
 
 				resp := httpGetBasicAuth(path, "user", initialPassword)
 				if resp.StatusCode != http.StatusOK {
@@ -855,12 +841,8 @@ func TestHTTPLogin(t *testing.T) {
 				httpRequest(http.MethodPut, cfgPath, cfg.GUI, "", "", testAPIKey, "", "", "", nil, t)
 			}
 			{
-				baseURL, cancel, err := startHTTP(w)
+				baseURL := startHTTP(t, w)
 				path := baseURL + "/meta.js"
-				t.Cleanup(cancel)
-				if err != nil {
-					t.Fatal(err)
-				}
 
 				resp := httpGetBasicAuth(path, "user", initialPassword)
 				if resp.StatusCode != http.StatusForbidden {
@@ -885,11 +867,7 @@ func TestHtmlFormLogin(t *testing.T) {
 		Password:            "$2a$10$IdIZTxTg/dCNuNEGlmLynOjqg4B1FvDKuIV5e0BB3pnWVHNb8.GSq", // bcrypt of "räksmörgås" in UTF-8
 		SendBasicAuthPrompt: false,
 	})
-	baseURL, cancel, err := startHTTP(cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Cleanup(cancel)
+	baseURL := startHTTP(t, cfg)
 
 	loginUrl := baseURL + "/rest/noauth/auth/password"
 	resourceUrl := baseURL + "/meta.js"
@@ -1030,11 +1008,7 @@ func TestApiCache(t *testing.T) {
 		RawAddress: "127.0.0.1:0",
 		APIKey:     testAPIKey,
 	})
-	baseURL, cancel, err := startHTTP(cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Cleanup(cancel)
+	baseURL := startHTTP(t, cfg)
 
 	httpGet := func(url string, bearer string) *http.Response {
 		return httpGet(url, "", "", "", bearer, nil, t)
@@ -1059,11 +1033,11 @@ func TestApiCache(t *testing.T) {
 	})
 }
 
-func startHTTP(cfg config.Wrapper) (string, context.CancelFunc, error) {
-	return startHTTPWithShutdownTimeout(cfg, 0)
+func startHTTP(t *testing.T, cfg config.Wrapper) string {
+	return startHTTPWithShutdownTimeout(t, cfg, 0)
 }
 
-func startHTTPWithShutdownTimeout(cfg config.Wrapper, shutdownTimeout time.Duration) (string, context.CancelFunc, error) {
+func startHTTPWithShutdownTimeout(t *testing.T, cfg config.Wrapper, shutdownTimeout time.Duration) string {
 	m := new(modelmocks.Model)
 	assetDir := "../../gui"
 	eventSub := new(eventmocks.BufferedSubscription)
@@ -1086,12 +1060,18 @@ func startHTTPWithShutdownTimeout(cfg config.Wrapper, shutdownTimeout time.Durat
 
 	// Instantiate the API service
 	urService := ur.New(cfg, m, connections, false)
-	mdb, _ := db.NewLowlevel(backend.OpenMemory(), events.NoopLogger)
-	kdb := db.NewMiscDataNamespace(mdb)
+	mdb, err := sqlite.OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		mdb.Close()
+	})
+	kdb := db.NewMiscDB(mdb)
 	svc := New(protocol.LocalDeviceID, cfg, assetDir, "syncthing", m, eventSub, diskEventSub, events.NoopLogger, discoverer, connections, urService, mockedSummary, errorLog, systemLog, false, kdb).(*service)
 	svc.started = addrChan
 
-	if shutdownTimeout > 0*time.Millisecond {
+	if shutdownTimeout > 0 {
 		svc.shutdownTimeout = shutdownTimeout
 	}
 
@@ -1101,14 +1081,14 @@ func startHTTPWithShutdownTimeout(cfg config.Wrapper, shutdownTimeout time.Durat
 	})
 	supervisor.Add(svc)
 	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
 	supervisor.ServeBackground(ctx)
 
 	// Make sure the API service is listening, and get the URL to use.
 	addr := <-addrChan
 	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
 	if err != nil {
-		cancel()
-		return "", cancel, fmt.Errorf("weird address from API service: %w", err)
+		t.Fatal(fmt.Errorf("weird address from API service: %w", err))
 	}
 
 	host, _, _ := net.SplitHostPort(cfg.GUI().RawAddress)
@@ -1117,17 +1097,13 @@ func startHTTPWithShutdownTimeout(cfg config.Wrapper, shutdownTimeout time.Durat
 	}
 	baseURL := fmt.Sprintf("http://%s", net.JoinHostPort(host, strconv.Itoa(tcpAddr.Port)))
 
-	return baseURL, cancel, nil
+	return baseURL
 }
 
 func TestCSRFRequired(t *testing.T) {
 	t.Parallel()
 
-	baseURL, cancel, err := startHTTP(apiCfg)
-	if err != nil {
-		t.Fatal("Unexpected error from getting base URL:", err)
-	}
-	t.Cleanup(cancel)
+	baseURL := startHTTP(t, apiCfg)
 
 	cli := &http.Client{
 		Timeout: time.Minute,
@@ -1245,11 +1221,7 @@ func TestCSRFRequired(t *testing.T) {
 func TestRandomString(t *testing.T) {
 	t.Parallel()
 
-	baseURL, cancel, err := startHTTP(apiCfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer cancel()
+	baseURL := startHTTP(t, apiCfg)
 	cli := &http.Client{
 		Timeout: time.Second,
 	}
@@ -1304,7 +1276,7 @@ func TestConfigPostOK(t *testing.T) {
 		]
 	}`))
 
-	resp, err := testConfigPost(cfg)
+	resp, err := testConfigPost(t, cfg)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1325,7 +1297,7 @@ func TestConfigPostDupFolder(t *testing.T) {
 		]
 	}`))
 
-	resp, err := testConfigPost(cfg)
+	resp, err := testConfigPost(t, cfg)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1334,12 +1306,10 @@ func TestConfigPostDupFolder(t *testing.T) {
 	}
 }
 
-func testConfigPost(data io.Reader) (*http.Response, error) {
-	baseURL, cancel, err := startHTTP(apiCfg)
-	if err != nil {
-		return nil, err
-	}
-	defer cancel()
+func testConfigPost(t *testing.T, data io.Reader) (*http.Response, error) {
+	t.Helper()
+
+	baseURL := startHTTP(t, apiCfg)
 	cli := &http.Client{
 		Timeout: time.Second,
 	}
@@ -1356,11 +1326,7 @@ func TestHostCheck(t *testing.T) {
 
 	cfg := newMockedConfig()
 	cfg.GUIReturns(config.GUIConfiguration{RawAddress: "127.0.0.1:0"})
-	baseURL, cancel, err := startHTTP(cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer cancel()
+	baseURL := startHTTP(t, cfg)
 
 	// A normal HTTP get to the localhost-bound service should succeed
 
@@ -1419,11 +1385,7 @@ func TestHostCheck(t *testing.T) {
 		RawAddress:            "127.0.0.1:0",
 		InsecureSkipHostCheck: true,
 	})
-	baseURL, cancel, err = startHTTP(cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer cancel()
+	baseURL = startHTTP(t, cfg)
 
 	// A request with a suspicious Host header should be allowed
 
@@ -1445,11 +1407,7 @@ func TestHostCheck(t *testing.T) {
 		cfg.GUIReturns(config.GUIConfiguration{
 			RawAddress: "0.0.0.0:0",
 		})
-		baseURL, cancel, err = startHTTP(cfg)
-		if err != nil {
-			t.Fatal(err)
-		}
-		defer cancel()
+		baseURL = startHTTP(t, cfg)
 
 		// A request with a suspicious Host header should be allowed
 
@@ -1476,11 +1434,7 @@ func TestHostCheck(t *testing.T) {
 	cfg.GUIReturns(config.GUIConfiguration{
 		RawAddress: "[::1]:0",
 	})
-	baseURL, cancel, err = startHTTP(cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer cancel()
+	baseURL = startHTTP(t, cfg)
 
 	// A normal HTTP get to the localhost-bound service should succeed
 
@@ -1568,11 +1522,7 @@ func TestAddressIsLocalhost(t *testing.T) {
 func TestAccessControlAllowOriginHeader(t *testing.T) {
 	t.Parallel()
 
-	baseURL, cancel, err := startHTTP(apiCfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer cancel()
+	baseURL := startHTTP(t, apiCfg)
 	cli := &http.Client{
 		Timeout: time.Second,
 	}
@@ -1596,11 +1546,7 @@ func TestAccessControlAllowOriginHeader(t *testing.T) {
 func TestOptionsRequest(t *testing.T) {
 	t.Parallel()
 
-	baseURL, cancel, err := startHTTP(apiCfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer cancel()
+	baseURL := startHTTP(t, apiCfg)
 	cli := &http.Client{
 		Timeout: time.Second,
 	}
@@ -1632,8 +1578,14 @@ func TestEventMasks(t *testing.T) {
 	cfg := newMockedConfig()
 	defSub := new(eventmocks.BufferedSubscription)
 	diskSub := new(eventmocks.BufferedSubscription)
-	mdb, _ := db.NewLowlevel(backend.OpenMemory(), events.NoopLogger)
-	kdb := db.NewMiscDataNamespace(mdb)
+	mdb, err := sqlite.OpenTemp()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		mdb.Close()
+	})
+	kdb := db.NewMiscDB(mdb)
 	svc := New(protocol.LocalDeviceID, cfg, "", "syncthing", nil, defSub, diskSub, events.NoopLogger, nil, nil, nil, nil, nil, nil, false, kdb).(*service)
 
 	if mask := svc.getEventMask(""); mask != DefaultEventMask {
@@ -1780,11 +1732,7 @@ func TestConfigChanges(t *testing.T) {
 	cfgCtx, cfgCancel := context.WithCancel(context.Background())
 	go w.Serve(cfgCtx)
 	defer cfgCancel()
-	baseURL, cancel, err := startHTTP(w)
-	if err != nil {
-		t.Fatal("Unexpected error from getting base URL:", err)
-	}
-	defer cancel()
+	baseURL := startHTTP(t, w)
 
 	cli := &http.Client{
 		Timeout: time.Minute,

lib/api/testdata/config/config.xml

@@ -18,7 +18,6 @@
         <disableSparseFiles>false</disableSparseFiles>
         <disableTempIndexes>false</disableTempIndexes>
         <paused>false</paused>
-        <weakHashThresholdPct>25</weakHashThresholdPct>
         <markerName>.stfolder</markerName>
         <useLargeBlocks>true</useLargeBlocks>
     </folder>
@@ -39,7 +38,6 @@
         <disableSparseFiles>false</disableSparseFiles>
         <disableTempIndexes>false</disableTempIndexes>
         <paused>false</paused>
-        <weakHashThresholdPct>25</weakHashThresholdPct>
         <markerName>.stfolder</markerName>
         <useLargeBlocks>true</useLargeBlocks>
     </folder>

lib/api/tokenmanager.go

@@ -14,9 +14,9 @@ import (
 
 	"google.golang.org/protobuf/proto"
 
+	"github.com/syncthing/syncthing/internal/db"
 	"github.com/syncthing/syncthing/internal/gen/apiproto"
 	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/sync"
@@ -24,7 +24,7 @@ import (
 
 type tokenManager struct {
 	key      string
-	miscDB   *db.NamespacedKV
+	miscDB   *db.Typed
 	lifetime time.Duration
 	maxItems int
 
@@ -35,7 +35,7 @@ type tokenManager struct {
 	saveTimer *time.Timer
 }
 
-func newTokenManager(key string, miscDB *db.NamespacedKV, lifetime time.Duration, maxItems int) *tokenManager {
+func newTokenManager(key string, miscDB *db.Typed, lifetime time.Duration, maxItems int) *tokenManager {
 	var tokens apiproto.TokenSet
 	if bs, ok, _ := miscDB.Bytes(key); ok {
 		_ = proto.Unmarshal(bs, &tokens) // best effort
@@ -152,7 +152,7 @@ type tokenCookieManager struct {
 	tokens     *tokenManager
 }
 
-func newTokenCookieManager(shortID string, guiCfg config.GUIConfiguration, evLogger events.Logger, miscDB *db.NamespacedKV) *tokenCookieManager {
+func newTokenCookieManager(shortID string, guiCfg config.GUIConfiguration, evLogger events.Logger, miscDB *db.Typed) *tokenCookieManager {
 	return &tokenCookieManager{
 		cookieName: "sessionid-" + shortID,
 		shortID:    shortID,

lib/build/build.go

@@ -18,7 +18,7 @@ import (
 	"time"
 )
 
-const Codename = "Gold Grasshopper"
+const Codename = "Hafnium Hornet"
 
 var (
 	// Injected by build script
@@ -28,6 +28,9 @@ var (
 	Stamp   = "0"
 	Tags    = ""
 
+	// Added to by other packages
+	extraTags []string
+
 	// Set by init()
 	Date        time.Time
 	IsRelease   bool
@@ -43,6 +46,11 @@ var (
 		"STNORESTART",
 		"STNOUPGRADE",
 	}
+	replaceTags = map[string]string{
+		"sqlite_omit_load_extension": "",
+		"osusergo":                   "",
+		"netgo":                      "",
+	}
 )
 
 const versionExtraAllowedChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-. "
@@ -108,8 +116,23 @@ func TagsList() []string {
 	if Extra != "" {
 		tags = append(tags, Extra)
 	}
+	tags = append(tags, extraTags...)
+
+	// Replace any tag values we want to have more user friendly versions,
+	// or be removed
+	for i, tag := range tags {
+		if repl, ok := replaceTags[tag]; ok {
+			tags[i] = repl
+		}
+	}
 
 	sort.Strings(tags)
+
+	// Remove any empty tags, which will be at the front of the list now
+	for len(tags) > 0 && tags[0] == "" {
+		tags = tags[1:]
+	}
+
 	return tags
 }
 
@@ -124,3 +147,8 @@ func filterString(s, allowedChars string) string {
 	}
 	return res.String()
 }
+
+func AddTag(tag string) {
+	extraTags = append(extraTags, tag)
+	LongVersion = LongVersionFor("syncthing")
+}

lib/config/config_test.go

@@ -115,10 +115,9 @@ func TestDefaultValues(t *testing.T) {
 					CleanupIntervalS: 3600,
 					Params:           map[string]string{},
 				},
-				MaxConflicts:         10,
-				WeakHashThresholdPct: 25,
-				MarkerName:           ".stfolder",
-				MaxConcurrentWrites:  2,
+				MaxConflicts:        10,
+				MarkerName:          ".stfolder",
+				MaxConcurrentWrites: 2,
 				XattrFilter: XattrFilter{
 					Entries:            []XattrFilterEntry{},
 					MaxSingleEntrySize: 1024,
@@ -193,10 +192,9 @@ func TestDeviceConfig(t *testing.T) {
 					FSType:           FilesystemTypeBasic,
 					Params:           map[string]string{},
 				},
-				WeakHashThresholdPct: 25,
-				MarkerName:           DefaultMarkerName,
-				JunctionsAsDirs:      true,
-				MaxConcurrentWrites:  maxConcurrentWritesDefault,
+				MarkerName:          DefaultMarkerName,
+				JunctionsAsDirs:     true,
+				MaxConcurrentWrites: maxConcurrentWritesDefault,
 				XattrFilter: XattrFilter{
 					MaxSingleEntrySize: 1024,
 					MaxTotalSize:       4096,
@@ -491,7 +489,7 @@ func TestIssue1262(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	actual := cfg.Folders()["test"].Filesystem(nil).URI()
+	actual := cfg.Folders()["test"].Filesystem().URI()
 	expected := `e:\`
 
 	if actual != expected {
@@ -529,7 +527,7 @@ func TestFolderPath(t *testing.T) {
 		Path:           "~/tmp",
 	}
 
-	realPath := folder.Filesystem(nil).URI()
+	realPath := folder.Filesystem().URI()
 	if !filepath.IsAbs(realPath) {
 		t.Error(realPath, "should be absolute")
 	}

lib/config/folderconfiguration.go

@@ -22,7 +22,6 @@ import (
 	"github.com/shirou/gopsutil/v4/disk"
 
 	"github.com/syncthing/syncthing/lib/build"
-	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/structutil"
@@ -73,7 +72,6 @@ type FolderConfiguration struct {
 	DisableSparseFiles      bool                        `json:"disableSparseFiles" xml:"disableSparseFiles"`
 	DisableTempIndexes      bool                        `json:"disableTempIndexes" xml:"disableTempIndexes"`
 	Paused                  bool                        `json:"paused" xml:"paused"`
-	WeakHashThresholdPct    int                         `json:"weakHashThresholdPct" xml:"weakHashThresholdPct"`
 	MarkerName              string                      `json:"markerName" xml:"markerName"`
 	CopyOwnershipFromParent bool                        `json:"copyOwnershipFromParent" xml:"copyOwnershipFromParent"`
 	RawModTimeWindowS       int                         `json:"modTimeWindowS" xml:"modTimeWindowS"`
@@ -123,26 +121,24 @@ func (f FolderConfiguration) Copy() FolderConfiguration {
 // Filesystem creates a filesystem for the path and options of this folder.
 // The fset parameter may be nil, in which case no mtime handling on top of
 // the filesystem is provided.
-func (f FolderConfiguration) Filesystem(fset *db.FileSet) fs.Filesystem {
+func (f FolderConfiguration) Filesystem(extraOpts ...fs.Option) fs.Filesystem {
 	// This is intentionally not a pointer method, because things like
 	// cfg.Folders["default"].Filesystem(nil) should be valid.
-	opts := make([]fs.Option, 0, 3)
+	var opts []fs.Option
 	if f.FilesystemType == FilesystemTypeBasic && f.JunctionsAsDirs {
 		opts = append(opts, new(fs.OptionJunctionsAsDirs))
 	}
 	if !f.CaseSensitiveFS {
 		opts = append(opts, new(fs.OptionDetectCaseConflicts))
 	}
-	if fset != nil {
-		opts = append(opts, fset.MtimeOption())
-	}
+	opts = append(opts, extraOpts...)
 	return fs.NewFilesystem(f.FilesystemType.ToFS(), f.Path, opts...)
 }
 
 func (f FolderConfiguration) ModTimeWindow() time.Duration {
 	dur := time.Duration(f.RawModTimeWindowS) * time.Second
 	if f.RawModTimeWindowS < 1 && build.IsAndroid {
-		if usage, err := disk.Usage(f.Filesystem(nil).URI()); err != nil {
+		if usage, err := disk.Usage(f.Filesystem().URI()); err != nil {
 			dur = 2 * time.Second
 			l.Debugf(`Detecting FS at "%v" on android: Setting mtime window to 2s: err == "%v"`, f.Path, err)
 		} else if strings.HasPrefix(strings.ToLower(usage.Fstype), "ext2") || strings.HasPrefix(strings.ToLower(usage.Fstype), "ext3") || strings.HasPrefix(strings.ToLower(usage.Fstype), "ext4") {
@@ -166,7 +162,7 @@ func (f *FolderConfiguration) CreateMarker() error {
 		return nil
 	}
 
-	ffs := f.Filesystem(nil)
+	ffs := f.Filesystem()
 
 	// Create the marker as a directory
 	err := ffs.Mkdir(DefaultMarkerName, 0o755)
@@ -193,7 +189,7 @@ func (f *FolderConfiguration) CreateMarker() error {
 }
 
 func (f *FolderConfiguration) RemoveMarker() error {
-	ffs := f.Filesystem(nil)
+	ffs := f.Filesystem()
 	_ = ffs.Remove(filepath.Join(DefaultMarkerName, f.markerFilename()))
 	return ffs.Remove(DefaultMarkerName)
 }
@@ -213,7 +209,7 @@ func (f *FolderConfiguration) markerContents() []byte {
 
 // CheckPath returns nil if the folder root exists and contains the marker file
 func (f *FolderConfiguration) CheckPath() error {
-	return f.checkFilesystemPath(f.Filesystem(nil), ".")
+	return f.checkFilesystemPath(f.Filesystem(), ".")
 }
 
 func (f *FolderConfiguration) checkFilesystemPath(ffs fs.Filesystem, path string) error {
@@ -256,7 +252,7 @@ func (f *FolderConfiguration) CreateRoot() (err error) {
 		permBits = 0o700
 	}
 
-	filesystem := f.Filesystem(nil)
+	filesystem := f.Filesystem()
 
 	if _, err = filesystem.Stat("."); fs.IsNotExist(err) {
 		err = filesystem.MkdirAll(".", permBits)
@@ -314,10 +310,6 @@ func (f *FolderConfiguration) prepare(myID protocol.DeviceID, existingDevices ma
 		f.Versioning.CleanupIntervalS = 0
 	}
 
-	if f.WeakHashThresholdPct == 0 {
-		f.WeakHashThresholdPct = 25
-	}
-
 	if f.MarkerName == "" {
 		f.MarkerName = DefaultMarkerName
 	}
@@ -371,7 +363,7 @@ func (f *FolderConfiguration) CheckAvailableSpace(req uint64) error {
 	if val <= 0 {
 		return nil
 	}
-	fs := f.Filesystem(nil)
+	fs := f.Filesystem()
 	usage, err := fs.Usage(".")
 	if err != nil {
 		return nil //nolint: nilerr

lib/config/migrations.go

@@ -208,7 +208,7 @@ func migrateToConfigV23(cfg *Configuration) {
 	// marker name in later versions.
 
 	for i := range cfg.Folders {
-		fs := cfg.Folders[i].Filesystem(nil)
+		fs := cfg.Folders[i].Filesystem()
 		// Invalid config posted, or tests.
 		if fs == nil {
 			continue
@@ -244,18 +244,18 @@ func migrateToConfigV21(cfg *Configuration) {
 		switch folder.Versioning.Type {
 		case "simple", "trashcan":
 			// Clean out symlinks in the known place
-			cleanSymlinks(folder.Filesystem(nil), ".stversions")
+			cleanSymlinks(folder.Filesystem(), ".stversions")
 		case "staggered":
 			versionDir := folder.Versioning.Params["versionsPath"]
 			if versionDir == "" {
 				// default place
-				cleanSymlinks(folder.Filesystem(nil), ".stversions")
+				cleanSymlinks(folder.Filesystem(), ".stversions")
 			} else if filepath.IsAbs(versionDir) {
 				// absolute
 				cleanSymlinks(fs.NewFilesystem(fs.FilesystemTypeBasic, versionDir), ".")
 			} else {
 				// relative to folder
-				cleanSymlinks(folder.Filesystem(nil), versionDir)
+				cleanSymlinks(folder.Filesystem(), versionDir)
 			}
 		}
 	}

lib/config/optionsconfiguration.go

@@ -61,7 +61,6 @@ type OptionsConfiguration struct {
 	StunKeepaliveStartS         int      `json:"stunKeepaliveStartS" xml:"stunKeepaliveStartS" default:"180"`
 	StunKeepaliveMinS           int      `json:"stunKeepaliveMinS" xml:"stunKeepaliveMinS" default:"20"`
 	RawStunServers              []string `json:"stunServers" xml:"stunServer" default:"default"`
-	DatabaseTuning              Tuning   `json:"databaseTuning" xml:"databaseTuning" restart:"true"`
 	RawMaxCIRequestKiB          int      `json:"maxConcurrentIncomingRequestKiB" xml:"maxConcurrentIncomingRequestKiB"`
 	AnnounceLANAddresses        bool     `json:"announceLANAddresses" xml:"announceLANAddresses" default:"true"`
 	SendFullIndexOnUpgrade      bool     `json:"sendFullIndexOnUpgrade" xml:"sendFullIndexOnUpgrade"`

lib/config/testdata/overridenvalues.xml

@@ -70,7 +70,6 @@
             <disableSparseFiles>false</disableSparseFiles>
             <disableTempIndexes>false</disableTempIndexes>
             <paused>false</paused>
-            <weakHashThresholdPct>25</weakHashThresholdPct>
             <markerName>.stfolder</markerName>
             <copyOwnershipFromParent>false</copyOwnershipFromParent>
             <modTimeWindowS>0</modTimeWindowS>

lib/config/tuning.go

@@ -1,46 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package config
-
-type Tuning int32
-
-const (
-	TuningAuto  Tuning = 0
-	TuningSmall Tuning = 1
-	TuningLarge Tuning = 2
-)
-
-func (t Tuning) String() string {
-	switch t {
-	case TuningAuto:
-		return "auto"
-	case TuningSmall:
-		return "small"
-	case TuningLarge:
-		return "large"
-	default:
-		return "unknown"
-	}
-}
-
-func (t Tuning) MarshalText() ([]byte, error) {
-	return []byte(t.String()), nil
-}
-
-func (t *Tuning) UnmarshalText(bs []byte) error {
-	switch string(bs) {
-	case "auto":
-		*t = TuningAuto
-	case "small":
-		*t = TuningSmall
-	case "large":
-		*t = TuningLarge
-	default:
-		*t = TuningAuto
-	}
-	return nil
-}

lib/config/tuning_test.go

@@ -1,26 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package config_test
-
-import (
-	"testing"
-
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/db/backend"
-)
-
-func TestTuningMatches(t *testing.T) {
-	if int(config.TuningAuto) != int(backend.TuningAuto) {
-		t.Error("mismatch for TuningAuto")
-	}
-	if int(config.TuningSmall) != int(backend.TuningSmall) {
-		t.Error("mismatch for TuningSmall")
-	}
-	if int(config.TuningLarge) != int(backend.TuningLarge) {
-		t.Error("mismatch for TuningLarge")
-	}
-}

lib/db/.gitignore

@@ -1,2 +0,0 @@
-!*.zip
-testdata/*.db

lib/db/backend/backend_test.go

@@ -1,76 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package backend
-
-import "testing"
-
-// testBackendBehavior is the generic test suite that must be fulfilled by
-// every backend implementation. It should be called by each implementation
-// as (part of) their test suite.
-func testBackendBehavior(t *testing.T, open func() Backend) {
-	t.Run("WriteIsolation", func(t *testing.T) { testWriteIsolation(t, open) })
-	t.Run("DeleteNonexisten", func(t *testing.T) { testDeleteNonexistent(t, open) })
-	t.Run("IteratorClosedDB", func(t *testing.T) { testIteratorClosedDB(t, open) })
-}
-
-func testWriteIsolation(t *testing.T, open func() Backend) {
-	// Values written during a transaction should not be read back, our
-	// updateGlobal depends on this.
-
-	db := open()
-	defer db.Close()
-
-	// Sanity check
-	_ = db.Put([]byte("a"), []byte("a"))
-	v, _ := db.Get([]byte("a"))
-	if string(v) != "a" {
-		t.Fatal("read back should work")
-	}
-
-	// Now in a transaction we should still see the old value
-	tx, _ := db.NewWriteTransaction()
-	defer tx.Release()
-	_ = tx.Put([]byte("a"), []byte("b"))
-	v, _ = tx.Get([]byte("a"))
-	if string(v) != "a" {
-		t.Fatal("read in transaction should read the old value")
-	}
-}
-
-func testDeleteNonexistent(t *testing.T, open func() Backend) {
-	// Deleting a non-existent key is not an error
-
-	db := open()
-	defer db.Close()
-
-	err := db.Delete([]byte("a"))
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-// Either creating the iterator or the .Error() method of the returned iterator
-// should return an error and IsClosed(err) == true.
-func testIteratorClosedDB(t *testing.T, open func() Backend) {
-	db := open()
-
-	_ = db.Put([]byte("a"), []byte("a"))
-
-	db.Close()
-
-	it, err := db.NewPrefixIterator(nil)
-	if err != nil {
-		if !IsClosed(err) {
-			t.Error("NewPrefixIterator: IsClosed(err) == false:", err)
-		}
-		return
-	}
-	it.Next()
-	if err := it.Error(); !IsClosed(err) {
-		t.Error("Next: IsClosed(err) == false:", err)
-	}
-}

lib/db/backend/debug.go

@@ -1,13 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package backend
-
-import (
-	"github.com/syncthing/syncthing/lib/logger"
-)
-
-var l = logger.DefaultLogger.NewFacility("backend", "The database backend")

lib/db/backend/leveldb_backend.go

@@ -1,233 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package backend
-
-import (
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/iterator"
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-const (
-	// Never flush transactions smaller than this, even on Checkpoint().
-	// This just needs to be just large enough to avoid flushing
-	// transactions when they are super tiny, thus creating millions of tiny
-	// transactions unnecessarily.
-	dbFlushBatchMin = 64 << KiB
-	// Once a transaction reaches this size, flush it unconditionally. This
-	// should be large enough to avoid forcing a flush between Checkpoint()
-	// calls in loops where we do those, so in principle just large enough
-	// to hold a FileInfo plus corresponding version list and metadata
-	// updates or two.
-	dbFlushBatchMax = 1 << MiB
-)
-
-// leveldbBackend implements Backend on top of a leveldb
-type leveldbBackend struct {
-	ldb      *leveldb.DB
-	closeWG  *closeWaitGroup
-	location string
-}
-
-func newLeveldbBackend(ldb *leveldb.DB, location string) *leveldbBackend {
-	return &leveldbBackend{
-		ldb:      ldb,
-		closeWG:  &closeWaitGroup{},
-		location: location,
-	}
-}
-
-func (b *leveldbBackend) NewReadTransaction() (ReadTransaction, error) {
-	return b.newSnapshot()
-}
-
-func (b *leveldbBackend) newSnapshot() (leveldbSnapshot, error) {
-	rel, err := newReleaser(b.closeWG)
-	if err != nil {
-		return leveldbSnapshot{}, err
-	}
-	snap, err := b.ldb.GetSnapshot()
-	if err != nil {
-		rel.Release()
-		return leveldbSnapshot{}, wrapLeveldbErr(err)
-	}
-	return leveldbSnapshot{
-		snap: snap,
-		rel:  rel,
-	}, nil
-}
-
-func (b *leveldbBackend) NewWriteTransaction(hooks ...CommitHook) (WriteTransaction, error) {
-	rel, err := newReleaser(b.closeWG)
-	if err != nil {
-		return nil, err
-	}
-	snap, err := b.newSnapshot()
-	if err != nil {
-		rel.Release()
-		return nil, err // already wrapped
-	}
-	return &leveldbTransaction{
-		leveldbSnapshot: snap,
-		ldb:             b.ldb,
-		batch:           new(leveldb.Batch),
-		rel:             rel,
-		commitHooks:     hooks,
-		inFlush:         false,
-	}, nil
-}
-
-func (b *leveldbBackend) Close() error {
-	b.closeWG.CloseWait()
-	return wrapLeveldbErr(b.ldb.Close())
-}
-
-func (b *leveldbBackend) Get(key []byte) ([]byte, error) {
-	val, err := b.ldb.Get(key, nil)
-	return val, wrapLeveldbErr(err)
-}
-
-func (b *leveldbBackend) NewPrefixIterator(prefix []byte) (Iterator, error) {
-	return &leveldbIterator{b.ldb.NewIterator(util.BytesPrefix(prefix), nil)}, nil
-}
-
-func (b *leveldbBackend) NewRangeIterator(first, last []byte) (Iterator, error) {
-	return &leveldbIterator{b.ldb.NewIterator(&util.Range{Start: first, Limit: last}, nil)}, nil
-}
-
-func (b *leveldbBackend) Put(key, val []byte) error {
-	return wrapLeveldbErr(b.ldb.Put(key, val, nil))
-}
-
-func (b *leveldbBackend) Delete(key []byte) error {
-	return wrapLeveldbErr(b.ldb.Delete(key, nil))
-}
-
-func (b *leveldbBackend) Compact() error {
-	// Race is detected during testing when db is closed while compaction
-	// is ongoing.
-	err := b.closeWG.Add(1)
-	if err != nil {
-		return err
-	}
-	defer b.closeWG.Done()
-	return wrapLeveldbErr(b.ldb.CompactRange(util.Range{}))
-}
-
-func (b *leveldbBackend) Location() string {
-	return b.location
-}
-
-// leveldbSnapshot implements backend.ReadTransaction
-type leveldbSnapshot struct {
-	snap *leveldb.Snapshot
-	rel  *releaser
-}
-
-func (l leveldbSnapshot) Get(key []byte) ([]byte, error) {
-	val, err := l.snap.Get(key, nil)
-	return val, wrapLeveldbErr(err)
-}
-
-func (l leveldbSnapshot) NewPrefixIterator(prefix []byte) (Iterator, error) {
-	return l.snap.NewIterator(util.BytesPrefix(prefix), nil), nil
-}
-
-func (l leveldbSnapshot) NewRangeIterator(first, last []byte) (Iterator, error) {
-	return l.snap.NewIterator(&util.Range{Start: first, Limit: last}, nil), nil
-}
-
-func (l leveldbSnapshot) Release() {
-	l.snap.Release()
-	l.rel.Release()
-}
-
-// leveldbTransaction implements backend.WriteTransaction using a batch (not
-// an actual leveldb transaction)
-type leveldbTransaction struct {
-	leveldbSnapshot
-	ldb         *leveldb.DB
-	batch       *leveldb.Batch
-	rel         *releaser
-	commitHooks []CommitHook
-	inFlush     bool
-}
-
-func (t *leveldbTransaction) Delete(key []byte) error {
-	t.batch.Delete(key)
-	return t.checkFlush(dbFlushBatchMax)
-}
-
-func (t *leveldbTransaction) Put(key, val []byte) error {
-	t.batch.Put(key, val)
-	return t.checkFlush(dbFlushBatchMax)
-}
-
-func (t *leveldbTransaction) Checkpoint() error {
-	return t.checkFlush(dbFlushBatchMin)
-}
-
-func (t *leveldbTransaction) Commit() error {
-	err := wrapLeveldbErr(t.flush())
-	t.leveldbSnapshot.Release()
-	t.rel.Release()
-	return err
-}
-
-func (t *leveldbTransaction) Release() {
-	t.leveldbSnapshot.Release()
-	t.rel.Release()
-}
-
-// checkFlush flushes and resets the batch if its size exceeds the given size.
-func (t *leveldbTransaction) checkFlush(size int) error {
-	// Hooks might put values in the database, which triggers a checkFlush which might trigger a flush,
-	// which might trigger the hooks.
-	// Don't recurse...
-	if t.inFlush || len(t.batch.Dump()) < size {
-		return nil
-	}
-	return t.flush()
-}
-
-func (t *leveldbTransaction) flush() error {
-	t.inFlush = true
-	defer func() { t.inFlush = false }()
-
-	for _, hook := range t.commitHooks {
-		if err := hook(t); err != nil {
-			return err
-		}
-	}
-	if t.batch.Len() == 0 {
-		return nil
-	}
-	if err := t.ldb.Write(t.batch, nil); err != nil {
-		return wrapLeveldbErr(err)
-	}
-	t.batch.Reset()
-	return nil
-}
-
-type leveldbIterator struct {
-	iterator.Iterator
-}
-
-func (it *leveldbIterator) Error() error {
-	return wrapLeveldbErr(it.Iterator.Error())
-}
-
-// wrapLeveldbErr wraps errors so that the backend package can recognize them
-func wrapLeveldbErr(err error) error {
-	switch err {
-	case leveldb.ErrClosed:
-		return errClosed
-	case leveldb.ErrNotFound:
-		return errNotFound
-	}
-	return err
-}

lib/db/backend/leveldb_open.go

@@ -1,231 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package backend
-
-import (
-	"fmt"
-	"os"
-	"strconv"
-	"strings"
-
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/errors"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/syndtr/goleveldb/leveldb/storage"
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-const (
-	dbMaxOpenFiles = 100
-
-	// A large database is > 200 MiB. It's a mostly arbitrary value, but
-	// it's also the case that each file is 2 MiB by default and when we
-	// have dbMaxOpenFiles of them we will need to start thrashing fd:s.
-	// Switching to large database settings causes larger files to be used
-	// when compacting, reducing the number.
-	dbLargeThreshold = dbMaxOpenFiles * (2 << MiB)
-
-	KiB = 10
-	MiB = 20
-)
-
-// OpenLevelDB attempts to open the database at the given location, and runs
-// recovery on it if opening fails. Worst case, if recovery is not possible,
-// the database is erased and created from scratch.
-func OpenLevelDB(location string, tuning Tuning) (Backend, error) {
-	opts := optsFor(location, tuning)
-	ldb, err := open(location, opts)
-	if err != nil {
-		return nil, err
-	}
-	return newLeveldbBackend(ldb, location), nil
-}
-
-// OpenLevelDBAuto is OpenLevelDB with TuningAuto tuning.
-func OpenLevelDBAuto(location string) (Backend, error) {
-	return OpenLevelDB(location, TuningAuto)
-}
-
-// OpenLevelDBRO attempts to open the database at the given location, read
-// only.
-func OpenLevelDBRO(location string) (Backend, error) {
-	opts := &opt.Options{
-		OpenFilesCacheCapacity: dbMaxOpenFiles,
-		ReadOnly:               true,
-	}
-	ldb, err := open(location, opts)
-	if err != nil {
-		return nil, err
-	}
-	return newLeveldbBackend(ldb, location), nil
-}
-
-// OpenLevelDBMemory returns a new Backend referencing an in-memory database.
-func OpenLevelDBMemory() Backend {
-	ldb, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	return newLeveldbBackend(ldb, "")
-}
-
-// optsFor returns the database options to use when opening a database with
-// the given location and tuning. Settings can be overridden by debug
-// environment variables.
-func optsFor(location string, tuning Tuning) *opt.Options {
-	large := false
-	switch tuning {
-	case TuningLarge:
-		large = true
-	case TuningAuto:
-		large = dbIsLarge(location)
-	}
-
-	var (
-		// Set defaults used for small databases.
-		defaultBlockCacheCapacity            = 0 // 0 means let leveldb use default
-		defaultBlockSize                     = 0
-		defaultCompactionTableSize           = 0
-		defaultCompactionTableSizeMultiplier = 0
-		defaultWriteBuffer                   = 16 << MiB                      // increased from leveldb default of 4 MiB
-		defaultCompactionL0Trigger           = opt.DefaultCompactionL0Trigger // explicit because we use it as base for other stuff
-	)
-
-	if large {
-		// Change the parameters for better throughput at the price of some
-		// RAM and larger files. This results in larger batches of writes
-		// and compaction at a lower frequency.
-		l.Infoln("Using large-database tuning")
-
-		defaultBlockCacheCapacity = 64 << MiB
-		defaultBlockSize = 64 << KiB
-		defaultCompactionTableSize = 16 << MiB
-		defaultCompactionTableSizeMultiplier = 20 // 2.0 after division by ten
-		defaultWriteBuffer = 64 << MiB
-		defaultCompactionL0Trigger = 8 // number of l0 files
-	}
-
-	opts := &opt.Options{
-		BlockCacheCapacity:            debugEnvValue("BlockCacheCapacity", defaultBlockCacheCapacity),
-		BlockCacheEvictRemoved:        debugEnvValue("BlockCacheEvictRemoved", 0) != 0,
-		BlockRestartInterval:          debugEnvValue("BlockRestartInterval", 0),
-		BlockSize:                     debugEnvValue("BlockSize", defaultBlockSize),
-		CompactionExpandLimitFactor:   debugEnvValue("CompactionExpandLimitFactor", 0),
-		CompactionGPOverlapsFactor:    debugEnvValue("CompactionGPOverlapsFactor", 0),
-		CompactionL0Trigger:           debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger),
-		CompactionSourceLimitFactor:   debugEnvValue("CompactionSourceLimitFactor", 0),
-		CompactionTableSize:           debugEnvValue("CompactionTableSize", defaultCompactionTableSize),
-		CompactionTableSizeMultiplier: float64(debugEnvValue("CompactionTableSizeMultiplier", defaultCompactionTableSizeMultiplier)) / 10.0,
-		CompactionTotalSize:           debugEnvValue("CompactionTotalSize", 0),
-		CompactionTotalSizeMultiplier: float64(debugEnvValue("CompactionTotalSizeMultiplier", 0)) / 10.0,
-		DisableBufferPool:             debugEnvValue("DisableBufferPool", 0) != 0,
-		DisableBlockCache:             debugEnvValue("DisableBlockCache", 0) != 0,
-		DisableCompactionBackoff:      debugEnvValue("DisableCompactionBackoff", 0) != 0,
-		DisableLargeBatchTransaction:  debugEnvValue("DisableLargeBatchTransaction", 0) != 0,
-		NoSync:                        debugEnvValue("NoSync", 0) != 0,
-		NoWriteMerge:                  debugEnvValue("NoWriteMerge", 0) != 0,
-		OpenFilesCacheCapacity:        debugEnvValue("OpenFilesCacheCapacity", dbMaxOpenFiles),
-		WriteBuffer:                   debugEnvValue("WriteBuffer", defaultWriteBuffer),
-		// The write slowdown and pause can be overridden, but even if they
-		// are not and the compaction trigger is overridden we need to
-		// adjust so that we don't pause writes for L0 compaction before we
-		// even *start* L0 compaction...
-		WriteL0SlowdownTrigger: debugEnvValue("WriteL0SlowdownTrigger", 2*debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger)),
-		WriteL0PauseTrigger:    debugEnvValue("WriteL0SlowdownTrigger", 3*debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger)),
-	}
-
-	return opts
-}
-
-func open(location string, opts *opt.Options) (*leveldb.DB, error) {
-	db, err := leveldb.OpenFile(location, opts)
-	if leveldbIsCorrupted(err) {
-		db, err = leveldb.RecoverFile(location, opts)
-	}
-	if leveldbIsCorrupted(err) {
-		// The database is corrupted, and we've tried to recover it but it
-		// didn't work. At this point there isn't much to do beyond dropping
-		// the database and reindexing...
-		l.Infoln("Database corruption detected, unable to recover. Reinitializing...")
-		if err := os.RemoveAll(location); err != nil {
-			return nil, &errorSuggestion{err, "failed to delete corrupted database"}
-		}
-		db, err = leveldb.OpenFile(location, opts)
-	}
-	if err != nil {
-		return nil, &errorSuggestion{err, "is another instance of Syncthing running?"}
-	}
-
-	if debugEnvValue("CompactEverything", 0) != 0 {
-		if err := db.CompactRange(util.Range{}); err != nil {
-			l.Warnln("Compacting database:", err)
-		}
-	}
-
-	return db, nil
-}
-
-func debugEnvValue(key string, def int) int {
-	v, err := strconv.ParseInt(os.Getenv("STDEBUG_"+key), 10, 63)
-	if err != nil {
-		return def
-	}
-	return int(v)
-}
-
-// A "better" version of leveldb's errors.IsCorrupted.
-func leveldbIsCorrupted(err error) bool {
-	switch {
-	case err == nil:
-		return false
-
-	case errors.IsCorrupted(err):
-		return true
-
-	case strings.Contains(err.Error(), "corrupted"):
-		return true
-	}
-
-	return false
-}
-
-// dbIsLarge returns whether the estimated size of the database at location
-// is large enough to warrant optimization for large databases.
-func dbIsLarge(location string) bool {
-	if ^uint(0)>>63 == 0 {
-		// We're compiled for a 32 bit architecture. We've seen trouble with
-		// large settings there.
-		// (https://forum.syncthing.net/t/many-small-ldb-files-with-database-tuning/13842)
-		return false
-	}
-
-	entries, err := os.ReadDir(location)
-	if err != nil {
-		return false
-	}
-
-	var size int64
-	for _, entry := range entries {
-		if entry.Name() == "LOG" {
-			// don't count the size
-			continue
-		}
-		fi, err := entry.Info()
-		if err != nil {
-			continue
-		}
-		size += fi.Size()
-	}
-
-	return size > dbLargeThreshold
-}
-
-type errorSuggestion struct {
-	inner      error
-	suggestion string
-}
-
-func (e *errorSuggestion) Error() string {
-	return fmt.Sprintf("%s (%s)", e.inner.Error(), e.suggestion)
-}

lib/db/backend/leveldb_test.go

@@ -1,13 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package backend
-
-import "testing"
-
-func TestLevelDBBackendBehavior(t *testing.T) {
-	testBackendBehavior(t, OpenLevelDBMemory)
-}

lib/db/benchmark_test.go

@@ -1,344 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package db_test
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-var files, oneFile, firstHalf, secondHalf, changed100, unchanged100 []protocol.FileInfo
-
-func lazyInitBenchFiles() {
-	if files != nil {
-		return
-	}
-
-	files = make([]protocol.FileInfo, 0, 1000)
-	for i := 0; i < 1000; i++ {
-		files = append(files, protocol.FileInfo{
-			Name:    fmt.Sprintf("file%d", i),
-			Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}},
-			Blocks:  genBlocks(i),
-		})
-	}
-
-	middle := len(files) / 2
-	firstHalf = files[:middle]
-	secondHalf = files[middle:]
-	oneFile = firstHalf[middle-1 : middle]
-
-	unchanged100 := files[100:200]
-	changed100 := append([]protocol.FileInfo{}, unchanged100...)
-	for i := range changed100 {
-		changed100[i].Version = changed100[i].Version.Copy().Update(myID)
-	}
-}
-
-func getBenchFileSet(b testing.TB) (*db.Lowlevel, *db.FileSet) {
-	lazyInitBenchFiles()
-
-	ldb := newLowlevelMemory(b)
-	benchS := newFileSet(b, "test)", ldb)
-	replace(benchS, remoteDevice0, files)
-	replace(benchS, protocol.LocalDeviceID, firstHalf)
-
-	return ldb, benchS
-}
-
-func BenchmarkReplaceAll(b *testing.B) {
-	ldb := newLowlevelMemory(b)
-	defer ldb.Close()
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		m := newFileSet(b, "test)", ldb)
-		replace(m, protocol.LocalDeviceID, files)
-	}
-
-	b.ReportAllocs()
-}
-
-func BenchmarkUpdateOneChanged(b *testing.B) {
-	ldb, benchS := getBenchFileSet(b)
-	defer ldb.Close()
-
-	changed := make([]protocol.FileInfo, 1)
-	changed[0] = oneFile[0]
-	changed[0].Version = changed[0].Version.Copy().Update(myID)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		if i%2 == 0 {
-			benchS.Update(protocol.LocalDeviceID, changed)
-		} else {
-			benchS.Update(protocol.LocalDeviceID, oneFile)
-		}
-	}
-
-	b.ReportAllocs()
-}
-
-func BenchmarkUpdate100Changed(b *testing.B) {
-	ldb, benchS := getBenchFileSet(b)
-	defer ldb.Close()
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		if i%2 == 0 {
-			benchS.Update(protocol.LocalDeviceID, changed100)
-		} else {
-			benchS.Update(protocol.LocalDeviceID, unchanged100)
-		}
-	}
-
-	b.ReportAllocs()
-}
-
-func setup10Remotes(benchS *db.FileSet) {
-	idBase := remoteDevice1.String()[1:]
-	first := 'J'
-	for i := 0; i < 10; i++ {
-		id, _ := protocol.DeviceIDFromString(fmt.Sprintf("%v%s", first+rune(i), idBase))
-		if i%2 == 0 {
-			benchS.Update(id, changed100)
-		} else {
-			benchS.Update(id, unchanged100)
-		}
-	}
-}
-
-func BenchmarkUpdate100Changed10Remotes(b *testing.B) {
-	ldb, benchS := getBenchFileSet(b)
-	defer ldb.Close()
-
-	setup10Remotes(benchS)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		if i%2 == 0 {
-			benchS.Update(protocol.LocalDeviceID, changed100)
-		} else {
-			benchS.Update(protocol.LocalDeviceID, unchanged100)
-		}
-	}
-
-	b.ReportAllocs()
-}
-
-func BenchmarkUpdate100ChangedRemote(b *testing.B) {
-	ldb, benchS := getBenchFileSet(b)
-	defer ldb.Close()
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		if i%2 == 0 {
-			benchS.Update(remoteDevice0, changed100)
-		} else {
-			benchS.Update(remoteDevice0, unchanged100)
-		}
-	}
-
-	b.ReportAllocs()
-}
-
-func BenchmarkUpdate100ChangedRemote10Remotes(b *testing.B) {
-	ldb, benchS := getBenchFileSet(b)
-	defer ldb.Close()
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		if i%2 == 0 {
-			benchS.Update(remoteDevice0, changed100)
-		} else {
-			benchS.Update(remoteDevice0, unchanged100)
-		}
-	}
-
-	b.ReportAllocs()
-}
-
-func BenchmarkUpdateOneUnchanged(b *testing.B) {
-	ldb, benchS := getBenchFileSet(b)
-	defer ldb.Close()
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		benchS.Update(protocol.LocalDeviceID, oneFile)
-	}
-
-	b.ReportAllocs()
-}
-
-func BenchmarkNeedHalf(b *testing.B) {
-	ldb, benchS := getBenchFileSet(b)
-	defer ldb.Close()
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		count := 0
-		snap := snapshot(b, benchS)
-		snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
-			count++
-			return true
-		})
-		snap.Release()
-		if count != len(secondHalf) {
-			b.Errorf("wrong length %d != %d", count, len(secondHalf))
-		}
-	}
-
-	b.ReportAllocs()
-}
-
-func BenchmarkNeedHalfRemote(b *testing.B) {
-	ldb := newLowlevelMemory(b)
-	defer ldb.Close()
-	fset := newFileSet(b, "test)", ldb)
-	replace(fset, remoteDevice0, firstHalf)
-	replace(fset, protocol.LocalDeviceID, files)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		count := 0
-		snap := snapshot(b, fset)
-		snap.WithNeed(remoteDevice0, func(fi protocol.FileInfo) bool {
-			count++
-			return true
-		})
-		snap.Release()
-		if count != len(secondHalf) {
-			b.Errorf("wrong length %d != %d", count, len(secondHalf))
-		}
-	}
-
-	b.ReportAllocs()
-}
-
-func BenchmarkHave(b *testing.B) {
-	ldb, benchS := getBenchFileSet(b)
-	defer ldb.Close()
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		count := 0
-		snap := snapshot(b, benchS)
-		snap.WithHave(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
-			count++
-			return true
-		})
-		snap.Release()
-		if count != len(firstHalf) {
-			b.Errorf("wrong length %d != %d", count, len(firstHalf))
-		}
-	}
-
-	b.ReportAllocs()
-}
-
-func BenchmarkGlobal(b *testing.B) {
-	ldb, benchS := getBenchFileSet(b)
-	defer ldb.Close()
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		count := 0
-		snap := snapshot(b, benchS)
-		snap.WithGlobal(func(fi protocol.FileInfo) bool {
-			count++
-			return true
-		})
-		snap.Release()
-		if count != len(files) {
-			b.Errorf("wrong length %d != %d", count, len(files))
-		}
-	}
-
-	b.ReportAllocs()
-}
-
-func BenchmarkNeedHalfTruncated(b *testing.B) {
-	ldb, benchS := getBenchFileSet(b)
-	defer ldb.Close()
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		count := 0
-		snap := snapshot(b, benchS)
-		snap.WithNeedTruncated(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
-			count++
-			return true
-		})
-		snap.Release()
-		if count != len(secondHalf) {
-			b.Errorf("wrong length %d != %d", count, len(secondHalf))
-		}
-	}
-
-	b.ReportAllocs()
-}
-
-func BenchmarkHaveTruncated(b *testing.B) {
-	ldb, benchS := getBenchFileSet(b)
-	defer ldb.Close()
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		count := 0
-		snap := snapshot(b, benchS)
-		snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
-			count++
-			return true
-		})
-		snap.Release()
-		if count != len(firstHalf) {
-			b.Errorf("wrong length %d != %d", count, len(firstHalf))
-		}
-	}
-
-	b.ReportAllocs()
-}
-
-func BenchmarkGlobalTruncated(b *testing.B) {
-	ldb, benchS := getBenchFileSet(b)
-	defer ldb.Close()
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		count := 0
-		snap := snapshot(b, benchS)
-		snap.WithGlobalTruncated(func(fi protocol.FileInfo) bool {
-			count++
-			return true
-		})
-		snap.Release()
-		if count != len(files) {
-			b.Errorf("wrong length %d != %d", count, len(files))
-		}
-	}
-
-	b.ReportAllocs()
-}
-
-func BenchmarkNeedCount(b *testing.B) {
-	ldb, benchS := getBenchFileSet(b)
-	defer ldb.Close()
-
-	benchS.Update(protocol.LocalDeviceID, changed100)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		snap := snapshot(b, benchS)
-		_ = snap.NeedSize(protocol.LocalDeviceID)
-		snap.Release()
-	}
-
-	b.ReportAllocs()
-}

lib/db/blockmap.go

@@ -1,64 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package db
-
-import (
-	"encoding/binary"
-	"fmt"
-
-	"github.com/syncthing/syncthing/lib/osutil"
-)
-
-type BlockFinder struct {
-	db *Lowlevel
-}
-
-func NewBlockFinder(db *Lowlevel) *BlockFinder {
-	return &BlockFinder{
-		db: db,
-	}
-}
-
-func (f *BlockFinder) String() string {
-	return fmt.Sprintf("BlockFinder@%p", f)
-}
-
-// Iterate takes an iterator function which iterates over all matching blocks
-// for the given hash. The iterator function has to return either true (if
-// they are happy with the block) or false to continue iterating for whatever
-// reason. The iterator finally returns the result, whether or not a
-// satisfying block was eventually found.
-func (f *BlockFinder) Iterate(folders []string, hash []byte, iterFn func(string, string, int32) bool) bool {
-	t, err := f.db.newReadOnlyTransaction()
-	if err != nil {
-		return false
-	}
-	defer t.close()
-
-	var key []byte
-	for _, folder := range folders {
-		key, err = f.db.keyer.GenerateBlockMapKey(key, []byte(folder), hash, nil)
-		if err != nil {
-			return false
-		}
-		iter, err := t.NewPrefixIterator(key)
-		if err != nil {
-			return false
-		}
-
-		for iter.Next() && iter.Error() == nil {
-			file := string(f.db.keyer.NameFromBlockMapKey(iter.Key()))
-			index := int32(binary.BigEndian.Uint32(iter.Value()))
-			if iterFn(folder, osutil.NativeFilename(file), index) {
-				iter.Release()
-				return true
-			}
-		}
-		iter.Release()
-	}
-	return false
-}

lib/db/blockmap_test.go

@@ -1,260 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package db
-
-import (
-	"encoding/binary"
-	"testing"
-
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-var (
-	f1, f2, f3 protocol.FileInfo
-	folders    = []string{"folder1", "folder2"}
-)
-
-func init() {
-	blocks := genBlocks(30)
-
-	f1 = protocol.FileInfo{
-		Name:   "f1",
-		Blocks: blocks[:10],
-	}
-
-	f2 = protocol.FileInfo{
-		Name:   "f2",
-		Blocks: blocks[10:20],
-	}
-
-	f3 = protocol.FileInfo{
-		Name:   "f3",
-		Blocks: blocks[20:],
-	}
-}
-
-func setup(t testing.TB) (*Lowlevel, *BlockFinder) {
-	t.Helper()
-	db := newLowlevelMemory(t)
-	return db, NewBlockFinder(db)
-}
-
-func dbEmpty(db *Lowlevel) bool {
-	iter, err := db.NewPrefixIterator([]byte{KeyTypeBlock})
-	if err != nil {
-		panic(err)
-	}
-	defer iter.Release()
-	return !iter.Next()
-}
-
-func addToBlockMap(db *Lowlevel, folder []byte, fs []protocol.FileInfo) error {
-	t, err := db.newReadWriteTransaction()
-	if err != nil {
-		return err
-	}
-	defer t.close()
-
-	var keyBuf []byte
-	blockBuf := make([]byte, 4)
-	for _, f := range fs {
-		if !f.IsDirectory() && !f.IsDeleted() && !f.IsInvalid() {
-			name := []byte(f.Name)
-			for i, block := range f.Blocks {
-				binary.BigEndian.PutUint32(blockBuf, uint32(i))
-				keyBuf, err = t.keyer.GenerateBlockMapKey(keyBuf, folder, block.Hash, name)
-				if err != nil {
-					return err
-				}
-				if err := t.Put(keyBuf, blockBuf); err != nil {
-					return err
-				}
-			}
-		}
-	}
-	return t.Commit()
-}
-
-func discardFromBlockMap(db *Lowlevel, folder []byte, fs []protocol.FileInfo) error {
-	t, err := db.newReadWriteTransaction()
-	if err != nil {
-		return err
-	}
-	defer t.close()
-
-	var keyBuf []byte
-	for _, ef := range fs {
-		if !ef.IsDirectory() && !ef.IsDeleted() && !ef.IsInvalid() {
-			name := []byte(ef.Name)
-			for _, block := range ef.Blocks {
-				keyBuf, err = t.keyer.GenerateBlockMapKey(keyBuf, folder, block.Hash, name)
-				if err != nil {
-					return err
-				}
-				if err := t.Delete(keyBuf); err != nil {
-					return err
-				}
-			}
-		}
-	}
-	return t.Commit()
-}
-
-func TestBlockMapAddUpdateWipe(t *testing.T) {
-	db, f := setup(t)
-	defer db.Close()
-
-	if !dbEmpty(db) {
-		t.Fatal("db not empty")
-	}
-
-	folder := []byte("folder1")
-
-	f3.Type = protocol.FileInfoTypeDirectory
-
-	if err := addToBlockMap(db, folder, []protocol.FileInfo{f1, f2, f3}); err != nil {
-		t.Fatal(err)
-	}
-
-	f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
-		if folder != "folder1" || file != "f1" || index != 0 {
-			t.Fatal("Mismatch")
-		}
-		return true
-	})
-
-	f.Iterate(folders, f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
-		if folder != "folder1" || file != "f2" || index != 0 {
-			t.Fatal("Mismatch")
-		}
-		return true
-	})
-
-	f.Iterate(folders, f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
-		t.Fatal("Unexpected block")
-		return true
-	})
-
-	if err := discardFromBlockMap(db, folder, []protocol.FileInfo{f1, f2, f3}); err != nil {
-		t.Fatal(err)
-	}
-
-	f1.Deleted = true
-	f2.LocalFlags = protocol.FlagLocalMustRescan // one of the invalid markers
-
-	if err := addToBlockMap(db, folder, []protocol.FileInfo{f1, f2, f3}); err != nil {
-		t.Fatal(err)
-	}
-
-	f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
-		t.Fatal("Unexpected block")
-		return false
-	})
-
-	f.Iterate(folders, f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
-		t.Fatal("Unexpected block")
-		return false
-	})
-
-	f.Iterate(folders, f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
-		if folder != "folder1" || file != "f3" || index != 0 {
-			t.Fatal("Mismatch")
-		}
-		return true
-	})
-
-	if err := db.dropFolder(folder); err != nil {
-		t.Fatal(err)
-	}
-
-	if !dbEmpty(db) {
-		t.Fatal("db not empty")
-	}
-
-	// Should not add
-	if err := addToBlockMap(db, folder, []protocol.FileInfo{f1, f2}); err != nil {
-		t.Fatal(err)
-	}
-
-	if !dbEmpty(db) {
-		t.Fatal("db not empty")
-	}
-
-	f1.Deleted = false
-	f1.LocalFlags = 0
-	f2.Deleted = false
-	f2.LocalFlags = 0
-	f3.Deleted = false
-	f3.LocalFlags = 0
-}
-
-func TestBlockFinderLookup(t *testing.T) {
-	db, f := setup(t)
-	defer db.Close()
-
-	folder1 := []byte("folder1")
-	folder2 := []byte("folder2")
-
-	if err := addToBlockMap(db, folder1, []protocol.FileInfo{f1}); err != nil {
-		t.Fatal(err)
-	}
-	if err := addToBlockMap(db, folder2, []protocol.FileInfo{f1}); err != nil {
-		t.Fatal(err)
-	}
-
-	counter := 0
-	f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
-		counter++
-		switch counter {
-		case 1:
-			if folder != "folder1" || file != "f1" || index != 0 {
-				t.Fatal("Mismatch")
-			}
-		case 2:
-			if folder != "folder2" || file != "f1" || index != 0 {
-				t.Fatal("Mismatch")
-			}
-		default:
-			t.Fatal("Unexpected block")
-		}
-		return false
-	})
-
-	if counter != 2 {
-		t.Fatal("Incorrect count", counter)
-	}
-
-	if err := discardFromBlockMap(db, folder1, []protocol.FileInfo{f1}); err != nil {
-		t.Fatal(err)
-	}
-
-	f1.Deleted = true
-
-	if err := addToBlockMap(db, folder1, []protocol.FileInfo{f1}); err != nil {
-		t.Fatal(err)
-	}
-
-	counter = 0
-	f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
-		counter++
-		switch counter {
-		case 1:
-			if folder != "folder2" || file != "f1" || index != 0 {
-				t.Fatal("Mismatch")
-			}
-		default:
-			t.Fatal("Unexpected block")
-		}
-		return false
-	})
-
-	if counter != 1 {
-		t.Fatal("Incorrect count")
-	}
-
-	f1.Deleted = false
-}

lib/db/db_test.go

@@ -1,701 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package db
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"testing"
-
-	"github.com/syncthing/syncthing/lib/db/backend"
-	"github.com/syncthing/syncthing/lib/events"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-func genBlocks(n int) []protocol.BlockInfo {
-	b := make([]protocol.BlockInfo, n)
-	for i := range b {
-		h := make([]byte, 32)
-		for j := range h {
-			h[j] = byte(i + j)
-		}
-		b[i].Size = i
-		b[i].Hash = h
-	}
-	return b
-}
-
-const myID = 1
-
-var (
-	remoteDevice0, remoteDevice1 protocol.DeviceID
-	invalid                      = "invalid"
-	slashPrefixed                = "/notgood"
-	haveUpdate0to3               map[protocol.DeviceID][]protocol.FileInfo
-)
-
-func init() {
-	remoteDevice0, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
-	remoteDevice1, _ = protocol.DeviceIDFromString("I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU")
-	haveUpdate0to3 = map[protocol.DeviceID][]protocol.FileInfo{
-		protocol.LocalDeviceID: {
-			protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
-			protocol.FileInfo{Name: slashPrefixed, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
-		},
-		remoteDevice0: {
-			protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
-			protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), RawInvalid: true},
-			protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
-		},
-		remoteDevice1: {
-			protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
-			protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), RawInvalid: true},
-			protocol.FileInfo{Name: invalid, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), RawInvalid: true},
-		},
-	}
-}
-
-// TestRepairSequence checks that a few hand-crafted messed-up sequence entries get fixed.
-func TestRepairSequence(t *testing.T) {
-	db := newLowlevelMemory(t)
-	defer db.Close()
-
-	folderStr := "test"
-	folder := []byte(folderStr)
-	id := protocol.LocalDeviceID
-	short := protocol.LocalDeviceID.Short()
-
-	files := []protocol.FileInfo{
-		{Name: "fine", Blocks: genBlocks(1)},
-		{Name: "duplicate", Blocks: genBlocks(2)},
-		{Name: "missing", Blocks: genBlocks(3)},
-		{Name: "overwriting", Blocks: genBlocks(4)},
-		{Name: "inconsistent", Blocks: genBlocks(5)},
-		{Name: "inconsistentNotIndirected", Blocks: genBlocks(2)},
-	}
-	for i, f := range files {
-		files[i].Version = f.Version.Update(short)
-	}
-
-	trans, err := db.newReadWriteTransaction()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer trans.close()
-
-	addFile := func(f protocol.FileInfo, seq int64) {
-		dk, err := trans.keyer.GenerateDeviceFileKey(nil, folder, id[:], []byte(f.Name))
-		if err != nil {
-			t.Fatal(err)
-		}
-		if err := trans.putFile(dk, f); err != nil {
-			t.Fatal(err)
-		}
-		sk, err := trans.keyer.GenerateSequenceKey(nil, folder, seq)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if err := trans.Put(sk, dk); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Plain normal entry
-	var seq int64 = 1
-	files[0].Sequence = 1
-	addFile(files[0], seq)
-
-	// Second entry once updated with original sequence still in place
-	f := files[1]
-	f.Sequence = int64(len(files) + 1)
-	addFile(f, f.Sequence)
-	// Original sequence entry
-	seq++
-	sk, err := trans.keyer.GenerateSequenceKey(nil, folder, seq)
-	if err != nil {
-		t.Fatal(err)
-	}
-	dk, err := trans.keyer.GenerateDeviceFileKey(nil, folder, id[:], []byte(f.Name))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := trans.Put(sk, dk); err != nil {
-		t.Fatal(err)
-	}
-
-	// File later overwritten thus missing sequence entry
-	seq++
-	files[2].Sequence = seq
-	addFile(files[2], seq)
-
-	// File overwriting previous sequence entry (no seq bump)
-	seq++
-	files[3].Sequence = seq
-	addFile(files[3], seq)
-
-	// Inconistent files
-	seq++
-	files[4].Sequence = 101
-	addFile(files[4], seq)
-	seq++
-	files[5].Sequence = 102
-	addFile(files[5], seq)
-
-	// And a sequence entry pointing at nothing because why not
-	sk, err = trans.keyer.GenerateSequenceKey(nil, folder, 100001)
-	if err != nil {
-		t.Fatal(err)
-	}
-	dk, err = trans.keyer.GenerateDeviceFileKey(nil, folder, id[:], []byte("nonexisting"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := trans.Put(sk, dk); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := trans.Commit(); err != nil {
-		t.Fatal(err)
-	}
-
-	// Loading the metadata for the first time means a "re"calculation happens,
-	// along which the sequences get repaired too.
-	db.gcMut.RLock()
-	_, err = db.loadMetadataTracker(folderStr)
-	db.gcMut.RUnlock()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Check the db
-	ro, err := db.newReadOnlyTransaction()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer ro.close()
-
-	it, err := ro.NewPrefixIterator([]byte{KeyTypeDevice})
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer it.Release()
-	for it.Next() {
-		fi, err := ro.unmarshalTrunc(it.Value(), true)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if sk, err = ro.keyer.GenerateSequenceKey(sk, folder, fi.SequenceNo()); err != nil {
-			t.Fatal(err)
-		}
-		dk, err := ro.Get(sk)
-		if backend.IsNotFound(err) {
-			t.Error("Missing sequence entry for", fi.FileName())
-		} else if err != nil {
-			t.Fatal(err)
-		}
-		if !bytes.Equal(it.Key(), dk) {
-			t.Errorf("Wrong key for %v, expected %s, got %s", f.FileName(), it.Key(), dk)
-		}
-	}
-	if err := it.Error(); err != nil {
-		t.Fatal(err)
-	}
-	it.Release()
-
-	it, err = ro.NewPrefixIterator([]byte{KeyTypeSequence})
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer it.Release()
-	for it.Next() {
-		fi, ok, err := ro.getFileTrunc(it.Value(), false)
-		if err != nil {
-			t.Fatal(err)
-		}
-		seq := ro.keyer.SequenceFromSequenceKey(it.Key())
-		if !ok {
-			t.Errorf("Sequence entry %v points at nothing", seq)
-		} else if fi.SequenceNo() != seq {
-			t.Errorf("Inconsistent sequence entry for %v: %v != %v", fi.FileName(), fi.SequenceNo(), seq)
-		}
-		if len(fi.Blocks) == 0 {
-			t.Error("Missing blocks in", fi.FileName())
-		}
-	}
-	if err := it.Error(); err != nil {
-		t.Fatal(err)
-	}
-	it.Release()
-}
-
-func TestDowngrade(t *testing.T) {
-	db := newLowlevelMemory(t)
-	defer db.Close()
-	// sets the min version etc
-	if err := UpdateSchema(db); err != nil {
-		t.Fatal(err)
-	}
-
-	// Bump the database version to something newer than we actually support
-	miscDB := NewMiscDataNamespace(db)
-	if err := miscDB.PutInt64("dbVersion", dbVersion+1); err != nil {
-		t.Fatal(err)
-	}
-	l.Infoln(dbVersion)
-
-	// Pretend we just opened the DB and attempt to update it again
-	err := UpdateSchema(db)
-
-	if err, ok := err.(*databaseDowngradeError); !ok {
-		t.Fatal("Expected error due to database downgrade, got", err)
-	} else if err.minSyncthingVersion != dbMinSyncthingVersion {
-		t.Fatalf("Error has %v as min Syncthing version, expected %v", err.minSyncthingVersion, dbMinSyncthingVersion)
-	}
-}
-
-func TestCheckGlobals(t *testing.T) {
-	db := newLowlevelMemory(t)
-	defer db.Close()
-
-	fs := newFileSet(t, "test", db)
-
-	// Add any file
-	name := "foo"
-	fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
-		{
-			Name:    name,
-			Type:    protocol.FileInfoTypeFile,
-			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}},
-		},
-	})
-
-	// Remove just the file entry
-	if err := db.dropPrefix([]byte{KeyTypeDevice}); err != nil {
-		t.Fatal(err)
-	}
-
-	// Clean up global entry of the now missing file
-	if repaired, err := db.checkGlobals(fs.folder); err != nil {
-		t.Fatal(err)
-	} else if repaired != 1 {
-		t.Error("Expected 1 repaired global item, got", repaired)
-	}
-
-	// Check that the global entry is gone
-	gk, err := db.keyer.GenerateGlobalVersionKey(nil, []byte(fs.folder), []byte(name))
-	if err != nil {
-		t.Fatal(err)
-	}
-	_, err = db.Get(gk)
-	if !backend.IsNotFound(err) {
-		t.Error("Expected key missing error, got", err)
-	}
-}
-
-func TestDropDuplicates(t *testing.T) {
-	names := []string{
-		"foo",
-		"bar",
-		"dcxvoijnds",
-		"3d/dsfase/4/ss2",
-	}
-	tcs := []struct{ in, out []int }{
-		{[]int{0}, []int{0}},
-		{[]int{0, 1}, []int{0, 1}},
-		{[]int{0, 1, 0, 1}, []int{0, 1}},
-		{[]int{0, 1, 1, 1, 1}, []int{0, 1}},
-		{[]int{0, 0, 0, 1}, []int{0, 1}},
-		{[]int{0, 1, 2, 3}, []int{0, 1, 2, 3}},
-		{[]int{3, 2, 1, 0, 0, 1, 2, 3}, []int{0, 1, 2, 3}},
-		{[]int{0, 1, 1, 3, 0, 1, 0, 1, 2, 3}, []int{0, 1, 2, 3}},
-	}
-
-	for tci, tc := range tcs {
-		inp := make([]protocol.FileInfo, len(tc.in))
-		expSeq := make(map[string]int)
-		for i, j := range tc.in {
-			inp[i] = protocol.FileInfo{Name: names[j], Sequence: int64(i)}
-			expSeq[names[j]] = i
-		}
-		outp := normalizeFilenamesAndDropDuplicates(inp)
-		if len(outp) != len(tc.out) {
-			t.Errorf("tc %v: Expected %v entries, got %v", tci, len(tc.out), len(outp))
-			continue
-		}
-		for i, f := range outp {
-			if exp := names[tc.out[i]]; exp != f.Name {
-				t.Errorf("tc %v: Got file %v at pos %v, expected %v", tci, f.Name, i, exp)
-			}
-			if exp := int64(expSeq[outp[i].Name]); exp != f.Sequence {
-				t.Errorf("tc %v: Got sequence %v at pos %v, expected %v", tci, f.Sequence, i, exp)
-			}
-		}
-	}
-}
-
-func TestGCIndirect(t *testing.T) {
-	// Verify that the gcIndirect run actually removes block lists.
-
-	db := newLowlevelMemory(t)
-	defer db.Close()
-	meta := newMetadataTracker(db.keyer, events.NoopLogger)
-
-	// Add three files with different block lists
-
-	files := []protocol.FileInfo{
-		{Name: "a", Blocks: genBlocks(100)},
-		{Name: "b", Blocks: genBlocks(200)},
-		{Name: "c", Blocks: genBlocks(300)},
-	}
-
-	db.updateLocalFiles([]byte("folder"), files, meta)
-
-	// Run a GC pass
-
-	db.gcIndirect(context.Background())
-
-	// Verify that we have three different block lists
-
-	n, err := numBlockLists(db)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if n != len(files) {
-		t.Fatal("expected each file to have a block list")
-	}
-
-	// Change the block lists for each file
-
-	for i := range files {
-		files[i].Version = files[i].Version.Update(42)
-		files[i].Blocks = genBlocks(len(files[i].Blocks) + 1)
-	}
-
-	db.updateLocalFiles([]byte("folder"), files, meta)
-
-	// Verify that we now have *six* different block lists
-
-	n, err = numBlockLists(db)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if n != 2*len(files) {
-		t.Fatal("expected both old and new block lists to exist")
-	}
-
-	// Run a GC pass
-
-	db.gcIndirect(context.Background())
-
-	// Verify that we now have just the three we need, again
-
-	n, err = numBlockLists(db)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if n != len(files) {
-		t.Fatal("expected GC to collect all but the needed ones")
-	}
-
-	// Double check the correctness by loading the block lists and comparing with what we stored
-
-	tr, err := db.newReadOnlyTransaction()
-	if err != nil {
-		t.Fatal()
-	}
-	defer tr.Release()
-	for _, f := range files {
-		fi, ok, err := tr.getFile([]byte("folder"), protocol.LocalDeviceID[:], []byte(f.Name))
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !ok {
-			t.Fatal("mysteriously missing")
-		}
-		if len(fi.Blocks) != len(f.Blocks) {
-			t.Fatal("block list mismatch")
-		}
-		for i := range fi.Blocks {
-			if !bytes.Equal(fi.Blocks[i].Hash, f.Blocks[i].Hash) {
-				t.Fatal("hash mismatch")
-			}
-		}
-	}
-}
-
-func TestUpdateTo14(t *testing.T) {
-	db := newLowlevelMemory(t)
-	defer db.Close()
-
-	folderStr := "default"
-	folder := []byte(folderStr)
-	name := []byte("foo")
-	file := protocol.FileInfo{Name: string(name), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(blocksIndirectionCutoff - 1)}
-	file.BlocksHash = protocol.BlocksHash(file.Blocks)
-	fileWOBlocks := file
-	fileWOBlocks.Blocks = nil
-	meta, err := db.loadMetadataTracker(folderStr)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Initially add the correct file the usual way, all good here.
-	if err := db.updateLocalFiles(folder, []protocol.FileInfo{file}, meta); err != nil {
-		t.Fatal(err)
-	}
-
-	// Simulate the previous bug, where .putFile could write a file info without
-	// blocks, even though the file has them (and thus a non-nil BlocksHash).
-	trans, err := db.newReadWriteTransaction()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer trans.close()
-	key, err := db.keyer.GenerateDeviceFileKey(nil, folder, protocol.LocalDeviceID[:], name)
-	if err != nil {
-		t.Fatal(err)
-	}
-	fiBs := mustMarshal(fileWOBlocks.ToWire(true))
-	if err := trans.Put(key, fiBs); err != nil {
-		t.Fatal(err)
-	}
-	if err := trans.Commit(); err != nil {
-		t.Fatal(err)
-	}
-	trans.close()
-
-	// Run migration, pretending were still on schema 13.
-	if err := (&schemaUpdater{db}).updateSchemaTo14(13); err != nil {
-		t.Fatal(err)
-	}
-
-	// checks
-	ro, err := db.newReadOnlyTransaction()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer ro.close()
-	if f, ok, err := ro.getFileByKey(key); err != nil {
-		t.Fatal(err)
-	} else if !ok {
-		t.Error("file missing")
-	} else if !f.MustRescan() {
-		t.Error("file not marked as MustRescan")
-	}
-
-	if vl, err := ro.getGlobalVersions(nil, folder, name); err != nil {
-		t.Fatal(err)
-	} else if fv, ok := vlGetGlobal(vl); !ok {
-		t.Error("missing global")
-	} else if !fvIsInvalid(fv) {
-		t.Error("global not marked as invalid")
-	}
-}
-
-func TestFlushRecursion(t *testing.T) {
-	// Verify that a commit hook can write to the transaction without
-	// causing another flush and thus recursion.
-
-	db := newLowlevelMemory(t)
-	defer db.Close()
-
-	// A commit hook that writes a small piece of data to the transaction.
-	hookFired := 0
-	hook := func(tx backend.WriteTransaction) error {
-		err := tx.Put([]byte(fmt.Sprintf("hook-key-%d", hookFired)), []byte(fmt.Sprintf("hook-value-%d", hookFired)))
-		if err != nil {
-			t.Fatal(err)
-		}
-		hookFired++
-		return nil
-	}
-
-	// A transaction.
-	tx, err := db.NewWriteTransaction(hook)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer tx.Release()
-
-	// Write stuff until the transaction flushes, thus firing the hook.
-	i := 0
-	for hookFired == 0 {
-		err := tx.Put([]byte(fmt.Sprintf("key-%d", i)), []byte(fmt.Sprintf("value-%d", i)))
-		if err != nil {
-			t.Fatal(err)
-		}
-		i++
-	}
-
-	// The hook should have fired precisely once.
-	if hookFired != 1 {
-		t.Error("expect one hook fire, not", hookFired)
-	}
-}
-
-func TestCheckLocalNeed(t *testing.T) {
-	db := newLowlevelMemory(t)
-	defer db.Close()
-
-	folderStr := "test"
-	fs := newFileSet(t, folderStr, db)
-
-	// Add files such that we are in sync for a and b, and need c and d.
-	files := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}},
-		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}},
-		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}},
-		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}},
-	}
-	fs.Update(protocol.LocalDeviceID, files)
-	files[2].Version = files[2].Version.Update(remoteDevice0.Short())
-	files[3].Version = files[2].Version.Update(remoteDevice0.Short())
-	fs.Update(remoteDevice0, files)
-
-	checkNeed := func() {
-		snap := snapshot(t, fs)
-		defer snap.Release()
-		c := snap.NeedSize(protocol.LocalDeviceID)
-		if c.Files != 2 {
-			t.Errorf("Expected 2 needed files locally, got %v in meta", c.Files)
-		}
-		needed := make([]protocol.FileInfo, 0, 2)
-		snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileInfo) bool {
-			needed = append(needed, fi)
-			return true
-		})
-		if l := len(needed); l != 2 {
-			t.Errorf("Expected 2 needed files locally, got %v in db", l)
-		} else if needed[0].Name != "c" || needed[1].Name != "d" {
-			t.Errorf("Expected files c and d to be needed, got %v and %v", needed[0].Name, needed[1].Name)
-		}
-	}
-
-	checkNeed()
-
-	trans, err := db.newReadWriteTransaction()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer trans.close()
-
-	// Add "b" to needed and remove "d"
-	folder := []byte(folderStr)
-	key, err := trans.keyer.GenerateNeedFileKey(nil, folder, []byte(files[1].Name))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err = trans.Put(key, nil); err != nil {
-		t.Fatal(err)
-	}
-	key, err = trans.keyer.GenerateNeedFileKey(nil, folder, []byte(files[3].Name))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err = trans.Delete(key); err != nil {
-		t.Fatal(err)
-	}
-	if err := trans.Commit(); err != nil {
-		t.Fatal(err)
-	}
-
-	if repaired, err := db.checkLocalNeed(folder); err != nil {
-		t.Fatal(err)
-	} else if repaired != 2 {
-		t.Error("Expected 2 repaired local need items, got", repaired)
-	}
-
-	checkNeed()
-}
-
-func TestDuplicateNeedCount(t *testing.T) {
-	db := newLowlevelMemory(t)
-	defer db.Close()
-
-	folder := "test"
-
-	fs := newFileSet(t, folder, db)
-	files := []protocol.FileInfo{{Name: "foo", Version: protocol.Vector{}.Update(myID), Sequence: 1}}
-	fs.Update(protocol.LocalDeviceID, files)
-	files[0].Version = files[0].Version.Update(remoteDevice0.Short())
-	fs.Update(remoteDevice0, files)
-
-	db.checkRepair()
-
-	fs = newFileSet(t, folder, db)
-	found := false
-	for _, c := range fs.meta.counts.Counts {
-		if protocol.LocalDeviceID == c.DeviceID && c.LocalFlags == needFlag {
-			if found {
-				t.Fatal("second need count for local device encountered")
-			}
-			found = true
-		}
-	}
-	if !found {
-		t.Fatal("no need count for local device encountered")
-	}
-}
-
-func TestNeedAfterDropGlobal(t *testing.T) {
-	db := newLowlevelMemory(t)
-	defer db.Close()
-
-	folder := "test"
-
-	fs := newFileSet(t, folder, db)
-
-	// Initial:
-	// Three devices and a file "test": local has Version 1, remoteDevice0
-	// Version 2 and remoteDevice2 doesn't have it.
-	// All of them have "bar", just so the db knows about remoteDevice2.
-	files := []protocol.FileInfo{
-		{Name: "foo", Version: protocol.Vector{}.Update(myID), Sequence: 1},
-		{Name: "bar", Version: protocol.Vector{}.Update(myID), Sequence: 2},
-	}
-	fs.Update(protocol.LocalDeviceID, files)
-	files[0].Version = files[0].Version.Update(myID)
-	fs.Update(remoteDevice0, files)
-	fs.Update(remoteDevice1, files[1:])
-
-	// remoteDevice1 needs one file: test
-	snap := snapshot(t, fs)
-	c := snap.NeedSize(remoteDevice1)
-	if c.Files != 1 {
-		t.Errorf("Expected 1 needed files initially, got %v", c.Files)
-	}
-	snap.Release()
-
-	// Drop remoteDevice0, i.e. remove all their files from db.
-	// That changes the global file, which is now what local has.
-	fs.Drop(remoteDevice0)
-
-	// remoteDevice1 still needs test.
-	snap = snapshot(t, fs)
-	c = snap.NeedSize(remoteDevice1)
-	if c.Files != 1 {
-		t.Errorf("Expected still 1 needed files, got %v", c.Files)
-	}
-	snap.Release()
-}
-
-func numBlockLists(db *Lowlevel) (int, error) {
-	it, err := db.Backend.NewPrefixIterator([]byte{KeyTypeBlockList})
-	if err != nil {
-		return 0, err
-	}
-	defer it.Release()
-	n := 0
-	for it.Next() {
-		n++
-	}
-	if err := it.Error(); err != nil {
-		return 0, err
-	}
-	return n, nil
-}

lib/db/keyer_test.go

@@ -1,80 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package db
-
-import (
-	"bytes"
-	"testing"
-)
-
-func TestDeviceKey(t *testing.T) {
-	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
-	dev := []byte("device67890123456789012345678901")
-	name := []byte("name")
-
-	db := newLowlevelMemory(t)
-	defer db.Close()
-
-	key, err := db.keyer.GenerateDeviceFileKey(nil, fld, dev, name)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	fld2, ok := db.keyer.FolderFromDeviceFileKey(key)
-	if !ok {
-		t.Fatal("unexpectedly not found")
-	}
-	if !bytes.Equal(fld2, fld) {
-		t.Errorf("wrong folder %q != %q", fld2, fld)
-	}
-	dev2, ok := db.keyer.DeviceFromDeviceFileKey(key)
-	if !ok {
-		t.Fatal("unexpectedly not found")
-	}
-	if !bytes.Equal(dev2, dev) {
-		t.Errorf("wrong device %q != %q", dev2, dev)
-	}
-	name2 := db.keyer.NameFromDeviceFileKey(key)
-	if !bytes.Equal(name2, name) {
-		t.Errorf("wrong name %q != %q", name2, name)
-	}
-}
-
-func TestGlobalKey(t *testing.T) {
-	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
-	name := []byte("name")
-
-	db := newLowlevelMemory(t)
-	defer db.Close()
-
-	key, err := db.keyer.GenerateGlobalVersionKey(nil, fld, name)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	name2 := db.keyer.NameFromGlobalVersionKey(key)
-	if !bytes.Equal(name2, name) {
-		t.Errorf("wrong name %q != %q", name2, name)
-	}
-}
-
-func TestSequenceKey(t *testing.T) {
-	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
-
-	db := newLowlevelMemory(t)
-	defer db.Close()
-
-	const seq = 1234567890
-	key, err := db.keyer.GenerateSequenceKey(nil, fld, seq)
-	if err != nil {
-		t.Fatal(err)
-	}
-	outSeq := db.keyer.SequenceFromSequenceKey(key)
-	if outSeq != seq {
-		t.Errorf("sequence number mangled, %d != %d", outSeq, seq)
-	}
-}