chore: switch database engine to sqlite (fixes #9954) (#9965)

Switch the database from LevelDB to SQLite, for greater stability and simpler code. Co-authored-by: Tommy van der Vorst <tommy@pixelspark.nl> Co-authored-by: bt90 <btom1990@googlemail.com>
2026-02-06 12:12:09 -05:00 · 2025-03-29 12:50:08 +00:00
parent b1c8f88a44
commit 025905fcdf
146 changed files with 8315 additions and 11984 deletions
--- a/.github/workflows/build-syncthing.yaml
+++ b/.github/workflows/build-syncthing.yaml
@@ -13,8 +13,6 @@ env:
  GO_VERSION: "~1.24.0"

  # Optimize compatibility on the slow archictures.
-  GO386: softfloat
-  GOARM: "5"
  GOMIPS: softfloat

  # Avoid hilarious amounts of obscuring log output when running tests.
@@ -24,6 +22,8 @@ env:
  BUILD_USER: builder
  BUILD_HOST: github.syncthing.net

+  TAGS: "netgo osusergo sqlite_omit_load_extension"
+
 # A note on actions and third party code... The actions under actions/ (like
 # `uses: actions/checkout`) are maintained by GitHub, and we need to trust
 # GitHub to maintain their code and infrastructure or we're in deep shit in
@@ -85,6 +85,7 @@ jobs:
          LOKI_USER: ${{ secrets.LOKI_USER }}
          LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD }}
          LOKI_LABELS: "go=${{ matrix.go }},runner=${{ matrix.runner }},repo=${{ github.repository }},ref=${{ github.ref }}"
+          CGO_ENABLED: "1"

  #
  # Meta checks for formatting, copyright, etc
@@ -136,17 +137,8 @@ jobs:

  package-windows:
    name: Package for Windows
-    runs-on: windows-latest
+    runs-on: ubuntu-latest
    steps:
-      - name: Set git to use LF
-        # Without this, the checkout will happen with CRLF line endings,
-        # which is fine for the source code but messes up tests that depend
-        # on data on disk being as expected. Ideally, those tests should be
-        # fixed, but not today.
-        run: |
-          git config --global core.autocrlf false
-          git config --global core.eol lf
-
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
@@ -158,17 +150,14 @@ jobs:
          cache: false
          check-latest: true

-      - name: Get actual Go version
-        run: |
-          go version
-          echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
+      - uses: mlugg/setup-zig@v1

      - uses: actions/cache@v4
        with:
          path: |
-            ~\AppData\Local\go-build
-            ~\go\pkg\mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-windows-${{ hashFiles('**/go.sum') }}

      - name: Install dependencies
        run: |
@@ -176,15 +165,14 @@ jobs:

      - name: Create packages
        run: |
-          $targets = 'syncthing', 'stdiscosrv', 'strelaysrv'
-          $archs = 'amd64', 'arm', 'arm64', '386'
-          foreach ($arch in $archs) {
-            foreach ($tgt in $targets) {
-              go run build.go -goarch $arch zip $tgt
-            }
-          }
+          for tgt in syncthing stdiscosrv strelaysrv ; do
+            go run build.go -tags "${{env.TAGS}}" -goos windows -goarch amd64 -cc "zig cc -target x86_64-windows" zip $tgt
+            go run build.go -tags "${{env.TAGS}}" -goos windows -goarch 386 -cc "zig cc -target x86-windows" zip $tgt
+            go run build.go -tags "${{env.TAGS}}" -goos windows -goarch arm64 -cc "zig cc -target aarch64-windows" zip $tgt
+            # go run build.go -tags "${{env.TAGS}}" -goos windows -goarch arm -cc "zig cc -target thumb-windows" zip $tgt # failes with linker errors
+          done
        env:
-          CGO_ENABLED: "0"
+          CGO_ENABLED: "1"

      - name: Archive artifacts
        uses: actions/upload-artifact@v4
@@ -194,7 +182,7 @@ jobs:

  codesign-windows:
    name: Codesign for Windows
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
+    if: github.repository_owner == 'syncthing' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
    environment: release
    runs-on: windows-latest
    needs:
@@ -269,6 +257,8 @@ jobs:
          go version
          echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV

+      - uses: mlugg/setup-zig@v1
+
      - uses: actions/cache@v4
        with:
          path: |
@@ -278,14 +268,25 @@ jobs:

      - name: Create packages
        run: |
-          archs=$(go tool dist list | grep linux | sed 's#linux/##')
-          for goarch in $archs ; do
-            for tgt in syncthing stdiscosrv strelaysrv ; do
-              go run build.go -goarch "$goarch" tar "$tgt"
-            done
+          sudo apt-get install -y gcc-mips64-linux-gnuabi64 gcc-mips64el-linux-gnuabi64
+          for tgt in syncthing stdiscosrv strelaysrv ; do
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch amd64 -cc "zig cc -target x86_64-linux-musl" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch 386 -cc "zig cc -target x86-linux-musl" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch arm -cc "zig cc -target arm-linux-musleabi" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch arm64 -cc "zig cc -target aarch64-linux-musl" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch mips -cc "zig cc -target mips-linux-musleabi" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch mipsle -cc "zig cc -target mipsel-linux-musleabi" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch mips64 -cc mips64-linux-gnuabi64-gcc tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch mips64le -cc mips64el-linux-gnuabi64-gcc tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch riscv64 -cc "zig cc -target riscv64-linux-musl" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch s390x -cc "zig cc -target s390x-linux-musl" tar "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch loong64 -cc "zig cc -target loongarch64-linux-musl" tar "$tgt"
+            # go run build.go -tags "${{env.TAGS}}" -goos linux -goarch ppc64 -cc "zig cc -target powerpc64-linux-musl" tar "$tgt" # fails with linkmode not supported
+            go run build.go -tags "${{env.TAGS}}" -goos linux -goarch ppc64le -cc "zig cc -target powerpc64le-linux-musl" tar "$tgt"
          done
        env:
-          CGO_ENABLED: "0"
+          CGO_ENABLED: "1"
+          EXTRA_LDFLAGS: "-linkmode=external -extldflags=-static"

      - name: Archive artifacts
        uses: actions/upload-artifact@v4
@@ -303,6 +304,8 @@ jobs:
    name: Package for macOS
    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
    environment: release
+    env:
+      CODESIGN_IDENTITY: ${{ secrets.CODESIGN_IDENTITY }}
    runs-on: macos-latest
    steps:
      - uses: actions/checkout@v4
@@ -329,6 +332,7 @@ jobs:
          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}

      - name: Import signing certificate
+        if: env.CODESIGN_IDENTITY != ''
        run: |
          # Set up a run-specific keychain, making it available for the
          # `codesign` tool.
@@ -356,7 +360,7 @@ jobs:
      - name: Create package (amd64)
        run: |
          for tgt in syncthing stdiscosrv strelaysrv ; do
-            go run build.go -goarch amd64 zip "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -goarch amd64 zip "$tgt"
          done
        env:
          CGO_ENABLED: "1"
@@ -372,7 +376,7 @@ jobs:
          EOT
          chmod 755 xgo.sh
          for tgt in syncthing stdiscosrv strelaysrv ; do
-            go run build.go -gocmd ./xgo.sh -goarch arm64 zip "$tgt"
+            go run build.go -tags "${{env.TAGS}}" -gocmd ./xgo.sh -goarch arm64 zip "$tgt"
          done
        env:
          CGO_ENABLED: "1"
@@ -401,7 +405,7 @@ jobs:

  notarize-macos:
    name: Notarize for macOS
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
+    if: github.repository_owner == 'syncthing' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
    environment: release
    needs:
      - package-macos
@@ -483,7 +487,7 @@ jobs:
            goarch="${plat#*/}"
            echo "::group ::$plat"
            for tgt in syncthing stdiscosrv strelaysrv ; do
-              if ! go run build.go -goos "$goos" -goarch "$goarch" tar "$tgt" 2>/dev/null; then
+              if ! go run build.go -goos "$goos" -goarch "$goarch" tar "$tgt" ; then
                echo "::warning ::Failed to build $tgt for $plat"
              fi
            done
@@ -545,7 +549,7 @@ jobs:

  sign-for-upgrade:
    name: Sign for upgrade
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-')  || startsWith(github.ref, 'refs/tags/v'))
+    if: github.repository_owner == 'syncthing' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-')  || startsWith(github.ref, 'refs/tags/v'))
    environment: release
    needs:
      - codesign-windows
@@ -663,6 +667,8 @@ jobs:
        run: |
          gem install fpm

+      - uses: mlugg/setup-zig@v1
+
      - uses: actions/cache@v4
        with:
          path: |
@@ -670,15 +676,17 @@ jobs:
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-debian-${{ hashFiles('**/go.sum') }}

-      - name: Package for Debian
+      - name: Package for Debian (CGO)
        run: |
-          for arch in amd64 i386 armhf armel arm64 ; do
-            for tgt in syncthing stdiscosrv strelaysrv ; do
-              go run build.go -no-upgrade -installsuffix=no-upgrade -goarch "$arch" deb "$tgt"
-            done
+          for tgt in syncthing stdiscosrv strelaysrv ; do
+            go run build.go -no-upgrade -installsuffix=no-upgrade -tags "${{env.TAGS}}" -goos linux -goarch amd64 -cc "zig cc -target x86_64-linux-musl" deb "$tgt"
+            go run build.go -no-upgrade -installsuffix=no-upgrade -tags "${{env.TAGS}}" -goos linux -goarch arm -cc "zig cc -target arm-linux-musleabi" deb "$tgt"
+            go run build.go -no-upgrade -installsuffix=no-upgrade -tags "${{env.TAGS}}" -goos linux -goarch arm64 -cc "zig cc -target aarch64-linux-musl" deb "$tgt"
          done
        env:
          BUILD_USER: debian
+          CGO_ENABLED: "1"
+          EXTRA_LDFLAGS: "-linkmode=external -extldflags=-static"

      - name: Archive artifacts
        uses: actions/upload-artifact@v4
@@ -692,7 +700,7 @@ jobs:

  publish-nightly:
    name: Publish nightly build
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && startsWith(github.ref, 'refs/heads/release-nightly')
+    if: github.repository_owner == 'syncthing' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && startsWith(github.ref, 'refs/heads/release-nightly')
    environment: release
    needs:
      - sign-for-upgrade
@@ -742,7 +750,7 @@ jobs:

  publish-release-files:
    name: Publish release files
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v'))
+    if: github.repository_owner == 'syncthing' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v'))
    environment: release
    needs:
      - sign-for-upgrade
@@ -809,7 +817,7 @@ jobs:

  publish-apt:
    name: Publish APT
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
+    if: github.repository_owner == 'syncthing' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v1'))
    environment: release
    needs:
      - package-debian
@@ -836,7 +844,9 @@ jobs:
      - name: Prepare packages
        run: |
          kind=stable
-          if [[ $VERSION == *-rc.[0-9] ]] ; then
+          if [[ $VERSION == v2* ]] ; then
+            kind=v2
+          elif [[ $VERSION == *-rc.[0-9] ]] ; then
            kind=candidate
          elif [[ $VERSION == *-* ]] ; then
            kind=nightly
@@ -888,8 +898,10 @@ jobs:
  docker-syncthing:
    name: Build and push Docker images
    runs-on: ubuntu-latest
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release' || github.ref == 'refs/heads/infrastructure' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
    environment: docker
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
    permissions:
      contents: read
      packages: write
@@ -902,13 +914,13 @@ jobs:
        include:
          - pkg: syncthing
            dockerfile: Dockerfile
-            image: syncthing/syncthing
+            image: syncthing
          - pkg: strelaysrv
            dockerfile: Dockerfile.strelaysrv
-            image: syncthing/relaysrv
+            image: relaysrv
          - pkg: stdiscosrv
            dockerfile: Dockerfile.stdiscosrv
-            image: syncthing/discosrv
+            image: discosrv
    steps:
      - uses: actions/checkout@v4
        with:
@@ -926,6 +938,8 @@ jobs:
          go version
          echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV

+      - uses: mlugg/setup-zig@v1
+
      - uses: actions/cache@v4
        with:
          path: |
@@ -933,33 +947,34 @@ jobs:
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-docker-${{ matrix.pkg }}-${{ hashFiles('**/go.sum') }}

-      - name: Build binaries
+      - name: Build binaries (CGO)
        run: |
-          for arch in amd64 arm64 arm; do
-            go run build.go -goos linux -goarch "$arch" -no-upgrade build ${{ matrix.pkg }}
-            mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-"$arch"
-          done
-        env:
-          CGO_ENABLED: "0"
-          BUILD_USER: docker
+          # amd64
+          go run build.go -goos linux -goarch amd64 -tags "${{env.TAGS}}" -cc "zig cc -target x86_64-linux-musl" -no-upgrade build ${{ matrix.pkg }}
+          mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-amd64

-      - name: Check if we will be able to push images
-        run: |
-          if [[ "${{ secrets.DOCKERHUB_TOKEN }}" != "" ]]; then
-            echo "DOCKER_PUSH=true" >> $GITHUB_ENV;
-          fi
+          # arm64
+          go run build.go -goos linux -goarch arm64 -tags "${{env.TAGS}}" -cc "zig cc -target aarch64-linux-musl" -no-upgrade build ${{ matrix.pkg }}
+          mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-arm64
+
+          # arm
+          go run build.go -goos linux -goarch arm -tags "${{env.TAGS}}" -cc "zig cc -target arm-linux-musleabi" -no-upgrade build ${{ matrix.pkg }}
+          mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-arm
+        env:
+          CGO_ENABLED: "1"
+          BUILD_USER: docker
+          EXTRA_LDFLAGS: "-linkmode=external -extldflags=-static"

      - name: Login to Docker Hub
        uses: docker/login-action@v3
-        if: env.DOCKER_PUSH == 'true'
+        if: env.DOCKERHUB_USERNAME != ''
        with:
          registry: docker.io
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          username: ${{ env.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Login to GHCR
        uses: docker/login-action@v3
-        if: env.DOCKER_PUSH == 'true'
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
@@ -972,18 +987,31 @@ jobs:
        run: |
          version=$(go run build.go version)
          version=${version#v}
+          repo=ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}
+          ref="${{github.ref_name}}"
+          ref=${ref//\//-} # slashes to dashes
+
+          # List of tags for ghcr.io
          if [[ $version == @([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]) ]] ; then
-            echo Release version, pushing to :latest and version tags
            major=${version%.*.*}
            minor=${version%.*}
-            tags=docker.io/${{ matrix.image }}:$version,ghcr.io/${{ matrix.image }}:$version,docker.io/${{ matrix.image }}:$major,ghcr.io/${{ matrix.image }}:$major,docker.io/${{ matrix.image }}:$minor,ghcr.io/${{ matrix.image }}:$minor,docker.io/${{ matrix.image }}:latest,ghcr.io/${{ matrix.image }}:latest
+            tags=$repo:$version,$repo:$major,$repo:$minor,$repo:latest
          elif [[ $version == *-rc.@([0-9]|[0-9][0-9]) ]] ; then
-            echo Release candidate, pushing to :rc and version tags
-            tags=docker.io/${{ matrix.image }}:$version,ghcr.io/${{ matrix.image }}:$version,docker.io/${{ matrix.image }}:rc,ghcr.io/${{ matrix.image }}:rc
+            tags=$repo:$version,$repo:rc
+          elif [[ $ref == "main" ]] ; then
+            tags=$repo:edge
          else
-            echo Development version, pushing to :edge
-            tags=docker.io/${{ matrix.image }}:edge,ghcr.io/${{ matrix.image }}:edge
+            tags=$repo:$ref
          fi
+
+          # If we have a Docker Hub secret, also push to there.
+          if [[ $DOCKERHUB_USERNAME != "" ]] ; then
+            dockerhubtags="${tags//ghcr.io\/syncthing/docker.io\/syncthing}"
+            tags="$tags,$dockerhubtags"
+          fi
+
+          echo Pushing to $tags
+
          echo "DOCKER_TAGS=$tags" >> $GITHUB_ENV
          echo "VERSION=$version" >> $GITHUB_ENV

@@ -993,8 +1021,8 @@ jobs:
          context: .
          file: ${{ matrix.dockerfile }}
          platforms: linux/amd64,linux/arm64,linux/arm/7
-          push: ${{ env.DOCKER_PUSH == 'true' }}
          tags: ${{ env.DOCKER_TAGS }}
+          push: true
          labels: |
            org.opencontainers.image.version=${{ env.VERSION }}
            org.opencontainers.image.revision=${{ github.sha }}