mirror of
https://github.com/syncthing/syncthing.git
synced 2026-03-26 18:21:41 -04:00
This changes the files table to use normalisation for the names and
versions. The idea is that these are often common between all remote
devices, and repeating an integer is more efficient than repeating a
long string. A new benchmark bears this out; for a database with 100k
files shared between 31 devices, with some worst case assumption on
version vector size, the database is reduced in size by 50% and the test
finishes quicker:
Current:
db_bench_test.go:322: Total size: 6263.70 MiB
--- PASS: TestBenchmarkSizeManyFilesRemotes (1084.89s)
New:
db_bench_test.go:326: Total size: 3049.95 MiB
--- PASS: TestBenchmarkSizeManyFilesRemotes (776.97s)
The other benchmarks end up about the same within the margin of
variability, with one possible exception being that RemoteNeed seems to
be a little slower on average:
old files/s new files/s
Update/n=RemoteNeed/size=1000-8 5.051k 4.654k
Update/n=RemoteNeed/size=2000-8 5.201k 4.384k
Update/n=RemoteNeed/size=4000-8 4.943k 4.242k
Update/n=RemoteNeed/size=8000-8 5.099k 3.527k
Update/n=RemoteNeed/size=16000-8 3.686k 3.847k
Update/n=RemoteNeed/size=30000-8 4.456k 3.482k
I'm not sure why, possibly that query can be optimised anyhow.
Signed-off-by: Jakob Borg <jakob@kastelo.net>
164 lines
4.4 KiB
Go
164 lines
4.4 KiB
Go
// Copyright (C) 2014 The Syncthing Authors.
|
|
//
|
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
|
|
// Package osutil implements utilities for native OS support.
|
|
package osutil
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/syncthing/syncthing/lib/build"
|
|
"github.com/syncthing/syncthing/lib/fs"
|
|
)
|
|
|
|
// Try to keep this entire operation atomic-like. We shouldn't be doing this
|
|
// often enough that there is any contention on this lock.
|
|
var renameLock sync.Mutex
|
|
|
|
// RenameOrCopy renames a file, leaving source file intact in case of failure.
|
|
// Tries hard to succeed on various systems by temporarily tweaking directory
|
|
// permissions and removing the destination file when necessary.
|
|
func RenameOrCopy(method fs.CopyRangeMethod, src, dst fs.Filesystem, from, to string) error {
|
|
renameLock.Lock()
|
|
defer renameLock.Unlock()
|
|
|
|
return withPreparedTarget(dst, from, to, func() error {
|
|
// Optimisation 1
|
|
if src.Type() == dst.Type() && src.URI() == dst.URI() {
|
|
return src.Rename(from, to)
|
|
}
|
|
|
|
// "Optimisation" 2
|
|
// Try to find a common prefix between the two filesystems, use that as the base for the new one
|
|
// and try a rename.
|
|
if src.Type() == dst.Type() {
|
|
commonPrefix := fs.CommonPrefix(src.URI(), dst.URI())
|
|
if len(commonPrefix) > 0 {
|
|
commonFs := fs.NewFilesystem(src.Type(), commonPrefix)
|
|
err := commonFs.Rename(
|
|
filepath.Join(strings.TrimPrefix(src.URI(), commonPrefix), from),
|
|
filepath.Join(strings.TrimPrefix(dst.URI(), commonPrefix), to),
|
|
)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
|
|
// Everything is sad, do a copy and delete.
|
|
if _, err := dst.Stat(to); !fs.IsNotExist(err) {
|
|
err := dst.Remove(to)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
err := copyFileContents(method, src, dst, from, to)
|
|
if err != nil {
|
|
_ = dst.Remove(to)
|
|
return err
|
|
}
|
|
|
|
return withPreparedTarget(src, from, from, func() error {
|
|
return src.Remove(from)
|
|
})
|
|
})
|
|
}
|
|
|
|
// Copy copies the file content from source to destination.
|
|
// Tries hard to succeed on various systems by temporarily tweaking directory
|
|
// permissions and removing the destination file when necessary.
|
|
func Copy(method fs.CopyRangeMethod, src, dst fs.Filesystem, from, to string) error {
|
|
return withPreparedTarget(dst, from, to, func() error {
|
|
return copyFileContents(method, src, dst, from, to)
|
|
})
|
|
}
|
|
|
|
// Tries hard to succeed on various systems by temporarily tweaking directory
|
|
// permissions and removing the destination file when necessary.
|
|
func withPreparedTarget(filesystem fs.Filesystem, from, to string, f func() error) error {
|
|
// Make sure the destination directory is writeable
|
|
toDir := filepath.Dir(to)
|
|
if info, err := filesystem.Stat(toDir); err == nil && info.IsDir() && info.Mode()&0o200 == 0 {
|
|
filesystem.Chmod(toDir, 0o755)
|
|
defer filesystem.Chmod(toDir, info.Mode())
|
|
}
|
|
|
|
// On Windows, make sure the destination file is writeable (or we can't delete it)
|
|
if build.IsWindows {
|
|
filesystem.Chmod(to, 0o666)
|
|
if !strings.EqualFold(from, to) {
|
|
err := filesystem.Remove(to)
|
|
if err != nil && !fs.IsNotExist(err) {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return f()
|
|
}
|
|
|
|
// copyFileContents copies the contents of the file named src to the file named
|
|
// by dst. The file will be created if it does not already exist. If the
|
|
// destination file exists, all its contents will be replaced by the contents
|
|
// of the source file.
|
|
func copyFileContents(method fs.CopyRangeMethod, srcFs, dstFs fs.Filesystem, src, dst string) (err error) {
|
|
in, err := srcFs.Open(src)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer in.Close()
|
|
out, err := dstFs.Create(dst)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer func() {
|
|
cerr := out.Close()
|
|
if err == nil {
|
|
err = cerr
|
|
}
|
|
}()
|
|
inFi, err := in.Stat()
|
|
if err != nil {
|
|
return
|
|
}
|
|
err = fs.CopyRange(method, in, out, 0, 0, inFi.Size())
|
|
return
|
|
}
|
|
|
|
func IsDeleted(ffs fs.Filesystem, name string) bool {
|
|
if _, err := ffs.Lstat(name); err != nil {
|
|
if fs.IsNotExist(err) || fs.IsErrCaseConflict(err) {
|
|
return true
|
|
}
|
|
}
|
|
switch TraversesSymlink(ffs, filepath.Dir(name)).(type) {
|
|
case *NotADirectoryError, *TraversesSymlinkError:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func DirSize(location string) int64 {
|
|
entries, err := os.ReadDir(location)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
|
|
var size int64
|
|
for _, entry := range entries {
|
|
fi, err := entry.Info()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
size += fi.Size()
|
|
}
|
|
|
|
return size
|
|
}
|