Compare commits

...

21 Commits

Author SHA1 Message Date
Jakob Borg
a2da31056b Increase deadlock timeout, make configurable (fixes #389, fixes #393) 2014-06-26 11:29:41 +02:00
Jakob Borg
2383579a64 Remove spurious debug output in .stignore handling 2014-06-23 21:54:28 +02:00
Jakob Borg
68750211ef Connection notices are informational 2014-06-23 15:38:37 +02:00
Jakob Borg
db3e3ade80 No need to hold a write lock in Override 2014-06-23 11:52:13 +02:00
Jakob Borg
e6f04ed238 Don't whine about unexpected EOFs 2014-06-23 10:52:09 +02:00
Jakob Borg
a6eb690e31 Ensure correct version string format 2014-06-23 10:40:09 +02:00
Jakob Borg
77fe8449ba Test script for REST interface 2014-06-22 18:18:21 +02:00
Jakob Borg
33e9a35f08 Don't deadlock on connect close while sending Index (fixes #386) 2014-06-22 08:17:58 +02:00
Jakob Borg
4ab4816556 Detect deadlock in model and panic 2014-06-21 12:35:53 +02:00
Jakob Borg
8e8a579bb2 Asset update for previous commit 2014-06-20 11:40:38 +02:00
Jakob Borg
efbdf72d20 Lower CPU usage at idle by reducing db polling 2014-06-20 00:28:45 +02:00
Jakob Borg
0e59b5678a Further clarify message ordering requirements (ref #377) 2014-06-19 01:59:58 +02:00
Jakob Borg
de75550415 Clarify requirements on config messages (ref #377) 2014-06-19 01:27:03 +02:00
Jakob Borg
4dbce32738 Simplify memory handling 2014-06-19 01:02:32 +02:00
Jakob Borg
b05fcbc9d7 Simplify usage reporting config options (fixes #370) 2014-06-18 12:54:30 +02:00
Jakob Borg
d09c71b688 Avoid build error in Go1.2 2014-06-18 11:02:59 +02:00
Jakob Borg
874d6760d4 Handle .stignore correctly on Windows (fixes #369) 2014-06-16 16:19:14 +02:00
Jakob Borg
26ebbee877 Hard override on changes from master repo 2014-06-16 10:47:02 +02:00
Jakob Borg
12eda0449a Build and memSize impl for Solaris 2014-06-16 10:19:32 +02:00
Jakob Borg
5a98f4e47c Mark repos with missing dir as invalid on startup (fixes #311) 2014-06-16 09:33:52 +02:00
Jakob Borg
964c903a68 Only keep track of version (not modified) for sent index 2014-06-16 07:40:17 +02:00
28 changed files with 555 additions and 185 deletions

2
Godeps/Godeps.json generated
View File

@@ -1,6 +1,6 @@
{
"ImportPath": "github.com/calmh/syncthing",
"GoVersion": "go1.2.2",
"GoVersion": "go1.3",
"Packages": [
"./cmd/syncthing",
"./cmd/assets",

View File

File diff suppressed because one or more lines are too long

View File

@@ -52,7 +52,7 @@ func (b *Beacon) Recv() ([]byte, net.Addr) {
}
func (b *Beacon) reader() {
var bs = make([]byte, 65536)
bs := make([]byte, 65536)
for {
n, addr, err := b.conn.ReadFrom(bs)
if err != nil {
@@ -62,8 +62,11 @@ func (b *Beacon) reader() {
if debug {
l.Debugf("recv %d bytes from %s", n, addr)
}
c := make([]byte, n)
copy(c, bs)
select {
case b.outbox <- recv{bs[:n], addr}:
case b.outbox <- recv{c, addr}:
default:
if debug {
l.Debugln("dropping message")

View File

@@ -1,50 +0,0 @@
// Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
// Use of this source code is governed by an MIT-style license that can be
// found in the LICENSE file.
// Package buffers manages a set of reusable byte buffers.
package buffers
const (
largeMin = 1024
)
var (
smallBuffers = make(chan []byte, 32)
largeBuffers = make(chan []byte, 32)
)
func Get(size int) []byte {
var ch = largeBuffers
if size < largeMin {
ch = smallBuffers
}
var buf []byte
select {
case buf = <-ch:
default:
}
if len(buf) < size {
return make([]byte, size)
}
return buf[:size]
}
func Put(buf []byte) {
buf = buf[:cap(buf)]
if len(buf) == 0 {
return
}
var ch = largeBuffers
if len(buf) < largeMin {
ch = smallBuffers
}
select {
case ch <- buf:
default:
}
}

View File

@@ -142,7 +142,7 @@ case "$1" in
godep go build ./cmd/stpidx
godep go build ./cmd/stcli
for os in darwin-amd64 linux-386 linux-amd64 freebsd-amd64 windows-amd64 windows-386 ; do
for os in darwin-amd64 linux-386 linux-amd64 freebsd-amd64 windows-amd64 windows-386 solaris-amd64 ; do
export GOOS=${os%-*}
export GOARCH=${os#*-}

View File

@@ -91,6 +91,7 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
router.Get("/", getRoot)
router.Get("/rest/version", restGetVersion)
router.Get("/rest/model", restGetModel)
router.Get("/rest/model/version", restGetModelVersion)
router.Get("/rest/need", restGetNeed)
router.Get("/rest/connections", restGetConnections)
router.Get("/rest/config", restGetConfig)
@@ -108,6 +109,7 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
router.Post("/rest/error", restPostError)
router.Post("/rest/error/clear", restClearErrors)
router.Post("/rest/discovery/hint", restPostDiscoveryHint)
router.Post("/rest/model/override", restPostOverride)
mr := martini.New()
mr.Use(csrfMiddleware)
@@ -143,6 +145,17 @@ func restGetVersion() string {
return Version
}
func restGetModelVersion(m *model.Model, w http.ResponseWriter, r *http.Request) {
var qs = r.URL.Query()
var repo = qs.Get("repo")
var res = make(map[string]interface{})
res["version"] = m.Version(repo)
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(res)
}
func restGetModel(m *model.Model, w http.ResponseWriter, r *http.Request) {
var qs = r.URL.Query()
var repo = qs.Get("repo")
@@ -167,24 +180,31 @@ func restGetModel(m *model.Model, w http.ResponseWriter, r *http.Request) {
res["inSyncFiles"], res["inSyncBytes"] = globalFiles-needFiles, globalBytes-needBytes
res["state"] = m.State(repo)
res["version"] = m.Version(repo)
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(res)
}
func restPostOverride(m *model.Model, r *http.Request) {
var qs = r.URL.Query()
var repo = qs.Get("repo")
m.Override(repo)
}
func restGetNeed(m *model.Model, w http.ResponseWriter, r *http.Request) {
var qs = r.URL.Query()
var repo = qs.Get("repo")
files := m.NeedFilesRepo(repo)
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(files)
}
func restGetConnections(m *model.Model, w http.ResponseWriter) {
var res = m.ConnectionStats()
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(res)
}
@@ -193,6 +213,7 @@ func restGetConfig(w http.ResponseWriter) {
if encCfg.GUI.Password != "" {
encCfg.GUI.Password = unchangedPassword
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(encCfg)
}
@@ -243,30 +264,18 @@ func restPostConfig(req *http.Request, m *model.Model) {
}
}
if newCfg.Options.UREnabled && !cfg.Options.UREnabled {
if newCfg.Options.URAccepted > cfg.Options.URAccepted {
// UR was enabled
cfg.Options.UREnabled = true
cfg.Options.URDeclined = false
cfg.Options.URAccepted = usageReportVersion
// Set the corresponding options in newCfg so we don't trigger the restart check if this was the only option change
newCfg.Options.URDeclined = false
newCfg.Options.URAccepted = usageReportVersion
err := sendUsageReport(m)
if err != nil {
l.Infoln("Usage report:", err)
}
go usageReportingLoop(m)
} else if !newCfg.Options.UREnabled && cfg.Options.UREnabled {
} else if newCfg.Options.URAccepted < cfg.Options.URAccepted {
// UR was disabled
cfg.Options.UREnabled = false
cfg.Options.URDeclined = true
cfg.Options.URAccepted = 0
// Set the corresponding options in newCfg so we don't trigger the restart check if this was the only option change
newCfg.Options.URDeclined = true
newCfg.Options.URAccepted = 0
newCfg.Options.URAccepted = -1
stopUsageReporting()
} else {
cfg.Options.URDeclined = newCfg.Options.URDeclined
}
if !reflect.DeepEqual(cfg.Options, newCfg.Options) || !reflect.DeepEqual(cfg.GUI, newCfg.GUI) {
@@ -281,21 +290,25 @@ func restPostConfig(req *http.Request, m *model.Model) {
}
func restGetConfigInSync(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(map[string]bool{"configInSync": configInSync})
}
func restPostRestart(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
flushResponse(`{"ok": "restarting"}`, w)
go restart()
}
func restPostReset(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
flushResponse(`{"ok": "resetting repos"}`, w)
resetRepositories()
go restart()
}
func restPostShutdown(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
flushResponse(`{"ok": "shutting down"}`, w)
go shutdown()
}
@@ -330,11 +343,12 @@ func restGetSystem(w http.ResponseWriter) {
cpuUsageLock.RUnlock()
res["cpuPercent"] = cpusum / 10
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(res)
}
func restGetErrors(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
guiErrorsMut.Lock()
json.NewEncoder(w).Encode(guiErrors)
guiErrorsMut.Unlock()
@@ -371,10 +385,12 @@ func restPostDiscoveryHint(r *http.Request) {
}
func restGetDiscovery(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(discoverer.All())
}
func restGetReport(w http.ResponseWriter, m *model.Model) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(reportData(m))
}

View File

@@ -18,6 +18,7 @@ import (
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"runtime/debug"
"runtime/pprof"
@@ -48,6 +49,14 @@ var (
var l = logger.DefaultLogger
func init() {
if Version != "unknown-dev" {
// If not a generic dev build, version string should come from git describe
exp := regexp.MustCompile(`^v\d+\.\d+\.\d+(-\d+-g[0-9a-f]+)?(-dirty)?$`)
if !exp.MatchString(Version) {
l.Fatalf("Invalid version string %q;\n\tdoes not match regexp %v", Version, exp)
}
}
stamp, _ := strconv.Atoi(BuildStamp)
BuildDate = time.Unix(int64(stamp), 0)
@@ -106,7 +115,9 @@ The following enviroment variables are interpreted by syncthing:
STCPUPROFILE Write CPU profile to the specified file.
STGUIASSETS Directory to load GUI assets from. Overrides compiled in assets.`
STGUIASSETS Directory to load GUI assets from. Overrides compiled in assets.
STDEADLOCKTIMEOUT Alter deadlock detection timeout (seconds; default 1200).`
)
func init() {
@@ -279,11 +290,28 @@ func main() {
m := model.NewModel(confDir, &cfg, "syncthing", Version)
for _, repo := range cfg.Repositories {
nextRepo:
for i, repo := range cfg.Repositories {
if repo.Invalid != "" {
continue
}
repo.Directory = expandTilde(repo.Directory)
// Safety check. If the cached index contains files but the repository
// doesn't exist, we have a problem. We would assume that all files
// have been deleted which might not be the case, so abort instead.
id := fmt.Sprintf("%x", sha1.Sum([]byte(repo.Directory)))
idxFile := filepath.Join(confDir, id+".idx.gz")
if _, err := os.Stat(idxFile); err == nil {
if fi, err := os.Stat(repo.Directory); err != nil || !fi.IsDir() {
cfg.Repositories[i].Invalid = "repo directory missing"
continue nextRepo
}
}
ensureDir(repo.Directory, -1)
m.AddRepo(repo)
}
@@ -327,29 +355,6 @@ func main() {
l.Infoln("Populating repository index")
m.LoadIndexes(confDir)
for _, repo := range cfg.Repositories {
if repo.Invalid != "" {
continue
}
dir := expandTilde(repo.Directory)
// Safety check. If the cached index contains files but the repository
// doesn't exist, we have a problem. We would assume that all files
// have been deleted which might not be the case, so abort instead.
if files, _, _ := m.LocalSize(repo.ID); files > 0 {
if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
l.Warnf("Configured repository %q has index but directory %q is missing; not starting.", repo.ID, repo.Directory)
l.Fatalf("Ensure that directory is present or remove repository from configuration.")
}
}
// Ensure that repository directories exist for newly configured repositories.
ensureDir(dir, -1)
}
m.CleanRepos()
m.ScanRepos()
m.SaveIndexes(confDir)
@@ -421,11 +426,11 @@ func main() {
}
}
if cfg.Options.UREnabled && cfg.Options.URAccepted < usageReportVersion {
if cfg.Options.URAccepted > 0 && cfg.Options.URAccepted < usageReportVersion {
l.Infoln("Anonymous usage report has changed; revoking acceptance")
cfg.Options.UREnabled = false
cfg.Options.URAccepted = 0
}
if cfg.Options.UREnabled {
if cfg.Options.URAccepted >= usageReportVersion {
go usageReportingLoop(m)
go func() {
time.Sleep(10 * time.Minute)
@@ -700,6 +705,9 @@ next:
wr = &limitedWriter{conn, rateBucket}
}
protoConn := protocol.NewConnection(remoteID, conn, wr, m)
l.Infof("Connection to %s established at %v", remoteID, conn.RemoteAddr())
m.AddConnection(conn, protoConn)
continue next
}

View File

@@ -0,0 +1,22 @@
// +build solaris
package main
import (
"os/exec"
"strconv"
)
func memorySize() (uint64, error) {
cmd := exec.Command("prtconf", "-m")
out, err := cmd.CombinedOutput()
if err != nil {
return 0, err
}
mb, err := strconv.ParseUint(string(out), 10, 64)
if err != nil {
return 0, err
}
return mb * 1024 * 1024, nil
}

View File

@@ -1,4 +1,4 @@
// +build freebsd solaris
// +build freebsd
package main

View File

@@ -2,6 +2,8 @@
// Use of this source code is governed by an MIT-style license that can be
// found in the LICENSE file.
// +build !solaris,!windows
package main
import (

View File

@@ -0,0 +1,9 @@
// +build windows solaris
package main
import "errors"
func upgrade() error {
return errors.New("Upgrade currently unsupported on Windows")
}

View File

@@ -25,6 +25,7 @@ func reportData(m *model.Model) map[string]interface{} {
res := make(map[string]interface{})
res["uniqueID"] = strings.ToLower(certID([]byte(myID)))[:6]
res["version"] = Version
res["longVersion"] = LongVersion
res["platform"] = runtime.GOOS + "-" + runtime.GOARCH
res["numRepos"] = len(cfg.Repositories)
res["numNodes"] = len(cfg.Nodes)
@@ -107,7 +108,10 @@ loop:
}
func stopUsageReporting() {
stopUsageReportingCh <- struct{}{}
select {
case stopUsageReportingCh <- struct{}{}:
default:
}
}
// Returns CPU performance as a measure of single threaded SHA-256 MiB/s

View File

@@ -16,9 +16,9 @@ import (
"strconv"
"strings"
"github.com/calmh/syncthing/scanner"
"code.google.com/p/go.crypto/bcrypt"
"github.com/calmh/syncthing/logger"
"github.com/calmh/syncthing/scanner"
)
var l = logger.DefaultLogger
@@ -156,11 +156,10 @@ type OptionsConfiguration struct {
MaxChangeKbps int `xml:"maxChangeKbps" default:"10000"`
StartBrowser bool `xml:"startBrowser" default:"true"`
UPnPEnabled bool `xml:"upnpEnabled" default:"true"`
URAccepted int `xml:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
UREnabled bool `xml:"urEnabled"` // If true, send usage reporting data
URDeclined bool `xml:"urDeclined"` // If true, don't ask again
URAccepted int `xml:"urAccepted"` // Accepted usage reporting version
Deprecated_UREnabled bool `xml:"urEnabled,omitempty" json:"-"`
Deprecated_URDeclined bool `xml:"urDeclined,omitempty" json:"-"`
Deprecated_ReadOnly bool `xml:"readOnly,omitempty" json:"-"`
Deprecated_GUIEnabled bool `xml:"guiEnabled,omitempty" json:"-"`
Deprecated_GUIAddress string `xml:"guiAddress,omitempty" json:"-"`
@@ -345,6 +344,12 @@ func Load(rd io.Reader, myID string) (Configuration, error) {
}
}
if cfg.Options.Deprecated_URDeclined {
cfg.Options.URAccepted = -1
}
cfg.Options.Deprecated_URDeclined = false
cfg.Options.Deprecated_UREnabled = false
// Upgrade to v2 configuration if appropriate
if cfg.Version == 1 {
convertV1V2(&cfg)

View File

@@ -14,7 +14,6 @@ import (
"time"
"github.com/calmh/syncthing/beacon"
"github.com/calmh/syncthing/buffers"
)
type Discoverer struct {
@@ -329,11 +328,8 @@ func (d *Discoverer) externalLookup(node string) []string {
}
return nil
}
buffers.Put(buf)
buf = buffers.Get(2048)
defer buffers.Put(buf)
buf = make([]byte, 2048)
n, err := conn.Read(buf)
if err != nil {
if err, ok := err.(net.Error); ok && err.Timeout() {

BIN
files/testdata/index.db vendored Normal file
View File

Binary file not shown.

View File

@@ -103,9 +103,20 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
getFailed();
});
Object.keys($scope.repos).forEach(function (id) {
$http.get(urlbase + '/model?repo=' + encodeURIComponent(id)).success(function (data) {
$scope.model[id] = data;
});
if (typeof $scope.model[id] === 'undefined') {
// Never fetched before
$http.get(urlbase + '/model?repo=' + encodeURIComponent(id)).success(function (data) {
$scope.model[id] = data;
});
} else {
$http.get(urlbase + '/model/version?repo=' + encodeURIComponent(id)).success(function (data) {
if (data.version > $scope.model[id].version) {
$http.get(urlbase + '/model?repo=' + encodeURIComponent(id)).success(function (data) {
$scope.model[id] = data;
});
}
});
}
});
$http.get(urlbase + '/connections').success(function (data) {
var now = Date.now(),
@@ -279,8 +290,9 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
$scope.editSettings = function () {
// Make a working copy
$scope.config.workingOptions = angular.copy($scope.config.Options);
$scope.config.workingGUI = angular.copy($scope.config.GUI);
$scope.tmpOptions = angular.copy($scope.config.Options);
$scope.tmpOptions.UREnabled = ($scope.tmpOptions.URAccepted > 0);
$scope.tmpGUI = angular.copy($scope.config.GUI);
$('#settings').modal({backdrop: 'static', keyboard: true});
};
@@ -296,17 +308,24 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
$scope.saveSettings = function () {
// Make sure something changed
var changed = ! angular.equals($scope.config.Options, $scope.config.workingOptions) ||
! angular.equals($scope.config.GUI, $scope.config.workingGUI);
if(changed){
// see if protocol will need to be changed on restart
if($scope.config.GUI.UseTLS !== $scope.config.workingGUI.UseTLS){
var changed = !angular.equals($scope.config.Options, $scope.tmpOptions) ||
!angular.equals($scope.config.GUI, $scope.tmpGUI);
if (changed) {
// Check if usage reporting has been enabled or disabled
if ($scope.tmpOptions.UREnabled && $scope.tmpOptions.URAccepted <= 0) {
$scope.tmpOptions.URAccepted = 1000;
} else if (!$scope.tmpOptions.UREnabled && $scope.tmpOptions.URAccepted > 0){
$scope.tmpOptions.URAccepted = -1;
}
// Check if protocol will need to be changed on restart
if($scope.config.GUI.UseTLS !== $scope.tmpGUI.UseTLS){
$scope.protocolChanged = true;
}
// Apply new settings locally
$scope.config.Options = angular.copy($scope.config.workingOptions);
$scope.config.GUI = angular.copy($scope.config.workingGUI);
$scope.config.Options = angular.copy($scope.tmpOptions);
$scope.config.GUI = angular.copy($scope.tmpGUI);
$scope.config.Options.ListenAddress = $scope.config.Options.ListenStr.split(',').map(function (x) { return x.trim(); });
$scope.saveConfig();
@@ -561,7 +580,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
$scope.refresh();
if (!$scope.config.Options.UREnabled && !$scope.config.Options.URDeclined) {
if ($scope.config.Options.URAccepted == 0) {
// If usage reporting has been neither accepted nor declined,
// we want to ask the user to make a choice. But we don't want
// to bug them during initial setup, so we set a cookie with
@@ -590,15 +609,13 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
};
$scope.acceptUR = function () {
$scope.config.Options.UREnabled = true;
$scope.config.Options.URDeclined = false;
$scope.config.Options.URAccepted = 1000; // Larger than the largest existing report version
$scope.saveConfig();
$('#ur').modal('hide');
};
$scope.declineUR = function () {
$scope.config.Options.UREnabled = false;
$scope.config.Options.URDeclined = true;
$scope.config.Options.URAccepted = -1;
$scope.saveConfig();
$('#ur').modal('hide');
};
@@ -627,6 +644,12 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
}
};
$scope.override = function (repo) {
$http.post(urlbase + "/model/override?repo=" + encodeURIComponent(repo)).success(function () {
$scope.refresh();
});
};
$scope.init();
setInterval($scope.refresh, 10000);
});

View File

@@ -206,7 +206,10 @@ found in the LICENSE file.
</tbody>
</table>
</div>
<span class="pull-right"><a class="btn btn-sm btn-primary" href="" ng-click="editRepo(repo)"><span class="glyphicon glyphicon-pencil"></span>&emsp;Edit</a></span>
<span class="pull-right">
<a class="btn btn-sm btn-primary" href="" ng-click="editRepo(repo)"><span class="glyphicon glyphicon-pencil"></span>&emsp;Edit</a>
<a class="btn btn-sm btn-danger" ng-if="repo.ReadOnly && model[repo.ID].needFiles > 0" ng-click="override(repo.ID)" href=""><span class="glyphicon glyphicon-upload"></span>&emsp;Override Changes</a>
</span>
</div>
</div>
</div>
@@ -580,11 +583,11 @@ found in the LICENSE file.
<div class="form-group" ng-repeat="setting in settings">
<div ng-if="setting.type == 'text' || setting.type == 'number'">
<label for="{{setting.id}}">{{setting.descr}}</label>
<input id="{{setting.id}}" class="form-control" type="{{setting.type}}" ng-model="config.workingOptions[setting.id]"></input>
<input id="{{setting.id}}" class="form-control" type="{{setting.type}}" ng-model="tmpOptions[setting.id]"></input>
</div>
<div class="checkbox" ng-if="setting.type == 'bool'">
<label>
{{setting.descr}} <input id="{{setting.id}}" type="checkbox" ng-model="config.workingOptions[setting.id]"></input>
{{setting.descr}} <input id="{{setting.id}}" type="checkbox" ng-model="tmpOptions[setting.id]"></input>
</label>
</div>
</div>
@@ -593,17 +596,17 @@ found in the LICENSE file.
<div class="form-group" ng-repeat="setting in guiSettings">
<div ng-if="setting.type == 'text' || setting.type == 'number' || setting.type == 'password'">
<label for="{{setting.id}}">{{setting.descr}}</label>
<input id="{{setting.id}}" class="form-control" type="{{setting.type}}" ng-model="config.workingGUI[setting.id]"></input>
<input id="{{setting.id}}" class="form-control" type="{{setting.type}}" ng-model="tmpGUI[setting.id]"></input>
</div>
<div class="checkbox" ng-if="setting.type == 'bool'">
<label>
{{setting.descr}} <input id="{{setting.id}}" type="checkbox" ng-model="config.workingGUI[setting.id]"></input>
{{setting.descr}} <input id="{{setting.id}}" type="checkbox" ng-model="tmpGUI[setting.id]"></input>
</label>
</div>
<div ng-if="setting.type == 'apikey'">
<label>{{setting.descr}} (<a href="http://discourse.syncthing.net/t/v0-8-14-api-keys/335">Usage</a>)</label>
<div class="well well-sm text-monospace">{{config.workingGUI[setting.id] || "-"}}</div>
<button type="button" class="btn btn-sm btn-default" ng-click="setAPIKey(config.workingGUI)">Generate</button>
<div class="well well-sm text-monospace">{{tmpGUI[setting.id] || "-"}}</div>
<button type="button" class="btn btn-sm btn-default" ng-click="setAPIKey(tmpGUI)">Generate</button>
</div>
</div>
</div>

View File

@@ -14,3 +14,4 @@ dirs-*
*.out
csrftokens.txt
s4d
http

View File

@@ -25,6 +25,8 @@
<gui enabled="true" tls="false">
<address>127.0.0.1:8081</address>
<apikey>abc123</apikey>
<user>testuser</user>
<password>testpass</password>
</gui>
<options>
<listenAddress>127.0.0.1:22001</listenAddress>

232
integration/http.go Normal file
View File

@@ -0,0 +1,232 @@
// Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
// Use of this source code is governed by an MIT-style license that can be
// found in the LICENSE file.
// +build ignore
package main
import (
"bufio"
"flag"
"io/ioutil"
"log"
"net/http"
"os"
"regexp"
"testing"
)
var (
target string
authUser string
authPass string
csrfToken string
csrfFile string
apiKey string
)
var jsonEndpoints = []string{
"/rest/model?repo=default",
"/rest/model/version?repo=default",
"/rest/need",
"/rest/connections",
"/rest/config",
"/rest/config/sync",
"/rest/system",
"/rest/errors",
// "/rest/discovery",
"/rest/report",
}
func main() {
flag.StringVar(&target, "target", "localhost:8080", "Test target")
flag.StringVar(&authUser, "user", "", "Username")
flag.StringVar(&authPass, "pass", "", "Password")
flag.StringVar(&csrfFile, "csrf", "", "CSRF token file")
flag.StringVar(&apiKey, "api", "", "API key")
flag.Parse()
if len(csrfFile) > 0 {
fd, err := os.Open(csrfFile)
if err != nil {
log.Fatal(err)
}
s := bufio.NewScanner(fd)
for s.Scan() {
csrfToken = s.Text()
}
fd.Close()
}
var tests []testing.InternalTest
tests = append(tests, testing.InternalTest{"TestGetIndex", TestGetIndex})
tests = append(tests, testing.InternalTest{"TestGetVersion", TestGetVersion})
tests = append(tests, testing.InternalTest{"TestGetVersionNoCSRF", TestGetVersion})
tests = append(tests, testing.InternalTest{"TestJSONEndpoints", TestJSONEndpoints})
if len(authUser) > 0 || len(apiKey) > 0 {
tests = append(tests, testing.InternalTest{"TestJSONEndpointsNoAuth", TestJSONEndpointsNoAuth})
tests = append(tests, testing.InternalTest{"TestJSONEndpointsIncorrectAuth", TestJSONEndpointsIncorrectAuth})
}
if len(csrfToken) > 0 {
tests = append(tests, testing.InternalTest{"TestJSONEndpointsNoCSRF", TestJSONEndpointsNoCSRF})
}
testing.Main(matcher, tests, nil, nil)
}
func matcher(s0, s1 string) (bool, error) {
return true, nil
}
func TestGetIndex(t *testing.T) {
res, err := get("/index.html")
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 200 {
t.Errorf("Status %d != 200", res.StatusCode)
}
if res.ContentLength < 1024 {
t.Errorf("Length %d < 1024", res.ContentLength)
}
res.Body.Close()
res, err = get("/")
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 200 {
t.Errorf("Status %d != 200", res.StatusCode)
}
if res.ContentLength < 1024 {
t.Errorf("Length %d < 1024", res.ContentLength)
}
res.Body.Close()
}
func TestGetVersion(t *testing.T) {
res, err := get("/rest/version")
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 200 {
t.Fatalf("Status %d != 200", res.StatusCode)
}
ver, err := ioutil.ReadAll(res.Body)
if err != nil {
t.Fatal(err)
}
res.Body.Close()
if !regexp.MustCompile(`v\d+\.\d+\.\d+`).Match(ver) {
t.Errorf("Invalid version %q", ver)
}
}
func TestGetVersionNoCSRF(t *testing.T) {
r, err := http.NewRequest("GET", "http://"+target+"/rest/version", nil)
if err != nil {
t.Fatal(err)
}
if len(authUser) > 0 {
r.SetBasicAuth(authUser, authPass)
}
res, err := http.DefaultClient.Do(r)
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 403 {
t.Fatalf("Status %d != 403", res.StatusCode)
}
}
func TestJSONEndpoints(t *testing.T) {
for _, p := range jsonEndpoints {
res, err := get(p)
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 200 {
t.Errorf("Status %d != 200 for %q", res.StatusCode, p)
}
if ct := res.Header.Get("Content-Type"); ct != "application/json; charset=utf-8" {
t.Errorf("Content-Type %q != \"application/json\" for %q", ct, p)
}
}
}
func TestJSONEndpointsNoCSRF(t *testing.T) {
for _, p := range jsonEndpoints {
r, err := http.NewRequest("GET", "http://"+target+p, nil)
if err != nil {
t.Fatal(err)
}
if len(authUser) > 0 {
r.SetBasicAuth(authUser, authPass)
}
res, err := http.DefaultClient.Do(r)
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 403 && res.StatusCode != 401 {
t.Fatalf("Status %d != 403/401 for %q", res.StatusCode, p)
}
}
}
func TestJSONEndpointsNoAuth(t *testing.T) {
for _, p := range jsonEndpoints {
r, err := http.NewRequest("GET", "http://"+target+p, nil)
if err != nil {
t.Fatal(err)
}
if len(csrfToken) > 0 {
r.Header.Set("X-CSRF-Token", csrfToken)
}
res, err := http.DefaultClient.Do(r)
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 403 && res.StatusCode != 401 {
t.Fatalf("Status %d != 403/401 for %q", res.StatusCode, p)
}
}
}
func TestJSONEndpointsIncorrectAuth(t *testing.T) {
for _, p := range jsonEndpoints {
r, err := http.NewRequest("GET", "http://"+target+p, nil)
if err != nil {
t.Fatal(err)
}
if len(csrfToken) > 0 {
r.Header.Set("X-CSRF-Token", csrfToken)
}
r.SetBasicAuth("wronguser", "wrongpass")
res, err := http.DefaultClient.Do(r)
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 403 && res.StatusCode != 401 {
t.Fatalf("Status %d != 403/401 for %q", res.StatusCode, p)
}
}
}
func get(path string) (*http.Response, error) {
r, err := http.NewRequest("GET", "http://"+target+path, nil)
if err != nil {
return nil, err
}
if len(authUser) > 0 {
r.SetBasicAuth(authUser, authPass)
}
if len(csrfToken) > 0 {
r.Header.Set("X-CSRF-Token", csrfToken)
}
if len(apiKey) > 0 {
r.Header.Set("X-API-Key", apiKey)
}
return http.DefaultClient.Do(r)
}

View File

@@ -13,12 +13,30 @@ id3=373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA
go build genfiles.go
go build md5r.go
go build json.go
go build http.go
start() {
echo "Starting..."
for i in 1 2 3 4 ; do
STPROFILER=":909$i" syncthing -home "h$i" > "$i.out" 2>&1 &
done
# Test REST API
sleep 2
curl -s -o /dev/null http://testuser:testpass@localhost:8081/index.html
curl -s -o /dev/null http://localhost:8082/index.html
sleep 1
./http -target localhost:8081 -user testuser -pass testpass -csrf h1/csrftokens.txt || stop 1
./http -target localhost:8081 -api abc123 || stop 1
./http -target localhost:8082 -csrf h2/csrftokens.txt || stop 1
./http -target localhost:8082 -api abc123 || stop 1
}
stop() {
for i in 1 2 3 4 ; do
curl -HX-API-Key:abc123 -X POST "http://localhost:808$i/rest/shutdown"
done
exit $1
}
clean() {
@@ -83,8 +101,7 @@ testConvergence() {
fi
done
if [[ $ok != 7 ]] ; then
pkill syncthing
exit 1
stop 1
fi
}
@@ -157,6 +174,4 @@ for ((t = 1; t <= $iterations; t++)) ; do
testConvergence
done
for i in 1 2 3 4 ; do
curl -HX-API-Key:abc123 -X POST "http://localhost:808$i/rest/shutdown"
done
stop 0

View File

@@ -13,10 +13,10 @@ import (
"net"
"os"
"path/filepath"
"strconv"
"sync"
"time"
"github.com/calmh/syncthing/buffers"
"github.com/calmh/syncthing/cid"
"github.com/calmh/syncthing/config"
"github.com/calmh/syncthing/files"
@@ -98,6 +98,16 @@ func NewModel(indexDir string, cfg *config.Configuration, clientName, clientVers
sup: suppressor{threshold: int64(cfg.Options.MaxChangeKbps)},
}
var timeout = 20 * 60 // seconds
if t := os.Getenv("STDEADLOCKTIMEOUT"); len(t) > 0 {
it, err := strconv.Atoi(t)
if err == nil {
timeout = it
}
}
deadlockDetect(&m.rmut, time.Duration(timeout)*time.Second)
deadlockDetect(&m.smut, time.Duration(timeout)*time.Second)
deadlockDetect(&m.pmut, time.Duration(timeout)*time.Second)
go m.broadcastIndexLoop()
return m
}
@@ -364,15 +374,7 @@ func (m *Model) ClusterConfig(nodeID string, config protocol.ClusterConfigMessag
// Close removes the peer from the model and closes the underlying connection if possible.
// Implements the protocol.Model interface.
func (m *Model) Close(node string, err error) {
if debug {
l.Debugf("%s: %v", node, err)
}
if err != io.EOF {
l.Warnf("Connection to %s closed: %v", node, err)
} else if _, ok := err.(ClusterConfigMismatch); ok {
l.Warnf("Connection to %s closed: %v", node, err)
}
l.Infof("Connection to %s closed: %v", node, err)
cid := m.cm.Get(node)
m.rmut.RLock()
@@ -433,7 +435,7 @@ func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]by
}
defer fd.Close()
buf := buffers.Get(int(size))
buf := make([]byte, size)
_, err = fd.ReadAt(buf, offset)
if err != nil {
return nil, err
@@ -851,3 +853,42 @@ func (m *Model) State(repo string) string {
return "unknown"
}
}
func (m *Model) Override(repo string) {
fs := m.NeedFilesRepo(repo)
m.rmut.RLock()
r := m.repoFiles[repo]
m.rmut.RUnlock()
for i := range fs {
f := &fs[i]
h := r.Get(cid.LocalID, f.Name)
if h.Name != f.Name {
// We are missing the file
f.Flags |= protocol.FlagDeleted
f.Blocks = nil
} else {
// We have the file, replace with our version
*f = h
}
f.Version = lamport.Default.Tick(f.Version)
}
r.Update(cid.LocalID, fs)
}
// Version returns the change version for the given repository. This is
// guaranteed to increment if the contents of the local or global repository
// has changed.
func (m *Model) Version(repo string) uint64 {
var ver uint64
m.rmut.Lock()
for _, n := range m.repoNodes[repo] {
ver += m.repoFiles[repo].Changes(m.cm.Get(n))
}
m.rmut.Unlock()
return ver
}

View File

@@ -12,7 +12,6 @@ import (
"runtime"
"time"
"github.com/calmh/syncthing/buffers"
"github.com/calmh/syncthing/cid"
"github.com/calmh/syncthing/config"
"github.com/calmh/syncthing/osutil"
@@ -137,6 +136,7 @@ func (p *puller) run() {
walkTicker := time.Tick(time.Duration(p.cfg.Options.RescanIntervalS) * time.Second)
timeout := time.Tick(5 * time.Second)
changed := true
var prevVer uint64
for {
// Run the pulling loop as long as there are blocks to fetch
@@ -199,8 +199,11 @@ func (p *puller) run() {
default:
}
// Queue more blocks to fetch, if any
p.queueNeededBlocks()
if v := p.model.Version(p.repoCfg.ID); v > prevVer {
// Queue more blocks to fetch, if any
p.queueNeededBlocks()
prevVer = v
}
}
}
@@ -339,7 +342,6 @@ func (p *puller) handleRequestResult(res requestResult) {
}
_, of.err = of.file.WriteAt(res.data, res.offset)
buffers.Put(res.data)
of.outstanding--
p.openFiles[f.Name] = of
@@ -490,12 +492,11 @@ func (p *puller) handleCopyBlock(b bqBlock) {
defer exfd.Close()
for _, b := range b.copy {
bs := buffers.Get(int(b.Size))
bs := make([]byte, b.Size)
_, of.err = exfd.ReadAt(bs, b.Offset)
if of.err == nil {
_, of.err = of.file.WriteAt(bs, b.Offset)
}
buffers.Put(bs)
if of.err != nil {
if debug {
l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err)

View File

@@ -7,6 +7,8 @@ package model
import (
"fmt"
"path/filepath"
"sync"
"time"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
@@ -90,3 +92,27 @@ func compareClusterConfig(local, remote protocol.ClusterConfigMessage) error {
return nil
}
func deadlockDetect(mut sync.Locker, timeout time.Duration) {
go func() {
for {
time.Sleep(timeout / 4)
ok := make(chan bool, 2)
go func() {
mut.Lock()
mut.Unlock()
ok <- true
}()
go func() {
time.Sleep(timeout)
ok <- false
}()
if r := <-ok; !r {
panic("deadlock detected")
}
}
}()
}

View File

@@ -59,10 +59,11 @@ or certificate pinning combined with some out of band first
verification. The reference implementation uses preshared certificate
fingerprints (SHA-256) referred to as "Node IDs".
There is no required order or synchronization among BEP messages - any
message type may be sent at any time and the sender need not await a
response to one message before sending another. Responses MUST however
be sent in the same order as the requests are received.
There is no required order or synchronization among BEP messages except
as noted per message type - any message type may be sent at any time and
the sender need not await a response to one message before sending
another. Responses MUST however be sent in the same order as the
requests are received.
The underlying transport protocol MUST be TCP.
@@ -118,8 +119,9 @@ normalization form C.
### Cluster Config (Type = 0)
This informational message provides information about the cluster
configuration, as it pertains to the current connection. It is sent by
both sides after connection establishment.
configuration as it pertains to the current connection. A Cluster Config
message MUST be the first message sent on a BEP connection. Additional
Cluster Config messages MUST NOT be sent after the initial exchange.
#### Graphical Representation
@@ -295,11 +297,12 @@ peers acting in a specific manner as a result of sent options.
### Index (Type = 1)
The Index message defines the contents of the senders repository. An
Index message MUST be sent by each node immediately upon connection. A
node with no data to advertise MUST send an empty Index message (a file
list of zero length). If the repository contents change from non-empty
to empty, an empty Index message MUST be sent. There is no response to
the Index message.
Index message MUST be sent for each repository mentioned in the Cluster
Config message. An Index message for a repository MUST be sent before
any other message referring to that repository. A node with no data to
advertise MUST send an empty Index message (a file list of zero length).
If the repository contents change from non-empty to empty, an empty
Index message MUST be sent. There is no response to the Index message.
#### Graphical Representation

View File

@@ -81,10 +81,12 @@ type rawConnection struct {
xw *xdr.Writer
wmut sync.Mutex
indexSent map[string]map[string][2]int64
indexSent map[string]map[string]uint64
awaiting []chan asyncResult
imut sync.Mutex
idxMut sync.Mutex // ensures serialization of Index calls
nextID chan int
outbox chan []encodable
closed chan struct{}
@@ -122,7 +124,7 @@ func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver M
wb: wb,
xw: xdr.NewWriter(wb),
awaiting: make([]chan asyncResult, 0x1000),
indexSent: make(map[string]map[string][2]int64),
indexSent: make(map[string]map[string]uint64),
outbox: make(chan []encodable),
nextID: make(chan int),
closed: make(chan struct{}),
@@ -143,33 +145,36 @@ func (c *rawConnection) ID() string {
// Index writes the list of file information to the connected peer node
func (c *rawConnection) Index(repo string, idx []FileInfo) {
c.idxMut.Lock()
defer c.idxMut.Unlock()
c.imut.Lock()
var msgType int
if c.indexSent[repo] == nil {
// This is the first time we send an index.
msgType = messageTypeIndex
c.indexSent[repo] = make(map[string][2]int64)
c.indexSent[repo] = make(map[string]uint64)
for _, f := range idx {
c.indexSent[repo][f.Name] = [2]int64{f.Modified, int64(f.Version)}
c.indexSent[repo][f.Name] = f.Version
}
} else {
// We have sent one full index. Only send updates now.
msgType = messageTypeIndexUpdate
var diff []FileInfo
for _, f := range idx {
if vs, ok := c.indexSent[repo][f.Name]; !ok || f.Modified != vs[0] || int64(f.Version) != vs[1] {
if vs, ok := c.indexSent[repo][f.Name]; !ok || f.Version != vs {
diff = append(diff, f)
c.indexSent[repo][f.Name] = [2]int64{f.Modified, int64(f.Version)}
c.indexSent[repo][f.Name] = f.Version
}
}
idx = diff
}
c.imut.Unlock()
if len(idx) > 0 {
c.send(header{0, -1, msgType}, IndexMessage{repo, idx})
}
c.imut.Unlock()
}
// Request returns the bytes for the specified block after fetching them from the connected peer.

View File

@@ -7,6 +7,7 @@ package scanner
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
@@ -104,13 +105,14 @@ func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.W
}
if pn, sn := filepath.Split(rn); sn == w.IgnoreFile {
pn := strings.Trim(pn, "/")
pn := filepath.Clean(pn)
bs, _ := ioutil.ReadFile(p)
lines := bytes.Split(bs, []byte("\n"))
var patterns []string
for _, line := range lines {
if len(line) > 0 {
patterns = append(patterns, string(line))
lineStr := strings.TrimSpace(string(line))
if len(lineStr) > 0 {
patterns = append(patterns, lineStr)
}
}
ign[pn] = patterns
@@ -282,7 +284,7 @@ func (w *Walker) cleanTempFile(path string, info os.FileInfo, err error) error {
func (w *Walker) ignoreFile(patterns map[string][]string, file string) bool {
first, last := filepath.Split(file)
for prefix, pats := range patterns {
if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") {
if prefix == "." || prefix == first || strings.HasPrefix(first, fmt.Sprintf("%s%c", prefix, os.PathSeparator)) {
for _, pattern := range pats {
if match, _ := filepath.Match(pattern, last); match {
return true

View File

@@ -22,7 +22,7 @@ var testdata = []struct {
}
var correctIgnores = map[string][]string{
"": {".*", "quux"},
".": {".*", "quux"},
}
func TestWalk(t *testing.T) {
@@ -88,7 +88,7 @@ func TestWalkError(t *testing.T) {
func TestIgnore(t *testing.T) {
var patterns = map[string][]string{
"": {"t2"},
".": {"t2"},
"foo": {"bar", "z*"},
"foo/baz": {"quux", ".*"},
}
@@ -97,6 +97,7 @@ func TestIgnore(t *testing.T) {
r bool
}{
{"foo/bar", true},
{"foofoo", false},
{"foo/quux", false},
{"foo/zuux", true},
{"foo/qzuux", false},