Fix ui, hide report date

This commit is contained in:
Audrius Butkevicius
2020-06-20 17:46:44 +01:00
parent 3fcf22ed5d
commit 16f2445764
22 changed files with 826 additions and 648 deletions

View File

@@ -114,6 +114,23 @@ func statsForInts(data []int) [4]float64 {
return res
}
func statsForInt64s(data []int64) [4]float64 {
var res [4]float64
if len(data) == 0 {
return res
}
sort.Slice(data, func(a, b int) bool {
return data[a] < data[b]
})
res[0] = float64(data[int(float64(len(data))*0.05)])
res[1] = float64(data[len(data)/2])
res[2] = float64(data[int(float64(len(data))*0.95)])
res[3] = float64(data[len(data)-1])
return res
}
func statsForFloats(data []float64) [4]float64 {
var res [4]float64
if len(data) == 0 {

View File

@@ -10,9 +10,7 @@ import (
"bytes"
"crypto/tls"
"database/sql"
"database/sql/driver"
"encoding/json"
"errors"
"fmt"
"html/template"
"io"
@@ -29,8 +27,9 @@ import (
"time"
"unicode"
"github.com/lib/pq"
geoip2 "github.com/oschwald/geoip2-golang"
"github.com/syncthing/syncthing/lib/ur/contract"
)
var (
@@ -103,376 +102,6 @@ func getEnvDefault(key, def string) string {
return def
}
type IntMap map[string]int
func (p IntMap) Value() (driver.Value, error) {
return json.Marshal(p)
}
func (p *IntMap) Scan(src interface{}) error {
source, ok := src.([]byte)
if !ok {
return errors.New("Type assertion .([]byte) failed.")
}
var i map[string]int
err := json.Unmarshal(source, &i)
if err != nil {
return err
}
*p = i
return nil
}
type report struct {
Received time.Time // Only from DB
UniqueID string
Version string
LongVersion string
Platform string
NumFolders int
NumDevices int
TotFiles int
FolderMaxFiles int
TotMiB int
FolderMaxMiB int
MemoryUsageMiB int
SHA256Perf float64
MemorySize int
// v2 fields
URVersion int
NumCPU int
FolderUses struct {
SendOnly int
ReceiveOnly int
IgnorePerms int
IgnoreDelete int
AutoNormalize int
SimpleVersioning int
ExternalVersioning int
StaggeredVersioning int
TrashcanVersioning int
}
DeviceUses struct {
Introducer int
CustomCertName int
CompressAlways int
CompressMetadata int
CompressNever int
DynamicAddr int
StaticAddr int
}
Announce struct {
GlobalEnabled bool
LocalEnabled bool
DefaultServersDNS int
DefaultServersIP int
OtherServers int
}
Relays struct {
Enabled bool
DefaultServers int
OtherServers int
}
UsesRateLimit bool
UpgradeAllowedManual bool
UpgradeAllowedAuto bool
// V2.5 fields (fields that were in v2 but never added to the database
UpgradeAllowedPre bool
RescanIntvs pq.Int64Array
// v3 fields
Uptime int
NATType string
AlwaysLocalNets bool
CacheIgnoredFiles bool
OverwriteRemoteDeviceNames bool
ProgressEmitterEnabled bool
CustomDefaultFolderPath bool
WeakHashSelection string
CustomTrafficClass bool
CustomTempIndexMinBlocks bool
TemporariesDisabled bool
TemporariesCustom bool
LimitBandwidthInLan bool
CustomReleaseURL bool
RestartOnWakeup bool
CustomStunServers bool
FolderUsesV3 struct {
ScanProgressDisabled int
ConflictsDisabled int
ConflictsUnlimited int
ConflictsOther int
DisableSparseFiles int
DisableTempIndexes int
AlwaysWeakHash int
CustomWeakHashThreshold int
FsWatcherEnabled int
PullOrder IntMap
FilesystemType IntMap
FsWatcherDelays pq.Int64Array
}
GUIStats struct {
Enabled int
UseTLS int
UseAuth int
InsecureAdminAccess int
Debugging int
InsecureSkipHostCheck int
InsecureAllowFrameLoading int
ListenLocal int
ListenUnspecified int
Theme IntMap
}
BlockStats struct {
Total int
Renamed int
Reused int
Pulled int
CopyOrigin int
CopyOriginShifted int
CopyElsewhere int
}
TransportStats IntMap
IgnoreStats struct {
Lines int
Inverts int
Folded int
Deletable int
Rooted int
Includes int
EscapedIncludes int
DoubleStars int
Stars int
}
// V3 fields added late in the RC
WeakHashEnabled bool
// Generated
Date string
Address string
}
func (r *report) Validate() error {
if r.UniqueID == "" || r.Version == "" || r.Platform == "" {
return errors.New("missing required field")
}
if len(r.Date) != 8 {
return errors.New("date not initialized")
}
// Some fields may not be null.
if r.RescanIntvs == nil {
r.RescanIntvs = []int64{}
}
if r.FolderUsesV3.FsWatcherDelays == nil {
r.FolderUsesV3.FsWatcherDelays = []int64{}
}
return nil
}
func (r *report) FieldPointers() []interface{} {
// All the fields of the report, in the same order as the database fields.
return []interface{}{
&r.Received, &r.UniqueID, &r.Version, &r.LongVersion, &r.Platform,
&r.NumFolders, &r.NumDevices, &r.TotFiles, &r.FolderMaxFiles,
&r.TotMiB, &r.FolderMaxMiB, &r.MemoryUsageMiB, &r.SHA256Perf,
&r.MemorySize, &r.Date,
// V2
&r.URVersion, &r.NumCPU, &r.FolderUses.SendOnly, &r.FolderUses.IgnorePerms,
&r.FolderUses.IgnoreDelete, &r.FolderUses.AutoNormalize, &r.DeviceUses.Introducer,
&r.DeviceUses.CustomCertName, &r.DeviceUses.CompressAlways,
&r.DeviceUses.CompressMetadata, &r.DeviceUses.CompressNever,
&r.DeviceUses.DynamicAddr, &r.DeviceUses.StaticAddr,
&r.Announce.GlobalEnabled, &r.Announce.LocalEnabled,
&r.Announce.DefaultServersDNS, &r.Announce.DefaultServersIP,
&r.Announce.OtherServers, &r.Relays.Enabled, &r.Relays.DefaultServers,
&r.Relays.OtherServers, &r.UsesRateLimit, &r.UpgradeAllowedManual,
&r.UpgradeAllowedAuto, &r.FolderUses.SimpleVersioning,
&r.FolderUses.ExternalVersioning, &r.FolderUses.StaggeredVersioning,
&r.FolderUses.TrashcanVersioning,
// V2.5
&r.UpgradeAllowedPre, &r.RescanIntvs,
// V3
&r.Uptime, &r.NATType, &r.AlwaysLocalNets, &r.CacheIgnoredFiles,
&r.OverwriteRemoteDeviceNames, &r.ProgressEmitterEnabled, &r.CustomDefaultFolderPath,
&r.WeakHashSelection, &r.CustomTrafficClass, &r.CustomTempIndexMinBlocks,
&r.TemporariesDisabled, &r.TemporariesCustom, &r.LimitBandwidthInLan,
&r.CustomReleaseURL, &r.RestartOnWakeup, &r.CustomStunServers,
&r.FolderUsesV3.ScanProgressDisabled, &r.FolderUsesV3.ConflictsDisabled,
&r.FolderUsesV3.ConflictsUnlimited, &r.FolderUsesV3.ConflictsOther,
&r.FolderUsesV3.DisableSparseFiles, &r.FolderUsesV3.DisableTempIndexes,
&r.FolderUsesV3.AlwaysWeakHash, &r.FolderUsesV3.CustomWeakHashThreshold,
&r.FolderUsesV3.FsWatcherEnabled,
&r.FolderUsesV3.PullOrder, &r.FolderUsesV3.FilesystemType,
&r.FolderUsesV3.FsWatcherDelays,
&r.GUIStats.Enabled, &r.GUIStats.UseTLS, &r.GUIStats.UseAuth,
&r.GUIStats.InsecureAdminAccess,
&r.GUIStats.Debugging, &r.GUIStats.InsecureSkipHostCheck,
&r.GUIStats.InsecureAllowFrameLoading, &r.GUIStats.ListenLocal,
&r.GUIStats.ListenUnspecified, &r.GUIStats.Theme,
&r.BlockStats.Total, &r.BlockStats.Renamed,
&r.BlockStats.Reused, &r.BlockStats.Pulled, &r.BlockStats.CopyOrigin,
&r.BlockStats.CopyOriginShifted, &r.BlockStats.CopyElsewhere,
&r.TransportStats,
&r.IgnoreStats.Lines, &r.IgnoreStats.Inverts, &r.IgnoreStats.Folded,
&r.IgnoreStats.Deletable, &r.IgnoreStats.Rooted, &r.IgnoreStats.Includes,
&r.IgnoreStats.EscapedIncludes, &r.IgnoreStats.DoubleStars, &r.IgnoreStats.Stars,
// V3 added late in the RC
&r.WeakHashEnabled,
&r.Address,
// Receive only folders
&r.FolderUses.ReceiveOnly,
}
}
func (r *report) FieldNames() []string {
// The database fields that back this struct in PostgreSQL
return []string{
// V1
"Received",
"UniqueID",
"Version",
"LongVersion",
"Platform",
"NumFolders",
"NumDevices",
"TotFiles",
"FolderMaxFiles",
"TotMiB",
"FolderMaxMiB",
"MemoryUsageMiB",
"SHA256Perf",
"MemorySize",
"Date",
// V2
"ReportVersion",
"NumCPU",
"FolderRO",
"FolderIgnorePerms",
"FolderIgnoreDelete",
"FolderAutoNormalize",
"DeviceIntroducer",
"DeviceCustomCertName",
"DeviceCompressAlways",
"DeviceCompressMetadata",
"DeviceCompressNever",
"DeviceDynamicAddr",
"DeviceStaticAddr",
"AnnounceGlobalEnabled",
"AnnounceLocalEnabled",
"AnnounceDefaultServersDNS",
"AnnounceDefaultServersIP",
"AnnounceOtherServers",
"RelayEnabled",
"RelayDefaultServers",
"RelayOtherServers",
"RateLimitEnabled",
"UpgradeAllowedManual",
"UpgradeAllowedAuto",
// v0.12.19+
"FolderSimpleVersioning",
"FolderExternalVersioning",
"FolderStaggeredVersioning",
"FolderTrashcanVersioning",
// V2.5
"UpgradeAllowedPre",
"RescanIntvs",
// V3
"Uptime",
"NATType",
"AlwaysLocalNets",
"CacheIgnoredFiles",
"OverwriteRemoteDeviceNames",
"ProgressEmitterEnabled",
"CustomDefaultFolderPath",
"WeakHashSelection",
"CustomTrafficClass",
"CustomTempIndexMinBlocks",
"TemporariesDisabled",
"TemporariesCustom",
"LimitBandwidthInLan",
"CustomReleaseURL",
"RestartOnWakeup",
"CustomStunServers",
"FolderScanProgressDisabled",
"FolderConflictsDisabled",
"FolderConflictsUnlimited",
"FolderConflictsOther",
"FolderDisableSparseFiles",
"FolderDisableTempIndexes",
"FolderAlwaysWeakHash",
"FolderCustomWeakHashThreshold",
"FolderFsWatcherEnabled",
"FolderPullOrder",
"FolderFilesystemType",
"FolderFsWatcherDelays",
"GUIEnabled",
"GUIUseTLS",
"GUIUseAuth",
"GUIInsecureAdminAccess",
"GUIDebugging",
"GUIInsecureSkipHostCheck",
"GUIInsecureAllowFrameLoading",
"GUIListenLocal",
"GUIListenUnspecified",
"GUITheme",
"BlocksTotal",
"BlocksRenamed",
"BlocksReused",
"BlocksPulled",
"BlocksCopyOrigin",
"BlocksCopyOriginShifted",
"BlocksCopyElsewhere",
"Transport",
"IgnoreLines",
"IgnoreInverts",
"IgnoreFolded",
"IgnoreDeletable",
"IgnoreRooted",
"IgnoreIncludes",
"IgnoreEscapedIncludes",
"IgnoreDoubleStars",
"IgnoreStars",
// V3 added late in the RC
"WeakHashEnabled",
"Address",
// Receive only folders
"FolderRecvOnly",
}
}
func setupDB(db *sql.DB) error {
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS Reports (
Received TIMESTAMP NOT NULL,
@@ -673,8 +302,9 @@ func setupDB(db *sql.DB) error {
return nil
}
func insertReport(db *sql.DB, r report) error {
r.Received = time.Now().UTC()
func insertReport(db *sql.DB, r contract.Report) error {
time := time.Now().UTC()
r.Received = &time
fields := r.FieldPointers()
params := make([]string, len(fields))
for i := range params {
@@ -861,7 +491,7 @@ func newDataHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
addr = ""
}
var rep report
var rep contract.Report
rep.Date = time.Now().UTC().Format("20060102")
rep.Address = addr
@@ -1069,11 +699,11 @@ func getReport(db *sql.DB) map[string]interface{} {
var numDevices []int
var totFiles []int
var maxFiles []int
var totMiB []int
var maxMiB []int
var memoryUsage []int
var totMiB []int64
var maxMiB []int64
var memoryUsage []int64
var sha256Perf []float64
var memorySize []int
var memorySize []int64
var uptime []int
var compilers []string
var builders []string
@@ -1112,7 +742,7 @@ func getReport(db *sql.DB) map[string]interface{} {
var numCPU []int
var rep report
var rep contract.Report
rows, err := db.Query(`SELECT ` + strings.Join(rep.FieldNames(), ",") + ` FROM Reports WHERE Received > now() - '1 day'::INTERVAL`)
if err != nil {
@@ -1173,19 +803,19 @@ func getReport(db *sql.DB) map[string]interface{} {
maxFiles = append(maxFiles, rep.FolderMaxFiles)
}
if rep.TotMiB > 0 {
totMiB = append(totMiB, rep.TotMiB*(1<<20))
totMiB = append(totMiB, int64(rep.TotMiB)*(1<<20))
}
if rep.FolderMaxMiB > 0 {
maxMiB = append(maxMiB, rep.FolderMaxMiB*(1<<20))
maxMiB = append(maxMiB, int64(rep.FolderMaxMiB)*(1<<20))
}
if rep.MemoryUsageMiB > 0 {
memoryUsage = append(memoryUsage, rep.MemoryUsageMiB*(1<<20))
memoryUsage = append(memoryUsage, int64(rep.MemoryUsageMiB)*(1<<20))
}
if rep.SHA256Perf > 0 {
sha256Perf = append(sha256Perf, rep.SHA256Perf*(1<<20))
}
if rep.MemorySize > 0 {
memorySize = append(memorySize, rep.MemorySize*(1<<20))
memorySize = append(memorySize, int64(rep.MemorySize)*(1<<20))
}
if rep.Uptime > 0 {
uptime = append(uptime, rep.Uptime)
@@ -1336,14 +966,14 @@ func getReport(db *sql.DB) map[string]interface{} {
})
categories = append(categories, category{
Values: statsForInts(totMiB),
Values: statsForInt64s(totMiB),
Descr: "Data Managed per Device",
Unit: "B",
Type: NumberBinary,
})
categories = append(categories, category{
Values: statsForInts(maxMiB),
Values: statsForInt64s(maxMiB),
Descr: "Data in Largest Folder",
Unit: "B",
Type: NumberBinary,
@@ -1360,14 +990,14 @@ func getReport(db *sql.DB) map[string]interface{} {
})
categories = append(categories, category{
Values: statsForInts(memoryUsage),
Values: statsForInt64s(memoryUsage),
Descr: "Memory Usage",
Unit: "B",
Type: NumberBinary,
})
categories = append(categories, category{
Values: statsForInts(memorySize),
Values: statsForInt64s(memorySize),
Descr: "System Memory",
Unit: "B",
Type: NumberBinary,

View File

@@ -32,8 +32,6 @@ angular.module('syncthing.core')
$scope.protocolChanged = false;
$scope.reportData = {};
$scope.reportDataPreview = '';
$scope.reportDataPreviewVersion = '';
$scope.reportDataPreviewDiff = false;
$scope.reportPreview = false;
$scope.folders = {};
$scope.seenError = '';
@@ -2322,13 +2320,13 @@ angular.module('syncthing.core')
$scope.reportPreview = true;
};
$scope.refreshReportDataPreview = function () {
$scope.refreshReportDataPreview = function (ver, diff) {
$scope.reportDataPreview = '';
if (!$scope.reportDataPreviewVersion) {
if (!ver) {
return;
}
var version = parseInt($scope.reportDataPreviewVersion);
if ($scope.reportDataPreviewDiff && version > 2) {
var version = parseInt(ver);
if (diff && version > 2) {
$q.all([
$http.get(urlbase + '/svc/report?version=' + version),
$http.get(urlbase + '/svc/report?version=' + (version - 1)),

View File

@@ -6,13 +6,13 @@
<p translate>The aggregated statistics are publicly available at the URL below.</p>
<p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p>
<label translate>Version</label>
<select id="urPreviewVersion" class="form-control" ng-model="$parent.$parent.reportDataPreviewVersion" ng-change="refreshReportDataPreview()">
<select id="urPreviewVersion" class="form-control" ng-model="reportDataPreviewVersion" ng-change="refreshReportDataPreview(reportDataPreviewVersion, reportDataPreviewDiff)">
<option selected value translate>Select a version</option>
<option ng-repeat="n in urVersions()" value="{{n}}">{{'Version' | translate}} {{n}}</option>
</select>
<div class="checkbox" ng-if="$parent.$parent.reportDataPreviewVersion > 2">
<div class="checkbox" ng-if="reportDataPreviewVersion > 2">
<label>
<input type="checkbox" ng-model="$parent.$parent.$parent.reportDataPreviewDiff" ng-change="refreshReportDataPreview()" />
<input type="checkbox" ng-model="reportDataPreviewDiff" ng-change="refreshReportDataPreview(reportDataPreviewVersion, reportDataPreviewDiff)" />
<span translate>Show diff with previous version</span>
</label>
</div>

View File

@@ -1059,10 +1059,15 @@ func (s *service) getSupportBundle(w http.ResponseWriter, r *http.Request) {
}
// Report Data as a JSON
if usageReportingData, err := json.MarshalIndent(s.urService.ReportData(context.TODO()), "", " "); err != nil {
l.Warnln("Support bundle: failed to create versionPlatform.json:", err)
if r, err := s.urService.ReportData(context.TODO()); err != nil {
l.Warnln("Support bundle: failed to create usage-reporting.json.txt:", err)
} else {
files = append(files, fileEntry{name: "usage-reporting.json.txt", data: usageReportingData})
if usageReportingData, err := json.MarshalIndent(r, "", " "); err != nil {
l.Warnln("Support bundle: failed to serialize usage-reporting.json.txt", err)
} else {
files = append(files, fileEntry{name: "usage-reporting.json.txt", data: usageReportingData})
}
}
// Heap and CPU Proofs as a pprof extension
@@ -1144,7 +1149,13 @@ func (s *service) getReport(w http.ResponseWriter, r *http.Request) {
if val, _ := strconv.Atoi(r.URL.Query().Get("version")); val > 0 {
version = val
}
sendJSON(w, s.urService.ReportDataPreview(context.TODO(), version))
if r, err := s.urService.ReportDataPreview(context.TODO(), version); err != nil {
http.Error(w, err.Error(), 500)
return
} else {
sendJSON(w, r)
}
}
func (s *service) getRandomString(w http.ResponseWriter, r *http.Request) {

View File

@@ -183,7 +183,7 @@ func migrateToConfigV21(cfg *Configuration) {
}
switch folder.Versioning.Type {
case "simple", "trashcan":
// Clean out symlinks in the known place
// ClearForVersion out symlinks in the known place
cleanSymlinks(folder.Filesystem(), ".stversions")
case "staggered":
versionDir := folder.Versioning.Params["versionsPath"]

View File

@@ -505,7 +505,7 @@ func TestCheckGlobals(t *testing.T) {
t.Fatal(err)
}
// Clean up global entry of the now missing file
// ClearForVersion up global entry of the now missing file
if err := db.checkGlobals([]byte(fs.folder)); err != nil {
t.Fatal(err)
}

View File

@@ -386,7 +386,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
// and it's ok to release twice.
defer snap.Release()
// Clean the list of subitems to ensure that we start at a known
// ClearForVersion the list of subitems to ensure that we start at a known
// directory, and don't scan subdirectories of things we've already
// scanned.
subDirs = unifySubs(subDirs, func(file string) bool {

View File

@@ -34,6 +34,7 @@ import (
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/stats"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/ur/contract"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/versioner"
)
@@ -101,7 +102,7 @@ type Model interface {
ConnectionStats() map[string]interface{}
DeviceStatistics() (map[string]stats.DeviceStatistics, error)
FolderStatistics() (map[string]stats.FolderStatistics, error)
UsageReportingStats(version int, preview bool) map[string]interface{}
UsageReportingStats(report *contract.Report, version int, preview bool)
StartDeadlockDetector(timeout time.Duration)
GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{}
@@ -443,7 +444,7 @@ func (m *model) stopFolder(cfg config.FolderConfiguration, err error) {
// Need to hold lock on m.fmut when calling this.
func (m *model) cleanupFolderLocked(cfg config.FolderConfiguration) {
// Clean up our config maps
// clear up our config maps
delete(m.folderCfgs, cfg.ID)
delete(m.folderFiles, cfg.ID)
delete(m.folderIgnores, cfg.ID)
@@ -517,49 +518,49 @@ func (m *model) newFolder(cfg config.FolderConfiguration) {
m.addAndStartFolderLocked(cfg, fset)
}
func (m *model) UsageReportingStats(version int, preview bool) map[string]interface{} {
stats := make(map[string]interface{})
func (m *model) UsageReportingStats(report *contract.Report, version int, preview bool) {
if version >= 3 {
// Block stats
blockStatsMut.Lock()
copyBlockStats := make(map[string]int)
for k, v := range blockStats {
copyBlockStats[k] = v
switch k {
case "total":
report.BlockStats.Total = v
case "renamed":
report.BlockStats.Renamed = v
case "reused":
report.BlockStats.Reused = v
case "pulled":
report.BlockStats.Pulled = v
case "copyOrigin":
report.BlockStats.CopyOrigin = v
case "copyOriginShifted":
report.BlockStats.CopyOriginShifted = v
case "copyElsewhere":
report.BlockStats.CopyElsewhere = v
}
// Reset counts, as these are incremental
if !preview {
blockStats[k] = 0
}
}
blockStatsMut.Unlock()
stats["blockStats"] = copyBlockStats
// Transport stats
m.pmut.RLock()
transportStats := make(map[string]int)
for _, conn := range m.conn {
transportStats[conn.Transport()]++
report.TransportStats[conn.Transport()]++
}
m.pmut.RUnlock()
stats["transportStats"] = transportStats
// Ignore stats
ignoreStats := map[string]int{
"lines": 0,
"inverts": 0,
"folded": 0,
"deletable": 0,
"rooted": 0,
"includes": 0,
"escapedIncludes": 0,
"doubleStars": 0,
"stars": 0,
}
var seenPrefix [3]bool
for folder := range m.cfg.Folders() {
lines, _, err := m.GetIgnores(folder)
if err != nil {
continue
}
ignoreStats["lines"] += len(lines)
report.IgnoreStats.Lines += len(lines)
for _, line := range lines {
// Allow prefixes to be specified in any order, but only once.
@@ -567,15 +568,15 @@ func (m *model) UsageReportingStats(version int, preview bool) map[string]interf
if strings.HasPrefix(line, "!") && !seenPrefix[0] {
seenPrefix[0] = true
line = line[1:]
ignoreStats["inverts"] += 1
report.IgnoreStats.Inverts++
} else if strings.HasPrefix(line, "(?i)") && !seenPrefix[1] {
seenPrefix[1] = true
line = line[4:]
ignoreStats["folded"] += 1
report.IgnoreStats.Folded++
} else if strings.HasPrefix(line, "(?d)") && !seenPrefix[2] {
seenPrefix[2] = true
line = line[4:]
ignoreStats["deletable"] += 1
report.IgnoreStats.Deletable++
} else {
seenPrefix[0] = false
seenPrefix[1] = false
@@ -589,28 +590,26 @@ func (m *model) UsageReportingStats(version int, preview bool) map[string]interf
line = strings.TrimPrefix(line, "**/")
if strings.HasPrefix(line, "/") {
ignoreStats["rooted"] += 1
report.IgnoreStats.Rooted++
} else if strings.HasPrefix(line, "#include ") {
ignoreStats["includes"] += 1
report.IgnoreStats.Includes++
if strings.Contains(line, "..") {
ignoreStats["escapedIncludes"] += 1
report.IgnoreStats.EscapedIncludes++
}
}
if strings.Contains(line, "**") {
ignoreStats["doubleStars"] += 1
report.IgnoreStats.DoubleStars++
// Remove not to trip up star checks.
line = strings.Replace(line, "**", "", -1)
}
if strings.Contains(line, "*") {
ignoreStats["stars"] += 1
report.IgnoreStats.Stars++
}
}
}
stats["ignoreStats"] = ignoreStats
}
return stats
}
type ConnectionInfo struct {

View File

@@ -179,7 +179,7 @@ func (t *ProgressEmitter) computeProgressUpdates() []progressUpdate {
}
}
// Clean up sentDownloadStates for devices which we are no longer connected to.
// ClearForVersion up sentDownloadStates for devices which we are no longer connected to.
for id := range t.sentDownloadStates {
_, ok := t.connections[id]
if !ok {

View File

@@ -415,8 +415,8 @@ func TestSendDownloadProgressMessages(t *testing.T) {
expectEmpty()
// Device is no longer subscribed to a particular folder
delete(p.registry["folder"], "1") // Clean up first
delete(p.registry["folder2"], "2") // Clean up first
delete(p.registry["folder"], "1") // ClearForVersion up first
delete(p.registry["folder2"], "2") // ClearForVersion up first
sendMsgs(p)
expect(-1, state1, protocol.UpdateTypeForget, v1, nil, true)

View File

@@ -79,7 +79,7 @@ func (s *sentFolderDownloadState) update(pullers []*sharedPullerState) []protoco
if !pullerVersion.Equal(localFile.version) || !pullerCreated.Equal(localFile.created) {
// The version has changed or the puller was reconstrcuted due to failure.
// Clean up whatever we had for the old file, and advertise the new file.
// ClearForVersion up whatever we had for the old file, and advertise the new file.
updates = append(updates, protocol.FileDownloadProgressUpdate{
Name: name,
Version: localFile.version,

View File

@@ -602,7 +602,7 @@ func checkFilename(name string) error {
cleanedName := path.Clean(name)
if cleanedName != name {
// The filename on the wire should be in canonical format. If
// Clean() managed to clean it up, there was something wrong with
// ClearForVersion() managed to clean it up, there was something wrong with
// it.
return errUncleanFilename
}
@@ -618,7 +618,7 @@ func checkFilename(name string) error {
}
if strings.HasPrefix(name, "../") {
// Starting with a dotdot is not allowed. Any other dotdots would
// have been handled by the Clean() call at the top.
// have been handled by the ClearForVersion() call at the top.
return errInvalidFilename
}
return nil

453
lib/ur/contract/contract.go Normal file
View File

@@ -0,0 +1,453 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package contract
import (
"database/sql/driver"
"encoding/json"
"errors"
"reflect"
"sort"
"strconv"
"time"
"github.com/lib/pq"
)
type IntMap map[string]int
func (p IntMap) Value() (driver.Value, error) {
return json.Marshal(p)
}
func (p *IntMap) Scan(src interface{}) error {
source, ok := src.([]byte)
if !ok {
return errors.New("Type assertion .([]byte) failed.")
}
var i map[string]int
err := json.Unmarshal(source, &i)
if err != nil {
return err
}
*p = i
return nil
}
type Report struct {
// Generated
Received *time.Time `json:"received,omitempty"` // Only from DB
Date string `json:"date,omitempty"`
Address string `json:"address,omitempty"`
// v1 fields
UniqueID string `json:"uniqueID,omitempty" since:"1"`
Version string `json:"version,omitempty" since:"1"`
LongVersion string `json:"longVersion,omitempty" since:"1"`
Platform string `json:"platform,omitempty" since:"1"`
NumFolders int `json:"numFolders,omitempty" since:"1"`
NumDevices int `json:"numDevices,omitempty" since:"1"`
TotFiles int `json:"totFiles,omitempty" since:"1"`
FolderMaxFiles int `json:"folderMaxFiles,omitempty" since:"1"`
TotMiB int `json:"totMiB,omitempty" since:"1"`
FolderMaxMiB int `json:"folderMaxMiB,omitempty" since:"1"`
MemoryUsageMiB int `json:"memoryUsageMiB,omitempty" since:"1"`
SHA256Perf float64 `json:"sha256Perf,omitempty" since:"1"`
HashPerf float64 `json:"hashPerf,omitempty" since:"1"` // Was previously not stored server-side
MemorySize int `json:"memorySize,omitempty" since:"1"`
// v2 fields
URVersion int `json:"urVersion,omitempty" since:"2"`
NumCPU int `json:"numCPU,omitempty" since:"2"`
FolderUses struct {
SendOnly int `json:"sendonly,omitempty" since:"2"`
SendReceive int `json:"sendreceive,omitempty" since:"2"` // Was previously not stored server-side
ReceiveOnly int `json:"receiveonly,omitempty" since:"2"`
IgnorePerms int `json:"ignorePerms,omitempty" since:"2"`
IgnoreDelete int `json:"ignoreDelete,omitempty" since:"2"`
AutoNormalize int `json:"autoNormalize,omitempty" since:"2"`
SimpleVersioning int `json:"simpleVersioning,omitempty" since:"2"`
ExternalVersioning int `json:"externalVersioning,omitempty" since:"2"`
StaggeredVersioning int `json:"staggeredVersioning,omitempty" since:"2"`
TrashcanVersioning int `json:"trashcanVersioning,omitempty" since:"2"`
} `json:"folderUses,omitempty" since:"2"`
DeviceUses struct {
Introducer int `json:"introducer,omitempty" since:"2"`
CustomCertName int `json:"customCertName,omitempty" since:"2"`
CompressAlways int `json:"compressAlways,omitempty" since:"2"`
CompressMetadata int `json:"compressMetadata,omitempty" since:"2"`
CompressNever int `json:"compressNever,omitempty" since:"2"`
DynamicAddr int `json:"dynamicAddr,omitempty" since:"2"`
StaticAddr int `json:"staticAddr,omitempty" since:"2"`
} `json:"deviceUses,omitempty" since:"2"`
Announce struct {
GlobalEnabled bool `json:"globalEnabled,omitempty" since:"2"`
LocalEnabled bool `json:"localEnabled,omitempty" since:"2"`
DefaultServersDNS int `json:"defaultServersDNS,omitempty" since:"2"`
DefaultServersIP int `json:"defaultServersIP,omitempty" since:"2"` // Deprecated and not provided client-side anymore
OtherServers int `json:"otherServers,omitempty" since:"2"`
} `json:"announce,omitempty" since:"2"`
Relays struct {
Enabled bool `json:"enabled,omitempty" since:"2"`
DefaultServers int `json:"defaultServers,omitempty" since:"2"`
OtherServers int `json:"otherServers,omitempty" since:"2"`
} `json:"relays,omitempty" since:"2"`
UsesRateLimit bool `json:"usesRateLimit,omitempty" since:"2"`
UpgradeAllowedManual bool `json:"upgradeAllowedManual,omitempty" since:"2"`
UpgradeAllowedAuto bool `json:"upgradeAllowedAuto,omitempty" since:"2"`
// V2.5 fields (fields that were in v2 but never added to the database
UpgradeAllowedPre bool `json:"upgradeAllowedPre,omitempty" since:"2"`
RescanIntvs pq.Int64Array `json:"rescanIntvs,omitempty" since:"2"`
// v3 fields
Uptime int `json:"uptime,omitempty" since:"3"`
NATType string `json:"natType,omitempty" since:"3"`
AlwaysLocalNets bool `json:"alwaysLocalNets,omitempty" since:"3"`
CacheIgnoredFiles bool `json:"cacheIgnoredFiles,omitempty" since:"3"`
OverwriteRemoteDeviceNames bool `json:"overwriteRemoteDeviceNames,omitempty" since:"3"`
ProgressEmitterEnabled bool `json:"progressEmitterEnabled,omitempty" since:"3"`
CustomDefaultFolderPath bool `json:"customDefaultFolderPath,omitempty" since:"3"`
WeakHashSelection string `json:"weakHashSelection,omitempty" since:"3"` // Deprecated and not provided client-side anymore
CustomTrafficClass bool `json:"customTrafficClass,omitempty" since:"3"`
CustomTempIndexMinBlocks bool `json:"customTempIndexMinBlocks,omitempty" since:"3"`
TemporariesDisabled bool `json:"temporariesDisabled,omitempty" since:"3"`
TemporariesCustom bool `json:"temporariesCustom,omitempty" since:"3"`
LimitBandwidthInLan bool `json:"limitBandwidthInLan,omitempty" since:"3"`
CustomReleaseURL bool `json:"customReleaseURL,omitempty" since:"3"`
RestartOnWakeup bool `json:"restartOnWakeup,omitempty" since:"3"`
CustomStunServers bool `json:"customStunServers,omitempty" since:"3"`
FolderUsesV3 struct {
ScanProgressDisabled int `json:"scanProgressDisabled,omitempty" since:"3"`
ConflictsDisabled int `json:"conflictsDisabled,omitempty" since:"3"`
ConflictsUnlimited int `json:"conflictsUnlimited,omitempty" since:"3"`
ConflictsOther int `json:"conflictsOther,omitempty" since:"3"`
DisableSparseFiles int `json:"disableSparseFiles,omitempty" since:"3"`
DisableTempIndexes int `json:"disableTempIndexes,omitempty" since:"3"`
AlwaysWeakHash int `json:"alwaysWeakHash,omitempty" since:"3"`
CustomWeakHashThreshold int `json:"customWeakHashThreshold,omitempty" since:"3"`
FsWatcherEnabled int `json:"fsWatcherEnabled,omitempty" since:"3"`
PullOrder IntMap `json:"pullOrder,omitempty" since:"3"`
FilesystemType IntMap `json:"filesystemType,omitempty" since:"3"`
FsWatcherDelays pq.Int64Array `json:"fsWatcherDelays,omitempty" since:"3"`
} `json:"folderUsesV3,omitempty" since:"3"`
GUIStats struct {
Enabled int `json:"enabled,omitempty" since:"3"`
UseTLS int `json:"useTLS,omitempty" since:"3"`
UseAuth int `json:"useAuth,omitempty" since:"3"`
InsecureAdminAccess int `json:"insecureAdminAccess,omitempty" since:"3"`
Debugging int `json:"debugging,omitempty" since:"3"`
InsecureSkipHostCheck int `json:"insecureSkipHostCheck,omitempty" since:"3"`
InsecureAllowFrameLoading int `json:"insecureAllowFrameLoading,omitempty" since:"3"`
ListenLocal int `json:"listenLocal,omitempty" since:"3"`
ListenUnspecified int `json:"listenUnspecified,omitempty" since:"3"`
Theme IntMap `json:"theme,omitempty" since:"3"`
} `json:"guiStats,omitempty" since:"3"`
BlockStats struct {
Total int `json:"total,omitempty" since:"3"`
Renamed int `json:"renamed,omitempty" since:"3"`
Reused int `json:"reused,omitempty" since:"3"`
Pulled int `json:"pulled,omitempty" since:"3"`
CopyOrigin int `json:"copyOrigin,omitempty" since:"3"`
CopyOriginShifted int `json:"copyOriginShifted,omitempty" since:"3"`
CopyElsewhere int `json:"copyElsewhere,omitempty" since:"3"`
} `json:"blockStats,omitempty" since:"3"`
TransportStats IntMap `json:"transportStats,omitempty" since:"3"`
IgnoreStats struct {
Lines int `json:"lines,omitempty" since:"3"`
Inverts int `json:"inverts,omitempty" since:"3"`
Folded int `json:"folded,omitempty" since:"3"`
Deletable int `json:"deletable,omitempty" since:"3"`
Rooted int `json:"rooted,omitempty" since:"3"`
Includes int `json:"includes,omitempty" since:"3"`
EscapedIncludes int `json:"escapedIncludes,omitempty" since:"3"`
DoubleStars int `json:"doubleStars,omitempty" since:"3"`
Stars int `json:"stars,omitempty" since:"3"`
} `json:"ignoreStats,omitempty" since:"3"`
// V3 fields added late in the RC
WeakHashEnabled bool `json:"weakHashEnabled,omitempty" since:"3"` // Deprecated and not provided client-side anymore
}
func New() *Report {
r := &Report{}
r.FolderUsesV3.PullOrder = make(IntMap)
r.FolderUsesV3.FilesystemType = make(IntMap)
r.GUIStats.Theme = make(IntMap)
r.TransportStats = make(IntMap)
r.RescanIntvs = make(pq.Int64Array, 0)
r.FolderUsesV3.FsWatcherDelays = make(pq.Int64Array, 0)
return r
}
func (r *Report) Validate() error {
if r.UniqueID == "" || r.Version == "" || r.Platform == "" {
return errors.New("missing required field")
}
if len(r.Date) != 8 {
return errors.New("date not initialized")
}
// Some fields may not be null.
if r.RescanIntvs == nil {
r.RescanIntvs = []int64{}
}
if r.FolderUsesV3.FsWatcherDelays == nil {
r.FolderUsesV3.FsWatcherDelays = []int64{}
}
return nil
}
func (r *Report) ClearForVersion(version int) error {
return clear(r, version)
}
func (r *Report) FieldPointers() []interface{} {
// All the fields of the Report, in the same order as the database fields.
return []interface{}{
&r.Received, &r.UniqueID, &r.Version, &r.LongVersion, &r.Platform,
&r.NumFolders, &r.NumDevices, &r.TotFiles, &r.FolderMaxFiles,
&r.TotMiB, &r.FolderMaxMiB, &r.MemoryUsageMiB, &r.SHA256Perf,
&r.MemorySize, &r.Date,
// V2
&r.URVersion, &r.NumCPU, &r.FolderUses.SendOnly, &r.FolderUses.IgnorePerms,
&r.FolderUses.IgnoreDelete, &r.FolderUses.AutoNormalize, &r.DeviceUses.Introducer,
&r.DeviceUses.CustomCertName, &r.DeviceUses.CompressAlways,
&r.DeviceUses.CompressMetadata, &r.DeviceUses.CompressNever,
&r.DeviceUses.DynamicAddr, &r.DeviceUses.StaticAddr,
&r.Announce.GlobalEnabled, &r.Announce.LocalEnabled,
&r.Announce.DefaultServersDNS, &r.Announce.DefaultServersIP,
&r.Announce.OtherServers, &r.Relays.Enabled, &r.Relays.DefaultServers,
&r.Relays.OtherServers, &r.UsesRateLimit, &r.UpgradeAllowedManual,
&r.UpgradeAllowedAuto, &r.FolderUses.SimpleVersioning,
&r.FolderUses.ExternalVersioning, &r.FolderUses.StaggeredVersioning,
&r.FolderUses.TrashcanVersioning,
// V2.5
&r.UpgradeAllowedPre, &r.RescanIntvs,
// V3
&r.Uptime, &r.NATType, &r.AlwaysLocalNets, &r.CacheIgnoredFiles,
&r.OverwriteRemoteDeviceNames, &r.ProgressEmitterEnabled, &r.CustomDefaultFolderPath,
&r.WeakHashSelection, &r.CustomTrafficClass, &r.CustomTempIndexMinBlocks,
&r.TemporariesDisabled, &r.TemporariesCustom, &r.LimitBandwidthInLan,
&r.CustomReleaseURL, &r.RestartOnWakeup, &r.CustomStunServers,
&r.FolderUsesV3.ScanProgressDisabled, &r.FolderUsesV3.ConflictsDisabled,
&r.FolderUsesV3.ConflictsUnlimited, &r.FolderUsesV3.ConflictsOther,
&r.FolderUsesV3.DisableSparseFiles, &r.FolderUsesV3.DisableTempIndexes,
&r.FolderUsesV3.AlwaysWeakHash, &r.FolderUsesV3.CustomWeakHashThreshold,
&r.FolderUsesV3.FsWatcherEnabled,
&r.FolderUsesV3.PullOrder, &r.FolderUsesV3.FilesystemType,
&r.FolderUsesV3.FsWatcherDelays,
&r.GUIStats.Enabled, &r.GUIStats.UseTLS, &r.GUIStats.UseAuth,
&r.GUIStats.InsecureAdminAccess,
&r.GUIStats.Debugging, &r.GUIStats.InsecureSkipHostCheck,
&r.GUIStats.InsecureAllowFrameLoading, &r.GUIStats.ListenLocal,
&r.GUIStats.ListenUnspecified, &r.GUIStats.Theme,
&r.BlockStats.Total, &r.BlockStats.Renamed,
&r.BlockStats.Reused, &r.BlockStats.Pulled, &r.BlockStats.CopyOrigin,
&r.BlockStats.CopyOriginShifted, &r.BlockStats.CopyElsewhere,
&r.TransportStats,
&r.IgnoreStats.Lines, &r.IgnoreStats.Inverts, &r.IgnoreStats.Folded,
&r.IgnoreStats.Deletable, &r.IgnoreStats.Rooted, &r.IgnoreStats.Includes,
&r.IgnoreStats.EscapedIncludes, &r.IgnoreStats.DoubleStars, &r.IgnoreStats.Stars,
// V3 added late in the RC
&r.WeakHashEnabled,
&r.Address,
// Receive only folders
&r.FolderUses.ReceiveOnly,
}
}
func (r *Report) FieldNames() []string {
// The database fields that back this struct in PostgreSQL
return []string{
// V1
"Received",
"UniqueID",
"Version",
"LongVersion",
"Platform",
"NumFolders",
"NumDevices",
"TotFiles",
"FolderMaxFiles",
"TotMiB",
"FolderMaxMiB",
"MemoryUsageMiB",
"SHA256Perf",
"MemorySize",
"Date",
// V2
"ReportVersion",
"NumCPU",
"FolderRO",
"FolderIgnorePerms",
"FolderIgnoreDelete",
"FolderAutoNormalize",
"DeviceIntroducer",
"DeviceCustomCertName",
"DeviceCompressAlways",
"DeviceCompressMetadata",
"DeviceCompressNever",
"DeviceDynamicAddr",
"DeviceStaticAddr",
"AnnounceGlobalEnabled",
"AnnounceLocalEnabled",
"AnnounceDefaultServersDNS",
"AnnounceDefaultServersIP",
"AnnounceOtherServers",
"RelayEnabled",
"RelayDefaultServers",
"RelayOtherServers",
"RateLimitEnabled",
"UpgradeAllowedManual",
"UpgradeAllowedAuto",
// v0.12.19+
"FolderSimpleVersioning",
"FolderExternalVersioning",
"FolderStaggeredVersioning",
"FolderTrashcanVersioning",
// V2.5
"UpgradeAllowedPre",
"RescanIntvs",
// V3
"Uptime",
"NATType",
"AlwaysLocalNets",
"CacheIgnoredFiles",
"OverwriteRemoteDeviceNames",
"ProgressEmitterEnabled",
"CustomDefaultFolderPath",
"WeakHashSelection",
"CustomTrafficClass",
"CustomTempIndexMinBlocks",
"TemporariesDisabled",
"TemporariesCustom",
"LimitBandwidthInLan",
"CustomReleaseURL",
"RestartOnWakeup",
"CustomStunServers",
"FolderScanProgressDisabled",
"FolderConflictsDisabled",
"FolderConflictsUnlimited",
"FolderConflictsOther",
"FolderDisableSparseFiles",
"FolderDisableTempIndexes",
"FolderAlwaysWeakHash",
"FolderCustomWeakHashThreshold",
"FolderFsWatcherEnabled",
"FolderPullOrder",
"FolderFilesystemType",
"FolderFsWatcherDelays",
"GUIEnabled",
"GUIUseTLS",
"GUIUseAuth",
"GUIInsecureAdminAccess",
"GUIDebugging",
"GUIInsecureSkipHostCheck",
"GUIInsecureAllowFrameLoading",
"GUIListenLocal",
"GUIListenUnspecified",
"GUITheme",
"BlocksTotal",
"BlocksRenamed",
"BlocksReused",
"BlocksPulled",
"BlocksCopyOrigin",
"BlocksCopyOriginShifted",
"BlocksCopyElsewhere",
"Transport",
"IgnoreLines",
"IgnoreInverts",
"IgnoreFolded",
"IgnoreDeletable",
"IgnoreRooted",
"IgnoreIncludes",
"IgnoreEscapedIncludes",
"IgnoreDoubleStars",
"IgnoreStars",
// V3 added late in the RC
"WeakHashEnabled",
"Address",
// Receive only folders
"FolderRecvOnly",
}
}
func clear(v interface{}, since int) error {
s := reflect.ValueOf(v).Elem()
t := s.Type()
for i := 0; i < s.NumField(); i++ {
f := s.Field(i)
tag := t.Field(i).Tag
v := tag.Get("since")
if len(v) == 0 {
f.Set(reflect.Zero(f.Type()))
continue
}
vn, err := strconv.Atoi(v)
if err != nil {
return err
}
if vn > since {
f.Set(reflect.Zero(f.Type()))
continue
}
// Dive deeper
if f.Kind() == reflect.Ptr {
f = f.Elem()
}
if f.Kind() == reflect.Struct {
if err := clear(f.Addr().Interface(), since); err != nil {
return err
}
}
}
return nil
}
func SortPqInt64Array(slice pq.Int64Array) {
sort.Slice(slice, func(a, b int) bool {
return slice[a] < slice[b]
})
}

View File

@@ -0,0 +1,130 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package contract
import (
"reflect"
"testing"
)
type PtrStruct struct {
A string `since:"2"`
B IntMap `since:"3"`
}
type Nested struct {
A float32 `since:"4"`
B [4]int `since:"5"`
C bool `since:"1"`
}
type TestStruct struct {
A int
B map[string]string `since:"1"`
C []string `since:"2"`
Nested Nested `since:"3"`
Ptr *PtrStruct `since:"2"`
}
func testValue() TestStruct {
return TestStruct{
A: 1,
B: map[string]string{
"foo": "bar",
},
C: []string{"a", "b"},
Nested: Nested{
A: 0.10,
B: [4]int{1, 2, 3, 4},
C: true,
},
Ptr: &PtrStruct{
A: "value",
B: map[string]int{
"x": 1,
"b": 2,
},
},
}
}
func TestClean(t *testing.T) {
expect(t, 0, TestStruct{})
expect(t, 1, TestStruct{
// A unset, since it does not have "since"
B: map[string]string{
"foo": "bar",
},
})
expect(t, 2, TestStruct{
// A unset, since it does not have "since"
B: map[string]string{
"foo": "bar",
},
C: []string{"a", "b"},
Ptr: &PtrStruct{
A: "value",
},
})
expect(t, 3, TestStruct{
// A unset, since it does not have "since"
B: map[string]string{
"foo": "bar",
},
C: []string{"a", "b"},
Nested: Nested{
C: true,
},
Ptr: &PtrStruct{
A: "value",
B: map[string]int{
"x": 1,
"b": 2,
},
},
})
expect(t, 4, TestStruct{
// A unset, since it does not have "since"
B: map[string]string{
"foo": "bar",
},
C: []string{"a", "b"},
Nested: Nested{
A: 0.10,
C: true,
},
Ptr: &PtrStruct{
A: "value",
B: map[string]int{
"x": 1,
"b": 2,
},
},
})
x := testValue()
x.A = 0
expect(t, 5, x)
expect(t, 6, x)
}
func expect(t *testing.T, since int, b interface{}) {
t.Helper()
x := testValue()
if err := clear(&x, since); err != nil {
t.Fatal(err.Error())
}
if !reflect.DeepEqual(x, b) {
t.Errorf("%#v != %#v", x, b)
}
}

View File

@@ -8,7 +8,10 @@ package ur
import "golang.org/x/sys/unix"
func memorySize() (int64, error) {
func memorySize() int64 {
mem, err := unix.SysctlUint64("hw.memsize")
return int64(mem), err
if err != nil {
return 0
}
return mem
}

View File

@@ -8,32 +8,31 @@ package ur
import (
"bufio"
"errors"
"os"
"strconv"
"strings"
)
func memorySize() (int64, error) {
func memorySize() int64 {
f, err := os.Open("/proc/meminfo")
if err != nil {
return 0, err
return 0
}
s := bufio.NewScanner(f)
if !s.Scan() {
return 0, errors.New("/proc/meminfo parse error 1")
return 0
}
l := s.Text()
fs := strings.Fields(l)
if len(fs) != 3 || fs[2] != "kB" {
return 0, errors.New("/proc/meminfo parse error 2")
return 0
}
kb, err := strconv.ParseInt(fs[1], 10, 64)
if err != nil {
return 0, err
return 0
}
return kb * 1024, nil
return kb * 1024
}

View File

@@ -7,25 +7,24 @@
package ur
import (
"errors"
"os/exec"
"strconv"
"strings"
)
func memorySize() (int64, error) {
func memorySize() int64 {
cmd := exec.Command("/sbin/sysctl", "hw.physmem64")
out, err := cmd.Output()
if err != nil {
return 0, err
return 0
}
fs := strings.Fields(string(out))
if len(fs) != 3 {
return 0, errors.New("sysctl parse error")
return 0
}
bytes, err := strconv.ParseInt(fs[2], 10, 64)
if err != nil {
return 0, err
return 0
}
return bytes, nil
return bytes
}

View File

@@ -13,16 +13,16 @@ import (
"strconv"
)
func memorySize() (int64, error) {
func memorySize() int64 {
cmd := exec.Command("prtconf", "-m")
out, err := cmd.CombinedOutput()
if err != nil {
return 0, err
return 0
}
mb, err := strconv.ParseInt(string(out), 10, 64)
if err != nil {
return 0, err
return 0
}
return mb * 1024 * 1024, nil
return mb * 1024 * 1024
}

View File

@@ -8,8 +8,6 @@
package ur
import "errors"
func memorySize() (int64, error) {
return 0, errors.New("not implemented")
func memorySize() int64 {
return 0
}

View File

@@ -17,14 +17,14 @@ var (
globalMemoryStatusEx, _ = syscall.GetProcAddress(kernel32, "GlobalMemoryStatusEx")
)
func memorySize() (int64, error) {
func memorySize() int64 {
var memoryStatusEx [64]byte
binary.LittleEndian.PutUint32(memoryStatusEx[:], 64)
ret, _, callErr := syscall.Syscall(uintptr(globalMemoryStatusEx), 1, uintptr(unsafe.Pointer(&memoryStatusEx[0])), 0, 0)
ret, _, _ := syscall.Syscall(uintptr(globalMemoryStatusEx), 1, uintptr(unsafe.Pointer(&memoryStatusEx[0])), 0, 0)
if ret == 0 {
return 0, callErr
return 0
}
return int64(binary.LittleEndian.Uint64(memoryStatusEx[8:])), nil
return int64(binary.LittleEndian.Uint64(memoryStatusEx[8:]))
}

View File

@@ -15,7 +15,6 @@ import (
"net"
"net/http"
"runtime"
"sort"
"strings"
"sync"
"time"
@@ -28,6 +27,7 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/syncthing/syncthing/lib/ur/contract"
"github.com/syncthing/syncthing/lib/util"
"github.com/thejerf/suture"
@@ -63,27 +63,19 @@ func New(cfg config.Wrapper, m model.Model, connectionsService connections.Servi
// ReportData returns the data to be sent in a usage report with the currently
// configured usage reporting version.
func (s *Service) ReportData(ctx context.Context) map[string]interface{} {
func (s *Service) ReportData(ctx context.Context) (*contract.Report, error) {
urVersion := s.cfg.Options().URAccepted
return s.reportData(ctx, urVersion, false)
}
// ReportDataPreview returns a preview of the data to be sent in a usage report
// with the given version.
func (s *Service) ReportDataPreview(ctx context.Context, urVersion int) map[string]interface{} {
func (s *Service) ReportDataPreview(ctx context.Context, urVersion int) (*contract.Report, error) {
return s.reportData(ctx, urVersion, true)
}
func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) map[string]interface{} {
func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (*contract.Report, error) {
opts := s.cfg.Options()
res := make(map[string]interface{})
res["urVersion"] = urVersion
res["uniqueID"] = opts.URUniqueID
res["version"] = build.Version
res["longVersion"] = build.LongVersion
res["platform"] = runtime.GOOS + "-" + runtime.GOARCH
res["numFolders"] = len(s.cfg.Folders())
res["numDevices"] = len(s.cfg.Devices())
var totFiles, maxFiles int
var totBytes, maxBytes int64
@@ -104,264 +96,211 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) m
}
}
res["totFiles"] = totFiles
res["folderMaxFiles"] = maxFiles
res["totMiB"] = totBytes / 1024 / 1024
res["folderMaxMiB"] = maxBytes / 1024 / 1024
var mem runtime.MemStats
runtime.ReadMemStats(&mem)
res["memoryUsageMiB"] = (mem.Sys - mem.HeapReleased) / 1024 / 1024
res["sha256Perf"] = CpuBench(ctx, 5, 125*time.Millisecond, false)
res["hashPerf"] = CpuBench(ctx, 5, 125*time.Millisecond, true)
bytes, err := memorySize()
if err == nil {
res["memorySize"] = bytes / 1024 / 1024
}
res["numCPU"] = runtime.NumCPU()
report := contract.New()
report.URVersion = urVersion
report.UniqueID = opts.URUniqueID
report.Version = build.Version
report.LongVersion = build.LongVersion
report.Platform = runtime.GOOS + "-" + runtime.GOARCH
report.NumFolders = len(s.cfg.Folders())
report.NumDevices = len(s.cfg.Devices())
report.TotFiles = totFiles
report.FolderMaxFiles = maxFiles
report.TotMiB = int(totBytes / 1024 / 1024)
report.FolderMaxMiB = int(maxBytes / 1024 / 1024)
report.MemoryUsageMiB = int((mem.Sys - mem.HeapReleased) / 1024 / 1024)
report.SHA256Perf = CpuBench(ctx, 5, 125*time.Millisecond, false)
report.HashPerf = CpuBench(ctx, 5, 125*time.Millisecond, true)
report.MemorySize = int(memorySize() / 1024 / 1024)
report.NumCPU = runtime.NumCPU()
var rescanIntvs []int
folderUses := map[string]int{
"sendonly": 0,
"sendreceive": 0,
"receiveonly": 0,
"ignorePerms": 0,
"ignoreDelete": 0,
"autoNormalize": 0,
"simpleVersioning": 0,
"externalVersioning": 0,
"staggeredVersioning": 0,
"trashcanVersioning": 0,
}
for _, cfg := range s.cfg.Folders() {
rescanIntvs = append(rescanIntvs, cfg.RescanIntervalS)
report.RescanIntvs = append(report.RescanIntvs, int64(cfg.RescanIntervalS))
switch cfg.Type {
case config.FolderTypeSendOnly:
folderUses["sendonly"]++
report.FolderUses.SendOnly++
case config.FolderTypeSendReceive:
folderUses["sendreceive"]++
report.FolderUses.SendReceive++
case config.FolderTypeReceiveOnly:
folderUses["receiveonly"]++
report.FolderUses.ReceiveOnly++
}
if cfg.IgnorePerms {
folderUses["ignorePerms"]++
report.FolderUses.IgnorePerms++
}
if cfg.IgnoreDelete {
folderUses["ignoreDelete"]++
report.FolderUses.IgnoreDelete++
}
if cfg.AutoNormalize {
folderUses["autoNormalize"]++
report.FolderUses.AutoNormalize++
}
if cfg.Versioning.Type != "" {
folderUses[cfg.Versioning.Type+"Versioning"]++
switch cfg.Versioning.Type {
case "":
// None
case "simple":
report.FolderUses.SimpleVersioning++
case "staggered":
report.FolderUses.StaggeredVersioning++
case "external":
report.FolderUses.ExternalVersioning++
case "trashcan":
report.FolderUses.TrashcanVersioning++
default:
l.Warnf("Unhandled versioning type for usage reports: %s", cfg.Versioning.Type)
}
}
sort.Ints(rescanIntvs)
res["rescanIntvs"] = rescanIntvs
res["folderUses"] = folderUses
contract.SortPqInt64Array(report.RescanIntvs)
deviceUses := map[string]int{
"introducer": 0,
"customCertName": 0,
"compressAlways": 0,
"compressMetadata": 0,
"compressNever": 0,
"dynamicAddr": 0,
"staticAddr": 0,
}
for _, cfg := range s.cfg.Devices() {
if cfg.Introducer {
deviceUses["introducer"]++
report.DeviceUses.Introducer++
}
if cfg.CertName != "" && cfg.CertName != "syncthing" {
deviceUses["customCertName"]++
report.DeviceUses.CustomCertName++
}
if cfg.Compression == protocol.CompressAlways {
deviceUses["compressAlways"]++
} else if cfg.Compression == protocol.CompressMetadata {
deviceUses["compressMetadata"]++
} else if cfg.Compression == protocol.CompressNever {
deviceUses["compressNever"]++
switch cfg.Compression {
case protocol.CompressAlways:
report.DeviceUses.CompressAlways++
case protocol.CompressMetadata:
report.DeviceUses.CompressMetadata++
case protocol.CompressNever:
report.DeviceUses.CompressNever++
default:
l.Warnf("Unhandled versioning type for usage reports: %s", cfg.Compression)
}
for _, addr := range cfg.Addresses {
if addr == "dynamic" {
deviceUses["dynamicAddr"]++
report.DeviceUses.DynamicAddr++
} else {
deviceUses["staticAddr"]++
report.DeviceUses.StaticAddr++
}
}
}
res["deviceUses"] = deviceUses
defaultAnnounceServersDNS, defaultAnnounceServersIP, otherAnnounceServers := 0, 0, 0
report.Announce.GlobalEnabled = opts.GlobalAnnEnabled
report.Announce.LocalEnabled = opts.LocalAnnEnabled
for _, addr := range opts.RawGlobalAnnServers {
if addr == "default" || addr == "default-v4" || addr == "default-v6" {
defaultAnnounceServersDNS++
report.Announce.DefaultServersDNS++
} else {
otherAnnounceServers++
report.Announce.OtherServers++
}
}
res["announce"] = map[string]interface{}{
"globalEnabled": opts.GlobalAnnEnabled,
"localEnabled": opts.LocalAnnEnabled,
"defaultServersDNS": defaultAnnounceServersDNS,
"defaultServersIP": defaultAnnounceServersIP,
"otherServers": otherAnnounceServers,
}
defaultRelayServers, otherRelayServers := 0, 0
report.Relays.Enabled = opts.RelaysEnabled
for _, addr := range s.cfg.Options().ListenAddresses() {
switch {
case addr == "dynamic+https://relays.syncthing.net/endpoint":
defaultRelayServers++
report.Relays.DefaultServers++
case strings.HasPrefix(addr, "relay://") || strings.HasPrefix(addr, "dynamic+http"):
otherRelayServers++
report.Relays.OtherServers++
}
}
res["relays"] = map[string]interface{}{
"enabled": defaultRelayServers+otherAnnounceServers > 0,
"defaultServers": defaultRelayServers,
"otherServers": otherRelayServers,
}
res["usesRateLimit"] = opts.MaxRecvKbps > 0 || opts.MaxSendKbps > 0
report.UsesRateLimit = opts.MaxRecvKbps > 0 || opts.MaxSendKbps > 0
report.UpgradeAllowedManual = !(upgrade.DisabledByCompilation || s.noUpgrade)
report.UpgradeAllowedAuto = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0
report.UpgradeAllowedPre = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases
res["upgradeAllowedManual"] = !(upgrade.DisabledByCompilation || s.noUpgrade)
res["upgradeAllowedAuto"] = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0
res["upgradeAllowedPre"] = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases
// V3
if urVersion >= 3 {
res["uptime"] = s.UptimeS()
res["natType"] = s.connectionsService.NATType()
res["alwaysLocalNets"] = len(opts.AlwaysLocalNets) > 0
res["cacheIgnoredFiles"] = opts.CacheIgnoredFiles
res["overwriteRemoteDeviceNames"] = opts.OverwriteRemoteDevNames
res["progressEmitterEnabled"] = opts.ProgressUpdateIntervalS > -1
res["customDefaultFolderPath"] = opts.DefaultFolderPath != "~"
res["customTrafficClass"] = opts.TrafficClass != 0
res["customTempIndexMinBlocks"] = opts.TempIndexMinBlocks != 10
res["temporariesDisabled"] = opts.KeepTemporariesH == 0
res["temporariesCustom"] = opts.KeepTemporariesH != 24
res["limitBandwidthInLan"] = opts.LimitBandwidthInLan
res["customReleaseURL"] = opts.ReleasesURL != "https://upgrades.syncthing.net/meta.json"
res["restartOnWakeup"] = opts.RestartOnWakeup
report.Uptime = s.UptimeS()
report.NATType = s.connectionsService.NATType()
report.AlwaysLocalNets = len(opts.AlwaysLocalNets) > 0
report.CacheIgnoredFiles = opts.CacheIgnoredFiles
report.OverwriteRemoteDeviceNames = opts.OverwriteRemoteDevNames
report.ProgressEmitterEnabled = opts.ProgressUpdateIntervalS > -1
report.CustomDefaultFolderPath = opts.DefaultFolderPath != "~"
report.CustomTrafficClass = opts.TrafficClass != 0
report.CustomTempIndexMinBlocks = opts.TempIndexMinBlocks != 10
report.TemporariesDisabled = opts.KeepTemporariesH == 0
report.TemporariesCustom = opts.KeepTemporariesH != 24
report.LimitBandwidthInLan = opts.LimitBandwidthInLan
report.CustomReleaseURL = opts.ReleasesURL != "https=//upgrades.syncthing.net/meta.json"
report.RestartOnWakeup = opts.RestartOnWakeup
report.CustomStunServers = len(opts.RawStunServers) != 1 || opts.RawStunServers[0] != "default"
folderUsesV3 := map[string]int{
"scanProgressDisabled": 0,
"conflictsDisabled": 0,
"conflictsUnlimited": 0,
"conflictsOther": 0,
"disableSparseFiles": 0,
"disableTempIndexes": 0,
"alwaysWeakHash": 0,
"customWeakHashThreshold": 0,
"fsWatcherEnabled": 0,
}
pullOrder := make(map[string]int)
filesystemType := make(map[string]int)
var fsWatcherDelays []int
for _, cfg := range s.cfg.Folders() {
if cfg.ScanProgressIntervalS < 0 {
folderUsesV3["scanProgressDisabled"]++
report.FolderUsesV3.ScanProgressDisabled++
}
if cfg.MaxConflicts == 0 {
folderUsesV3["conflictsDisabled"]++
report.FolderUsesV3.ConflictsDisabled++
} else if cfg.MaxConflicts < 0 {
folderUsesV3["conflictsUnlimited"]++
report.FolderUsesV3.ConflictsUnlimited++
} else {
folderUsesV3["conflictsOther"]++
report.FolderUsesV3.ConflictsOther++
}
if cfg.DisableSparseFiles {
folderUsesV3["disableSparseFiles"]++
report.FolderUsesV3.DisableSparseFiles++
}
if cfg.DisableTempIndexes {
folderUsesV3["disableTempIndexes"]++
report.FolderUsesV3.DisableTempIndexes++
}
if cfg.WeakHashThresholdPct < 0 {
folderUsesV3["alwaysWeakHash"]++
report.FolderUsesV3.AlwaysWeakHash++
} else if cfg.WeakHashThresholdPct != 25 {
folderUsesV3["customWeakHashThreshold"]++
report.FolderUsesV3.CustomWeakHashThreshold++
}
if cfg.FSWatcherEnabled {
folderUsesV3["fsWatcherEnabled"]++
report.FolderUsesV3.FsWatcherEnabled++
}
pullOrder[cfg.Order.String()]++
filesystemType[cfg.FilesystemType.String()]++
fsWatcherDelays = append(fsWatcherDelays, cfg.FSWatcherDelayS)
report.FolderUsesV3.PullOrder[cfg.Order.String()]++
report.FolderUsesV3.FilesystemType[cfg.FilesystemType.String()]++
report.FolderUsesV3.FsWatcherDelays = append(report.FolderUsesV3.FsWatcherDelays, int64(cfg.FSWatcherDelayS))
}
sort.Ints(fsWatcherDelays)
folderUsesV3Interface := map[string]interface{}{
"pullOrder": pullOrder,
"filesystemType": filesystemType,
"fsWatcherDelays": fsWatcherDelays,
}
for key, value := range folderUsesV3 {
folderUsesV3Interface[key] = value
}
res["folderUsesV3"] = folderUsesV3Interface
contract.SortPqInt64Array(report.FolderUsesV3.FsWatcherDelays)
guiCfg := s.cfg.GUI()
// Anticipate multiple GUI configs in the future, hence store counts.
guiStats := map[string]int{
"enabled": 0,
"useTLS": 0,
"useAuth": 0,
"insecureAdminAccess": 0,
"debugging": 0,
"insecureSkipHostCheck": 0,
"insecureAllowFrameLoading": 0,
"listenLocal": 0,
"listenUnspecified": 0,
}
theme := make(map[string]int)
if guiCfg.Enabled {
guiStats["enabled"]++
report.GUIStats.Enabled++
if guiCfg.UseTLS() {
guiStats["useTLS"]++
report.GUIStats.UseTLS++
}
if len(guiCfg.User) > 0 && len(guiCfg.Password) > 0 {
guiStats["useAuth"]++
report.GUIStats.UseAuth++
}
if guiCfg.InsecureAdminAccess {
guiStats["insecureAdminAccess"]++
report.GUIStats.InsecureAdminAccess++
}
if guiCfg.Debugging {
guiStats["debugging"]++
report.GUIStats.Debugging++
}
if guiCfg.InsecureSkipHostCheck {
guiStats["insecureSkipHostCheck"]++
report.GUIStats.InsecureSkipHostCheck++
}
if guiCfg.InsecureAllowFrameLoading {
guiStats["insecureAllowFrameLoading"]++
report.GUIStats.InsecureAllowFrameLoading++
}
addr, err := net.ResolveTCPAddr("tcp", guiCfg.Address())
if err == nil {
if addr.IP.IsLoopback() {
guiStats["listenLocal"]++
report.GUIStats.ListenLocal++
} else if addr.IP.IsUnspecified() {
guiStats["listenUnspecified"]++
report.GUIStats.ListenUnspecified++
}
}
theme[guiCfg.Theme]++
report.GUIStats.Theme[guiCfg.Theme]++
}
guiStatsInterface := map[string]interface{}{
"theme": theme,
}
for key, value := range guiStats {
guiStatsInterface[key] = value
}
res["guiStats"] = guiStatsInterface
}
for key, value := range s.model.UsageReportingStats(urVersion, preview) {
res[key] = value
s.model.UsageReportingStats(report, urVersion, preview)
if err := report.ClearForVersion(urVersion); err != nil {
return nil, err
}
return res
return report, nil
}
func (s *Service) UptimeS() int {
@@ -369,7 +308,10 @@ func (s *Service) UptimeS() int {
}
func (s *Service) sendUsageReport(ctx context.Context) error {
d := s.ReportData(ctx)
d, err := s.ReportData(ctx)
if err != nil {
return err
}
var b bytes.Buffer
if err := json.NewEncoder(&b).Encode(d); err != nil {
return err
@@ -384,12 +326,11 @@ func (s *Service) sendUsageReport(ctx context.Context) error {
},
},
}
req, err := http.NewRequest("POST", s.cfg.Options().URURL, &b)
req, err := http.NewRequestWithContext(ctx, "POST", s.cfg.Options().URURL, &b)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Cancel = ctx.Done()
resp, err := client.Do(req)
if err != nil {
return err