diff --git a/cmd/infra/stcrashreceiver/main.go b/cmd/infra/stcrashreceiver/main.go index 315bcd0ea..30c834c2a 100644 --- a/cmd/infra/stcrashreceiver/main.go +++ b/cmd/infra/stcrashreceiver/main.go @@ -123,7 +123,7 @@ func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http return } - if ignore.match(bs) { + if _, ok := ignore.match(bs); ok { result = "ignored" return } @@ -216,14 +216,14 @@ func loadIgnorePatterns(path string) (*ignorePatterns, error) { return &ignorePatterns{patterns: patterns}, nil } -func (i *ignorePatterns) match(report []byte) bool { +func (i *ignorePatterns) match(report []byte) (string, bool) { if i == nil { - return false + return "", false } for _, re := range i.patterns { if re.Match(report) { - return true + return re.String(), true } } - return false + return "", false } diff --git a/cmd/infra/stcrashreceiver/metrics.go b/cmd/infra/stcrashreceiver/metrics.go index 8ccbc8fa3..c073c8c39 100644 --- a/cmd/infra/stcrashreceiver/metrics.go +++ b/cmd/infra/stcrashreceiver/metrics.go @@ -37,4 +37,9 @@ var ( Subsystem: "crashreceiver", Name: "diskstore_oldest_age_seconds", }) + metricSentryReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "syncthing", + Subsystem: "crashreceiver", + Name: "sentry_reports_total", + }, []string{"result"}) ) diff --git a/cmd/infra/stcrashreceiver/sentry.go b/cmd/infra/stcrashreceiver/sentry.go index dcc60f069..fd5ed2bfb 100644 --- a/cmd/infra/stcrashreceiver/sentry.go +++ b/cmd/infra/stcrashreceiver/sentry.go @@ -10,6 +10,7 @@ import ( "bytes" "context" "errors" + "fmt" "io" "log" "regexp" @@ -52,11 +53,15 @@ func (s *sentryService) Serve(ctx context.Context) { pkt, err := parseCrashReport(req.reportID, req.data) if err != nil { log.Println("Failed to parse crash report:", err) + metricSentryReportsTotal.WithLabelValues("parse_failure").Inc() continue } if err := sendReport(s.dsn, pkt, req.userID); err != nil { log.Println("Failed to send crash report:", err) + metricSentryReportsTotal.WithLabelValues("send_failure").Inc() + continue } + metricSentryReportsTotal.WithLabelValues("success").Inc() case <-ctx.Done(): return @@ -69,6 +74,7 @@ func (s *sentryService) Send(reportID, userID string, data []byte) bool { case s.inbox <- sentryRequest{reportID, userID, data}: return true default: + metricCrashReportsTotal.WithLabelValues("overflow").Inc() return false } } @@ -108,7 +114,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) { version, err := build.ParseVersion(string(parts[0])) if err != nil { - return nil, err + return nil, fmt.Errorf("%w in %q", err, parts[0]) } report = parts[1] diff --git a/cmd/infra/stcrashreceiver/stcrashreceiver.go b/cmd/infra/stcrashreceiver/stcrashreceiver.go index d45e057ed..18776f338 100644 --- a/cmd/infra/stcrashreceiver/stcrashreceiver.go +++ b/cmd/infra/stcrashreceiver/stcrashreceiver.go @@ -7,21 +7,18 @@ package main import ( + "bytes" "io" "log" "net/http" "path" "strings" - "sync" ) type crashReceiver struct { store *diskStore sentry *sentryService ignore *ignorePatterns - - ignoredMut sync.RWMutex - ignored map[string]struct{} } func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) { @@ -69,12 +66,6 @@ func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http // serveHead responds to HEAD requests by checking if the named report // already exists in the system. func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) { - r.ignoredMut.RLock() - _, ignored := r.ignored[reportID] - r.ignoredMut.RUnlock() - if ignored { - return // found - } if !r.store.Exists(reportID) { http.Error(w, "Not found", http.StatusNotFound) } @@ -87,17 +78,7 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht metricCrashReportsTotal.WithLabelValues(result).Inc() }() - r.ignoredMut.RLock() - _, ignored := r.ignored[reportID] - r.ignoredMut.RUnlock() - if ignored { - result = "ignored_cached" - io.Copy(io.Discard, req.Body) - return // found - } - // Read at most maxRequestSize of report data. - log.Println("Receiving report", reportID) lr := io.LimitReader(req.Body, maxRequestSize) bs, err := io.ReadAll(lr) if err != nil { @@ -106,14 +87,11 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht return } - if r.ignore.match(bs) { - r.ignoredMut.Lock() - if r.ignored == nil { - r.ignored = make(map[string]struct{}) - } - r.ignored[reportID] = struct{}{} - r.ignoredMut.Unlock() + first := string(bytes.TrimSpace(bytes.Split(bs, []byte("\n"))[0])) + + if pat, ok := r.ignore.match(bs); ok { result = "ignored" + log.Printf("Ignored report %s, matched: %s (%s)", reportID[:8], pat, first) return } @@ -121,13 +99,15 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht // Store the report if !r.store.Put(reportID, bs) { - log.Println("Failed to store report (queue full):", reportID) + log.Println("Failed to store report (queue full):", reportID[:8]) result = "queue_failure" } // Send the report to Sentry if !r.sentry.Send(reportID, userIDFor(req), bs) { - log.Println("Failed to send report to sentry (queue full):", reportID) + log.Println("Failed to send report to sentry (queue full):", reportID[:8]) result = "sentry_failure" } + + log.Printf("Received report %s (%s)", reportID[:8], first) }