diff --git a/build.sh b/build.sh index 643d5b4d4..5ea6c7fc3 100755 --- a/build.sh +++ b/build.sh @@ -23,6 +23,7 @@ case "${1:-default}" in prerelease) script authors + script copyrights build weblate pushd man ; ./refresh.sh ; popd git add -A gui man AUTHORS diff --git a/gui/default/syncthing/core/aboutModalView.html b/gui/default/syncthing/core/aboutModalView.html index e95a20348..8e82ba32f 100644 --- a/gui/default/syncthing/core/aboutModalView.html +++ b/gui/default/syncthing/core/aboutModalView.html @@ -38,45 +38,94 @@ Jakob Borg, Audrius Butkevicius, Jesse Lucas, Simon Frei, Tomasz Wilczyński, Al

Syncthing includes the following software or portions thereof:

diff --git a/script/copyrights.go b/script/copyrights.go new file mode 100644 index 000000000..f192893c2 --- /dev/null +++ b/script/copyrights.go @@ -0,0 +1,489 @@ +// Copyright (C) 2025 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +//go:build ignore +// +build ignore + +// Updates the list of software copyrights in aboutModalView.html based on the +// output of `go mod graph`. + +package main + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "os/exec" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "golang.org/x/net/html" +) + +var copyrightMap = map[string]string{ + // https://github.com/aws/aws-sdk-go/blob/main/NOTICE.txt#L2 + "aws/aws-sdk-go": "Copyright © 2015 Amazon.com, Inc. or its affiliates, Copyright 2014-2015 Stripe, Inc", + // https://github.com/ccding/go-stun/blob/master/main.go#L1 + "ccding/go-stun": "Copyright © 2016 Cong Ding", + // https://github.com/search?q=repo%3Acertifi%2Fgocertifi%20copyright&type=code + // "certifi/gocertifi": "No copyrights found", + // https://github.com/search?q=repo%3Aebitengine%2Fpurego%20copyright&type=code + "ebitengine/purego": "Copyright © 2022 The Ebitengine Authors", + // https://github.com/search?q=repo%3Agoogle%2Fpprof%20copyright&type=code + "google/pprof": "Copyright © 2016 Google Inc", + // https://github.com/greatroar/blobloom/blob/master/README.md?plain=1#L74 + "greatroar/blobloom": "Copyright © 2020-2024 the Blobloom authors", + // https://github.com/jmespath/go-jmespath/blob/master/NOTICE#L2 + "jmespath/go-jmespath": "Copyright © 2015 James Saryerwinnie", + // https://github.com/maxmind/geoipupdate/blob/main/README.md?plain=1#L140 + "maxmind/geoipupdate": "Copyright © 2018-2024 by MaxMind, Inc", + // https://github.com/search?q=repo%3Apuzpuzpuz%2Fxsync%20copyright&type=code + // "puzpuzpuz/xsync": "No copyrights found", + // https://github.com/search?q=repo%3Atklauser%2Fnumcpus%20copyright&type=code + "tklauser/numcpus": "Copyright © 2018-2024 Tobias Klauser", + // https://github.com/search?q=repo%3Auber-go%2Fmock%20copyright&type=code + "go.uber.org/mock": "Copyright © 2010-2022 Google LLC", +} + +var urlMap = map[string]string{ + "fontawesome.io": "https://github.com/FortAwesome/Font-Awesome", + "go.uber.org/automaxprocs": "https://github.com/uber-go/automaxprocs", + "go.uber.org/mock": "https://github.com/uber-go/mock", + "google.golang.org/protobuf": "https://github.com/protocolbuffers/protobuf-go", + "gopkg.in/yaml.v2": "", // ignore, as gopkg.in/yaml.v3 supersedes + "gopkg.in/yaml.v3": "https://github.com/go-yaml/yaml", + "sigs.k8s.io/yaml": "https://github.com/kubernetes-sigs/yaml", +} + +const htmlFile = "gui/default/syncthing/core/aboutModalView.html" + +type Type int + +const ( + // TypeJS defines non-Go copyright notices + TypeJS Type = iota + // TypeKeep defines Go copyright notices for packages that are still used. + TypeKeep + // TypeToss defines Go copyright notices for packages that are no longer used. + TypeToss + // TypeNew defines Go copyright notices for new packages found via `go mod graph`. + TypeNew +) + +type CopyrightNotice struct { + Type Type + Name string + HTML string + Module string + URL string + Copyright string + RepoURL string + RepoCopyrights []string +} + +var copyrightRe = regexp.MustCompile(`(?s)id="copyright-notices">(.+?)`) + +func main() { + bs := readAll(htmlFile) + matches := copyrightRe.FindStringSubmatch(string(bs)) + + if len(matches) <= 1 { + log.Fatal("Cannot find id copyright-notices in ", htmlFile) + } + + modules := getModules() + + notices := parseCopyrightNotices(matches[1]) + old := len(notices) + + // match up modules to notices + matched := map[string]bool{} + removes := 0 + for i, notice := range notices { + if notice.Type == TypeJS { + continue + } + found := "" + for _, module := range modules { + if strings.Contains(module, notice.Name) { + found = module + + break + } + } + if found != "" { + matched[found] = true + notices[i].Module = found + + continue + } + removes++ + fmt.Printf("Removing: %-40s %-55s %s\n", notice.Name, notice.URL, notice.Copyright) + notices[i].Type = TypeToss + } + + // add new modules to notices + adds := 0 + for _, module := range modules { + _, ok := matched[module] + if ok { + continue + } + + adds++ + notice := CopyrightNotice{} + notice.Name = module + if strings.HasPrefix(notice.Name, "github.com/") { + notice.Name = strings.ReplaceAll(notice.Name, "github.com/", "") + } + notice.Type = TypeNew + + url, ok := urlMap[module] + if ok { + notice.URL = url + notice.RepoURL = url + } else { + notice.URL = "https://" + module + notice.RepoURL = "https://" + module + } + notices = append(notices, notice) + } + + if removes == 0 && adds == 0 { + // authors.go is quiet, so let's be quiet too. + // fmt.Printf("No changes detected in %d modules and %d notices\n", len(modules), len(notices)) + os.Exit(0) + } + + // get copyrights via Github API for new modules + notfound := 0 + for i, n := range notices { + if n.Type != TypeNew { + continue + } + copyright, ok := copyrightMap[n.Name] + if ok { + notices[i].Copyright = copyright + + continue + } + notices[i].Copyright = defaultCopyright(n) + + if strings.Contains(n.URL, "github.com/") { + notices[i].RepoURL = notices[i].URL + owner, repo := parseGitHubURL(n.URL) + licenseText := getLicenseText(owner, repo) + notices[i].RepoCopyrights = extractCopyrights(licenseText, n) + + if len(notices[i].RepoCopyrights) > 0 { + notices[i].Copyright = notices[i].RepoCopyrights[0] + } + + notices[i].HTML = fmt.Sprintf("
  • %s, %s.
    • ", n.URL, n.Name, notices[i].Copyright) + if len(notices[i].RepoCopyrights) > 0 { + continue + } + } + fmt.Printf("Copyright not found: %-30s : using %q\n", n.Name, notices[i].Copyright) + notfound++ + } + + replacements := write(notices, bs) + fmt.Printf("Removed: %3d\n", removes) + fmt.Printf("Added: %3d\n", adds) + fmt.Printf("Copyrights not found: %3d\n", notfound) + fmt.Printf("Old package count: %3d\n", old) + fmt.Printf("New package count: %3d\n", replacements) +} + +func write(notices []CopyrightNotice, bs []byte) int { + keys := make([]string, 0, len(notices)) + + noticeMap := make(map[string]CopyrightNotice, 0) + + for _, n := range notices { + if n.Type != TypeKeep && n.Type != TypeNew { + continue + } + if n.Type == TypeNew { + fmt.Printf("Adding: %-40s %-55s %s\n", n.Name, n.URL, n.Copyright) + } + keys = append(keys, n.Name) + noticeMap[n.Name] = n + } + + slices.Sort(keys) + + indent := " " + replacements := []string{} + for _, n := range notices { + if n.Type != TypeJS { + continue + } + replacements = append(replacements, indent+n.HTML) + } + + for _, k := range keys { + n := noticeMap[k] + line := fmt.Sprintf("%s
  • %s, %s.
    • ", indent, n.URL, n.Name, n.Copyright) + replacements = append(replacements, line) + } + replacement := strings.Join(replacements, "\n") + + bs = copyrightRe.ReplaceAll(bs, []byte("id=\"copyright-notices\">\n"+replacement+"\n ")) + writeFile(htmlFile, string(bs)) + + return len(replacements) +} + +func readAll(path string) []byte { + fd, err := os.Open(path) + if err != nil { + log.Fatal(err) + } + defer fd.Close() + + bs, err := io.ReadAll(fd) + if err != nil { + log.Fatal(err) + } + + return bs +} + +func writeFile(path string, data string) { + err := os.WriteFile(path, []byte(data), 0o644) + if err != nil { + log.Fatal(err) + } +} + +func getModules() []string { + cmd := exec.Command("go", "mod", "graph") + output, err := cmd.Output() + if err != nil { + log.Fatal(err) + } + + seen := make(map[string]struct{}) + scanner := bufio.NewScanner(bytes.NewReader(output)) + + for scanner.Scan() { + line := scanner.Text() + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + + if !strings.HasPrefix(fields[0], "github.com/syncthing/syncthing") { + continue + } + + // Get left-hand side of dependency pair (before '@') + mod := strings.SplitN(fields[1], "@", 2)[0] + + // Keep only first 3 path components + parts := strings.Split(mod, "/") + if len(parts) == 1 { + continue + } + short := strings.Join(parts[:min(len(parts), 3)], "/") + + if strings.HasPrefix(short, "golang.org/x") || + strings.HasPrefix(short, "github.com/prometheus") || + short == "go" { + + continue + } + + seen[short] = struct{}{} + } + + adds := make([]string, 0) + for k := range seen { + adds = append(adds, k) + } + + slices.Sort(adds) + + return adds +} + +func parseCopyrightNotices(input string) []CopyrightNotice { + doc, err := html.Parse(strings.NewReader("")) + if err != nil { + log.Fatal(err) + } + + var notices []CopyrightNotice + + typ := TypeJS + + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "li" { + var notice CopyrightNotice + var aFound bool + + for c := n.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.ElementNode && c.Data == "a" { + aFound = true + for _, attr := range c.Attr { + if attr.Key == "href" { + notice.URL = attr.Val + } + } + if c.FirstChild != nil && c.FirstChild.Type == html.TextNode { + notice.Name = strings.TrimSpace(c.FirstChild.Data) + } + } else if c.Type == html.TextNode && aFound { + // Anything after is considered the copyright + notice.Copyright = strings.TrimSpace(html.UnescapeString(c.Data)) + notice.Copyright = strings.Trim(notice.Copyright, "., ") + } + if typ == TypeJS && strings.Contains(notice.URL, "AudriusButkevicius") { + typ = TypeKeep + } + notice.Type = typ + var buf strings.Builder + _ = html.Render(&buf, n) + notice.HTML = buf.String() + } + + notice.Copyright = strings.ReplaceAll(notice.Copyright, "©", "©") + notice.HTML = strings.ReplaceAll(notice.HTML, "©", "©") + notices = append(notices, notice) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + + f(doc) + + return notices +} + +func parseGitHubURL(u string) (string, string) { + parsed, err := url.Parse(u) + if err != nil { + log.Fatal(err) + } + parts := strings.Split(strings.Trim(parsed.Path, "/"), "/") + if len(parts) < 2 { + log.Fatal(fmt.Errorf("invalid GitHub URL: %q", parsed.Path)) + } + + return parts[0], parts[1] +} + +func getLicenseText(owner, repo string) string { + url := fmt.Sprintf("https://api.github.com/repos/%s/%s/license", owner, repo) + req, _ := http.NewRequest("GET", url, nil) + req.Header.Set("Accept", "application/vnd.github.v3+json") + + if token := os.Getenv("GITHUB_TOKEN"); token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Fatal(err) + } + defer resp.Body.Close() + + var result struct { + Content string `json:"content"` + Encoding string `json:"encoding"` + } + body, _ := io.ReadAll(resp.Body) + err = json.Unmarshal(body, &result) + if err != nil { + log.Fatal(err) + } + + if result.Encoding != "base64" { + log.Fatal(fmt.Sprintf("unexpected encoding: %s", result.Encoding)) + } + + decoded, err := base64.StdEncoding.DecodeString(result.Content) + if err != nil { + log.Fatal(err) + } + + return string(decoded) +} + +func extractCopyrights(license string, notice CopyrightNotice) []string { + lines := strings.Split(license, "\n") + + re := regexp.MustCompile(`(?i)^\s*(copyright\s*(?:©|\(c\)|©|19|20).*)$`) + + copyrights := []string{} + + for _, line := range lines { + if matches := re.FindStringSubmatch(strings.TrimSpace(line)); len(matches) == 2 { + copyright := strings.TrimSpace(matches[1]) + re := regexp.MustCompile(`(?i)all rights reserved`) + copyright = re.ReplaceAllString(copyright, "") + copyright = strings.ReplaceAll(copyright, "©", "©") + copyright = strings.ReplaceAll(copyright, "(C)", "©") + copyright = strings.ReplaceAll(copyright, "(c)", "©") + copyright = strings.Trim(copyright, "., ") + copyrights = append(copyrights, copyright) + } + } + + if len(copyrights) > 0 { + return copyrights + } + + return []string{} +} + +func defaultCopyright(n CopyrightNotice) string { + year := time.Now().Format("2006") + + return fmt.Sprintf("Copyright © %v, the %s authors", year, n.Name) +} + +func writeNotices(path string, notices []CopyrightNotice) { + s := "" + for i, n := range notices { + s += "# : " + strconv.Itoa(i) + "\n" + n.String() + } + writeFile(path, s) +} + +func (n CopyrightNotice) String() string { + return fmt.Sprintf("Type : %v\nHTML : %v\nName : %v\nModule : %v\nURL : %v\nCopyright: %v\nRepoURL : %v\nRepoCopys: %v\n\n", + n.Type, n.HTML, n.Name, n.Module, n.URL, n.Copyright, n.RepoURL, strings.Join(n.RepoCopyrights, ",")) +} + +func (t Type) String() string { + switch t { + case TypeJS: + return "TypeJS" + case TypeKeep: + return "TypeKeep" + case TypeToss: + return "TypeToss" + case TypeNew: + return "TypeNew" + default: + return "unknown" + } +}