From 93ae30d88978009c1d8a6525832d8fc376aa2c91 Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Tue, 22 Apr 2025 22:41:05 -0700 Subject: [PATCH] chore(gui): update dependency copyrights, add script for periodic maintenance (#10067) ### Purpose This PR parses the output of `go mod graph` and updates the copyright list in our [about modal](https://github.com/syncthing/syncthing/blob/486eebc4ac911b86b3d110dd0f7a955edd0f5106/gui/default/syncthing/core/aboutModalView.html#L38). If there are no changes, the program is silent. Otherwise, it reports what additions, and deletions it made. It does not rewrite existing copyright notices, but it does remove notices that we no longer use, as well as add new ones. It uses a GitHub API to try to determine the copyright string in the license file. If one is not found, it defaults to `Copyright © the authors`. If a proper copyright is found, simply update the notice in `aboutModalView.html`, and it will be used. --- build.sh | 1 + .../syncthing/core/aboutModalView.html | 85 ++- script/copyrights.go | 489 ++++++++++++++++++ 3 files changed, 557 insertions(+), 18 deletions(-) create mode 100644 script/copyrights.go diff --git a/build.sh b/build.sh index 643d5b4d4..5ea6c7fc3 100755 --- a/build.sh +++ b/build.sh @@ -23,6 +23,7 @@ case "${1:-default}" in prerelease) script authors + script copyrights build weblate pushd man ; ./refresh.sh ; popd git add -A gui man AUTHORS diff --git a/gui/default/syncthing/core/aboutModalView.html b/gui/default/syncthing/core/aboutModalView.html index e95a20348..8e82ba32f 100644 --- a/gui/default/syncthing/core/aboutModalView.html +++ b/gui/default/syncthing/core/aboutModalView.html @@ -38,45 +38,94 @@ Jakob Borg, Audrius Butkevicius, Jesse Lucas, Simon Frei, Tomasz Wilczyński, Al

Syncthing includes the following software or portions thereof:

diff --git a/script/copyrights.go b/script/copyrights.go new file mode 100644 index 000000000..f192893c2 --- /dev/null +++ b/script/copyrights.go @@ -0,0 +1,489 @@ +// Copyright (C) 2025 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +//go:build ignore +// +build ignore + +// Updates the list of software copyrights in aboutModalView.html based on the +// output of `go mod graph`. + +package main + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "os/exec" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "golang.org/x/net/html" +) + +var copyrightMap = map[string]string{ + // https://github.com/aws/aws-sdk-go/blob/main/NOTICE.txt#L2 + "aws/aws-sdk-go": "Copyright © 2015 Amazon.com, Inc. or its affiliates, Copyright 2014-2015 Stripe, Inc", + // https://github.com/ccding/go-stun/blob/master/main.go#L1 + "ccding/go-stun": "Copyright © 2016 Cong Ding", + // https://github.com/search?q=repo%3Acertifi%2Fgocertifi%20copyright&type=code + // "certifi/gocertifi": "No copyrights found", + // https://github.com/search?q=repo%3Aebitengine%2Fpurego%20copyright&type=code + "ebitengine/purego": "Copyright © 2022 The Ebitengine Authors", + // https://github.com/search?q=repo%3Agoogle%2Fpprof%20copyright&type=code + "google/pprof": "Copyright © 2016 Google Inc", + // https://github.com/greatroar/blobloom/blob/master/README.md?plain=1#L74 + "greatroar/blobloom": "Copyright © 2020-2024 the Blobloom authors", + // https://github.com/jmespath/go-jmespath/blob/master/NOTICE#L2 + "jmespath/go-jmespath": "Copyright © 2015 James Saryerwinnie", + // https://github.com/maxmind/geoipupdate/blob/main/README.md?plain=1#L140 + "maxmind/geoipupdate": "Copyright © 2018-2024 by MaxMind, Inc", + // https://github.com/search?q=repo%3Apuzpuzpuz%2Fxsync%20copyright&type=code + // "puzpuzpuz/xsync": "No copyrights found", + // https://github.com/search?q=repo%3Atklauser%2Fnumcpus%20copyright&type=code + "tklauser/numcpus": "Copyright © 2018-2024 Tobias Klauser", + // https://github.com/search?q=repo%3Auber-go%2Fmock%20copyright&type=code + "go.uber.org/mock": "Copyright © 2010-2022 Google LLC", +} + +var urlMap = map[string]string{ + "fontawesome.io": "https://github.com/FortAwesome/Font-Awesome", + "go.uber.org/automaxprocs": "https://github.com/uber-go/automaxprocs", + "go.uber.org/mock": "https://github.com/uber-go/mock", + "google.golang.org/protobuf": "https://github.com/protocolbuffers/protobuf-go", + "gopkg.in/yaml.v2": "", // ignore, as gopkg.in/yaml.v3 supersedes + "gopkg.in/yaml.v3": "https://github.com/go-yaml/yaml", + "sigs.k8s.io/yaml": "https://github.com/kubernetes-sigs/yaml", +} + +const htmlFile = "gui/default/syncthing/core/aboutModalView.html" + +type Type int + +const ( + // TypeJS defines non-Go copyright notices + TypeJS Type = iota + // TypeKeep defines Go copyright notices for packages that are still used. + TypeKeep + // TypeToss defines Go copyright notices for packages that are no longer used. + TypeToss + // TypeNew defines Go copyright notices for new packages found via `go mod graph`. + TypeNew +) + +type CopyrightNotice struct { + Type Type + Name string + HTML string + Module string + URL string + Copyright string + RepoURL string + RepoCopyrights []string +} + +var copyrightRe = regexp.MustCompile(`(?s)id="copyright-notices">(.+?)`) + +func main() { + bs := readAll(htmlFile) + matches := copyrightRe.FindStringSubmatch(string(bs)) + + if len(matches) <= 1 { + log.Fatal("Cannot find id copyright-notices in ", htmlFile) + } + + modules := getModules() + + notices := parseCopyrightNotices(matches[1]) + old := len(notices) + + // match up modules to notices + matched := map[string]bool{} + removes := 0 + for i, notice := range notices { + if notice.Type == TypeJS { + continue + } + found := "" + for _, module := range modules { + if strings.Contains(module, notice.Name) { + found = module + + break + } + } + if found != "" { + matched[found] = true + notices[i].Module = found + + continue + } + removes++ + fmt.Printf("Removing: %-40s %-55s %s\n", notice.Name, notice.URL, notice.Copyright) + notices[i].Type = TypeToss + } + + // add new modules to notices + adds := 0 + for _, module := range modules { + _, ok := matched[module] + if ok { + continue + } + + adds++ + notice := CopyrightNotice{} + notice.Name = module + if strings.HasPrefix(notice.Name, "github.com/") { + notice.Name = strings.ReplaceAll(notice.Name, "github.com/", "") + } + notice.Type = TypeNew + + url, ok := urlMap[module] + if ok { + notice.URL = url + notice.RepoURL = url + } else { + notice.URL = "https://" + module + notice.RepoURL = "https://" + module + } + notices = append(notices, notice) + } + + if removes == 0 && adds == 0 { + // authors.go is quiet, so let's be quiet too. + // fmt.Printf("No changes detected in %d modules and %d notices\n", len(modules), len(notices)) + os.Exit(0) + } + + // get copyrights via Github API for new modules + notfound := 0 + for i, n := range notices { + if n.Type != TypeNew { + continue + } + copyright, ok := copyrightMap[n.Name] + if ok { + notices[i].Copyright = copyright + + continue + } + notices[i].Copyright = defaultCopyright(n) + + if strings.Contains(n.URL, "github.com/") { + notices[i].RepoURL = notices[i].URL + owner, repo := parseGitHubURL(n.URL) + licenseText := getLicenseText(owner, repo) + notices[i].RepoCopyrights = extractCopyrights(licenseText, n) + + if len(notices[i].RepoCopyrights) > 0 { + notices[i].Copyright = notices[i].RepoCopyrights[0] + } + + notices[i].HTML = fmt.Sprintf("
  • %s, %s.
    • ", n.URL, n.Name, notices[i].Copyright) + if len(notices[i].RepoCopyrights) > 0 { + continue + } + } + fmt.Printf("Copyright not found: %-30s : using %q\n", n.Name, notices[i].Copyright) + notfound++ + } + + replacements := write(notices, bs) + fmt.Printf("Removed: %3d\n", removes) + fmt.Printf("Added: %3d\n", adds) + fmt.Printf("Copyrights not found: %3d\n", notfound) + fmt.Printf("Old package count: %3d\n", old) + fmt.Printf("New package count: %3d\n", replacements) +} + +func write(notices []CopyrightNotice, bs []byte) int { + keys := make([]string, 0, len(notices)) + + noticeMap := make(map[string]CopyrightNotice, 0) + + for _, n := range notices { + if n.Type != TypeKeep && n.Type != TypeNew { + continue + } + if n.Type == TypeNew { + fmt.Printf("Adding: %-40s %-55s %s\n", n.Name, n.URL, n.Copyright) + } + keys = append(keys, n.Name) + noticeMap[n.Name] = n + } + + slices.Sort(keys) + + indent := " " + replacements := []string{} + for _, n := range notices { + if n.Type != TypeJS { + continue + } + replacements = append(replacements, indent+n.HTML) + } + + for _, k := range keys { + n := noticeMap[k] + line := fmt.Sprintf("%s
  • %s, %s.
    • ", indent, n.URL, n.Name, n.Copyright) + replacements = append(replacements, line) + } + replacement := strings.Join(replacements, "\n") + + bs = copyrightRe.ReplaceAll(bs, []byte("id=\"copyright-notices\">\n"+replacement+"\n ")) + writeFile(htmlFile, string(bs)) + + return len(replacements) +} + +func readAll(path string) []byte { + fd, err := os.Open(path) + if err != nil { + log.Fatal(err) + } + defer fd.Close() + + bs, err := io.ReadAll(fd) + if err != nil { + log.Fatal(err) + } + + return bs +} + +func writeFile(path string, data string) { + err := os.WriteFile(path, []byte(data), 0o644) + if err != nil { + log.Fatal(err) + } +} + +func getModules() []string { + cmd := exec.Command("go", "mod", "graph") + output, err := cmd.Output() + if err != nil { + log.Fatal(err) + } + + seen := make(map[string]struct{}) + scanner := bufio.NewScanner(bytes.NewReader(output)) + + for scanner.Scan() { + line := scanner.Text() + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + + if !strings.HasPrefix(fields[0], "github.com/syncthing/syncthing") { + continue + } + + // Get left-hand side of dependency pair (before '@') + mod := strings.SplitN(fields[1], "@", 2)[0] + + // Keep only first 3 path components + parts := strings.Split(mod, "/") + if len(parts) == 1 { + continue + } + short := strings.Join(parts[:min(len(parts), 3)], "/") + + if strings.HasPrefix(short, "golang.org/x") || + strings.HasPrefix(short, "github.com/prometheus") || + short == "go" { + + continue + } + + seen[short] = struct{}{} + } + + adds := make([]string, 0) + for k := range seen { + adds = append(adds, k) + } + + slices.Sort(adds) + + return adds +} + +func parseCopyrightNotices(input string) []CopyrightNotice { + doc, err := html.Parse(strings.NewReader("
      " + input + "
    ")) + if err != nil { + log.Fatal(err) + } + + var notices []CopyrightNotice + + typ := TypeJS + + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "li" { + var notice CopyrightNotice + var aFound bool + + for c := n.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.ElementNode && c.Data == "a" { + aFound = true + for _, attr := range c.Attr { + if attr.Key == "href" { + notice.URL = attr.Val + } + } + if c.FirstChild != nil && c.FirstChild.Type == html.TextNode { + notice.Name = strings.TrimSpace(c.FirstChild.Data) + } + } else if c.Type == html.TextNode && aFound { + // Anything after is considered the copyright + notice.Copyright = strings.TrimSpace(html.UnescapeString(c.Data)) + notice.Copyright = strings.Trim(notice.Copyright, "., ") + } + if typ == TypeJS && strings.Contains(notice.URL, "AudriusButkevicius") { + typ = TypeKeep + } + notice.Type = typ + var buf strings.Builder + _ = html.Render(&buf, n) + notice.HTML = buf.String() + } + + notice.Copyright = strings.ReplaceAll(notice.Copyright, "©", "©") + notice.HTML = strings.ReplaceAll(notice.HTML, "©", "©") + notices = append(notices, notice) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + + f(doc) + + return notices +} + +func parseGitHubURL(u string) (string, string) { + parsed, err := url.Parse(u) + if err != nil { + log.Fatal(err) + } + parts := strings.Split(strings.Trim(parsed.Path, "/"), "/") + if len(parts) < 2 { + log.Fatal(fmt.Errorf("invalid GitHub URL: %q", parsed.Path)) + } + + return parts[0], parts[1] +} + +func getLicenseText(owner, repo string) string { + url := fmt.Sprintf("https://api.github.com/repos/%s/%s/license", owner, repo) + req, _ := http.NewRequest("GET", url, nil) + req.Header.Set("Accept", "application/vnd.github.v3+json") + + if token := os.Getenv("GITHUB_TOKEN"); token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Fatal(err) + } + defer resp.Body.Close() + + var result struct { + Content string `json:"content"` + Encoding string `json:"encoding"` + } + body, _ := io.ReadAll(resp.Body) + err = json.Unmarshal(body, &result) + if err != nil { + log.Fatal(err) + } + + if result.Encoding != "base64" { + log.Fatal(fmt.Sprintf("unexpected encoding: %s", result.Encoding)) + } + + decoded, err := base64.StdEncoding.DecodeString(result.Content) + if err != nil { + log.Fatal(err) + } + + return string(decoded) +} + +func extractCopyrights(license string, notice CopyrightNotice) []string { + lines := strings.Split(license, "\n") + + re := regexp.MustCompile(`(?i)^\s*(copyright\s*(?:©|\(c\)|©|19|20).*)$`) + + copyrights := []string{} + + for _, line := range lines { + if matches := re.FindStringSubmatch(strings.TrimSpace(line)); len(matches) == 2 { + copyright := strings.TrimSpace(matches[1]) + re := regexp.MustCompile(`(?i)all rights reserved`) + copyright = re.ReplaceAllString(copyright, "") + copyright = strings.ReplaceAll(copyright, "©", "©") + copyright = strings.ReplaceAll(copyright, "(C)", "©") + copyright = strings.ReplaceAll(copyright, "(c)", "©") + copyright = strings.Trim(copyright, "., ") + copyrights = append(copyrights, copyright) + } + } + + if len(copyrights) > 0 { + return copyrights + } + + return []string{} +} + +func defaultCopyright(n CopyrightNotice) string { + year := time.Now().Format("2006") + + return fmt.Sprintf("Copyright © %v, the %s authors", year, n.Name) +} + +func writeNotices(path string, notices []CopyrightNotice) { + s := "" + for i, n := range notices { + s += "# : " + strconv.Itoa(i) + "\n" + n.String() + } + writeFile(path, s) +} + +func (n CopyrightNotice) String() string { + return fmt.Sprintf("Type : %v\nHTML : %v\nName : %v\nModule : %v\nURL : %v\nCopyright: %v\nRepoURL : %v\nRepoCopys: %v\n\n", + n.Type, n.HTML, n.Name, n.Module, n.URL, n.Copyright, n.RepoURL, strings.Join(n.RepoCopyrights, ",")) +} + +func (t Type) String() string { + switch t { + case TypeJS: + return "TypeJS" + case TypeKeep: + return "TypeKeep" + case TypeToss: + return "TypeToss" + case TypeNew: + return "TypeNew" + default: + return "unknown" + } +}