// Package gettool combines and replaces curl, tar and gunzip, sha256sum and a bunch of Makefile scripts // to quickly download, verify and install OS-specific version of tools (typically from GitHub) // in a platform-agnostic manner without external tooling. package main import ( "bufio" _ "embed" "flag" "fmt" "log" "path/filepath" "runtime" "sort" "strings" "github.com/pkg/errors" "github.com/kopia/kopia/tools/gettool/autodownload" ) // ToolInfo encapsulates all information required to download a tool. type ToolInfo struct { urlTemplate string osMap map[string]string archMap map[string]string stripPathComponents int unsupportedArch map[string]bool unsupportedOSArch map[string]bool } func (ti ToolInfo) actualURL(version, goos, goarch string) string { if ti.unsupportedArch[goarch] { return "" } if ti.unsupportedOSArch[goos+"/"+goarch] { return "" } u := ti.urlTemplate u = strings.ReplaceAll(u, "VERSION", version) u = strings.ReplaceAll(u, "GOARCH", replacementFromMap(goarch, ti.archMap)) u = strings.ReplaceAll(u, "GOOS", replacementFromMap(goos, ti.osMap)) u = strings.ReplaceAll(u, "EXT", replacementFromMap(goos, map[string]string{ "windows": "zip", "linux": "tar.gz", "darwin": "tar.gz", })) return u } var tools = map[string]ToolInfo{ "linter": { urlTemplate: "https://github.com/golangci/golangci-lint/releases/download/vVERSION/golangci-lint-VERSION-GOOS-GOARCH.EXT", archMap: map[string]string{ "arm": "armv6", }, stripPathComponents: 1, }, "hugo": { urlTemplate: "https://github.com/gohugoio/hugo/releases/download/vVERSION/hugo_extended_VERSION_GOOS-GOARCH.EXT", archMap: map[string]string{ "amd64": "64bit", "arm": "ARM", "arm64": "ARM64", }, osMap: map[string]string{ "linux": "Linux", "darwin": "macOS", }, unsupportedArch: map[string]bool{ "arm": true, }, unsupportedOSArch: map[string]bool{ "linux/arm64": true, }, }, "gotestsum": { urlTemplate: "https://github.com/gotestyourself/gotestsum/releases/download/vVERSION/gotestsum_VERSION_GOOS_GOARCH.tar.gz", archMap: map[string]string{ "arm": "armv6", }, }, "kopia": { urlTemplate: "https://github.com/kopia/kopia/releases/download/vVERSION/kopia-VERSION-GOOS-GOARCH.EXT", archMap: map[string]string{ "amd64": "x64", }, osMap: map[string]string{ "darwin": "macOS", }, stripPathComponents: 1, }, "rclone": { urlTemplate: "https://github.com/rclone/rclone/releases/download/vVERSION/rclone-vVERSION-GOOS-GOARCH.zip", osMap: map[string]string{"darwin": "osx"}, stripPathComponents: 1, }, "goreleaser": { urlTemplate: "https://github.com/goreleaser/goreleaser/releases/download/VERSION/goreleaser_GOOS_GOARCH.EXT", archMap: map[string]string{ "amd64": "x86_64", "arm": "armv6", }, osMap: map[string]string{ "darwin": "Darwin", "linux": "Linux", "windows": "Windows", }, }, "node": { urlTemplate: "https://nodejs.org/dist/vVERSION/node-vVERSION-GOOS-GOARCH.EXT", osMap: map[string]string{"windows": "win"}, archMap: map[string]string{"arm": "armv7l", "amd64": "x64"}, stripPathComponents: 1, }, } var ( tool = flag.String("tool", "", "Name of the tool:version") outputDir = flag.String("output-dir", "", "Output directory") testAll = flag.Bool("test-all", false, "Unpacks the package for all GOOS/ARCH combinations") regenerateChecksums = flag.Bool("regenerate-checksums", false, "Regenerate checksums") ) var buildArchitectures = []struct { goos string goarch string }{ {"linux", "amd64"}, {"linux", "arm64"}, {"linux", "arm"}, {"darwin", "amd64"}, {"darwin", "arm64"}, {"windows", "amd64"}, } func replacementFromMap(defaultValue string, m map[string]string) string { if v, ok := m[defaultValue]; ok { return v } return defaultValue } //go:embed checksums.txt var checksumsFileContents string func parseEmbeddedChecksums() map[string]string { m := map[string]string{} s := bufio.NewScanner(strings.NewReader(checksumsFileContents)) for s.Scan() { p := strings.Split(s.Text(), ": ") m[p[0]] = p[1] } return m } func main() { flag.Parse() if *outputDir == "" { log.Fatalf("--output-dir must be set") } checksums := parseEmbeddedChecksums() var errorCount int for _, toolNameVersion := range strings.Split(*tool, ",") { parts := strings.Split(toolNameVersion, ":") // nolint:gomnd if len(parts) != 2 { log.Fatalf("invalid tool spec, must be tool:version[,tool:version]") } toolName := parts[0] toolVersion := parts[1] if err := downloadTool(toolName, toolVersion, checksums, &errorCount); err != nil { log.Fatalf("unable to download %v version %v: %v", toolName, toolVersion, err) } } // all good if errorCount == 0 && !*regenerateChecksums { return } // on failure print current checksums, so they can be copy/pasted as the new baseline var lines []string for k, v := range checksums { lines = append(lines, fmt.Sprintf("%v: %v", k, v)) } sort.Strings(lines) for _, l := range lines { fmt.Println(l) } if *regenerateChecksums { return } log.Fatalf("Error(s) encountered, see log messages above.") } func downloadTool(toolName, toolVersion string, checksums map[string]string, errorCount *int) error { t, ok := tools[toolName] if !ok { return errors.Errorf("unsupported tool: %q", toolName) } if *testAll { for _, ba := range buildArchitectures { u := t.actualURL(toolVersion, ba.goos, ba.goarch) if u == "" { continue } if err := autodownload.Download(u, filepath.Join(*outputDir, ba.goos, ba.goarch), checksums, t.stripPathComponents); err != nil { log.Printf("ERROR %v: %v", u, err) *errorCount++ } } return nil } if *regenerateChecksums { for _, ba := range buildArchitectures { u := t.actualURL(toolVersion, ba.goos, ba.goarch) if u == "" { continue } if checksums[u] != "" { continue } log.Printf("downloading %v...", u) if err := autodownload.Download(u, filepath.Join(*outputDir, ba.goos, ba.goarch), checksums, t.stripPathComponents); err != nil { log.Printf("ERROR %v: %v", u, err) *errorCount++ } } return nil } u := t.actualURL(toolVersion, runtime.GOOS, runtime.GOARCH) if u == "" { log.Fatalf("Tool '%v' is not supported on %v/%v", toolName, runtime.GOOS, runtime.GOARCH) } fmt.Printf("Downloading %v version %v from %v...\n", toolName, toolVersion, u) if err := autodownload.Download(u, *outputDir, checksums, t.stripPathComponents); err != nil { return errors.Wrap(err, "unable to download") } return nil }