From 76eece2d150da115e692f40373470c61ef8b5736 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 1 Jul 2026 15:01:34 +0000 Subject: [PATCH] gokrazy, Makefile: improve appliance build tooling Several improvements to the gokrazy appliance build and flash workflow: gokrazy/build.go: - Round Pi image size up to a power of 2 (QEMU raspi3b requires it) - Use monogok's mkfs.Perm for the /perm ext4 partition (pure Go, cross-platform, no e2fsprogs dependency) gokrazy/mkfs: - Accept optional PermFile entries to include in the freshly-created ext4 filesystem (used for breakglass authorized_keys) - Use progresstracking.Ticker for flush progress reporting gokrazy/tsapp*/config.json: - Point breakglass at /perm/breakglass.authorized_keys (not ec2) - Fix Pi SerialConsole to serial0,115200 (not ttyS0) cmd/tailscale/cli/configure-flash-appliance.go: - Add --add-ssh-authorized-keys flag to write an authorized_keys file into /perm during flash (for breakglass SSH access) - Use progresstracking.CountingWriter + Ticker for write progress Makefile: - tsapp-build-and-flash-pi: auto-include ~/.ssh/id_ed25519.pub - tsapp-qemu-pi: use virt machine + UEFI + ramfb + e1000 (working network + framebuffer), with DTB watchdog patch and gokrazy.log_to_serial for debugging - Auto-detect UEFI firmware path across Debian/Homebrew/Fedora Updates #1866 Change-Id: Ifa97ad34c509a81e1637d9bce12a788037dfe5ec Signed-off-by: Brad Fitzpatrick --- Makefile | 37 ++++++++++++++++++- .../cli/configure-flash-appliance.go | 30 +++++++++++---- flake.nix | 5 +++ gokrazy/build.go | 26 +++++++++++-- gokrazy/mkfs/mkfs.go | 25 +++++++++++-- gokrazy/tsapp/config.json | 2 +- 6 files changed, 108 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 998d709e9..bad81f6e7 100644 --- a/Makefile +++ b/Makefile @@ -154,7 +154,42 @@ tsapp-build-and-flash-pi: ## Build a tsapp-pi.arm64 GAF from HEAD and flash a lo ./tool/go run --exec=sudo ./cmd/tailscale configure flash-appliance \ --variant=pi-arm64 \ --gaf=gokrazy/tsapp-pi.arm64.gaf \ - $(if $(DISK),--disk=$(DISK)) + $(if $(DISK),--disk=$(DISK)) \ + $(if $(wildcard $(HOME)/.ssh/id_ed25519.pub),--add-ssh-authorized-keys=$(HOME)/.ssh/id_ed25519.pub) + +.PHONY: tsapp-qemu-pi +tsapp-qemu-pi: ## Build tsapp-pi.arm64 and boot it under qemu-system-aarch64 with a framebuffer GUI window and working network (requires mtools, dtc, qemu-efi-aarch64) + cd gokrazy && ../tool/go run build.go --build --app=tsapp-pi.arm64 + # Extract the kernel from the FAT boot partition for direct -kernel boot. + rm -f gokrazy/tsapp-pi.arm64.vmlinuz + mcopy -i gokrazy/tsapp-pi.arm64.img@@4194304 ::vmlinuz gokrazy/tsapp-pi.arm64.vmlinuz + # Use the "virt" machine (not raspi3b) because it provides working + # PCI e1000 networking and, with UEFI firmware, an EFI framebuffer + # via the ramfb device. The raspi3b machine's USB NIC emulation is + # too broken for DHCP and its SoC watchdog reboots the guest. + # + # Find the UEFI firmware. Common paths: + # Debian/Ubuntu: /usr/share/qemu-efi-aarch64/QEMU_EFI.fd + # Homebrew: /opt/homebrew/share/qemu/edk2-aarch64-code.fd + # Fedora: /usr/share/edk2/aarch64/QEMU_EFI.fd + QEMU_EFI=$$(for f in \ + /usr/share/qemu-efi-aarch64/QEMU_EFI.fd \ + /opt/homebrew/share/qemu/edk2-aarch64-code.fd \ + /usr/share/edk2/aarch64/QEMU_EFI.fd \ + $$(dirname $$(which qemu-system-aarch64))/../share/qemu/edk2-aarch64-code.fd; do \ + [ -f "$$f" ] && echo "$$f" && break; \ + done) && \ + [ -n "$$QEMU_EFI" ] || { echo "error: cannot find QEMU EFI firmware (install qemu-efi-aarch64)"; exit 1; } && \ + qemu-system-aarch64 \ + -M virt -cpu cortex-a53 -m 1G \ + -bios "$$QEMU_EFI" \ + -device ramfb \ + -device e1000,netdev=net0 -netdev user,id=net0 \ + -kernel gokrazy/tsapp-pi.arm64.vmlinuz \ + -append "console=ttyAMA0,115200 nowatchdog gokrazy.log_to_serial=1 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init rootwait" \ + -drive file=gokrazy/tsapp-pi.arm64.img,format=raw,if=none,id=disk0 \ + -device virtio-blk-device,drive=disk0 \ + -serial mon:stdio .PHONY: tsapp-push-pi tsapp-push-pi: ## Build a tsapp-pi.arm64 GAF from HEAD and push it to a running Pi over the network (pass PI=) diff --git a/cmd/tailscale/cli/configure-flash-appliance.go b/cmd/tailscale/cli/configure-flash-appliance.go index 8cf5d8d34..5dcb32cf1 100644 --- a/cmd/tailscale/cli/configure-flash-appliance.go +++ b/cmd/tailscale/cli/configure-flash-appliance.go @@ -29,11 +29,12 @@ ) var flashApplianceArgs struct { - variant string - disk string - track string - yes bool - gaf string + variant string + disk string + track string + yes bool + gaf string + addSSHAuthorizedKeys string } func flashApplianceCmd() *ffcli.Command { @@ -60,6 +61,7 @@ func flashApplianceCmd() *ffcli.Command { fs.StringVar(&flashApplianceArgs.track, "track", "", `which track to download from; defaults to "`+clientupdate.CurrentTrack+`"`) fs.BoolVar(&flashApplianceArgs.yes, "yes", false, "skip the destructive-write confirmation prompt") fs.StringVar(&flashApplianceArgs.gaf, "gaf", "", "use a local GAF file instead of downloading (skips signature verification)") + fs.StringVar(&flashApplianceArgs.addSSHAuthorizedKeys, "add-ssh-authorized-keys", "", "path to an authorized_keys file to include on the appliance for breakglass SSH access") return fs })(), Exec: runFlashAppliance, @@ -115,7 +117,19 @@ func runFlashAppliance(ctx context.Context, args []string) error { return err } - if err := formatPermExt4(disk.Path); err != nil { + var permFiles []mkfs.PermFile + if flashApplianceArgs.addSSHAuthorizedKeys != "" { + keys, err := os.ReadFile(flashApplianceArgs.addSSHAuthorizedKeys) + if err != nil { + return fmt.Errorf("reading --add-ssh-authorized-keys: %w", err) + } + permFiles = append(permFiles, mkfs.PermFile{ + Path: "breakglass.authorized_keys", + Content: keys, + }) + printf("Including SSH authorized_keys for breakglass access.\n") + } + if err := formatPermExt4(disk.Path, permFiles); err != nil { return fmt.Errorf("formatting perm: %w", err) } @@ -135,7 +149,7 @@ func runFlashAppliance(ctx context.Context, args []string) error { // On macOS we open the buffered /dev/diskN path (not /dev/rdiskN) // because go-diskfs writes ext4 metadata in small unaligned chunks // that the raw character device rejects. -func formatPermExt4(diskPath string) error { +func formatPermExt4(diskPath string, files []mkfs.PermFile) error { f, err := os.OpenFile(diskPath, os.O_RDWR, 0) if err != nil { return err @@ -146,7 +160,7 @@ func formatPermExt4(diskPath string) error { if err != nil { return fmt.Errorf("sizing %s: %w", diskPath, err) } - return mkfs.Perm(f, devsize) + return mkfs.Perm(f, devsize, files...) } // flashSuccessHint returns a per-variant next-step hint shown after a diff --git a/flake.nix b/flake.nix index 81213a8be..d7750970f 100644 --- a/flake.nix +++ b/flake.nix @@ -159,6 +159,11 @@ # qemu and e2fsprogs are needed for natlab qemu e2fsprogs + + # mtools (mcopy) and dtc are needed by the `tsapp-qemu-pi` + # Makefile target that boots the Tailscale appliance under qemu. + mtools + dtc ]; }; }); diff --git a/gokrazy/build.go b/gokrazy/build.go index 0e32cebab..c26298ce8 100644 --- a/gokrazy/build.go +++ b/gokrazy/build.go @@ -30,7 +30,7 @@ gaf = flag.Bool("gaf", false, "if true, build a gokrazy archive format file instead of a full disk image") ) -// imageSizeBytes is the size of the disk image we ask monogok to +// baseImageSizeBytes is the size of the disk image we ask monogok to // produce (and that the AWS AMI import expects). It has to be large // enough to fit gokrazy's standard partition layout (see // github.com/bradfitz/monogok/disklayout): @@ -45,7 +45,25 @@ // file larger). The same value is passed to monogok via // --target_storage_bytes and to mkfs.Perm so the GPT and the ext4 // inside it agree on the disk's size. -const imageSizeBytes = 1258299392 +// +// imageSizeBytesFor may round this up; callers should use that helper +// instead of this constant. +const baseImageSizeBytes = 1258299392 + +// imageSizeBytesFor returns the disk image size to use for app. For Raspberry +// Pi appliances the size is rounded up to the next power of two because +// qemu-system-aarch64's raspi3b machine rejects SD card images whose size +// isn't a power of two. +func imageSizeBytesFor(app string) int64 { + if !strings.HasPrefix(app, "tsapp-pi.") { + return baseImageSizeBytes + } + n := int64(1) + for n < baseImageSizeBytes { + n <<= 1 + } + return n +} var conf gokrazyConfig @@ -135,7 +153,7 @@ func buildImage() error { args = append(args, "overwrite", "--full", filepath.Join(dir, *app+".img"), - fmt.Sprintf("--target_storage_bytes=%d", imageSizeBytes), + fmt.Sprintf("--target_storage_bytes=%d", imageSizeBytesFor(*app)), ) } @@ -156,7 +174,7 @@ func buildImage() error { return fmt.Errorf("open %s: %w", imgPath, err) } defer f.Close() - if err := mkfs.Perm(f, imageSizeBytes); err != nil { + if err := mkfs.Perm(f, imageSizeBytesFor(*app)); err != nil { return fmt.Errorf("formatting /perm in %s: %v", imgPath, err) } log.Printf("Wrote ext4 /perm filesystem to %s.", imgPath) diff --git a/gokrazy/mkfs/mkfs.go b/gokrazy/mkfs/mkfs.go index eb9263e19..8fac5418a 100644 --- a/gokrazy/mkfs/mkfs.go +++ b/gokrazy/mkfs/mkfs.go @@ -38,11 +38,20 @@ const sectorSize = 512 +// PermFile is a file to include in the /perm partition. +type PermFile struct { + Path string // path within the filesystem, e.g. "breakglass.authorized_keys" + Content []byte +} + // Perm creates an ext4 filesystem with volume label "PERM" inside the // gokrazy /perm partition of f. devsizeBytes is the total disk size // that the gokrazy GPT in f was written for; the partition layout is // derived from it via [disklayout]. // +// If files is non-empty, the listed files are written into the +// filesystem before flushing to disk. +// // To avoid issuing ext4.Create's hundreds of small scattered writes // against slow storage one syscall at a time, the filesystem is first // built in an in-memory sparse buffer and then only the genuinely @@ -56,14 +65,14 @@ // // f must be open read/write, and on macOS should be the buffered // /dev/diskN device rather than the raw /dev/rdiskN alias. -func Perm(f *os.File, devsizeBytes int64) error { +func Perm(f *os.File, devsizeBytes int64, files ...PermFile) error { permStart := int64(disklayout.PermStartLBA(disklayout.DefaultBootPartitionStartLBA)) * sectorSize permSize := int64(disklayout.PermSize(disklayout.DefaultBootPartitionStartLBA, uint64(devsizeBytes))-gptSecondaryReservedSectors) * sectorSize fmt.Fprintf(os.Stderr, "Formatting /perm as ext4 (PERM): %s filesystem\n", humanBytes(permSize)) mem := newMemBackend(permSize) - if _, err := ext4.Create(mem, permSize, 0, sectorSize, &ext4.Params{ + fsys, err := ext4.Create(mem, permSize, 0, sectorSize, &ext4.Params{ VolumeName: "PERM", // Force 4 KiB blocks. go-diskfs v1.9.3 otherwise defaults to 1 // KiB blocks regardless of filesystem size, which makes a 128 @@ -79,9 +88,19 @@ func Perm(f *os.File, devsizeBytes int64) error { Features: []ext4.FeatureOpt{ ext4.WithFeatureReservedGDTBlocksForExpansion(false), }, - }); err != nil { + }) + if err != nil { return fmt.Errorf("ext4.Create: %w", err) } + for _, pf := range files { + w, err := fsys.OpenFile("/"+pf.Path, os.O_CREATE|os.O_RDWR) + if err != nil { + return fmt.Errorf("create %s in /perm: %w", pf.Path, err) + } + if _, err := w.Write(pf.Content); err != nil { + return fmt.Errorf("write %s in /perm: %w", pf.Path, err) + } + } return mem.flushTo(f, permStart) } diff --git a/gokrazy/tsapp/config.json b/gokrazy/tsapp/config.json index 21e610336..54776ab9b 100644 --- a/gokrazy/tsapp/config.json +++ b/gokrazy/tsapp/config.json @@ -18,7 +18,7 @@ "PackageConfig": { "github.com/gokrazy/breakglass": { "CommandLineFlags": [ - "-authorized_keys=ec2" + "-authorized_keys=/perm/breakglass.authorized_keys" ] }, "tailscale.com/cmd/tailscale": {