gokrazy, Makefile: improve appliance build tooling

Several improvements to the gokrazy appliance build and flash workflow:

gokrazy/build.go:
  - Round Pi image size up to a power of 2 (QEMU raspi3b requires it)
  - Use monogok's mkfs.Perm for the /perm ext4 partition (pure Go,
    cross-platform, no e2fsprogs dependency)

gokrazy/mkfs:
  - Accept optional PermFile entries to include in the freshly-created
    ext4 filesystem (used for breakglass authorized_keys)
  - Use progresstracking.Ticker for flush progress reporting

gokrazy/tsapp*/config.json:
  - Point breakglass at /perm/breakglass.authorized_keys (not ec2)
  - Fix Pi SerialConsole to serial0,115200 (not ttyS0)

cmd/tailscale/cli/configure-flash-appliance.go:
  - Add --add-ssh-authorized-keys flag to write an authorized_keys
    file into /perm during flash (for breakglass SSH access)
  - Use progresstracking.CountingWriter + Ticker for write progress

Makefile:
  - tsapp-build-and-flash-pi: auto-include ~/.ssh/id_ed25519.pub
  - tsapp-qemu-pi: use virt machine + UEFI + ramfb + e1000 (working
    network + framebuffer), with DTB watchdog patch and
    gokrazy.log_to_serial for debugging
  - Auto-detect UEFI firmware path across Debian/Homebrew/Fedora

Updates #1866

Change-Id: Ifa97ad34c509a81e1637d9bce12a788037dfe5ec
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

Makefile

@@ -154,7 +154,42 @@ tsapp-build-and-flash-pi: ## Build a tsapp-pi.arm64 GAF from HEAD and flash a lo
 	./tool/go run --exec=sudo ./cmd/tailscale configure flash-appliance \
 		--variant=pi-arm64 \
 		--gaf=gokrazy/tsapp-pi.arm64.gaf \
-		$(if $(DISK),--disk=$(DISK))
+		$(if $(DISK),--disk=$(DISK)) \
+		$(if $(wildcard $(HOME)/.ssh/id_ed25519.pub),--add-ssh-authorized-keys=$(HOME)/.ssh/id_ed25519.pub)
+
+.PHONY: tsapp-qemu-pi
+tsapp-qemu-pi: ## Build tsapp-pi.arm64 and boot it under qemu-system-aarch64 with a framebuffer GUI window and working network (requires mtools, dtc, qemu-efi-aarch64)
+	cd gokrazy && ../tool/go run build.go --build --app=tsapp-pi.arm64
+	# Extract the kernel from the FAT boot partition for direct -kernel boot.
+	rm -f gokrazy/tsapp-pi.arm64.vmlinuz
+	mcopy -i gokrazy/tsapp-pi.arm64.img@@4194304 ::vmlinuz gokrazy/tsapp-pi.arm64.vmlinuz
+	# Use the "virt" machine (not raspi3b) because it provides working
+	# PCI e1000 networking and, with UEFI firmware, an EFI framebuffer
+	# via the ramfb device. The raspi3b machine's USB NIC emulation is
+	# too broken for DHCP and its SoC watchdog reboots the guest.
+	#
+	# Find the UEFI firmware. Common paths:
+	#   Debian/Ubuntu: /usr/share/qemu-efi-aarch64/QEMU_EFI.fd
+	#   Homebrew:      /opt/homebrew/share/qemu/edk2-aarch64-code.fd
+	#   Fedora:        /usr/share/edk2/aarch64/QEMU_EFI.fd
+	QEMU_EFI=$$(for f in \
+		/usr/share/qemu-efi-aarch64/QEMU_EFI.fd \
+		/opt/homebrew/share/qemu/edk2-aarch64-code.fd \
+		/usr/share/edk2/aarch64/QEMU_EFI.fd \
+		$$(dirname $$(which qemu-system-aarch64))/../share/qemu/edk2-aarch64-code.fd; do \
+		[ -f "$$f" ] && echo "$$f" && break; \
+	done) && \
+	[ -n "$$QEMU_EFI" ] || { echo "error: cannot find QEMU EFI firmware (install qemu-efi-aarch64)"; exit 1; } && \
+	qemu-system-aarch64 \
+		-M virt -cpu cortex-a53 -m 1G \
+		-bios "$$QEMU_EFI" \
+		-device ramfb \
+		-device e1000,netdev=net0 -netdev user,id=net0 \
+		-kernel gokrazy/tsapp-pi.arm64.vmlinuz \
+		-append "console=ttyAMA0,115200 nowatchdog gokrazy.log_to_serial=1 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init rootwait" \
+		-drive file=gokrazy/tsapp-pi.arm64.img,format=raw,if=none,id=disk0 \
+		-device virtio-blk-device,drive=disk0 \
+		-serial mon:stdio
 
 .PHONY: tsapp-push-pi
 tsapp-push-pi: ## Build a tsapp-pi.arm64 GAF from HEAD and push it to a running Pi over the network (pass PI=<ip>)

cmd/tailscale/cli/configure-flash-appliance.go

@@ -29,11 +29,12 @@
 )
 
 var flashApplianceArgs struct {
-	variant string
-	disk    string
-	track   string
-	yes     bool
-	gaf     string
+	variant              string
+	disk                 string
+	track                string
+	yes                  bool
+	gaf                  string
+	addSSHAuthorizedKeys string
 }
 
 func flashApplianceCmd() *ffcli.Command {
@@ -60,6 +61,7 @@ func flashApplianceCmd() *ffcli.Command {
 			fs.StringVar(&flashApplianceArgs.track, "track", "", `which track to download from; defaults to "`+clientupdate.CurrentTrack+`"`)
 			fs.BoolVar(&flashApplianceArgs.yes, "yes", false, "skip the destructive-write confirmation prompt")
 			fs.StringVar(&flashApplianceArgs.gaf, "gaf", "", "use a local GAF file instead of downloading (skips signature verification)")
+			fs.StringVar(&flashApplianceArgs.addSSHAuthorizedKeys, "add-ssh-authorized-keys", "", "path to an authorized_keys file to include on the appliance for breakglass SSH access")
 			return fs
 		})(),
 		Exec: runFlashAppliance,
@@ -115,7 +117,19 @@ func runFlashAppliance(ctx context.Context, args []string) error {
 		return err
 	}
 
-	if err := formatPermExt4(disk.Path); err != nil {
+	var permFiles []mkfs.PermFile
+	if flashApplianceArgs.addSSHAuthorizedKeys != "" {
+		keys, err := os.ReadFile(flashApplianceArgs.addSSHAuthorizedKeys)
+		if err != nil {
+			return fmt.Errorf("reading --add-ssh-authorized-keys: %w", err)
+		}
+		permFiles = append(permFiles, mkfs.PermFile{
+			Path:    "breakglass.authorized_keys",
+			Content: keys,
+		})
+		printf("Including SSH authorized_keys for breakglass access.\n")
+	}
+	if err := formatPermExt4(disk.Path, permFiles); err != nil {
 		return fmt.Errorf("formatting perm: %w", err)
 	}
 
@@ -135,7 +149,7 @@ func runFlashAppliance(ctx context.Context, args []string) error {
 // On macOS we open the buffered /dev/diskN path (not /dev/rdiskN)
 // because go-diskfs writes ext4 metadata in small unaligned chunks
 // that the raw character device rejects.
-func formatPermExt4(diskPath string) error {
+func formatPermExt4(diskPath string, files []mkfs.PermFile) error {
 	f, err := os.OpenFile(diskPath, os.O_RDWR, 0)
 	if err != nil {
 		return err
@@ -146,7 +160,7 @@ func formatPermExt4(diskPath string) error {
 	if err != nil {
 		return fmt.Errorf("sizing %s: %w", diskPath, err)
 	}
-	return mkfs.Perm(f, devsize)
+	return mkfs.Perm(f, devsize, files...)
 }
 
 // flashSuccessHint returns a per-variant next-step hint shown after a

flake.nix

@@ -159,6 +159,11 @@
           # qemu and e2fsprogs are needed for natlab
           qemu
           e2fsprogs
+
+          # mtools (mcopy) and dtc are needed by the `tsapp-qemu-pi`
+          # Makefile target that boots the Tailscale appliance under qemu.
+          mtools
+          dtc
         ];
       };
     });

gokrazy/build.go

@@ -30,7 +30,7 @@
 	gaf    = flag.Bool("gaf", false, "if true, build a gokrazy archive format file instead of a full disk image")
 )
 
-// imageSizeBytes is the size of the disk image we ask monogok to
+// baseImageSizeBytes is the size of the disk image we ask monogok to
 // produce (and that the AWS AMI import expects). It has to be large
 // enough to fit gokrazy's standard partition layout (see
 // github.com/bradfitz/monogok/disklayout):
@@ -45,7 +45,25 @@
 // file larger). The same value is passed to monogok via
 // --target_storage_bytes and to mkfs.Perm so the GPT and the ext4
 // inside it agree on the disk's size.
-const imageSizeBytes = 1258299392
+//
+// imageSizeBytesFor may round this up; callers should use that helper
+// instead of this constant.
+const baseImageSizeBytes = 1258299392
+
+// imageSizeBytesFor returns the disk image size to use for app. For Raspberry
+// Pi appliances the size is rounded up to the next power of two because
+// qemu-system-aarch64's raspi3b machine rejects SD card images whose size
+// isn't a power of two.
+func imageSizeBytesFor(app string) int64 {
+	if !strings.HasPrefix(app, "tsapp-pi.") {
+		return baseImageSizeBytes
+	}
+	n := int64(1)
+	for n < baseImageSizeBytes {
+		n <<= 1
+	}
+	return n
+}
 
 var conf gokrazyConfig
 
@@ -135,7 +153,7 @@ func buildImage() error {
 		args = append(args,
 			"overwrite",
 			"--full", filepath.Join(dir, *app+".img"),
-			fmt.Sprintf("--target_storage_bytes=%d", imageSizeBytes),
+			fmt.Sprintf("--target_storage_bytes=%d", imageSizeBytesFor(*app)),
 		)
 	}
 
@@ -156,7 +174,7 @@ func buildImage() error {
 		return fmt.Errorf("open %s: %w", imgPath, err)
 	}
 	defer f.Close()
-	if err := mkfs.Perm(f, imageSizeBytes); err != nil {
+	if err := mkfs.Perm(f, imageSizeBytesFor(*app)); err != nil {
 		return fmt.Errorf("formatting /perm in %s: %v", imgPath, err)
 	}
 	log.Printf("Wrote ext4 /perm filesystem to %s.", imgPath)

gokrazy/mkfs/mkfs.go

@@ -38,11 +38,20 @@
 
 const sectorSize = 512
 
+// PermFile is a file to include in the /perm partition.
+type PermFile struct {
+	Path    string // path within the filesystem, e.g. "breakglass.authorized_keys"
+	Content []byte
+}
+
 // Perm creates an ext4 filesystem with volume label "PERM" inside the
 // gokrazy /perm partition of f. devsizeBytes is the total disk size
 // that the gokrazy GPT in f was written for; the partition layout is
 // derived from it via [disklayout].
 //
+// If files is non-empty, the listed files are written into the
+// filesystem before flushing to disk.
+//
 // To avoid issuing ext4.Create's hundreds of small scattered writes
 // against slow storage one syscall at a time, the filesystem is first
 // built in an in-memory sparse buffer and then only the genuinely
@@ -56,14 +65,14 @@
 //
 // f must be open read/write, and on macOS should be the buffered
 // /dev/diskN device rather than the raw /dev/rdiskN alias.
-func Perm(f *os.File, devsizeBytes int64) error {
+func Perm(f *os.File, devsizeBytes int64, files ...PermFile) error {
 	permStart := int64(disklayout.PermStartLBA(disklayout.DefaultBootPartitionStartLBA)) * sectorSize
 	permSize := int64(disklayout.PermSize(disklayout.DefaultBootPartitionStartLBA, uint64(devsizeBytes))-gptSecondaryReservedSectors) * sectorSize
 
 	fmt.Fprintf(os.Stderr, "Formatting /perm as ext4 (PERM): %s filesystem\n", humanBytes(permSize))
 
 	mem := newMemBackend(permSize)
-	if _, err := ext4.Create(mem, permSize, 0, sectorSize, &ext4.Params{
+	fsys, err := ext4.Create(mem, permSize, 0, sectorSize, &ext4.Params{
 		VolumeName: "PERM",
 		// Force 4 KiB blocks. go-diskfs v1.9.3 otherwise defaults to 1
 		// KiB blocks regardless of filesystem size, which makes a 128
@@ -79,9 +88,19 @@ func Perm(f *os.File, devsizeBytes int64) error {
 		Features: []ext4.FeatureOpt{
 			ext4.WithFeatureReservedGDTBlocksForExpansion(false),
 		},
-	}); err != nil {
+	})
+	if err != nil {
 		return fmt.Errorf("ext4.Create: %w", err)
 	}
+	for _, pf := range files {
+		w, err := fsys.OpenFile("/"+pf.Path, os.O_CREATE|os.O_RDWR)
+		if err != nil {
+			return fmt.Errorf("create %s in /perm: %w", pf.Path, err)
+		}
+		if _, err := w.Write(pf.Content); err != nil {
+			return fmt.Errorf("write %s in /perm: %w", pf.Path, err)
+		}
+	}
 	return mem.flushTo(f, permStart)
 }
 

gokrazy/tsapp/config.json

@@ -18,7 +18,7 @@
     "PackageConfig": {
         "github.com/gokrazy/breakglass": {
             "CommandLineFlags": [
-                "-authorized_keys=ec2"
+                "-authorized_keys=/perm/breakglass.authorized_keys"
             ]
         },
         "tailscale.com/cmd/tailscale": {