feat(release): build NetBSD x64 binaries natively in a VM (#17830)
Build and publish the CLI for NetBSD x64. The cross images cannot link pnpm for x86_64-unknown-netbsd: their NetBSD 9 sysroot has no libexecinfo, which Rust's std links, and no getentropy, which aws-lc calls. NetBSD 9 is end-of-life, and a native build on NetBSD 10.2 needs no workaround. The NetBSD entry of the CLI release build matrix runs on a GitHub-hosted runner, which has KVM. A composite action installs NetBSD 10.2 with anita from the official release files and boots it, and .github/scripts/netbsd-vm/vm.sh builds inside it and copies the artifacts back to where the existing archive, attest, and upload steps expect them. The release files are checked against a committed SHA512 list, and anita and rustup-init against pinned SHA-256 hashes. A separate workflow runs the same build, plus an install, on changes to the VM files and weekly, since only a release tag runs the release jobs. --------- Co-authored-by: Alessio Attilio <attilio.alessio@protonmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
13 files changed
+324
-1
No files matched your search
@@ -0,0 +1,39 @@
|
||||
"""Log in on a fresh anita-installed NetBSD VM's serial console and start sshd.
|
||||
|
||||
Usage: enable-ssh.py <serial-port> <authorized-key-file>
|
||||
"""
|
||||
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
||||
import pexpect.fdpexpect
|
||||
|
||||
port = int(sys.argv[1])
|
||||
with open(sys.argv[2], encoding='utf-8') as key_file:
|
||||
authorized_key = key_file.read().strip()
|
||||
|
||||
deadline = time.monotonic() + 120
|
||||
while True:
|
||||
try:
|
||||
console = socket.create_connection(('127.0.0.1', port))
|
||||
break
|
||||
except OSError:
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
time.sleep(1)
|
||||
|
||||
session = pexpect.fdpexpect.fdspawn(console.fileno(), timeout=600, encoding='utf-8', logfile=sys.stdout)
|
||||
session.send('\r')
|
||||
session.expect('login:')
|
||||
session.sendline('root')
|
||||
session.expect('# ')
|
||||
for command in [
|
||||
'echo sshd=YES >> /etc/rc.conf && echo dhcpcd=YES >> /etc/rc.conf',
|
||||
'echo "PermitRootLogin prohibit-password" >> /etc/ssh/sshd_config',
|
||||
f'mkdir -p /root/.ssh && echo "{authorized_key}" > /root/.ssh/authorized_keys && chmod -R go= /root/.ssh',
|
||||
# The quotes keep the console's echo of this line from matching the marker.
|
||||
'/etc/rc.d/dhcpcd start && /etc/rc.d/sshd start && echo "SSH-""READY"',
|
||||
]:
|
||||
session.sendline(command)
|
||||
session.expect('SSH-READY')
|
||||
@@ -0,0 +1,12 @@
|
||||
03e0876550f050776349cf487f73f313c9bad634bec1bcbf22ef0b297d1af61db587a3f08693adad2936adfa3663c6510e7d12a46fb806652cf904eb33c490be amd64/binary/sets/base.tar.xz
|
||||
9163f96f57e98e8de219034a5c74f5e3502385a8168f4da667408f98921f8bf4e4919775ccd06e3ea33ed93149c62b01f724f2d81bcf7e057ac0d8bfc5a4d51b amd64/binary/sets/comp.tar.xz
|
||||
1dd44ea51b3e98162d089ea3edaf7719b4243c717c3d132bdd361c233156ec5c72d0efc36448175210bd0e2fc269a54773c8769cd3859cfdb7eb4112827dda27 amd64/binary/sets/etc.tar.xz
|
||||
6a237a8afe29b19ea896d2eaf7c8a066e2f8350e1dd44d56fce9c5b086a27e0f56f341a7b39678442bb35bc680385f688940c8bde3b2010fbdf6a6ecba9a2150 amd64/binary/sets/gpufw.tar.xz
|
||||
2d526a976ef34bc152d6cbe2020dc6a54ada98e599337623616bae52d22828f1b9ef9d10533b05232dba3dc41af2810b1b2ead8eecb1455575068f8124579e1e amd64/binary/sets/kern-GENERIC.tar.xz
|
||||
025a2cb4807fa1d0985015accecfff7d640a52d0f9a0c1bad8be549fe0da11279a3b9d3bf51781a5a1101bc0c6557c2ae2ea2cfc8f2983788e70e5f94fd1d706 amd64/binary/sets/misc.tar.xz
|
||||
8a447f4ccceec4bc16e2b0c6b940801d9a37397df3af1b977c22dedae4fd071f277f98929b8b549885fc2d91bf145550264b83bb41b59876b80f7602d8298825 amd64/binary/sets/modules.tar.xz
|
||||
5553ac509ad08d5c4f6eecf143459110109dfc85242ebf38942b95350e938e1f6e39c886b2d99ffe7ea57b878b8d2a4b177da271c4d3cf143fff2952e99e1475 amd64/binary/sets/rescue.tar.xz
|
||||
f16997d2340ce115960bb694f35118b237afdf3e548eabd8028e93f36f4fbb658bfcd201cb7fb029e3a2117b5485c029084ad818d36d27e0168e8adfeaf6327a amd64/binary/sets/tests.tar.xz
|
||||
0cf9b3e0a0464ab994cb1430439cbde4de42baffcd613312ebf0cd058815841a3f7073cac9578dadc8646e286a3d3cec886bdcabbb50ee3f3651ccd591e73f96 amd64/binary/kernel/netbsd-INSTALL.gz
|
||||
ec87ab091d0b562c841234f17040190192d533479c707a51e65bf31914a5634233276f23027fd0a6f98dfbabfb8744196e2c23872209c123d29af8abfbd7cff5 amd64/installation/cdrom/boot-com.iso
|
||||
9af9403e4aa64649be3a8902bf043556ab3097e146a9ac92bab59d51eb3a529d91e8089cc1b7207266a4d9dafe52877e67947fe5c6661d70d17d86a9f7119585 amd64/installation/misc/pxeboot_ia32.bin
|
||||
Executable
+142
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env bash
|
||||
# Builds release artifacts natively in a NetBSD VM that qemu runs with KVM.
|
||||
#
|
||||
# The `cross` NetBSD images cannot link pnpm: their NetBSD 9 sysroot has no
|
||||
# libexecinfo (which Rust's std links) and no getentropy (which aws-lc calls).
|
||||
# A native build on NetBSD 10 needs neither workaround.
|
||||
#
|
||||
# Every download is checked against a hash committed here: the NetBSD release
|
||||
# files against release.sha512, anita and rustup-init against the constants
|
||||
# below. The binaries built in the VM are attested and published.
|
||||
#
|
||||
# Usage:
|
||||
# vm.sh start install NetBSD, boot it with this checkout attached, install Rust
|
||||
# vm.sh build <args> run `cargo build <args>` in the VM and copy the
|
||||
# top-level files of target/<triple>/<profile> back here
|
||||
# vm.sh run <command> run a shell command in the VM
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
NETBSD_RELEASE=10.2
|
||||
ANITA_VERSION=2.18
|
||||
ANITA_SHA256=fa5ac3a8b3e30a3a35df9254bbee213b28210a396544592c3e604a1e53a294ce
|
||||
RUSTUP_VERSION=1.29.3
|
||||
RUSTUP_SHA256=b3b81e3e737bc9465acfa5872c9c0afee7c6430bceaa52c038fbd9f3c327b51d
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
repo_root=$(git -C "$script_dir" rev-parse --show-toplevel)
|
||||
vm_dir=${RUNNER_TEMP:-/tmp}/netbsd-vm
|
||||
ssh_port=2222
|
||||
serial_port=4444
|
||||
|
||||
# The VM only listens on the runner's loopback and is created fresh by `start`,
|
||||
# so there is no host key worth pinning.
|
||||
vm_ssh() {
|
||||
ssh -i "$vm_dir/key" -p "$ssh_port" \
|
||||
-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
-o LogLevel=ERROR -o ServerAliveInterval=30 \
|
||||
root@127.0.0.1 "$@"
|
||||
}
|
||||
|
||||
download() {
|
||||
curl --fail --silent --show-error --location --retry 3 --output "$2" "$1"
|
||||
}
|
||||
|
||||
install_netbsd() {
|
||||
local release_dir="$vm_dir/release"
|
||||
local path
|
||||
while read -r _ path; do
|
||||
mkdir -p "$release_dir/$(dirname "$path")"
|
||||
download "https://cdn.netbsd.org/pub/NetBSD/NetBSD-$NETBSD_RELEASE/$path" "$release_dir/$path"
|
||||
done < "$script_dir/release.sha512"
|
||||
(cd "$release_dir" && sha512sum --check --quiet "$script_dir/release.sha512")
|
||||
|
||||
download "https://www.gson.org/netbsd/anita/download/anita-$ANITA_VERSION.tar.gz" "$vm_dir/anita.tar.gz"
|
||||
echo "$ANITA_SHA256 $vm_dir/anita.tar.gz" | sha256sum --check --quiet
|
||||
tar xzf "$vm_dir/anita.tar.gz" -C "$vm_dir"
|
||||
|
||||
PYTHONPATH="$vm_dir/anita-$ANITA_VERSION" python3 "$vm_dir/anita-$ANITA_VERSION/anita" \
|
||||
--workdir "$vm_dir/work" --disk-size 40G --memory-size 4G \
|
||||
--vmm-args "-enable-kvm -cpu host" \
|
||||
install "$release_dir/amd64/"
|
||||
}
|
||||
|
||||
pack_source() {
|
||||
# `.pnpm/crates` links into the pnpm store, so it is packed with the links
|
||||
# followed: the VM builds offline from it.
|
||||
git -C "$repo_root" ls-files -z | tar -C "$repo_root" --null -T - -cf "$vm_dir/source.tar"
|
||||
tar -C "$repo_root" -rhf "$vm_dir/source.tar" .pnpm/crates
|
||||
}
|
||||
|
||||
boot() {
|
||||
local memory_mb
|
||||
memory_mb=$(( $(awk '/^MemTotal:/ { print $2 }' /proc/meminfo) * 3 / 4 / 1024 ))
|
||||
# The source travels as a raw second disk, which the VM reads with tar.
|
||||
qemu-system-x86_64 -enable-kvm -cpu host -smp "$(nproc)" -m "${memory_mb}M" \
|
||||
-drive "file=$vm_dir/work/wd0.img,format=raw,media=disk" \
|
||||
-drive "file=$vm_dir/source.tar,format=raw,media=disk,snapshot=on" \
|
||||
-nic "user,model=e1000,hostfwd=tcp:127.0.0.1:$ssh_port-:22" \
|
||||
-display none -serial "tcp:127.0.0.1:$serial_port,server,nowait" -daemonize
|
||||
|
||||
ssh-keygen -q -t ed25519 -N '' -f "$vm_dir/key"
|
||||
python3 "$script_dir/enable-ssh.py" "$serial_port" "$vm_dir/key.pub"
|
||||
for _ in $(seq 1 30); do
|
||||
vm_ssh true && return 0
|
||||
sleep 2
|
||||
done
|
||||
echo "::error::the NetBSD VM did not accept SSH connections" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
install_rust() {
|
||||
download "https://static.rust-lang.org/rustup/archive/$RUSTUP_VERSION/x86_64-unknown-netbsd/rustup-init" "$vm_dir/rustup-init"
|
||||
echo "$RUSTUP_SHA256 $vm_dir/rustup-init" | sha256sum --check --quiet
|
||||
vm_ssh 'cat > /tmp/rustup-init && chmod +x /tmp/rustup-init' < "$vm_dir/rustup-init"
|
||||
|
||||
local channel
|
||||
channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' "$repo_root/rust-toolchain.toml")
|
||||
vm_ssh "mkdir /build && cd /build && tar xf /dev/rwd1d && /tmp/rustup-init -y --profile minimal --default-toolchain $channel"
|
||||
}
|
||||
|
||||
build() {
|
||||
local target='' profile=release
|
||||
local arg previous=''
|
||||
for arg in "$@"; do
|
||||
case "$previous" in
|
||||
--target) target=$arg ;;
|
||||
--profile) profile=$arg ;;
|
||||
esac
|
||||
case "$arg" in
|
||||
--target=*) target=${arg#--target=} ;;
|
||||
--profile=*) profile=${arg#--profile=} ;;
|
||||
esac
|
||||
previous=$arg
|
||||
done
|
||||
if [ -z "$target" ]; then
|
||||
echo "::error::vm.sh build needs --target, so the output lands where the release steps look for it" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
vm_ssh ". \$HOME/.cargo/env && cd /build && cargo build --offline $(printf '%q ' "$@")"
|
||||
local output_dir="target/$target/$profile"
|
||||
mkdir -p "$repo_root/$output_dir"
|
||||
vm_ssh "cd /build/$output_dir && tar cf - \$(find . -maxdepth 1 -type f)" | tar xf - -C "$repo_root/$output_dir"
|
||||
}
|
||||
|
||||
command=${1:-}
|
||||
shift || true
|
||||
case "$command" in
|
||||
start)
|
||||
mkdir -p "$vm_dir"
|
||||
install_netbsd
|
||||
pack_source
|
||||
boot
|
||||
install_rust
|
||||
;;
|
||||
build) build "$@" ;;
|
||||
run) vm_ssh "$@" ;;
|
||||
*)
|
||||
echo "usage: vm.sh start | build <cargo args> | run <command>" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
Reference in new issue
Block a user