diff --git a/build.sh b/build.sh index d0c390d..cca872c 100755 --- a/build.sh +++ b/build.sh @@ -49,6 +49,8 @@ DEBUG_TAR=${OUTPUT}_debug-x86-64.tar # Output debug archive path (.zst will be a ROOTFS_CAIBX=${OUTPUT}_root-x86-64.caibx ROOTFS_EROFS=${OUTPUT}_root-x86-64.erofs # Output erofs image path IMG=${OUTPUT}.raw # Output raw image path +ISO=${OUTPUT}.iso +ISO_STAGING="kde-linux.cache/iso-staging" EFI_BASE=kde-linux_${VERSION} # Base name of the UKI in the image's ESP (exported so it can be used in basic-test-efi-addon.sh) EFI=${EFI_BASE}+3.efi # Name of primary UKI in the image's ESP @@ -178,6 +180,58 @@ cp --reflink=auto "$ROOTFS_EROFS" kde-linux.cache/root.raw touch "$IMG" systemd-repart --no-pager --empty=allow --size=auto --dry-run=no --root=kde-linux.cache --definitions=mkosi.repart "$IMG" +# Build ISO for live booting +echo "Building bootable ISO..." +# Clean and create ISO staging directory +rm -rf "$ISO_STAGING" +mkdir -p "$ISO_STAGING/live" "$ISO_STAGING/boot/EFI/Linux" + +# Copy the erofs root filesystem +cp "$ROOTFS_EROFS" "$ISO_STAGING/live/filesystem.erofs" + +# Use the ISO-specific UKI +cp "${OUTPUT}"/live-iso.efi "$ISO_STAGING/boot/EFI/Linux/BOOTX64.EFI" + +# Create ESP image for El Torito +ESP_IMG="kde-linux.cache/esp-iso.img" +fallocate -l 328M "$ESP_IMG" +mkfs.fat -F 32 "$ESP_IMG" + +# Mount and populate ESP +ESP_MNT="kde-linux.cache/esp-iso.mnt" +mkdir -p "$ESP_MNT" +mount "$ESP_IMG" "$ESP_MNT" +mkdir -p "$ESP_MNT/EFI/BOOT" +cp "$ISO_STAGING/boot/EFI/Linux/BOOTX64.EFI" "$ESP_MNT/EFI/BOOT/" +umount "$ESP_MNT" + +# Create ISO with xorriso +xorriso -as mkisofs \ + -o "$ISO" \ + -iso-level 3 \ + -full-iso9660-filenames \ + -joliet -joliet-long \ + -rational-rock \ + -volid "KDE_LINUX_${VERSION}" \ + -appid "KDE Linux Live ${VERSION}" \ + -publisher "KDE" \ + -preparer "kde-linux build system" \ + -e esp-iso.img \ + -no-emul-boot \ + -eltorito-platform efi \ + -isohybrid-gpt-basdat \ + "$ISO_STAGING" \ + -graft-points \ + "esp-iso.img=$ESP_IMG" + +# Make ISO readable +chmod go+r "$ISO" + +echo "ISO created: $ISO ($(du -h "$ISO" | cut -f1))" + +# Create torrent for ISO +./torrent-create.rb "$VERSION" "$OUTPUT" "$ISO" + # Incase the owner is root chown -R user:user mkosi.output diff --git a/mkosi.extra/usr/lib/initcpio/install/systemd-extension b/mkosi.extra/usr/lib/initcpio/install/systemd-extension index 80203dc..b969357 100644 --- a/mkosi.extra/usr/lib/initcpio/install/systemd-extension +++ b/mkosi.extra/usr/lib/initcpio/install/systemd-extension @@ -10,6 +10,7 @@ build() { /usr/lib/systemd/system-generators/00-kde-linux-os-release \ /usr/lib/systemd/system-generators/kde-linux-live-generator \ /usr/lib/systemd/system-generators/kde-linux-mount-generator \ + /usr/lib/systemd/system-generators/kde-linux-iso-mount-generator \ /usr/lib/systemd/systemd-bootchart \ /usr/lib/etc-factory \ /usr/bin/btrfs diff --git a/mkosi.extra/usr/lib/rebuild-efi b/mkosi.extra/usr/lib/rebuild-efi index cd8bb2f..b90e956 100755 --- a/mkosi.extra/usr/lib/rebuild-efi +++ b/mkosi.extra/usr/lib/rebuild-efi @@ -24,15 +24,34 @@ if ! dkms autoinstall -k "$kernel_version"; then exit 1 fi - # NOTE: plymouth MUST be after systemd as per the wiki! cat <<- EOF > mkinitcpio.conf -MODULES=(overlay) +MODULES=(overlay loop isofs udf sr_mod cdrom) BINARIES=() FILES=() HOOKS=(base systemd modconf kms keyboard block sd-encrypt filesystems fsck systemd-extension plymouth microcode sd-shutdown) EOF +# ISO Live UKI - boots from ISO media +echo "rw \ + systemd.volatile=overlay systemd.firstboot=false systemd.hostname=kde-linux kde-linux.live=1 kde-linux.iso=1 plasma.live.user=live \ + lsm=landlock,lockdown,yama,integrity,bpf \ + zswap.enabled=0 \ + preempt=full threadirqs \ + nohz=on nohz_full=all \ + rcu_nocbs=all rcutree.enable_rcu_lazy=1 \ + amdgpu.dcdebugmask=0x10 \ + vt.global_cursor_default=0 quiet splash loglevel=3 iso_label=KDE_LINUX_${IMAGE_VERSION}" > cmdline.iso + +mkinitcpio --config mkinitcpio.conf --generate initrd.iso --kernel "$kernel_version" + +ukify build \ + --linux /usr/lib/modules/$kernel_version/vmlinuz \ + --initrd initrd.iso \ + --cmdline @cmdline.iso \ + --output live-iso.efi + +# Raw Live UKI - boots from .raw disk image (USB/DD) echo "rw \ systemd.volatile=overlay systemd.firstboot=false systemd.hostname=kde-linux kde-linux.live=1 plasma.live.user=live \ lsm=landlock,lockdown,yama,integrity,bpf \ @@ -41,14 +60,17 @@ echo "rw \ nohz=on nohz_full=all \ rcu_nocbs=all rcutree.enable_rcu_lazy=1 \ amdgpu.dcdebugmask=0x10 \ - vt.global_cursor_default=0 quiet splash loglevel=3" > cmdline -mkinitcpio --config mkinitcpio.conf --generate initrd --kernel "$kernel_version" + vt.global_cursor_default=0 quiet splash loglevel=3" > cmdline.raw-live + +mkinitcpio --config mkinitcpio.conf --generate initrd.raw-live --kernel "$kernel_version" + ukify build \ --linux /usr/lib/modules/$kernel_version/vmlinuz \ - --initrd initrd \ - --cmdline @cmdline \ + --initrd initrd.raw-live \ + --cmdline @cmdline.raw-live \ --output live.efi +# Installed system UKI # "preempt=full threadirqs" reduces latency especially for audio and gaming workflows. # "nohz=on nohz_full=all" stops waking up a CPU more than once a second if it has fewer @@ -72,14 +94,16 @@ echo "rw rootflags=subvol=@system,compress=zstd:-4,lazytime \ amdgpu.dcdebugmask=0x10 \ systemd.hostname=kde-linux \ vt.global_cursor_default=0 quiet splash loglevel=3" > cmdline + mkinitcpio --config mkinitcpio.conf --generate initrd --kernel "$kernel_version" + ukify build \ --linux /usr/lib/modules/$kernel_version/vmlinuz \ --initrd initrd \ --cmdline @cmdline \ --output kde-linux.efi -# Mock artifact for upgrades, see build.sh +# Mock artifact for upgrades ukify build \ --cmdline "kde-linux.erofs.silence" \ --output erofs.addon.efi diff --git a/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-iso-mount-generator b/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-iso-mount-generator new file mode 100755 index 0000000..564769d --- /dev/null +++ b/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-iso-mount-generator @@ -0,0 +1,151 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL +# SPDX-FileCopyrightText: 2026 Hadi Chokr + +set -euo pipefail + +. /usr/lib/os-release + +normal_dir="$1" +early_dir="$2" +late_dir="$3" + +exec >/dev/kmsg 2>&1 + +# Only run in initrd during ISO boot +if [[ "${SYSTEMD_IN_INITRD:-}" != "1" ]]; then + exit 0 +fi + +if ! grep -q "kde-linux.iso=1" /proc/cmdline; then + exit 0 +fi + +echo "ISO boot mode detected. Locating boot device..." + +# find the block device we booted from (the ISO) + +find_iso_device() { + local iso_label dev candidate tmpdir + + # 1. Try explicit iso_label= from kernel cmdline + iso_label=$(sed -n 's/.*iso_label=\([^ ]*\).*/\1/p' /proc/cmdline) + if [[ -n "$iso_label" ]] && [[ -e "/dev/disk/by-label/$iso_label" ]]; then + echo "/dev/disk/by-label/$iso_label" + return 0 + fi + + # 2. Try to find the device we actually booted from via EFI or kernel + # The UKI is loaded directly from the ESP in the ISO's El Torito image. + # The boot device is typically the first optical drive (sr0) in VMs, + # but we scan all optical drives to be safe. + for dev in /dev/sr* /dev/cdrom*; do + [[ -b "$dev" ]] || continue + + # Check if this device contains our ISO signature: /live/filesystem.erofs + tmpdir=$(mktemp -d) + if mount -t iso9660 -o ro "$dev" "$tmpdir" 2>/dev/null; then + if [[ -f "$tmpdir/live/filesystem.erofs" ]]; then + umount "$tmpdir" + rmdir "$tmpdir" + echo "$dev" + return 0 + fi + umount "$tmpdir" + fi + rmdir "$tmpdir" + done + + # 3. Last resort: use blkid to find a device with filesystem type iso9660 + # and the correct volume label pattern (if we know the label). + if [[ -n "$iso_label" ]]; then + candidate=$(blkid -l -t LABEL="$iso_label" -o device 2>/dev/null | head -n1) + if [[ -n "$candidate" ]] && [[ -b "$candidate" ]]; then + echo "$candidate" + return 0 + fi + fi + + return 1 +} + +ISO_DEVICE=$(find_iso_device) +if [[ -z "$ISO_DEVICE" ]]; then + echo "ERROR: Could not locate boot ISO device" >&2 + exit 1 +fi + +echo "Boot ISO device: $ISO_DEVICE" + +# Generate mount units +# Early mount: the ISO itself, before initrd-root-fs.target +cat <<- EOF > "$early_dir/run-iso.mount" +# Generated by $(basename "$0") +[Unit] +Description=Mount ISO filesystem (live boot) +DefaultDependencies=no +Before=initrd-root-fs.target +After=blockdev@${ISO_DEVICE##*/}.target +Requires=blockdev@${ISO_DEVICE##*/}.target +JobTimeoutSec=30 +[Mount] +What=${ISO_DEVICE} +Where=/run/iso +Type=iso9660 +Options=ro,x-systemd.device-timeout=30 +DirectoryMode=0755 +EOF + +# Normal mount: the root erofs image from inside the ISO +cat <<- EOF > "$normal_dir/sysusr-usr.mount" +# Generated by $(basename "$0") +[Unit] +Description=Mount root filesystem from ISO +Before=initrd-usr-fs.target +After=run-iso.mount +[Mount] +What=/run/iso/live/filesystem.erofs +Where=/sysusr/usr +Options=ro,loop,X-mount.subdir=usr +DirectoryMode=0755 +EOF + +mkdir -p "$normal_dir/initrd-usr-fs.target.requires" +ln -sf ../sysusr-usr.mount "$normal_dir/initrd-usr-fs.target.requires/sysusr-usr.mount" + +# Bind mount /sysusr/usr → /sysroot/usr (required by systemd) +cat <<- EOF > "$normal_dir/sysroot-usr.mount" +# Generated by $(basename "$0") +[Unit] +Description=Bind mount /usr to sysroot +Before=initrd-usr-fs.target +After=sysusr-usr.mount +[Mount] +What=/sysusr/usr +Where=/sysroot/usr +Options=bind +EOF + +mkdir -p "$normal_dir/initrd-fs.target.requires" +ln -sf ../sysroot-usr.mount "$normal_dir/initrd-fs.target.requires/sysroot-usr.mount" + +# Post-switch-root mount: keep the ISO accessible at /run/iso +cat <<- EOF > "$normal_dir/run-iso.mount" +# Generated by $(basename "$0") +[Unit] +Description=Keep ISO mounted in booted system +After=initrd-switch-root.target +[Mount] +What=${ISO_DEVICE} +Where=/run/iso +Type=iso9660 +Options=ro +DirectoryMode=0755 +[Install] +WantedBy=local-fs.target +EOF + +mkdir -p "$normal_dir/local-fs.target.wants" +ln -sf ../run-iso.mount "$normal_dir/local-fs.target.wants/run-iso.mount" + +echo "ISO mount units generated successfully." diff --git a/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-live-generator b/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-live-generator index 3295990..0b50d3c 100755 --- a/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-live-generator +++ b/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-live-generator @@ -1,6 +1,7 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL # SPDX-FileCopyrightText: 2024 Harald Sitter +# SPDX-FileCopyrightText: 2026 Hadi Chokr set -eux @@ -17,25 +18,46 @@ if ! grep "kde-linux.live=1" /proc/cmdline; then exit 0 fi -if [ "$(readlink --canonicalize /dev/disk/by-partlabel/KDELinuxLive)" != "$(readlink --canonicalize /dev/gpt-auto-root)" ]; then - echo "gpt-auto-root is not KDELinuxLive" - exit 0 -fi +# Check if this is ISO boot mode +if grep "kde-linux.iso=1" /proc/cmdline; then + echo "ISO boot mode detected" -cat <<- EOF > "$normal_dir/var-lib-flatpak.mount" + # Mount flatpak from the root filesystem (same as raw mode) + cat <<- EOF > "$normal_dir/var-lib-flatpak.mount" # Generated by $(basename "$0") [Unit] -Description=Mount unit for /var/lib/flatpak +Description=Mount unit for /var/lib/flatpak (ISO mode) Before=kde-linux-volatile-var-lib-flatpak.service - [Mount] What=/usr/share/factory/var/lib/flatpak Where=/var/lib/flatpak Options=bind - [Install] WantedBy=multi-user.target EOF +else + echo "Raw disk boot mode detected" + + # Check for raw live boot + if [ "$(readlink --canonicalize /dev/disk/by-partlabel/KDELinuxLive)" != "$(readlink --canonicalize /dev/gpt-auto-root)" ]; then + echo "gpt-auto-root is not KDELinuxLive" + exit 0 + fi + + # Mount flatpak from raw image factory directory + cat <<- EOF > "$normal_dir/var-lib-flatpak.mount" +# Generated by $(basename "$0") +[Unit] +Description=Mount unit for /var/lib/flatpak (Raw mode) +Before=kde-linux-volatile-var-lib-flatpak.service +[Mount] +What=/usr/share/factory/var/lib/flatpak +Where=/var/lib/flatpak +Options=bind +[Install] +WantedBy=multi-user.target +EOF +fi mkdir "$normal_dir/multi-user.target.wants/" || true ln -sf ../var-lib-flatpak.mount "$normal_dir/multi-user.target.wants/var-lib-flatpak.mount" diff --git a/upload-to-storage.sh b/upload-to-storage.sh index afeddd3..cb5adb8 100755 --- a/upload-to-storage.sh +++ b/upload-to-storage.sh @@ -8,16 +8,15 @@ set -eu # Output Directory OUTDIR="${OUTDIR:-mkosi.output}" -# Do not blow the lid off the storage for now. Reset the tree and only publish a select few files +# Reset upload tree mv upload-tree upload-tree-old || true if [ ! -d upload-tree ]; then mkdir upload-tree - for f in "$OUTDIR"/*.raw "$OUTDIR"/*.erofs "$OUTDIR"/*.efi; do + for f in "$OUTDIR"/*.raw "$OUTDIR"/*.iso "$OUTDIR"/*.erofs "$OUTDIR"/*.efi; do if [[ $f == *.test.raw ]]; then - # Skip test images continue fi - mv "$f" upload-tree/ + [ -f "$f" ] && mv "$f" upload-tree/ done fi @@ -25,6 +24,6 @@ go -C ./token-redeemer/ run . go -C ./uploader/ run . --remote "s3+https://storage.kde.org/ci-artifacts/$CI_PROJECT_PATH/j/$CI_JOB_ID" echo "𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏" -echo "You can find the raw disk images at:" +echo "Artifacts available at:" echo "https://qoomon.github.io/aws-s3-bucket-browser/index.html?bucket=https://storage.kde.org/ci-artifacts/#$CI_PROJECT_PATH/j/$CI_JOB_ID/" echo "𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏" diff --git a/upload.sh b/upload.sh index 29f80ce..5f63d93 100755 --- a/upload.sh +++ b/upload.sh @@ -7,7 +7,7 @@ set -eux OUTDIR=mkosi.output -# For the vacuum helper and this script +# SSH configuration export SSH_IDENTITY="$PWD/.secure_files/ssh.key" export SSH_USER=kdeos export SSH_HOST=origin.files.kde.org @@ -17,60 +17,60 @@ export SSH_REALLY_DELETE=1 chmod 600 "$SSH_IDENTITY" +# Run vacuum helper go -C ./upload-vacuum/ build -o upload-vacuum . ./upload-vacuum/upload-vacuum -# The following variables are for this script only. Not shared with the vacuum helper. -sudo chown -Rvf "$(id -u):$(id -g)" "$PWD/.secure_files" # Make sure we have access +# GPG setup +sudo chown -Rvf "$(id -u):$(id -g)" "$PWD/.secure_files" export GNUPGHOME="$PWD/.secure_files/gpg" gpg --verbose --no-options --homedir="$GNUPGHOME" --import "$PWD/.secure_files/gpg.private.key" + REMOTE_ROOT=$SSH_USER@$SSH_HOST:$SSH_ROOT_PATH REMOTE_PATH=$SSH_USER@$SSH_HOST:$SSH_PATH -# You can use `ssh-keyscan origin.files.kde.org` to get the host key + +# Add SSH host key echo "origin.files.kde.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILUjdH4S7otYIdLUkOZK+owIiByjNQPzGi7GQ5HOWjO6" >> ~/.ssh/known_hosts -# The initial SHA256SUMS file is created by the vacuum script based on what is left on the server. We append to it. - +# Generate checksums sudo chown -R "$USER":"$USER" "$OUTDIR" cd "$OUTDIR" -# We need shell globs here! More readable this way. Ignore shellcheck. -# shellcheck disable=SC2129 sha256sum -- *.efi >> SHA256SUMS sha256sum -- *.tar.zst >> SHA256SUMS sha256sum -- *.erofs >> SHA256SUMS -# Don't put .erofs.caibx into the SHA256SUMS, it will break file matching. -# https://github.com/systemd/systemd/issues/38605 sha256sum -- *-x86-64.caibx >> SHA256SUMS +sha256sum -- *.iso >> SHA256SUMS gpg --homedir="$GNUPGHOME" --output SHA256SUMS.gpg --detach-sign SHA256SUMS -scp -i "$SSH_IDENTITY" ./*.raw ./*.torrent "$REMOTE_ROOT" +# Upload to SSH server +scp -i "$SSH_IDENTITY" ./*.raw ./*.iso ./*.torrent "$REMOTE_ROOT" scp -i "$SSH_IDENTITY" ./*.efi ./*.tar.zst ./*.erofs ./*.caibx "$REMOTE_PATH" -scp -i "$SSH_IDENTITY" SHA256SUMS SHA256SUMS.gpg "$REMOTE_PATH" # upload as last artifact to finalize the upload - -# The new s3 based upload system +scp -i "$SSH_IDENTITY" SHA256SUMS SHA256SUMS.gpg "$REMOTE_PATH" +# S3 upload S3_STORE="s3+https://storage.kde.org/kde-linux/sysupdate/store/" S3_TARGET="s3+https://storage.kde.org/kde-linux/testing/" -## Upload to the chunk store directly +## Upload chunks to store go install -v github.com/folbricht/desync/cmd/desync@latest go -C ../token-redeemer/ run . + ~/go/bin/desync chop \ --concurrency "$(nproc)" \ --store "$S3_STORE" \ ./*-x86-64.caibx \ ./*-x86-64.erofs -## Prepare the image upload tree +## Prepare upload tree cd .. rm -rf upload-tree mkdir -p upload-tree/sysupdate/v2 -mv "$OUTDIR"/*.raw "$OUTDIR"/*.torrent upload-tree/ +mv "$OUTDIR"/*.raw "$OUTDIR"/*.iso "$OUTDIR"/*.torrent upload-tree/ mv "$OUTDIR"/*.efi "$OUTDIR"/*.tar.zst "$OUTDIR"/*.erofs "$OUTDIR"/*.caibx "$OUTDIR"/SHA256SUMS "$OUTDIR"/SHA256SUMS.gpg upload-tree/sysupdate/v2/ -### Upload +## Upload to S3 go -C ./token-redeemer/ run . go -C ./uploader/ run . --remote "$S3_TARGET"