From 8f52fd06c5d5d8ae70a5388aa59daa61b32f2e0f Mon Sep 17 00:00:00 2001 From: Hadi Chokr Date: Thu, 23 Apr 2026 16:06:58 +0200 Subject: [PATCH] Make our Image comply with ISO9660. Signed-off-by: Hadi Chokr --- basic-test-efi-addon.sh | 14 ++-- basic-test.py | 65 ++++++++++++------ build.sh | 66 ++++++++++++------- mkosi.extra/live/usr/lib/basic-test | 15 ++++- .../lib/initcpio/install/systemd-extension | 12 +++- upload-to-storage.sh | 2 +- upload.sh | 4 +- 7 files changed, 124 insertions(+), 54 deletions(-) diff --git a/basic-test-efi-addon.sh b/basic-test-efi-addon.sh index 56003f6..4064907 100755 --- a/basic-test-efi-addon.sh +++ b/basic-test-efi-addon.sh @@ -1,13 +1,17 @@ #!/bin/bash # SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL # SPDX-FileCopyrightText: 2025 Harald Sitter +# SPDX-FileCopyrightText: 2026 Hadi Chokr set -eux -cd boot/EFI/Linux/ +# ADDON_DIR is set by basic-test.py to the .extra.d directory inside the mounted ESP +if [ -z "$ADDON_DIR" ]; then + echo "ERROR: ADDON_DIR environment variable not set" + exit 1 +fi -dir="$UKI.extra.d" -[ -d "$dir" ] || mkdir "$dir" +# Create the addon UKI (systemd-stub addon) that appends the test cmdline ukify build \ - --cmdline "kde-linux.basic-test=1 kde-linux.basic-test-callback=http://10.0.2.2:${PORT}/good" \ - --output "$dir/basic-test.addon.efi" + --cmdline "kde-linux.basic-test=1 kde-linux.basic-test-callback=http://10.0.2.2:${PORT}/good" \ + --output "$ADDON_DIR/basic-test.addon.efi" diff --git a/basic-test.py b/basic-test.py index 884221a..254bbb5 100755 --- a/basic-test.py +++ b/basic-test.py @@ -1,6 +1,7 @@ -#!/bin/env python3 +#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL # SPDX-FileCopyrightText: 2025 Harald Sitter +# SPDX-FileCopyrightText: 2026 Hadi Chokr import atexit import http.server @@ -8,7 +9,7 @@ import sys import subprocess import os import time - +import tempfile from pathlib import Path class Handler(http.server.BaseHTTPRequestHandler): @@ -31,32 +32,58 @@ img = sys.argv[1] if not img: print("No image specified") sys.exit(1) -test_img = img.replace('.raw', '.test.raw') efi_base = sys.argv[2] if not efi_base: print("No EFI base image specified") sys.exit(1) +# Always test as ISO (a valid .iso9660 is also a valid .raw) +test_img = img.replace('.raw', '.test.iso').replace('.iso', '.test.iso') subprocess.check_call(['cp', '--reflink=auto', img, test_img]) -subprocess.check_call(['systemd-dissect', test_img, '--with', f'{os.path.dirname(os.path.realpath(__file__))}/basic-test-efi-addon.sh'], - env={'PORT': str(server.server_port), - 'UKI': efi_base}, - stdout=sys.stdout, stderr=sys.stderr) + +# Inject the EFI addon into the ESP partition of the test ISO +script_dir = os.path.dirname(os.path.realpath(__file__)) +addon_src = f'{script_dir}/basic-test-efi-addon.sh' + +with tempfile.TemporaryDirectory() as mnt: + # Find the ESP partition offset and size using sfdisk + sfdisk = subprocess.check_output(['sfdisk', '--json', test_img]).decode() + import json + parts = json.loads(sfdisk)['partitiontable']['partitions'] + esp = next(p for p in parts if p.get('type', '') == 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B') + sector_size = json.loads(sfdisk)['partitiontable']['sectorsize'] + offset = esp['start'] * sector_size + size = esp['size'] * sector_size + + subprocess.check_call([ + 'mount', '-o', f'loop,offset={offset},sizelimit={size}', + test_img, mnt + ]) + try: + efi_extra_dir = f'{mnt}/EFI/Linux/{efi_base}.extra.d' + os.makedirs(efi_extra_dir, exist_ok=True) + subprocess.check_call([ + 'bash', addon_src + ], env={ + 'PORT': str(server.server_port), + 'UKI': efi_base, + 'ADDON_DIR': efi_extra_dir, + }) + finally: + subprocess.check_call(['umount', mnt]) + +qemu_cmd = [ + "qemu-system-x86_64", + "-cdrom", test_img, + "-m", "4G", + "-enable-kvm", + "-cpu", "host", + "-bios", "/usr/share/OVMF/x64/OVMF.4m.fd", +] # I ought to point out that this leaks the process in case of failure. It will however get reaped by the docker container shutdown. -qemu = subprocess.Popen([ - "qemu-system-x86_64", - "-drive", - f"file={test_img},format=raw", - "-m", - "4G", - "-enable-kvm", - "-cpu", - "host", - "-bios", - "/usr/share/OVMF/x64/OVMF.4m.fd", -]) +qemu = subprocess.Popen(qemu_cmd) atexit.register(lambda: (qemu.kill())) def on_timeout(): diff --git a/build.sh b/build.sh index e2730cd..308e28d 100755 --- a/build.sh +++ b/build.sh @@ -2,9 +2,10 @@ # SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL # SPDX-FileCopyrightText: 2023 Harald Sitter # SPDX-FileCopyrightText: 2024 Bruno Pajdek +# SPDX-FileCopyrightText: 2026 Hadi Chokr # Build image using mkosi, well, somewhat. mkosi is actually a bit too inflexible for our purposes so we generate a OS -# tree using mkosi and then construct shipable raw images (for installation) and tarballs (for systemd-sysupdate) +# tree using mkosi and then construct shipable .iso9660 (and gpt raw disk images) for installation and tarballs (for systemd-sysupdate) # ourselves. set -ex @@ -48,13 +49,14 @@ DEBUG_TAR=${OUTPUT}_debug-x86-64.tar # Output debug archive path (.zst will be a # We'll rename things accordingly via sysupdate.d files. ROOTFS_CAIBX=${OUTPUT}_root-x86-64.caibx ROOTFS_EROFS=${OUTPUT}_root-x86-64.erofs # Output erofs image path -IMG=${OUTPUT}.raw # Output raw image path +ISO="${OUTPUT}.iso" # both a valid GPT disk image and a bootable ISO EFI_BASE=kde-linux_${VERSION} # Base name of the UKI in the image's ESP (exported so it can be used in basic-test-efi-addon.sh) -EFI=${EFI_BASE}+3.efi # Name of primary UKI in the image's ESP +EFI=${EFI_BASE}+3.efi # Name of primary UKI in the image's ESP (with tries counter for installed system) +LIVE_EFI=${EFI_BASE}.efi # Name of live UKI in the ESP (no tries counter — ESP is read-only on ISO) # Clean up old build artifacts. -rm --recursive --force kde-linux.cache/*.raw kde-linux.cache/*.mnt +rm --recursive --force kde-linux.cache/*.raw kde-linux.cache/*.iso kde-linux.cache/*.mnt # FIXME: temporary hack to work around repo priorities being off in the CI image cat <<- EOF > mkosi.sandbox/etc/pacman.conf @@ -130,42 +132,50 @@ rm -rfv "${OUTPUT}/efi" [ -d "${OUTPUT}/usr/share/factory/boot/EFI" ] || mkdir --mode 0700 "${OUTPUT}/usr/share/factory/boot/EFI" [ -d "${OUTPUT}/usr/share/factory/boot/EFI/Linux" ] || mkdir --mode 0700 "${OUTPUT}/usr/share/factory/boot/EFI/Linux" [ -d "${OUTPUT}/usr/share/factory/boot/EFI/Linux/$EFI_BASE.efi.extra.d" ] || mkdir --mode 0700 "${OUTPUT}/usr/share/factory/boot/EFI/Linux/$EFI_BASE.efi.extra.d" + +# Save the main UKI (with tries counter) aside as it must NOT go into factory/boot yet +# so it doesn't end up on the live ESP. cp -v "${OUTPUT}"/kde-linux.efi "$MAIN_UKI" -mv -v "${OUTPUT}"/kde-linux.efi "${OUTPUT}/usr/share/factory/boot/EFI/Linux/$EFI" -mv -v "${OUTPUT}"/live.efi "$LIVE_UKI" +rm -v "${OUTPUT}"/kde-linux.efi mv -v "${OUTPUT}"/erofs.addon.efi "${OUTPUT}_erofs.addon.efi" +mv -v "${OUTPUT}"/live.efi "$LIVE_UKI" make_debug_archive -# Now let's actually build a live raw image. First, the ESP. +# Now let's actually build the live ESP. # We use kde-linux.cache instead of /tmp as usual because we'll probably run out of space there. -# Since we're building a live image, replace the main UKI with the live one. -mv "$LIVE_UKI" "${OUTPUT}/usr/share/factory/boot/EFI/Linux/$EFI" +# Only LIVE_EFI (no tries counter) goes into factory/boot for the ESP. +# The installed system UKI ($EFI with +3) is added AFTER the ESP is built. +mv "$LIVE_UKI" "${OUTPUT}/usr/share/factory/boot/EFI/Linux/$LIVE_EFI" # Change to kde-linux.cache since we'll be working there. cd kde-linux.cache -# Create a 260M large FAT32 filesystem inside of esp.raw. -fallocate -l 260M esp.raw +# Create a 280M large FAT32 filesystem inside of esp.raw. +fallocate -l 280M esp.raw mkfs.fat -F 32 esp.raw # Mount it to esp.raw.mnt. -mkdir -p esp.raw.mnt # The -p prevents failure if directory already exists +mkdir -p esp.raw.mnt mount esp.raw esp.raw.mnt # Copy everything from /usr/share/factory/boot into esp.raw.mnt. +# At this point only LIVE_EFI is in factory/boot/EFI/Linux/ so the installed UKI (+3) is not there yet. cp --archive --recursive "${OUTPUT}/usr/share/factory/boot/." esp.raw.mnt # We're done, unmount esp.raw.mnt. umount esp.raw.mnt -# Now, the root. +cd .. # and back to root -# Copy back the main UKI for the root. +# Now add the installed system UKI (with tries counter) to factory/boot for the erofs rootfs. +# This happens AFTER the ESP build so it doesn't land on the live ESP. cp "$MAIN_UKI" "${OUTPUT}/usr/share/factory/boot/EFI/Linux/$EFI" -cd .. # and back to root +# Remove the live UKI from factory as it was only needed for the ESP build. +# The erofs rootfs should only contain the installed system UKI (+3). +rm "${OUTPUT}/usr/share/factory/boot/EFI/Linux/$LIVE_EFI" # Drop flatpak data from erofs. They are in the usr/share/factory and deployed from there. rm -rf "$OUTPUT/var/lib/flatpak" @@ -174,18 +184,30 @@ mkdir "$OUTPUT/var/lib/flatpak" # but keep a mountpoint around for the live sess time mkfs.erofs -zzstd -C 65536 --chunksize 65536 "$ROOTFS_EROFS" "$OUTPUT" > erofs.log 2>&1 cp --reflink=auto "$ROOTFS_EROFS" kde-linux.cache/root.raw -# Now assemble the two generated images using systemd-repart and the definitions in mkosi.repart into $IMG. -touch "$IMG" -systemd-repart --no-pager --empty=allow --size=auto --dry-run=no --root=kde-linux.cache --definitions=mkosi.repart "$IMG" +# Now assemble the image using systemd-repart and the definitions in mkosi.repart into $ISO. +# The resulting file is both a valid GPT disk image and a bootable El Torito ISO. +touch "$ISO" +systemd-repart \ + --no-pager \ + --empty=allow \ + --size=auto \ + --dry-run=no \ + --root=kde-linux.cache \ + --definitions=mkosi.repart \ + --el-torito=true \ + --el-torito-volume="KDE LINUX $VERSION" \ + --el-torito-publisher="KDE" \ + "$ISO" + +# Test the ISO (which is also a valid GPT image so no need to test as .raw separately) +./basic-test.py "$ISO" "$LIVE_EFI" || exit 1 +rm ./mkosi.output/*.test.iso # Incase the owner is root chown -R user:user mkosi.output -./basic-test.py "$IMG" "$EFI_BASE.efi" || exit 1 -rm ./mkosi.output/*.test.raw - # Create a torrent for the image -./torrent-create.rb "$VERSION" "$OUTPUT" "$IMG" +./torrent-create.rb "$VERSION" "$OUTPUT" "$ISO" go install -v github.com/folbricht/desync/cmd/desync@latest ~/go/bin/desync make --print-stats --chunk-size 1024:2048:4096 "$ROOTFS_CAIBX" "$ROOTFS_EROFS" diff --git a/mkosi.extra/live/usr/lib/basic-test b/mkosi.extra/live/usr/lib/basic-test index f5ab86a..aa52c30 100755 --- a/mkosi.extra/live/usr/lib/basic-test +++ b/mkosi.extra/live/usr/lib/basic-test @@ -1,12 +1,19 @@ #!/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL # SPDX-FileCopyrightText: 2025 Harald Sitter +# SPDX-FileCopyrightText: 2026 Hadi Chokr import subprocess import time from pathlib import Path import json +# Units that are known to fail on a read-only ISO (e.g., /boot is not writable) +IGNORED_FAILED_UNITS = { + "systemd-boot-update.service", + "systemd-boot-random-seed.service", +} + callback = None with open('/proc/cmdline') as cmdline: data = cmdline.read() @@ -28,14 +35,16 @@ while True: # 1000 is the uid of the live user. always. if Path('/run/user/1000/kde-linux-bless-session').is_file(): failed = json.loads(subprocess.check_output(["systemctl", "--failed", "--output=json"])) - if len(failed) > 0: + # Filter out ignored units + relevant_failed = [unit for unit in failed if unit['unit'] not in IGNORED_FAILED_UNITS] + if len(relevant_failed) > 0: with open('data.file', 'w') as f: - for unit in failed: + for unit in relevant_failed: f.write("\n") f.write(json.dumps(unit)) f.write("\n") try: - f.write(subprocess.check_output(['journalctl', '--no-pager', f'_SYSTEMD_UNIT={unit['unit']}']).decode('utf-8')) + f.write(subprocess.check_output(['journalctl', '--no-pager', f'_SYSTEMD_UNIT={unit["unit"]}']).decode('utf-8')) except Exception as e: f.write(f"Failed to get journal for {unit['unit']}: {e}\n") f.write("\n") # make sure we have a final newline diff --git a/mkosi.extra/usr/lib/initcpio/install/systemd-extension b/mkosi.extra/usr/lib/initcpio/install/systemd-extension index 80203dc..d4905f3 100644 --- a/mkosi.extra/usr/lib/initcpio/install/systemd-extension +++ b/mkosi.extra/usr/lib/initcpio/install/systemd-extension @@ -12,15 +12,23 @@ build() { /usr/lib/systemd/system-generators/kde-linux-mount-generator \ /usr/lib/systemd/systemd-bootchart \ /usr/lib/etc-factory \ - /usr/bin/btrfs + /usr/lib/udev/cdrom_id \ + /usr/bin/btrfs \ + /usr/bin/blkid \ + /usr/bin/systemd-dissect + # The loop service will make the ISO be found by gpt-auto-root map add_systemd_unit \ + systemd-loop@.service \ systemd-volatile-root.service \ systemd-bootchart.service \ etc-factory.service # Make double sure the dissection rule is present. Without it booting doesn't work because we can't find a root. # Notably not getting added by the release initcpio combined with aur systemd-git at the time of writing. + # 60-cdrom will make parsing the ISO in the Initrd easier and 99-systemd will triger the loop service for GPT mismatch map add_udev_rule \ - 90-image-dissect.rules + 60-cdrom_id.rules \ + 90-image-dissect.rules \ + 99-systemd.rules } diff --git a/upload-to-storage.sh b/upload-to-storage.sh index afeddd3..b7b48e9 100755 --- a/upload-to-storage.sh +++ b/upload-to-storage.sh @@ -12,7 +12,7 @@ OUTDIR="${OUTDIR:-mkosi.output}" mv upload-tree upload-tree-old || true if [ ! -d upload-tree ]; then mkdir upload-tree - for f in "$OUTDIR"/*.raw "$OUTDIR"/*.erofs "$OUTDIR"/*.efi; do + for f in "$OUTDIR"/*.iso "$OUTDIR"/*.erofs "$OUTDIR"/*.efi; do if [[ $f == *.test.raw ]]; then # Skip test images continue diff --git a/upload.sh b/upload.sh index 5527894..4efe9c1 100755 --- a/upload.sh +++ b/upload.sh @@ -45,7 +45,7 @@ sha256sum -- *-x86-64.caibx >> SHA256SUMS gpg --homedir="$GNUPGHOME" --output SHA256SUMS.gpg --detach-sign SHA256SUMS -scp -i "$SSH_IDENTITY" ./*.raw ./*.torrent "$REMOTE_ROOT" +scp -i "$SSH_IDENTITY" ./*.iso ./*.torrent "$REMOTE_ROOT" scp -i "$SSH_IDENTITY" ./*.efi ./*.tar.zst ./*.erofs ./*.caibx "$REMOTE_PATH" scp -i "$SSH_IDENTITY" SHA256SUMS SHA256SUMS.gpg "$REMOTE_PATH" # upload as last artifact to finalize the upload @@ -71,7 +71,7 @@ rm -rf upload-tree V2_TREE="upload-tree/testing/sysupdate/v2" mkdir -p "$V2_TREE" -mv "$OUTDIR"/*.raw "$OUTDIR"/*.torrent upload-tree/testing/ +mv "$OUTDIR"/*.iso "$OUTDIR"/*.torrent upload-tree/testing/ mv "$OUTDIR"/*.efi "$OUTDIR"/*.tar.zst "$OUTDIR"/*.erofs "$OUTDIR"/*.caibx "$V2_TREE/" ### Upload