From fc86e72fcc6c26e94585e31dc4ad3d90cbc2a15d Mon Sep 17 00:00:00 2001 From: Hadi Chokr Date: Wed, 15 Apr 2026 14:53:52 +0200 Subject: [PATCH] Reset --- build.sh | 54 ------- .../lib/initcpio/install/systemd-extension | 1 - mkosi.extra/usr/lib/rebuild-efi | 38 +---- .../kde-linux-iso-mount-generator | 151 ------------------ .../kde-linux-live-generator | 58 +++---- upload-to-storage.sh | 9 +- upload.sh | 36 ++--- 7 files changed, 48 insertions(+), 299 deletions(-) delete mode 100755 mkosi.extra/usr/lib/systemd/system-generators/kde-linux-iso-mount-generator diff --git a/build.sh b/build.sh index cca872c..d0c390d 100755 --- a/build.sh +++ b/build.sh @@ -49,8 +49,6 @@ DEBUG_TAR=${OUTPUT}_debug-x86-64.tar # Output debug archive path (.zst will be a ROOTFS_CAIBX=${OUTPUT}_root-x86-64.caibx ROOTFS_EROFS=${OUTPUT}_root-x86-64.erofs # Output erofs image path IMG=${OUTPUT}.raw # Output raw image path -ISO=${OUTPUT}.iso -ISO_STAGING="kde-linux.cache/iso-staging" EFI_BASE=kde-linux_${VERSION} # Base name of the UKI in the image's ESP (exported so it can be used in basic-test-efi-addon.sh) EFI=${EFI_BASE}+3.efi # Name of primary UKI in the image's ESP @@ -180,58 +178,6 @@ cp --reflink=auto "$ROOTFS_EROFS" kde-linux.cache/root.raw touch "$IMG" systemd-repart --no-pager --empty=allow --size=auto --dry-run=no --root=kde-linux.cache --definitions=mkosi.repart "$IMG" -# Build ISO for live booting -echo "Building bootable ISO..." -# Clean and create ISO staging directory -rm -rf "$ISO_STAGING" -mkdir -p "$ISO_STAGING/live" "$ISO_STAGING/boot/EFI/Linux" - -# Copy the erofs root filesystem -cp "$ROOTFS_EROFS" "$ISO_STAGING/live/filesystem.erofs" - -# Use the ISO-specific UKI -cp "${OUTPUT}"/live-iso.efi "$ISO_STAGING/boot/EFI/Linux/BOOTX64.EFI" - -# Create ESP image for El Torito -ESP_IMG="kde-linux.cache/esp-iso.img" -fallocate -l 328M "$ESP_IMG" -mkfs.fat -F 32 "$ESP_IMG" - -# Mount and populate ESP -ESP_MNT="kde-linux.cache/esp-iso.mnt" -mkdir -p "$ESP_MNT" -mount "$ESP_IMG" "$ESP_MNT" -mkdir -p "$ESP_MNT/EFI/BOOT" -cp "$ISO_STAGING/boot/EFI/Linux/BOOTX64.EFI" "$ESP_MNT/EFI/BOOT/" -umount "$ESP_MNT" - -# Create ISO with xorriso -xorriso -as mkisofs \ - -o "$ISO" \ - -iso-level 3 \ - -full-iso9660-filenames \ - -joliet -joliet-long \ - -rational-rock \ - -volid "KDE_LINUX_${VERSION}" \ - -appid "KDE Linux Live ${VERSION}" \ - -publisher "KDE" \ - -preparer "kde-linux build system" \ - -e esp-iso.img \ - -no-emul-boot \ - -eltorito-platform efi \ - -isohybrid-gpt-basdat \ - "$ISO_STAGING" \ - -graft-points \ - "esp-iso.img=$ESP_IMG" - -# Make ISO readable -chmod go+r "$ISO" - -echo "ISO created: $ISO ($(du -h "$ISO" | cut -f1))" - -# Create torrent for ISO -./torrent-create.rb "$VERSION" "$OUTPUT" "$ISO" - # Incase the owner is root chown -R user:user mkosi.output diff --git a/mkosi.extra/usr/lib/initcpio/install/systemd-extension b/mkosi.extra/usr/lib/initcpio/install/systemd-extension index b969357..80203dc 100644 --- a/mkosi.extra/usr/lib/initcpio/install/systemd-extension +++ b/mkosi.extra/usr/lib/initcpio/install/systemd-extension @@ -10,7 +10,6 @@ build() { /usr/lib/systemd/system-generators/00-kde-linux-os-release \ /usr/lib/systemd/system-generators/kde-linux-live-generator \ /usr/lib/systemd/system-generators/kde-linux-mount-generator \ - /usr/lib/systemd/system-generators/kde-linux-iso-mount-generator \ /usr/lib/systemd/systemd-bootchart \ /usr/lib/etc-factory \ /usr/bin/btrfs diff --git a/mkosi.extra/usr/lib/rebuild-efi b/mkosi.extra/usr/lib/rebuild-efi index b90e956..cd8bb2f 100755 --- a/mkosi.extra/usr/lib/rebuild-efi +++ b/mkosi.extra/usr/lib/rebuild-efi @@ -24,34 +24,15 @@ if ! dkms autoinstall -k "$kernel_version"; then exit 1 fi + # NOTE: plymouth MUST be after systemd as per the wiki! cat <<- EOF > mkinitcpio.conf -MODULES=(overlay loop isofs udf sr_mod cdrom) +MODULES=(overlay) BINARIES=() FILES=() HOOKS=(base systemd modconf kms keyboard block sd-encrypt filesystems fsck systemd-extension plymouth microcode sd-shutdown) EOF -# ISO Live UKI - boots from ISO media -echo "rw \ - systemd.volatile=overlay systemd.firstboot=false systemd.hostname=kde-linux kde-linux.live=1 kde-linux.iso=1 plasma.live.user=live \ - lsm=landlock,lockdown,yama,integrity,bpf \ - zswap.enabled=0 \ - preempt=full threadirqs \ - nohz=on nohz_full=all \ - rcu_nocbs=all rcutree.enable_rcu_lazy=1 \ - amdgpu.dcdebugmask=0x10 \ - vt.global_cursor_default=0 quiet splash loglevel=3 iso_label=KDE_LINUX_${IMAGE_VERSION}" > cmdline.iso - -mkinitcpio --config mkinitcpio.conf --generate initrd.iso --kernel "$kernel_version" - -ukify build \ - --linux /usr/lib/modules/$kernel_version/vmlinuz \ - --initrd initrd.iso \ - --cmdline @cmdline.iso \ - --output live-iso.efi - -# Raw Live UKI - boots from .raw disk image (USB/DD) echo "rw \ systemd.volatile=overlay systemd.firstboot=false systemd.hostname=kde-linux kde-linux.live=1 plasma.live.user=live \ lsm=landlock,lockdown,yama,integrity,bpf \ @@ -60,17 +41,14 @@ echo "rw \ nohz=on nohz_full=all \ rcu_nocbs=all rcutree.enable_rcu_lazy=1 \ amdgpu.dcdebugmask=0x10 \ - vt.global_cursor_default=0 quiet splash loglevel=3" > cmdline.raw-live - -mkinitcpio --config mkinitcpio.conf --generate initrd.raw-live --kernel "$kernel_version" - + vt.global_cursor_default=0 quiet splash loglevel=3" > cmdline +mkinitcpio --config mkinitcpio.conf --generate initrd --kernel "$kernel_version" ukify build \ --linux /usr/lib/modules/$kernel_version/vmlinuz \ - --initrd initrd.raw-live \ - --cmdline @cmdline.raw-live \ + --initrd initrd \ + --cmdline @cmdline \ --output live.efi -# Installed system UKI # "preempt=full threadirqs" reduces latency especially for audio and gaming workflows. # "nohz=on nohz_full=all" stops waking up a CPU more than once a second if it has fewer @@ -94,16 +72,14 @@ echo "rw rootflags=subvol=@system,compress=zstd:-4,lazytime \ amdgpu.dcdebugmask=0x10 \ systemd.hostname=kde-linux \ vt.global_cursor_default=0 quiet splash loglevel=3" > cmdline - mkinitcpio --config mkinitcpio.conf --generate initrd --kernel "$kernel_version" - ukify build \ --linux /usr/lib/modules/$kernel_version/vmlinuz \ --initrd initrd \ --cmdline @cmdline \ --output kde-linux.efi -# Mock artifact for upgrades +# Mock artifact for upgrades, see build.sh ukify build \ --cmdline "kde-linux.erofs.silence" \ --output erofs.addon.efi diff --git a/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-iso-mount-generator b/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-iso-mount-generator deleted file mode 100755 index 564769d..0000000 --- a/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-iso-mount-generator +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL -# SPDX-FileCopyrightText: 2026 Hadi Chokr - -set -euo pipefail - -. /usr/lib/os-release - -normal_dir="$1" -early_dir="$2" -late_dir="$3" - -exec >/dev/kmsg 2>&1 - -# Only run in initrd during ISO boot -if [[ "${SYSTEMD_IN_INITRD:-}" != "1" ]]; then - exit 0 -fi - -if ! grep -q "kde-linux.iso=1" /proc/cmdline; then - exit 0 -fi - -echo "ISO boot mode detected. Locating boot device..." - -# find the block device we booted from (the ISO) - -find_iso_device() { - local iso_label dev candidate tmpdir - - # 1. Try explicit iso_label= from kernel cmdline - iso_label=$(sed -n 's/.*iso_label=\([^ ]*\).*/\1/p' /proc/cmdline) - if [[ -n "$iso_label" ]] && [[ -e "/dev/disk/by-label/$iso_label" ]]; then - echo "/dev/disk/by-label/$iso_label" - return 0 - fi - - # 2. Try to find the device we actually booted from via EFI or kernel - # The UKI is loaded directly from the ESP in the ISO's El Torito image. - # The boot device is typically the first optical drive (sr0) in VMs, - # but we scan all optical drives to be safe. - for dev in /dev/sr* /dev/cdrom*; do - [[ -b "$dev" ]] || continue - - # Check if this device contains our ISO signature: /live/filesystem.erofs - tmpdir=$(mktemp -d) - if mount -t iso9660 -o ro "$dev" "$tmpdir" 2>/dev/null; then - if [[ -f "$tmpdir/live/filesystem.erofs" ]]; then - umount "$tmpdir" - rmdir "$tmpdir" - echo "$dev" - return 0 - fi - umount "$tmpdir" - fi - rmdir "$tmpdir" - done - - # 3. Last resort: use blkid to find a device with filesystem type iso9660 - # and the correct volume label pattern (if we know the label). - if [[ -n "$iso_label" ]]; then - candidate=$(blkid -l -t LABEL="$iso_label" -o device 2>/dev/null | head -n1) - if [[ -n "$candidate" ]] && [[ -b "$candidate" ]]; then - echo "$candidate" - return 0 - fi - fi - - return 1 -} - -ISO_DEVICE=$(find_iso_device) -if [[ -z "$ISO_DEVICE" ]]; then - echo "ERROR: Could not locate boot ISO device" >&2 - exit 1 -fi - -echo "Boot ISO device: $ISO_DEVICE" - -# Generate mount units -# Early mount: the ISO itself, before initrd-root-fs.target -cat <<- EOF > "$early_dir/run-iso.mount" -# Generated by $(basename "$0") -[Unit] -Description=Mount ISO filesystem (live boot) -DefaultDependencies=no -Before=initrd-root-fs.target -After=blockdev@${ISO_DEVICE##*/}.target -Requires=blockdev@${ISO_DEVICE##*/}.target -JobTimeoutSec=30 -[Mount] -What=${ISO_DEVICE} -Where=/run/iso -Type=iso9660 -Options=ro,x-systemd.device-timeout=30 -DirectoryMode=0755 -EOF - -# Normal mount: the root erofs image from inside the ISO -cat <<- EOF > "$normal_dir/sysusr-usr.mount" -# Generated by $(basename "$0") -[Unit] -Description=Mount root filesystem from ISO -Before=initrd-usr-fs.target -After=run-iso.mount -[Mount] -What=/run/iso/live/filesystem.erofs -Where=/sysusr/usr -Options=ro,loop,X-mount.subdir=usr -DirectoryMode=0755 -EOF - -mkdir -p "$normal_dir/initrd-usr-fs.target.requires" -ln -sf ../sysusr-usr.mount "$normal_dir/initrd-usr-fs.target.requires/sysusr-usr.mount" - -# Bind mount /sysusr/usr → /sysroot/usr (required by systemd) -cat <<- EOF > "$normal_dir/sysroot-usr.mount" -# Generated by $(basename "$0") -[Unit] -Description=Bind mount /usr to sysroot -Before=initrd-usr-fs.target -After=sysusr-usr.mount -[Mount] -What=/sysusr/usr -Where=/sysroot/usr -Options=bind -EOF - -mkdir -p "$normal_dir/initrd-fs.target.requires" -ln -sf ../sysroot-usr.mount "$normal_dir/initrd-fs.target.requires/sysroot-usr.mount" - -# Post-switch-root mount: keep the ISO accessible at /run/iso -cat <<- EOF > "$normal_dir/run-iso.mount" -# Generated by $(basename "$0") -[Unit] -Description=Keep ISO mounted in booted system -After=initrd-switch-root.target -[Mount] -What=${ISO_DEVICE} -Where=/run/iso -Type=iso9660 -Options=ro -DirectoryMode=0755 -[Install] -WantedBy=local-fs.target -EOF - -mkdir -p "$normal_dir/local-fs.target.wants" -ln -sf ../run-iso.mount "$normal_dir/local-fs.target.wants/run-iso.mount" - -echo "ISO mount units generated successfully." diff --git a/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-live-generator b/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-live-generator index 0b50d3c..3295990 100755 --- a/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-live-generator +++ b/mkosi.extra/usr/lib/systemd/system-generators/kde-linux-live-generator @@ -1,7 +1,6 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL # SPDX-FileCopyrightText: 2024 Harald Sitter -# SPDX-FileCopyrightText: 2026 Hadi Chokr set -eux @@ -18,46 +17,25 @@ if ! grep "kde-linux.live=1" /proc/cmdline; then exit 0 fi -# Check if this is ISO boot mode -if grep "kde-linux.iso=1" /proc/cmdline; then - echo "ISO boot mode detected" - - # Mount flatpak from the root filesystem (same as raw mode) - cat <<- EOF > "$normal_dir/var-lib-flatpak.mount" -# Generated by $(basename "$0") -[Unit] -Description=Mount unit for /var/lib/flatpak (ISO mode) -Before=kde-linux-volatile-var-lib-flatpak.service -[Mount] -What=/usr/share/factory/var/lib/flatpak -Where=/var/lib/flatpak -Options=bind -[Install] -WantedBy=multi-user.target -EOF -else - echo "Raw disk boot mode detected" - - # Check for raw live boot - if [ "$(readlink --canonicalize /dev/disk/by-partlabel/KDELinuxLive)" != "$(readlink --canonicalize /dev/gpt-auto-root)" ]; then - echo "gpt-auto-root is not KDELinuxLive" - exit 0 - fi - - # Mount flatpak from raw image factory directory - cat <<- EOF > "$normal_dir/var-lib-flatpak.mount" -# Generated by $(basename "$0") -[Unit] -Description=Mount unit for /var/lib/flatpak (Raw mode) -Before=kde-linux-volatile-var-lib-flatpak.service -[Mount] -What=/usr/share/factory/var/lib/flatpak -Where=/var/lib/flatpak -Options=bind -[Install] -WantedBy=multi-user.target -EOF +if [ "$(readlink --canonicalize /dev/disk/by-partlabel/KDELinuxLive)" != "$(readlink --canonicalize /dev/gpt-auto-root)" ]; then + echo "gpt-auto-root is not KDELinuxLive" + exit 0 fi +cat <<- EOF > "$normal_dir/var-lib-flatpak.mount" +# Generated by $(basename "$0") +[Unit] +Description=Mount unit for /var/lib/flatpak +Before=kde-linux-volatile-var-lib-flatpak.service + +[Mount] +What=/usr/share/factory/var/lib/flatpak +Where=/var/lib/flatpak +Options=bind + +[Install] +WantedBy=multi-user.target +EOF + mkdir "$normal_dir/multi-user.target.wants/" || true ln -sf ../var-lib-flatpak.mount "$normal_dir/multi-user.target.wants/var-lib-flatpak.mount" diff --git a/upload-to-storage.sh b/upload-to-storage.sh index cb5adb8..afeddd3 100755 --- a/upload-to-storage.sh +++ b/upload-to-storage.sh @@ -8,15 +8,16 @@ set -eu # Output Directory OUTDIR="${OUTDIR:-mkosi.output}" -# Reset upload tree +# Do not blow the lid off the storage for now. Reset the tree and only publish a select few files mv upload-tree upload-tree-old || true if [ ! -d upload-tree ]; then mkdir upload-tree - for f in "$OUTDIR"/*.raw "$OUTDIR"/*.iso "$OUTDIR"/*.erofs "$OUTDIR"/*.efi; do + for f in "$OUTDIR"/*.raw "$OUTDIR"/*.erofs "$OUTDIR"/*.efi; do if [[ $f == *.test.raw ]]; then + # Skip test images continue fi - [ -f "$f" ] && mv "$f" upload-tree/ + mv "$f" upload-tree/ done fi @@ -24,6 +25,6 @@ go -C ./token-redeemer/ run . go -C ./uploader/ run . --remote "s3+https://storage.kde.org/ci-artifacts/$CI_PROJECT_PATH/j/$CI_JOB_ID" echo "𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏" -echo "Artifacts available at:" +echo "You can find the raw disk images at:" echo "https://qoomon.github.io/aws-s3-bucket-browser/index.html?bucket=https://storage.kde.org/ci-artifacts/#$CI_PROJECT_PATH/j/$CI_JOB_ID/" echo "𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏" diff --git a/upload.sh b/upload.sh index 5f63d93..29f80ce 100755 --- a/upload.sh +++ b/upload.sh @@ -7,7 +7,7 @@ set -eux OUTDIR=mkosi.output -# SSH configuration +# For the vacuum helper and this script export SSH_IDENTITY="$PWD/.secure_files/ssh.key" export SSH_USER=kdeos export SSH_HOST=origin.files.kde.org @@ -17,60 +17,60 @@ export SSH_REALLY_DELETE=1 chmod 600 "$SSH_IDENTITY" -# Run vacuum helper go -C ./upload-vacuum/ build -o upload-vacuum . ./upload-vacuum/upload-vacuum -# GPG setup -sudo chown -Rvf "$(id -u):$(id -g)" "$PWD/.secure_files" +# The following variables are for this script only. Not shared with the vacuum helper. +sudo chown -Rvf "$(id -u):$(id -g)" "$PWD/.secure_files" # Make sure we have access export GNUPGHOME="$PWD/.secure_files/gpg" gpg --verbose --no-options --homedir="$GNUPGHOME" --import "$PWD/.secure_files/gpg.private.key" - REMOTE_ROOT=$SSH_USER@$SSH_HOST:$SSH_ROOT_PATH REMOTE_PATH=$SSH_USER@$SSH_HOST:$SSH_PATH - -# Add SSH host key +# You can use `ssh-keyscan origin.files.kde.org` to get the host key echo "origin.files.kde.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILUjdH4S7otYIdLUkOZK+owIiByjNQPzGi7GQ5HOWjO6" >> ~/.ssh/known_hosts -# Generate checksums +# The initial SHA256SUMS file is created by the vacuum script based on what is left on the server. We append to it. + sudo chown -R "$USER":"$USER" "$OUTDIR" cd "$OUTDIR" +# We need shell globs here! More readable this way. Ignore shellcheck. +# shellcheck disable=SC2129 sha256sum -- *.efi >> SHA256SUMS sha256sum -- *.tar.zst >> SHA256SUMS sha256sum -- *.erofs >> SHA256SUMS +# Don't put .erofs.caibx into the SHA256SUMS, it will break file matching. +# https://github.com/systemd/systemd/issues/38605 sha256sum -- *-x86-64.caibx >> SHA256SUMS -sha256sum -- *.iso >> SHA256SUMS gpg --homedir="$GNUPGHOME" --output SHA256SUMS.gpg --detach-sign SHA256SUMS -# Upload to SSH server -scp -i "$SSH_IDENTITY" ./*.raw ./*.iso ./*.torrent "$REMOTE_ROOT" +scp -i "$SSH_IDENTITY" ./*.raw ./*.torrent "$REMOTE_ROOT" scp -i "$SSH_IDENTITY" ./*.efi ./*.tar.zst ./*.erofs ./*.caibx "$REMOTE_PATH" -scp -i "$SSH_IDENTITY" SHA256SUMS SHA256SUMS.gpg "$REMOTE_PATH" +scp -i "$SSH_IDENTITY" SHA256SUMS SHA256SUMS.gpg "$REMOTE_PATH" # upload as last artifact to finalize the upload + +# The new s3 based upload system -# S3 upload S3_STORE="s3+https://storage.kde.org/kde-linux/sysupdate/store/" S3_TARGET="s3+https://storage.kde.org/kde-linux/testing/" -## Upload chunks to store +## Upload to the chunk store directly go install -v github.com/folbricht/desync/cmd/desync@latest go -C ../token-redeemer/ run . - ~/go/bin/desync chop \ --concurrency "$(nproc)" \ --store "$S3_STORE" \ ./*-x86-64.caibx \ ./*-x86-64.erofs -## Prepare upload tree +## Prepare the image upload tree cd .. rm -rf upload-tree mkdir -p upload-tree/sysupdate/v2 -mv "$OUTDIR"/*.raw "$OUTDIR"/*.iso "$OUTDIR"/*.torrent upload-tree/ +mv "$OUTDIR"/*.raw "$OUTDIR"/*.torrent upload-tree/ mv "$OUTDIR"/*.efi "$OUTDIR"/*.tar.zst "$OUTDIR"/*.erofs "$OUTDIR"/*.caibx "$OUTDIR"/SHA256SUMS "$OUTDIR"/SHA256SUMS.gpg upload-tree/sysupdate/v2/ -## Upload to S3 +### Upload go -C ./token-redeemer/ run . go -C ./uploader/ run . --remote "$S3_TARGET"