From cfc528f6d017a1b526bb11c7b44722b21496084a Mon Sep 17 00:00:00 2001 From: Thomas Duckworth Date: Tue, 23 Jun 2026 13:36:14 +1000 Subject: [PATCH] Reintroduce OpenQA testing Integrates OpenQA image testing into CI. Builds images once in the imaging job, then passes the resulting image and update channel to the OpenQA pipeline in kde-linux/os-autoinst-distri-kdelinux. For regular test builds, upload the built ISO and a temporary sysupdate tree to ci-artifacts on storage.kde.org. The resulting IMAGE_URL, STAGING_CHANNEL_URL and ephemeral update signing key are exported through the imaging job dotenv artifact so OpenQA can boot the image and run upgrade tests against the build under test. For protected default-branch builds, stage the release artifacts under a per-job staging prefix in the kde-linux bucket. OpenQA tests the staged ISO and sysupdate channel before the publish job is allowed to run. Publishing downloads the staged artifacts, uploads the public ISO/torrent and sysupdate assets to files.kde.org, chunks and uploads the root to the desync chunk store, then merges the staged S3 tree into the live tree on the bucket and regenerates the published SHA256SUMS. --- .gitlab-ci.yml | 79 +++++-- basic-test-efi-addon.sh | 17 -- basic-test.py | 105 --------- build.sh | 6 +- minioauth/go.mod | 31 +++ minioauth/go.mod.license | 2 + minioauth/go.sum | 44 ++++ minioauth/go.sum.license | 2 + minioauth/minioauth.go | 71 ++++++ mkosi.extra/live/usr/lib/basic-test | 57 ----- mkosi.extra/live/usr/lib/live-setup | 4 - .../system/kde-linux-basic-test.service | 10 - publisher/go.mod | 34 +++ publisher/go.mod.license | 2 + publisher/go.sum | 44 ++++ publisher/go.sum.license | 2 + publisher/main.go | 220 +++++++++++++++++ upload-to-storage.sh | 43 +++- upload-vacuum-v3/awsconfig.go | 35 --- upload-vacuum-v3/go.mod | 3 + upload-vacuum-v3/main.go | 32 +-- upload-vacuum/main.go | 14 +- upload.sh | 221 ++++++++++++------ uploader/awsconfig.go | 35 --- uploader/go.mod | 3 + uploader/main.go | 32 +-- 26 files changed, 725 insertions(+), 423 deletions(-) delete mode 100755 basic-test-efi-addon.sh delete mode 100755 basic-test.py create mode 100644 minioauth/go.mod create mode 100644 minioauth/go.mod.license create mode 100644 minioauth/go.sum create mode 100644 minioauth/go.sum.license create mode 100644 minioauth/minioauth.go delete mode 100755 mkosi.extra/live/usr/lib/basic-test delete mode 100644 mkosi.extra/live/usr/lib/systemd/system/kde-linux-basic-test.service create mode 100644 publisher/go.mod create mode 100644 publisher/go.mod.license create mode 100644 publisher/go.sum create mode 100644 publisher/go.sum.license create mode 100644 publisher/main.go delete mode 100644 upload-vacuum-v3/awsconfig.go delete mode 100644 uploader/awsconfig.go diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ceed57d..e45be6a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,19 +10,33 @@ include: stages: - validate - start + - test + - publish imaging: stage: start tags: - VM - amd64 + # IMAGING_MODE selects what we do with the built image: + # test - throwaway image, uploaded to ci-artifacts and given to OpenQA for testing. + # stage - signed and staged to storage for publishing, gated on OpenQA tests. rules: - if: $CI_MERGE_REQUEST_DRAFT == "true" when: manual + variables: + IMAGING_MODE: test - if: $CI_COMMIT_BRANCH =~ /^work\// when: manual - - if: $CI_COMMIT_REF_PROTECTED != 'true' || $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME || $CI_PROJECT_PATH != 'kde-linux/kde-linux' + variables: + IMAGING_MODE: test + - if: $CI_COMMIT_REF_PROTECTED == 'true' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME && $CI_PROJECT_PATH == 'kde-linux/kde-linux' when: always + variables: + IMAGING_MODE: stage + - when: always + variables: + IMAGING_MODE: test allow_failure: false image: storage.kde.org/vm-images/kde-linux-builder id_tokens: &tokens @@ -31,36 +45,67 @@ imaging: variables: &variables RUNNER_AFTER_SCRIPT_TIMEOUT: 1h script: - - sudo cp /etc/pacman.conf.nolinux /etc/pacman.conf # can be dropped once an image rebuild has happened after 2026-05-27 - - sudo ./build.sh --force --debug # bootstap.sh happens as part of CI image build + - sudo ./build.sh --force --debug + - if [ "$IMAGING_MODE" = stage ]; then ./upload.sh --stage; fi + - if [ "$IMAGING_MODE" = test ]; then ./upload-to-storage.sh; fi # Need this to have something to pass to OpenQA. after_script: + - if [ "$IMAGING_MODE" = stage ] && [ "$CI_JOB_STATUS" = success ]; then exit 0; fi + - if [ "$IMAGING_MODE" = test ]; then exit 0; fi - ./upload-to-storage.sh || true - artifacts: &artifacts + artifacts: expire_in: 4 weeks when: always paths: - "**/erofs.log" + reports: + dotenv: build.env -imaging+publish: - stage: start +# Run OpenQA testing on the built image. +trigger-openqa: + stage: test + rules: + - if: $CI_PROJECT_PATH != 'kde-linux/kde-linux' + when: never + - if: $CI_MERGE_REQUEST_DRAFT == "true" + when: manual + - if: $CI_COMMIT_BRANCH =~ /^work\// + when: manual + - when: on_success + needs: + - job: imaging + artifacts: true + trigger: + project: kde-linux/os-autoinst-distri-kdelinux + branch: $OPENQA_BRANCH + strategy: depend + variables: + IMAGE_URL: $IMAGE_URL + STAGING_CHANNEL_URL: $STAGING_CHANNEL_URL + SYSUPDATE_PUBKEY_B64: $SYSUPDATE_PUBKEY_B64 + OPENQA_BRANCH: master + +# Publish the staged image in storage. +publish: + stage: publish tags: - VM - amd64 rules: - # NOTE: these are different from the regular imaging job! - if: $CI_COMMIT_REF_PROTECTED == 'true' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME && $CI_PROJECT_PATH == 'kde-linux/kde-linux' - when: always + when: on_success + needs: + - job: imaging + artifacts: true + - job: trigger-openqa + artifacts: false allow_failure: false image: storage.kde.org/vm-images/kde-linux-builder id_tokens: *tokens variables: *variables script: - - sudo cp /etc/pacman.conf.nolinux /etc/pacman.conf # can be dropped once an image rebuild has happened after 2026-05-27 - - sudo pacman --sync --refresh --noconfirm curl which git - - curl -s https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer | bash - - sudo ./build.sh --force --debug # bootstap.sh happens as part of CI image build - - ./upload.sh - after_script: - # Upload broken images to Harald for inspection - - "[ $CI_JOB_STATUS == 'success' ] || ./upload-to-storage.sh || true" - artifacts: *artifacts + - ./upload.sh --publish + artifacts: + expire_in: 4 weeks + when: always + paths: + - "**/erofs.log" diff --git a/basic-test-efi-addon.sh b/basic-test-efi-addon.sh deleted file mode 100755 index 4064907..0000000 --- a/basic-test-efi-addon.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL -# SPDX-FileCopyrightText: 2025 Harald Sitter -# SPDX-FileCopyrightText: 2026 Hadi Chokr - -set -eux - -# ADDON_DIR is set by basic-test.py to the .extra.d directory inside the mounted ESP -if [ -z "$ADDON_DIR" ]; then - echo "ERROR: ADDON_DIR environment variable not set" - exit 1 -fi - -# Create the addon UKI (systemd-stub addon) that appends the test cmdline -ukify build \ - --cmdline "kde-linux.basic-test=1 kde-linux.basic-test-callback=http://10.0.2.2:${PORT}/good" \ - --output "$ADDON_DIR/basic-test.addon.efi" diff --git a/basic-test.py b/basic-test.py deleted file mode 100755 index 254bbb5..0000000 --- a/basic-test.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL -# SPDX-FileCopyrightText: 2025 Harald Sitter -# SPDX-FileCopyrightText: 2026 Hadi Chokr - -import atexit -import http.server -import sys -import subprocess -import os -import time -import tempfile -from pathlib import Path - -class Handler(http.server.BaseHTTPRequestHandler): - def do_GET(self): - if self.path == '/good': - sys.exit(0) - if self.path == '/bad': - print("==Received /bad callback==") - content_len = int(self.headers.get('Content-Length')) - body = self.rfile.read(content_len) - print(body.decode('utf-8')) - sys.exit(1) - self.send_response(200) - self.end_headers() - -server = http.server.HTTPServer(server_address=('', 0), RequestHandlerClass=Handler) -print("serving at port", server.server_port) - -img = sys.argv[1] -if not img: - print("No image specified") - sys.exit(1) - -efi_base = sys.argv[2] -if not efi_base: - print("No EFI base image specified") - sys.exit(1) - -# Always test as ISO (a valid .iso9660 is also a valid .raw) -test_img = img.replace('.raw', '.test.iso').replace('.iso', '.test.iso') -subprocess.check_call(['cp', '--reflink=auto', img, test_img]) - -# Inject the EFI addon into the ESP partition of the test ISO -script_dir = os.path.dirname(os.path.realpath(__file__)) -addon_src = f'{script_dir}/basic-test-efi-addon.sh' - -with tempfile.TemporaryDirectory() as mnt: - # Find the ESP partition offset and size using sfdisk - sfdisk = subprocess.check_output(['sfdisk', '--json', test_img]).decode() - import json - parts = json.loads(sfdisk)['partitiontable']['partitions'] - esp = next(p for p in parts if p.get('type', '') == 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B') - sector_size = json.loads(sfdisk)['partitiontable']['sectorsize'] - offset = esp['start'] * sector_size - size = esp['size'] * sector_size - - subprocess.check_call([ - 'mount', '-o', f'loop,offset={offset},sizelimit={size}', - test_img, mnt - ]) - try: - efi_extra_dir = f'{mnt}/EFI/Linux/{efi_base}.extra.d' - os.makedirs(efi_extra_dir, exist_ok=True) - subprocess.check_call([ - 'bash', addon_src - ], env={ - 'PORT': str(server.server_port), - 'UKI': efi_base, - 'ADDON_DIR': efi_extra_dir, - }) - finally: - subprocess.check_call(['umount', mnt]) - -qemu_cmd = [ - "qemu-system-x86_64", - "-cdrom", test_img, - "-m", "4G", - "-enable-kvm", - "-cpu", "host", - "-bios", "/usr/share/OVMF/x64/OVMF.4m.fd", -] - -# I ought to point out that this leaks the process in case of failure. It will however get reaped by the docker container shutdown. -qemu = subprocess.Popen(qemu_cmd) -atexit.register(lambda: (qemu.kill())) - -def on_timeout(): - print("\n\n\n== Test timed out ==") - print("Download the image for inspection from http://images.kde-linux.haraldsitter.eu/") - print("(location may have changed to something like https://qoomon.github.io/aws-s3-bucket-browser/index.html?bucket=https://storage.kde.org/ci-artifacts/#).") - print("Once downloaded run:\n") - print(f"./basic-test.py {img} {efi_base}\n\n\n") - - qemu.kill() - sys.exit(2) - -server.timeout = 5 * 60 # 5 minutes -server.handle_timeout = on_timeout -while True: # kinda garbage but there seems to be no nice (non-private) poll-or-timeout api - server.handle_request() - time.sleep(8) -qemu.kill() -sys.exit(1) # if we get here we timed out = fail diff --git a/build.sh b/build.sh index 224c6a3..e7865ec 100755 --- a/build.sh +++ b/build.sh @@ -50,7 +50,7 @@ ROOTFS_CAIBX=${OUTPUT}_root-x86-64.caibx ROOTFS_EROFS=${OUTPUT}_root-x86-64.erofs # Output erofs image path ISO="${OUTPUT}.iso" # both a valid GPT disk image and a bootable ISO -EFI_BASE=kde-linux_${VERSION} # Base name of the UKI in the image's ESP (exported so it can be used in basic-test-efi-addon.sh) +EFI_BASE=kde-linux_${VERSION} # Base name of the UKI in the image's ESP EFI=${EFI_BASE}+3.efi # Name of primary UKI in the image's ESP (with tries counter for installed system) LIVE_EFI=${EFI_BASE}.efi # Name of live UKI in the ESP (no tries counter — ESP is read-only on ISO) @@ -204,10 +204,6 @@ systemd-repart \ --el-torito-publisher="KDE" \ "$ISO" -# Test the ISO (which is also a valid GPT image so no need to test as .raw separately) -./basic-test.py "$ISO" "$LIVE_EFI" || exit 1 -rm ./mkosi.output/*.test.iso - # Incase the owner is root chown -R user:user mkosi.output diff --git a/minioauth/go.mod b/minioauth/go.mod new file mode 100644 index 0000000..49a5020 --- /dev/null +++ b/minioauth/go.mod @@ -0,0 +1,31 @@ +module invent.kde.org/kde-linux/kde-linux/minioauth + +go 1.23.0 + +toolchain go1.24.4 + +require ( + github.com/minio/minio-go/v7 v7.0.95 + gopkg.in/ini.v1 v1.67.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/minio/crc64nvme v1.0.2 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.3.0 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.26.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/minioauth/go.mod.license b/minioauth/go.mod.license new file mode 100644 index 0000000..65051d9 --- /dev/null +++ b/minioauth/go.mod.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: CC0-1.0 +SPDX-FileCopyrightText: None diff --git a/minioauth/go.sum b/minioauth/go.sum new file mode 100644 index 0000000..577a665 --- /dev/null +++ b/minioauth/go.sum @@ -0,0 +1,44 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= +github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/minioauth/go.sum.license b/minioauth/go.sum.license new file mode 100644 index 0000000..65051d9 --- /dev/null +++ b/minioauth/go.sum.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: CC0-1.0 +SPDX-FileCopyrightText: None diff --git a/minioauth/minioauth.go b/minioauth/minioauth.go new file mode 100644 index 0000000..3df9643 --- /dev/null +++ b/minioauth/minioauth.go @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL +// SPDX-FileCopyrightText: 2025 Harald Sitter +// SPDX-FileCopyrightText: 2026 Thomas Duckworth + +package minioauth + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "gopkg.in/ini.v1" +) + +type AWSSection struct { + AccessKeyID string `ini:"aws_access_key_id"` + SecretKey string `ini:"aws_secret_access_key"` + SessionToken string `ini:"aws_session_token"` +} + +func readConfigAWS(section string) (AWSSection, error) { + awsSection := AWSSection{} + + awsConfigPath := filepath.Join(os.Getenv("HOME"), ".aws", "credentials") + cfg, err := ini.Load(awsConfigPath) + if err != nil { + return awsSection, fmt.Errorf("failed to load AWS credentials file: %w", err) + } + + err = cfg.Section(section).MapTo(&awsSection) + if err != nil { + return awsSection, fmt.Errorf("failed to map AWS credentials section: %w", err) + } + + return awsSection, nil +} + +func Connect(endpoint string) (*minio.Client, error) { + awsSection, err := readConfigAWS("default") + if err != nil { + return nil, err + } + if awsSection.AccessKeyID == "" { + return nil, errors.New("AWS access key ID is empty") + } + if awsSection.SecretKey == "" { + return nil, errors.New("AWS secret access key is empty") + } + if awsSection.SessionToken == "" { + return nil, errors.New("AWS session token is empty") + } + + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(awsSection.AccessKeyID, awsSection.SecretKey, awsSection.SessionToken), + Secure: true, + TrailingHeaders: true, + }) + if err != nil { + return nil, fmt.Errorf("failed to create MinIO client: %w", err) + } + + return minioClient, nil +} + +func ListBuckets(client *minio.Client) ([]minio.BucketInfo, error) { + return client.ListBuckets(context.Background()) +} diff --git a/mkosi.extra/live/usr/lib/basic-test b/mkosi.extra/live/usr/lib/basic-test deleted file mode 100755 index aa52c30..0000000 --- a/mkosi.extra/live/usr/lib/basic-test +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL -# SPDX-FileCopyrightText: 2025 Harald Sitter -# SPDX-FileCopyrightText: 2026 Hadi Chokr - -import subprocess -import time -from pathlib import Path -import json - -# Units that are known to fail on a read-only ISO (e.g., /boot is not writable) -IGNORED_FAILED_UNITS = { - "systemd-boot-update.service", - "systemd-boot-random-seed.service", -} - -callback = None -with open('/proc/cmdline') as cmdline: - data = cmdline.read() - entries = data.split(' ') - for entry in entries: - if entry.startswith('kde-linux.basic-test-callback='): - # We don't support quoting of any kind. - callback = entry.split('=', 2)[1] - callback = callback.strip() - break - -if callback is None: - print('No callback specified. set kde-linux.basic-test-callback=') - exit(1) - -bad_callback = callback.replace('/good', '/bad') - -while True: - # 1000 is the uid of the live user. always. - if Path('/run/user/1000/kde-linux-bless-session').is_file(): - failed = json.loads(subprocess.check_output(["systemctl", "--failed", "--output=json"])) - # Filter out ignored units - relevant_failed = [unit for unit in failed if unit['unit'] not in IGNORED_FAILED_UNITS] - if len(relevant_failed) > 0: - with open('data.file', 'w') as f: - for unit in relevant_failed: - f.write("\n") - f.write(json.dumps(unit)) - f.write("\n") - try: - f.write(subprocess.check_output(['journalctl', '--no-pager', f'_SYSTEMD_UNIT={unit["unit"]}']).decode('utf-8')) - except Exception as e: - f.write(f"Failed to get journal for {unit['unit']}: {e}\n") - f.write("\n") # make sure we have a final newline - subprocess.check_call(['curl', - '--request', 'GET', - '--data-binary', '@data.file', # use binary as otherwise curl will eat the newlines - bad_callback]) - else: - subprocess.check_call(['curl', callback]) - time.sleep(30) diff --git a/mkosi.extra/live/usr/lib/live-setup b/mkosi.extra/live/usr/lib/live-setup index dea1f09..75a05d8 100755 --- a/mkosi.extra/live/usr/lib/live-setup +++ b/mkosi.extra/live/usr/lib/live-setup @@ -90,7 +90,3 @@ cp /usr/share/applications/calamares.desktop /home/live/Desktop/ chmod +x /home/live/Desktop/calamares.desktop # without this there'd be a dialog asking if it really should execute chown -R live:live /home/live/ - -# Start the basic-test (if it wants to be started) -systemctl daemon-reload -systemctl start --no-block kde-linux-basic-test.service diff --git a/mkosi.extra/live/usr/lib/systemd/system/kde-linux-basic-test.service b/mkosi.extra/live/usr/lib/systemd/system/kde-linux-basic-test.service deleted file mode 100644 index 1e448aa..0000000 --- a/mkosi.extra/live/usr/lib/systemd/system/kde-linux-basic-test.service +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL -# SPDX-FileCopyrightText: 2025 Harald Sitter - -[Unit] -Description=Test of KDE Linux disk image -ConditionKernelCommandLine=kde-linux.basic-test=1 - -[Service] -Type=simple -ExecStart=/usr/lib/basic-test diff --git a/publisher/go.mod b/publisher/go.mod new file mode 100644 index 0000000..2002b77 --- /dev/null +++ b/publisher/go.mod @@ -0,0 +1,34 @@ +module invent.kde.org/kde-linux/kde-linux/publisher + +go 1.23.0 + +toolchain go1.24.4 + +require ( + github.com/minio/minio-go/v7 v7.0.95 + invent.kde.org/kde-linux/kde-linux/minioauth v0.0.0 + gopkg.in/ini.v1 v1.67.0 +) + +replace invent.kde.org/kde-linux/kde-linux/minioauth => ../minioauth + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/minio/crc64nvme v1.0.2 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.3.0 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.26.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/publisher/go.mod.license b/publisher/go.mod.license new file mode 100644 index 0000000..23c1849 --- /dev/null +++ b/publisher/go.mod.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: CC0-1.0 +SPDX-FileCopyrightText: none diff --git a/publisher/go.sum b/publisher/go.sum new file mode 100644 index 0000000..577a665 --- /dev/null +++ b/publisher/go.sum @@ -0,0 +1,44 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= +github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/publisher/go.sum.license b/publisher/go.sum.license new file mode 100644 index 0000000..23c1849 --- /dev/null +++ b/publisher/go.sum.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: CC0-1.0 +SPDX-FileCopyrightText: none diff --git a/publisher/main.go b/publisher/main.go new file mode 100644 index 0000000..32a39f7 --- /dev/null +++ b/publisher/main.go @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL +// SPDX-FileCopyrightText: 2024-2025 Harald Sitter +// SPDX-FileCopyrightText: 2026 Thomas Duckworth + +package main + +import ( + "context" + "flag" + "log" + "net/url" + "path/filepath" + "sort" + "strings" + + "invent.kde.org/kde-linux/kde-linux/minioauth" + + "github.com/minio/minio-go/v7" +) + +func connectToMinIO(endpoint string) *minio.Client { + minioClient, err := minioauth.Connect(endpoint) + if err != nil { + log.Fatalln("Failed to connect to MinIO:", err) + } + + buckets, err := minioauth.ListBuckets(minioClient) + if err != nil { + log.Fatalln("Failed to list buckets:", err) + } + for _, bucket := range buckets { + log.Println(bucket) + } + + return minioClient +} + +func prefix(path string) string { + trimmed := strings.Trim(path, "/") + if trimmed == "" { + return "" + } + return trimmed + "/" +} + +func destinationPrefix(path string) string { + path = strings.Trim(path, "/") + if path == "" { + log.Fatalln("Remote path must be a non-empty staging path") + } + + parts := strings.Split(path, "/") + for i, part := range parts { + if part != "staging" { + continue + } + if i == len(parts)-1 { + log.Fatalln("Remote path must include a staging directory name") + } + if i == 0 { + return "" + } + return strings.Join(parts[:i], "/") + "/" + } + + log.Fatalln("Remote path must contain a staging directory") + return "" +} + +func isChannelCommitMarker(key string) bool { + base := filepath.Base(key) + return base == "SHA256SUMS" || base == "SHA256SUMS.gpg" +} + +func removeObject(ctx context.Context, client *minio.Client, bucket string, objectKey string) { + log.Printf("Removing staged object %s", objectKey) + if err := client.RemoveObject(ctx, bucket, objectKey, minio.RemoveObjectOptions{}); err != nil { + log.Fatalf("Failed to remove %s: %v", objectKey, err) + } +} + +// publish merges a staging tree into its channel root. +func publish(client *minio.Client, bucket string, path string) { + ctx := context.Background() + srcPrefix := prefix(path) + + destPrefix := destinationPrefix(path) + + objects := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{ + Prefix: srcPrefix, + Recursive: true, + }) + + var objectKeys []string + for object := range objects { + if object.Err != nil { + log.Fatalln("Failed to list objects:", object.Err) + } + objectKeys = append(objectKeys, object.Key) + } + sort.Strings(objectKeys) + + for _, objectKey := range objectKeys { + rel := strings.TrimPrefix(objectKey, srcPrefix) + if rel == "" { + continue + } + if isChannelCommitMarker(rel) { + log.Printf("Skipping staged commit marker %s", objectKey) + removeObject(ctx, client, bucket, objectKey) + continue + } + dstKey := destPrefix + rel + if dstKey == objectKey { + continue + } + + log.Printf("Publishing %s -> %s", objectKey, dstKey) + + _, err := client.ComposeObject(ctx, + minio.CopyDestOptions{Bucket: bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: bucket, Object: objectKey}, + ) + if err != nil { + log.Fatalf("Failed to copy %s to %s: %v", objectKey, dstKey, err) + } + + removeObject(ctx, client, bucket, objectKey) + } +} + +// artifactSuffixes are the build artifacts we download so they can be pushed to +// files.kde.org and the chunk store. sha256sums are regenerated at publish time, so they're excluded here +var artifactSuffixes = []string{".iso", ".torrent", ".efi", ".tar.zst", ".erofs", ".caibx"} + +func isArtifact(key string) bool { + for _, suffix := range artifactSuffixes { + if strings.HasSuffix(key, suffix) { + return true + } + } + return false +} + +// download gets every build artifact under path to publish to files.kde.org and the chunk store +func download(client *minio.Client, bucket string, path string, output string) { + ctx := context.Background() + + objects := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{ + Prefix: prefix(path), + Recursive: true, + }) + + for object := range objects { + if object.Err != nil { + log.Fatalln("Failed to list objects:", object.Err) + } + + if !isArtifact(object.Key) { + continue + } + + dst := filepath.Join(output, filepath.Base(object.Key)) + log.Printf("Downloading %s -> %s", object.Key, dst) + if err := client.FGetObject(ctx, bucket, object.Key, dst, minio.GetObjectOptions{}); err != nil { + log.Fatalf("Failed to download %s: %v", object.Key, err) + } + } +} + +// Either downloads every build artifact in the staging directory into --output, +// or promotes the staging tree into the channel root. +func main() { + remote := flag.String("remote", "", "remote url to publish from, e.g. s3+https://storage.kde.org/kde-linux/staging/1") + output := flag.String("output", ".", "directory to download artifacts into") + downloadMode := flag.Bool("download", false, "download artifacts from the staging tree") + promoteMode := flag.Bool("promote", false, "promote the staging tree into the channel root") + flag.Parse() + + if *downloadMode == *promoteMode { + log.Fatalln("Must choose exactly one of --download or --promote") + } + + remoteURI, err := url.Parse(*remote) + if err != nil { + log.Fatalln("Failed to parse remote URL:", err) + } + if remoteURI.Scheme != "s3+https" { + log.Fatalln("Unsupported remote scheme:", remoteURI.Scheme) + } + if remoteURI.Host != "storage.kde.org" { + log.Fatalln("Unsupported remote host:", remoteURI.Host) + } + parts := strings.SplitN(remoteURI.Path[1:], "/", 2) + if len(parts) != 2 { + log.Fatalln("Invalid remote path, expected format: /bucket/path") + } + bucket := parts[0] + path := parts[1] + if bucket == "" { + log.Fatalln("Invalid remote path, expected format: /bucket/path") + } + if path == "" { + log.Println("Warning: path is empty, uploading to bucket root") + path = "/" + } + + log.Println("Connecting to MinIO at", remoteURI.Host) + minioClient := connectToMinIO(remoteURI.Host) + + if *downloadMode { + log.Println("Downloading artifacts from bucket", bucket, "with path prefix", path, "into", *output) + download(minioClient, bucket, path, *output) + } + + if *promoteMode { + log.Println("Publishing to bucket", bucket, "from source path", path) + publish(minioClient, bucket, path) + } +} diff --git a/upload-to-storage.sh b/upload-to-storage.sh index b8cb2f6..985272a 100755 --- a/upload-to-storage.sh +++ b/upload-to-storage.sh @@ -8,23 +8,46 @@ set -eu # Output Directory OUTDIR="${OUTDIR:-mkosi.output}" -# Do not blow the lid off the storage for now. Reset the tree and only publish a select few files +# We need to wire up an ephemeral image-signing key pair for OpenQA. We can't use the production key, but we still need +# to test if the image can be upgraded to. Hence, create a key pair and pass it into OpenQA, where it will be injected +# into the system. +GNUPGHOME="$PWD/.openqa-gpg" +rm -rf "$GNUPGHOME" +mkdir -p "$GNUPGHOME" +chmod 700 "$GNUPGHOME" +gpg --batch --pinentry-mode loopback --homedir="$GNUPGHOME" --passphrase "" --quick-generate-key "KDE Linux openQA " ed25519 sign 3d +SYSUPDATE_PUBKEY_B64=$(gpg --homedir="$GNUPGHOME" --export --armor "KDE Linux openQA " | base64 -w0) + mv upload-tree upload-tree-old || true if [ ! -d upload-tree ]; then - mkdir upload-tree - for f in "$OUTDIR"/*.iso "$OUTDIR"/*.erofs "$OUTDIR"/*.efi; do - if [[ $f == *.test.iso ]]; then - # Skip test images - continue - fi - mv "$f" upload-tree/ - done + mkdir -p upload-tree/sysupdate/v2 + mv "$OUTDIR"/*.iso upload-tree/ + mv "$OUTDIR"/*.efi "$OUTDIR"/*.tar.zst "$OUTDIR"/*.erofs "$OUTDIR"/*.caibx upload-tree/sysupdate/v2/ + ( + cd upload-tree/sysupdate/v2 + # shellcheck disable=SC2129 + sha256sum -- *.efi >> SHA256SUMS + sha256sum -- *.tar.zst >> SHA256SUMS + sha256sum -- *.erofs >> SHA256SUMS + # Don't put .erofs.caibx into SHA256SUMS, it will break file matching. + # https://github.com/systemd/systemd/issues/38605 + sha256sum -- *-x86-64.caibx >> SHA256SUMS + + # Sign, so test images can actually verify and upgrade from artifacts. + gpg --homedir="$GNUPGHOME" --output SHA256SUMS.gpg --detach-sign SHA256SUMS + ) fi go -C ./token-redeemer/ run . go -C ./uploader/ run . --remote "s3+https://storage.kde.org/ci-artifacts/$CI_PROJECT_PATH/j/$CI_JOB_ID" +# Point OpenQA at the image we just uploaded. +ISO_FILE=$(find upload-tree -maxdepth 1 -name '*.iso' | head -1 | xargs -r basename) +echo "IMAGE_URL=https://storage.kde.org/ci-artifacts/$CI_PROJECT_PATH/j/$CI_JOB_ID/$ISO_FILE" >> build.env +echo "STAGING_CHANNEL_URL=https://storage.kde.org/ci-artifacts/$CI_PROJECT_PATH/j/$CI_JOB_ID/sysupdate/v2/" >> build.env +echo "SYSUPDATE_PUBKEY_B64=$SYSUPDATE_PUBKEY_B64" >> build.env + echo "𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏" -echo "You can find the raw disk images at:" +echo "You can find the raw disk images and sysupdate tree at:" echo "https://qoomon.github.io/aws-s3-bucket-browser/index.html?bucket=https://storage.kde.org/ci-artifacts/#$CI_PROJECT_PATH/j/$CI_JOB_ID/" echo "𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏" diff --git a/upload-vacuum-v3/awsconfig.go b/upload-vacuum-v3/awsconfig.go deleted file mode 100644 index 74a797b..0000000 --- a/upload-vacuum-v3/awsconfig.go +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL -// SPDX-FileCopyrightText: 2025 Harald Sitter - -package main - -import ( - "errors" - "os" - "path/filepath" - - "gopkg.in/ini.v1" -) - -type AWSSection struct { - AccessKeyId string `ini:"aws_access_key_id"` - SecretKey string `ini:"aws_secret_access_key"` - SessionToken string `ini:"aws_session_token"` -} - -func readConfigAWS(section string) (AWSSection, error) { - awsSection := AWSSection{} - - awsConfigPath := filepath.Join(os.Getenv("HOME"), ".aws", "credentials") - cfg, err := ini.Load(awsConfigPath) - if err != nil { - return awsSection, errors.New("failed to load AWS credentials file: " + err.Error()) - } - - err = cfg.Section(section).MapTo(&awsSection) - if err != nil { - return awsSection, errors.New("failed to map AWS credentials section: " + err.Error()) - } - - return awsSection, nil -} diff --git a/upload-vacuum-v3/go.mod b/upload-vacuum-v3/go.mod index 1f6bc1c..f4c3ad4 100644 --- a/upload-vacuum-v3/go.mod +++ b/upload-vacuum-v3/go.mod @@ -6,10 +6,13 @@ require ( github.com/minio/minio-go/v7 v7.0.100 github.com/pkg/sftp v1.13.6 golang.org/x/crypto v0.50.0 + invent.kde.org/kde-linux/kde-linux/minioauth v0.0.0 gopkg.in/ini.v1 v1.67.1 gopkg.in/yaml.v2 v2.4.0 ) +replace invent.kde.org/kde-linux/kde-linux/minioauth => ../minioauth + require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-ini/ini v1.67.0 // indirect diff --git a/upload-vacuum-v3/main.go b/upload-vacuum-v3/main.go index a4c5676..f9af9ad 100644 --- a/upload-vacuum-v3/main.go +++ b/upload-vacuum-v3/main.go @@ -17,9 +17,9 @@ import ( "strings" "gopkg.in/yaml.v2" + "invent.kde.org/kde-linux/kde-linux/minioauth" "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" ) type release struct { @@ -29,36 +29,12 @@ type release struct { func connectToMinIO() *minio.Client { endpoint := "storage.kde.org" - // Below is a huge code dupe from uploader/main.go, should be shared somehow - awsSection, err := readConfigAWS("default") + minioClient, err := minioauth.Connect(endpoint) if err != nil { - log.Fatalln("Failed to read AWS config:", err) - } - accessKeyID := awsSection.AccessKeyId - if accessKeyID == "" { - log.Fatalln("AWS access key ID is empty") - } - secretAccessKey := awsSection.SecretKey - if secretAccessKey == "" { - log.Fatalln("AWS secret access key is empty") - } - sessionToken := awsSection.SessionToken - if secretAccessKey == "" { - log.Fatalln("AWS session token is empty") - } - useSSL := true - - // Initialize minio client object. - minioClient, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, sessionToken), - Secure: useSSL, - TrailingHeaders: true, - }) - if err != nil { - log.Fatalln("Failed to create MinIO client:", err) + log.Fatalln("Failed to connect to MinIO:", err) } - buckets, err := minioClient.ListBuckets(context.Background()) + buckets, err := minioauth.ListBuckets(minioClient) if err != nil { log.Fatalln("Failed to list buckets:", err) } diff --git a/upload-vacuum/main.go b/upload-vacuum/main.go index c64cacb..45fbea8 100644 --- a/upload-vacuum/main.go +++ b/upload-vacuum/main.go @@ -33,8 +33,8 @@ func connectToHost(user, host, identity string) (*ssh.Client, *ssh.Session, erro log.Fatalf("unable to parse private key: %v", err) } - // You can use `ssh-keyscan origin.files.kde.org` to get the host key - _, _, hostKey, _, _, err := ssh.ParseKnownHosts([]byte("origin.files.kde.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILUjdH4S7otYIdLUkOZK+owIiByjNQPzGi7GQ5HOWjO6")) + // You can use `ssh-keyscan tinami.kde.org` to get the host key + _, _, hostKey, _, _, err := ssh.ParseKnownHosts([]byte("tinami.kde.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILUjdH4S7otYIdLUkOZK+owIiByjNQPzGi7GQ5HOWjO6")) if err != nil { log.Fatalf("unable to parse host public key: %v", err) } @@ -120,6 +120,7 @@ func readSHA256SUMS(client *sftp.Client, path string) map[string]string { func readSHA256s(toKeep []string, releases map[string]release, existingSums map[string]string) []string { sha256s := []string{} + rootPath := strings.TrimRight(os.Getenv("SSH_ROOT_PATH"), "/") for _, key := range toKeep { artifacts := releases[key].artifacts sort.Strings(artifacts) // Sort artifacts to ensure consistent order @@ -129,7 +130,7 @@ func readSHA256s(toKeep []string, releases map[string]release, existingSums map[ // https://github.com/systemd/systemd/issues/38605 continue } - if strings.HasPrefix(artifact, "/home/kdeos/kde-linux/kdeos_") { + if rootPath != "" && strings.HasPrefix(artifact, rootPath+"/kdeos_") { // HACK 2025-08-20 sha256s of the files are broken, only drop this if when they are fixed (possibly just a matter of time) continue } @@ -235,7 +236,10 @@ func main() { identity := os.Getenv("SSH_IDENTITY") host := os.Getenv("SSH_HOST") user := os.Getenv("SSH_USER") - path := os.Getenv("SSH_PATH") + path := os.Getenv("SSH_SYSUPDATE_PATH") + if path == "" { + path = os.Getenv("SSH_PATH") + } root_path := os.Getenv("SSH_ROOT_PATH") var errs []error @@ -249,7 +253,7 @@ func main() { errs = append(errs, errors.New("SSH_USER not set")) } if path == "" { - errs = append(errs, errors.New("SSH_PATH not set")) + errs = append(errs, errors.New("SSH_SYSUPDATE_PATH or SSH_PATH not set")) } if root_path == "" { errs = append(errs, errors.New("SSH_ROOT_PATH not set")) diff --git a/upload.sh b/upload.sh index 4efe9c1..8b145e9 100755 --- a/upload.sh +++ b/upload.sh @@ -2,89 +2,174 @@ # SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL # SPDX-FileCopyrightText: 2024 Harald Sitter # SPDX-FileCopyrightText: 2026 Hadi Chokr +# SPDX-FileCopyrightText: 2026 Thomas Duckworth set -eux +if [[ -z "${1:-}" ]]; then + echo "Choice between --stage and --publish must be provided as argument." >&2 + exit 1 +fi + +STAGE= +PUBLISH= +while [[ $# -gt 0 ]]; do + case "$1" in + --stage) + STAGE=1 + shift + ;; + --publish) + PUBLISH=1 + shift + ;; + *) + echo "Unknown option $1." + exit 1 + ;; + esac +done + +if [[ "$STAGE" -eq 1 && "$PUBLISH" -eq 1 ]] || [[ "$STAGE" -ne 1 && "$PUBLISH" -ne 1 ]]; then + echo "Must choose exactly one of --stage or --publish." + exit 1 +fi + OUTDIR=mkosi.output # For the vacuum helper and this script export SSH_IDENTITY="$PWD/.secure_files/ssh.key" export SSH_USER=kdeos -export SSH_HOST=origin.files.kde.org -export SSH_ROOT_PATH=/home/kdeos/kde-linux/ -export SSH_PATH=$SSH_ROOT_PATH/sysupdate/v2/ +export SSH_HOST=tinami.kde.org +export SSH_ROOT_PATH=/srv/archives/files/kde-linux +export SSH_SYSUPDATE_PATH=$SSH_ROOT_PATH/sysupdate/v2/ export SSH_REALLY_DELETE=1 - -chmod 600 "$SSH_IDENTITY" - -go -C ./upload-vacuum/ build -o upload-vacuum . -./upload-vacuum/upload-vacuum - -# The following variables are for this script only. Not shared with the vacuum helper. -sudo chown -Rvf "$(id -u):$(id -g)" "$PWD/.secure_files" # Make sure we have access -export GNUPGHOME="$PWD/.secure_files/gpg" -gpg --verbose --no-options --homedir="$GNUPGHOME" --import "$PWD/.secure_files/gpg.private.key" -REMOTE_ROOT=$SSH_USER@$SSH_HOST:$SSH_ROOT_PATH -REMOTE_PATH=$SSH_USER@$SSH_HOST:$SSH_PATH -# You can use `ssh-keyscan origin.files.kde.org` to get the host key -echo "origin.files.kde.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILUjdH4S7otYIdLUkOZK+owIiByjNQPzGi7GQ5HOWjO6" >> ~/.ssh/known_hosts - -# The initial SHA256SUMS file is created by the vacuum script based on what is left on the server. We append to it. - -sudo chown -R "$USER":"$USER" "$OUTDIR" -cd "$OUTDIR" - -# We need shell globs here! More readable this way. Ignore shellcheck. -# shellcheck disable=SC2129 -sha256sum -- *.efi >> SHA256SUMS -sha256sum -- *.tar.zst >> SHA256SUMS -sha256sum -- *.erofs >> SHA256SUMS -# Don't put .erofs.caibx into the SHA256SUMS, it will break file matching. -# https://github.com/systemd/systemd/issues/38605 -sha256sum -- *-x86-64.caibx >> SHA256SUMS - -gpg --homedir="$GNUPGHOME" --output SHA256SUMS.gpg --detach-sign SHA256SUMS - -scp -i "$SSH_IDENTITY" ./*.iso ./*.torrent "$REMOTE_ROOT" -scp -i "$SSH_IDENTITY" ./*.efi ./*.tar.zst ./*.erofs ./*.caibx "$REMOTE_PATH" -scp -i "$SSH_IDENTITY" SHA256SUMS SHA256SUMS.gpg "$REMOTE_PATH" # upload as last artifact to finalize the upload - -# The new s3 based upload system - export VACUUM_REALLY_DELETE=1 -S3_STORE="s3+https://storage.kde.org/kde-linux/sysupdate/store/" +export GNUPGHOME="$PWD/.secure_files/gpg" + +# upload tree built during staging +V2_TREE="upload-tree/sysupdate/v2" + S3_TARGET="s3+https://storage.kde.org/kde-linux/" +S3_STORE="${S3_TARGET}sysupdate/store/" +S3_TARGET_STAGING="${S3_TARGET_STAGING:-}" -## Upload to the chunk store directly -go install -v github.com/folbricht/desync/cmd/desync@latest -export PATH="$HOME/go/bin:$PATH" -go -C ../token-redeemer/ run . -desync chop \ - --concurrency "$(nproc)" \ - --store "$S3_STORE" \ - ./*-x86-64.caibx \ - ./*-x86-64.erofs +# files.kde.org scp targets. We don't stage on files.kde.org, only on the storage.kde.org bucket. +# We download from the bucket then upload directly to files.kde.org to publish there. +REMOTE_ROOT_PATH=$SSH_USER@$SSH_HOST:$SSH_ROOT_PATH +REMOTE_SYSUPDATE_PATH=$SSH_USER@$SSH_HOST:$SSH_SYSUPDATE_PATH +# You can use `ssh-keyscan tinami.kde.org` to get the host key +echo "tinami.kde.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILUjdH4S7otYIdLUkOZK+owIiByjNQPzGi7GQ5HOWjO6" >> ~/.ssh/known_hosts -## Prepare the image upload tree -cd .. -rm -rf upload-tree -V2_TREE="upload-tree/testing/sysupdate/v2" -mkdir -p "$V2_TREE" +stage() { + S3_TARGET_STAGING="${S3_TARGET_STAGING:-${S3_TARGET}testing/staging/${CI_PIPELINE_ID}/${CI_JOB_ID}}" -mv "$OUTDIR"/*.iso "$OUTDIR"/*.torrent upload-tree/testing/ -mv "$OUTDIR"/*.efi "$OUTDIR"/*.tar.zst "$OUTDIR"/*.erofs "$OUTDIR"/*.caibx "$V2_TREE/" + # Stage the freshly built image into the bucket. + sudo chown -R "$USER":"$USER" "$OUTDIR" -### Upload -go -C ./token-redeemer/ run . -go -C ./uploader/ run . --remote "$S3_TARGET" + ( + cd "$OUTDIR" + # We need shell globs here! More readable this way. Ignore shellcheck. + # shellcheck disable=SC2129 + sha256sum -- *.efi >> SHA256SUMS + sha256sum -- *.tar.zst >> SHA256SUMS + sha256sum -- *.erofs >> SHA256SUMS + # Don't put .erofs.caibx into the SHA256SUMS, it will break file matching. + # https://github.com/systemd/systemd/issues/38605 + sha256sum -- *-x86-64.caibx >> SHA256SUMS -### Clean up previous images (this replaces the upload-tree with only SHA256SUMS!) -go -C ./token-redeemer/ run . -go -C ./upload-vacuum-v3/ run . + gpg --homedir="$GNUPGHOME" --output SHA256SUMS.gpg --detach-sign SHA256SUMS + ) -#### Sign SHA256SUMS -gpg --homedir="$GNUPGHOME" --output "$V2_TREE/SHA256SUMS.gpg" --detach-sign "$V2_TREE/SHA256SUMS" + # Prepare the staging upload tree. + rm -rf upload-tree + mkdir -p "$V2_TREE" + mv "$OUTDIR"/*.iso "$OUTDIR"/*.torrent upload-tree/ + mv "$OUTDIR"/*.efi "$OUTDIR"/*.tar.zst "$OUTDIR"/*.erofs "$OUTDIR"/*.caibx "$V2_TREE/" + mv "$OUTDIR"/SHA256SUMS "$OUTDIR"/SHA256SUMS.gpg "$V2_TREE/" -#### Upload SHA256SUMS -go -C ./token-redeemer/ run . -go -C ./uploader/ run . --remote "$S3_TARGET" + # Upload to the per-job staging prefix in the bucket. + go -C ./token-redeemer/ run . + go -C ./uploader/ run . --remote "$S3_TARGET_STAGING" + + # Emit the exact staging target for the publish job, plus public URLs for + # OpenQA to test the staged image and sysupdate channel. + echo "S3_TARGET_STAGING=$S3_TARGET_STAGING" >> build.env + echo "IMAGE_URL=${S3_TARGET_STAGING#s3+}/$(basename upload-tree/*.iso)" >> build.env + echo "STAGING_CHANNEL_URL=${S3_TARGET_STAGING#s3+}/sysupdate/v2/" >> build.env +} + +publish() { + if [[ -z "$S3_TARGET_STAGING" ]]; then + echo "S3_TARGET_STAGING must be supplied by the imaging job dotenv artifact." >&2 + exit 1 + fi + + # Pull the staged build down from the bucket into ./publish-artifacts. + rm -rf publish-artifacts + mkdir -p publish-artifacts + go -C ./token-redeemer/ run . + go -C ./publisher/ build -o publisher . + ./publisher/publisher --remote "$S3_TARGET_STAGING" --output publish-artifacts --download + + # Publish to files.kde.org. + go -C ./upload-vacuum/ build -o upload-vacuum . + ( + cd publish-artifacts + ../upload-vacuum/upload-vacuum + + # shellcheck disable=SC2129 + sha256sum -- *.efi >> SHA256SUMS + sha256sum -- *.tar.zst >> SHA256SUMS + sha256sum -- *.erofs >> SHA256SUMS + # https://github.com/systemd/systemd/issues/38605 + sha256sum -- *-x86-64.caibx >> SHA256SUMS + + gpg --homedir="$GNUPGHOME" --output SHA256SUMS.gpg --detach-sign SHA256SUMS + + scp -i "$SSH_IDENTITY" ./*.iso ./*.torrent "$REMOTE_ROOT_PATH" + scp -i "$SSH_IDENTITY" ./*.efi ./*.tar.zst ./*.erofs ./*.caibx "$REMOTE_SYSUPDATE_PATH" + ) + + # Push the rootfs chunks into the S3 chunk store. + go install -v github.com/folbricht/desync/cmd/desync@f67d01e + export PATH="$HOME/go/bin:$PATH" + go -C ./token-redeemer/ run . + desync chop \ + --concurrency "$(nproc)" \ + --store "$S3_STORE" \ + publish-artifacts/*-x86-64.caibx \ + publish-artifacts/*-x86-64.erofs + + # Upload sums to files.kde.org only after the chunk store is ready. + scp -i "$SSH_IDENTITY" publish-artifacts/SHA256SUMS publish-artifacts/SHA256SUMS.gpg "$REMOTE_SYSUPDATE_PATH" + + # Merge the staged tree into the live S3 tree only after files.kde.org and the chunk store are ready. + go -C ./token-redeemer/ run . + ./publisher/publisher --remote "$S3_TARGET_STAGING" --promote + + # Regenerate, re-sign and re-upload SHA256SUMS. + go -C ./token-redeemer/ run . + go -C ./upload-vacuum-v3/ run . + + gpg --homedir="$GNUPGHOME" \ + --output "upload-tree/testing/sysupdate/v2/SHA256SUMS.gpg" \ + --detach-sign "upload-tree/testing/sysupdate/v2/SHA256SUMS" + go -C ./token-redeemer/ run . + go -C ./uploader/ run . --remote "$S3_TARGET" +} + +curl -s https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer | bash + +sudo chown -Rvf "$(id -u):$(id -g)" "$PWD/.secure_files" # Make sure we have access +chmod 600 "$SSH_IDENTITY" +gpg --verbose --no-options --homedir="$GNUPGHOME" --import "$PWD/.secure_files/gpg.private.key" + + +if [[ "${STAGE}" -eq 1 ]]; then + stage +fi + +if [[ "${PUBLISH}" -eq 1 ]]; then + publish +fi diff --git a/uploader/awsconfig.go b/uploader/awsconfig.go deleted file mode 100644 index 74a797b..0000000 --- a/uploader/awsconfig.go +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL -// SPDX-FileCopyrightText: 2025 Harald Sitter - -package main - -import ( - "errors" - "os" - "path/filepath" - - "gopkg.in/ini.v1" -) - -type AWSSection struct { - AccessKeyId string `ini:"aws_access_key_id"` - SecretKey string `ini:"aws_secret_access_key"` - SessionToken string `ini:"aws_session_token"` -} - -func readConfigAWS(section string) (AWSSection, error) { - awsSection := AWSSection{} - - awsConfigPath := filepath.Join(os.Getenv("HOME"), ".aws", "credentials") - cfg, err := ini.Load(awsConfigPath) - if err != nil { - return awsSection, errors.New("failed to load AWS credentials file: " + err.Error()) - } - - err = cfg.Section(section).MapTo(&awsSection) - if err != nil { - return awsSection, errors.New("failed to map AWS credentials section: " + err.Error()) - } - - return awsSection, nil -} diff --git a/uploader/go.mod b/uploader/go.mod index 48cea83..055035a 100644 --- a/uploader/go.mod +++ b/uploader/go.mod @@ -6,9 +6,12 @@ toolchain go1.24.4 require ( github.com/minio/minio-go/v7 v7.0.95 + invent.kde.org/kde-linux/kde-linux/minioauth v0.0.0 gopkg.in/ini.v1 v1.67.0 ) +replace invent.kde.org/kde-linux/kde-linux/minioauth => ../minioauth + require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/uploader/main.go b/uploader/main.go index d8d5451..efcd895 100644 --- a/uploader/main.go +++ b/uploader/main.go @@ -17,40 +17,18 @@ import ( "strings" "sync" + "invent.kde.org/kde-linux/kde-linux/minioauth" + "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" ) func connectToMinIO(endpoint string) *minio.Client { - awsSection, err := readConfigAWS("default") + minioClient, err := minioauth.Connect(endpoint) if err != nil { - log.Fatalln("Failed to read AWS config:", err) - } - accessKeyID := awsSection.AccessKeyId - if accessKeyID == "" { - log.Fatalln("AWS access key ID is empty") - } - secretAccessKey := awsSection.SecretKey - if secretAccessKey == "" { - log.Fatalln("AWS secret access key is empty") - } - sessionToken := awsSection.SessionToken - if secretAccessKey == "" { - log.Fatalln("AWS session token is empty") - } - useSSL := true - - // Initialize minio client object. - minioClient, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, sessionToken), - Secure: useSSL, - TrailingHeaders: true, - }) - if err != nil { - log.Fatalln("Failed to create MinIO client:", err) + log.Fatalln("Failed to connect to MinIO:", err) } - buckets, err := minioClient.ListBuckets(context.Background()) + buckets, err := minioauth.ListBuckets(minioClient) if err != nil { log.Fatalln("Failed to list buckets:", err) }