diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ceed57d..e45be6a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,19 +10,33 @@ include: stages: - validate - start + - test + - publish imaging: stage: start tags: - VM - amd64 + # IMAGING_MODE selects what we do with the built image: + # test - throwaway image, uploaded to ci-artifacts and given to OpenQA for testing. + # stage - signed and staged to storage for publishing, gated on OpenQA tests. rules: - if: $CI_MERGE_REQUEST_DRAFT == "true" when: manual + variables: + IMAGING_MODE: test - if: $CI_COMMIT_BRANCH =~ /^work\// when: manual - - if: $CI_COMMIT_REF_PROTECTED != 'true' || $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME || $CI_PROJECT_PATH != 'kde-linux/kde-linux' + variables: + IMAGING_MODE: test + - if: $CI_COMMIT_REF_PROTECTED == 'true' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME && $CI_PROJECT_PATH == 'kde-linux/kde-linux' when: always + variables: + IMAGING_MODE: stage + - when: always + variables: + IMAGING_MODE: test allow_failure: false image: storage.kde.org/vm-images/kde-linux-builder id_tokens: &tokens @@ -31,36 +45,67 @@ imaging: variables: &variables RUNNER_AFTER_SCRIPT_TIMEOUT: 1h script: - - sudo cp /etc/pacman.conf.nolinux /etc/pacman.conf # can be dropped once an image rebuild has happened after 2026-05-27 - - sudo ./build.sh --force --debug # bootstap.sh happens as part of CI image build + - sudo ./build.sh --force --debug + - if [ "$IMAGING_MODE" = stage ]; then ./upload.sh --stage; fi + - if [ "$IMAGING_MODE" = test ]; then ./upload-to-storage.sh; fi # Need this to have something to pass to OpenQA. after_script: + - if [ "$IMAGING_MODE" = stage ] && [ "$CI_JOB_STATUS" = success ]; then exit 0; fi + - if [ "$IMAGING_MODE" = test ]; then exit 0; fi - ./upload-to-storage.sh || true - artifacts: &artifacts + artifacts: expire_in: 4 weeks when: always paths: - "**/erofs.log" + reports: + dotenv: build.env -imaging+publish: - stage: start +# Run OpenQA testing on the built image. +trigger-openqa: + stage: test + rules: + - if: $CI_PROJECT_PATH != 'kde-linux/kde-linux' + when: never + - if: $CI_MERGE_REQUEST_DRAFT == "true" + when: manual + - if: $CI_COMMIT_BRANCH =~ /^work\// + when: manual + - when: on_success + needs: + - job: imaging + artifacts: true + trigger: + project: kde-linux/os-autoinst-distri-kdelinux + branch: $OPENQA_BRANCH + strategy: depend + variables: + IMAGE_URL: $IMAGE_URL + STAGING_CHANNEL_URL: $STAGING_CHANNEL_URL + SYSUPDATE_PUBKEY_B64: $SYSUPDATE_PUBKEY_B64 + OPENQA_BRANCH: master + +# Publish the staged image in storage. +publish: + stage: publish tags: - VM - amd64 rules: - # NOTE: these are different from the regular imaging job! - if: $CI_COMMIT_REF_PROTECTED == 'true' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME && $CI_PROJECT_PATH == 'kde-linux/kde-linux' - when: always + when: on_success + needs: + - job: imaging + artifacts: true + - job: trigger-openqa + artifacts: false allow_failure: false image: storage.kde.org/vm-images/kde-linux-builder id_tokens: *tokens variables: *variables script: - - sudo cp /etc/pacman.conf.nolinux /etc/pacman.conf # can be dropped once an image rebuild has happened after 2026-05-27 - - sudo pacman --sync --refresh --noconfirm curl which git - - curl -s https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer | bash - - sudo ./build.sh --force --debug # bootstap.sh happens as part of CI image build - - ./upload.sh - after_script: - # Upload broken images to Harald for inspection - - "[ $CI_JOB_STATUS == 'success' ] || ./upload-to-storage.sh || true" - artifacts: *artifacts + - ./upload.sh --publish + artifacts: + expire_in: 4 weeks + when: always + paths: + - "**/erofs.log" diff --git a/basic-test-efi-addon.sh b/basic-test-efi-addon.sh deleted file mode 100755 index 4064907..0000000 --- a/basic-test-efi-addon.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL -# SPDX-FileCopyrightText: 2025 Harald Sitter -# SPDX-FileCopyrightText: 2026 Hadi Chokr - -set -eux - -# ADDON_DIR is set by basic-test.py to the .extra.d directory inside the mounted ESP -if [ -z "$ADDON_DIR" ]; then - echo "ERROR: ADDON_DIR environment variable not set" - exit 1 -fi - -# Create the addon UKI (systemd-stub addon) that appends the test cmdline -ukify build \ - --cmdline "kde-linux.basic-test=1 kde-linux.basic-test-callback=http://10.0.2.2:${PORT}/good" \ - --output "$ADDON_DIR/basic-test.addon.efi" diff --git a/basic-test.py b/basic-test.py deleted file mode 100755 index 254bbb5..0000000 --- a/basic-test.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL -# SPDX-FileCopyrightText: 2025 Harald Sitter -# SPDX-FileCopyrightText: 2026 Hadi Chokr - -import atexit -import http.server -import sys -import subprocess -import os -import time -import tempfile -from pathlib import Path - -class Handler(http.server.BaseHTTPRequestHandler): - def do_GET(self): - if self.path == '/good': - sys.exit(0) - if self.path == '/bad': - print("==Received /bad callback==") - content_len = int(self.headers.get('Content-Length')) - body = self.rfile.read(content_len) - print(body.decode('utf-8')) - sys.exit(1) - self.send_response(200) - self.end_headers() - -server = http.server.HTTPServer(server_address=('', 0), RequestHandlerClass=Handler) -print("serving at port", server.server_port) - -img = sys.argv[1] -if not img: - print("No image specified") - sys.exit(1) - -efi_base = sys.argv[2] -if not efi_base: - print("No EFI base image specified") - sys.exit(1) - -# Always test as ISO (a valid .iso9660 is also a valid .raw) -test_img = img.replace('.raw', '.test.iso').replace('.iso', '.test.iso') -subprocess.check_call(['cp', '--reflink=auto', img, test_img]) - -# Inject the EFI addon into the ESP partition of the test ISO -script_dir = os.path.dirname(os.path.realpath(__file__)) -addon_src = f'{script_dir}/basic-test-efi-addon.sh' - -with tempfile.TemporaryDirectory() as mnt: - # Find the ESP partition offset and size using sfdisk - sfdisk = subprocess.check_output(['sfdisk', '--json', test_img]).decode() - import json - parts = json.loads(sfdisk)['partitiontable']['partitions'] - esp = next(p for p in parts if p.get('type', '') == 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B') - sector_size = json.loads(sfdisk)['partitiontable']['sectorsize'] - offset = esp['start'] * sector_size - size = esp['size'] * sector_size - - subprocess.check_call([ - 'mount', '-o', f'loop,offset={offset},sizelimit={size}', - test_img, mnt - ]) - try: - efi_extra_dir = f'{mnt}/EFI/Linux/{efi_base}.extra.d' - os.makedirs(efi_extra_dir, exist_ok=True) - subprocess.check_call([ - 'bash', addon_src - ], env={ - 'PORT': str(server.server_port), - 'UKI': efi_base, - 'ADDON_DIR': efi_extra_dir, - }) - finally: - subprocess.check_call(['umount', mnt]) - -qemu_cmd = [ - "qemu-system-x86_64", - "-cdrom", test_img, - "-m", "4G", - "-enable-kvm", - "-cpu", "host", - "-bios", "/usr/share/OVMF/x64/OVMF.4m.fd", -] - -# I ought to point out that this leaks the process in case of failure. It will however get reaped by the docker container shutdown. -qemu = subprocess.Popen(qemu_cmd) -atexit.register(lambda: (qemu.kill())) - -def on_timeout(): - print("\n\n\n== Test timed out ==") - print("Download the image for inspection from http://images.kde-linux.haraldsitter.eu/") - print("(location may have changed to something like https://qoomon.github.io/aws-s3-bucket-browser/index.html?bucket=https://storage.kde.org/ci-artifacts/#).") - print("Once downloaded run:\n") - print(f"./basic-test.py {img} {efi_base}\n\n\n") - - qemu.kill() - sys.exit(2) - -server.timeout = 5 * 60 # 5 minutes -server.handle_timeout = on_timeout -while True: # kinda garbage but there seems to be no nice (non-private) poll-or-timeout api - server.handle_request() - time.sleep(8) -qemu.kill() -sys.exit(1) # if we get here we timed out = fail diff --git a/build.sh b/build.sh index 224c6a3..e7865ec 100755 --- a/build.sh +++ b/build.sh @@ -50,7 +50,7 @@ ROOTFS_CAIBX=${OUTPUT}_root-x86-64.caibx ROOTFS_EROFS=${OUTPUT}_root-x86-64.erofs # Output erofs image path ISO="${OUTPUT}.iso" # both a valid GPT disk image and a bootable ISO -EFI_BASE=kde-linux_${VERSION} # Base name of the UKI in the image's ESP (exported so it can be used in basic-test-efi-addon.sh) +EFI_BASE=kde-linux_${VERSION} # Base name of the UKI in the image's ESP EFI=${EFI_BASE}+3.efi # Name of primary UKI in the image's ESP (with tries counter for installed system) LIVE_EFI=${EFI_BASE}.efi # Name of live UKI in the ESP (no tries counter — ESP is read-only on ISO) @@ -204,10 +204,6 @@ systemd-repart \ --el-torito-publisher="KDE" \ "$ISO" -# Test the ISO (which is also a valid GPT image so no need to test as .raw separately) -./basic-test.py "$ISO" "$LIVE_EFI" || exit 1 -rm ./mkosi.output/*.test.iso - # Incase the owner is root chown -R user:user mkosi.output diff --git a/minioauth/go.mod b/minioauth/go.mod new file mode 100644 index 0000000..49a5020 --- /dev/null +++ b/minioauth/go.mod @@ -0,0 +1,31 @@ +module invent.kde.org/kde-linux/kde-linux/minioauth + +go 1.23.0 + +toolchain go1.24.4 + +require ( + github.com/minio/minio-go/v7 v7.0.95 + gopkg.in/ini.v1 v1.67.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/minio/crc64nvme v1.0.2 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.3.0 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.26.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/minioauth/go.mod.license b/minioauth/go.mod.license new file mode 100644 index 0000000..65051d9 --- /dev/null +++ b/minioauth/go.mod.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: CC0-1.0 +SPDX-FileCopyrightText: None diff --git a/minioauth/go.sum b/minioauth/go.sum new file mode 100644 index 0000000..577a665 --- /dev/null +++ b/minioauth/go.sum @@ -0,0 +1,44 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= +github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/minioauth/go.sum.license b/minioauth/go.sum.license new file mode 100644 index 0000000..65051d9 --- /dev/null +++ b/minioauth/go.sum.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: CC0-1.0 +SPDX-FileCopyrightText: None diff --git a/minioauth/minioauth.go b/minioauth/minioauth.go new file mode 100644 index 0000000..3df9643 --- /dev/null +++ b/minioauth/minioauth.go @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL +// SPDX-FileCopyrightText: 2025 Harald Sitter +// SPDX-FileCopyrightText: 2026 Thomas Duckworth + +package minioauth + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "gopkg.in/ini.v1" +) + +type AWSSection struct { + AccessKeyID string `ini:"aws_access_key_id"` + SecretKey string `ini:"aws_secret_access_key"` + SessionToken string `ini:"aws_session_token"` +} + +func readConfigAWS(section string) (AWSSection, error) { + awsSection := AWSSection{} + + awsConfigPath := filepath.Join(os.Getenv("HOME"), ".aws", "credentials") + cfg, err := ini.Load(awsConfigPath) + if err != nil { + return awsSection, fmt.Errorf("failed to load AWS credentials file: %w", err) + } + + err = cfg.Section(section).MapTo(&awsSection) + if err != nil { + return awsSection, fmt.Errorf("failed to map AWS credentials section: %w", err) + } + + return awsSection, nil +} + +func Connect(endpoint string) (*minio.Client, error) { + awsSection, err := readConfigAWS("default") + if err != nil { + return nil, err + } + if awsSection.AccessKeyID == "" { + return nil, errors.New("AWS access key ID is empty") + } + if awsSection.SecretKey == "" { + return nil, errors.New("AWS secret access key is empty") + } + if awsSection.SessionToken == "" { + return nil, errors.New("AWS session token is empty") + } + + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(awsSection.AccessKeyID, awsSection.SecretKey, awsSection.SessionToken), + Secure: true, + TrailingHeaders: true, + }) + if err != nil { + return nil, fmt.Errorf("failed to create MinIO client: %w", err) + } + + return minioClient, nil +} + +func ListBuckets(client *minio.Client) ([]minio.BucketInfo, error) { + return client.ListBuckets(context.Background()) +} diff --git a/mkosi.extra/live/usr/lib/basic-test b/mkosi.extra/live/usr/lib/basic-test deleted file mode 100755 index aa52c30..0000000 --- a/mkosi.extra/live/usr/lib/basic-test +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL -# SPDX-FileCopyrightText: 2025 Harald Sitter -# SPDX-FileCopyrightText: 2026 Hadi Chokr - -import subprocess -import time -from pathlib import Path -import json - -# Units that are known to fail on a read-only ISO (e.g., /boot is not writable) -IGNORED_FAILED_UNITS = { - "systemd-boot-update.service", - "systemd-boot-random-seed.service", -} - -callback = None -with open('/proc/cmdline') as cmdline: - data = cmdline.read() - entries = data.split(' ') - for entry in entries: - if entry.startswith('kde-linux.basic-test-callback='): - # We don't support quoting of any kind. - callback = entry.split('=', 2)[1] - callback = callback.strip() - break - -if callback is None: - print('No callback specified. set kde-linux.basic-test-callback=') - exit(1) - -bad_callback = callback.replace('/good', '/bad') - -while True: - # 1000 is the uid of the live user. always. - if Path('/run/user/1000/kde-linux-bless-session').is_file(): - failed = json.loads(subprocess.check_output(["systemctl", "--failed", "--output=json"])) - # Filter out ignored units - relevant_failed = [unit for unit in failed if unit['unit'] not in IGNORED_FAILED_UNITS] - if len(relevant_failed) > 0: - with open('data.file', 'w') as f: - for unit in relevant_failed: - f.write("\n") - f.write(json.dumps(unit)) - f.write("\n") - try: - f.write(subprocess.check_output(['journalctl', '--no-pager', f'_SYSTEMD_UNIT={unit["unit"]}']).decode('utf-8')) - except Exception as e: - f.write(f"Failed to get journal for {unit['unit']}: {e}\n") - f.write("\n") # make sure we have a final newline - subprocess.check_call(['curl', - '--request', 'GET', - '--data-binary', '@data.file', # use binary as otherwise curl will eat the newlines - bad_callback]) - else: - subprocess.check_call(['curl', callback]) - time.sleep(30) diff --git a/mkosi.extra/live/usr/lib/live-setup b/mkosi.extra/live/usr/lib/live-setup index dea1f09..75a05d8 100755 --- a/mkosi.extra/live/usr/lib/live-setup +++ b/mkosi.extra/live/usr/lib/live-setup @@ -90,7 +90,3 @@ cp /usr/share/applications/calamares.desktop /home/live/Desktop/ chmod +x /home/live/Desktop/calamares.desktop # without this there'd be a dialog asking if it really should execute chown -R live:live /home/live/ - -# Start the basic-test (if it wants to be started) -systemctl daemon-reload -systemctl start --no-block kde-linux-basic-test.service diff --git a/mkosi.extra/live/usr/lib/systemd/system/kde-linux-basic-test.service b/mkosi.extra/live/usr/lib/systemd/system/kde-linux-basic-test.service deleted file mode 100644 index 1e448aa..0000000 --- a/mkosi.extra/live/usr/lib/systemd/system/kde-linux-basic-test.service +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL -# SPDX-FileCopyrightText: 2025 Harald Sitter - -[Unit] -Description=Test of KDE Linux disk image -ConditionKernelCommandLine=kde-linux.basic-test=1 - -[Service] -Type=simple -ExecStart=/usr/lib/basic-test diff --git a/publisher/go.mod b/publisher/go.mod new file mode 100644 index 0000000..2002b77 --- /dev/null +++ b/publisher/go.mod @@ -0,0 +1,34 @@ +module invent.kde.org/kde-linux/kde-linux/publisher + +go 1.23.0 + +toolchain go1.24.4 + +require ( + github.com/minio/minio-go/v7 v7.0.95 + invent.kde.org/kde-linux/kde-linux/minioauth v0.0.0 + gopkg.in/ini.v1 v1.67.0 +) + +replace invent.kde.org/kde-linux/kde-linux/minioauth => ../minioauth + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/minio/crc64nvme v1.0.2 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.3.0 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.26.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/publisher/go.mod.license b/publisher/go.mod.license new file mode 100644 index 0000000..23c1849 --- /dev/null +++ b/publisher/go.mod.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: CC0-1.0 +SPDX-FileCopyrightText: none diff --git a/publisher/go.sum b/publisher/go.sum new file mode 100644 index 0000000..577a665 --- /dev/null +++ b/publisher/go.sum @@ -0,0 +1,44 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= +github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/publisher/go.sum.license b/publisher/go.sum.license new file mode 100644 index 0000000..23c1849 --- /dev/null +++ b/publisher/go.sum.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: CC0-1.0 +SPDX-FileCopyrightText: none diff --git a/publisher/main.go b/publisher/main.go new file mode 100644 index 0000000..32a39f7 --- /dev/null +++ b/publisher/main.go @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL +// SPDX-FileCopyrightText: 2024-2025 Harald Sitter +// SPDX-FileCopyrightText: 2026 Thomas Duckworth + +package main + +import ( + "context" + "flag" + "log" + "net/url" + "path/filepath" + "sort" + "strings" + + "invent.kde.org/kde-linux/kde-linux/minioauth" + + "github.com/minio/minio-go/v7" +) + +func connectToMinIO(endpoint string) *minio.Client { + minioClient, err := minioauth.Connect(endpoint) + if err != nil { + log.Fatalln("Failed to connect to MinIO:", err) + } + + buckets, err := minioauth.ListBuckets(minioClient) + if err != nil { + log.Fatalln("Failed to list buckets:", err) + } + for _, bucket := range buckets { + log.Println(bucket) + } + + return minioClient +} + +func prefix(path string) string { + trimmed := strings.Trim(path, "/") + if trimmed == "" { + return "" + } + return trimmed + "/" +} + +func destinationPrefix(path string) string { + path = strings.Trim(path, "/") + if path == "" { + log.Fatalln("Remote path must be a non-empty staging path") + } + + parts := strings.Split(path, "/") + for i, part := range parts { + if part != "staging" { + continue + } + if i == len(parts)-1 { + log.Fatalln("Remote path must include a staging directory name") + } + if i == 0 { + return "" + } + return strings.Join(parts[:i], "/") + "/" + } + + log.Fatalln("Remote path must contain a staging directory") + return "" +} + +func isChannelCommitMarker(key string) bool { + base := filepath.Base(key) + return base == "SHA256SUMS" || base == "SHA256SUMS.gpg" +} + +func removeObject(ctx context.Context, client *minio.Client, bucket string, objectKey string) { + log.Printf("Removing staged object %s", objectKey) + if err := client.RemoveObject(ctx, bucket, objectKey, minio.RemoveObjectOptions{}); err != nil { + log.Fatalf("Failed to remove %s: %v", objectKey, err) + } +} + +// publish merges a staging tree into its channel root. +func publish(client *minio.Client, bucket string, path string) { + ctx := context.Background() + srcPrefix := prefix(path) + + destPrefix := destinationPrefix(path) + + objects := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{ + Prefix: srcPrefix, + Recursive: true, + }) + + var objectKeys []string + for object := range objects { + if object.Err != nil { + log.Fatalln("Failed to list objects:", object.Err) + } + objectKeys = append(objectKeys, object.Key) + } + sort.Strings(objectKeys) + + for _, objectKey := range objectKeys { + rel := strings.TrimPrefix(objectKey, srcPrefix) + if rel == "" { + continue + } + if isChannelCommitMarker(rel) { + log.Printf("Skipping staged commit marker %s", objectKey) + removeObject(ctx, client, bucket, objectKey) + continue + } + dstKey := destPrefix + rel + if dstKey == objectKey { + continue + } + + log.Printf("Publishing %s -> %s", objectKey, dstKey) + + _, err := client.ComposeObject(ctx, + minio.CopyDestOptions{Bucket: bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: bucket, Object: objectKey}, + ) + if err != nil { + log.Fatalf("Failed to copy %s to %s: %v", objectKey, dstKey, err) + } + + removeObject(ctx, client, bucket, objectKey) + } +} + +// artifactSuffixes are the build artifacts we download so they can be pushed to +// files.kde.org and the chunk store. sha256sums are regenerated at publish time, so they're excluded here +var artifactSuffixes = []string{".iso", ".torrent", ".efi", ".tar.zst", ".erofs", ".caibx"} + +func isArtifact(key string) bool { + for _, suffix := range artifactSuffixes { + if strings.HasSuffix(key, suffix) { + return true + } + } + return false +} + +// download gets every build artifact under path to publish to files.kde.org and the chunk store +func download(client *minio.Client, bucket string, path string, output string) { + ctx := context.Background() + + objects := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{ + Prefix: prefix(path), + Recursive: true, + }) + + for object := range objects { + if object.Err != nil { + log.Fatalln("Failed to list objects:", object.Err) + } + + if !isArtifact(object.Key) { + continue + } + + dst := filepath.Join(output, filepath.Base(object.Key)) + log.Printf("Downloading %s -> %s", object.Key, dst) + if err := client.FGetObject(ctx, bucket, object.Key, dst, minio.GetObjectOptions{}); err != nil { + log.Fatalf("Failed to download %s: %v", object.Key, err) + } + } +} + +// Either downloads every build artifact in the staging directory into --output, +// or promotes the staging tree into the channel root. +func main() { + remote := flag.String("remote", "", "remote url to publish from, e.g. s3+https://storage.kde.org/kde-linux/staging/1") + output := flag.String("output", ".", "directory to download artifacts into") + downloadMode := flag.Bool("download", false, "download artifacts from the staging tree") + promoteMode := flag.Bool("promote", false, "promote the staging tree into the channel root") + flag.Parse() + + if *downloadMode == *promoteMode { + log.Fatalln("Must choose exactly one of --download or --promote") + } + + remoteURI, err := url.Parse(*remote) + if err != nil { + log.Fatalln("Failed to parse remote URL:", err) + } + if remoteURI.Scheme != "s3+https" { + log.Fatalln("Unsupported remote scheme:", remoteURI.Scheme) + } + if remoteURI.Host != "storage.kde.org" { + log.Fatalln("Unsupported remote host:", remoteURI.Host) + } + parts := strings.SplitN(remoteURI.Path[1:], "/", 2) + if len(parts) != 2 { + log.Fatalln("Invalid remote path, expected format: /bucket/path") + } + bucket := parts[0] + path := parts[1] + if bucket == "" { + log.Fatalln("Invalid remote path, expected format: /bucket/path") + } + if path == "" { + log.Println("Warning: path is empty, uploading to bucket root") + path = "/" + } + + log.Println("Connecting to MinIO at", remoteURI.Host) + minioClient := connectToMinIO(remoteURI.Host) + + if *downloadMode { + log.Println("Downloading artifacts from bucket", bucket, "with path prefix", path, "into", *output) + download(minioClient, bucket, path, *output) + } + + if *promoteMode { + log.Println("Publishing to bucket", bucket, "from source path", path) + publish(minioClient, bucket, path) + } +} diff --git a/upload-to-storage.sh b/upload-to-storage.sh index b8cb2f6..985272a 100755 --- a/upload-to-storage.sh +++ b/upload-to-storage.sh @@ -8,23 +8,46 @@ set -eu # Output Directory OUTDIR="${OUTDIR:-mkosi.output}" -# Do not blow the lid off the storage for now. Reset the tree and only publish a select few files +# We need to wire up an ephemeral image-signing key pair for OpenQA. We can't use the production key, but we still need +# to test if the image can be upgraded to. Hence, create a key pair and pass it into OpenQA, where it will be injected +# into the system. +GNUPGHOME="$PWD/.openqa-gpg" +rm -rf "$GNUPGHOME" +mkdir -p "$GNUPGHOME" +chmod 700 "$GNUPGHOME" +gpg --batch --pinentry-mode loopback --homedir="$GNUPGHOME" --passphrase "" --quick-generate-key "KDE Linux openQA " ed25519 sign 3d +SYSUPDATE_PUBKEY_B64=$(gpg --homedir="$GNUPGHOME" --export --armor "KDE Linux openQA " | base64 -w0) + mv upload-tree upload-tree-old || true if [ ! -d upload-tree ]; then - mkdir upload-tree - for f in "$OUTDIR"/*.iso "$OUTDIR"/*.erofs "$OUTDIR"/*.efi; do - if [[ $f == *.test.iso ]]; then - # Skip test images - continue - fi - mv "$f" upload-tree/ - done + mkdir -p upload-tree/sysupdate/v2 + mv "$OUTDIR"/*.iso upload-tree/ + mv "$OUTDIR"/*.efi "$OUTDIR"/*.tar.zst "$OUTDIR"/*.erofs "$OUTDIR"/*.caibx upload-tree/sysupdate/v2/ + ( + cd upload-tree/sysupdate/v2 + # shellcheck disable=SC2129 + sha256sum -- *.efi >> SHA256SUMS + sha256sum -- *.tar.zst >> SHA256SUMS + sha256sum -- *.erofs >> SHA256SUMS + # Don't put .erofs.caibx into SHA256SUMS, it will break file matching. + # https://github.com/systemd/systemd/issues/38605 + sha256sum -- *-x86-64.caibx >> SHA256SUMS + + # Sign, so test images can actually verify and upgrade from artifacts. + gpg --homedir="$GNUPGHOME" --output SHA256SUMS.gpg --detach-sign SHA256SUMS + ) fi go -C ./token-redeemer/ run . go -C ./uploader/ run . --remote "s3+https://storage.kde.org/ci-artifacts/$CI_PROJECT_PATH/j/$CI_JOB_ID" +# Point OpenQA at the image we just uploaded. +ISO_FILE=$(find upload-tree -maxdepth 1 -name '*.iso' | head -1 | xargs -r basename) +echo "IMAGE_URL=https://storage.kde.org/ci-artifacts/$CI_PROJECT_PATH/j/$CI_JOB_ID/$ISO_FILE" >> build.env +echo "STAGING_CHANNEL_URL=https://storage.kde.org/ci-artifacts/$CI_PROJECT_PATH/j/$CI_JOB_ID/sysupdate/v2/" >> build.env +echo "SYSUPDATE_PUBKEY_B64=$SYSUPDATE_PUBKEY_B64" >> build.env + echo "𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏" -echo "You can find the raw disk images at:" +echo "You can find the raw disk images and sysupdate tree at:" echo "https://qoomon.github.io/aws-s3-bucket-browser/index.html?bucket=https://storage.kde.org/ci-artifacts/#$CI_PROJECT_PATH/j/$CI_JOB_ID/" echo "𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏𓃀𓂝𓏏" diff --git a/upload-vacuum-v3/awsconfig.go b/upload-vacuum-v3/awsconfig.go deleted file mode 100644 index 74a797b..0000000 --- a/upload-vacuum-v3/awsconfig.go +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL -// SPDX-FileCopyrightText: 2025 Harald Sitter - -package main - -import ( - "errors" - "os" - "path/filepath" - - "gopkg.in/ini.v1" -) - -type AWSSection struct { - AccessKeyId string `ini:"aws_access_key_id"` - SecretKey string `ini:"aws_secret_access_key"` - SessionToken string `ini:"aws_session_token"` -} - -func readConfigAWS(section string) (AWSSection, error) { - awsSection := AWSSection{} - - awsConfigPath := filepath.Join(os.Getenv("HOME"), ".aws", "credentials") - cfg, err := ini.Load(awsConfigPath) - if err != nil { - return awsSection, errors.New("failed to load AWS credentials file: " + err.Error()) - } - - err = cfg.Section(section).MapTo(&awsSection) - if err != nil { - return awsSection, errors.New("failed to map AWS credentials section: " + err.Error()) - } - - return awsSection, nil -} diff --git a/upload-vacuum-v3/go.mod b/upload-vacuum-v3/go.mod index 1f6bc1c..f4c3ad4 100644 --- a/upload-vacuum-v3/go.mod +++ b/upload-vacuum-v3/go.mod @@ -6,10 +6,13 @@ require ( github.com/minio/minio-go/v7 v7.0.100 github.com/pkg/sftp v1.13.6 golang.org/x/crypto v0.50.0 + invent.kde.org/kde-linux/kde-linux/minioauth v0.0.0 gopkg.in/ini.v1 v1.67.1 gopkg.in/yaml.v2 v2.4.0 ) +replace invent.kde.org/kde-linux/kde-linux/minioauth => ../minioauth + require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-ini/ini v1.67.0 // indirect diff --git a/upload-vacuum-v3/main.go b/upload-vacuum-v3/main.go index a4c5676..f9af9ad 100644 --- a/upload-vacuum-v3/main.go +++ b/upload-vacuum-v3/main.go @@ -17,9 +17,9 @@ import ( "strings" "gopkg.in/yaml.v2" + "invent.kde.org/kde-linux/kde-linux/minioauth" "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" ) type release struct { @@ -29,36 +29,12 @@ type release struct { func connectToMinIO() *minio.Client { endpoint := "storage.kde.org" - // Below is a huge code dupe from uploader/main.go, should be shared somehow - awsSection, err := readConfigAWS("default") + minioClient, err := minioauth.Connect(endpoint) if err != nil { - log.Fatalln("Failed to read AWS config:", err) - } - accessKeyID := awsSection.AccessKeyId - if accessKeyID == "" { - log.Fatalln("AWS access key ID is empty") - } - secretAccessKey := awsSection.SecretKey - if secretAccessKey == "" { - log.Fatalln("AWS secret access key is empty") - } - sessionToken := awsSection.SessionToken - if secretAccessKey == "" { - log.Fatalln("AWS session token is empty") - } - useSSL := true - - // Initialize minio client object. - minioClient, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, sessionToken), - Secure: useSSL, - TrailingHeaders: true, - }) - if err != nil { - log.Fatalln("Failed to create MinIO client:", err) + log.Fatalln("Failed to connect to MinIO:", err) } - buckets, err := minioClient.ListBuckets(context.Background()) + buckets, err := minioauth.ListBuckets(minioClient) if err != nil { log.Fatalln("Failed to list buckets:", err) } diff --git a/upload-vacuum/main.go b/upload-vacuum/main.go index c64cacb..45fbea8 100644 --- a/upload-vacuum/main.go +++ b/upload-vacuum/main.go @@ -33,8 +33,8 @@ func connectToHost(user, host, identity string) (*ssh.Client, *ssh.Session, erro log.Fatalf("unable to parse private key: %v", err) } - // You can use `ssh-keyscan origin.files.kde.org` to get the host key - _, _, hostKey, _, _, err := ssh.ParseKnownHosts([]byte("origin.files.kde.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILUjdH4S7otYIdLUkOZK+owIiByjNQPzGi7GQ5HOWjO6")) + // You can use `ssh-keyscan tinami.kde.org` to get the host key + _, _, hostKey, _, _, err := ssh.ParseKnownHosts([]byte("tinami.kde.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILUjdH4S7otYIdLUkOZK+owIiByjNQPzGi7GQ5HOWjO6")) if err != nil { log.Fatalf("unable to parse host public key: %v", err) } @@ -120,6 +120,7 @@ func readSHA256SUMS(client *sftp.Client, path string) map[string]string { func readSHA256s(toKeep []string, releases map[string]release, existingSums map[string]string) []string { sha256s := []string{} + rootPath := strings.TrimRight(os.Getenv("SSH_ROOT_PATH"), "/") for _, key := range toKeep { artifacts := releases[key].artifacts sort.Strings(artifacts) // Sort artifacts to ensure consistent order @@ -129,7 +130,7 @@ func readSHA256s(toKeep []string, releases map[string]release, existingSums map[ // https://github.com/systemd/systemd/issues/38605 continue } - if strings.HasPrefix(artifact, "/home/kdeos/kde-linux/kdeos_") { + if rootPath != "" && strings.HasPrefix(artifact, rootPath+"/kdeos_") { // HACK 2025-08-20 sha256s of the files are broken, only drop this if when they are fixed (possibly just a matter of time) continue } @@ -235,7 +236,10 @@ func main() { identity := os.Getenv("SSH_IDENTITY") host := os.Getenv("SSH_HOST") user := os.Getenv("SSH_USER") - path := os.Getenv("SSH_PATH") + path := os.Getenv("SSH_SYSUPDATE_PATH") + if path == "" { + path = os.Getenv("SSH_PATH") + } root_path := os.Getenv("SSH_ROOT_PATH") var errs []error @@ -249,7 +253,7 @@ func main() { errs = append(errs, errors.New("SSH_USER not set")) } if path == "" { - errs = append(errs, errors.New("SSH_PATH not set")) + errs = append(errs, errors.New("SSH_SYSUPDATE_PATH or SSH_PATH not set")) } if root_path == "" { errs = append(errs, errors.New("SSH_ROOT_PATH not set")) diff --git a/upload.sh b/upload.sh index 4efe9c1..8b145e9 100755 --- a/upload.sh +++ b/upload.sh @@ -2,89 +2,174 @@ # SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL # SPDX-FileCopyrightText: 2024 Harald Sitter # SPDX-FileCopyrightText: 2026 Hadi Chokr +# SPDX-FileCopyrightText: 2026 Thomas Duckworth set -eux +if [[ -z "${1:-}" ]]; then + echo "Choice between --stage and --publish must be provided as argument." >&2 + exit 1 +fi + +STAGE= +PUBLISH= +while [[ $# -gt 0 ]]; do + case "$1" in + --stage) + STAGE=1 + shift + ;; + --publish) + PUBLISH=1 + shift + ;; + *) + echo "Unknown option $1." + exit 1 + ;; + esac +done + +if [[ "$STAGE" -eq 1 && "$PUBLISH" -eq 1 ]] || [[ "$STAGE" -ne 1 && "$PUBLISH" -ne 1 ]]; then + echo "Must choose exactly one of --stage or --publish." + exit 1 +fi + OUTDIR=mkosi.output # For the vacuum helper and this script export SSH_IDENTITY="$PWD/.secure_files/ssh.key" export SSH_USER=kdeos -export SSH_HOST=origin.files.kde.org -export SSH_ROOT_PATH=/home/kdeos/kde-linux/ -export SSH_PATH=$SSH_ROOT_PATH/sysupdate/v2/ +export SSH_HOST=tinami.kde.org +export SSH_ROOT_PATH=/srv/archives/files/kde-linux +export SSH_SYSUPDATE_PATH=$SSH_ROOT_PATH/sysupdate/v2/ export SSH_REALLY_DELETE=1 - -chmod 600 "$SSH_IDENTITY" - -go -C ./upload-vacuum/ build -o upload-vacuum . -./upload-vacuum/upload-vacuum - -# The following variables are for this script only. Not shared with the vacuum helper. -sudo chown -Rvf "$(id -u):$(id -g)" "$PWD/.secure_files" # Make sure we have access -export GNUPGHOME="$PWD/.secure_files/gpg" -gpg --verbose --no-options --homedir="$GNUPGHOME" --import "$PWD/.secure_files/gpg.private.key" -REMOTE_ROOT=$SSH_USER@$SSH_HOST:$SSH_ROOT_PATH -REMOTE_PATH=$SSH_USER@$SSH_HOST:$SSH_PATH -# You can use `ssh-keyscan origin.files.kde.org` to get the host key -echo "origin.files.kde.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILUjdH4S7otYIdLUkOZK+owIiByjNQPzGi7GQ5HOWjO6" >> ~/.ssh/known_hosts - -# The initial SHA256SUMS file is created by the vacuum script based on what is left on the server. We append to it. - -sudo chown -R "$USER":"$USER" "$OUTDIR" -cd "$OUTDIR" - -# We need shell globs here! More readable this way. Ignore shellcheck. -# shellcheck disable=SC2129 -sha256sum -- *.efi >> SHA256SUMS -sha256sum -- *.tar.zst >> SHA256SUMS -sha256sum -- *.erofs >> SHA256SUMS -# Don't put .erofs.caibx into the SHA256SUMS, it will break file matching. -# https://github.com/systemd/systemd/issues/38605 -sha256sum -- *-x86-64.caibx >> SHA256SUMS - -gpg --homedir="$GNUPGHOME" --output SHA256SUMS.gpg --detach-sign SHA256SUMS - -scp -i "$SSH_IDENTITY" ./*.iso ./*.torrent "$REMOTE_ROOT" -scp -i "$SSH_IDENTITY" ./*.efi ./*.tar.zst ./*.erofs ./*.caibx "$REMOTE_PATH" -scp -i "$SSH_IDENTITY" SHA256SUMS SHA256SUMS.gpg "$REMOTE_PATH" # upload as last artifact to finalize the upload - -# The new s3 based upload system - export VACUUM_REALLY_DELETE=1 -S3_STORE="s3+https://storage.kde.org/kde-linux/sysupdate/store/" +export GNUPGHOME="$PWD/.secure_files/gpg" + +# upload tree built during staging +V2_TREE="upload-tree/sysupdate/v2" + S3_TARGET="s3+https://storage.kde.org/kde-linux/" +S3_STORE="${S3_TARGET}sysupdate/store/" +S3_TARGET_STAGING="${S3_TARGET_STAGING:-}" -## Upload to the chunk store directly -go install -v github.com/folbricht/desync/cmd/desync@latest -export PATH="$HOME/go/bin:$PATH" -go -C ../token-redeemer/ run . -desync chop \ - --concurrency "$(nproc)" \ - --store "$S3_STORE" \ - ./*-x86-64.caibx \ - ./*-x86-64.erofs +# files.kde.org scp targets. We don't stage on files.kde.org, only on the storage.kde.org bucket. +# We download from the bucket then upload directly to files.kde.org to publish there. +REMOTE_ROOT_PATH=$SSH_USER@$SSH_HOST:$SSH_ROOT_PATH +REMOTE_SYSUPDATE_PATH=$SSH_USER@$SSH_HOST:$SSH_SYSUPDATE_PATH +# You can use `ssh-keyscan tinami.kde.org` to get the host key +echo "tinami.kde.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILUjdH4S7otYIdLUkOZK+owIiByjNQPzGi7GQ5HOWjO6" >> ~/.ssh/known_hosts -## Prepare the image upload tree -cd .. -rm -rf upload-tree -V2_TREE="upload-tree/testing/sysupdate/v2" -mkdir -p "$V2_TREE" +stage() { + S3_TARGET_STAGING="${S3_TARGET_STAGING:-${S3_TARGET}testing/staging/${CI_PIPELINE_ID}/${CI_JOB_ID}}" -mv "$OUTDIR"/*.iso "$OUTDIR"/*.torrent upload-tree/testing/ -mv "$OUTDIR"/*.efi "$OUTDIR"/*.tar.zst "$OUTDIR"/*.erofs "$OUTDIR"/*.caibx "$V2_TREE/" + # Stage the freshly built image into the bucket. + sudo chown -R "$USER":"$USER" "$OUTDIR" -### Upload -go -C ./token-redeemer/ run . -go -C ./uploader/ run . --remote "$S3_TARGET" + ( + cd "$OUTDIR" + # We need shell globs here! More readable this way. Ignore shellcheck. + # shellcheck disable=SC2129 + sha256sum -- *.efi >> SHA256SUMS + sha256sum -- *.tar.zst >> SHA256SUMS + sha256sum -- *.erofs >> SHA256SUMS + # Don't put .erofs.caibx into the SHA256SUMS, it will break file matching. + # https://github.com/systemd/systemd/issues/38605 + sha256sum -- *-x86-64.caibx >> SHA256SUMS -### Clean up previous images (this replaces the upload-tree with only SHA256SUMS!) -go -C ./token-redeemer/ run . -go -C ./upload-vacuum-v3/ run . + gpg --homedir="$GNUPGHOME" --output SHA256SUMS.gpg --detach-sign SHA256SUMS + ) -#### Sign SHA256SUMS -gpg --homedir="$GNUPGHOME" --output "$V2_TREE/SHA256SUMS.gpg" --detach-sign "$V2_TREE/SHA256SUMS" + # Prepare the staging upload tree. + rm -rf upload-tree + mkdir -p "$V2_TREE" + mv "$OUTDIR"/*.iso "$OUTDIR"/*.torrent upload-tree/ + mv "$OUTDIR"/*.efi "$OUTDIR"/*.tar.zst "$OUTDIR"/*.erofs "$OUTDIR"/*.caibx "$V2_TREE/" + mv "$OUTDIR"/SHA256SUMS "$OUTDIR"/SHA256SUMS.gpg "$V2_TREE/" -#### Upload SHA256SUMS -go -C ./token-redeemer/ run . -go -C ./uploader/ run . --remote "$S3_TARGET" + # Upload to the per-job staging prefix in the bucket. + go -C ./token-redeemer/ run . + go -C ./uploader/ run . --remote "$S3_TARGET_STAGING" + + # Emit the exact staging target for the publish job, plus public URLs for + # OpenQA to test the staged image and sysupdate channel. + echo "S3_TARGET_STAGING=$S3_TARGET_STAGING" >> build.env + echo "IMAGE_URL=${S3_TARGET_STAGING#s3+}/$(basename upload-tree/*.iso)" >> build.env + echo "STAGING_CHANNEL_URL=${S3_TARGET_STAGING#s3+}/sysupdate/v2/" >> build.env +} + +publish() { + if [[ -z "$S3_TARGET_STAGING" ]]; then + echo "S3_TARGET_STAGING must be supplied by the imaging job dotenv artifact." >&2 + exit 1 + fi + + # Pull the staged build down from the bucket into ./publish-artifacts. + rm -rf publish-artifacts + mkdir -p publish-artifacts + go -C ./token-redeemer/ run . + go -C ./publisher/ build -o publisher . + ./publisher/publisher --remote "$S3_TARGET_STAGING" --output publish-artifacts --download + + # Publish to files.kde.org. + go -C ./upload-vacuum/ build -o upload-vacuum . + ( + cd publish-artifacts + ../upload-vacuum/upload-vacuum + + # shellcheck disable=SC2129 + sha256sum -- *.efi >> SHA256SUMS + sha256sum -- *.tar.zst >> SHA256SUMS + sha256sum -- *.erofs >> SHA256SUMS + # https://github.com/systemd/systemd/issues/38605 + sha256sum -- *-x86-64.caibx >> SHA256SUMS + + gpg --homedir="$GNUPGHOME" --output SHA256SUMS.gpg --detach-sign SHA256SUMS + + scp -i "$SSH_IDENTITY" ./*.iso ./*.torrent "$REMOTE_ROOT_PATH" + scp -i "$SSH_IDENTITY" ./*.efi ./*.tar.zst ./*.erofs ./*.caibx "$REMOTE_SYSUPDATE_PATH" + ) + + # Push the rootfs chunks into the S3 chunk store. + go install -v github.com/folbricht/desync/cmd/desync@f67d01e + export PATH="$HOME/go/bin:$PATH" + go -C ./token-redeemer/ run . + desync chop \ + --concurrency "$(nproc)" \ + --store "$S3_STORE" \ + publish-artifacts/*-x86-64.caibx \ + publish-artifacts/*-x86-64.erofs + + # Upload sums to files.kde.org only after the chunk store is ready. + scp -i "$SSH_IDENTITY" publish-artifacts/SHA256SUMS publish-artifacts/SHA256SUMS.gpg "$REMOTE_SYSUPDATE_PATH" + + # Merge the staged tree into the live S3 tree only after files.kde.org and the chunk store are ready. + go -C ./token-redeemer/ run . + ./publisher/publisher --remote "$S3_TARGET_STAGING" --promote + + # Regenerate, re-sign and re-upload SHA256SUMS. + go -C ./token-redeemer/ run . + go -C ./upload-vacuum-v3/ run . + + gpg --homedir="$GNUPGHOME" \ + --output "upload-tree/testing/sysupdate/v2/SHA256SUMS.gpg" \ + --detach-sign "upload-tree/testing/sysupdate/v2/SHA256SUMS" + go -C ./token-redeemer/ run . + go -C ./uploader/ run . --remote "$S3_TARGET" +} + +curl -s https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer | bash + +sudo chown -Rvf "$(id -u):$(id -g)" "$PWD/.secure_files" # Make sure we have access +chmod 600 "$SSH_IDENTITY" +gpg --verbose --no-options --homedir="$GNUPGHOME" --import "$PWD/.secure_files/gpg.private.key" + + +if [[ "${STAGE}" -eq 1 ]]; then + stage +fi + +if [[ "${PUBLISH}" -eq 1 ]]; then + publish +fi diff --git a/uploader/awsconfig.go b/uploader/awsconfig.go deleted file mode 100644 index 74a797b..0000000 --- a/uploader/awsconfig.go +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL -// SPDX-FileCopyrightText: 2025 Harald Sitter - -package main - -import ( - "errors" - "os" - "path/filepath" - - "gopkg.in/ini.v1" -) - -type AWSSection struct { - AccessKeyId string `ini:"aws_access_key_id"` - SecretKey string `ini:"aws_secret_access_key"` - SessionToken string `ini:"aws_session_token"` -} - -func readConfigAWS(section string) (AWSSection, error) { - awsSection := AWSSection{} - - awsConfigPath := filepath.Join(os.Getenv("HOME"), ".aws", "credentials") - cfg, err := ini.Load(awsConfigPath) - if err != nil { - return awsSection, errors.New("failed to load AWS credentials file: " + err.Error()) - } - - err = cfg.Section(section).MapTo(&awsSection) - if err != nil { - return awsSection, errors.New("failed to map AWS credentials section: " + err.Error()) - } - - return awsSection, nil -} diff --git a/uploader/go.mod b/uploader/go.mod index 48cea83..055035a 100644 --- a/uploader/go.mod +++ b/uploader/go.mod @@ -6,9 +6,12 @@ toolchain go1.24.4 require ( github.com/minio/minio-go/v7 v7.0.95 + invent.kde.org/kde-linux/kde-linux/minioauth v0.0.0 gopkg.in/ini.v1 v1.67.0 ) +replace invent.kde.org/kde-linux/kde-linux/minioauth => ../minioauth + require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/uploader/main.go b/uploader/main.go index d8d5451..efcd895 100644 --- a/uploader/main.go +++ b/uploader/main.go @@ -17,40 +17,18 @@ import ( "strings" "sync" + "invent.kde.org/kde-linux/kde-linux/minioauth" + "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" ) func connectToMinIO(endpoint string) *minio.Client { - awsSection, err := readConfigAWS("default") + minioClient, err := minioauth.Connect(endpoint) if err != nil { - log.Fatalln("Failed to read AWS config:", err) - } - accessKeyID := awsSection.AccessKeyId - if accessKeyID == "" { - log.Fatalln("AWS access key ID is empty") - } - secretAccessKey := awsSection.SecretKey - if secretAccessKey == "" { - log.Fatalln("AWS secret access key is empty") - } - sessionToken := awsSection.SessionToken - if secretAccessKey == "" { - log.Fatalln("AWS session token is empty") - } - useSSL := true - - // Initialize minio client object. - minioClient, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, sessionToken), - Secure: useSSL, - TrailingHeaders: true, - }) - if err != nil { - log.Fatalln("Failed to create MinIO client:", err) + log.Fatalln("Failed to connect to MinIO:", err) } - buckets, err := minioClient.ListBuckets(context.Background()) + buckets, err := minioauth.ListBuckets(minioClient) if err != nil { log.Fatalln("Failed to list buckets:", err) }