From 18c547ba292786104d701fbf2b72e785044ecebf Mon Sep 17 00:00:00 2001 From: James Rich Date: Tue, 26 May 2026 09:53:23 -0500 Subject: [PATCH] fix(flatpak-ops): capture build-logic bootstrap via init script PR #5599's BuildOperationListener attached too late: build-logic's own plugin resolutions (kotlin-dsl plugin marker, detekt, etc.) happen before the root project applies meshtastic.flatpak-ops, so those URLs never reached the manifest. Vid's flatpak-builder run then failed with 'Plugin [org.gradle.kotlin.kotlin-dsl:6.5.7] was not found' under --offline Gradle. Fix: move listener registration into a Gradle init script (gradle/init-scripts/flatpak-ops.init.gradle.kts) passed via -I. The init script fires before any project or plugin resolution, so build-logic bootstrap downloads are captured. The flatpak-ops plugin now reads the shared URL set from gradle.extensions; if the init script isn't loaded, it falls back to a local listener and warns. CI workflows + scripts/verify-flatpak/verify.sh updated to pass -I gradle/init-scripts/flatpak-ops.init.gradle.kts. Also expand verify.sh to optionally run a full flatpak-builder build (not just --download-only), with macOS refusing full-build mode because nested bwrap fails under Docker Desktop's seccomp. Adds --download-only and --skip-regen flags. Verified on macOS via --download-only: manifest grew to 2744 entries and now contains org.gradle.kotlin.kotlin-dsl.gradle.plugin (the artifact that broke vid's CI). Full-build verification pending on Linux. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/release.yml | 1 + .github/workflows/reusable-check.yml | 1 + .../meshtastic/flatpakops/FlatpakOpsPlugin.kt | 24 +++- .../init-scripts/flatpak-ops.init.gradle.kts | 44 +++++++ scripts/verify-flatpak/README.md | 17 ++- scripts/verify-flatpak/verify.sh | 113 +++++++++++------- 6 files changed, 149 insertions(+), 51 deletions(-) create mode 100644 gradle/init-scripts/flatpak-ops.init.gradle.kts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86b3c37a9..cd116603c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -378,6 +378,7 @@ jobs: run: > ./gradlew --no-build-cache --no-configuration-cache -Dgradle.user.home=${{ runner.temp }}/flatpak-gradle-home + -I gradle/init-scripts/flatpak-ops.init.gradle.kts :desktopApp:assemble :captureFlatpakSources - name: Stage manifest diff --git a/.github/workflows/reusable-check.yml b/.github/workflows/reusable-check.yml index cfb7ffaa6..932659de2 100644 --- a/.github/workflows/reusable-check.yml +++ b/.github/workflows/reusable-check.yml @@ -580,6 +580,7 @@ jobs: run: > ./gradlew --no-build-cache --no-configuration-cache -Dgradle.user.home=${{ runner.temp }}/flatpak-gradle-home + -I gradle/init-scripts/flatpak-ops.init.gradle.kts :desktopApp:assemble :captureFlatpakSources - name: Stage manifest diff --git a/build-logic/flatpak-ops/src/main/kotlin/org/meshtastic/flatpakops/FlatpakOpsPlugin.kt b/build-logic/flatpak-ops/src/main/kotlin/org/meshtastic/flatpakops/FlatpakOpsPlugin.kt index e63178969..169e3fb8a 100644 --- a/build-logic/flatpak-ops/src/main/kotlin/org/meshtastic/flatpakops/FlatpakOpsPlugin.kt +++ b/build-logic/flatpak-ops/src/main/kotlin/org/meshtastic/flatpakops/FlatpakOpsPlugin.kt @@ -50,12 +50,24 @@ class FlatpakOpsPlugin : Plugin { override fun apply(target: Project) { check(target == target.rootProject) { "meshtastic.flatpak-ops must be applied to the root project" } - val capturedUrls: MutableSet = ConcurrentHashMap.newKeySet() - val manager: BuildOperationListenerManager = - (target as ProjectInternal).services.get(BuildOperationListenerManager::class.java) - - val listener = OpListener(capturedUrls) - manager.addListener(listener) + // Prefer the URL set populated by gradle/init-scripts/flatpak-ops.init.gradle.kts. + // The init script attaches its listener BEFORE any plugin/project resolution, so it + // captures bootstrap downloads (kotlin-dsl plugin marker, build-logic deps) that a + // listener registered here would miss. If the init script wasn't passed via -I, we + // fall back to a locally-attached listener — incomplete for build-logic deps but + // useful for developer debugging. + @Suppress("UNCHECKED_CAST") + val capturedUrls: MutableSet = + (target.gradle.extensions.findByName("flatpakOpsCapturedUrls") as? MutableSet) + ?: ConcurrentHashMap.newKeySet().also { fallback -> + val manager = + (target as ProjectInternal).services.get(BuildOperationListenerManager::class.java) + manager.addListener(OpListener(fallback)) + target.logger.warn( + "flatpak-ops: init script not loaded; build-logic bootstrap URLs will be missing. " + + "Pass -I gradle/init-scripts/flatpak-ops.init.gradle.kts for a complete manifest.", + ) + } val outputProvider = target.layout.buildDirectory.file("flatpak-ops-sources.json") diff --git a/gradle/init-scripts/flatpak-ops.init.gradle.kts b/gradle/init-scripts/flatpak-ops.init.gradle.kts new file mode 100644 index 000000000..ac4de7018 --- /dev/null +++ b/gradle/init-scripts/flatpak-ops.init.gradle.kts @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * Init script for meshtastic.flatpak-ops. Attaches a BuildOperationListener + * BEFORE any project or plugin resolution happens — which is necessary because + * the flatpak-ops plugin itself lives in build-logic, and any artifacts pulled + * to bootstrap build-logic (kotlin-dsl plugin marker, detekt, etc.) would be + * invisible to a listener registered later from a root-project plugin. + * + * Captured URLs are stored on `gradle.extensions` under the key below; the + * captureFlatpakSources task (registered by FlatpakOpsPlugin) reads them. + * + * Pass to Gradle via: + * ./gradlew -I gradle/init-scripts/flatpak-ops.init.gradle.kts ... + */ + +import org.gradle.api.internal.GradleInternal +import org.gradle.internal.operations.BuildOperationDescriptor +import org.gradle.internal.operations.BuildOperationListener +import org.gradle.internal.operations.BuildOperationListenerManager +import org.gradle.internal.operations.OperationFinishEvent +import org.gradle.internal.operations.OperationIdentifier +import org.gradle.internal.operations.OperationProgressEvent +import org.gradle.internal.operations.OperationStartEvent +import org.gradle.internal.resource.ExternalResourceReadBuildOperationType +import java.util.concurrent.ConcurrentHashMap + +val capturedUrls: MutableSet = ConcurrentHashMap.newKeySet() +gradle.extensions.add("flatpakOpsCapturedUrls", capturedUrls) + +val manager = + (gradle as GradleInternal).services.get(BuildOperationListenerManager::class.java) + +manager.addListener( + object : BuildOperationListener { + override fun started(op: BuildOperationDescriptor, e: OperationStartEvent) = Unit + override fun progress(id: OperationIdentifier, e: OperationProgressEvent) = Unit + override fun finished(op: BuildOperationDescriptor, e: OperationFinishEvent) { + val details = op.details as? ExternalResourceReadBuildOperationType.Details ?: return + if (e.failure != null) return + capturedUrls.add(details.location) + } + }, +) diff --git a/scripts/verify-flatpak/README.md b/scripts/verify-flatpak/README.md index 46b593e5a..616c97b07 100644 --- a/scripts/verify-flatpak/README.md +++ b/scripts/verify-flatpak/README.md @@ -34,9 +34,16 @@ instead of waiting on cross-repo CI. ## Usage ```bash -# Full offline build (~10–20 min the first time, faster after — Docker image is cached) +# Full offline build — Linux host required (~15–30 min first time) scripts/verify-flatpak/verify.sh +# URLs + sha256 verification only; skips the Gradle build phase. +# Works on macOS where nested bwrap fails under Docker Desktop's seccomp. +scripts/verify-flatpak/verify.sh --download-only + +# Reuse an already-generated flatpak-sources.json (don't re-run Gradle) +scripts/verify-flatpak/verify.sh --skip-regen + # Cross-arch test via QEMU emulation (slower) scripts/verify-flatpak/verify.sh --arch aarch64 @@ -44,6 +51,14 @@ scripts/verify-flatpak/verify.sh --arch aarch64 scripts/verify-flatpak/verify.sh --shell ``` +### macOS limitation + +`flatpak-builder` runs the build phase inside `bwrap` (bubblewrap). Nested +bwrap fails inside Docker Desktop on macOS with +`prctl(PR_SET_SECCOMP) EINVAL`. The script refuses to run a full build on +macOS by default — pass `--download-only` to validate URLs + sha256s without +executing the Gradle build, or run the full script on a Linux host. + ## Interpreting failures | Symptom | Likely cause | diff --git a/scripts/verify-flatpak/verify.sh b/scripts/verify-flatpak/verify.sh index 30e888824..87007f007 100755 --- a/scripts/verify-flatpak/verify.sh +++ b/scripts/verify-flatpak/verify.sh @@ -1,33 +1,42 @@ #!/usr/bin/env bash -# Local replica of vid's flatpak CI (vidplace7/org.meshtastic.desktop, .github/workflows/build-flatpak.yml) -# but flipped to true-offline mode: our flatpak-sources.json is included and --share=network is removed. +# Local replica of vid's flatpak CI (vidplace7/org.meshtastic.desktop, +# .github/workflows/build-flatpak.yml) but flipped to true-offline mode: our +# flatpak-sources.json is included and --share=network is removed from the +# build phase. # -# Goal: validate flatpak-sources.json without bugging vid to push & re-run his workflow. +# Goal: validate flatpak-sources.json end-to-end (download + verify sha256s + +# offline Gradle build) without bugging vid to push & re-run his workflow. # # Requirements: -# - Docker (Docker Desktop on macOS is fine; needs ~10GB free + ability to run --privileged) -# - This Meshtastic-Android checkout has produced flatpak-sources.json -# (run `./gradlew :desktopApp:assemble :captureFlatpakSources` first, or this script will do it) +# - Docker (Docker Desktop on macOS works for --download-only mode; full builds +# need a Linux host because flatpak-builder uses nested bwrap which fails +# under Docker Desktop's seccomp sandbox). +# - ~15GB free disk for the SDK + Gradle cache + builddir. # # Usage: -# scripts/verify-flatpak/verify.sh # full build, x86_64 -# scripts/verify-flatpak/verify.sh --arch aarch64 # cross-arch via QEMU emulation -# scripts/verify-flatpak/verify.sh --shell # drop into the container shell instead of building +# scripts/verify-flatpak/verify.sh # full offline build (Linux only) +# scripts/verify-flatpak/verify.sh --download-only # URLs+sha256 only (works on macOS) +# scripts/verify-flatpak/verify.sh --arch aarch64 # cross-arch via QEMU emulation +# scripts/verify-flatpak/verify.sh --shell # drop into builder container shell +# scripts/verify-flatpak/verify.sh --skip-regen # reuse existing flatpak-sources.json set -euo pipefail ARCH="x86_64" DROP_TO_SHELL=0 +DOWNLOAD_ONLY=0 +SKIP_REGEN=0 while [[ $# -gt 0 ]]; do case "$1" in --arch) ARCH="$2"; shift 2 ;; --shell) DROP_TO_SHELL=1; shift ;; - -h|--help) sed -n '2,17p' "$0"; exit 0 ;; + --download-only) DOWNLOAD_ONLY=1; shift ;; + --skip-regen) SKIP_REGEN=1; shift ;; + -h|--help) sed -n '2,22p' "$0"; exit 0 ;; *) echo "Unknown arg: $1" >&2; exit 2 ;; esac done -# Map flatpak arch names to docker platform names case "$ARCH" in x86_64) DOCKER_PLATFORM="linux/amd64" ;; aarch64) DOCKER_PLATFORM="linux/arm64" ;; @@ -38,11 +47,12 @@ REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel)" WORK="$REPO_ROOT/build/flatpak-verify" OVERLAY="$REPO_ROOT/scripts/verify-flatpak/desktop-offline.yaml" SOURCES_JSON="$REPO_ROOT/flatpak-sources.json" +GRADLE_HOME_ISOLATED="$REPO_ROOT/build/flatpak-gradle-home" VID_REPO="https://github.com/vidplace7/org.meshtastic.desktop.git" -# Image provides flatpak + flatpak-builder. The freedesktop 25.08 runtime declared in -# the manifest is pulled from flathub at build time (no 25.08 image exists yet; 24.08 is -# fine as the builder host because the SDK used at compile time comes from flathub). +# bilelmoussaoui's image is what vid's CI uses; freedesktop-24.08 is the latest +# tag available. The 25.08 runtime declared in the manifest is pulled from +# flathub at build time inside the container. BUILDER_IMAGE="bilelmoussaoui/flatpak-github-actions:freedesktop-24.08" step() { printf '\n\033[1;34m==> %s\033[0m\n' "$*"; } @@ -50,10 +60,23 @@ fail() { printf '\033[1;31m!! %s\033[0m\n' "$*" >&2; exit 1; } command -v docker >/dev/null 2>&1 || fail "docker is required; install Docker Desktop or equivalent." -step "Ensuring flatpak-sources.json is fresh" -if [[ ! -f "$SOURCES_JSON" ]]; then - (cd "$REPO_ROOT" && ./gradlew --no-build-cache --no-configuration-cache :desktopApp:assemble :captureFlatpakSources) +# Refuse full-build mode on macOS — nested bwrap fails under Docker Desktop's +# seccomp and the user will spend 20 minutes finding out. They can override +# with --download-only. +if [[ "$(uname -s)" == "Darwin" && $DOWNLOAD_ONLY -eq 0 && $DROP_TO_SHELL -eq 0 ]]; then + fail "Full flatpak-builder runs require a Linux host (nested bwrap fails under Docker Desktop on macOS). Re-run with --download-only, or use --shell to poke around manually." +fi + +if [[ $SKIP_REGEN -eq 0 ]]; then + step "Regenerating flatpak-sources.json via isolated Gradle home" + rm -rf "$GRADLE_HOME_ISOLATED" + (cd "$REPO_ROOT" && ./gradlew --no-build-cache --no-configuration-cache \ + -Dgradle.user.home="$GRADLE_HOME_ISOLATED" \ + -I gradle/init-scripts/flatpak-ops.init.gradle.kts \ + :desktopApp:assemble :captureFlatpakSources) cp "$REPO_ROOT/build/flatpak-ops-sources.json" "$SOURCES_JSON" +elif [[ ! -f "$SOURCES_JSON" ]]; then + fail "--skip-regen specified but $SOURCES_JSON does not exist." fi step "Preparing workspace at $WORK" @@ -70,8 +93,6 @@ step "Wiring overlay manifest + our flatpak-sources.json" cp "$OVERLAY" "$WORK/org.meshtastic.desktop/org.meshtastic.desktop.yaml" cp "$SOURCES_JSON" "$WORK/org.meshtastic.desktop/flatpak-sources.json" -# Materialize a clean copy of our checkout (excluding build outputs) for `type: dir`. -# flatpak-builder copies the whole tree — skip heavy/irrelevant paths. step "Snapshotting Meshtastic-Android checkout (excluding build/, .gradle/)" rsync -a --delete \ --exclude='/build/' \ @@ -85,32 +106,36 @@ rsync -a --delete \ step "Pulling builder image: $BUILDER_IMAGE ($DOCKER_PLATFORM)" docker pull --platform "$DOCKER_PLATFORM" "$BUILDER_IMAGE" >/dev/null +DOCKER_RUN_ARGS=( + --rm + --privileged + -v "$WORK/org.meshtastic.desktop:/work" + -w /work + --platform "$DOCKER_PLATFORM" + --security-opt seccomp=unconfined +) + if [[ $DROP_TO_SHELL -eq 1 ]]; then step "Dropping into builder shell — flatpak-builder is on PATH" - exec docker run --rm -it --privileged \ - -v "$WORK/org.meshtastic.desktop:/work" \ - -w /work \ - --platform "$DOCKER_PLATFORM" \ - --security-opt seccomp=unconfined \ - "$BUILDER_IMAGE" bash + exec docker run -it "${DOCKER_RUN_ARGS[@]}" "$BUILDER_IMAGE" bash fi -step "Running flatpak-builder (arch=$ARCH)" -docker run --rm --privileged \ - -v "$WORK/org.meshtastic.desktop:/work" \ - -w /work \ - --platform "$DOCKER_PLATFORM" \ - --security-opt seccomp=unconfined \ - "$BUILDER_IMAGE" \ - bash -c "set -e - flatpak remote-add --user --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo - # --download-only verifies every source URL + sha256 and exits before the bwrap - # sandbox phase. We do this because nested bwrap fails inside Docker Desktop on - # macOS (prctl(PR_SET_SECCOMP) EINVAL). For full sandbox build, run on Linux directly - # — or rely on vid's GHA CI which uses bare ubuntu-24.04 runners. - flatpak-builder --user --repo=repo --install-deps-from=flathub --force-clean \ - --disable-rofiles-fuse --download-only \ - builddir org.meshtastic.desktop.yaml - echo - echo '=== All sources downloaded and sha256-verified successfully ===' - " +# Build flatpak-builder invocation. --download-only mode skips the bwrap-based +# build phase, which is the part that fails under Docker Desktop on macOS. +if [[ $DOWNLOAD_ONLY -eq 1 ]]; then + BUILDER_EXTRA_FLAGS="--download-only" + SUCCESS_MSG="All sources downloaded and sha256-verified successfully (URLs + hashes OK; Gradle build NOT exercised)" +else + BUILDER_EXTRA_FLAGS="" + SUCCESS_MSG="Full offline build succeeded — flatpak-sources.json is complete and self-sufficient" +fi + +step "Running flatpak-builder (arch=$ARCH, mode=$([[ $DOWNLOAD_ONLY -eq 1 ]] && echo download-only || echo full-build))" +docker run "${DOCKER_RUN_ARGS[@]}" "$BUILDER_IMAGE" bash -c "set -e + flatpak remote-add --user --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo + flatpak-builder --user --repo=repo --install-deps-from=flathub --force-clean \ + --disable-rofiles-fuse $BUILDER_EXTRA_FLAGS \ + builddir org.meshtastic.desktop.yaml + echo + echo '=== $SUCCESS_MSG ===' +"