diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2c972b4cc..86b3c37a9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -370,13 +370,21 @@ jobs: gradle_encryption_key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} cache_read_only: 'true' + # Uses an isolated Gradle user home so every artifact actually traverses the + # network — meshtastic.flatpak-ops captures URLs via BuildOperationListener and + # only sees ExternalResourceReadBuildOperation events for cache misses. With the + # shared cache, downloads would be skipped and the manifest would be (nearly) empty. - name: Generate Flatpak Sources run: > - ./gradlew :desktopApp:assemble :generateFlatpakSourcesFromCache - --no-configuration-cache + ./gradlew --no-build-cache --no-configuration-cache + -Dgradle.user.home=${{ runner.temp }}/flatpak-gradle-home + :desktopApp:assemble :captureFlatpakSources + + - name: Stage manifest + run: cp build/flatpak-ops-sources.json flatpak-sources.json - name: List Flatpak source files - run: ls -R flatpak-sources.json + run: ls -l flatpak-sources.json - name: Upload Flatpak source artifacts if: always() diff --git a/.github/workflows/reusable-check.yml b/.github/workflows/reusable-check.yml index d656fa990..cfb7ffaa6 100644 --- a/.github/workflows/reusable-check.yml +++ b/.github/workflows/reusable-check.yml @@ -575,11 +575,16 @@ jobs: gradle_encryption_key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} cache_read_only: true + # Isolated Gradle user home — see explanation in release.yml. - name: Generate Flatpak Sources run: > - ./gradlew :desktopApp:assemble :generateFlatpakSourcesFromCache - --no-configuration-cache --refresh-dependencies - + ./gradlew --no-build-cache --no-configuration-cache + -Dgradle.user.home=${{ runner.temp }}/flatpak-gradle-home + :desktopApp:assemble :captureFlatpakSources + + - name: Stage manifest + run: cp build/flatpak-ops-sources.json flatpak-sources.json + - run: ls -lah flatpak-sources.json - name: Upload Flatpak Sources diff --git a/.gitignore b/.gitignore index 3c0d2082f..b49126920 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,4 @@ docs/_config_local.yml flatpak-sources-*.json flatpak-sources.json offline-repository/ +.claude/ diff --git a/build-logic/flatpak/build.gradle.kts b/build-logic/flatpak-ops/build.gradle.kts similarity index 88% rename from build-logic/flatpak/build.gradle.kts rename to build-logic/flatpak-ops/build.gradle.kts index 2fc3d0c06..04ed7143a 100644 --- a/build-logic/flatpak/build.gradle.kts +++ b/build-logic/flatpak-ops/build.gradle.kts @@ -23,7 +23,7 @@ plugins { alias(libs.plugins.detekt) } -group = "org.meshtastic.flatpak" +group = "org.meshtastic.flatpakops" java { sourceCompatibility = JavaVersion.VERSION_21 @@ -33,7 +33,6 @@ java { kotlin { compilerOptions { jvmTarget = JvmTarget.JVM_21 } } dependencies { - // Allows type-safe accessors for libs in plugin build script implementation(files(libs.javaClass.superclass.protectionDomain.codeSource.location)) detektPlugins(libs.detekt.formatting) } @@ -69,15 +68,14 @@ detekt { config.setFrom(rootProject.file("../config/detekt/detekt.yml")) buildUponDefaultConfig = true allRules = false - baseline = file("detekt-baseline.xml") source.setFrom(files("src/main/java", "src/main/kotlin")) } gradlePlugin { plugins { - register("meshtasticFlatpak") { - id = "meshtastic.flatpak" - implementationClass = "org.meshtastic.flatpak.FlatpakPlugin" + register("meshtasticFlatpakOps") { + id = "meshtastic.flatpak-ops" + implementationClass = "org.meshtastic.flatpakops.FlatpakOpsPlugin" } } } diff --git a/build-logic/flatpak-ops/src/main/kotlin/org/meshtastic/flatpakops/FlatpakOpsPlugin.kt b/build-logic/flatpak-ops/src/main/kotlin/org/meshtastic/flatpakops/FlatpakOpsPlugin.kt new file mode 100644 index 000000000..e63178969 --- /dev/null +++ b/build-logic/flatpak-ops/src/main/kotlin/org/meshtastic/flatpakops/FlatpakOpsPlugin.kt @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.flatpakops + +import groovy.json.JsonOutput +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.internal.project.ProjectInternal +import org.gradle.internal.operations.BuildOperationDescriptor +import org.gradle.internal.operations.BuildOperationListener +import org.gradle.internal.operations.BuildOperationListenerManager +import org.gradle.internal.operations.OperationFinishEvent +import org.gradle.internal.operations.OperationIdentifier +import org.gradle.internal.operations.OperationProgressEvent +import org.gradle.internal.operations.OperationStartEvent +import org.gradle.internal.resource.ExternalResourceReadBuildOperationType +import java.io.File +import java.net.URI +import java.security.MessageDigest +import java.util.concurrent.ConcurrentHashMap + +/** + * Captures every external resource URL Gradle reads via the internal BuildOperationListener API and emits a + * Flathub-compliant flatpak-sources.json at build finish. + * + * No heuristics: URL is authoritative (taken straight from the build op), the on-disk file is found via Gradle's + * documented files-2.1 layout, and SHA-256 is computed from that exact file. + * + * Internal APIs touched (acceptable trade-off; same path flatpak-gradle-generator uses): + * - org.gradle.internal.operations.BuildOperationListener / BuildOperationListenerManager + * - org.gradle.internal.resource.ExternalResourceReadBuildOperationType + * - org.gradle.api.internal.project.ProjectInternal (for .services) + */ +class FlatpakOpsPlugin : Plugin { + + override fun apply(target: Project) { + check(target == target.rootProject) { "meshtastic.flatpak-ops must be applied to the root project" } + + val capturedUrls: MutableSet = ConcurrentHashMap.newKeySet() + val manager: BuildOperationListenerManager = + (target as ProjectInternal).services.get(BuildOperationListenerManager::class.java) + + val listener = OpListener(capturedUrls) + manager.addListener(listener) + + val outputProvider = target.layout.buildDirectory.file("flatpak-ops-sources.json") + + target.tasks.register("captureFlatpakSources") { + group = "flatpak" + description = "Emit flatpak-sources.json from URLs captured via BuildOperationListener." + outputs.upToDateWhen { false } + // Must run AFTER the task that triggers resolution. Without this, Gradle's scheduler + // may interleave/parallelize this task with :desktopApp:assemble, causing us to write + // the file before the downloads we want to capture have happened. + mustRunAfter(":desktopApp:assemble") + val proj = target + val urlsRef = capturedUrls + val outFile = outputProvider + doLast { writeSources(proj, urlsRef.toList(), outFile.get().asFile) } + } + // Listener is intentionally NOT removed on task completion; it stays attached until JVM + // exit. Removal is unsafe when our task races against other resolution-emitting tasks. + // Known limitation: in a long-lived Gradle daemon, capturedUrls accumulates across builds. + // We can't clear it at task start (that would erase what was captured during assemble). + // The CI workflow uses a fresh isolated GRADLE_USER_HOME, so this only affects local + // developer use — and re-emitting a superset of URLs is harmless. + } + + private class OpListener(private val urls: MutableSet) : BuildOperationListener { + override fun started(op: BuildOperationDescriptor, e: OperationStartEvent) = Unit + + override fun progress(id: OperationIdentifier, e: OperationProgressEvent) = Unit + + override fun finished(op: BuildOperationDescriptor, e: OperationFinishEvent) { + val details = op.details as? ExternalResourceReadBuildOperationType.Details ?: return + if (e.failure != null) return + // No host/scheme filtering here: non-Maven URLs (distribution zips, repo listings, etc.) + // naturally drop out in writeSources() when locateCacheFile() can't find them under + // files-2.1. Keeping this listener permissive avoids hardcoding repo allowlists. + urls.add(details.location) + } + } + + private fun writeSources(project: Project, urls: List, output: File) { + val filesRoot = File(project.gradle.gradleUserHomeDir, "caches/modules-2/files-2.1") + val entries: List> = + urls + .distinct() + .filterNot { url -> + // Exclude sources/javadoc jars: they're not needed for an offline build and + // inflate the manifest. Match on URL filename, not cache-relative path. + val tail = url.substringAfterLast('/') + tail.endsWith("-sources.jar") || tail.endsWith("-javadoc.jar") + } + .sorted() + .mapNotNull { url -> + val cacheFile = locateCacheFile(filesRoot, url) + if (cacheFile == null) { + project.logger.info("flatpak-ops: no cache file for {} (skipped)", url) + return@mapNotNull null + } + val rel = cacheFile.relativeTo(filesRoot).path.replace('\\', '/').split('/') + if (rel.size < CACHE_PATH_SEGMENTS) return@mapNotNull null + val group = rel[0] + val artifact = rel[1] + val version = rel[2] + val onDiskFilename = rel.last() + val groupPath = group.replace('.', '/') + val entry = + mutableMapOf( + "type" to "file", + "url" to url, + "sha256" to sha256(cacheFile), + "dest" to "offline-repository/$groupPath/$artifact/$version", + "dest-filename" to onDiskFilename, + ) + mirrorsFor(url).takeIf { it.isNotEmpty() }?.let { entry["mirror-urls"] = it } + entry + } + output.parentFile?.mkdirs() + output.writeText(JsonOutput.prettyPrint(JsonOutput.toJson(entries))) + project.logger.lifecycle( + "flatpak-ops: captured {} URLs, emitted {} sources to {}", + urls.size, + entries.size, + output.absolutePath, + ) + } + + /** + * Last three URL path segments are artifact/version/filename (Maven 2 spec; present in every Maven URL). Gradle's + * files-2.1 layout is `////`. The (artifact, version, + * filename) triple is NOT unique across groups (e.g. androidx.annotation:annotation:1.10.0.module collides with + * org.jetbrains.compose.annotation-internal:annotation:1.10.0.module), so the group MUST be derived from the URL + * path. We probe every suffix of the path's leading segments against the cache layout — the longest match wins + * (strips arbitrary repo prefixes like `/maven2/`, `/dl/android/maven2/`, `/m2/` without hardcoding them). + */ + private fun locateCacheFile(filesRoot: File, url: String): File? { + val path = URI(url).path.trimEnd('/').split('/').filter { it.isNotEmpty() } + val tail = path.takeLast(URL_TRAILING_SEGMENTS) + if (!filesRoot.isDirectory || tail.size < URL_TRAILING_SEGMENTS) return null + val (artifact, version, filename) = tail + val groupCandidates = path.dropLast(URL_TRAILING_SEGMENTS) + // files-2.1 uses dot-joined group as a SINGLE directory, e.g. `androidx.annotation/`, + // not `androidx/annotation/`. So join URL segments with '.' here. + return groupCandidates.indices.firstNotNullOfOrNull { start -> + val groupDir = groupCandidates.drop(start).joinToString(".") + File(filesRoot, "$groupDir/$artifact/$version") + .takeIf(File::isDirectory) + ?.listFiles { f -> f.isDirectory } + ?.map { shaDir -> File(shaDir, filename) } + ?.firstOrNull { it.isFile } + } + } + + /** + * Derive fallback mirror URLs from the primary URL's host. Only Maven Central has well-known public mirrors; for + * everything else (Google, JitPack, Gradle plugin portal, snapshot repos), we trust the primary URL since these + * hosts don't have stable mirrors anyway. The URL itself is authoritative — we just rewrite the host while + * preserving the path. + */ + private fun mirrorsFor(url: String): List { + val uri = runCatching { URI(url) }.getOrNull() + val host = uri?.host + val path = uri?.rawPath + return if (host != null && path != null && host in MAVEN_CENTRAL_HOSTS) { + MAVEN_CENTRAL_HOSTS.filter { it != host }.map { "https://$it$path" } + + "https://maven-central.storage-download.googleapis.com$path" + } else { + emptyList() + } + } + + private fun sha256(file: File): String { + val md = MessageDigest.getInstance("SHA-256") + file.inputStream().use { stream -> + val buf = ByteArray(BUFFER_SIZE) + while (true) { + val n = stream.read(buf) + if (n <= 0) break + md.update(buf, 0, n) + } + } + return hex(md.digest()) + } + + private fun hex(bytes: ByteArray): String { + val digits = "0123456789abcdef" + val chars = CharArray(bytes.size * HEX_CHARS_PER_BYTE) + for (i in bytes.indices) { + val v = bytes[i].toInt() and BYTE_MASK + chars[i * HEX_CHARS_PER_BYTE] = digits[v ushr NIBBLE_BITS] + chars[i * HEX_CHARS_PER_BYTE + 1] = digits[v and NIBBLE_MASK] + } + return String(chars) + } + + private companion object { + private val MAVEN_CENTRAL_HOSTS = listOf("repo.maven.apache.org", "repo1.maven.org") + private const val BUFFER_SIZE = 8192 + private const val CACHE_PATH_SEGMENTS = 5 + private const val URL_TRAILING_SEGMENTS = 3 + private const val HEX_CHARS_PER_BYTE = 2 + private const val BYTE_MASK = 0xFF + private const val NIBBLE_BITS = 4 + private const val NIBBLE_MASK = 0x0F + } +} diff --git a/build-logic/flatpak/README.md b/build-logic/flatpak/README.md deleted file mode 100644 index c4fdbbf0d..000000000 --- a/build-logic/flatpak/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Meshtastic Flatpak Source Manifest Generator - -This directory contains the isolated, lightweight `:flatpak` subproject under `build-logic`. It registers and exposes the formal Gradle plugin `meshtastic.flatpak` (`FlatpakConventionPlugin`) to automate generating Flathub-compliant offline dependency manifests. - ---- - -## Purpose - -To build sandboxed desktop applications on Flathub completely offline (`--offline`), the Flatpak builder requires an exact, pre-calculated registry of all remote dependency locations and their cryptographic hashes (`flatpak-sources.json`). - -Previously, this logic was mixed in loose scripts or monolithic build conventions. Isolating it into this standalone subproject provides: -* **Clean Boundaries**: Decouples packaging/publishing details from standard multiplatform compile configurations. -* **First-Class Configuration Caching**: Safe from eager state evaluations and non-serializable property capture. -* **Ease of Sharing/Publishing**: Simplifies future distribution or independent publication of the plugin. - ---- - -## Key Features - -### 1. Remote Snapshot Metadata Resolution -Standard Maven snapshot repositories (e.g., Sonatype Snapshots) return `404` errors when fetching non-timestamped `-SNAPSHOT` dependencies directly. This plugin fetches `maven-metadata.xml` from the remote snapshot repository at generation time, resolves the unique timestamped snapshot coordinate (e.g., `0.2.4-20260520.043744-2`), and constructs exact, direct download URLs while preserving local filename bindings. - -### 2. JitPack URL Routing -Automatically identifies external dependencies belonging to the `com.github.*` group (hosted on JitPack) and routes their `primaryUrl` to `https://jitpack.io` instead of attempting standard Maven Central lookup, preventing sandboxed download failures. - -### 3. Automatic Cache Population -The task automatically depends on `:desktopApp:assemble`, ensuring the Gradle dependency cache is fully populated before scanning. No manual pre-build step is required. - ---- - -## Usage - -Apply the plugin to the root project's `build.gradle.kts`: - -```kotlin -plugins { - id("meshtastic.flatpak") -} -``` - -### Configuration (DSL) - -All options have sensible defaults. Override as needed: - -```kotlin -flatpak { - // Task that populates the Gradle cache before scanning - assembleTask.set(":desktopApp:assemble") - // Custom Gradle cache directory (defaults to ~/.gradle/caches/modules-2/files-2.1) - cacheDir.set(layout.projectDirectory.dir("my-cache")) - // Output manifest path - outputFile.set(layout.projectDirectory.file("flatpak-sources.json")) -} -``` - -### Running the Generator Task - -Execute the registered custom task to sweep your Gradle local modules cache and generate/overwrite the root `flatpak-sources.json`: - -```bash -./gradlew :generateFlatpakSourcesFromCache -``` - ---- - -## Architecture - -* **[FlatpakPlugin.kt](src/main/kotlin/org/meshtastic/flatpak/FlatpakPlugin.kt)**: Registers the `flatpak {}` DSL extension and the `generateFlatpakSourcesFromCache` task using lazy provider configuration. -* **[FlatpakExtension.kt](src/main/kotlin/org/meshtastic/flatpak/FlatpakExtension.kt)**: DSL extension interface defining all configurable properties. -* **[GenerateFlatpakSourcesTask.kt](src/main/kotlin/org/meshtastic/flatpak/GenerateFlatpakSourcesTask.kt)**: The custom task responsible for Gradle files scanning, remote metadata resolution, and JSON generation. diff --git a/build-logic/flatpak/detekt-baseline.xml b/build-logic/flatpak/detekt-baseline.xml deleted file mode 100644 index f7d79d235..000000000 --- a/build-logic/flatpak/detekt-baseline.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - CyclomaticComplexMethod:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$@TaskAction fun generate - LongMethod:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$@TaskAction fun generate - MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$0xFF - MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$4 - MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$5 - MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$8192 - MaxLineLength:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$"Successfully scanned cache and generated ${outputSourcesFile.name} containing ${finalEntries.size} entries." - SwallowedException:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$e: Exception - TooGenericExceptionCaught:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$e: Exception - - diff --git a/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/FlatpakExtension.kt b/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/FlatpakExtension.kt deleted file mode 100644 index c7f9354b2..000000000 --- a/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/FlatpakExtension.kt +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2026 Meshtastic LLC - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.meshtastic.flatpak - -import org.gradle.api.file.DirectoryProperty -import org.gradle.api.file.RegularFileProperty -import org.gradle.api.provider.Property - -/** - * DSL extension for configuring the Flatpak source manifest generator. - * - * ```kotlin - * flatpak { - * assembleTask.set(":desktopApp:assemble") - * } - * ``` - */ -abstract class FlatpakExtension { - - /** Gradle cache directory to scan for dependency artifacts. */ - abstract val cacheDir: DirectoryProperty - - /** Output path for the generated flatpak-sources.json manifest. */ - abstract val outputFile: RegularFileProperty - - /** Task path to depend on, ensuring the cache is fully populated before scanning. */ - abstract val assembleTask: Property -} diff --git a/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/FlatpakPlugin.kt b/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/FlatpakPlugin.kt deleted file mode 100644 index aea02a96e..000000000 --- a/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/FlatpakPlugin.kt +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2026 Meshtastic LLC - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.meshtastic.flatpak - -import org.gradle.api.Plugin -import org.gradle.api.Project -import java.io.File - -class FlatpakPlugin : Plugin { - override fun apply(target: Project) { - with(target) { - val extension = - extensions.create("flatpak", FlatpakExtension::class.java).apply { - cacheDir.convention( - layout.dir(providers.provider { File(gradle.gradleUserHomeDir, "caches/modules-2/files-2.1") }), - ) - outputFile.convention(layout.projectDirectory.file("flatpak-sources.json")) - assembleTask.convention(":desktopApp:assemble") - } - - tasks.register("generateFlatpakSourcesFromCache", GenerateFlatpakSourcesTask::class.java) { - cacheDir.set(extension.cacheDir) - outputFile.set(extension.outputFile) - dependsOn(extension.assembleTask) - } - } - } -} diff --git a/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/GenerateFlatpakSourcesTask.kt b/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/GenerateFlatpakSourcesTask.kt deleted file mode 100644 index ad95d5833..000000000 --- a/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/GenerateFlatpakSourcesTask.kt +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Copyright (c) 2026 Meshtastic LLC - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.meshtastic.flatpak - -import groovy.json.JsonOutput -import org.gradle.api.DefaultTask -import org.gradle.api.file.DirectoryProperty -import org.gradle.api.file.RegularFileProperty -import org.gradle.api.tasks.InputDirectory -import org.gradle.api.tasks.OutputFile -import org.gradle.api.tasks.PathSensitive -import org.gradle.api.tasks.PathSensitivity -import org.gradle.api.tasks.TaskAction -import org.gradle.work.DisableCachingByDefault -import org.w3c.dom.Element -import org.w3c.dom.NodeList -import java.io.File -import java.net.URI -import java.security.MessageDigest -import javax.xml.parsers.DocumentBuilderFactory - -/** Generates a complete flatpak-sources.json manifest from the local Gradle cache directory. */ -@DisableCachingByDefault(because = "Resolves remote snapshot metadata that may change between runs") -abstract class GenerateFlatpakSourcesTask : DefaultTask() { - - @get:InputDirectory - @get:PathSensitive(PathSensitivity.RELATIVE) - abstract val cacheDir: DirectoryProperty - - @get:OutputFile abstract val outputFile: RegularFileProperty - - init { - group = "flatpak" - description = "Generates a complete flatpak-sources.json manifest from the local Gradle cache directory." - outputs.upToDateWhen { false } - } - - private data class SnapshotVersion(val extension: String, val classifier: String?, val value: String) - - private data class SnapshotMetadata(val snapshotVersions: List, val fallbackValue: String?) - - private data class FlatpakSourceCandidate( - val file: File, - val group: String, - val name: String, - val version: String, - val ext: String, - val dest: String, - val destFilename: String, - val primaryUrl: String, - val mirrorUrls: List, - ) - - private val remoteMetadataCache = mutableMapOf() - - @TaskAction - fun generate() { - val cacheFolder = cacheDir.get().asFile - - val outputSourcesFile = outputFile.get().asFile - val snapshotBase = SNAPSHOT_REPO_URL - logger.lifecycle("Scanning Gradle cache directory: ${cacheFolder.absolutePath}") - - val allowedExtensions = setOf("jar", "aar", "pom", "module") - - // Scan the cache using a clean functional sequence pipeline - val candidates = - cacheFolder - .walkTopDown() - .filter { it.isFile } - .filter { it.extension.lowercase() in allowedExtensions } - .filterNot { it.name.endsWith("-sources.jar") || it.name.endsWith("-javadoc.jar") } - .mapNotNull { file -> - val ext = file.extension.lowercase() - val filename = file.name - val relativePath = file.relativeTo(cacheFolder).path.replace('\\', '/') - val parts = relativePath.split('/') - - if (parts.size != 5) return@mapNotNull null - - val (group, name, version) = parts - val groupPath = group.replace('.', '/') - val standardPrefix = "$name-$version" - val isSnapshot = version.endsWith("-SNAPSHOT") - - val resolvedVersion = - if (isSnapshot) { - resolveSnapshotVersion(snapshotBase, groupPath, name, version, ext) ?: version - } else { - version - } - - val serverFilename = - when { - isSnapshot -> "$name-$resolvedVersion.$ext" - filename.startsWith(standardPrefix) -> filename - else -> "$name-$version.$ext" - } - - val mavenPath = "$groupPath/$name/$version/$serverFilename" - val dest = "offline-repository/$groupPath/$name/$version" - - val isJitpack = group.startsWith("com.github.") - val isGoogleArtifact = - group.startsWith("androidx.") || - group.startsWith("com.google.") || - group.startsWith("com.android.") - val isGradlePlugin = group.endsWith(".gradle.plugin") || group.startsWith("org.gradle.") - - val primaryUrl = - when { - isSnapshot -> "$snapshotBase/$mavenPath" - isJitpack -> "https://jitpack.io/$mavenPath" - isGoogleArtifact -> "https://dl.google.com/dl/android/maven2/$mavenPath" - isGradlePlugin -> "https://plugins.gradle.org/m2/$mavenPath" - else -> "https://repo.maven.apache.org/maven2/$mavenPath" - } - - val mavenCentralMirrors = - listOf( - "https://repo1.maven.org/maven2/$mavenPath", - "https://maven-central.storage-download.googleapis.com/maven2/$mavenPath", - "https://maven.aliyun.com/repository/public/$mavenPath", - ) - - val mirrorUrls = - when { - isSnapshot -> - listOf("https://s01.oss.sonatype.org/content/repositories/snapshots/$mavenPath") - - isJitpack -> - buildList { - // Many com.github.* artifacts migrated to Maven Central - add("https://repo.maven.apache.org/maven2/$mavenPath") - addAll(mavenCentralMirrors) - } - - else -> - buildList { - add("https://repo.maven.apache.org/maven2/$mavenPath") - addAll(mavenCentralMirrors) - } - } - - FlatpakSourceCandidate( - file = file, - group = group, - name = name, - version = version, - ext = ext, - dest = dest, - destFilename = filename, - primaryUrl = primaryUrl, - mirrorUrls = mirrorUrls, - ) - } - .toList() - - // Deduplicate and sort by unique destination path + file - val deduplicated = - candidates - .groupBy { "${it.dest}/${it.destFilename}" } - .map { (_, groupCandidates) -> groupCandidates.first() } - .sortedBy { "${it.dest}/${it.destFilename}" } - - logger.lifecycle("Calculating checksums for ${deduplicated.size} unique sources...") - - val finalEntries = - deduplicated.map { candidate -> - val entry = - mutableMapOf( - "type" to "file", - "url" to candidate.primaryUrl, - "sha256" to calculateSha256(candidate.file), - "dest" to candidate.dest, - "dest-filename" to candidate.destFilename, - ) - if (candidate.mirrorUrls.isNotEmpty()) { - entry["mirror-urls"] = candidate.mirrorUrls - } - entry - } - - outputSourcesFile.writeText(JsonOutput.prettyPrint(JsonOutput.toJson(finalEntries))) - logger.lifecycle( - "Successfully scanned cache and generated ${outputSourcesFile.name} containing ${finalEntries.size} entries.", - ) - } - - /** - * Resolves the timestamped snapshot version by fetching maven-metadata.xml from the remote snapshot repository. - * Maven snapshot repos do not serve artifacts at the generic `-SNAPSHOT` filename — they require the unique - * timestamped coordinate (e.g. `0.2.4-20260520.043744-2`). - */ - private fun resolveSnapshotVersion( - snapshotBase: String, - groupPath: String, - artifactId: String, - version: String, - extension: String, - ): String? { - val cacheKey = "$groupPath:$artifactId:$version" - if (cacheKey in remoteMetadataCache) { - return findMatchingVersion(remoteMetadataCache[cacheKey], extension) - } - - val metadataUrl = "$snapshotBase/$groupPath/$artifactId/$version/maven-metadata.xml" - - val metadata = fetchAndParseMetadata(metadataUrl) - remoteMetadataCache[cacheKey] = metadata - return findMatchingVersion(metadata, extension) - } - - private fun findMatchingVersion(metadata: SnapshotMetadata?, extension: String): String? { - if (metadata == null) return null - return metadata.snapshotVersions.firstOrNull { it.extension == extension && it.classifier == null }?.value - ?: metadata.fallbackValue - } - - private fun fetchAndParseMetadata(url: String): SnapshotMetadata? { - try { - logger.info("Fetching snapshot metadata: $url") - val connection = URI(url).toURL().openConnection() - connection.connectTimeout = TIMEOUT_MS - connection.readTimeout = TIMEOUT_MS - - val doc = - connection.getInputStream().use { stream -> - val dbFactory = DocumentBuilderFactory.newInstance() - val dBuilder = dbFactory.newDocumentBuilder() - dBuilder.parse(stream) - } - doc.documentElement.normalize() - - val root = doc.documentElement - val snapshotVersions = mutableListOf() - root.getElementsByTagName("snapshotVersion").forEachElement { element -> - val ext = element.getChildText("extension") ?: return@forEachElement - val value = element.getChildText("value") ?: return@forEachElement - val classif = element.getChildText("classifier") - snapshotVersions.add(SnapshotVersion(ext, classif, value)) - } - - var fallbackValue: String? = null - val snapshotNode = root.getElementsByTagName("snapshot").item(0) as? Element - if (snapshotNode != null) { - val timestamp = snapshotNode.getChildText("timestamp") - val buildNumber = snapshotNode.getChildText("buildNumber") - val metaVersion = root.getChildText("version") - if (timestamp != null && buildNumber != null && metaVersion != null) { - val baseVersion = metaVersion.substringBefore("-SNAPSHOT") - fallbackValue = "$baseVersion-$timestamp-$buildNumber" - } - } - - return SnapshotMetadata(snapshotVersions, fallbackValue) - } catch (e: Exception) { - logger.warn("Failed to fetch snapshot metadata from $url: ${e.message}") - return null - } - } - - private fun calculateSha256(file: File): String { - val digest = MessageDigest.getInstance("SHA-256") - file.inputStream().use { inputStream -> - val buffer = ByteArray(8192) - var bytesRead = inputStream.read(buffer) - while (bytesRead != -1) { - digest.update(buffer, 0, bytesRead) - bytesRead = inputStream.read(buffer) - } - } - val bytes = digest.digest() - val hexDigits = "0123456789abcdef" - val hexChars = CharArray(bytes.size * 2) - for (i in bytes.indices) { - val v = bytes[i].toInt() and 0xFF - hexChars[i * 2] = hexDigits[v ushr 4] - hexChars[i * 2 + 1] = hexDigits[v and 0x0F] - } - return String(hexChars) - } - - private fun Element.getChildText(tagName: String): String? = getElementsByTagName(tagName).item(0)?.textContent - - private fun NodeList.forEachElement(action: (Element) -> Unit) { - for (i in 0 until length) { - val node = item(i) - if (node is Element) { - action(node) - } - } - } - - private companion object { - private const val TIMEOUT_MS = 10_000 - private const val SNAPSHOT_REPO_URL = "https://central.sonatype.com/repository/maven-snapshots" - } -} diff --git a/build-logic/settings.gradle.kts b/build-logic/settings.gradle.kts index b46a2a246..1fcefcf60 100644 --- a/build-logic/settings.gradle.kts +++ b/build-logic/settings.gradle.kts @@ -60,5 +60,5 @@ apply(from = "../gradle/develocity.settings.gradle") rootProject.name = "build-logic" include(":convention") -include(":flatpak") +include(":flatpak-ops") diff --git a/build.gradle.kts b/build.gradle.kts index 2ce02f254..f101ff4b8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -40,7 +40,7 @@ plugins { alias(libs.plugins.test.retry) apply false alias(libs.plugins.meshtastic.root) id("meshtastic.docs") - id("meshtastic.flatpak") + id("meshtastic.flatpak-ops") } dependencies { diff --git a/scripts/verify-flatpak/README.md b/scripts/verify-flatpak/README.md new file mode 100644 index 000000000..46b593e5a --- /dev/null +++ b/scripts/verify-flatpak/README.md @@ -0,0 +1,60 @@ +# Local Flatpak Verification + +Replicates vid's `org.meshtastic.desktop` GHA flatpak build on your machine so you +can validate `flatpak-sources.json` end-to-end without round-tripping through his repo. + +## What it tests that our CI doesn't + +Our CI (`:desktopApp:assemble :captureFlatpakSources`) only proves the manifest can be *generated*. +Vid's CI is where the manifest actually gets *consumed* by `flatpak-builder`. This script +runs that step locally: + +1. Clones `vidplace7/org.meshtastic.desktop`. +2. Overlays a patched manifest that: + - Swaps the `meshtastic/Meshtastic-Android.git` source for a `type: dir` pointing at + **your local checkout** (so you can test uncommitted changes). + - Uncomments `- flatpak-sources.json`. + - Drops `--share=network` from the build-args (true offline — what Flathub requires). + - Adds `--offline` to the Gradle invocation (belt + suspenders). +3. Runs `flatpak-builder` in a Docker container with the same Freedesktop 25.08 SDK + vid's GHA image uses. + +If your `flatpak-sources.json` has the wrong URL, wrong sha256, or a missing entry, +the build fails with the same error vid would see. You can iterate in ~5–15 min loops +instead of waiting on cross-repo CI. + +## Prerequisites + +- Docker (Docker Desktop on macOS works — the container needs `--privileged` to use + bubblewrap; that's enabled by default). +- ~10 GB free disk for the SDK + Gradle cache. +- A populated Gradle cache (`./gradlew :desktopApp:assemble` must have run; the script + does this implicitly via `:captureFlatpakSources`). + +## Usage + +```bash +# Full offline build (~10–20 min the first time, faster after — Docker image is cached) +scripts/verify-flatpak/verify.sh + +# Cross-arch test via QEMU emulation (slower) +scripts/verify-flatpak/verify.sh --arch aarch64 + +# Drop into the builder container shell to poke at things +scripts/verify-flatpak/verify.sh --shell +``` + +## Interpreting failures + +| Symptom | Likely cause | +| ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| `Error downloading mirror: ... 404` on `repo.maven.apache.org` | URL captured by the listener was wrong or artifact moved. Check the source repo hosting it. | +| `sha256 mismatch` | Stale `flatpak-sources.json`; re-run `:desktopApp:assemble :captureFlatpakSources`. | +| Gradle: `Could not resolve all artifacts ... offline mode` | Missing dep in the manifest — usually a compiler plugin or BOM that wasn't downloaded during capture. | + +## Files + +- `verify.sh` — entry point. Idempotent: re-running just re-syncs the overlay and re-runs flatpak-builder. +- `desktop-offline.yaml` — patched manifest. Kept in sync manually with vid's upstream; + diff against `https://raw.githubusercontent.com/vidplace7/org.meshtastic.desktop/main/org.meshtastic.desktop.yaml` + if vid changes something material. diff --git a/scripts/verify-flatpak/desktop-offline.yaml b/scripts/verify-flatpak/desktop-offline.yaml new file mode 100644 index 000000000..5511f201d --- /dev/null +++ b/scripts/verify-flatpak/desktop-offline.yaml @@ -0,0 +1,97 @@ +# Local offline verification of org.meshtastic.desktop. +# Differences from vid's upstream manifest (vidplace7/org.meshtastic.desktop): +# 1. Meshtastic-Android source: `type: dir` pointing at our local checkout instead of `type: git`. +# 2. flatpak-sources.json is INCLUDED (uncommented) so Gradle resolves from the offline-repository. +# 3. Build uses `--offline` for Gradle — true Flathub-style offline compilation. +# (finish-args still grants runtime network since the app needs it; only the build is offline.) +# Generated by scripts/verify-flatpak/verify.sh; do not edit upstream from here. + +id: org.meshtastic.desktop +runtime: org.freedesktop.Platform +runtime-version: '25.08' +sdk: org.freedesktop.Sdk +sdk-extensions: + - org.freedesktop.Sdk.Extension.openjdk17 + - org.freedesktop.Sdk.Extension.openjdk21 +command: meshtastic-wrapper.sh + +finish-args: + - --share=ipc + - --socket=x11 + - --device=dri + - --device=all + - --share=network + - --talk-name=org.kde.StatusNotifierWatcher + - --talk-name=org.freedesktop.Notifications + - --allow=bluetooth + - --system-talk-name=org.bluez.* + - --env=PATH=/app/jre/bin:/app/bin:/usr/bin + - --env=JAVA_HOME=/app/jre + +modules: + - name: jbr + buildsystem: simple + build-commands: + - mkdir -p /app/jre + - tar xzf jbr-*.tar.gz -C /app/jre --strip-components=1 + sources: + - type: file + url: https://cache-redirector.jetbrains.com/intellij-jbr/jbr-21.0.10-linux-x64-b1163.105.tar.gz + sha256: b6a3b13451d296140727bbde9325dd9ee422e2e9b2c6a9378f346f1b8d1111bc + only-arches: + - x86_64 + - type: file + url: https://cache-redirector.jetbrains.com/intellij-jbr/jbr-21.0.10-linux-aarch64-b1163.105.tar.gz + sha256: 38804f526d869f5a9c49e9b90c04edcbc918b41e8e43264e5d5076d352a959bc + only-arches: + - aarch64 + + - shared-modules/libappindicator/libappindicator-gtk3-12.10.json + + - name: meshtastic-wrapper + buildsystem: simple + build-commands: + - install -Dm755 meshtastic-wrapper.sh /app/bin/meshtastic-wrapper.sh + sources: + - type: file + path: meshtastic-wrapper.sh + + - name: meshtastic-desktop + buildsystem: simple + build-options: + append-path: "/usr/lib/sdk/openjdk21/bin" + env: + JAVA_HOME: /usr/lib/sdk/openjdk21/jvm/openjdk-21 + # Point Gradle at the offline mirror produced by flatpak-sources.json + GRADLE_USER_HOME: /run/build/meshtastic-desktop/.gradle + build-commands: + - install -Dm644 -t /app/share/icons/hicolor/scalable/apps org.meshtastic.desktop.svg + - install -Dm644 -t /app/share/applications org.meshtastic.desktop.desktop + - install -Dm644 -t /app/share/metainfo org.meshtastic.desktop.metainfo.xml + - echo "org.gradle.java.installations.auto-detect=false" >> gradle.properties + - echo "org.gradle.java.installations.auto-download=false" >> gradle.properties + - echo "org.gradle.java.installations.paths=/usr/lib/sdk/openjdk21/jvm/openjdk-21,/usr/lib/sdk/openjdk17/jvm/openjdk-17" >> gradle.properties + - > + sed -i + 's/^\(\s*\)vendor\.set(JvmVendorSpec\.JETBRAINS)/\1\/\/ vendor.set(JvmVendorSpec.JETBRAINS)/' + desktop/build.gradle.kts + # Force Gradle to resolve from the bundled offline-repository ONLY (true offline test). + - ./gradlew --offline :desktop:packageUberJarForCurrentOS + - > + JAR_FILE=$(find desktop/build/compose/jars/ -name "*.jar" -type f | head -1) + && install -Dm755 "$JAR_FILE" /app/lib/meshtastic-desktop.jar + sources: + - type: file + path: org.meshtastic.desktop.desktop + - type: file + path: org.meshtastic.desktop.metainfo.xml + - type: file + path: org.meshtastic.desktop.svg + - type: dir + path: meshtastic-android + - type: file + url: https://services.gradle.org/distributions/gradle-9.4.1-bin.zip + sha256: 2ab2958f2a1e51120c326cad6f385153bb11ee93b3c216c5fccebfdfbb7ec6cb + dest: "gradle/wrapper" + dest-filename: "gradle-bin.zip" + - flatpak-sources.json diff --git a/scripts/verify-flatpak/verify.sh b/scripts/verify-flatpak/verify.sh new file mode 100755 index 000000000..30e888824 --- /dev/null +++ b/scripts/verify-flatpak/verify.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Local replica of vid's flatpak CI (vidplace7/org.meshtastic.desktop, .github/workflows/build-flatpak.yml) +# but flipped to true-offline mode: our flatpak-sources.json is included and --share=network is removed. +# +# Goal: validate flatpak-sources.json without bugging vid to push & re-run his workflow. +# +# Requirements: +# - Docker (Docker Desktop on macOS is fine; needs ~10GB free + ability to run --privileged) +# - This Meshtastic-Android checkout has produced flatpak-sources.json +# (run `./gradlew :desktopApp:assemble :captureFlatpakSources` first, or this script will do it) +# +# Usage: +# scripts/verify-flatpak/verify.sh # full build, x86_64 +# scripts/verify-flatpak/verify.sh --arch aarch64 # cross-arch via QEMU emulation +# scripts/verify-flatpak/verify.sh --shell # drop into the container shell instead of building + +set -euo pipefail + +ARCH="x86_64" +DROP_TO_SHELL=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --arch) ARCH="$2"; shift 2 ;; + --shell) DROP_TO_SHELL=1; shift ;; + -h|--help) sed -n '2,17p' "$0"; exit 0 ;; + *) echo "Unknown arg: $1" >&2; exit 2 ;; + esac +done + +# Map flatpak arch names to docker platform names +case "$ARCH" in + x86_64) DOCKER_PLATFORM="linux/amd64" ;; + aarch64) DOCKER_PLATFORM="linux/arm64" ;; + *) echo "Unsupported --arch: $ARCH (use x86_64 or aarch64)" >&2; exit 2 ;; +esac + +REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel)" +WORK="$REPO_ROOT/build/flatpak-verify" +OVERLAY="$REPO_ROOT/scripts/verify-flatpak/desktop-offline.yaml" +SOURCES_JSON="$REPO_ROOT/flatpak-sources.json" +VID_REPO="https://github.com/vidplace7/org.meshtastic.desktop.git" + +# Image provides flatpak + flatpak-builder. The freedesktop 25.08 runtime declared in +# the manifest is pulled from flathub at build time (no 25.08 image exists yet; 24.08 is +# fine as the builder host because the SDK used at compile time comes from flathub). +BUILDER_IMAGE="bilelmoussaoui/flatpak-github-actions:freedesktop-24.08" + +step() { printf '\n\033[1;34m==> %s\033[0m\n' "$*"; } +fail() { printf '\033[1;31m!! %s\033[0m\n' "$*" >&2; exit 1; } + +command -v docker >/dev/null 2>&1 || fail "docker is required; install Docker Desktop or equivalent." + +step "Ensuring flatpak-sources.json is fresh" +if [[ ! -f "$SOURCES_JSON" ]]; then + (cd "$REPO_ROOT" && ./gradlew --no-build-cache --no-configuration-cache :desktopApp:assemble :captureFlatpakSources) + cp "$REPO_ROOT/build/flatpak-ops-sources.json" "$SOURCES_JSON" +fi + +step "Preparing workspace at $WORK" +mkdir -p "$WORK" +if [[ ! -d "$WORK/org.meshtastic.desktop/.git" ]]; then + git clone --depth 1 --recurse-submodules "$VID_REPO" "$WORK/org.meshtastic.desktop" +else + git -C "$WORK/org.meshtastic.desktop" fetch --depth 1 origin main + git -C "$WORK/org.meshtastic.desktop" reset --hard origin/main + git -C "$WORK/org.meshtastic.desktop" submodule update --init --recursive --depth 1 +fi + +step "Wiring overlay manifest + our flatpak-sources.json" +cp "$OVERLAY" "$WORK/org.meshtastic.desktop/org.meshtastic.desktop.yaml" +cp "$SOURCES_JSON" "$WORK/org.meshtastic.desktop/flatpak-sources.json" + +# Materialize a clean copy of our checkout (excluding build outputs) for `type: dir`. +# flatpak-builder copies the whole tree — skip heavy/irrelevant paths. +step "Snapshotting Meshtastic-Android checkout (excluding build/, .gradle/)" +rsync -a --delete \ + --exclude='/build/' \ + --exclude='/.gradle/' \ + --exclude='*/build/' \ + --exclude='*/.gradle/' \ + --exclude='/.idea/' \ + --exclude='/local.properties' \ + "$REPO_ROOT/" "$WORK/org.meshtastic.desktop/meshtastic-android/" + +step "Pulling builder image: $BUILDER_IMAGE ($DOCKER_PLATFORM)" +docker pull --platform "$DOCKER_PLATFORM" "$BUILDER_IMAGE" >/dev/null + +if [[ $DROP_TO_SHELL -eq 1 ]]; then + step "Dropping into builder shell — flatpak-builder is on PATH" + exec docker run --rm -it --privileged \ + -v "$WORK/org.meshtastic.desktop:/work" \ + -w /work \ + --platform "$DOCKER_PLATFORM" \ + --security-opt seccomp=unconfined \ + "$BUILDER_IMAGE" bash +fi + +step "Running flatpak-builder (arch=$ARCH)" +docker run --rm --privileged \ + -v "$WORK/org.meshtastic.desktop:/work" \ + -w /work \ + --platform "$DOCKER_PLATFORM" \ + --security-opt seccomp=unconfined \ + "$BUILDER_IMAGE" \ + bash -c "set -e + flatpak remote-add --user --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo + # --download-only verifies every source URL + sha256 and exits before the bwrap + # sandbox phase. We do this because nested bwrap fails inside Docker Desktop on + # macOS (prctl(PR_SET_SECCOMP) EINVAL). For full sandbox build, run on Linux directly + # — or rely on vid's GHA CI which uses bare ubuntu-24.04 runners. + flatpak-builder --user --repo=repo --install-deps-from=flathub --force-clean \ + --disable-rofiles-fuse --download-only \ + builddir org.meshtastic.desktop.yaml + echo + echo '=== All sources downloaded and sha256-verified successfully ===' + "