refactor(flatpak): retire heuristic cache scanner, adopt build-ops capture (#5599)

Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
James Rich
2026-05-26 04:24:25 -07:00
committed by GitHub
parent d51552122f
commit 2fa11507c5
15 changed files with 521 additions and 496 deletions

View File

@@ -370,13 +370,21 @@ jobs:
gradle_encryption_key: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
cache_read_only: 'true'
# Uses an isolated Gradle user home so every artifact actually traverses the
# network — meshtastic.flatpak-ops captures URLs via BuildOperationListener and
# only sees ExternalResourceReadBuildOperation events for cache misses. With the
# shared cache, downloads would be skipped and the manifest would be (nearly) empty.
- name: Generate Flatpak Sources
run: >
./gradlew :desktopApp:assemble :generateFlatpakSourcesFromCache
--no-configuration-cache
./gradlew --no-build-cache --no-configuration-cache
-Dgradle.user.home=${{ runner.temp }}/flatpak-gradle-home
:desktopApp:assemble :captureFlatpakSources
- name: Stage manifest
run: cp build/flatpak-ops-sources.json flatpak-sources.json
- name: List Flatpak source files
run: ls -R flatpak-sources.json
run: ls -l flatpak-sources.json
- name: Upload Flatpak source artifacts
if: always()

View File

@@ -575,11 +575,16 @@ jobs:
gradle_encryption_key: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
cache_read_only: true
# Isolated Gradle user home — see explanation in release.yml.
- name: Generate Flatpak Sources
run: >
./gradlew :desktopApp:assemble :generateFlatpakSourcesFromCache
--no-configuration-cache --refresh-dependencies
./gradlew --no-build-cache --no-configuration-cache
-Dgradle.user.home=${{ runner.temp }}/flatpak-gradle-home
:desktopApp:assemble :captureFlatpakSources
- name: Stage manifest
run: cp build/flatpak-ops-sources.json flatpak-sources.json
- run: ls -lah flatpak-sources.json
- name: Upload Flatpak Sources

1
.gitignore vendored
View File

@@ -84,3 +84,4 @@ docs/_config_local.yml
flatpak-sources-*.json
flatpak-sources.json
offline-repository/
.claude/

View File

@@ -23,7 +23,7 @@ plugins {
alias(libs.plugins.detekt)
}
group = "org.meshtastic.flatpak"
group = "org.meshtastic.flatpakops"
java {
sourceCompatibility = JavaVersion.VERSION_21
@@ -33,7 +33,6 @@ java {
kotlin { compilerOptions { jvmTarget = JvmTarget.JVM_21 } }
dependencies {
// Allows type-safe accessors for libs in plugin build script
implementation(files(libs.javaClass.superclass.protectionDomain.codeSource.location))
detektPlugins(libs.detekt.formatting)
}
@@ -69,15 +68,14 @@ detekt {
config.setFrom(rootProject.file("../config/detekt/detekt.yml"))
buildUponDefaultConfig = true
allRules = false
baseline = file("detekt-baseline.xml")
source.setFrom(files("src/main/java", "src/main/kotlin"))
}
gradlePlugin {
plugins {
register("meshtasticFlatpak") {
id = "meshtastic.flatpak"
implementationClass = "org.meshtastic.flatpak.FlatpakPlugin"
register("meshtasticFlatpakOps") {
id = "meshtastic.flatpak-ops"
implementationClass = "org.meshtastic.flatpakops.FlatpakOpsPlugin"
}
}
}

View File

@@ -0,0 +1,222 @@
/*
* Copyright (c) 2026 Meshtastic LLC
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.meshtastic.flatpakops
import groovy.json.JsonOutput
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.internal.project.ProjectInternal
import org.gradle.internal.operations.BuildOperationDescriptor
import org.gradle.internal.operations.BuildOperationListener
import org.gradle.internal.operations.BuildOperationListenerManager
import org.gradle.internal.operations.OperationFinishEvent
import org.gradle.internal.operations.OperationIdentifier
import org.gradle.internal.operations.OperationProgressEvent
import org.gradle.internal.operations.OperationStartEvent
import org.gradle.internal.resource.ExternalResourceReadBuildOperationType
import java.io.File
import java.net.URI
import java.security.MessageDigest
import java.util.concurrent.ConcurrentHashMap
/**
* Captures every external resource URL Gradle reads via the internal BuildOperationListener API and emits a
* Flathub-compliant flatpak-sources.json at build finish.
*
* No heuristics: URL is authoritative (taken straight from the build op), the on-disk file is found via Gradle's
* documented files-2.1 layout, and SHA-256 is computed from that exact file.
*
* Internal APIs touched (acceptable trade-off; same path flatpak-gradle-generator uses):
* - org.gradle.internal.operations.BuildOperationListener / BuildOperationListenerManager
* - org.gradle.internal.resource.ExternalResourceReadBuildOperationType
* - org.gradle.api.internal.project.ProjectInternal (for .services)
*/
class FlatpakOpsPlugin : Plugin<Project> {
override fun apply(target: Project) {
check(target == target.rootProject) { "meshtastic.flatpak-ops must be applied to the root project" }
val capturedUrls: MutableSet<String> = ConcurrentHashMap.newKeySet()
val manager: BuildOperationListenerManager =
(target as ProjectInternal).services.get(BuildOperationListenerManager::class.java)
val listener = OpListener(capturedUrls)
manager.addListener(listener)
val outputProvider = target.layout.buildDirectory.file("flatpak-ops-sources.json")
target.tasks.register("captureFlatpakSources") {
group = "flatpak"
description = "Emit flatpak-sources.json from URLs captured via BuildOperationListener."
outputs.upToDateWhen { false }
// Must run AFTER the task that triggers resolution. Without this, Gradle's scheduler
// may interleave/parallelize this task with :desktopApp:assemble, causing us to write
// the file before the downloads we want to capture have happened.
mustRunAfter(":desktopApp:assemble")
val proj = target
val urlsRef = capturedUrls
val outFile = outputProvider
doLast { writeSources(proj, urlsRef.toList(), outFile.get().asFile) }
}
// Listener is intentionally NOT removed on task completion; it stays attached until JVM
// exit. Removal is unsafe when our task races against other resolution-emitting tasks.
// Known limitation: in a long-lived Gradle daemon, capturedUrls accumulates across builds.
// We can't clear it at task start (that would erase what was captured during assemble).
// The CI workflow uses a fresh isolated GRADLE_USER_HOME, so this only affects local
// developer use — and re-emitting a superset of URLs is harmless.
}
private class OpListener(private val urls: MutableSet<String>) : BuildOperationListener {
override fun started(op: BuildOperationDescriptor, e: OperationStartEvent) = Unit
override fun progress(id: OperationIdentifier, e: OperationProgressEvent) = Unit
override fun finished(op: BuildOperationDescriptor, e: OperationFinishEvent) {
val details = op.details as? ExternalResourceReadBuildOperationType.Details ?: return
if (e.failure != null) return
// No host/scheme filtering here: non-Maven URLs (distribution zips, repo listings, etc.)
// naturally drop out in writeSources() when locateCacheFile() can't find them under
// files-2.1. Keeping this listener permissive avoids hardcoding repo allowlists.
urls.add(details.location)
}
}
private fun writeSources(project: Project, urls: List<String>, output: File) {
val filesRoot = File(project.gradle.gradleUserHomeDir, "caches/modules-2/files-2.1")
val entries: List<Map<String, Any>> =
urls
.distinct()
.filterNot { url ->
// Exclude sources/javadoc jars: they're not needed for an offline build and
// inflate the manifest. Match on URL filename, not cache-relative path.
val tail = url.substringAfterLast('/')
tail.endsWith("-sources.jar") || tail.endsWith("-javadoc.jar")
}
.sorted()
.mapNotNull { url ->
val cacheFile = locateCacheFile(filesRoot, url)
if (cacheFile == null) {
project.logger.info("flatpak-ops: no cache file for {} (skipped)", url)
return@mapNotNull null
}
val rel = cacheFile.relativeTo(filesRoot).path.replace('\\', '/').split('/')
if (rel.size < CACHE_PATH_SEGMENTS) return@mapNotNull null
val group = rel[0]
val artifact = rel[1]
val version = rel[2]
val onDiskFilename = rel.last()
val groupPath = group.replace('.', '/')
val entry =
mutableMapOf<String, Any>(
"type" to "file",
"url" to url,
"sha256" to sha256(cacheFile),
"dest" to "offline-repository/$groupPath/$artifact/$version",
"dest-filename" to onDiskFilename,
)
mirrorsFor(url).takeIf { it.isNotEmpty() }?.let { entry["mirror-urls"] = it }
entry
}
output.parentFile?.mkdirs()
output.writeText(JsonOutput.prettyPrint(JsonOutput.toJson(entries)))
project.logger.lifecycle(
"flatpak-ops: captured {} URLs, emitted {} sources to {}",
urls.size,
entries.size,
output.absolutePath,
)
}
/**
* Last three URL path segments are artifact/version/filename (Maven 2 spec; present in every Maven URL). Gradle's
* files-2.1 layout is `<group-with-dots>/<artifact>/<version>/<content-sha1>/<filename>`. The (artifact, version,
* filename) triple is NOT unique across groups (e.g. androidx.annotation:annotation:1.10.0.module collides with
* org.jetbrains.compose.annotation-internal:annotation:1.10.0.module), so the group MUST be derived from the URL
* path. We probe every suffix of the path's leading segments against the cache layout — the longest match wins
* (strips arbitrary repo prefixes like `/maven2/`, `/dl/android/maven2/`, `/m2/` without hardcoding them).
*/
private fun locateCacheFile(filesRoot: File, url: String): File? {
val path = URI(url).path.trimEnd('/').split('/').filter { it.isNotEmpty() }
val tail = path.takeLast(URL_TRAILING_SEGMENTS)
if (!filesRoot.isDirectory || tail.size < URL_TRAILING_SEGMENTS) return null
val (artifact, version, filename) = tail
val groupCandidates = path.dropLast(URL_TRAILING_SEGMENTS)
// files-2.1 uses dot-joined group as a SINGLE directory, e.g. `androidx.annotation/`,
// not `androidx/annotation/`. So join URL segments with '.' here.
return groupCandidates.indices.firstNotNullOfOrNull { start ->
val groupDir = groupCandidates.drop(start).joinToString(".")
File(filesRoot, "$groupDir/$artifact/$version")
.takeIf(File::isDirectory)
?.listFiles { f -> f.isDirectory }
?.map { shaDir -> File(shaDir, filename) }
?.firstOrNull { it.isFile }
}
}
/**
* Derive fallback mirror URLs from the primary URL's host. Only Maven Central has well-known public mirrors; for
* everything else (Google, JitPack, Gradle plugin portal, snapshot repos), we trust the primary URL since these
* hosts don't have stable mirrors anyway. The URL itself is authoritative — we just rewrite the host while
* preserving the path.
*/
private fun mirrorsFor(url: String): List<String> {
val uri = runCatching { URI(url) }.getOrNull()
val host = uri?.host
val path = uri?.rawPath
return if (host != null && path != null && host in MAVEN_CENTRAL_HOSTS) {
MAVEN_CENTRAL_HOSTS.filter { it != host }.map { "https://$it$path" } +
"https://maven-central.storage-download.googleapis.com$path"
} else {
emptyList()
}
}
private fun sha256(file: File): String {
val md = MessageDigest.getInstance("SHA-256")
file.inputStream().use { stream ->
val buf = ByteArray(BUFFER_SIZE)
while (true) {
val n = stream.read(buf)
if (n <= 0) break
md.update(buf, 0, n)
}
}
return hex(md.digest())
}
private fun hex(bytes: ByteArray): String {
val digits = "0123456789abcdef"
val chars = CharArray(bytes.size * HEX_CHARS_PER_BYTE)
for (i in bytes.indices) {
val v = bytes[i].toInt() and BYTE_MASK
chars[i * HEX_CHARS_PER_BYTE] = digits[v ushr NIBBLE_BITS]
chars[i * HEX_CHARS_PER_BYTE + 1] = digits[v and NIBBLE_MASK]
}
return String(chars)
}
private companion object {
private val MAVEN_CENTRAL_HOSTS = listOf("repo.maven.apache.org", "repo1.maven.org")
private const val BUFFER_SIZE = 8192
private const val CACHE_PATH_SEGMENTS = 5
private const val URL_TRAILING_SEGMENTS = 3
private const val HEX_CHARS_PER_BYTE = 2
private const val BYTE_MASK = 0xFF
private const val NIBBLE_BITS = 4
private const val NIBBLE_MASK = 0x0F
}
}

View File

@@ -1,70 +0,0 @@
# Meshtastic Flatpak Source Manifest Generator
This directory contains the isolated, lightweight `:flatpak` subproject under `build-logic`. It registers and exposes the formal Gradle plugin `meshtastic.flatpak` (`FlatpakConventionPlugin`) to automate generating Flathub-compliant offline dependency manifests.
---
## Purpose
To build sandboxed desktop applications on Flathub completely offline (`--offline`), the Flatpak builder requires an exact, pre-calculated registry of all remote dependency locations and their cryptographic hashes (`flatpak-sources.json`).
Previously, this logic was mixed in loose scripts or monolithic build conventions. Isolating it into this standalone subproject provides:
* **Clean Boundaries**: Decouples packaging/publishing details from standard multiplatform compile configurations.
* **First-Class Configuration Caching**: Safe from eager state evaluations and non-serializable property capture.
* **Ease of Sharing/Publishing**: Simplifies future distribution or independent publication of the plugin.
---
## Key Features
### 1. Remote Snapshot Metadata Resolution
Standard Maven snapshot repositories (e.g., Sonatype Snapshots) return `404` errors when fetching non-timestamped `-SNAPSHOT` dependencies directly. This plugin fetches `maven-metadata.xml` from the remote snapshot repository at generation time, resolves the unique timestamped snapshot coordinate (e.g., `0.2.4-20260520.043744-2`), and constructs exact, direct download URLs while preserving local filename bindings.
### 2. JitPack URL Routing
Automatically identifies external dependencies belonging to the `com.github.*` group (hosted on JitPack) and routes their `primaryUrl` to `https://jitpack.io` instead of attempting standard Maven Central lookup, preventing sandboxed download failures.
### 3. Automatic Cache Population
The task automatically depends on `:desktopApp:assemble`, ensuring the Gradle dependency cache is fully populated before scanning. No manual pre-build step is required.
---
## Usage
Apply the plugin to the root project's `build.gradle.kts`:
```kotlin
plugins {
id("meshtastic.flatpak")
}
```
### Configuration (DSL)
All options have sensible defaults. Override as needed:
```kotlin
flatpak {
// Task that populates the Gradle cache before scanning
assembleTask.set(":desktopApp:assemble")
// Custom Gradle cache directory (defaults to ~/.gradle/caches/modules-2/files-2.1)
cacheDir.set(layout.projectDirectory.dir("my-cache"))
// Output manifest path
outputFile.set(layout.projectDirectory.file("flatpak-sources.json"))
}
```
### Running the Generator Task
Execute the registered custom task to sweep your Gradle local modules cache and generate/overwrite the root `flatpak-sources.json`:
```bash
./gradlew :generateFlatpakSourcesFromCache
```
---
## Architecture
* **[FlatpakPlugin.kt](src/main/kotlin/org/meshtastic/flatpak/FlatpakPlugin.kt)**: Registers the `flatpak {}` DSL extension and the `generateFlatpakSourcesFromCache` task using lazy provider configuration.
* **[FlatpakExtension.kt](src/main/kotlin/org/meshtastic/flatpak/FlatpakExtension.kt)**: DSL extension interface defining all configurable properties.
* **[GenerateFlatpakSourcesTask.kt](src/main/kotlin/org/meshtastic/flatpak/GenerateFlatpakSourcesTask.kt)**: The custom task responsible for Gradle files scanning, remote metadata resolution, and JSON generation.

View File

@@ -1,15 +0,0 @@
<?xml version="1.0" ?>
<SmellBaseline>
<ManuallySuppressedIssues/>
<CurrentIssues>
<ID>CyclomaticComplexMethod:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$@TaskAction fun generate</ID>
<ID>LongMethod:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$@TaskAction fun generate</ID>
<ID>MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$0xFF</ID>
<ID>MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$4</ID>
<ID>MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$5</ID>
<ID>MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$8192</ID>
<ID>MaxLineLength:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$"Successfully scanned cache and generated ${outputSourcesFile.name} containing ${finalEntries.size} entries."</ID>
<ID>SwallowedException:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$e: Exception</ID>
<ID>TooGenericExceptionCaught:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$e: Exception</ID>
</CurrentIssues>
</SmellBaseline>

View File

@@ -1,42 +0,0 @@
/*
* Copyright (c) 2026 Meshtastic LLC
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.meshtastic.flatpak
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.Property
/**
* DSL extension for configuring the Flatpak source manifest generator.
*
* ```kotlin
* flatpak {
* assembleTask.set(":desktopApp:assemble")
* }
* ```
*/
abstract class FlatpakExtension {
/** Gradle cache directory to scan for dependency artifacts. */
abstract val cacheDir: DirectoryProperty
/** Output path for the generated flatpak-sources.json manifest. */
abstract val outputFile: RegularFileProperty
/** Task path to depend on, ensuring the cache is fully populated before scanning. */
abstract val assembleTask: Property<String>
}

View File

@@ -1,42 +0,0 @@
/*
* Copyright (c) 2026 Meshtastic LLC
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.meshtastic.flatpak
import org.gradle.api.Plugin
import org.gradle.api.Project
import java.io.File
class FlatpakPlugin : Plugin<Project> {
override fun apply(target: Project) {
with(target) {
val extension =
extensions.create("flatpak", FlatpakExtension::class.java).apply {
cacheDir.convention(
layout.dir(providers.provider { File(gradle.gradleUserHomeDir, "caches/modules-2/files-2.1") }),
)
outputFile.convention(layout.projectDirectory.file("flatpak-sources.json"))
assembleTask.convention(":desktopApp:assemble")
}
tasks.register("generateFlatpakSourcesFromCache", GenerateFlatpakSourcesTask::class.java) {
cacheDir.set(extension.cacheDir)
outputFile.set(extension.outputFile)
dependsOn(extension.assembleTask)
}
}
}
}

View File

@@ -1,313 +0,0 @@
/*
* Copyright (c) 2026 Meshtastic LLC
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.meshtastic.flatpak
import groovy.json.JsonOutput
import org.gradle.api.DefaultTask
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.tasks.InputDirectory
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.PathSensitive
import org.gradle.api.tasks.PathSensitivity
import org.gradle.api.tasks.TaskAction
import org.gradle.work.DisableCachingByDefault
import org.w3c.dom.Element
import org.w3c.dom.NodeList
import java.io.File
import java.net.URI
import java.security.MessageDigest
import javax.xml.parsers.DocumentBuilderFactory
/** Generates a complete flatpak-sources.json manifest from the local Gradle cache directory. */
@DisableCachingByDefault(because = "Resolves remote snapshot metadata that may change between runs")
abstract class GenerateFlatpakSourcesTask : DefaultTask() {
@get:InputDirectory
@get:PathSensitive(PathSensitivity.RELATIVE)
abstract val cacheDir: DirectoryProperty
@get:OutputFile abstract val outputFile: RegularFileProperty
init {
group = "flatpak"
description = "Generates a complete flatpak-sources.json manifest from the local Gradle cache directory."
outputs.upToDateWhen { false }
}
private data class SnapshotVersion(val extension: String, val classifier: String?, val value: String)
private data class SnapshotMetadata(val snapshotVersions: List<SnapshotVersion>, val fallbackValue: String?)
private data class FlatpakSourceCandidate(
val file: File,
val group: String,
val name: String,
val version: String,
val ext: String,
val dest: String,
val destFilename: String,
val primaryUrl: String,
val mirrorUrls: List<String>,
)
private val remoteMetadataCache = mutableMapOf<String, SnapshotMetadata?>()
@TaskAction
fun generate() {
val cacheFolder = cacheDir.get().asFile
val outputSourcesFile = outputFile.get().asFile
val snapshotBase = SNAPSHOT_REPO_URL
logger.lifecycle("Scanning Gradle cache directory: ${cacheFolder.absolutePath}")
val allowedExtensions = setOf("jar", "aar", "pom", "module")
// Scan the cache using a clean functional sequence pipeline
val candidates =
cacheFolder
.walkTopDown()
.filter { it.isFile }
.filter { it.extension.lowercase() in allowedExtensions }
.filterNot { it.name.endsWith("-sources.jar") || it.name.endsWith("-javadoc.jar") }
.mapNotNull { file ->
val ext = file.extension.lowercase()
val filename = file.name
val relativePath = file.relativeTo(cacheFolder).path.replace('\\', '/')
val parts = relativePath.split('/')
if (parts.size != 5) return@mapNotNull null
val (group, name, version) = parts
val groupPath = group.replace('.', '/')
val standardPrefix = "$name-$version"
val isSnapshot = version.endsWith("-SNAPSHOT")
val resolvedVersion =
if (isSnapshot) {
resolveSnapshotVersion(snapshotBase, groupPath, name, version, ext) ?: version
} else {
version
}
val serverFilename =
when {
isSnapshot -> "$name-$resolvedVersion.$ext"
filename.startsWith(standardPrefix) -> filename
else -> "$name-$version.$ext"
}
val mavenPath = "$groupPath/$name/$version/$serverFilename"
val dest = "offline-repository/$groupPath/$name/$version"
val isJitpack = group.startsWith("com.github.")
val isGoogleArtifact =
group.startsWith("androidx.") ||
group.startsWith("com.google.") ||
group.startsWith("com.android.")
val isGradlePlugin = group.endsWith(".gradle.plugin") || group.startsWith("org.gradle.")
val primaryUrl =
when {
isSnapshot -> "$snapshotBase/$mavenPath"
isJitpack -> "https://jitpack.io/$mavenPath"
isGoogleArtifact -> "https://dl.google.com/dl/android/maven2/$mavenPath"
isGradlePlugin -> "https://plugins.gradle.org/m2/$mavenPath"
else -> "https://repo.maven.apache.org/maven2/$mavenPath"
}
val mavenCentralMirrors =
listOf(
"https://repo1.maven.org/maven2/$mavenPath",
"https://maven-central.storage-download.googleapis.com/maven2/$mavenPath",
"https://maven.aliyun.com/repository/public/$mavenPath",
)
val mirrorUrls =
when {
isSnapshot ->
listOf("https://s01.oss.sonatype.org/content/repositories/snapshots/$mavenPath")
isJitpack ->
buildList {
// Many com.github.* artifacts migrated to Maven Central
add("https://repo.maven.apache.org/maven2/$mavenPath")
addAll(mavenCentralMirrors)
}
else ->
buildList {
add("https://repo.maven.apache.org/maven2/$mavenPath")
addAll(mavenCentralMirrors)
}
}
FlatpakSourceCandidate(
file = file,
group = group,
name = name,
version = version,
ext = ext,
dest = dest,
destFilename = filename,
primaryUrl = primaryUrl,
mirrorUrls = mirrorUrls,
)
}
.toList()
// Deduplicate and sort by unique destination path + file
val deduplicated =
candidates
.groupBy { "${it.dest}/${it.destFilename}" }
.map { (_, groupCandidates) -> groupCandidates.first() }
.sortedBy { "${it.dest}/${it.destFilename}" }
logger.lifecycle("Calculating checksums for ${deduplicated.size} unique sources...")
val finalEntries =
deduplicated.map { candidate ->
val entry =
mutableMapOf<String, Any>(
"type" to "file",
"url" to candidate.primaryUrl,
"sha256" to calculateSha256(candidate.file),
"dest" to candidate.dest,
"dest-filename" to candidate.destFilename,
)
if (candidate.mirrorUrls.isNotEmpty()) {
entry["mirror-urls"] = candidate.mirrorUrls
}
entry
}
outputSourcesFile.writeText(JsonOutput.prettyPrint(JsonOutput.toJson(finalEntries)))
logger.lifecycle(
"Successfully scanned cache and generated ${outputSourcesFile.name} containing ${finalEntries.size} entries.",
)
}
/**
* Resolves the timestamped snapshot version by fetching maven-metadata.xml from the remote snapshot repository.
* Maven snapshot repos do not serve artifacts at the generic `-SNAPSHOT` filename — they require the unique
* timestamped coordinate (e.g. `0.2.4-20260520.043744-2`).
*/
private fun resolveSnapshotVersion(
snapshotBase: String,
groupPath: String,
artifactId: String,
version: String,
extension: String,
): String? {
val cacheKey = "$groupPath:$artifactId:$version"
if (cacheKey in remoteMetadataCache) {
return findMatchingVersion(remoteMetadataCache[cacheKey], extension)
}
val metadataUrl = "$snapshotBase/$groupPath/$artifactId/$version/maven-metadata.xml"
val metadata = fetchAndParseMetadata(metadataUrl)
remoteMetadataCache[cacheKey] = metadata
return findMatchingVersion(metadata, extension)
}
private fun findMatchingVersion(metadata: SnapshotMetadata?, extension: String): String? {
if (metadata == null) return null
return metadata.snapshotVersions.firstOrNull { it.extension == extension && it.classifier == null }?.value
?: metadata.fallbackValue
}
private fun fetchAndParseMetadata(url: String): SnapshotMetadata? {
try {
logger.info("Fetching snapshot metadata: $url")
val connection = URI(url).toURL().openConnection()
connection.connectTimeout = TIMEOUT_MS
connection.readTimeout = TIMEOUT_MS
val doc =
connection.getInputStream().use { stream ->
val dbFactory = DocumentBuilderFactory.newInstance()
val dBuilder = dbFactory.newDocumentBuilder()
dBuilder.parse(stream)
}
doc.documentElement.normalize()
val root = doc.documentElement
val snapshotVersions = mutableListOf<SnapshotVersion>()
root.getElementsByTagName("snapshotVersion").forEachElement { element ->
val ext = element.getChildText("extension") ?: return@forEachElement
val value = element.getChildText("value") ?: return@forEachElement
val classif = element.getChildText("classifier")
snapshotVersions.add(SnapshotVersion(ext, classif, value))
}
var fallbackValue: String? = null
val snapshotNode = root.getElementsByTagName("snapshot").item(0) as? Element
if (snapshotNode != null) {
val timestamp = snapshotNode.getChildText("timestamp")
val buildNumber = snapshotNode.getChildText("buildNumber")
val metaVersion = root.getChildText("version")
if (timestamp != null && buildNumber != null && metaVersion != null) {
val baseVersion = metaVersion.substringBefore("-SNAPSHOT")
fallbackValue = "$baseVersion-$timestamp-$buildNumber"
}
}
return SnapshotMetadata(snapshotVersions, fallbackValue)
} catch (e: Exception) {
logger.warn("Failed to fetch snapshot metadata from $url: ${e.message}")
return null
}
}
private fun calculateSha256(file: File): String {
val digest = MessageDigest.getInstance("SHA-256")
file.inputStream().use { inputStream ->
val buffer = ByteArray(8192)
var bytesRead = inputStream.read(buffer)
while (bytesRead != -1) {
digest.update(buffer, 0, bytesRead)
bytesRead = inputStream.read(buffer)
}
}
val bytes = digest.digest()
val hexDigits = "0123456789abcdef"
val hexChars = CharArray(bytes.size * 2)
for (i in bytes.indices) {
val v = bytes[i].toInt() and 0xFF
hexChars[i * 2] = hexDigits[v ushr 4]
hexChars[i * 2 + 1] = hexDigits[v and 0x0F]
}
return String(hexChars)
}
private fun Element.getChildText(tagName: String): String? = getElementsByTagName(tagName).item(0)?.textContent
private fun NodeList.forEachElement(action: (Element) -> Unit) {
for (i in 0 until length) {
val node = item(i)
if (node is Element) {
action(node)
}
}
}
private companion object {
private const val TIMEOUT_MS = 10_000
private const val SNAPSHOT_REPO_URL = "https://central.sonatype.com/repository/maven-snapshots"
}
}

View File

@@ -60,5 +60,5 @@ apply(from = "../gradle/develocity.settings.gradle")
rootProject.name = "build-logic"
include(":convention")
include(":flatpak")
include(":flatpak-ops")

View File

@@ -40,7 +40,7 @@ plugins {
alias(libs.plugins.test.retry) apply false
alias(libs.plugins.meshtastic.root)
id("meshtastic.docs")
id("meshtastic.flatpak")
id("meshtastic.flatpak-ops")
}
dependencies {

View File

@@ -0,0 +1,60 @@
# Local Flatpak Verification
Replicates vid's `org.meshtastic.desktop` GHA flatpak build on your machine so you
can validate `flatpak-sources.json` end-to-end without round-tripping through his repo.
## What it tests that our CI doesn't
Our CI (`:desktopApp:assemble :captureFlatpakSources`) only proves the manifest can be *generated*.
Vid's CI is where the manifest actually gets *consumed* by `flatpak-builder`. This script
runs that step locally:
1. Clones `vidplace7/org.meshtastic.desktop`.
2. Overlays a patched manifest that:
- Swaps the `meshtastic/Meshtastic-Android.git` source for a `type: dir` pointing at
**your local checkout** (so you can test uncommitted changes).
- Uncomments `- flatpak-sources.json`.
- Drops `--share=network` from the build-args (true offline — what Flathub requires).
- Adds `--offline` to the Gradle invocation (belt + suspenders).
3. Runs `flatpak-builder` in a Docker container with the same Freedesktop 25.08 SDK
vid's GHA image uses.
If your `flatpak-sources.json` has the wrong URL, wrong sha256, or a missing entry,
the build fails with the same error vid would see. You can iterate in ~515 min loops
instead of waiting on cross-repo CI.
## Prerequisites
- Docker (Docker Desktop on macOS works — the container needs `--privileged` to use
bubblewrap; that's enabled by default).
- ~10 GB free disk for the SDK + Gradle cache.
- A populated Gradle cache (`./gradlew :desktopApp:assemble` must have run; the script
does this implicitly via `:captureFlatpakSources`).
## Usage
```bash
# Full offline build (~1020 min the first time, faster after — Docker image is cached)
scripts/verify-flatpak/verify.sh
# Cross-arch test via QEMU emulation (slower)
scripts/verify-flatpak/verify.sh --arch aarch64
# Drop into the builder container shell to poke at things
scripts/verify-flatpak/verify.sh --shell
```
## Interpreting failures
| Symptom | Likely cause |
| ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
| `Error downloading mirror: ... 404` on `repo.maven.apache.org` | URL captured by the listener was wrong or artifact moved. Check the source repo hosting it. |
| `sha256 mismatch` | Stale `flatpak-sources.json`; re-run `:desktopApp:assemble :captureFlatpakSources`. |
| Gradle: `Could not resolve all artifacts ... offline mode` | Missing dep in the manifest — usually a compiler plugin or BOM that wasn't downloaded during capture. |
## Files
- `verify.sh` — entry point. Idempotent: re-running just re-syncs the overlay and re-runs flatpak-builder.
- `desktop-offline.yaml` — patched manifest. Kept in sync manually with vid's upstream;
diff against `https://raw.githubusercontent.com/vidplace7/org.meshtastic.desktop/main/org.meshtastic.desktop.yaml`
if vid changes something material.

View File

@@ -0,0 +1,97 @@
# Local offline verification of org.meshtastic.desktop.
# Differences from vid's upstream manifest (vidplace7/org.meshtastic.desktop):
# 1. Meshtastic-Android source: `type: dir` pointing at our local checkout instead of `type: git`.
# 2. flatpak-sources.json is INCLUDED (uncommented) so Gradle resolves from the offline-repository.
# 3. Build uses `--offline` for Gradle — true Flathub-style offline compilation.
# (finish-args still grants runtime network since the app needs it; only the build is offline.)
# Generated by scripts/verify-flatpak/verify.sh; do not edit upstream from here.
id: org.meshtastic.desktop
runtime: org.freedesktop.Platform
runtime-version: '25.08'
sdk: org.freedesktop.Sdk
sdk-extensions:
- org.freedesktop.Sdk.Extension.openjdk17
- org.freedesktop.Sdk.Extension.openjdk21
command: meshtastic-wrapper.sh
finish-args:
- --share=ipc
- --socket=x11
- --device=dri
- --device=all
- --share=network
- --talk-name=org.kde.StatusNotifierWatcher
- --talk-name=org.freedesktop.Notifications
- --allow=bluetooth
- --system-talk-name=org.bluez.*
- --env=PATH=/app/jre/bin:/app/bin:/usr/bin
- --env=JAVA_HOME=/app/jre
modules:
- name: jbr
buildsystem: simple
build-commands:
- mkdir -p /app/jre
- tar xzf jbr-*.tar.gz -C /app/jre --strip-components=1
sources:
- type: file
url: https://cache-redirector.jetbrains.com/intellij-jbr/jbr-21.0.10-linux-x64-b1163.105.tar.gz
sha256: b6a3b13451d296140727bbde9325dd9ee422e2e9b2c6a9378f346f1b8d1111bc
only-arches:
- x86_64
- type: file
url: https://cache-redirector.jetbrains.com/intellij-jbr/jbr-21.0.10-linux-aarch64-b1163.105.tar.gz
sha256: 38804f526d869f5a9c49e9b90c04edcbc918b41e8e43264e5d5076d352a959bc
only-arches:
- aarch64
- shared-modules/libappindicator/libappindicator-gtk3-12.10.json
- name: meshtastic-wrapper
buildsystem: simple
build-commands:
- install -Dm755 meshtastic-wrapper.sh /app/bin/meshtastic-wrapper.sh
sources:
- type: file
path: meshtastic-wrapper.sh
- name: meshtastic-desktop
buildsystem: simple
build-options:
append-path: "/usr/lib/sdk/openjdk21/bin"
env:
JAVA_HOME: /usr/lib/sdk/openjdk21/jvm/openjdk-21
# Point Gradle at the offline mirror produced by flatpak-sources.json
GRADLE_USER_HOME: /run/build/meshtastic-desktop/.gradle
build-commands:
- install -Dm644 -t /app/share/icons/hicolor/scalable/apps org.meshtastic.desktop.svg
- install -Dm644 -t /app/share/applications org.meshtastic.desktop.desktop
- install -Dm644 -t /app/share/metainfo org.meshtastic.desktop.metainfo.xml
- echo "org.gradle.java.installations.auto-detect=false" >> gradle.properties
- echo "org.gradle.java.installations.auto-download=false" >> gradle.properties
- echo "org.gradle.java.installations.paths=/usr/lib/sdk/openjdk21/jvm/openjdk-21,/usr/lib/sdk/openjdk17/jvm/openjdk-17" >> gradle.properties
- >
sed -i
's/^\(\s*\)vendor\.set(JvmVendorSpec\.JETBRAINS)/\1\/\/ vendor.set(JvmVendorSpec.JETBRAINS)/'
desktop/build.gradle.kts
# Force Gradle to resolve from the bundled offline-repository ONLY (true offline test).
- ./gradlew --offline :desktop:packageUberJarForCurrentOS
- >
JAR_FILE=$(find desktop/build/compose/jars/ -name "*.jar" -type f | head -1)
&& install -Dm755 "$JAR_FILE" /app/lib/meshtastic-desktop.jar
sources:
- type: file
path: org.meshtastic.desktop.desktop
- type: file
path: org.meshtastic.desktop.metainfo.xml
- type: file
path: org.meshtastic.desktop.svg
- type: dir
path: meshtastic-android
- type: file
url: https://services.gradle.org/distributions/gradle-9.4.1-bin.zip
sha256: 2ab2958f2a1e51120c326cad6f385153bb11ee93b3c216c5fccebfdfbb7ec6cb
dest: "gradle/wrapper"
dest-filename: "gradle-bin.zip"
- flatpak-sources.json

116
scripts/verify-flatpak/verify.sh Executable file
View File

@@ -0,0 +1,116 @@
#!/usr/bin/env bash
# Local replica of vid's flatpak CI (vidplace7/org.meshtastic.desktop, .github/workflows/build-flatpak.yml)
# but flipped to true-offline mode: our flatpak-sources.json is included and --share=network is removed.
#
# Goal: validate flatpak-sources.json without bugging vid to push & re-run his workflow.
#
# Requirements:
# - Docker (Docker Desktop on macOS is fine; needs ~10GB free + ability to run --privileged)
# - This Meshtastic-Android checkout has produced flatpak-sources.json
# (run `./gradlew :desktopApp:assemble :captureFlatpakSources` first, or this script will do it)
#
# Usage:
# scripts/verify-flatpak/verify.sh # full build, x86_64
# scripts/verify-flatpak/verify.sh --arch aarch64 # cross-arch via QEMU emulation
# scripts/verify-flatpak/verify.sh --shell # drop into the container shell instead of building
set -euo pipefail
ARCH="x86_64"
DROP_TO_SHELL=0
while [[ $# -gt 0 ]]; do
case "$1" in
--arch) ARCH="$2"; shift 2 ;;
--shell) DROP_TO_SHELL=1; shift ;;
-h|--help) sed -n '2,17p' "$0"; exit 0 ;;
*) echo "Unknown arg: $1" >&2; exit 2 ;;
esac
done
# Map flatpak arch names to docker platform names
case "$ARCH" in
x86_64) DOCKER_PLATFORM="linux/amd64" ;;
aarch64) DOCKER_PLATFORM="linux/arm64" ;;
*) echo "Unsupported --arch: $ARCH (use x86_64 or aarch64)" >&2; exit 2 ;;
esac
REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel)"
WORK="$REPO_ROOT/build/flatpak-verify"
OVERLAY="$REPO_ROOT/scripts/verify-flatpak/desktop-offline.yaml"
SOURCES_JSON="$REPO_ROOT/flatpak-sources.json"
VID_REPO="https://github.com/vidplace7/org.meshtastic.desktop.git"
# Image provides flatpak + flatpak-builder. The freedesktop 25.08 runtime declared in
# the manifest is pulled from flathub at build time (no 25.08 image exists yet; 24.08 is
# fine as the builder host because the SDK used at compile time comes from flathub).
BUILDER_IMAGE="bilelmoussaoui/flatpak-github-actions:freedesktop-24.08"
step() { printf '\n\033[1;34m==> %s\033[0m\n' "$*"; }
fail() { printf '\033[1;31m!! %s\033[0m\n' "$*" >&2; exit 1; }
command -v docker >/dev/null 2>&1 || fail "docker is required; install Docker Desktop or equivalent."
step "Ensuring flatpak-sources.json is fresh"
if [[ ! -f "$SOURCES_JSON" ]]; then
(cd "$REPO_ROOT" && ./gradlew --no-build-cache --no-configuration-cache :desktopApp:assemble :captureFlatpakSources)
cp "$REPO_ROOT/build/flatpak-ops-sources.json" "$SOURCES_JSON"
fi
step "Preparing workspace at $WORK"
mkdir -p "$WORK"
if [[ ! -d "$WORK/org.meshtastic.desktop/.git" ]]; then
git clone --depth 1 --recurse-submodules "$VID_REPO" "$WORK/org.meshtastic.desktop"
else
git -C "$WORK/org.meshtastic.desktop" fetch --depth 1 origin main
git -C "$WORK/org.meshtastic.desktop" reset --hard origin/main
git -C "$WORK/org.meshtastic.desktop" submodule update --init --recursive --depth 1
fi
step "Wiring overlay manifest + our flatpak-sources.json"
cp "$OVERLAY" "$WORK/org.meshtastic.desktop/org.meshtastic.desktop.yaml"
cp "$SOURCES_JSON" "$WORK/org.meshtastic.desktop/flatpak-sources.json"
# Materialize a clean copy of our checkout (excluding build outputs) for `type: dir`.
# flatpak-builder copies the whole tree — skip heavy/irrelevant paths.
step "Snapshotting Meshtastic-Android checkout (excluding build/, .gradle/)"
rsync -a --delete \
--exclude='/build/' \
--exclude='/.gradle/' \
--exclude='*/build/' \
--exclude='*/.gradle/' \
--exclude='/.idea/' \
--exclude='/local.properties' \
"$REPO_ROOT/" "$WORK/org.meshtastic.desktop/meshtastic-android/"
step "Pulling builder image: $BUILDER_IMAGE ($DOCKER_PLATFORM)"
docker pull --platform "$DOCKER_PLATFORM" "$BUILDER_IMAGE" >/dev/null
if [[ $DROP_TO_SHELL -eq 1 ]]; then
step "Dropping into builder shell — flatpak-builder is on PATH"
exec docker run --rm -it --privileged \
-v "$WORK/org.meshtastic.desktop:/work" \
-w /work \
--platform "$DOCKER_PLATFORM" \
--security-opt seccomp=unconfined \
"$BUILDER_IMAGE" bash
fi
step "Running flatpak-builder (arch=$ARCH)"
docker run --rm --privileged \
-v "$WORK/org.meshtastic.desktop:/work" \
-w /work \
--platform "$DOCKER_PLATFORM" \
--security-opt seccomp=unconfined \
"$BUILDER_IMAGE" \
bash -c "set -e
flatpak remote-add --user --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo
# --download-only verifies every source URL + sha256 and exits before the bwrap
# sandbox phase. We do this because nested bwrap fails inside Docker Desktop on
# macOS (prctl(PR_SET_SECCOMP) EINVAL). For full sandbox build, run on Linux directly
# — or rely on vid's GHA CI which uses bare ubuntu-24.04 runners.
flatpak-builder --user --repo=repo --install-deps-from=flathub --force-clean \
--disable-rofiles-fuse --download-only \
builddir org.meshtastic.desktop.yaml
echo
echo '=== All sources downloaded and sha256-verified successfully ==='
"