fix(flatpak): modernize snapshot URL resolution in source generator (#5552)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
James Rich
2026-05-20 15:27:29 -07:00
committed by GitHub
parent 11437fb6f4
commit 6a53022a50
6 changed files with 201 additions and 144 deletions

View File

@@ -17,15 +17,14 @@ Previously, this logic was mixed in loose scripts or monolithic build convention
## Key Features
### 1. Snapshot Metadata Harvesting
Standard Maven snapshot repositories (e.g., Sonatype Snapshots) return `404` errors when fetching non-timestamped `-SNAPSHOT` dependencies directly. This plugin dynamically locates and parses local cached `maven-metadata.xml` files inside Gradle's cached directories, resolves the unique timestamped snapshot coordinate, and constructs exact, direct download URLs while preserving local filename bindings.
### 1. Remote Snapshot Metadata Resolution
Standard Maven snapshot repositories (e.g., Sonatype Snapshots) return `404` errors when fetching non-timestamped `-SNAPSHOT` dependencies directly. This plugin fetches `maven-metadata.xml` from the remote snapshot repository at generation time, resolves the unique timestamped snapshot coordinate (e.g., `0.2.4-20260520.043744-2`), and constructs exact, direct download URLs while preserving local filename bindings.
### 2. JitPack URL Routing
Automatically identifies external dependencies belonging to the `com.github.*` group (hosted on JitPack) and routes their `primaryUrl` to `https://jitpack.io` instead of attempting standard Maven Central lookup, preventing sandboxed download failures.
### 3. High-Performance Optimizations
* **Single-Pass Metadata Indexing**: Scans cached metadata files exactly once on-demand, caching them in an in-memory `$O(1)$` lookup map.
* **Deferred Cryptographic Hashing**: Defers expensive SHA-256 calculation until after candidate files are fully deduplicated and sorted.
### 3. Automatic Cache Population
The task automatically depends on `:desktopApp:assemble`, ensuring the Gradle dependency cache is fully populated before scanning. No manual pre-build step is required.
---
@@ -39,6 +38,23 @@ plugins {
}
```
### Configuration (DSL)
All options have sensible defaults. Override as needed:
```kotlin
flatpak {
// Snapshot repository to fetch maven-metadata.xml from
snapshotRepoUrl.set("https://central.sonatype.com/repository/maven-snapshots")
// Task that populates the Gradle cache before scanning
assembleTask.set(":desktopApp:assemble")
// Custom Gradle cache directory (defaults to ~/.gradle/caches/modules-2/files-2.1)
cacheDir.set(layout.projectDirectory.dir("my-cache"))
// Output manifest path
outputFile.set(layout.projectDirectory.file("flatpak-sources.json"))
}
```
### Running the Generator Task
Execute the registered custom task to sweep your Gradle local modules cache and generate/overwrite the root `flatpak-sources.json`:
@@ -47,17 +63,10 @@ Execute the registered custom task to sweep your Gradle local modules cache and
./gradlew :generateFlatpakSourcesFromCache
```
### Custom Cache Directory
By default, the task scans the standard Gradle user home caches directory (`~/.gradle/caches/modules-2/files-2.1`). You can supply a custom cache directory using the `flatpak.cache.dir` Gradle property:
```bash
./gradlew :generateFlatpakSourcesFromCache -Pflatpak.cache.dir="/custom/cache/path"
```
---
## Architecture
* **[FlatpakConventionPlugin.kt](src/main/kotlin/FlatpakConventionPlugin.kt)**: Registers the `generateFlatpakSourcesFromCache` task using lazy provider configuration.
* **[GenerateFlatpakSourcesTask.kt](src/main/kotlin/org/meshtastic/flatpak/GenerateFlatpakSourcesTask.kt)**: The native JVM-based custom task responsible for Gradle files scanning, metadata harvesting, and JSON generation.
* **[FlatpakPlugin.kt](src/main/kotlin/org/meshtastic/flatpak/FlatpakPlugin.kt)**: Registers the `flatpak {}` DSL extension and the `generateFlatpakSourcesFromCache` task using lazy provider configuration.
* **[FlatpakExtension.kt](src/main/kotlin/org/meshtastic/flatpak/FlatpakExtension.kt)**: DSL extension interface defining all configurable properties.
* **[GenerateFlatpakSourcesTask.kt](src/main/kotlin/org/meshtastic/flatpak/GenerateFlatpakSourcesTask.kt)**: The custom task responsible for Gradle files scanning, remote metadata resolution, and JSON generation.

View File

@@ -77,7 +77,7 @@ gradlePlugin {
plugins {
register("meshtasticFlatpak") {
id = "meshtastic.flatpak"
implementationClass = "FlatpakConventionPlugin"
implementationClass = "org.meshtastic.flatpak.FlatpakPlugin"
}
}
}

View File

@@ -8,10 +8,7 @@
<ID>MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$4</ID>
<ID>MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$5</ID>
<ID>MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$8192</ID>
<ID>MaxLineLength:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$"Gradle cache directory does not exist or is not configured correctly. Please run a build first to populate the cache."</ID>
<ID>MaxLineLength:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$"Successfully scanned cache and generated ${outputSourcesFile.name} containing ${finalEntries.size} entries."</ID>
<ID>NestedBlockDepth:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$private fun populateMetadataCache</ID>
<ID>ReturnCount:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$private fun findSnapshotValue: String?</ID>
<ID>SwallowedException:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$e: Exception</ID>
<ID>TooGenericExceptionCaught:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$e: Exception</ID>
</CurrentIssues>

View File

@@ -0,0 +1,46 @@
/*
* Copyright (c) 2026 Meshtastic LLC
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.meshtastic.flatpak
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.Property
/**
* DSL extension for configuring the Flatpak source manifest generator.
*
* ```kotlin
* flatpak {
* snapshotRepoUrl.set("https://central.sonatype.com/repository/maven-snapshots")
* assembleTask.set(":desktopApp:assemble")
* }
* ```
*/
abstract class FlatpakExtension {
/** Gradle cache directory to scan for dependency artifacts. */
abstract val cacheDir: DirectoryProperty
/** Output path for the generated flatpak-sources.json manifest. */
abstract val outputFile: RegularFileProperty
/** Base URL of the Maven snapshot repository (no trailing slash). */
abstract val snapshotRepoUrl: Property<String>
/** Task path to depend on, ensuring the cache is fully populated before scanning. */
abstract val assembleTask: Property<String>
}

View File

@@ -14,25 +14,29 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.meshtastic.flatpak
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.meshtastic.flatpak.GenerateFlatpakSourcesTask
import java.io.File
class FlatpakConventionPlugin : Plugin<Project> {
class FlatpakPlugin : Plugin<Project> {
override fun apply(target: Project) {
with(target) {
val extension = extensions.create("flatpak", FlatpakExtension::class.java).apply {
cacheDir.convention(
layout.dir(providers.provider { File(gradle.gradleUserHomeDir, "caches/modules-2/files-2.1") }),
)
outputFile.convention(layout.projectDirectory.file("flatpak-sources.json"))
snapshotRepoUrl.convention("https://central.sonatype.com/repository/maven-snapshots")
assembleTask.convention(":desktopApp:assemble")
}
tasks.register("generateFlatpakSourcesFromCache", GenerateFlatpakSourcesTask::class.java) {
val customCachePath = providers.gradleProperty("flatpak.cache.dir").orNull
if (customCachePath != null) {
cacheDir.set(layout.projectDirectory.dir(customCachePath))
} else {
cacheDir.set(
layout.dir(providers.provider { File(gradle.gradleUserHomeDir, "caches/modules-2/files-2.1") }),
)
}
outputFile.set(layout.projectDirectory.file("flatpak-sources.json"))
cacheDir.set(extension.cacheDir)
outputFile.set(extension.outputFile)
snapshotRepoUrl.set(extension.snapshotRepoUrl)
dependsOn(extension.assembleTask)
}
}
}

View File

@@ -18,29 +18,40 @@ package org.meshtastic.flatpak
import groovy.json.JsonOutput
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.tasks.Internal
import org.gradle.api.provider.Property
import org.gradle.work.DisableCachingByDefault
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputDirectory
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.PathSensitive
import org.gradle.api.tasks.PathSensitivity
import org.gradle.api.tasks.TaskAction
import org.w3c.dom.Element
import org.w3c.dom.NodeList
import java.io.File
import java.net.URI
import java.security.MessageDigest
import javax.xml.parsers.DocumentBuilderFactory
/** Generates a complete flatpak-sources.json manifest from the local Gradle cache directory. */
@DisableCachingByDefault(because = "Resolves remote snapshot metadata that may change between runs")
abstract class GenerateFlatpakSourcesTask : DefaultTask() {
@get:Internal abstract val cacheDir: DirectoryProperty
@get:InputDirectory
@get:PathSensitive(PathSensitivity.RELATIVE)
abstract val cacheDir: DirectoryProperty
@get:OutputFile abstract val outputFile: RegularFileProperty
/** Base URL of the Maven snapshot repository (no trailing slash). */
@get:Input
abstract val snapshotRepoUrl: Property<String>
init {
group = "flatpak"
description = "Generates a complete flatpak-sources.json manifest from the local Gradle cache directory."
// Ensure the task always runs when executed
outputs.upToDateWhen { false }
}
@@ -60,17 +71,14 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() {
val mirrorUrls: List<String>,
)
private var metadataCache: Map<String, SnapshotMetadata>? = null
private val remoteMetadataCache = mutableMapOf<String, SnapshotMetadata?>()
@TaskAction
fun generate() {
val cacheFolder =
cacheDir.orNull?.asFile
?: throw GradleException(
"Gradle cache directory does not exist or is not configured correctly. Please run a build first to populate the cache.",
)
val cacheFolder = cacheDir.get().asFile
val outputSourcesFile = outputFile.get().asFile
val snapshotBase = snapshotRepoUrl.get()
logger.lifecycle("Scanning Gradle cache directory: ${cacheFolder.absolutePath}")
val allowedExtensions = setOf("jar", "aar", "pom", "module")
@@ -93,33 +101,19 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() {
val (group, name, version) = parts
val groupPath = group.replace('.', '/')
val standardPrefix = "$name-$version"
val isSnapshot = version.endsWith("-SNAPSHOT") || version.contains("-SNAPSHOT")
val classifier =
if (isSnapshot) {
val prefix = "$name-$version-"
filename.takeIf { it.startsWith(prefix) }?.removePrefix(prefix)?.removeSuffix(".$ext")
} else {
null
}
val isSnapshot = version.endsWith("-SNAPSHOT")
val resolvedVersion =
if (isSnapshot) {
val resourcesFolder = File(cacheFolder.parentFile, "resources-2.1")
findSnapshotValue(resourcesFolder, group, name, ext, classifier) ?: version
resolveSnapshotVersion(snapshotBase, groupPath, name, version, ext) ?: version
} else {
version
}
val serverFilename =
when {
isSnapshot -> {
val suffix = classifier?.let { "-$it" } ?: ""
"$name-$resolvedVersion$suffix.$ext"
}
isSnapshot -> "$name-$resolvedVersion.$ext"
filename.startsWith(standardPrefix) -> filename
else -> "$name-$version.$ext"
}
@@ -128,31 +122,30 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() {
val isJitpack = group.startsWith("com.github.")
val primaryUrl =
if (isSnapshot) {
"https://central.sonatype.com/repository/maven-snapshots/$mavenPath"
} else if (isJitpack) {
"https://jitpack.io/$mavenPath"
} else {
"https://repo.maven.apache.org/maven2/$mavenPath"
when {
isSnapshot ->
"$snapshotBase/$mavenPath"
isJitpack ->
"https://jitpack.io/$mavenPath"
else ->
"https://repo.maven.apache.org/maven2/$mavenPath"
}
val mirrorUrls =
when {
isSnapshot -> listOf("https://oss.sonatype.org/content/repositories/snapshots/$mavenPath")
isSnapshot -> listOf(
"https://s01.oss.sonatype.org/content/repositories/snapshots/$mavenPath",
)
isJitpack ->
listOf(
"https://repo.maven.apache.org/maven2/$mavenPath",
"https://maven-central.storage-download.googleapis.com/maven2/$mavenPath",
"https://maven.aliyun.com/repository/public/$mavenPath",
)
isJitpack -> emptyList()
else ->
listOf(
"https://dl.google.com/dl/android/maven2/$mavenPath",
"https://plugins.gradle.org/m2/$mavenPath",
"https://maven-central.storage-download.googleapis.com/maven2/$mavenPath",
"https://maven.aliyun.com/repository/public/$mavenPath",
)
}
@@ -181,14 +174,17 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() {
val finalEntries =
deduplicated.map { candidate ->
mapOf(
val entry = mutableMapOf<String, Any>(
"type" to "file",
"url" to candidate.primaryUrl,
"sha256" to calculateSha256(candidate.file),
"dest" to candidate.dest,
"dest-filename" to candidate.destFilename,
"mirror-urls" to candidate.mirrorUrls,
)
if (candidate.mirrorUrls.isNotEmpty()) {
entry["mirror-urls"] = candidate.mirrorUrls
}
entry
}
outputSourcesFile.writeText(JsonOutput.prettyPrint(JsonOutput.toJson(finalEntries)))
@@ -197,6 +193,80 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() {
)
}
/**
* Resolves the timestamped snapshot version by fetching maven-metadata.xml from the remote
* snapshot repository. Maven snapshot repos do not serve artifacts at the generic `-SNAPSHOT`
* filename — they require the unique timestamped coordinate (e.g. `0.2.4-20260520.043744-2`).
*/
private fun resolveSnapshotVersion(
snapshotBase: String,
groupPath: String,
artifactId: String,
version: String,
extension: String,
): String? {
val cacheKey = "$groupPath:$artifactId:$version"
if (cacheKey in remoteMetadataCache) {
return findMatchingVersion(remoteMetadataCache[cacheKey], extension)
}
val metadataUrl = "$snapshotBase/$groupPath/$artifactId/$version/maven-metadata.xml"
val metadata = fetchAndParseMetadata(metadataUrl)
remoteMetadataCache[cacheKey] = metadata
return findMatchingVersion(metadata, extension)
}
private fun findMatchingVersion(metadata: SnapshotMetadata?, extension: String): String? {
if (metadata == null) return null
return metadata.snapshotVersions
.firstOrNull { it.extension == extension && it.classifier == null }
?.value
?: metadata.fallbackValue
}
private fun fetchAndParseMetadata(url: String): SnapshotMetadata? {
try {
logger.info("Fetching snapshot metadata: $url")
val connection = URI(url).toURL().openConnection()
connection.connectTimeout = TIMEOUT_MS
connection.readTimeout = TIMEOUT_MS
val doc = connection.getInputStream().use { stream ->
val dbFactory = DocumentBuilderFactory.newInstance()
val dBuilder = dbFactory.newDocumentBuilder()
dBuilder.parse(stream)
}
doc.documentElement.normalize()
val root = doc.documentElement
val snapshotVersions = mutableListOf<SnapshotVersion>()
root.getElementsByTagName("snapshotVersion").forEachElement { element ->
val ext = element.getChildText("extension") ?: return@forEachElement
val value = element.getChildText("value") ?: return@forEachElement
val classif = element.getChildText("classifier")
snapshotVersions.add(SnapshotVersion(ext, classif, value))
}
var fallbackValue: String? = null
val snapshotNode = root.getElementsByTagName("snapshot").item(0) as? Element
if (snapshotNode != null) {
val timestamp = snapshotNode.getChildText("timestamp")
val buildNumber = snapshotNode.getChildText("buildNumber")
val metaVersion = root.getChildText("version")
if (timestamp != null && buildNumber != null && metaVersion != null) {
val baseVersion = metaVersion.substringBefore("-SNAPSHOT")
fallbackValue = "$baseVersion-$timestamp-$buildNumber"
}
}
return SnapshotMetadata(snapshotVersions, fallbackValue)
} catch (e: Exception) {
logger.warn("Failed to fetch snapshot metadata from $url: ${e.message}")
return null
}
}
private fun calculateSha256(file: File): String {
val digest = MessageDigest.getInstance("SHA-256")
file.inputStream().use { inputStream ->
@@ -218,10 +288,8 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() {
return String(hexChars)
}
// Modern helper extension to query DOM child element text safely
private fun Element.getChildText(tagName: String): String? = getElementsByTagName(tagName).item(0)?.textContent
// Modern helper extension to iterate DOM elements cleanly
private fun NodeList.forEachElement(action: (Element) -> Unit) {
for (i in 0 until length) {
val node = item(i)
@@ -231,74 +299,7 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() {
}
}
private fun populateMetadataCache(resourcesFolder: File) {
if (metadataCache != null || !resourcesFolder.exists()) return
val cache = mutableMapOf<String, SnapshotMetadata>()
resourcesFolder.walkTopDown().forEach { file ->
if (file.isFile && file.name == "maven-metadata.xml") {
try {
val dbFactory = DocumentBuilderFactory.newInstance()
val dBuilder = dbFactory.newDocumentBuilder()
val doc = dBuilder.parse(file)
doc.documentElement.normalize()
val root = doc.documentElement
val group = root.getChildText("groupId") ?: return@forEach
val name = root.getChildText("artifactId") ?: return@forEach
val key = "$group:$name"
val snapshotVersions = mutableListOf<SnapshotVersion>()
root.getElementsByTagName("snapshotVersion").forEachElement { element ->
val ext = element.getChildText("extension") ?: return@forEachElement
val value = element.getChildText("value") ?: return@forEachElement
val classif = element.getChildText("classifier")
snapshotVersions.add(SnapshotVersion(ext, classif, value))
}
var fallbackValue: String? = null
val snapshotNode = root.getElementsByTagName("snapshot").item(0) as? Element
if (snapshotNode != null) {
val timestamp = snapshotNode.getChildText("timestamp")
val buildNumber = snapshotNode.getChildText("buildNumber")
val version = root.getChildText("version")
if (timestamp != null && buildNumber != null && version != null) {
val baseVersion = version.substringBefore("-SNAPSHOT")
fallbackValue = "$baseVersion-$timestamp-$buildNumber"
}
}
cache[key] = SnapshotMetadata(snapshotVersions, fallbackValue)
} catch (e: Exception) {
// Ignore parsing errors for individual files
}
}
}
metadataCache = cache
}
private fun findSnapshotValue(
resourcesFolder: File,
group: String,
name: String,
extension: String,
classifier: String?,
): String? {
populateMetadataCache(resourcesFolder)
val metadata = metadataCache?.get("$group:$name") ?: return null
for (version in metadata.snapshotVersions) {
if (version.extension == extension) {
if (classifier == null && version.classifier == null) {
return version.value
}
if (classifier != null && classifier == version.classifier) {
return version.value
}
}
}
return metadata.fallbackValue
private companion object {
private const val TIMEOUT_MS = 10_000
}
}