From 6a53022a50fb0c151b61c92bc2ddee4ade287d54 Mon Sep 17 00:00:00 2001 From: James Rich <2199651+jamesarich@users.noreply.github.com> Date: Wed, 20 May 2026 15:27:29 -0700 Subject: [PATCH] fix(flatpak): modernize snapshot URL resolution in source generator (#5552) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build-logic/flatpak/README.md | 39 +-- build-logic/flatpak/build.gradle.kts | 2 +- build-logic/flatpak/detekt-baseline.xml | 3 - .../meshtastic/flatpak/FlatpakExtension.kt | 46 ++++ .../meshtastic/flatpak/FlatpakPlugin.kt} | 26 +- .../flatpak/GenerateFlatpakSourcesTask.kt | 229 +++++++++--------- 6 files changed, 201 insertions(+), 144 deletions(-) create mode 100644 build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/FlatpakExtension.kt rename build-logic/flatpak/src/main/kotlin/{FlatpakConventionPlugin.kt => org/meshtastic/flatpak/FlatpakPlugin.kt} (54%) diff --git a/build-logic/flatpak/README.md b/build-logic/flatpak/README.md index d9596f07d..c680b7331 100644 --- a/build-logic/flatpak/README.md +++ b/build-logic/flatpak/README.md @@ -17,15 +17,14 @@ Previously, this logic was mixed in loose scripts or monolithic build convention ## Key Features -### 1. Snapshot Metadata Harvesting -Standard Maven snapshot repositories (e.g., Sonatype Snapshots) return `404` errors when fetching non-timestamped `-SNAPSHOT` dependencies directly. This plugin dynamically locates and parses local cached `maven-metadata.xml` files inside Gradle's cached directories, resolves the unique timestamped snapshot coordinate, and constructs exact, direct download URLs while preserving local filename bindings. +### 1. Remote Snapshot Metadata Resolution +Standard Maven snapshot repositories (e.g., Sonatype Snapshots) return `404` errors when fetching non-timestamped `-SNAPSHOT` dependencies directly. This plugin fetches `maven-metadata.xml` from the remote snapshot repository at generation time, resolves the unique timestamped snapshot coordinate (e.g., `0.2.4-20260520.043744-2`), and constructs exact, direct download URLs while preserving local filename bindings. ### 2. JitPack URL Routing Automatically identifies external dependencies belonging to the `com.github.*` group (hosted on JitPack) and routes their `primaryUrl` to `https://jitpack.io` instead of attempting standard Maven Central lookup, preventing sandboxed download failures. -### 3. High-Performance Optimizations -* **Single-Pass Metadata Indexing**: Scans cached metadata files exactly once on-demand, caching them in an in-memory `$O(1)$` lookup map. -* **Deferred Cryptographic Hashing**: Defers expensive SHA-256 calculation until after candidate files are fully deduplicated and sorted. +### 3. Automatic Cache Population +The task automatically depends on `:desktopApp:assemble`, ensuring the Gradle dependency cache is fully populated before scanning. No manual pre-build step is required. --- @@ -39,6 +38,23 @@ plugins { } ``` +### Configuration (DSL) + +All options have sensible defaults. Override as needed: + +```kotlin +flatpak { + // Snapshot repository to fetch maven-metadata.xml from + snapshotRepoUrl.set("https://central.sonatype.com/repository/maven-snapshots") + // Task that populates the Gradle cache before scanning + assembleTask.set(":desktopApp:assemble") + // Custom Gradle cache directory (defaults to ~/.gradle/caches/modules-2/files-2.1) + cacheDir.set(layout.projectDirectory.dir("my-cache")) + // Output manifest path + outputFile.set(layout.projectDirectory.file("flatpak-sources.json")) +} +``` + ### Running the Generator Task Execute the registered custom task to sweep your Gradle local modules cache and generate/overwrite the root `flatpak-sources.json`: @@ -47,17 +63,10 @@ Execute the registered custom task to sweep your Gradle local modules cache and ./gradlew :generateFlatpakSourcesFromCache ``` -### Custom Cache Directory - -By default, the task scans the standard Gradle user home caches directory (`~/.gradle/caches/modules-2/files-2.1`). You can supply a custom cache directory using the `flatpak.cache.dir` Gradle property: - -```bash -./gradlew :generateFlatpakSourcesFromCache -Pflatpak.cache.dir="/custom/cache/path" -``` - --- ## Architecture -* **[FlatpakConventionPlugin.kt](src/main/kotlin/FlatpakConventionPlugin.kt)**: Registers the `generateFlatpakSourcesFromCache` task using lazy provider configuration. -* **[GenerateFlatpakSourcesTask.kt](src/main/kotlin/org/meshtastic/flatpak/GenerateFlatpakSourcesTask.kt)**: The native JVM-based custom task responsible for Gradle files scanning, metadata harvesting, and JSON generation. +* **[FlatpakPlugin.kt](src/main/kotlin/org/meshtastic/flatpak/FlatpakPlugin.kt)**: Registers the `flatpak {}` DSL extension and the `generateFlatpakSourcesFromCache` task using lazy provider configuration. +* **[FlatpakExtension.kt](src/main/kotlin/org/meshtastic/flatpak/FlatpakExtension.kt)**: DSL extension interface defining all configurable properties. +* **[GenerateFlatpakSourcesTask.kt](src/main/kotlin/org/meshtastic/flatpak/GenerateFlatpakSourcesTask.kt)**: The custom task responsible for Gradle files scanning, remote metadata resolution, and JSON generation. diff --git a/build-logic/flatpak/build.gradle.kts b/build-logic/flatpak/build.gradle.kts index 057d0495c..2fc3d0c06 100644 --- a/build-logic/flatpak/build.gradle.kts +++ b/build-logic/flatpak/build.gradle.kts @@ -77,7 +77,7 @@ gradlePlugin { plugins { register("meshtasticFlatpak") { id = "meshtastic.flatpak" - implementationClass = "FlatpakConventionPlugin" + implementationClass = "org.meshtastic.flatpak.FlatpakPlugin" } } } diff --git a/build-logic/flatpak/detekt-baseline.xml b/build-logic/flatpak/detekt-baseline.xml index eddf0ebdc..f7d79d235 100644 --- a/build-logic/flatpak/detekt-baseline.xml +++ b/build-logic/flatpak/detekt-baseline.xml @@ -8,10 +8,7 @@ MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$4 MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$5 MagicNumber:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$8192 - MaxLineLength:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$"Gradle cache directory does not exist or is not configured correctly. Please run a build first to populate the cache." MaxLineLength:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$"Successfully scanned cache and generated ${outputSourcesFile.name} containing ${finalEntries.size} entries." - NestedBlockDepth:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$private fun populateMetadataCache - ReturnCount:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$private fun findSnapshotValue: String? SwallowedException:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$e: Exception TooGenericExceptionCaught:GenerateFlatpakSourcesTask.kt:GenerateFlatpakSourcesTask$e: Exception diff --git a/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/FlatpakExtension.kt b/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/FlatpakExtension.kt new file mode 100644 index 000000000..1af44e1aa --- /dev/null +++ b/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/FlatpakExtension.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.flatpak + +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.provider.Property + +/** + * DSL extension for configuring the Flatpak source manifest generator. + * + * ```kotlin + * flatpak { + * snapshotRepoUrl.set("https://central.sonatype.com/repository/maven-snapshots") + * assembleTask.set(":desktopApp:assemble") + * } + * ``` + */ +abstract class FlatpakExtension { + + /** Gradle cache directory to scan for dependency artifacts. */ + abstract val cacheDir: DirectoryProperty + + /** Output path for the generated flatpak-sources.json manifest. */ + abstract val outputFile: RegularFileProperty + + /** Base URL of the Maven snapshot repository (no trailing slash). */ + abstract val snapshotRepoUrl: Property + + /** Task path to depend on, ensuring the cache is fully populated before scanning. */ + abstract val assembleTask: Property +} diff --git a/build-logic/flatpak/src/main/kotlin/FlatpakConventionPlugin.kt b/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/FlatpakPlugin.kt similarity index 54% rename from build-logic/flatpak/src/main/kotlin/FlatpakConventionPlugin.kt rename to build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/FlatpakPlugin.kt index 52a8279a3..16b278754 100644 --- a/build-logic/flatpak/src/main/kotlin/FlatpakConventionPlugin.kt +++ b/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/FlatpakPlugin.kt @@ -14,25 +14,29 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ +package org.meshtastic.flatpak import org.gradle.api.Plugin import org.gradle.api.Project -import org.meshtastic.flatpak.GenerateFlatpakSourcesTask import java.io.File -class FlatpakConventionPlugin : Plugin { +class FlatpakPlugin : Plugin { override fun apply(target: Project) { with(target) { + val extension = extensions.create("flatpak", FlatpakExtension::class.java).apply { + cacheDir.convention( + layout.dir(providers.provider { File(gradle.gradleUserHomeDir, "caches/modules-2/files-2.1") }), + ) + outputFile.convention(layout.projectDirectory.file("flatpak-sources.json")) + snapshotRepoUrl.convention("https://central.sonatype.com/repository/maven-snapshots") + assembleTask.convention(":desktopApp:assemble") + } + tasks.register("generateFlatpakSourcesFromCache", GenerateFlatpakSourcesTask::class.java) { - val customCachePath = providers.gradleProperty("flatpak.cache.dir").orNull - if (customCachePath != null) { - cacheDir.set(layout.projectDirectory.dir(customCachePath)) - } else { - cacheDir.set( - layout.dir(providers.provider { File(gradle.gradleUserHomeDir, "caches/modules-2/files-2.1") }), - ) - } - outputFile.set(layout.projectDirectory.file("flatpak-sources.json")) + cacheDir.set(extension.cacheDir) + outputFile.set(extension.outputFile) + snapshotRepoUrl.set(extension.snapshotRepoUrl) + dependsOn(extension.assembleTask) } } } diff --git a/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/GenerateFlatpakSourcesTask.kt b/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/GenerateFlatpakSourcesTask.kt index ca91c27d0..86dd9cd29 100644 --- a/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/GenerateFlatpakSourcesTask.kt +++ b/build-logic/flatpak/src/main/kotlin/org/meshtastic/flatpak/GenerateFlatpakSourcesTask.kt @@ -18,29 +18,40 @@ package org.meshtastic.flatpak import groovy.json.JsonOutput import org.gradle.api.DefaultTask -import org.gradle.api.GradleException import org.gradle.api.file.DirectoryProperty import org.gradle.api.file.RegularFileProperty -import org.gradle.api.tasks.Internal +import org.gradle.api.provider.Property +import org.gradle.work.DisableCachingByDefault +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputDirectory import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity import org.gradle.api.tasks.TaskAction import org.w3c.dom.Element import org.w3c.dom.NodeList import java.io.File +import java.net.URI import java.security.MessageDigest import javax.xml.parsers.DocumentBuilderFactory /** Generates a complete flatpak-sources.json manifest from the local Gradle cache directory. */ +@DisableCachingByDefault(because = "Resolves remote snapshot metadata that may change between runs") abstract class GenerateFlatpakSourcesTask : DefaultTask() { - @get:Internal abstract val cacheDir: DirectoryProperty + @get:InputDirectory + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val cacheDir: DirectoryProperty @get:OutputFile abstract val outputFile: RegularFileProperty + /** Base URL of the Maven snapshot repository (no trailing slash). */ + @get:Input + abstract val snapshotRepoUrl: Property + init { group = "flatpak" description = "Generates a complete flatpak-sources.json manifest from the local Gradle cache directory." - // Ensure the task always runs when executed outputs.upToDateWhen { false } } @@ -60,17 +71,14 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() { val mirrorUrls: List, ) - private var metadataCache: Map? = null + private val remoteMetadataCache = mutableMapOf() @TaskAction fun generate() { - val cacheFolder = - cacheDir.orNull?.asFile - ?: throw GradleException( - "Gradle cache directory does not exist or is not configured correctly. Please run a build first to populate the cache.", - ) + val cacheFolder = cacheDir.get().asFile val outputSourcesFile = outputFile.get().asFile + val snapshotBase = snapshotRepoUrl.get() logger.lifecycle("Scanning Gradle cache directory: ${cacheFolder.absolutePath}") val allowedExtensions = setOf("jar", "aar", "pom", "module") @@ -93,33 +101,19 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() { val (group, name, version) = parts val groupPath = group.replace('.', '/') val standardPrefix = "$name-$version" - val isSnapshot = version.endsWith("-SNAPSHOT") || version.contains("-SNAPSHOT") - - val classifier = - if (isSnapshot) { - val prefix = "$name-$version-" - filename.takeIf { it.startsWith(prefix) }?.removePrefix(prefix)?.removeSuffix(".$ext") - } else { - null - } + val isSnapshot = version.endsWith("-SNAPSHOT") val resolvedVersion = if (isSnapshot) { - val resourcesFolder = File(cacheFolder.parentFile, "resources-2.1") - findSnapshotValue(resourcesFolder, group, name, ext, classifier) ?: version + resolveSnapshotVersion(snapshotBase, groupPath, name, version, ext) ?: version } else { version } val serverFilename = when { - isSnapshot -> { - val suffix = classifier?.let { "-$it" } ?: "" - "$name-$resolvedVersion$suffix.$ext" - } - + isSnapshot -> "$name-$resolvedVersion.$ext" filename.startsWith(standardPrefix) -> filename - else -> "$name-$version.$ext" } @@ -128,31 +122,30 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() { val isJitpack = group.startsWith("com.github.") val primaryUrl = - if (isSnapshot) { - "https://central.sonatype.com/repository/maven-snapshots/$mavenPath" - } else if (isJitpack) { - "https://jitpack.io/$mavenPath" - } else { - "https://repo.maven.apache.org/maven2/$mavenPath" + when { + isSnapshot -> + "$snapshotBase/$mavenPath" + + isJitpack -> + "https://jitpack.io/$mavenPath" + + else -> + "https://repo.maven.apache.org/maven2/$mavenPath" } val mirrorUrls = when { - isSnapshot -> listOf("https://oss.sonatype.org/content/repositories/snapshots/$mavenPath") + isSnapshot -> listOf( + "https://s01.oss.sonatype.org/content/repositories/snapshots/$mavenPath", + ) - isJitpack -> - listOf( - "https://repo.maven.apache.org/maven2/$mavenPath", - "https://maven-central.storage-download.googleapis.com/maven2/$mavenPath", - "https://maven.aliyun.com/repository/public/$mavenPath", - ) + isJitpack -> emptyList() else -> listOf( "https://dl.google.com/dl/android/maven2/$mavenPath", "https://plugins.gradle.org/m2/$mavenPath", "https://maven-central.storage-download.googleapis.com/maven2/$mavenPath", - "https://maven.aliyun.com/repository/public/$mavenPath", ) } @@ -181,14 +174,17 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() { val finalEntries = deduplicated.map { candidate -> - mapOf( + val entry = mutableMapOf( "type" to "file", "url" to candidate.primaryUrl, "sha256" to calculateSha256(candidate.file), "dest" to candidate.dest, "dest-filename" to candidate.destFilename, - "mirror-urls" to candidate.mirrorUrls, ) + if (candidate.mirrorUrls.isNotEmpty()) { + entry["mirror-urls"] = candidate.mirrorUrls + } + entry } outputSourcesFile.writeText(JsonOutput.prettyPrint(JsonOutput.toJson(finalEntries))) @@ -197,6 +193,80 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() { ) } + /** + * Resolves the timestamped snapshot version by fetching maven-metadata.xml from the remote + * snapshot repository. Maven snapshot repos do not serve artifacts at the generic `-SNAPSHOT` + * filename — they require the unique timestamped coordinate (e.g. `0.2.4-20260520.043744-2`). + */ + private fun resolveSnapshotVersion( + snapshotBase: String, + groupPath: String, + artifactId: String, + version: String, + extension: String, + ): String? { + val cacheKey = "$groupPath:$artifactId:$version" + if (cacheKey in remoteMetadataCache) { + return findMatchingVersion(remoteMetadataCache[cacheKey], extension) + } + + val metadataUrl = "$snapshotBase/$groupPath/$artifactId/$version/maven-metadata.xml" + + val metadata = fetchAndParseMetadata(metadataUrl) + remoteMetadataCache[cacheKey] = metadata + return findMatchingVersion(metadata, extension) + } + + private fun findMatchingVersion(metadata: SnapshotMetadata?, extension: String): String? { + if (metadata == null) return null + return metadata.snapshotVersions + .firstOrNull { it.extension == extension && it.classifier == null } + ?.value + ?: metadata.fallbackValue + } + + private fun fetchAndParseMetadata(url: String): SnapshotMetadata? { + try { + logger.info("Fetching snapshot metadata: $url") + val connection = URI(url).toURL().openConnection() + connection.connectTimeout = TIMEOUT_MS + connection.readTimeout = TIMEOUT_MS + + val doc = connection.getInputStream().use { stream -> + val dbFactory = DocumentBuilderFactory.newInstance() + val dBuilder = dbFactory.newDocumentBuilder() + dBuilder.parse(stream) + } + doc.documentElement.normalize() + + val root = doc.documentElement + val snapshotVersions = mutableListOf() + root.getElementsByTagName("snapshotVersion").forEachElement { element -> + val ext = element.getChildText("extension") ?: return@forEachElement + val value = element.getChildText("value") ?: return@forEachElement + val classif = element.getChildText("classifier") + snapshotVersions.add(SnapshotVersion(ext, classif, value)) + } + + var fallbackValue: String? = null + val snapshotNode = root.getElementsByTagName("snapshot").item(0) as? Element + if (snapshotNode != null) { + val timestamp = snapshotNode.getChildText("timestamp") + val buildNumber = snapshotNode.getChildText("buildNumber") + val metaVersion = root.getChildText("version") + if (timestamp != null && buildNumber != null && metaVersion != null) { + val baseVersion = metaVersion.substringBefore("-SNAPSHOT") + fallbackValue = "$baseVersion-$timestamp-$buildNumber" + } + } + + return SnapshotMetadata(snapshotVersions, fallbackValue) + } catch (e: Exception) { + logger.warn("Failed to fetch snapshot metadata from $url: ${e.message}") + return null + } + } + private fun calculateSha256(file: File): String { val digest = MessageDigest.getInstance("SHA-256") file.inputStream().use { inputStream -> @@ -218,10 +288,8 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() { return String(hexChars) } - // Modern helper extension to query DOM child element text safely private fun Element.getChildText(tagName: String): String? = getElementsByTagName(tagName).item(0)?.textContent - // Modern helper extension to iterate DOM elements cleanly private fun NodeList.forEachElement(action: (Element) -> Unit) { for (i in 0 until length) { val node = item(i) @@ -231,74 +299,7 @@ abstract class GenerateFlatpakSourcesTask : DefaultTask() { } } - private fun populateMetadataCache(resourcesFolder: File) { - if (metadataCache != null || !resourcesFolder.exists()) return - val cache = mutableMapOf() - - resourcesFolder.walkTopDown().forEach { file -> - if (file.isFile && file.name == "maven-metadata.xml") { - try { - val dbFactory = DocumentBuilderFactory.newInstance() - val dBuilder = dbFactory.newDocumentBuilder() - val doc = dBuilder.parse(file) - doc.documentElement.normalize() - - val root = doc.documentElement - val group = root.getChildText("groupId") ?: return@forEach - val name = root.getChildText("artifactId") ?: return@forEach - - val key = "$group:$name" - val snapshotVersions = mutableListOf() - root.getElementsByTagName("snapshotVersion").forEachElement { element -> - val ext = element.getChildText("extension") ?: return@forEachElement - val value = element.getChildText("value") ?: return@forEachElement - val classif = element.getChildText("classifier") - - snapshotVersions.add(SnapshotVersion(ext, classif, value)) - } - - var fallbackValue: String? = null - val snapshotNode = root.getElementsByTagName("snapshot").item(0) as? Element - if (snapshotNode != null) { - val timestamp = snapshotNode.getChildText("timestamp") - val buildNumber = snapshotNode.getChildText("buildNumber") - val version = root.getChildText("version") - if (timestamp != null && buildNumber != null && version != null) { - val baseVersion = version.substringBefore("-SNAPSHOT") - fallbackValue = "$baseVersion-$timestamp-$buildNumber" - } - } - - cache[key] = SnapshotMetadata(snapshotVersions, fallbackValue) - } catch (e: Exception) { - // Ignore parsing errors for individual files - } - } - } - metadataCache = cache - } - - private fun findSnapshotValue( - resourcesFolder: File, - group: String, - name: String, - extension: String, - classifier: String?, - ): String? { - populateMetadataCache(resourcesFolder) - val metadata = metadataCache?.get("$group:$name") ?: return null - - for (version in metadata.snapshotVersions) { - if (version.extension == extension) { - if (classifier == null && version.classifier == null) { - return version.value - } - if (classifier != null && classifier == version.classifier) { - return version.value - } - } - } - - return metadata.fallbackValue + private companion object { + private const val TIMEOUT_MS = 10_000 } }