🏗️ subcommands scheduling and sending buildcycles

This adds 2 new sub commands:
* schedule_buildcycle - create a json format schedule for the next
  buildserver build cycle
* send_buildcycle - submit build requests from schedule json to buildbot

fdroidserver/__main__.py

@@ -78,6 +78,8 @@ COMMANDS_INTERNAL = [
     "pull_verify",
     "push",
     "schedule_verify",
+    "schedule_buildcycle",
+    "send_buildcycle",
     "up",
 ]
 

fdroidserver/schedule_buildcycle.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+#
+# schedule_buildcycle.py - part of the FDroid server tools
+# Copyright (C) 2024-2025, Michael Pöhn <michael@poehn.at>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import sys
+import json
+import time
+import urllib
+import logging
+import argparse
+import traceback
+
+from fdroidserver import _, common, metadata
+
+start_timestamp = time.gmtime()
+
+
+# by default fdroid builds time out after 2 hours
+# see: https://f-droid.org/en/docs/Build_Metadata_Reference/#build_timeout
+DEFAULT_BUILD_TIMEOUT = 7200
+
+
+def get_web_index(index_v2_url="https://f-droid.org/repo/index-v2.json"):
+    with urllib.request.urlopen(index_v2_url) as response:
+        raw = response.read().decode('utf-8')
+        return json.loads(raw)
+
+
+def published_apps(index_v2={}):
+    return index_v2.get("packages", {}).keys()
+
+
+def is_binary_artifact_present(appid, build):
+    """Check if a build artifact/result form a previous run exists.
+
+    Parameters
+    ----------
+    appid
+        app id you're looking for (e.g. 'org.fdroid.fdroid')
+    build
+        metadata build object you're checking
+
+    Returns
+    -------
+    True if a build artifact exists, otherwise False.
+    """
+    bin_dirs = ["archive", "repo", "unsigned"]
+    ext = common.get_output_extension(build)
+
+    for bin_dir in bin_dirs:
+        if os.path.exists(f"./{bin_dir}/{appid}_{build.versionCode}.{ext}"):
+            return True
+
+    return False
+
+
+def collect_schedule_entries(apps):
+    """Get list of schedule entries for next build run.
+
+    This function matches which builds in metadata are not built yet.
+
+    Parameters
+    ----------
+    apps
+        list of all metadata app objects of current repo
+
+    Returns
+    -------
+    list of schedule entries
+    """
+    schedule = []
+    for appid, app in apps.items():
+        enabled = not app.get("Disabled")
+        archived = app.get('ArchivePolicy') == 0
+        if enabled and not archived:
+            for build in app.get("Builds", {}):
+                if not build.get("disable"):
+                    if app.get("CurrentVersionCode") == build.get("versionCode"):
+                        if not is_binary_artifact_present(appid, build):
+                            schedule.append(
+                                {
+                                    "applicationId": appid,
+                                    "versionCode": build.get("versionCode"),
+                                    "timeout": int(
+                                        build.get("timeout") or DEFAULT_BUILD_TIMEOUT
+                                    ),
+                                }
+                            )
+    return schedule
+
+
+def schedule_buildcycle_wrapper(limit=None, offset=None, published_only=False):
+    apps = metadata.read_metadata()
+
+    if published_only:
+        pub_apps = published_apps(index_v2=get_web_index())
+        appids = [x for x in apps.keys()]
+        for appid in appids:
+            if appid not in pub_apps:
+                del apps[appid]
+
+    schedule = collect_schedule_entries(apps)
+
+    if offset:
+        schedule = schedule[offset:]
+    if limit:
+        schedule = schedule[:limit]
+
+    return schedule
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=_("""print not yet built apps in JSON fromat to STDOUT"""),
+    )
+    parser.add_argument(
+        "--pretty",
+        '-p',
+        action="store_true",
+        default=False,
+        help="pretty output formatting",
+    )
+    parser.add_argument(
+        "--limit",
+        "-l",
+        type=int,
+        help="limit the number of apps in output (e.g. if you wan to set "
+        "a batch size)",
+        default=None,
+    )
+    parser.add_argument(
+        "--offset",
+        "-o",
+        type=int,
+        help="offset the generated schedule (e.g. if you want to skip "
+        "building the first couple of apps for a batch)",
+        default=None,
+    )
+    parser.add_argument(
+        "--published-only",
+        action="store_true",
+        default=False,
+    )
+
+    # fdroid args/opts boilerplate
+    common.setup_global_opts(parser)
+    options = common.parse_args(parser)
+    common.get_config()  # set up for common functions
+    status_output = common.setup_status_output(start_timestamp)
+    common.write_running_status_json(status_output)
+
+    error = False
+    try:
+        schedule = schedule_buildcycle_wrapper(
+            limit=options.limit,
+            offset=options.offset,
+            published_only=options.published_only,
+        )
+        indent = 2 if options.pretty else None
+        print(json.dumps(schedule, indent=indent))
+    except Exception as e:
+        if options.verbose:
+            logging.error(traceback.format_exc())
+        else:
+            logging.error(e)
+        error = True
+        status_output['errors'] = [traceback.format_exc()]
+
+    common.write_status_json(status_output)
+    sys.exit(error)
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/send_buildcycle.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+#
+# send_buildcycle.py - part of the FDroid server tools
+# Copyright (C) 2024-2025, Hans-Christoph Steiner <hans@eds.org>
+# Copyright (C) 2024-2025, Michael Pöhn <michael@poehn.at>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+"""Sub-command for sending build requests to a F-Droid BuildBot instance."""
+
+
+import sys
+import json
+import uuid
+import shlex
+import shutil
+import logging
+import argparse
+import traceback
+import subprocess
+
+from pathlib import Path
+
+from fdroidserver import _, common, schedule_buildcycle
+
+
+def send_to_buildbot(
+    package_name,
+    version_code,
+    cycle_item=None,
+    cycle_count=None,
+    cycle_uuid=None,
+    timeout=None,
+):
+    """Use `buildbot sendchange` to submit builds to the queue.
+
+    This requires the automatically generated password to authenticate
+    to the buildbot instance, which is created at a static path by the
+    buildbot master:
+    https://gitlab.com/fdroid/buildbot/-/merge_requests/1
+
+    """
+    bb_bin = shutil.which("buildbot")
+    if not bb_bin:
+        raise Exception("'buildbot' not found, make sure it's installed correctly")
+
+    passwd_path = Path('/tmp/fdroid-buildbot-sendchange/passwd')
+    if not passwd_path.is_file():
+        raise FileNotFoundError(
+            f"'{passwd_path}' not found (file is managed by fdroid buildbot master)"
+        )
+    passwd = passwd_path.read_text().strip()
+
+    git_revision = str(
+        subprocess.check_output(["git", "-C", ".", "describe", "--always"]),
+        encoding="utf=8",
+    ).strip()
+    cmd = [
+        bb_bin,
+        "sendchange",
+        "--master={}".format("127.0.0.1:9999"),
+        "--auth=fdroid:{}".format(passwd),
+        "--branch=master",
+        "--repository='https://gitlab.com/fdroid/fdroiddata'",
+        "--revision={}".format(git_revision),
+        "--category=build",
+        "--who={}:{}".format(package_name, version_code),
+        "--project={}".format(package_name),
+        "--property=versionCode:{}".format(version_code),
+        "--property=packageName:{}".format(package_name),
+        "--property=timeout:{}".format(
+            timeout or schedule_buildcycle.DEFAULT_BUILD_TIMEOUT
+        ),
+    ]
+    if cycle_item:
+        cmd.append("--property=buildCycleItem:{}".format(cycle_item))
+    if cycle_count:
+        cmd.append("--property=buildCycleSize:{}".format(cycle_count))
+    if cycle_uuid:
+        cmd.append("--property=buildCycleUuid:{}".format(cycle_uuid))
+    cmd.append("metadata/{}.yml".format(package_name))
+
+    logging.info(f"sending buildbot build request for {package_name}:{version_code}")
+    logging.debug(shlex.join(cmd))
+    r = subprocess.run(
+        cmd,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    if r.returncode > 0:
+        raise Exception(
+            f"sending build request for '{package_name}:{version_code}' failed."
+            f"\nstdout: {r.stdout}\nstderr: {r.stderr}"
+        )
+
+
+def send_buildcycle_wrapper(
+    build_list=[], read_stdin=False, timeout=schedule_buildcycle.DEFAULT_BUILD_TIMEOUT
+):
+    if not read_stdin and len(build_list) <= 0:
+        raise Exception(
+            "you can not specify both APPID:VERCODE and -i at "
+            "the sametime (see -h for help)"
+        )
+
+    # generate a random unique indentifier for this build cycle
+    cycle_uuid = uuid.uuid4().hex
+
+    if read_stdin:
+        json_input = json.loads(sys.stdin.read())
+        count = len(json_input)
+        for i, appver in enumerate(json_input):
+            send_to_buildbot(
+                appver["applicationId"],
+                appver["versionCode"],
+                cycle_item=i + 1,
+                cycle_count=count,
+                cycle_uuid=cycle_uuid,
+                timeout=appver["timeout"],
+            )
+    else:
+        count = len(build_list)
+        for i, (appid, vercode) in enumerate(build_list):
+            # appid, vercode = appver
+            send_to_buildbot(
+                appid,
+                vercode,
+                cycle_item=i + 1,
+                cycle_count=count,
+                cycle_uuid=cycle_uuid,
+                timeout=timeout,
+            )
+
+
+def main():
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description=_(
+            "send change notifications to buildbot for kicking off app builds"
+        ),
+    )
+    common.setup_global_opts(parser)
+    parser.add_argument(
+        '--stdin',
+        "-i",
+        default=False,
+        action="store_true",
+        help="read JSON schedule data from stdin. "
+        "(typically created by `fdroid schedule_build`)",
+    )
+    parser.add_argument(
+        "--timeout",
+        "-t",
+        type=int,
+        default=schedule_buildcycle.DEFAULT_BUILD_TIMEOUT,
+        help="builds will get aborted when this time interval expires "
+        "(in seconds, defaults to 2 hours; will be ignored when --stdin is specified)",
+    )
+    parser.add_argument(
+        "APPID:VERCODE",
+        nargs="*",
+        help=_("app id and version code tuple 'APPID:VERCODE'"),
+    )
+    options = common.parse_args(parser)
+
+    try:
+        build_list = [
+            common.split_pkg_arg(x) for x in options.__dict__['APPID:VERCODE']
+        ]
+        send_buildcycle_wrapper(
+            build_list=build_list, read_stdin=options.stdin, timeout=options.timeout
+        )
+    except Exception as e:
+        if options.verbose:
+            logging.error(traceback.format_exc())
+        else:
+            logging.error(e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()