From 709c8fe675e78bd26cdf68ed2f342027f1641a07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sun, 16 Nov 2025 23:34:22 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=8F=97=EF=B8=8F=20subcommands=20schedulin?= =?UTF-8?q?g=20and=20sending=20buildcycles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds 2 new sub commands: * schedule_buildcycle - create a json format schedule for the next buildserver build cycle * send_buildcycle - submit build requests from schedule json to buildbot --- fdroidserver/__main__.py | 2 + fdroidserver/schedule_buildcycle.py | 190 +++++++++++++++++++++++++++ fdroidserver/send_buildcycle.py | 195 ++++++++++++++++++++++++++++ 3 files changed, 387 insertions(+) create mode 100644 fdroidserver/schedule_buildcycle.py create mode 100644 fdroidserver/send_buildcycle.py diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index c6e695f5..aa41024e 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -78,6 +78,8 @@ COMMANDS_INTERNAL = [ "pull_verify", "push", "schedule_verify", + "schedule_buildcycle", + "send_buildcycle", "up", ] diff --git a/fdroidserver/schedule_buildcycle.py b/fdroidserver/schedule_buildcycle.py new file mode 100644 index 00000000..0d5f7495 --- /dev/null +++ b/fdroidserver/schedule_buildcycle.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +# +# schedule_buildcycle.py - part of the FDroid server tools +# Copyright (C) 2024-2025, Michael Pöhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import os +import sys +import json +import time +import urllib +import logging +import argparse +import traceback + +from fdroidserver import _, common, metadata + +start_timestamp = time.gmtime() + + +# by default fdroid builds time out after 2 hours +# see: https://f-droid.org/en/docs/Build_Metadata_Reference/#build_timeout +DEFAULT_BUILD_TIMEOUT = 7200 + + +def get_web_index(index_v2_url="https://f-droid.org/repo/index-v2.json"): + with urllib.request.urlopen(index_v2_url) as response: + raw = response.read().decode('utf-8') + return json.loads(raw) + + +def published_apps(index_v2={}): + return index_v2.get("packages", {}).keys() + + +def is_binary_artifact_present(appid, build): + """Check if a build artifact/result form a previous run exists. + + Parameters + ---------- + appid + app id you're looking for (e.g. 'org.fdroid.fdroid') + build + metadata build object you're checking + + Returns + ------- + True if a build artifact exists, otherwise False. + """ + bin_dirs = ["archive", "repo", "unsigned"] + ext = common.get_output_extension(build) + + for bin_dir in bin_dirs: + if os.path.exists(f"./{bin_dir}/{appid}_{build.versionCode}.{ext}"): + return True + + return False + + +def collect_schedule_entries(apps): + """Get list of schedule entries for next build run. + + This function matches which builds in metadata are not built yet. + + Parameters + ---------- + apps + list of all metadata app objects of current repo + + Returns + ------- + list of schedule entries + """ + schedule = [] + for appid, app in apps.items(): + enabled = not app.get("Disabled") + archived = app.get('ArchivePolicy') == 0 + if enabled and not archived: + for build in app.get("Builds", {}): + if not build.get("disable"): + if app.get("CurrentVersionCode") == build.get("versionCode"): + if not is_binary_artifact_present(appid, build): + schedule.append( + { + "applicationId": appid, + "versionCode": build.get("versionCode"), + "timeout": int( + build.get("timeout") or DEFAULT_BUILD_TIMEOUT + ), + } + ) + return schedule + + +def schedule_buildcycle_wrapper(limit=None, offset=None, published_only=False): + apps = metadata.read_metadata() + + if published_only: + pub_apps = published_apps(index_v2=get_web_index()) + appids = [x for x in apps.keys()] + for appid in appids: + if appid not in pub_apps: + del apps[appid] + + schedule = collect_schedule_entries(apps) + + if offset: + schedule = schedule[offset:] + if limit: + schedule = schedule[:limit] + + return schedule + + +def main(): + parser = argparse.ArgumentParser( + description=_("""print not yet built apps in JSON fromat to STDOUT"""), + ) + parser.add_argument( + "--pretty", + '-p', + action="store_true", + default=False, + help="pretty output formatting", + ) + parser.add_argument( + "--limit", + "-l", + type=int, + help="limit the number of apps in output (e.g. if you wan to set " + "a batch size)", + default=None, + ) + parser.add_argument( + "--offset", + "-o", + type=int, + help="offset the generated schedule (e.g. if you want to skip " + "building the first couple of apps for a batch)", + default=None, + ) + parser.add_argument( + "--published-only", + action="store_true", + default=False, + ) + + # fdroid args/opts boilerplate + common.setup_global_opts(parser) + options = common.parse_args(parser) + common.get_config() # set up for common functions + status_output = common.setup_status_output(start_timestamp) + common.write_running_status_json(status_output) + + error = False + try: + schedule = schedule_buildcycle_wrapper( + limit=options.limit, + offset=options.offset, + published_only=options.published_only, + ) + indent = 2 if options.pretty else None + print(json.dumps(schedule, indent=indent)) + except Exception as e: + if options.verbose: + logging.error(traceback.format_exc()) + else: + logging.error(e) + error = True + status_output['errors'] = [traceback.format_exc()] + + common.write_status_json(status_output) + sys.exit(error) + + +if __name__ == "__main__": + main() diff --git a/fdroidserver/send_buildcycle.py b/fdroidserver/send_buildcycle.py new file mode 100644 index 00000000..f5920b17 --- /dev/null +++ b/fdroidserver/send_buildcycle.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +# +# send_buildcycle.py - part of the FDroid server tools +# Copyright (C) 2024-2025, Hans-Christoph Steiner +# Copyright (C) 2024-2025, Michael Pöhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +"""Sub-command for sending build requests to a F-Droid BuildBot instance.""" + + +import sys +import json +import uuid +import shlex +import shutil +import logging +import argparse +import traceback +import subprocess + +from pathlib import Path + +from fdroidserver import _, common, schedule_buildcycle + + +def send_to_buildbot( + package_name, + version_code, + cycle_item=None, + cycle_count=None, + cycle_uuid=None, + timeout=None, +): + """Use `buildbot sendchange` to submit builds to the queue. + + This requires the automatically generated password to authenticate + to the buildbot instance, which is created at a static path by the + buildbot master: + https://gitlab.com/fdroid/buildbot/-/merge_requests/1 + + """ + bb_bin = shutil.which("buildbot") + if not bb_bin: + raise Exception("'buildbot' not found, make sure it's installed correctly") + + passwd_path = Path('/tmp/fdroid-buildbot-sendchange/passwd') + if not passwd_path.is_file(): + raise FileNotFoundError( + f"'{passwd_path}' not found (file is managed by fdroid buildbot master)" + ) + passwd = passwd_path.read_text().strip() + + git_revision = str( + subprocess.check_output(["git", "-C", ".", "describe", "--always"]), + encoding="utf=8", + ).strip() + cmd = [ + bb_bin, + "sendchange", + "--master={}".format("127.0.0.1:9999"), + "--auth=fdroid:{}".format(passwd), + "--branch=master", + "--repository='https://gitlab.com/fdroid/fdroiddata'", + "--revision={}".format(git_revision), + "--category=build", + "--who={}:{}".format(package_name, version_code), + "--project={}".format(package_name), + "--property=versionCode:{}".format(version_code), + "--property=packageName:{}".format(package_name), + "--property=timeout:{}".format( + timeout or schedule_buildcycle.DEFAULT_BUILD_TIMEOUT + ), + ] + if cycle_item: + cmd.append("--property=buildCycleItem:{}".format(cycle_item)) + if cycle_count: + cmd.append("--property=buildCycleSize:{}".format(cycle_count)) + if cycle_uuid: + cmd.append("--property=buildCycleUuid:{}".format(cycle_uuid)) + cmd.append("metadata/{}.yml".format(package_name)) + + logging.info(f"sending buildbot build request for {package_name}:{version_code}") + logging.debug(shlex.join(cmd)) + r = subprocess.run( + cmd, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if r.returncode > 0: + raise Exception( + f"sending build request for '{package_name}:{version_code}' failed." + f"\nstdout: {r.stdout}\nstderr: {r.stderr}" + ) + + +def send_buildcycle_wrapper( + build_list=[], read_stdin=False, timeout=schedule_buildcycle.DEFAULT_BUILD_TIMEOUT +): + if not read_stdin and len(build_list) <= 0: + raise Exception( + "you can not specify both APPID:VERCODE and -i at " + "the sametime (see -h for help)" + ) + + # generate a random unique indentifier for this build cycle + cycle_uuid = uuid.uuid4().hex + + if read_stdin: + json_input = json.loads(sys.stdin.read()) + count = len(json_input) + for i, appver in enumerate(json_input): + send_to_buildbot( + appver["applicationId"], + appver["versionCode"], + cycle_item=i + 1, + cycle_count=count, + cycle_uuid=cycle_uuid, + timeout=appver["timeout"], + ) + else: + count = len(build_list) + for i, (appid, vercode) in enumerate(build_list): + # appid, vercode = appver + send_to_buildbot( + appid, + vercode, + cycle_item=i + 1, + cycle_count=count, + cycle_uuid=cycle_uuid, + timeout=timeout, + ) + + +def main(): + """CLI entry point.""" + parser = argparse.ArgumentParser( + description=_( + "send change notifications to buildbot for kicking off app builds" + ), + ) + common.setup_global_opts(parser) + parser.add_argument( + '--stdin', + "-i", + default=False, + action="store_true", + help="read JSON schedule data from stdin. " + "(typically created by `fdroid schedule_build`)", + ) + parser.add_argument( + "--timeout", + "-t", + type=int, + default=schedule_buildcycle.DEFAULT_BUILD_TIMEOUT, + help="builds will get aborted when this time interval expires " + "(in seconds, defaults to 2 hours; will be ignored when --stdin is specified)", + ) + parser.add_argument( + "APPID:VERCODE", + nargs="*", + help=_("app id and version code tuple 'APPID:VERCODE'"), + ) + options = common.parse_args(parser) + + try: + build_list = [ + common.split_pkg_arg(x) for x in options.__dict__['APPID:VERCODE'] + ] + send_buildcycle_wrapper( + build_list=build_list, read_stdin=options.stdin, timeout=options.timeout + ) + except Exception as e: + if options.verbose: + logging.error(traceback.format_exc()) + else: + logging.error(e) + sys.exit(1) + + +if __name__ == "__main__": + main()