Remove /opt/gradle related code

2025-12-23 23:57:51 -05:00 · 2025-11-17 23:03:41 +08:00
parent 47edd38d3d
commit b47ceb648b
11 changed files with 14 additions and 240 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -493,23 +493,6 @@ macOS:
    - fdroid="$(brew --prefix fdroidserver)/libexec/bin/python3 $PWD/fdroid" ./tests/run-tests


-gradle:
-  image: debian:trixie-slim
-  <<: *apt-template
-  rules:
-    - changes:
-        - .gitlab-ci.yml
-        - makebuildserver
-  script:
-    - apt-get install
-          ca-certificates
-          git
-          python3-colorama
-          python3-packaging
-          python3-requests
-    - ./tests/gradle-release-checksums.py
-
-
 # Run an actual build in a simple, faked version of the buildserver guest VM.
 fdroid build:
  image: registry.gitlab.com/fdroid/fdroidserver:buildserver
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -18,7 +18,6 @@ include examples/opensc-fdroid.cfg
 include examples/public-read-only-s3-bucket-policy.json
 include examples/template.yml
 include examples/Vagrantfile.yaml
-include gradlew-fdroid
 include LICENSE
 include locale/ba/LC_MESSAGES/fdroidserver.po
 include locale/bo/LC_MESSAGES/fdroidserver.po
@@ -586,7 +585,6 @@ include tests/gnupghome/random_seed
 include tests/gnupghome/secring.gpg
 include tests/gnupghome/trustdb.gpg
 include tests/gradle-maven-blocks.yaml
-include tests/gradle-release-checksums.py
 include tests/IsMD5Disabled.java
 include tests/issue-1128-min-sdk-30-poc.apk
 include tests/issue-1128-poc1.apk
@@ -860,7 +858,6 @@ include tests/test_checkupdates.py
 include tests/test_common.py
 include tests/test_deploy.py
 include tests/test_exception.py
-include tests/test_gradlew-fdroid
 include tests/test_import_subcommand.py
 include tests/test_index.py
 include tests/test_init.py
--- a/buildserver/config.buildserver.yml
+++ b/buildserver/config.buildserver.yml
@@ -1,2 +1 @@
 sdk_path: /opt/android-sdk
-gradle_version_dir: /opt/gradle/versions
--- a/buildserver/provision-gradle
+++ b/buildserver/provision-gradle
@@ -2,39 +2,11 @@

 set -ex

-
-# version compare magic
-vergte() {
-    printf '%s\n%s' "$1" "$2" | sort -C -V -r
-}
-
-test -e /opt/gradle/versions || mkdir -p /opt/gradle/versions
-cd /opt/gradle/versions
-
-glob="/vagrant/cache/gradle-*.zip"
-if compgen -G $glob; then # test if glob matches anything
-    f=$(ls -1 --sort=version --group-directories-first $glob | tail -1)
-    ver=`echo $f | sed 's,.*gradle-\([0-9][0-9.]*\).*\.zip,\1,'`
-    # only use versions greater or equal 2.2.1
-    if vergte $ver 2.2.1 && [ ! -d /opt/gradle/versions/${ver} ]; then
-        unzip -qq $f
-        mv gradle-${ver} /opt/gradle/versions/${ver}
-    fi
-fi
-
-chmod -R a+rX /opt/gradle
-
-test -e /opt/gradle/bin || mkdir -p /opt/gradle/bin
 git clone --depth 1 https://gitlab.com/fdroid/gradlew-fdroid.git /home/vagrant/gradlew-fdroid/
 chmod 0755 /home/vagrant/gradlew-fdroid/gradlew-fdroid
 chmod -R u+rwX,a+rX,go-w /home/vagrant/gradlew-fdroid/
-ln -fs /home/vagrant/gradlew-fdroid/gradlew-fdroid /opt/gradle/bin/gradle
 ln -fs /home/vagrant/gradlew-fdroid/gradlew-fdroid /usr/local/bin/

-chown -h vagrant:vagrant /opt/gradle/bin/gradle
-chown vagrant:vagrant /opt/gradle/versions
-chmod 0755 /opt/gradle/versions
-
 GRADLE_HOME=/home/vagrant/.gradle
 test -d $GRADLE_HOME/ || mkdir $GRADLE_HOME/
 cat <<EOF > $GRADLE_HOME/gradle.properties
--- a/fdroidserver/build.py
+++ b/fdroidserver/build.py
@@ -548,7 +548,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
            cmd += ['-P' + kv for kv in build.gradleprops]

        cmd += ['clean']
-        p = FDroidPopen(cmd, cwd=root_dir, envs={"GRADLE_VERSION_DIR": config['gradle_version_dir'], "CACHEDIR": config['cachedir']})
+        p = FDroidPopen(cmd, cwd=root_dir, envs={"CACHEDIR": config['cachedir']})

    elif bmethod == 'ant':
        logging.info("Cleaning Ant project...")
@@ -701,7 +701,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext

        cmd += gradletasks

-        p = FDroidPopen(cmd, cwd=root_dir, envs={"GRADLE_VERSION_DIR": config['gradle_version_dir'], "CACHEDIR": config['cachedir']})
+        p = FDroidPopen(cmd, cwd=root_dir, envs={"CACHEDIR": config['cachedir']})

    elif bmethod == 'ant':
        logging.info("Building Ant project...")
--- a/fdroidserver/build_local_run.py
+++ b/fdroidserver/build_local_run.py
@@ -176,7 +176,6 @@ def init_build(app, build, config):
            cmd,
            cwd=root_dir,
            envs={
-                "GRADLE_VERSION_DIR": config['gradle_version_dir'],
                "CACHEDIR": config['cachedir'],
            },
        )
@@ -420,7 +419,6 @@ def execute_build(app, build, config, gradletasks):
            cmd,
            cwd=root_dir,
            envs={
-                "GRADLE_VERSION_DIR": config['gradle_version_dir'],
                "CACHEDIR": config['cachedir'],
            },
        )
@@ -588,7 +586,7 @@ def execute_postbuild(app, build, src):

        if p.returncode != 0:
            raise exception.BuildException(
-                "Error running postbuild command for " f"{app.id}:{build.versionName}",
+                f"Error running postbuild command for {app.id}:{build.versionName}",
                p.output,
            )

--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -504,8 +504,6 @@ def fill_config_defaults(thisconfig):

    if 'cachedir_scanner' not in thisconfig:
        thisconfig['cachedir_scanner'] = str(Path(thisconfig['cachedir']) / 'scanner')
-    if 'gradle_version_dir' not in thisconfig:
-        thisconfig['gradle_version_dir'] = str(Path(thisconfig['cachedir']) / 'gradle')


 def get_config():
--- a/hooks/pre-commit
+++ b/hooks/pre-commit
@@ -8,7 +8,7 @@ exec 1>&2

 files=`git diff-index --cached HEAD 2>&1 | sed 's/^:.*     //' | uniq | cut -b100-500`
 if [ -z "$files" ]; then
-    PY_FILES="fdroid makebuildserver setup.py fdroidserver/*.py examples/*.py tests/*-release-checksums.py"
+    PY_FILES="fdroid makebuildserver setup.py fdroidserver/*.py examples/*.py"
    PY_TEST_FILES="tests/test_*.py"
    SH_FILES="hooks/pre-commit"
    BASH_FILES="jenkins-build-all jenkins-setup-build-environment jenkins-test completion/bash-completion buildserver/provision-*"
--- a/125
+++ b/125
@@ -1,23 +1,22 @@
 #!/usr/bin/env python3

-import os
-import re
-import requests
-import stat
-import sys
-import shutil
-import subprocess
-import vagrant
 import hashlib
-import yaml
 import json
 import logging
-from clint.textui import progress
+import os
+import re
+import shutil
+import stat
+import subprocess
+import sys
 from optparse import OptionParser
+
+import vagrant
+import yaml
+
 import fdroidserver.tail
 import fdroidserver.vmtools

-
 parser = OptionParser()
 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
                  help="Spew out even more information than normal")
@@ -26,11 +25,6 @@ parser.add_option("-c", "--clean", action="store_true", default=False,
                  help="Build from scratch, rather than attempting to update the existing server")
 parser.add_option('--skip-box-verification', action="store_true", default=False,
                  help="""Skip verifying the downloaded base box.""")
-parser.add_option('--skip-cache-update', action="store_true", default=False,
-                  help="""Skip downloading and checking cache."""
-                       """This assumes that the cache is already downloaded completely.""")
-parser.add_option('--copy-caches-from-host', action="store_true", default=False,
-                  help="""Copy gradle and pip caches into the buildserver VM""")
 parser.add_option('--keep-box-file', action="store_true", default=False,
                  help="""Box file will not be deleted after adding it to box storage"""
                       """ (KVM-only).""")
@@ -114,13 +108,6 @@ if config['vm_provider'] == 'libvirt':
        tmp = os.path.dirname(tmp)
    logging.debug('cache dir %s is accessible for libvirt vm.', config['cachedir'])

-CACHE_FILES = [
-    ('https://services.gradle.org/distributions/gradle-6.8.3-bin.zip',
-     '7faa7198769f872826c8ef4f1450f839ec27f0b4d5d1e51bade63667cbccd205'),
-    ('https://services.gradle.org/distributions/gradle-7.0.2-bin.zip',
-     '0e46229820205440b48a5501122002842b82886e76af35f0f3a069243dca4b3c'),
-]
-

 def sha256_for_file(path):
    with open(path, 'rb') as f:
@@ -159,64 +146,6 @@ def run_via_vagrant_ssh(v, cmdlist):
    v._run_vagrant_command(['ssh', '-c', cmd])


-def update_cache(cachedir):
-    count_files = 0
-    for srcurl, shasum in CACHE_FILES:
-        filename = os.path.basename(srcurl)
-        local_filename = os.path.join(cachedir, filename)
-        count_files = count_files + 1
-        if os.path.exists(local_filename):
-            if sha256_for_file(local_filename) == shasum:
-                logging.info("\t...shasum verified for '{filename}'\t({filecounter} of {filesum} files)".format(filename=local_filename, filecounter=count_files, filesum=len(CACHE_FILES)))
-                continue
-            local_length = os.path.getsize(local_filename)
-        else:
-            local_length = -1
-
-        resume_header = {}
-        download = True
-
-        try:
-            r = requests.head(srcurl, allow_redirects=True, timeout=60)
-            if r.status_code == 200:
-                content_length = int(r.headers.get('content-length'))
-            else:
-                content_length = local_length  # skip the download
-        except requests.exceptions.RequestException as e:
-            content_length = local_length  # skip the download
-            logging.warn('%s', e)
-
-        if local_length == content_length:
-            download = False
-        elif local_length > content_length:
-            logging.info('deleting corrupt file from cache: %s', local_filename)
-            os.remove(local_filename)
-            logging.info("Downloading %s to cache", filename)
-        elif local_length > -1 and local_length < content_length:
-            logging.info("Resuming download of %s", local_filename)
-            resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
-        else:
-            logging.info("Downloading %s to cache", filename)
-
-        if download:
-            r = requests.get(srcurl, headers=resume_header, stream=True,
-                             allow_redirects=True, timeout=60)
-            content_length = int(r.headers.get('content-length'))
-            with open(local_filename, 'ab') as f:
-                for chunk in progress.bar(r.iter_content(chunk_size=65536),
-                                          expected_size=(content_length / 65536) + 1):
-                    if chunk:  # filter out keep-alive new chunks
-                        f.write(chunk)
-
-        v = sha256_for_file(local_filename)
-        if v == shasum:
-            logging.info("\t...shasum verified for '{filename}'\t({filecounter} of {filesum} files)".format(filename=local_filename, filecounter=count_files, filesum=len(CACHE_FILES)))
-        else:
-            logging.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
-            os.remove(local_filename)
-            sys.exit(1)
-
-
 def debug_log_vagrant_vm(vm_dir, config):
    if options.verbosity >= 3:
        _vagrant_dir = os.path.join(vm_dir, '.vagrant')
@@ -236,11 +165,6 @@ def debug_log_vagrant_vm(vm_dir, config):
 def main():
    global config, tail

-    if options.skip_cache_update:
-        logging.info('skipping cache update and verification...')
-    else:
-        update_cache(config['cachedir'])
-
    # use VirtualBox software virtualization if hardware is not available,
    # like if this is being run in kvm or some other VM platform, like
    # http://jenkins.debian.net, the values are 'on' or 'off'
@@ -331,35 +255,6 @@ def main():
        logging.error("'vagrant up' failed. Consult %s", logfilename)
        sys.exit(1)

-    if options.copy_caches_from_host:
-        ssh_config = v.ssh_config()
-        user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
-        hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
-        port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
-        key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
-
-        for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
-            fullpath = os.path.join(os.getenv('HOME'), d)
-            os.system('date')
-            print('rsyncing', fullpath, 'into VM')
-            if os.path.isdir(fullpath):
-                ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
-                                        '-o StrictHostKeyChecking=no',
-                                        '-o UserKnownHostsFile=/dev/null',
-                                        '-o LogLevel=FATAL',
-                                        '-o IdentitiesOnly=yes',
-                                        '-o PasswordAuthentication=no'))
-                # TODO vagrant 1.5+ provides `vagrant rsync`
-                run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
-                subprocess.call(['rsync', '-ax', '--delete', '-e',
-                                 ssh_command,
-                                 fullpath + '/',
-                                 user + '@' + hostname + ':~/' + d + '/'])
-
-        # this file changes every time but should not be cached
-        run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
-        run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
-
    logging.info("Stopping build server VM")
    v.halt()

--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,7 +42,6 @@ force-exclude = '''(
  | fdroidserver/vmtools\.py
  | tests/config\.py
  | tests/extra/manual-vmtools-test\.py
-  | tests/gradle-release-checksums\.py
  | tests/openssl-version-check-test\.py
  | tests/valid-package-names/test\.py
  | tests/test_common\.py
--- a/tests/gradle-release-checksums.py
+++ b/tests/gradle-release-checksums.py
@@ -1,67 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import re
-import subprocess
-import sys
-
-import requests
-from colorama import Fore, Style
-from packaging.version import Version
-
-checksums = None
-versions = dict()
-
-while not checksums:
-    r = requests.get(
-        'https://gitlab.com/fdroid/gradle-transparency-log/-/raw/master/checksums.json',
-        timeout=300,
-    )
-    if r.status_code == 200:
-        checksums = r.json()
-
-gradle_bin_pat = re.compile(r'gradle-([0-9][0-9.]+[0-9])-bin.zip')
-for url, d in checksums.items():
-    m = gradle_bin_pat.search(url)
-    if m:
-        versions[m.group(1)] = d[0]['sha256']
-
-errors = 0
-makebuildserver = os.path.join(os.path.dirname(__file__), os.pardir, 'makebuildserver')
-with open(makebuildserver) as fp:
-    makebuildserver_current = fp.read()
-to_compile = re.search(r'CACHE_FILES = [^\]]+\]', makebuildserver_current).group()
-code = compile(to_compile, makebuildserver, 'exec')
-config = {}
-exec(code, None, config)  # nosec this is just a CI script
-makebuildserver_versions = []
-version_pat = re.compile(r'[0-9]+(\.[0-9]+)+')
-for url, checksum in config['CACHE_FILES']:
-    if 'gradle.org' in url:
-        m = version_pat.search(url.split('/')[-1])
-        if m:
-            makebuildserver_versions.append(m.group())
-            if checksum != versions[m.group()]:
-                print(Fore.RED
-                      + 'ERROR: checksum mismatch:', checksum, versions[m.group()]
-                      + Style.RESET_ALL)
-                errors += 1
-
-# error if makebuildserver is missing the latest version
-for version in sorted(versions.keys()):
-    if version not in makebuildserver_versions \
-       and Version(version) > Version(sorted(makebuildserver_versions)[-1]):
-        add_before = """    ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',"""
-        new = to_compile.replace(
-            add_before,
-            "    ('https://services.gradle.org/distributions/gradle-" + version + "-bin.zip',\n"
-            "     '" + versions[version] + "'),\n" + add_before
-        )
-        makebuildserver_current = makebuildserver_current.replace(to_compile, new)
-
-with open('makebuildserver', 'w') as fp:
-    fp.write(makebuildserver_current)
-
-p = subprocess.run(['git', '--no-pager', 'diff'])
-errors += p.returncode
-sys.exit(errors)