build: set open file limit based on how many apps are being processed

When running `fdroid build --all` on a buildserver with thousands of apps,
it was frequently hitting the open file limit.  This increases the open
file limit based on how many apps are being process.  It is doubled to
provide a margin of safety.

There are probably open file leaks which ideally would be fixed, but this
is also useful to make things more resilient to all the random stuff apps
include in their build systems.

fdroidserver/build.py

@@ -23,6 +23,7 @@ import shutil
 import glob
 import subprocess
 import re
+import resource
 import tarfile
 import traceback
 import time
@@ -1120,6 +1121,19 @@ def main():
     if not apps:
         raise FDroidException("No apps to process.")
 
+    # make sure enough open files are allowed to process everything
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    if len(apps) > soft:
+        try:
+            soft = len(apps) * 2
+            if soft > hard:
+                soft = hard
+            resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
+            logging.debug(_('Set open file limit to {integer}')
+                          .format(integer=soft))
+        except (OSError, ValueError) as e:
+            logging.warning(_('Setting open file limit failed: ') + str(e))
+
     if options.latest:
         for app in apps.values():
             for build in reversed(app.builds):

jenkins-build-all

@@ -31,7 +31,6 @@ else
         echo "No virtualization is used."
 fi
 sudo /bin/chmod -R a+rX /var/lib/libvirt/images
-ulimit -n 2048
 echo 'maximum allowed number of open file descriptors: ' `ulimit -n`
 ls -ld /var/lib/libvirt/images
 ls -l /var/lib/libvirt/images || echo no access