mirror of
https://github.com/f-droid/fdroidserver.git
synced 2026-03-21 16:31:42 -04:00
055759cf76
Initially, the scanner used libmagic which used magic numbers in the file's content to detect what kind of file it appears to be. Since that library isn't available on all systems, we added support for two other libraries, mimetypes amongst them. The issue with mimetypes is that it only uses the file's extension, not its actual content. So this ends in variable behaviour depending on what system you're using fdroidserver on. For example, an executable binary without extension would be ignored if mimetypes was being used. We now drop all libraries - mimetypes too as it depends on the system's mime.types file - and instead check extensions ourselves. On top of that, do a simple binary content check to find binary executables that don't have an extension. The new in-house code without any dependencies doesn't add any new checks, so no builds should break. The current checks still work: % fdroid scanner app.openconnect:1029 [...] Found executable binary at assets/raw/armeabi/curl Found executable binary at assets/raw/mips/curl Found executable binary at assets/raw/x86/curl Found JAR file at lib/XposedBridgeApi-54.jar Found JAR file at libs/acra-4.5.0.jar Found JAR file at libs/openconnect-wrapper.jar Found JAR file at libs/stoken-wrapper.jar Found shared library at libs/armeabi/libopenconnect.so Found shared library at libs/armeabi/libstoken.so Found shared library at libs/mips/libopenconnect.so Found shared library at libs/mips/libstoken.so Found shared library at libs/x86/libopenconnect.so Found shared library at libs/x86/libstoken.so
268 lines
9.1 KiB
Python
268 lines
9.1 KiB
Python
#!/usr/bin/env python2
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# scanner.py - part of the FDroid server tools
|
|
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import os
|
|
import re
|
|
import traceback
|
|
from argparse import ArgumentParser
|
|
import logging
|
|
|
|
import common
|
|
import metadata
|
|
from common import BuildException, VCSException
|
|
|
|
config = None
|
|
options = None
|
|
|
|
|
|
# Scan the source code in the given directory (and all subdirectories)
|
|
# and return the number of fatal problems encountered
|
|
def scan_source(build_dir, root_dir, thisbuild):
|
|
|
|
count = 0
|
|
|
|
# Common known non-free blobs (always lower case):
|
|
usual_suspects = [
|
|
re.compile(r'.*flurryagent', re.IGNORECASE),
|
|
re.compile(r'.*paypal.*mpl', re.IGNORECASE),
|
|
re.compile(r'.*google.*analytics', re.IGNORECASE),
|
|
re.compile(r'.*admob.*sdk.*android', re.IGNORECASE),
|
|
re.compile(r'.*google.*ad.*view', re.IGNORECASE),
|
|
re.compile(r'.*google.*admob', re.IGNORECASE),
|
|
re.compile(r'.*google.*play.*services', re.IGNORECASE),
|
|
re.compile(r'.*crittercism', re.IGNORECASE),
|
|
re.compile(r'.*heyzap', re.IGNORECASE),
|
|
re.compile(r'.*jpct.*ae', re.IGNORECASE),
|
|
re.compile(r'.*youtube.*android.*player.*api', re.IGNORECASE),
|
|
re.compile(r'.*bugsense', re.IGNORECASE),
|
|
re.compile(r'.*crashlytics', re.IGNORECASE),
|
|
re.compile(r'.*ouya.*sdk', re.IGNORECASE),
|
|
re.compile(r'.*libspen23', re.IGNORECASE),
|
|
]
|
|
|
|
scanignore = common.getpaths(build_dir, thisbuild, 'scanignore')
|
|
scandelete = common.getpaths(build_dir, thisbuild, 'scandelete')
|
|
|
|
scanignore_worked = set()
|
|
scandelete_worked = set()
|
|
|
|
def toignore(fd):
|
|
for p in scanignore:
|
|
if fd.startswith(p):
|
|
scanignore_worked.add(p)
|
|
return True
|
|
return False
|
|
|
|
def todelete(fd):
|
|
for p in scandelete:
|
|
if fd.startswith(p):
|
|
scandelete_worked.add(p)
|
|
return True
|
|
return False
|
|
|
|
def ignoreproblem(what, fd, fp):
|
|
logging.info('Ignoring %s at %s' % (what, fd))
|
|
return 0
|
|
|
|
def removeproblem(what, fd, fp):
|
|
logging.info('Removing %s at %s' % (what, fd))
|
|
os.remove(fp)
|
|
return 0
|
|
|
|
def warnproblem(what, fd):
|
|
logging.warn('Found %s at %s' % (what, fd))
|
|
|
|
def handleproblem(what, fd, fp):
|
|
if toignore(fd):
|
|
return ignoreproblem(what, fd, fp)
|
|
if todelete(fd):
|
|
return removeproblem(what, fd, fp)
|
|
logging.error('Found %s at %s' % (what, fd))
|
|
return 1
|
|
|
|
def is_executable(path):
|
|
return os.path.exists(path) and os.access(path, os.X_OK)
|
|
|
|
textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
|
|
|
|
def is_binary(path):
|
|
d = None
|
|
with open(path, 'rb') as f:
|
|
d = f.read(1024)
|
|
return bool(d.translate(None, textchars))
|
|
|
|
# Iterate through all files in the source code
|
|
for r, d, f in os.walk(build_dir, topdown=True):
|
|
|
|
# It's topdown, so checking the basename is enough
|
|
for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
|
|
if ignoredir in d:
|
|
d.remove(ignoredir)
|
|
|
|
for curfile in f:
|
|
|
|
# Path (relative) to the file
|
|
fp = os.path.join(r, curfile)
|
|
fd = fp[len(build_dir) + 1:]
|
|
|
|
ext = common.get_extension(fd)
|
|
|
|
if ext == 'so':
|
|
count += handleproblem('shared library', fd, fp)
|
|
elif ext == 'a':
|
|
count += handleproblem('static library', fd, fp)
|
|
elif ext == 'class':
|
|
count += handleproblem('Java compiled class', fd, fp)
|
|
elif ext == 'apk':
|
|
removeproblem('APK file', fd, fp)
|
|
|
|
elif ext == 'jar':
|
|
if any(suspect.match(curfile) for suspect in usual_suspects):
|
|
count += handleproblem('usual supect', fd, fp)
|
|
else:
|
|
warnproblem('JAR file', fd)
|
|
|
|
elif ext == 'java':
|
|
if not os.path.isfile(fp):
|
|
continue
|
|
for line in file(fp):
|
|
if 'DexClassLoader' in line:
|
|
count += handleproblem('DexClassLoader', fd, fp)
|
|
break
|
|
|
|
elif ext == 'gradle':
|
|
if not os.path.isfile(fp):
|
|
continue
|
|
for i, line in enumerate(file(fp)):
|
|
i = i + 1
|
|
if any(suspect.match(line) for suspect in usual_suspects):
|
|
count += handleproblem('usual suspect at line %d' % i, fd, fp)
|
|
break
|
|
|
|
elif is_binary(fp):
|
|
if is_executable(fp):
|
|
count += handleproblem('executable binary', fd, fp)
|
|
elif ext == '':
|
|
count += handleproblem('unknown binary', fd, fp)
|
|
|
|
for p in scanignore:
|
|
if p not in scanignore_worked:
|
|
logging.error('Unused scanignore path: %s' % p)
|
|
count += 1
|
|
|
|
for p in scandelete:
|
|
if p not in scandelete_worked:
|
|
logging.error('Unused scandelete path: %s' % p)
|
|
count += 1
|
|
|
|
# Presence of a jni directory without buildjni=yes might
|
|
# indicate a problem (if it's not a problem, explicitly use
|
|
# buildjni=no to bypass this check)
|
|
if (os.path.exists(os.path.join(root_dir, 'jni')) and
|
|
not thisbuild['buildjni']):
|
|
logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
|
|
count += 1
|
|
|
|
return count
|
|
|
|
|
|
def main():
|
|
|
|
global config, options
|
|
|
|
# Parse command line...
|
|
parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
|
|
common.setup_global_opts(parser)
|
|
parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
|
|
options = parser.parse_args()
|
|
|
|
config = common.read_config(options)
|
|
|
|
# Read all app and srclib metadata
|
|
allapps = metadata.read_metadata()
|
|
apps = common.read_app_args(options.appid, allapps, True)
|
|
|
|
probcount = 0
|
|
|
|
build_dir = 'build'
|
|
if not os.path.isdir(build_dir):
|
|
logging.info("Creating build directory")
|
|
os.makedirs(build_dir)
|
|
srclib_dir = os.path.join(build_dir, 'srclib')
|
|
extlib_dir = os.path.join(build_dir, 'extlib')
|
|
|
|
for appid, app in apps.iteritems():
|
|
|
|
if app['Disabled']:
|
|
logging.info("Skipping %s: disabled" % appid)
|
|
continue
|
|
if not app['builds']:
|
|
logging.info("Skipping %s: no builds specified" % appid)
|
|
continue
|
|
|
|
logging.info("Processing " + appid)
|
|
|
|
try:
|
|
|
|
if app['Repo Type'] == 'srclib':
|
|
build_dir = os.path.join('build', 'srclib', app['Repo'])
|
|
else:
|
|
build_dir = os.path.join('build', appid)
|
|
|
|
# Set up vcs interface and make sure we have the latest code...
|
|
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
|
|
|
|
for thisbuild in app['builds']:
|
|
|
|
if thisbuild['disable']:
|
|
logging.info("...skipping version %s - %s" % (
|
|
thisbuild['version'], thisbuild.get('disable', thisbuild['commit'][1:])))
|
|
else:
|
|
logging.info("...scanning version " + thisbuild['version'])
|
|
|
|
# Prepare the source code...
|
|
root_dir, _ = common.prepare_source(vcs, app, thisbuild,
|
|
build_dir, srclib_dir,
|
|
extlib_dir, False)
|
|
|
|
# Do the scan...
|
|
count = scan_source(build_dir, root_dir, thisbuild)
|
|
if count > 0:
|
|
logging.warn('Scanner found %d problems in %s (%s)' % (
|
|
count, appid, thisbuild['vercode']))
|
|
probcount += count
|
|
|
|
except BuildException as be:
|
|
logging.warn("Could not scan app %s due to BuildException: %s" % (
|
|
appid, be))
|
|
probcount += 1
|
|
except VCSException as vcse:
|
|
logging.warn("VCS error while scanning app %s: %s" % (appid, vcse))
|
|
probcount += 1
|
|
except Exception:
|
|
logging.warn("Could not scan app %s due to unknown error: %s" % (
|
|
appid, traceback.format_exc()))
|
|
probcount += 1
|
|
|
|
logging.info("Finished:")
|
|
print "%d problems found" % probcount
|
|
|
|
if __name__ == "__main__":
|
|
main()
|