mirror of
https://github.com/f-droid/fdroidserver.git
synced 2026-03-21 00:11:12 -04:00
d7d9e38ef7
This is much closer to what we did before with mimetypes. Using a whitelist turns out to be a bad idea since repositories seem to be randomly filled with executable images and documents, which trigger the scanner. In an ideal world the scanner would complain about all of those. For now, just warn about the possibility of them being hidden binaries.
275 lines
8.9 KiB
Python
275 lines
8.9 KiB
Python
#!/usr/bin/env python2
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# scanner.py - part of the FDroid server tools
|
|
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import os
|
|
import re
|
|
import traceback
|
|
from argparse import ArgumentParser
|
|
import logging
|
|
|
|
import common
|
|
import metadata
|
|
from common import BuildException, VCSException
|
|
|
|
config = None
|
|
options = None
|
|
|
|
|
|
# Scan the source code in the given directory (and all subdirectories)
|
|
# and return the number of fatal problems encountered
|
|
def scan_source(build_dir, root_dir, thisbuild):
|
|
|
|
count = 0
|
|
|
|
# Common known non-free blobs (always lower case):
|
|
usual_suspects = {
|
|
exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [
|
|
r'flurryagent',
|
|
r'paypal.*mpl',
|
|
r'google.*analytics',
|
|
r'admob.*sdk.*android',
|
|
r'google.*ad.*view',
|
|
r'google.*admob',
|
|
r'google.*play.*services',
|
|
r'crittercism',
|
|
r'heyzap',
|
|
r'jpct.*ae',
|
|
r'youtube.*android.*player.*api',
|
|
r'bugsense',
|
|
r'crashlytics',
|
|
r'ouya.*sdk',
|
|
r'libspen23',
|
|
]
|
|
}
|
|
|
|
def suspects_found(s):
|
|
for n, r in usual_suspects.iteritems():
|
|
if r.match(s):
|
|
yield n
|
|
|
|
scanignore = common.getpaths(build_dir, thisbuild, 'scanignore')
|
|
scandelete = common.getpaths(build_dir, thisbuild, 'scandelete')
|
|
|
|
scanignore_worked = set()
|
|
scandelete_worked = set()
|
|
|
|
def toignore(fd):
|
|
for p in scanignore:
|
|
if fd.startswith(p):
|
|
scanignore_worked.add(p)
|
|
return True
|
|
return False
|
|
|
|
def todelete(fd):
|
|
for p in scandelete:
|
|
if fd.startswith(p):
|
|
scandelete_worked.add(p)
|
|
return True
|
|
return False
|
|
|
|
def ignoreproblem(what, fd, fp):
|
|
logging.info('Ignoring %s at %s' % (what, fd))
|
|
return 0
|
|
|
|
def removeproblem(what, fd, fp):
|
|
logging.info('Removing %s at %s' % (what, fd))
|
|
os.remove(fp)
|
|
return 0
|
|
|
|
def warnproblem(what, fd):
|
|
logging.warn('Found %s at %s' % (what, fd))
|
|
|
|
def handleproblem(what, fd, fp):
|
|
if toignore(fd):
|
|
return ignoreproblem(what, fd, fp)
|
|
if todelete(fd):
|
|
return removeproblem(what, fd, fp)
|
|
logging.error('Found %s at %s' % (what, fd))
|
|
return 1
|
|
|
|
def is_executable(path):
|
|
return os.path.exists(path) and os.access(path, os.X_OK)
|
|
|
|
textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
|
|
|
|
def is_binary(path):
|
|
d = None
|
|
with open(path, 'rb') as f:
|
|
d = f.read(1024)
|
|
return bool(d.translate(None, textchars))
|
|
|
|
# Iterate through all files in the source code
|
|
for r, d, f in os.walk(build_dir, topdown=True):
|
|
|
|
# It's topdown, so checking the basename is enough
|
|
for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
|
|
if ignoredir in d:
|
|
d.remove(ignoredir)
|
|
|
|
for curfile in f:
|
|
|
|
# Path (relative) to the file
|
|
fp = os.path.join(r, curfile)
|
|
fd = fp[len(build_dir) + 1:]
|
|
|
|
ext = common.get_extension(fd)
|
|
|
|
if ext == 'so':
|
|
count += handleproblem('shared library', fd, fp)
|
|
elif ext == 'a':
|
|
count += handleproblem('static library', fd, fp)
|
|
elif ext == 'class':
|
|
count += handleproblem('Java compiled class', fd, fp)
|
|
elif ext == 'apk':
|
|
removeproblem('APK file', fd, fp)
|
|
|
|
elif ext == 'jar':
|
|
for name in suspects_found(curfile):
|
|
count += handleproblem('usual supect \'%s\'' % name, fd, fp)
|
|
warnproblem('JAR file', fd)
|
|
|
|
elif ext == 'java':
|
|
if not os.path.isfile(fp):
|
|
continue
|
|
for line in file(fp):
|
|
if 'DexClassLoader' in line:
|
|
count += handleproblem('DexClassLoader', fd, fp)
|
|
break
|
|
|
|
elif ext == 'gradle':
|
|
if not os.path.isfile(fp):
|
|
continue
|
|
for i, line in enumerate(file(fp)):
|
|
i = i + 1
|
|
for name in suspects_found(line):
|
|
count += handleproblem('usual supect \'%s\' at line %d' % (name, i), fd, fp)
|
|
|
|
elif ext in ['', 'bin', 'out', 'exe']:
|
|
if is_binary(fp):
|
|
count += handleproblem('binary', fd, fp)
|
|
|
|
elif is_executable(fp):
|
|
if is_binary(fp):
|
|
warnproblem('possible binary', fd)
|
|
|
|
for p in scanignore:
|
|
if p not in scanignore_worked:
|
|
logging.error('Unused scanignore path: %s' % p)
|
|
count += 1
|
|
|
|
for p in scandelete:
|
|
if p not in scandelete_worked:
|
|
logging.error('Unused scandelete path: %s' % p)
|
|
count += 1
|
|
|
|
# Presence of a jni directory without buildjni=yes might
|
|
# indicate a problem (if it's not a problem, explicitly use
|
|
# buildjni=no to bypass this check)
|
|
if (os.path.exists(os.path.join(root_dir, 'jni')) and
|
|
not thisbuild['buildjni']):
|
|
logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
|
|
count += 1
|
|
|
|
return count
|
|
|
|
|
|
def main():
|
|
|
|
global config, options
|
|
|
|
# Parse command line...
|
|
parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
|
|
common.setup_global_opts(parser)
|
|
parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
|
|
options = parser.parse_args()
|
|
|
|
config = common.read_config(options)
|
|
|
|
# Read all app and srclib metadata
|
|
allapps = metadata.read_metadata()
|
|
apps = common.read_app_args(options.appid, allapps, True)
|
|
|
|
probcount = 0
|
|
|
|
build_dir = 'build'
|
|
if not os.path.isdir(build_dir):
|
|
logging.info("Creating build directory")
|
|
os.makedirs(build_dir)
|
|
srclib_dir = os.path.join(build_dir, 'srclib')
|
|
extlib_dir = os.path.join(build_dir, 'extlib')
|
|
|
|
for appid, app in apps.iteritems():
|
|
|
|
if app['Disabled']:
|
|
logging.info("Skipping %s: disabled" % appid)
|
|
continue
|
|
if not app['builds']:
|
|
logging.info("Skipping %s: no builds specified" % appid)
|
|
continue
|
|
|
|
logging.info("Processing " + appid)
|
|
|
|
try:
|
|
|
|
if app['Repo Type'] == 'srclib':
|
|
build_dir = os.path.join('build', 'srclib', app['Repo'])
|
|
else:
|
|
build_dir = os.path.join('build', appid)
|
|
|
|
# Set up vcs interface and make sure we have the latest code...
|
|
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
|
|
|
|
for thisbuild in app['builds']:
|
|
|
|
if thisbuild['disable']:
|
|
logging.info("...skipping version %s - %s" % (
|
|
thisbuild['version'], thisbuild.get('disable', thisbuild['commit'][1:])))
|
|
else:
|
|
logging.info("...scanning version " + thisbuild['version'])
|
|
|
|
# Prepare the source code...
|
|
root_dir, _ = common.prepare_source(vcs, app, thisbuild,
|
|
build_dir, srclib_dir,
|
|
extlib_dir, False)
|
|
|
|
# Do the scan...
|
|
count = scan_source(build_dir, root_dir, thisbuild)
|
|
if count > 0:
|
|
logging.warn('Scanner found %d problems in %s (%s)' % (
|
|
count, appid, thisbuild['vercode']))
|
|
probcount += count
|
|
|
|
except BuildException as be:
|
|
logging.warn("Could not scan app %s due to BuildException: %s" % (
|
|
appid, be))
|
|
probcount += 1
|
|
except VCSException as vcse:
|
|
logging.warn("VCS error while scanning app %s: %s" % (appid, vcse))
|
|
probcount += 1
|
|
except Exception:
|
|
logging.warn("Could not scan app %s due to unknown error: %s" % (
|
|
appid, traceback.format_exc()))
|
|
probcount += 1
|
|
|
|
logging.info("Finished:")
|
|
print "%d problems found" % probcount
|
|
|
|
if __name__ == "__main__":
|
|
main()
|