mirror of
https://github.com/f-droid/fdroidserver.git
synced 2026-03-14 21:10:33 -04:00
24af895915
Without this, we get bunch of messages at the start of build, which end with "Applying patch", and then all the things hangs for several minutes (or more, in case of network problems, etc). So, consistently keep user aware od what's happening. Note that --verbose switch affects deatiledness and amount of output (in particular, if it's given, "build" command output goes to console instead of log).
1654 lines
60 KiB
Python
1654 lines
60 KiB
Python
# -*- coding: utf-8 -*-
|
|
#
|
|
# common.py - part of the FDroid server tools
|
|
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import glob, os, sys, re
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
import operator
|
|
import cgi
|
|
|
|
def getvcs(vcstype, remote, local, sdk_path):
|
|
if vcstype == 'git':
|
|
return vcs_git(remote, local, sdk_path)
|
|
if vcstype == 'svn':
|
|
return vcs_svn(remote, local, sdk_path)
|
|
if vcstype == 'git-svn':
|
|
return vcs_gitsvn(remote, local, sdk_path)
|
|
if vcstype == 'hg':
|
|
return vcs_hg(remote, local, sdk_path)
|
|
if vcstype == 'bzr':
|
|
return vcs_bzr(remote, local, sdk_path)
|
|
if vcstype == 'srclib':
|
|
if local != 'build/srclib/' + remote:
|
|
raise VCSException("Error: srclib paths are hard-coded!")
|
|
return getsrclib(remote, 'build/srclib', sdk_path, raw=True)
|
|
raise VCSException("Invalid vcs type " + vcstype)
|
|
|
|
def getsrclibvcs(name):
|
|
srclib_path = os.path.join('srclibs', name + ".txt")
|
|
if not os.path.exists(srclib_path):
|
|
raise VCSException("Missing srclib " + name)
|
|
return parse_srclib(srclib_path)['Repo Type']
|
|
|
|
class vcs:
|
|
def __init__(self, remote, local, sdk_path):
|
|
|
|
self.sdk_path = sdk_path
|
|
|
|
# It's possible to sneak a username and password in with
|
|
# the remote address... (this really only applies to svn
|
|
# and we should probably be more specific!)
|
|
index = remote.find('@')
|
|
if index != -1:
|
|
self.username = remote[:index]
|
|
remote = remote[index+1:]
|
|
index = self.username.find(':')
|
|
if index == -1:
|
|
raise VCSException("Password required with username")
|
|
self.password = self.username[index+1:]
|
|
self.username = self.username[:index]
|
|
else:
|
|
self.username = None
|
|
|
|
self.remote = remote
|
|
self.local = local
|
|
self.refreshed = False
|
|
self.srclib = None
|
|
|
|
# Take the local repository to a clean version of the given revision, which
|
|
# is specificed in the VCS's native format. Beforehand, the repository can
|
|
# be dirty, or even non-existent. If the repository does already exist
|
|
# locally, it will be updated from the origin, but only once in the
|
|
# lifetime of the vcs object.
|
|
# None is acceptable for 'rev' if you know you are cloning a clean copy of
|
|
# the repo - otherwise it must specify a valid revision.
|
|
def gotorevision(self, rev):
|
|
|
|
# The .fdroidvcs-id file for a repo tells us what VCS type
|
|
# and remote that directory was created from, allowing us to drop it
|
|
# automatically if either of those things changes.
|
|
fdpath = os.path.join(self.local, '..',
|
|
'.fdroidvcs-' + os.path.basename(self.local))
|
|
cdata = self.repotype() + ' ' + self.remote
|
|
writeback = True
|
|
deleterepo = False
|
|
if os.path.exists(self.local):
|
|
if os.path.exists(fdpath):
|
|
with open(fdpath, 'r') as f:
|
|
fsdata = f.read()
|
|
if fsdata == cdata:
|
|
writeback = False
|
|
else:
|
|
deleterepo = True
|
|
print "*** Repository details changed - deleting ***"
|
|
else:
|
|
deleterepo = True
|
|
print "*** Repository details missing - deleting ***"
|
|
if deleterepo:
|
|
shutil.rmtree(self.local)
|
|
|
|
self.gotorevisionx(rev)
|
|
|
|
# If necessary, write the .fdroidvcs file.
|
|
if writeback:
|
|
with open(fdpath, 'w') as f:
|
|
f.write(cdata)
|
|
|
|
# Derived classes need to implement this. It's called once basic checking
|
|
# has been performend.
|
|
def gotorevisionx(self, rev):
|
|
raise VCSException("This VCS type doesn't define gotorevisionx")
|
|
|
|
# Initialise and update submodules
|
|
def initsubmodules(self):
|
|
raise VCSException('Submodules not supported for this vcs type')
|
|
|
|
# Get a list of all known tags
|
|
def gettags(self):
|
|
raise VCSException('gettags not supported for this vcs type')
|
|
|
|
# Returns the srclib (name, path) used in setting up the current
|
|
# revision, or None.
|
|
def getsrclib(self):
|
|
return self.srclib
|
|
|
|
class vcs_git(vcs):
|
|
|
|
def repotype(self):
|
|
return 'git'
|
|
|
|
# If the local directory exists, but is somehow not a git repository, git
|
|
# will traverse up the directory tree until it finds one that is (i.e.
|
|
# fdroidserver) and then we'll proceed to destroy it! This is called as
|
|
# a safety check.
|
|
def checkrepo(self):
|
|
p = subprocess.Popen(['git', 'rev-parse', '--show-toplevel'],
|
|
stdout=subprocess.PIPE, cwd=self.local)
|
|
result = p.communicate()[0].rstrip()
|
|
if not result.endswith(self.local):
|
|
raise VCSException('Repository mismatch')
|
|
|
|
def gotorevisionx(self, rev):
|
|
if not os.path.exists(self.local):
|
|
# Brand new checkout...
|
|
if subprocess.call(['git', 'clone', self.remote, self.local]) != 0:
|
|
raise VCSException("Git clone failed")
|
|
self.checkrepo()
|
|
else:
|
|
self.checkrepo()
|
|
# Discard any working tree changes...
|
|
if subprocess.call(['git', 'reset', '--hard'], cwd=self.local) != 0:
|
|
raise VCSException("Git reset failed")
|
|
# Remove untracked files now, in case they're tracked in the target
|
|
# revision (it happens!)...
|
|
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
|
|
raise VCSException("Git clean failed")
|
|
if not self.refreshed:
|
|
# Get latest commits and tags from remote...
|
|
if subprocess.call(['git', 'fetch', 'origin'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Git fetch failed")
|
|
if subprocess.call(['git', 'fetch', '--tags', 'origin'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Git fetch failed")
|
|
self.refreshed = True
|
|
# Check out the appropriate revision...
|
|
if rev:
|
|
if subprocess.call(['git', 'checkout', rev], cwd=self.local) != 0:
|
|
raise VCSException("Git checkout failed")
|
|
# Get rid of any uncontrolled files left behind...
|
|
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
|
|
raise VCSException("Git clean failed")
|
|
|
|
def initsubmodules(self):
|
|
self.checkrepo()
|
|
if subprocess.call(['git', 'submodule', 'init'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Git submodule init failed")
|
|
if subprocess.call(['git', 'submodule', 'update'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Git submodule update failed")
|
|
|
|
def gettags(self):
|
|
self.checkrepo()
|
|
p = subprocess.Popen(['git', 'tag'],
|
|
stdout=subprocess.PIPE, cwd=self.local)
|
|
return p.communicate()[0].splitlines()
|
|
|
|
|
|
class vcs_gitsvn(vcs):
|
|
|
|
def repotype(self):
|
|
return 'git-svn'
|
|
|
|
# If the local directory exists, but is somehow not a git repository, git
|
|
# will traverse up the directory tree until it finds one that is (i.e.
|
|
# fdroidserver) and then we'll proceed to destory it! This is called as
|
|
# a safety check.
|
|
def checkrepo(self):
|
|
p = subprocess.Popen(['git', 'rev-parse', '--show-toplevel'],
|
|
stdout=subprocess.PIPE, cwd=self.local)
|
|
result = p.communicate()[0].rstrip()
|
|
if not result.endswith(self.local):
|
|
raise VCSException('Repository mismatch')
|
|
|
|
def gotorevisionx(self, rev):
|
|
if not os.path.exists(self.local):
|
|
# Brand new checkout...
|
|
gitsvn_cmd = ['git', 'svn', 'clone']
|
|
remote_split = self.remote.split(';')
|
|
if len(remote_split) > 1:
|
|
for i in remote_split[1:]:
|
|
if i.startswith('trunk='):
|
|
gitsvn_cmd += ['-T', i[6:]]
|
|
elif i.startswith('tags='):
|
|
gitsvn_cmd += ['-t', i[5:]]
|
|
elif i.startswith('branches='):
|
|
gitsvn_cmd += ['-b', i[9:]]
|
|
if subprocess.call(gitsvn_cmd + [remote_split[0], self.local]) != 0:
|
|
raise VCSException("Git clone failed")
|
|
else:
|
|
if subprocess.call(gitsvn_cmd + [self.remote, self.local]) != 0:
|
|
raise VCSException("Git clone failed")
|
|
self.checkrepo()
|
|
else:
|
|
self.checkrepo()
|
|
# Discard any working tree changes...
|
|
if subprocess.call(['git', 'reset', '--hard'], cwd=self.local) != 0:
|
|
raise VCSException("Git reset failed")
|
|
# Remove untracked files now, in case they're tracked in the target
|
|
# revision (it happens!)...
|
|
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
|
|
raise VCSException("Git clean failed")
|
|
if not self.refreshed:
|
|
# Get new commits and tags from repo...
|
|
if subprocess.call(['git', 'svn', 'rebase'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Git svn rebase failed")
|
|
self.refreshed = True
|
|
if rev:
|
|
nospaces_rev = rev.replace(' ', '%20')
|
|
# Try finding a svn tag
|
|
p = subprocess.Popen(['git', 'checkout', 'tags/' + nospaces_rev],
|
|
cwd=self.local, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
out, err = p.communicate()
|
|
if p.returncode == 0:
|
|
print out
|
|
else:
|
|
# No tag found, normal svn rev translation
|
|
# Translate svn rev into git format
|
|
p = subprocess.Popen(['git', 'svn', 'find-rev', 'r' + rev],
|
|
cwd=self.local, stdout=subprocess.PIPE)
|
|
git_rev = p.communicate()[0].rstrip()
|
|
if p.returncode != 0 or len(git_rev) == 0:
|
|
# Try a plain git checkout as a last resort
|
|
p = subprocess.Popen(['git', 'checkout', rev], cwd=self.local,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
out, err = p.communicate()
|
|
if p.returncode == 0:
|
|
print out
|
|
else:
|
|
raise VCSException("No git treeish found and direct git checkout failed")
|
|
else:
|
|
# Check out the git rev equivalent to the svn rev
|
|
p = subprocess.Popen(['git', 'checkout', git_rev], cwd=self.local,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
out, err = p.communicate()
|
|
if p.returncode == 0:
|
|
print out
|
|
else:
|
|
raise VCSException("Git svn checkout failed")
|
|
# Get rid of any uncontrolled files left behind...
|
|
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
|
|
raise VCSException("Git clean failed")
|
|
|
|
def gettags(self):
|
|
self.checkrepo()
|
|
return os.listdir(os.path.join(self.local, '.git/svn/refs/remotes/tags'))
|
|
|
|
class vcs_svn(vcs):
|
|
|
|
def repotype(self):
|
|
return 'svn'
|
|
|
|
def userargs(self):
|
|
if self.username is None:
|
|
return ['--non-interactive']
|
|
return ['--username', self.username,
|
|
'--password', self.password,
|
|
'--non-interactive']
|
|
|
|
def gotorevisionx(self, rev):
|
|
if not os.path.exists(self.local):
|
|
if subprocess.call(['svn', 'checkout', self.remote, self.local] +
|
|
self.userargs()) != 0:
|
|
raise VCSException("Svn checkout failed")
|
|
else:
|
|
for svncommand in (
|
|
'svn revert -R .',
|
|
r"svn status | awk '/\?/ {print $2}' | xargs rm -rf"):
|
|
if subprocess.call(svncommand, cwd=self.local,
|
|
shell=True) != 0:
|
|
raise VCSException("Svn reset ({0}) failed in {1}".format(svncommand, self.local))
|
|
if not self.refreshed:
|
|
if subprocess.call(['svn', 'update'] +
|
|
self.userargs(), cwd=self.local) != 0:
|
|
raise VCSException("Svn update failed")
|
|
self.refreshed = True
|
|
if rev:
|
|
revargs = ['-r', rev]
|
|
if subprocess.call(['svn', 'update', '--force'] + revargs +
|
|
self.userargs(), cwd=self.local) != 0:
|
|
raise VCSException("Svn update failed")
|
|
|
|
|
|
class vcs_hg(vcs):
|
|
|
|
def repotype(self):
|
|
return 'hg'
|
|
|
|
def gotorevisionx(self, rev):
|
|
if not os.path.exists(self.local):
|
|
if subprocess.call(['hg', 'clone', self.remote, self.local]) !=0:
|
|
raise VCSException("Hg clone failed")
|
|
else:
|
|
if subprocess.call('hg status -u | xargs rm -rf',
|
|
cwd=self.local, shell=True) != 0:
|
|
raise VCSException("Hg clean failed")
|
|
if not self.refreshed:
|
|
if subprocess.call(['hg', 'pull'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Hg pull failed")
|
|
self.refreshed = True
|
|
if rev:
|
|
revargs = [rev]
|
|
if subprocess.call(['hg', 'checkout', '-C'] + revargs,
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Hg checkout failed")
|
|
|
|
|
|
class vcs_bzr(vcs):
|
|
|
|
def repotype(self):
|
|
return 'bzr'
|
|
|
|
def gotorevisionx(self, rev):
|
|
if not os.path.exists(self.local):
|
|
if subprocess.call(['bzr', 'branch', self.remote, self.local]) != 0:
|
|
raise VCSException("Bzr branch failed")
|
|
else:
|
|
if subprocess.call(['bzr', 'clean-tree', '--force',
|
|
'--unknown', '--ignored'], cwd=self.local) != 0:
|
|
raise VCSException("Bzr revert failed")
|
|
if not self.refreshed:
|
|
if subprocess.call(['bzr', 'pull'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Bzr update failed")
|
|
self.refreshed = True
|
|
if rev:
|
|
revargs = ['-r', rev]
|
|
if subprocess.call(['bzr', 'revert'] + revargs,
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Bzr revert failed")
|
|
|
|
def __init__(self, remote, local, sdk_path):
|
|
|
|
self.sdk_path = sdk_path
|
|
|
|
index = remote.find('@')
|
|
if index != -1:
|
|
self.username = remote[:index]
|
|
remote = remote[index+1:]
|
|
index = self.username.find(':')
|
|
if index == -1:
|
|
raise VCSException("Password required with username")
|
|
self.password = self.username[index+1:]
|
|
self.username = self.username[:index]
|
|
else:
|
|
self.username = None
|
|
|
|
self.remote = remote
|
|
self.local = local
|
|
self.refreshed = False
|
|
self.srclib = None
|
|
|
|
|
|
# Get the type expected for a given metadata field.
|
|
def metafieldtype(name):
|
|
if name == 'Description':
|
|
return 'multiline'
|
|
if name == 'Requires Root':
|
|
return 'flag'
|
|
if name == 'Build Version':
|
|
return 'build'
|
|
if name == 'Use Built':
|
|
return 'obsolete'
|
|
return 'string'
|
|
|
|
|
|
# Parse metadata for a single application.
|
|
#
|
|
# 'metafile' - the filename to read. The package id for the application comes
|
|
# from this filename. Pass None to get a blank entry.
|
|
#
|
|
# Returns a dictionary containing all the details of the application. There are
|
|
# two major kinds of information in the dictionary. Keys beginning with capital
|
|
# letters correspond directory to identically named keys in the metadata file.
|
|
# Keys beginning with lower case letters are generated in one way or another,
|
|
# and are not found verbatim in the metadata.
|
|
#
|
|
# Known keys not originating from the metadata are:
|
|
#
|
|
# 'id' - the application's package ID
|
|
# 'builds' - a list of dictionaries containing build information
|
|
# for each defined build
|
|
# 'comments' - a list of comments from the metadata file. Each is
|
|
# a tuple of the form (field, comment) where field is
|
|
# the name of the field it preceded in the metadata
|
|
# file. Where field is None, the comment goes at the
|
|
# end of the file. Alternatively, 'build:version' is
|
|
# for a comment before a particular build version.
|
|
# 'descriptionlines' - original lines of description as formatted in the
|
|
# metadata file.
|
|
#
|
|
def parse_metadata(metafile, **kw):
|
|
|
|
def parse_buildline(lines):
|
|
value = "".join(lines)
|
|
parts = [p.replace("\\,", ",")
|
|
for p in re.split(r"(?<!\\),", value)]
|
|
if len(parts) < 3:
|
|
raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
|
|
thisbuild = {}
|
|
thisbuild['origlines'] = lines
|
|
thisbuild['version'] = parts[0]
|
|
thisbuild['vercode'] = parts[1]
|
|
try:
|
|
testvercode = int(thisbuild['vercode'])
|
|
except:
|
|
raise MetaDataException("Invalid version code for build in " + metafile.name)
|
|
thisbuild['commit'] = parts[2]
|
|
for p in parts[3:]:
|
|
pk, pv = p.split('=', 1)
|
|
thisbuild[pk] = pv
|
|
return thisbuild
|
|
|
|
def add_comments(key):
|
|
for comment in curcomments:
|
|
thisinfo['comments'].append((key, comment))
|
|
del curcomments[:]
|
|
|
|
thisinfo = {}
|
|
if metafile:
|
|
if not isinstance(metafile, file):
|
|
metafile = open(metafile, "r")
|
|
thisinfo['id'] = metafile.name[9:-4]
|
|
if kw.get("verbose", False):
|
|
print "Reading metadata for " + thisinfo['id']
|
|
else:
|
|
thisinfo['id'] = None
|
|
|
|
# Defaults for fields that come from metadata...
|
|
thisinfo['Name'] = None
|
|
thisinfo['Auto Name'] = ''
|
|
thisinfo['Category'] = 'None'
|
|
thisinfo['Description'] = []
|
|
thisinfo['Summary'] = ''
|
|
thisinfo['License'] = 'Unknown'
|
|
thisinfo['Web Site'] = ''
|
|
thisinfo['Source Code'] = ''
|
|
thisinfo['Issue Tracker'] = ''
|
|
thisinfo['Donate'] = None
|
|
thisinfo['FlattrID'] = None
|
|
thisinfo['Bitcoin'] = None
|
|
thisinfo['Disabled'] = None
|
|
thisinfo['AntiFeatures'] = None
|
|
thisinfo['Update Check Mode'] = 'None'
|
|
thisinfo['Auto Update Mode'] = 'None'
|
|
thisinfo['Current Version'] = ''
|
|
thisinfo['Current Version Code'] = '0'
|
|
thisinfo['Repo Type'] = ''
|
|
thisinfo['Repo'] = ''
|
|
thisinfo['Requires Root'] = False
|
|
thisinfo['No Source Since'] = ''
|
|
|
|
# General defaults...
|
|
thisinfo['builds'] = []
|
|
thisinfo['comments'] = []
|
|
|
|
if metafile is None:
|
|
return thisinfo
|
|
|
|
mode = 0
|
|
buildlines = []
|
|
curcomments = []
|
|
|
|
for line in metafile:
|
|
line = line.rstrip('\r\n')
|
|
if mode == 0:
|
|
if len(line) == 0:
|
|
continue
|
|
if line.startswith("#"):
|
|
curcomments.append(line)
|
|
continue
|
|
index = line.find(':')
|
|
if index == -1:
|
|
raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
|
|
field = line[:index]
|
|
value = line[index+1:]
|
|
|
|
# Translate obsolete fields...
|
|
if field == 'Market Version':
|
|
field = 'Current Version'
|
|
if field == 'Market Version Code':
|
|
field = 'Current Version Code'
|
|
|
|
fieldtype = metafieldtype(field)
|
|
if fieldtype != 'build':
|
|
add_comments(field)
|
|
if fieldtype == 'multiline':
|
|
mode = 1
|
|
thisinfo[field] = []
|
|
if len(value) > 0:
|
|
raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
|
|
elif fieldtype == 'string':
|
|
thisinfo[field] = value
|
|
elif fieldtype == 'flag':
|
|
if value == 'Yes':
|
|
thisinfo[field] = True
|
|
elif value == 'No':
|
|
thisinfo[field] = False
|
|
else:
|
|
raise MetaDataException("Expected Yes or No for " + field + " in " + metafile.name)
|
|
elif fieldtype == 'build':
|
|
if value.endswith("\\"):
|
|
mode = 2
|
|
buildlines = [value[:-1]]
|
|
else:
|
|
thisinfo['builds'].append(parse_buildline([value]))
|
|
add_comments('build:' + thisinfo['builds'][-1]['version'])
|
|
elif fieldtype == 'obsolete':
|
|
pass # Just throw it away!
|
|
else:
|
|
raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
|
|
elif mode == 1: # Multiline field
|
|
if line == '.':
|
|
mode = 0
|
|
else:
|
|
thisinfo[field].append(line)
|
|
elif mode == 2: # Line continuation mode in Build Version
|
|
if line.endswith("\\"):
|
|
buildlines.append(line[:-1])
|
|
else:
|
|
buildlines.append(line)
|
|
thisinfo['builds'].append(
|
|
parse_buildline(buildlines))
|
|
add_comments('build:' + thisinfo['builds'][-1]['version'])
|
|
mode = 0
|
|
add_comments(None)
|
|
|
|
# Mode at end of file should always be 0...
|
|
if mode == 1:
|
|
raise MetaDataException(field + " not terminated in " + metafile.name)
|
|
elif mode == 2:
|
|
raise MetaDataException("Unterminated continuation in " + metafile.name)
|
|
|
|
if len(thisinfo['Description']) == 0:
|
|
thisinfo['Description'].append('No description available')
|
|
|
|
# Ensure all AntiFeatures are recognised...
|
|
if thisinfo['AntiFeatures']:
|
|
parts = thisinfo['AntiFeatures'].split(",")
|
|
for part in parts:
|
|
if (part != "Ads" and
|
|
part != "Tracking" and
|
|
part != "NonFreeNet" and
|
|
part != "NonFreeDep" and
|
|
part != "NonFreeAdd"):
|
|
raise MetaDataException("Unrecognised antifeature '" + part + "' in " \
|
|
+ metafile.name)
|
|
|
|
return thisinfo
|
|
|
|
# Write a metadata file.
|
|
#
|
|
# 'dest' - The path to the output file
|
|
# 'app' - The app data
|
|
def write_metadata(dest, app):
|
|
|
|
def writecomments(key):
|
|
for pf, comment in app['comments']:
|
|
if pf == key:
|
|
mf.write(comment + '\n')
|
|
|
|
def writefield(field, value=None):
|
|
writecomments(field)
|
|
if value is None:
|
|
value = app[field]
|
|
mf.write(field + ':' + value + '\n')
|
|
|
|
mf = open(dest, 'w')
|
|
if app['Disabled']:
|
|
writefield('Disabled')
|
|
if app['AntiFeatures']:
|
|
writefield('AntiFeatures')
|
|
writefield('Category')
|
|
writefield('License')
|
|
writefield('Web Site')
|
|
writefield('Source Code')
|
|
writefield('Issue Tracker')
|
|
if app['Donate']:
|
|
writefield('Donate')
|
|
if app['FlattrID']:
|
|
writefield('FlattrID')
|
|
if app['Bitcoin']:
|
|
writefield('Bitcoin')
|
|
mf.write('\n')
|
|
if app['Name']:
|
|
writefield('Name')
|
|
writefield('Auto Name')
|
|
writefield('Summary')
|
|
writefield('Description', '')
|
|
for line in app['Description']:
|
|
mf.write(line + '\n')
|
|
mf.write('.\n')
|
|
mf.write('\n')
|
|
if app['Requires Root']:
|
|
writefield('Requires Root', 'Yes')
|
|
mf.write('\n')
|
|
if len(app['Repo Type']) > 0:
|
|
writefield('Repo Type')
|
|
writefield('Repo')
|
|
mf.write('\n')
|
|
for build in app['builds']:
|
|
writecomments('build:' + build['version'])
|
|
mf.write('Build Version:')
|
|
if 'origlines' in build:
|
|
# Keeping the original formatting if we loaded it from a file...
|
|
mf.write('\\\n'.join(build['origlines']) + '\n')
|
|
else:
|
|
mf.write(build['version'] + ',' + build['vercode'] + ',' +
|
|
build['commit'])
|
|
for key,value in build.iteritems():
|
|
if key not in ['version', 'vercode', 'commit']:
|
|
mf.write(',' + key + '=' + value)
|
|
mf.write('\n')
|
|
if len(app['builds']) > 0:
|
|
mf.write('\n')
|
|
writefield('Auto Update Mode')
|
|
writefield('Update Check Mode')
|
|
if len(app['Current Version']) > 0:
|
|
writefield('Current Version')
|
|
writefield('Current Version Code')
|
|
mf.write('\n')
|
|
if len(app['No Source Since']) > 0:
|
|
writefield('No Source Since')
|
|
mf.write('\n')
|
|
writecomments(None)
|
|
mf.close()
|
|
|
|
|
|
# Read all metadata. Returns a list of 'app' objects (which are dictionaries as
|
|
# returned by the parse_metadata function.
|
|
def read_metadata(verbose=False, xref=True):
|
|
apps = []
|
|
for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
|
|
apps.append(parse_metadata(metafile, verbose=verbose))
|
|
|
|
if xref:
|
|
# Parse all descriptions at load time, just to ensure cross-referencing
|
|
# errors are caught early rather than when they hit the build server.
|
|
def linkres(link):
|
|
for app in apps:
|
|
if app['id'] == link:
|
|
return ("fdroid.app:" + link, "Dummy name - don't know yet")
|
|
raise MetaDataException("Cannot resolve app id " + link)
|
|
for app in apps:
|
|
try:
|
|
description_html(app['Description'], linkres)
|
|
except Exception, e:
|
|
raise MetaDataException("Problem with description of " + app['id'] +
|
|
" - " + str(e))
|
|
|
|
return apps
|
|
|
|
# Formatter for descriptions. Create an instance, and call parseline() with
|
|
# each line of the description source from the metadata. At the end, call
|
|
# end() and then text_plain, text_wiki and text_html will contain the result.
|
|
class DescriptionFormatter:
|
|
stNONE = 0
|
|
stPARA = 1
|
|
stUL = 2
|
|
stOL = 3
|
|
bold = False
|
|
ital = False
|
|
state = stNONE
|
|
text_plain = ''
|
|
text_wiki = ''
|
|
text_html = ''
|
|
linkResolver = None
|
|
def __init__(self, linkres):
|
|
self.linkResolver = linkres
|
|
def endcur(self, notstates=None):
|
|
if notstates and self.state in notstates:
|
|
return
|
|
if self.state == self.stPARA:
|
|
self.endpara()
|
|
elif self.state == self.stUL:
|
|
self.endul()
|
|
elif self.state == self.stOL:
|
|
self.endol()
|
|
def endpara(self):
|
|
self.text_plain += '\n'
|
|
self.text_html += '</p>'
|
|
self.state = self.stNONE
|
|
def endul(self):
|
|
self.text_html += '</ul>'
|
|
self.state = self.stNONE
|
|
def endol(self):
|
|
self.text_html += '</ol>'
|
|
self.state = self.stNONE
|
|
|
|
def formatted(self, txt, html):
|
|
formatted = ''
|
|
if html:
|
|
txt = cgi.escape(txt)
|
|
while True:
|
|
index = txt.find("''")
|
|
if index == -1:
|
|
return formatted + txt
|
|
formatted += txt[:index]
|
|
txt = txt[index:]
|
|
if txt.startswith("'''"):
|
|
if html:
|
|
if self.bold:
|
|
formatted += '</b>'
|
|
else:
|
|
formatted += '<b>'
|
|
self.bold = not self.bold
|
|
txt = txt[3:]
|
|
else:
|
|
if html:
|
|
if self.ital:
|
|
formatted += '</i>'
|
|
else:
|
|
formatted += '<i>'
|
|
self.ital = not self.ital
|
|
txt = txt[2:]
|
|
|
|
|
|
def linkify(self, txt):
|
|
linkified_plain = ''
|
|
linkified_html = ''
|
|
while True:
|
|
index = txt.find("[")
|
|
if index == -1:
|
|
return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
|
|
linkified_plain += self.formatted(txt[:index], False)
|
|
linkified_html += self.formatted(txt[:index], True)
|
|
txt = txt[index:]
|
|
if txt.startswith("[["):
|
|
index = txt.find("]]")
|
|
if index == -1:
|
|
raise MetaDataException("Unterminated ]]")
|
|
url = txt[2:index]
|
|
if self.linkResolver:
|
|
url, urltext = self.linkResolver(url)
|
|
else:
|
|
urltext = url
|
|
linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
|
|
linkified_plain += urltext
|
|
txt = txt[index+2:]
|
|
else:
|
|
index = txt.find("]")
|
|
if index == -1:
|
|
raise MetaDataException("Unterminated ]")
|
|
url = txt[1:index]
|
|
index2 = url.find(' ')
|
|
if index2 == -1:
|
|
urltxt = url
|
|
else:
|
|
urltxt = url[index2 + 1:]
|
|
url = url[:index2]
|
|
linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
|
|
linkified_plain += urltxt
|
|
if urltxt != url:
|
|
linkified_plain += ' (' + url + ')'
|
|
txt = txt[index+1:]
|
|
|
|
def addtext(self, txt):
|
|
p, h = self.linkify(txt)
|
|
self.text_plain += p
|
|
self.text_html += h
|
|
|
|
def parseline(self, line):
|
|
self.text_wiki += line + '\n'
|
|
if len(line) == 0:
|
|
self.endcur()
|
|
elif line.startswith('*'):
|
|
self.endcur([self.stUL])
|
|
if self.state != self.stUL:
|
|
self.text_html += '<ul>'
|
|
self.state = self.stUL
|
|
self.text_html += '<li>'
|
|
self.text_plain += '*'
|
|
self.addtext(line[1:])
|
|
self.text_html += '</li>'
|
|
elif line.startswith('#'):
|
|
self.endcur([self.stOL])
|
|
if self.state != self.stOL:
|
|
self.text_html += '<ol>'
|
|
self.state = self.stOL
|
|
self.text_html += '<li>'
|
|
self.text_plain += '*' #TODO: lazy - put the numbers in!
|
|
self.addtext(line[1:])
|
|
self.text_html += '</li>'
|
|
else:
|
|
self.endcur([self.stPARA])
|
|
if self.state == self.stNONE:
|
|
self.text_html += '<p>'
|
|
self.state = self.stPARA
|
|
elif self.state == self.stPARA:
|
|
self.text_html += ' '
|
|
self.text_plain += ' '
|
|
self.addtext(line)
|
|
|
|
def end(self):
|
|
self.endcur()
|
|
|
|
# Parse multiple lines of description as written in a metadata file, returning
|
|
# a single string in plain text format.
|
|
def description_plain(lines, linkres):
|
|
ps = DescriptionFormatter(linkres)
|
|
for line in lines:
|
|
ps.parseline(line)
|
|
ps.end()
|
|
return ps.text_plain
|
|
|
|
# Parse multiple lines of description as written in a metadata file, returning
|
|
# a single string in wiki format.
|
|
def description_wiki(lines):
|
|
ps = DescriptionFormatter(None)
|
|
for line in lines:
|
|
ps.parseline(line)
|
|
ps.end()
|
|
return ps.text_wiki
|
|
|
|
# Parse multiple lines of description as written in a metadata file, returning
|
|
# a single string in HTML format.
|
|
def description_html(lines,linkres):
|
|
ps = DescriptionFormatter(linkres)
|
|
for line in lines:
|
|
ps.parseline(line)
|
|
ps.end()
|
|
return ps.text_html
|
|
|
|
def retrieve_string(xml_dir, string):
|
|
if not string.startswith('@string/'):
|
|
return string.replace("\\'","'")
|
|
string_search = re.compile(r'.*"'+string[8:]+'".*>([^<]+?)<.*').search
|
|
for xmlfile in glob.glob(os.path.join(xml_dir, '*.xml')):
|
|
for line in file(xmlfile):
|
|
matches = string_search(line)
|
|
if matches:
|
|
return retrieve_string(xml_dir, matches.group(1))
|
|
return ''
|
|
|
|
# Return list of existing files that will be used to find the highest vercode
|
|
def manifest_paths(app_dir, flavour):
|
|
|
|
possible_manifests = [ os.path.join(app_dir, 'AndroidManifest.xml'),
|
|
os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
|
|
os.path.join(app_dir, 'build.gradle') ]
|
|
|
|
if flavour is not None:
|
|
possible_manifests.append(
|
|
os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
|
|
|
|
return [path for path in possible_manifests if os.path.isfile(path)]
|
|
|
|
# Retrieve the package name
|
|
def fetch_real_name(app_dir, flavour):
|
|
app_search = re.compile(r'.*<application.*').search
|
|
name_search = re.compile(r'.*android:label="([^"]+)".*').search
|
|
app_found = False
|
|
name = None
|
|
for f in manifest_paths(app_dir, flavour):
|
|
if not f.endswith(".xml"):
|
|
continue
|
|
xml_dir = os.path.join(f[:-19], 'res', 'values')
|
|
for line in file(f):
|
|
if not app_found:
|
|
if app_search(line):
|
|
app_found = True
|
|
if app_found:
|
|
matches = name_search(line)
|
|
if matches:
|
|
return retrieve_string(xml_dir, matches.group(1))
|
|
return ''
|
|
|
|
# Retrieve the version name
|
|
def version_name(original, app_dir, flavour):
|
|
for f in manifest_paths(app_dir, flavour):
|
|
if not f.endswith(".xml"):
|
|
continue
|
|
xml_dir = os.path.join(f[:-19], 'res', 'values')
|
|
string = retrieve_string(xml_dir, original)
|
|
if len(string) > 0:
|
|
return string
|
|
return original
|
|
|
|
# Extract some information from the AndroidManifest.xml at the given path.
|
|
# Returns (version, vercode, package), any or all of which might be None.
|
|
# All values returned are strings.
|
|
def parse_androidmanifests(paths):
|
|
|
|
if not paths:
|
|
return (None, None, None)
|
|
|
|
vcsearch = re.compile(r'.*android:versionCode="([0-9]+?)".*').search
|
|
vnsearch = re.compile(r'.*android:versionName="([^"]+?)".*').search
|
|
psearch = re.compile(r'.*package="([^"]+)".*').search
|
|
|
|
vcsearch_g = re.compile(r'.*versionCode[ ]+?([0-9]+?).*').search
|
|
vnsearch_g = re.compile(r'.*versionName[ ]+?"([^"]+?)".*').search
|
|
psearch_g = re.compile(r'.*packageName[ ]+?"([^"]+)".*').search
|
|
|
|
max_version = None
|
|
max_vercode = None
|
|
max_package = None
|
|
|
|
for path in paths:
|
|
|
|
gradle = path.endswith("gradle")
|
|
version = None
|
|
vercode = None
|
|
# Remember package name, may be defined separately from version+vercode
|
|
package = max_package
|
|
|
|
for line in file(path):
|
|
if not package:
|
|
if gradle:
|
|
matches = psearch_g(line)
|
|
else:
|
|
matches = psearch(line)
|
|
if matches:
|
|
package = matches.group(1)
|
|
if not version:
|
|
if gradle:
|
|
matches = vnsearch_g(line)
|
|
else:
|
|
matches = vnsearch(line)
|
|
if matches:
|
|
version = matches.group(1)
|
|
if not vercode:
|
|
if gradle:
|
|
matches = vcsearch_g(line)
|
|
else:
|
|
matches = vcsearch(line)
|
|
if matches:
|
|
vercode = matches.group(1)
|
|
|
|
# Better some package name than nothing
|
|
if max_package is None:
|
|
max_package = package
|
|
|
|
if max_vercode is None or (vercode is not None and vercode > max_vercode):
|
|
max_version = version
|
|
max_vercode = vercode
|
|
max_package = package
|
|
|
|
if max_version is None:
|
|
max_version = "Unknown"
|
|
|
|
return (max_version, max_vercode, max_package)
|
|
|
|
class BuildException(Exception):
|
|
def __init__(self, value, stdout = None, stderr = None):
|
|
self.value = value
|
|
self.stdout = stdout
|
|
self.stderr = stderr
|
|
|
|
def get_wikitext(self):
|
|
ret = repr(self.value) + "\n"
|
|
if self.stdout:
|
|
ret += "=stdout=\n"
|
|
ret += "<pre>\n"
|
|
ret += str(self.stdout)
|
|
ret += "</pre>\n"
|
|
if self.stderr:
|
|
ret += "=stderr=\n"
|
|
ret += "<pre>\n"
|
|
ret += str(self.stderr)
|
|
ret += "</pre>\n"
|
|
return ret
|
|
|
|
def __str__(self):
|
|
ret = repr(self.value)
|
|
if self.stdout:
|
|
ret = ret + "\n==== stdout begin ====\n" + str(self.stdout) + "\n==== stdout end ===="
|
|
if self.stderr:
|
|
ret = ret + "\n==== stderr begin ====\n" + str(self.stderr) + "\n==== stderr end ===="
|
|
return ret
|
|
|
|
class VCSException(Exception):
|
|
def __init__(self, value):
|
|
self.value = value
|
|
|
|
def __str__(self):
|
|
return repr(self.value)
|
|
|
|
class MetaDataException(Exception):
|
|
def __init__(self, value):
|
|
self.value = value
|
|
|
|
def __str__(self):
|
|
return repr(self.value)
|
|
|
|
def parse_srclib(metafile, **kw):
|
|
|
|
thisinfo = {}
|
|
if metafile and not isinstance(metafile, file):
|
|
metafile = open(metafile, "r")
|
|
|
|
# Defaults for fields that come from metadata
|
|
thisinfo['Repo Type'] = ''
|
|
thisinfo['Repo'] = ''
|
|
thisinfo['Subdir'] = None
|
|
thisinfo['Prepare'] = None
|
|
thisinfo['Update Project'] = None
|
|
|
|
if metafile is None:
|
|
return thisinfo
|
|
|
|
mode = 0
|
|
buildlines = []
|
|
|
|
for line in metafile:
|
|
line = line.rstrip('\r\n')
|
|
if len(line) == 0:
|
|
continue
|
|
if line.startswith("#"):
|
|
continue
|
|
index = line.find(':')
|
|
if index == -1:
|
|
raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
|
|
field = line[:index]
|
|
value = line[index+1:]
|
|
|
|
if field == "Subdir":
|
|
thisinfo[field] = value.split(',')
|
|
else:
|
|
thisinfo[field] = value
|
|
|
|
return thisinfo
|
|
|
|
# Get the specified source library.
|
|
# Returns the path to it. Normally this is the path to be used when referencing
|
|
# it, which may be a subdirectory of the actual project. If you want the base
|
|
# directory of the project, pass 'basepath=True'.
|
|
def getsrclib(spec, srclib_dir, sdk_path, ndk_path="", mvn3="", basepath=False, raw=False, prepare=True, preponly=False):
|
|
|
|
if raw:
|
|
name = spec
|
|
ref = None
|
|
else:
|
|
name, ref = spec.split('@')
|
|
|
|
srclib_path = os.path.join('srclibs', name + ".txt")
|
|
|
|
if not os.path.exists(srclib_path):
|
|
raise BuildException('srclib ' + name + ' not found.')
|
|
|
|
srclib = parse_srclib(srclib_path)
|
|
|
|
sdir = os.path.join(srclib_dir, name)
|
|
|
|
if not preponly:
|
|
vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir, sdk_path)
|
|
vcs.srclib = (name, sdir)
|
|
vcs.gotorevision(ref)
|
|
|
|
if raw:
|
|
return vcs
|
|
|
|
libdir = None
|
|
|
|
if srclib["Subdir"] is not None:
|
|
for subdir in srclib["Subdir"]:
|
|
libdir_candidate = os.path.join(sdir, subdir)
|
|
if os.path.exists(libdir_candidate):
|
|
libdir = libdir_candidate
|
|
break
|
|
|
|
if libdir is None:
|
|
libdir = sdir
|
|
|
|
if prepare:
|
|
|
|
if srclib["Prepare"] is not None:
|
|
cmd = srclib["Prepare"].replace('$$SDK$$', sdk_path)
|
|
cmd = cmd.replace('$$NDK$$', ndk_path).replace('$$MVN$$', mvn3)
|
|
|
|
print "******************************* PREPARE " + cmd + " **************"
|
|
|
|
p = subprocess.Popen(['bash', '-c', cmd], cwd=libdir,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
out, err = p.communicate()
|
|
if p.returncode != 0:
|
|
raise BuildException("Error running prepare command for srclib "
|
|
+ name, out, err)
|
|
|
|
if srclib["Update Project"] == "Yes":
|
|
print "Updating srclib %s at path %s" % (name, libdir)
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p', libdir]) != 0:
|
|
raise BuildException( 'Error updating ' + name + ' project')
|
|
|
|
if basepath:
|
|
return sdir
|
|
return libdir
|
|
|
|
|
|
# Prepare the source code for a particular build
|
|
# 'vcs' - the appropriate vcs object for the application
|
|
# 'app' - the application details from the metadata
|
|
# 'build' - the build details from the metadata
|
|
# 'build_dir' - the path to the build directory, usually
|
|
# 'build/app.id'
|
|
# 'srclib_dir' - the path to the source libraries directory, usually
|
|
# 'build/srclib'
|
|
# 'extlib_dir' - the path to the external libraries directory, usually
|
|
# 'build/extlib'
|
|
# 'sdk_path' - the path to the Android SDK
|
|
# 'ndk_path' - the path to the Android NDK
|
|
# 'javacc_path' - the path to javacc
|
|
# 'mvn3' - the path to the maven 3 executable
|
|
# 'verbose' - optional: verbose or not (default=False)
|
|
# Returns the (root, srclibpaths) where:
|
|
# 'root' is the root directory, which may be the same as 'build_dir' or may
|
|
# be a subdirectory of it.
|
|
# 'srclibpaths' is information on the srclibs being used
|
|
def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path, ndk_path, javacc_path, mvn3, verbose=False, onserver=False):
|
|
|
|
# Optionally, the actual app source can be in a subdirectory...
|
|
if 'subdir' in build:
|
|
root_dir = os.path.join(build_dir, build['subdir'])
|
|
else:
|
|
root_dir = build_dir
|
|
|
|
# Get a working copy of the right revision...
|
|
print "Getting source for revision " + build['commit']
|
|
vcs.gotorevision(build['commit'])
|
|
|
|
# Check that a subdir (if we're using one) exists. This has to happen
|
|
# after the checkout, since it might not exist elsewhere...
|
|
if not os.path.exists(root_dir):
|
|
raise BuildException('Missing subdir ' + root_dir)
|
|
|
|
# Initialise submodules if requred...
|
|
if build.get('submodules', 'no') == 'yes':
|
|
if verbose: print "Initialising submodules..."
|
|
vcs.initsubmodules()
|
|
|
|
# Run an init command if one is required...
|
|
if 'init' in build:
|
|
init = build['init']
|
|
init = init.replace('$$SDK$$', sdk_path)
|
|
init = init.replace('$$NDK$$', ndk_path)
|
|
init = init.replace('$$MVN$$', mvn3)
|
|
if verbose: print "Doing init: exec '%s' in '%s'"%(init,root_dir)
|
|
if subprocess.call(['bash', '-c', init], cwd=root_dir) != 0:
|
|
raise BuildException("Error running init command")
|
|
|
|
# Generate (or update) the ant build file, build.xml...
|
|
updatemode = build.get('update', '.')
|
|
if (updatemode != 'no' and
|
|
'maven' not in build and 'gradle' not in build):
|
|
parms = [os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p', '.']
|
|
parms.append('--subprojects')
|
|
if 'target' in build:
|
|
parms.append('-t')
|
|
parms.append(build['target'])
|
|
update_dirs = updatemode.split(';')
|
|
# Force build.xml update if necessary...
|
|
if updatemode == 'force' or 'target' in build:
|
|
if updatemode == 'force':
|
|
update_dirs = ['.']
|
|
buildxml = os.path.join(root_dir, 'build.xml')
|
|
if os.path.exists(buildxml):
|
|
print 'Force-removing old build.xml'
|
|
os.remove(buildxml)
|
|
for d in update_dirs:
|
|
cwd = os.path.join(root_dir, d)
|
|
# Remove gen and bin dirs in libraries
|
|
# rid of them...
|
|
for baddir in ['gen', 'bin']:
|
|
badpath = os.path.join(cwd, baddir)
|
|
if os.path.exists(badpath):
|
|
shutil.rmtree(badpath)
|
|
if verbose:
|
|
print "Update of '%s': exec '%s' in '%s'"%\
|
|
(d," ".join(parms),cwd)
|
|
p = subprocess.Popen(parms, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
(out, err) = p.communicate()
|
|
if p.returncode != 0:
|
|
raise BuildException("Failed to update project with stdout '%s' and stderr '%s'"%(out,err))
|
|
# check to see whether an error was returned without a proper exit code (this is the case for the 'no target set or target invalid' error)
|
|
if err != "" and err.startswith("Error: "):
|
|
raise BuildException("Failed to update project with stdout '%s' and stderr '%s'"%(out,err))
|
|
|
|
# If the app has ant set up to sign the release, we need to switch
|
|
# that off, because we want the unsigned apk...
|
|
for propfile in ('build.properties', 'default.properties', 'ant.properties'):
|
|
if os.path.exists(os.path.join(root_dir, propfile)):
|
|
if subprocess.call(['sed','-i','s/^key.store/#/',
|
|
propfile], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend %s" % propfile)
|
|
|
|
# Update the local.properties file...
|
|
locprops = os.path.join(root_dir, 'local.properties')
|
|
if os.path.exists(locprops):
|
|
f = open(locprops, 'r')
|
|
props = f.read()
|
|
f.close()
|
|
# Fix old-fashioned 'sdk-location' by copying
|
|
# from sdk.dir, if necessary...
|
|
if build.get('oldsdkloc', 'no') == "yes":
|
|
sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
|
|
re.S|re.M).group(1)
|
|
props += "\nsdk-location=" + sdkloc + "\n"
|
|
# Add ndk location...
|
|
props+= "\nndk.dir=" + ndk_path + "\n"
|
|
# Add java.encoding if necessary...
|
|
if 'encoding' in build:
|
|
props += "\njava.encoding=" + build['encoding'] + "\n"
|
|
f = open(locprops, 'w')
|
|
f.write(props)
|
|
f.close()
|
|
|
|
# Insert version code and number into the manifest if necessary...
|
|
if 'forceversion' in build:
|
|
if subprocess.call(['sed','-r','-i',
|
|
's/android:versionName="[^"]+"/android:versionName="' + build['version'] + '"/g',
|
|
'AndroidManifest.xml'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend manifest")
|
|
if 'forcevercode' in build:
|
|
if subprocess.call(['sed','-r','-i',
|
|
's/android:versionCode="[^"]+"/android:versionCode="' + build['vercode'] + '"/g',
|
|
'AndroidManifest.xml'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend manifest")
|
|
|
|
# Delete unwanted file...
|
|
if 'rm' in build:
|
|
for part in build['rm'].split(';'):
|
|
dest = os.path.join(build_dir, part)
|
|
if os.path.exists(dest):
|
|
os.remove(dest)
|
|
|
|
# Fix apostrophes translation files if necessary...
|
|
if build.get('fixapos', 'no') == 'yes':
|
|
for root, dirs, files in os.walk(os.path.join(root_dir, 'res')):
|
|
for filename in files:
|
|
if filename.endswith('.xml'):
|
|
if subprocess.call(['sed','-i','s@' +
|
|
r"\([^\\]\)'@\1\\'" +
|
|
'@g',
|
|
os.path.join(root, filename)]) != 0:
|
|
raise BuildException("Failed to amend " + filename)
|
|
|
|
# Fix translation files if necessary...
|
|
if build.get('fixtrans', 'no') == 'yes':
|
|
for root, dirs, files in os.walk(os.path.join(root_dir, 'res')):
|
|
for filename in files:
|
|
if filename.endswith('.xml'):
|
|
f = open(os.path.join(root, filename))
|
|
changed = False
|
|
outlines = []
|
|
for line in f:
|
|
num = 1
|
|
index = 0
|
|
oldline = line
|
|
while True:
|
|
index = line.find("%", index)
|
|
if index == -1:
|
|
break
|
|
next = line[index+1:index+2]
|
|
if next == "s" or next == "d":
|
|
line = (line[:index+1] +
|
|
str(num) + "$" +
|
|
line[index+1:])
|
|
num += 1
|
|
index += 3
|
|
else:
|
|
index += 1
|
|
# We only want to insert the positional arguments
|
|
# when there is more than one argument...
|
|
if oldline != line:
|
|
if num > 2:
|
|
changed = True
|
|
else:
|
|
line = oldline
|
|
outlines.append(line)
|
|
f.close()
|
|
if changed:
|
|
f = open(os.path.join(root, filename), 'w')
|
|
f.writelines(outlines)
|
|
f.close()
|
|
|
|
# Add required external libraries...
|
|
if 'extlibs' in build:
|
|
print "Collecting prebuilt libraries..."
|
|
libsdir = os.path.join(root_dir, 'libs')
|
|
if not os.path.exists(libsdir):
|
|
os.mkdir(libsdir)
|
|
for lib in build['extlibs'].split(';'):
|
|
libf = os.path.basename(lib)
|
|
shutil.copyfile(os.path.join(extlib_dir, lib),
|
|
os.path.join(libsdir, libf))
|
|
|
|
# Get required source libraries...
|
|
srclibpaths = []
|
|
if 'srclibs' in build:
|
|
print "Collecting source libraries..."
|
|
for lib in build['srclibs'].split(';'):
|
|
name, _ = lib.split('@')
|
|
srclibpaths.append((name, getsrclib(lib, srclib_dir, sdk_path, ndk_path, mvn3, preponly=onserver)))
|
|
basesrclib = vcs.getsrclib()
|
|
# If one was used for the main source, add that too.
|
|
if basesrclib:
|
|
srclibpaths.append(basesrclib)
|
|
|
|
# There should never be bin, gen or native libs directories in the source, so just get
|
|
# rid of them...
|
|
for baddir in ['gen', 'bin', 'obj', 'libs/armeabi-v7a', 'libs/armeabi', 'libs/mips', 'libs/x86']:
|
|
badpath = os.path.join(root_dir, baddir)
|
|
if os.path.exists(badpath):
|
|
shutil.rmtree(badpath)
|
|
|
|
# Apply patches if any
|
|
if 'patch' in build:
|
|
for patch in build['patch'].split(';'):
|
|
print "Applying " + patch
|
|
patch_path = os.path.join('metadata', app['id'], patch)
|
|
if subprocess.call(['patch', '-p1',
|
|
'-i', os.path.abspath(patch_path)], cwd=build_dir) != 0:
|
|
raise BuildException("Failed to apply patch %s" % patch_path)
|
|
|
|
# Run a pre-build command if one is required...
|
|
if 'prebuild' in build:
|
|
prebuild = build['prebuild']
|
|
if verbose:
|
|
print "Running source init (prebuild) commands:" + prebuild
|
|
else:
|
|
print "Running source init (prebuild) commands..."
|
|
|
|
# Substitute source library paths into prebuild commands...
|
|
for name, libpath in srclibpaths:
|
|
libpath = os.path.relpath(libpath, root_dir)
|
|
prebuild = prebuild.replace('$$' + name + '$$', libpath)
|
|
prebuild = prebuild.replace('$$SDK$$', sdk_path)
|
|
prebuild = prebuild.replace('$$NDK$$', ndk_path)
|
|
prebuild = prebuild.replace('$$MVN3$$', mvn3)
|
|
p = subprocess.Popen(['bash', '-c', prebuild], cwd=root_dir,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
out, err = p.communicate()
|
|
if p.returncode != 0:
|
|
raise BuildException("Error running pre-build command", out, err)
|
|
|
|
print "Applying generic clean-ups..."
|
|
|
|
if build.get('anal-tics', 'no') == 'yes':
|
|
fp = os.path.join(root_dir, 'src', 'com', 'google', 'android', 'apps', 'analytics')
|
|
os.makedirs(fp)
|
|
with open(os.path.join(fp, 'GoogleAnalyticsTracker.java'), 'w') as f:
|
|
f.write("""
|
|
package com.google.android.apps.analytics;
|
|
public class GoogleAnalyticsTracker {
|
|
private static GoogleAnalyticsTracker instance;
|
|
private GoogleAnalyticsTracker() {
|
|
}
|
|
public static GoogleAnalyticsTracker getInstance() {
|
|
if(instance == null)
|
|
instance = new GoogleAnalyticsTracker();
|
|
return instance;
|
|
}
|
|
public void start(String i,int think ,Object not) {
|
|
}
|
|
public void dispatch() {
|
|
}
|
|
public void stop() {
|
|
}
|
|
public void setProductVersion(String uh, String hu) {
|
|
}
|
|
public void trackEvent(String that,String just,String aint,int happening) {
|
|
}
|
|
public void trackPageView(String nope) {
|
|
}
|
|
public void setCustomVar(int mind,String your,String own,int business) {
|
|
}
|
|
}
|
|
""")
|
|
|
|
# Special case init functions for funambol...
|
|
if build.get('initfun', 'no') == "yes":
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'<taskdef resource="net/sf/antcontrib/antcontrib.properties" />' +
|
|
'@' +
|
|
'<taskdef resource="net/sf/antcontrib/antcontrib.properties">' +
|
|
'<classpath>' +
|
|
'<pathelement location="/usr/share/java/ant-contrib.jar"/>' +
|
|
'</classpath>' +
|
|
'</taskdef>' +
|
|
'@g',
|
|
'build.xml'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend build.xml")
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'\${user.home}/funambol/build/android/build.properties' +
|
|
'@' +
|
|
'build.properties' +
|
|
'@g',
|
|
'build.xml'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend build.xml")
|
|
|
|
buildxml = os.path.join(root_dir, 'build.xml')
|
|
f = open(buildxml, 'r')
|
|
xml = f.read()
|
|
f.close()
|
|
xmlout = ""
|
|
mode = 0
|
|
for line in xml.splitlines():
|
|
if mode == 0:
|
|
if line.find("jarsigner") != -1:
|
|
mode = 1
|
|
else:
|
|
xmlout += line + "\n"
|
|
else:
|
|
if line.find("/exec") != -1:
|
|
mode += 1
|
|
if mode == 3:
|
|
mode =0
|
|
f = open(buildxml, 'w')
|
|
f.write(xmlout)
|
|
f.close()
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'platforms/android-2.0' +
|
|
'@' +
|
|
'platforms/android-8' +
|
|
'@g',
|
|
'build.xml'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend build.xml")
|
|
|
|
shutil.copyfile(
|
|
os.path.join(root_dir, "build.properties.example"),
|
|
os.path.join(root_dir, "build.properties"))
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'javacchome=.*'+
|
|
'@' +
|
|
'javacchome=' + javacc_path +
|
|
'@g',
|
|
'build.properties'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend build.properties")
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'sdk-folder=.*'+
|
|
'@' +
|
|
'sdk-folder=' + sdk_path +
|
|
'@g',
|
|
'build.properties'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend build.properties")
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'android.sdk.version.*'+
|
|
'@' +
|
|
'android.sdk.version=2.0' +
|
|
'@g',
|
|
'build.properties'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend build.properties")
|
|
|
|
return (root_dir, srclibpaths)
|
|
|
|
|
|
# Scan the source code in the given directory (and all subdirectories)
|
|
# and return a list of potential problems.
|
|
def scan_source(build_dir, root_dir, thisbuild):
|
|
|
|
problems = []
|
|
|
|
# Common known non-free blobs:
|
|
usual_suspects = ['flurryagent',
|
|
'paypal_mpl',
|
|
'libgoogleanalytics',
|
|
'admob-sdk-android',
|
|
'googleadview',
|
|
'googleadmobadssdk',
|
|
'google-play-services',
|
|
'crittercism',
|
|
'heyzap',
|
|
'jpct-ae']
|
|
|
|
if 'scanignore' in thisbuild:
|
|
ignore = thisbuild['scanignore'].split(';')
|
|
else:
|
|
ignore = []
|
|
|
|
# Iterate through all files in the source code...
|
|
for r,d,f in os.walk(build_dir):
|
|
for curfile in f:
|
|
|
|
if '/.hg' in r or '/.git' in r or '/.svn' in r:
|
|
continue
|
|
|
|
# Path (relative) to the file...
|
|
fp = os.path.join(r, curfile)
|
|
|
|
# Check if this file has been explicitly excluded from scanning...
|
|
ignorethis = False
|
|
for i in ignore:
|
|
if fp.startswith(i):
|
|
ignorethis = True
|
|
break
|
|
if ignorethis:
|
|
continue
|
|
|
|
for suspect in usual_suspects:
|
|
if suspect in curfile.lower():
|
|
msg = 'Found probable non-free blob ' + fp
|
|
problems.append(msg)
|
|
|
|
if curfile.endswith('.apk'):
|
|
msg = 'Found apk file, which should not be in the source - ' + fp
|
|
problems.append(msg)
|
|
|
|
elif curfile.endswith('.elf'):
|
|
msg = 'Found .elf at ' + fp
|
|
problems.append(msg)
|
|
|
|
elif curfile.endswith('.so'):
|
|
msg = 'Found .so at ' + fp
|
|
problems.append(msg)
|
|
|
|
elif curfile.endswith('.java'):
|
|
for line in file(fp):
|
|
if 'DexClassLoader' in line:
|
|
msg = 'Found DexClassLoader in ' + fp
|
|
problems.append(msg)
|
|
|
|
# Presence of a jni directory without buildjni=yes might
|
|
# indicate a problem... (if it's not a problem, explicitly use
|
|
# buildjni=no to bypass this check)
|
|
if (os.path.exists(os.path.join(root_dir, 'jni')) and
|
|
thisbuild.get('buildjni') is None):
|
|
msg = 'Found jni directory, but buildjni is not enabled'
|
|
problems.append(msg)
|
|
|
|
return problems
|
|
|
|
|
|
class KnownApks:
|
|
|
|
def __init__(self):
|
|
self.path = os.path.join('stats', 'known_apks.txt')
|
|
self.apks = {}
|
|
if os.path.exists(self.path):
|
|
for line in file( self.path):
|
|
t = line.rstrip().split(' ')
|
|
if len(t) == 2:
|
|
self.apks[t[0]] = (t[1], None)
|
|
else:
|
|
self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
|
|
self.changed = False
|
|
|
|
def writeifchanged(self):
|
|
if self.changed:
|
|
if not os.path.exists('stats'):
|
|
os.mkdir('stats')
|
|
f = open(self.path, 'w')
|
|
lst = []
|
|
for apk, app in self.apks.iteritems():
|
|
appid, added = app
|
|
line = apk + ' ' + appid
|
|
if added:
|
|
line += ' ' + time.strftime('%Y-%m-%d', added)
|
|
lst.append(line)
|
|
for line in sorted(lst):
|
|
f.write(line + '\n')
|
|
f.close()
|
|
|
|
# Record an apk (if it's new, otherwise does nothing)
|
|
# Returns the date it was added.
|
|
def recordapk(self, apk, app):
|
|
if not apk in self.apks:
|
|
self.apks[apk] = (app, time.gmtime(time.time()))
|
|
self.changed = True
|
|
_, added = self.apks[apk]
|
|
return added
|
|
|
|
# Look up information - given the 'apkname', returns (app id, date added/None).
|
|
# Or returns None for an unknown apk.
|
|
def getapp(self, apkname):
|
|
if apkname in self.apks:
|
|
return self.apks[apkname]
|
|
return None
|
|
|
|
# Get the most recent 'num' apps added to the repo, as a list of package ids
|
|
# with the most recent first.
|
|
def getlatest(self, num):
|
|
apps = {}
|
|
for apk, app in self.apks.iteritems():
|
|
appid, added = app
|
|
if added:
|
|
if appid in apps:
|
|
if apps[appid] > added:
|
|
apps[appid] = added
|
|
else:
|
|
apps[appid] = added
|
|
sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
|
|
lst = []
|
|
for app, added in sortedapps:
|
|
lst.append(app)
|
|
lst.reverse()
|
|
return lst
|
|
|
|
def isApkDebuggable(apkfile):
|
|
"""Returns True if the given apk file is debuggable
|
|
|
|
:param apkfile: full path to the apk to check"""
|
|
|
|
execfile('config.py', globals())
|
|
|
|
p = subprocess.Popen([os.path.join(sdk_path, 'build-tools', build_tools, 'aapt'),
|
|
'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
|
|
stdout=subprocess.PIPE)
|
|
output = p.communicate()[0]
|
|
if p.returncode != 0:
|
|
print "ERROR: Failed to get apk manifest information"
|
|
sys.exit(1)
|
|
for line in output.splitlines():
|
|
if line.find('android:debuggable') != -1 and not line.endswith('0x0'):
|
|
return True
|
|
return False
|
|
|
|
|