nzbget/linux/build-nzbget

#!/bin/bash
#
#  This file is part of nzbget
#
#  Copyright (C) 2015-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

# Setup strict bash error handling
set -o nounset
set -o errexit

# Uncomment next line for debuging
#set -x

ALLTARGETS="dist i686 x86_64 armel armhf aarch64 mipsel mipseb ppc6xx ppc500 x86_64-bsd i686-ndk armhf-ndk aarch64-ndk"
ROOT=`pwd`
ROOTPATH=$PATH
OUTPUTDIR=$ROOT/output
CROSSCLANG="clang-4.0"

# Configuration
BUILD=no
PLATFORMS=""
TARGETS=""
OUTPUTS=""
REVISION=""
BRANCH=""
CONFIGS=""
CLEAN=no
PCH=no
COREX="1"

PrintUsage()
{
    echo "Usage:"
    echo "  $(basename $0) [targets] [output] [revision] [reppath] [configs] [cleanup] [pch] [corex]"
    echo "    targets   : all (default) $ALLTARGETS"
    echo "    output    : bin installer"
    echo "    revision  : head (default) work <commit-hash>"
    echo "    branch    : develop (default) master tags/XXX branches/XXX"
    echo "    configs   : release debug (default) release-nostrip debug-strip"
    echo "    cleanup   : cleanup output directory before building"
    echo "    pch       : create and use precompiled headers"
    echo "    corex     : multicore make (x is a number of threads)"
    echo
}

ParseCommandLine()
{
    for PARAM in "$@"
    do
        case $PARAM in
            release|release-nostrip|debug|debug-strip)
                # using xargs to trim spaces
                CONFIGS=`echo "$CONFIGS $PARAM" | xargs`
                ;;
            master|develop|tags/*)
                BRANCH="$PARAM"
                ;;
            branches/*)
                BRANCH="${PARAM:9}"
                ;;
            head|work|[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*)
                REVISION="$PARAM"
                ;;
            clean|cleanup)
                CLEAN=yes
                ;;
            pch)
                PCH=yes
                ;;
            core[1-9])
                COREX="${PARAM:4}"
                ;;
            bin|installer)
                # using xargs to trim spaces
                OUTPUTS=`echo "$OUTPUTS $PARAM" | xargs`
                ;;
            *)
                if [[ " $ALLTARGETS " == *" $PARAM "* ]]; then
                    # using xargs to trim spaces
                    TARGETS=`echo "$TARGETS $PARAM" | xargs`
                    if [ "$PARAM" == "all" ]; then
                        PARAM=$ALLTARGETS
                    fi
                elif [ -d toolchain/$PARAM ]; then
                    # non-standard target but the toolchain exist
                    # using xargs to trim spaces
                    TARGETS=`echo "$TARGETS $PARAM" | xargs`
                else
                    echo "Invalid parameter: $PARAM"
                    exit 1
                fi
                ;;
        esac

    done

    if [ "$TARGETS" == "" ]; then
        TARGETS="$ALLTARGETS"
    fi

    if [ "$OUTPUTS" == "" ]; then
        OUTPUTS="bin installer"
    fi

    if [ "$REVISION" == "" ]; then
        REVISION="head"
    fi

    if [ "$BRANCH" == "" ]; then
        BRANCH="develop"
    fi

    if [ "$CONFIGS" == "" ]; then
        CONFIGS="release debug"
    fi

    for TARGET in $TARGETS
    do
        if [[ $TARGET == *-bsd ]]; then
            PLATFORM="freebsd"
        elif [[ $TARGET == *-ndk ]]; then
            PLATFORM="android"
        else
            PLATFORM="linux"
        fi
        if [[ " $PLATFORMS " != *" $PLATFORM "* ]]; then
              PLATFORMS=`echo "$PLATFORMS $PLATFORM" | xargs`
        fi
    done
}

PrintConfig()
{
    echo "Active configuration:"
    echo "  platforms: $PLATFORMS"
    echo "  targets  : $TARGETS"
    echo "  outputs  : $OUTPUTS"
    echo "  revision : $REVISION"
    echo "  branch   : $BRANCH"
    echo "  configs  : $CONFIGS"
    echo "  cleanup  : $CLEAN"
    echo "  pch      : $PCH"
    echo "  cores    : $COREX"
    echo 
}

UpdateFromRepository()
{
    if [ ! -d nzbget ]; then
        echo "Initial checkout"
        git clone https://github.com/nzbget/nzbget.git
    fi

    cd nzbget

    BUILDDIR=$ROOT/nzbget

    cd $BUILDDIR

    if [ $REVISION != "work" ]; then
        echo "Updating to $REVISION"
        git pull
        git checkout $BRANCH
        if [ $REVISION != "head" ]; then
            git checkout $REVISION
        fi
        touch Makefile.in configure config.h.in

        echo "Updating root certificates"
        cd ../setup
        curl --remote-name --time-cond cacert.pem https://curl.se/ca/cacert.pem
        # remove expired DST Root CA X3 certificate
        sed '/^DST Root CA X3$/,/^-----END CERTIFICATE-----$/d;' -i cacert.pem
        cd $BUILDDIR
    fi
}

DetectVersionRevision()
{
    VERSION=`grep "AC_INIT(nzbget, " configure.ac`
    VERSION=`expr "$VERSION" : '.*, \(.*\),.*)'`

    # Building revision name
    B=`git branch | sed -n -e 's/^\* \(.*\)/\1/p' | sed 's/ /-/g' | sed 's/(//g' | sed 's/)//g'`
    M=`git status --porcelain`
    if [ "$M" != "" ]; then
        M="M"
    fi
    if [ "$B" == "master" ]; then
        REVISION="$M"
    elif [ "$B" == "develop" ]; then
        REVISION=`git rev-list HEAD | wc -l | xargs`
        REVISION="${REVISION}$M"
    else
        REVISION=`git rev-list HEAD | wc -l | xargs`
        REVISION="${REVISION}$M-($B)"
    fi
}

ConstructOutputFilename()
{
    BASENAME="nzbget-$VERSION"
    if [ `expr "$VERSION" : ".*-testing"` != 0 ]; then
        BASENAME="$BASENAME-r$REVISION"
    fi
}

ConstructSuffix()
{
    case $CONFIG in
        release)
            SUFFIX=""
            ;;
        debug)
            SUFFIX="-debug"
            ;;
        debug-strip)
            SUFFIX="-debug-strip"
            ;;
        release-nostrip)
            SUFFIX="-nostrip"
            ;;
    esac

    if [ $PLATFORM == "freebsd" ]; then
        PLATSUFF="-bsd"
    elif [ $PLATFORM == "android" ]; then
        PLATSUFF="-ndk"
    else
        PLATSUFF=""
    fi
}

ConstructArchParams()
{
    ARCH=$TARGET
    case $TARGET in
        mipseb)
            ARCH=mips
            ;;
        arm*)
            ARCH=arm
            ;;
        ppc*)
            ARCH=powerpc
            ;;
    esac

    TOOLCHAIN_ROOT=$ROOT/toolchain/$TARGET$PLATSUFF
    TOOLPATH=$TOOLCHAIN_ROOT/output/host/usr/bin
    PATH=$TOOLPATH:$ROOTPATH
    STAGING="$TOOLCHAIN_ROOT/output/staging"
    case $PLATFORM in
        linux)
            TOOLKIND=buildroot
            ARCH=$ARCH-linux
        ;;
        android)
            TOOLKIND=ndk
            ARCH=$ARCH-linux-android
            if [ "$ARCH" == "arm-linux-android" ]; then
                ARCH="arm-linux-androideabi"
            fi
        ;;
        freebsd)
            TOOLKIND=crossclang
            ARCH=$ARCH-pc-freebsd
            SYSROOT="$TOOLCHAIN_ROOT/sysroot"
        ;;
    esac
}

ConfigureTarget()
{
    STRIP=""
    if [ $CONFIG == "debug-strip" -o $CONFIG == "release" ]; then
        STRIP="-s"
    fi

    DEBUG=""
    if [ $CONFIG != "release" ]; then
        DEBUG="-g"
    fi

    LUBACKTRACE=""
    if [ -f "$STAGING/lib/libubacktrace.so.1" ] ; then
        LUBACKTRACE="-lubacktrace"
    fi

    case "$TOOLKIND-$CONFIG" in
        buildroot-debug|buildroot-debug-strip)
            LDFLAGS="-static $STRIP" \
                CXXFLAGS="-std=c++14 -g -fasynchronous-unwind-tables" \
                LIBS="-lcrypto -ldl -lz -Wl,--whole-archive -lpthread -Wl,--no-whole-archive $LUBACKTRACE" \
                ./configure --disable-dependency-tracking --host=$ARCH --enable-debug
            ;;
        buildroot-release|buildroot-release-nostrip)
            LDFLAGS="-static $STRIP" \
                CXXFLAGS="-std=c++14 -O2 $DEBUG" \
                LIBS="-lcrypto -ldl -lz -Wl,--whole-archive -lpthread -Wl,--no-whole-archive" \
                ./configure --disable-dependency-tracking --host=$ARCH
            ;;
        ndk-debug|ndk-debug-strip)
            LDFLAGS="-static -static-libstdc++ $STRIP" \
                CXXFLAGS="-std=c++14 -g -fasynchronous-unwind-tables" \
                CXX="$ARCH-clang++" \
                PKG_CONFIG_LIBDIR="$STAGING/usr/lib/pkgconfig" \
                ./configure --disable-dependency-tracking --disable-largefile --host=$ARCH --enable-debug
            ;;
        ndk-release|ndk-release-nostrip)
            LDFLAGS="-static -static-libstdc++ $STRIP" \
                CXXFLAGS="-std=c++14 -O2 $DEBUG" \
                CXX="$ARCH-clang++" \
                PKG_CONFIG_LIBDIR="$STAGING/usr/lib/pkgconfig" \
                ./configure --disable-dependency-tracking --disable-largefile --host=$ARCH
            ;;
        crossclang-debug|crossclang-debug-strip)
            CXX="$CROSSCLANG" \
                LDFLAGS="-static $STRIP -fuse-ld=lld --target=$ARCH -lc++ -lm --sysroot=$SYSROOT" \
                CXXFLAGS="-g --target=$ARCH --sysroot=$SYSROOT -I$SYSROOT/usr/include/c++/v1" \
                PKG_CONFIG_LIBDIR="$STAGING/usr/lib/pkgconfig" \
                ./configure --disable-dependency-tracking --host=$ARCH --enable-debug
                ;;
        crossclang-release|crossclang-release-nostrip)
            CXX="$CROSSCLANG" \
                LDFLAGS="-static $STRIP -fuse-ld=lld --target=$ARCH -lc++ -lm --sysroot=$SYSROOT" \
                CXXFLAGS="-O2 $DEBUG --target=$ARCH --sysroot=$SYSROOT -I$SYSROOT/usr/include/c++/v1" \
                PKG_CONFIG_LIBDIR="$STAGING/usr/lib/pkgconfig" \
                ./configure --disable-dependency-tracking --host=$ARCH --enable-debug
                ;;
    esac
}

PrecompileHeaders()
{
    rm -f nzbget.h.*

    OPTIM=""
    if [ $CONFIG == "release" -o $CONFIG == "release-nostrip" ]; then
        OPTIM="-O2"
    fi

    if [ $PCH == "yes" -a $TOOLKIND == "buildroot" ]; then
        $ARCH-g++ -std=c++14 -DHAVE_CONFIG_H \
            -I. -I$STAGING/usr/include -I$STAGING/usr/include/libxml2 \
            -g $OPTIM daemon/main/nzbget.h -o nzbget.h.gch
    fi

    if [ $PCH == "yes" -a $TOOLKIND == "ndk" ]; then
        $ARCH-clang++ -std=c++14 -DHAVE_CONFIG_H \
            -I. -I$STAGING/usr/include -I$STAGING/usr/include/libxml2 \
            -g $OPTIM -x c++-header daemon/main/nzbget.h -o nzbget.h.pch
        CXXFLAGS=`sed -n 's:^CXXFLAGS =.\(.*\):\1:p' Makefile`
        sed 's:^CXXFLAGS = :CXXFLAGS = -include-pch nzbget.h.pch :' -i Makefile
        sed "s: CXXFLAGS+=: CXXFLAGS=$CXXFLAGS :" -i Makefile
    fi
}

PackTarget()
{
    rm -r -f $OUTPUTDIR/install
    make DESTDIR=$OUTPUTDIR/install install

    cd $OUTPUTDIR
    rm -r -f nzbget
    mkdir -p nzbget
    mv install/usr/local/bin/nzbget nzbget
    mv install/usr/local/share/doc/nzbget/* nzbget
    mv install/usr/local/share/nzbget/webui nzbget
    mv install/usr/local/share/nzbget/scripts nzbget
    CONFTEMPLATE=nzbget/webui/nzbget.conf.template
    mv install/usr/local/share/nzbget/nzbget.conf $CONFTEMPLATE

    rm -r -f $OUTPUTDIR/install

    # adjusting nzbget.conf
    sed 's:^MainDir=.*:MainDir=${AppDir}/downloads:' -i $CONFTEMPLATE
    sed 's:^DestDir=.*:DestDir=${MainDir}/completed:' -i $CONFTEMPLATE
    sed 's:^InterDir=.*:InterDir=${MainDir}/intermediate:' -i $CONFTEMPLATE
    sed 's:^WebDir=.*:WebDir=${AppDir}/webui:' -i $CONFTEMPLATE
    sed 's:^ScriptDir=.*:ScriptDir=${AppDir}/scripts:' -i $CONFTEMPLATE
    sed 's:^LogFile=.*:LogFile=${MainDir}/nzbget.log:' -i $CONFTEMPLATE
    sed 's:^ConfigTemplate=.*:ConfigTemplate=${AppDir}/webui/nzbget.conf.template:' -i $CONFTEMPLATE
    sed 's:^AuthorizedIP=.*:AuthorizedIP=127.0.0.1:' -i $CONFTEMPLATE

    tar -czf $BASENAME-bin-$PLATFORM-$TARGET$SUFFIX.tar.gz nzbget

    rm -r -f nzbget
}

BuildTarget()
{
    echo "Building $TARGET ($PLATFORM $CONFIG)"

    cd $BUILDDIR

    ConstructArchParams

    ConfigureTarget

    make clean

    PrecompileHeaders

    make -j $COREX

    PackTarget

    echo "Completed build in `pwd` for $TARGET ($CONFIG)"
}

BuildInstaller()
{
    echo "Creating installer for $PLATFORM $CONFIG..."

    cd $OUTPUTDIR

    # checking if all targets exists
    for TARGET in $TARGETS
    do
        ALLEXISTS="yes"
        if [ $TARGET != "dist" ]; then
            if [ ! -f $BASENAME-bin-$PLATFORM-$TARGET$SUFFIX.tar.gz ]; then
                echo "Could not find $BASENAME-bin-$PLATFORM-$TARGET$SUFFIX.tar.gz"
                ALLEXISTS="no"
            fi
        fi
    done

    if [ "$ALLEXISTS" == "no" ]; then
        exit 1;
    fi

    echo "Unpacking targets..."
    rm -r -f nzbget
    for TARGET in $TARGETS
    do
        ALLEXISTS="yes"
        if [ "$TARGET" != "dist" ]; then
            tar -xzf $BASENAME-bin-$PLATFORM-$TARGET$SUFFIX.tar.gz
            mv nzbget/nzbget nzbget/nzbget-$TARGET
            cp ../setup/unrar-$TARGET$PLATSUFF nzbget/unrar-$TARGET
            cp ../setup/7za-$TARGET$PLATSUFF nzbget/7za-$TARGET
        fi
    done

    # adjusting nzbget.conf
    sed 's:^UnrarCmd=unrar:UnrarCmd=${AppDir}/unrar:' -i nzbget/webui/nzbget.conf.template
    sed 's:^SevenZipCmd=7z:SevenZipCmd=${AppDir}/7za:' -i nzbget/webui/nzbget.conf.template
    sed 's:^CertStore=.*:CertStore=${AppDir}/cacert.pem:' -i nzbget/webui/nzbget.conf.template
    sed 's:^CertCheck=.*:CertCheck=yes:' -i nzbget/webui/nzbget.conf.template

    INSTFILE=$BASENAME-bin-$PLATFORM$SUFFIX.run

    echo "Building installer package..."
    cp $BUILDDIR/linux/installer.sh $INSTFILE
    cp $BUILDDIR/linux/package-info.json nzbget/webui
    cp $BUILDDIR/linux/install-update.sh nzbget
    cp $BUILDDIR/pubkey.pem nzbget
    cp ../setup/license-unrar.txt nzbget
    cp ../setup/license-7zip.txt nzbget
    cp ../setup/cacert.pem nzbget

    # adjusting update config file
    sed "s:linux:$PLATFORM:" -i nzbget/webui/package-info.json
    sed "s:linux:$PLATFORM:" -i nzbget/install-update.sh

    # creating payload
    cd nzbget
    tar czf - * > ../$INSTFILE.data
    cd ..

    # creating installer script
    sed "s:^TITLE=$:TITLE=\"$BASENAME$SUFFIX\":" -i $INSTFILE
    sed "s:^PLATFORM=$:PLATFORM=\"$PLATFORM\":" -i $INSTFILE
    DISTTARGETS="${TARGETS/dist/}"
    DISTTARGETS=`echo "$DISTTARGETS" | xargs`
    sed "s:^DISTARCHS=$:DISTARCHS=\"$DISTTARGETS\":" -i $INSTFILE

    MD5=`md5sum "$INSTFILE.data" | cut -b-32`
    sed "s:^MD5=$:MD5=\"$MD5\":" -i $INSTFILE

    PAYLOAD=`stat -c%s "$INSTFILE.data"`
    PAYLOADLEN=${#PAYLOAD}

    HEADER=`stat -c%s "$INSTFILE"`
    HEADERLEN=${#HEADER}
    HEADER=`expr $HEADER + $HEADERLEN + $PAYLOADLEN`

    TOTAL=`expr $HEADER + $PAYLOAD`
    TOTALLEN=${#TOTAL}

    HEADER=`expr $HEADER - $PAYLOADLEN + $TOTALLEN`
    TOTAL=`expr $TOTAL - $PAYLOADLEN + $TOTALLEN`

    sed "s:^HEADER=$:HEADER=$HEADER:" -i $INSTFILE
    sed "s:^TOTAL=$:TOTAL=$TOTAL:" -i $INSTFILE

    # attaching payload
    cat $INSTFILE.data >> $INSTFILE
    rm $INSTFILE.data
    chmod +x $INSTFILE

    rm -r nzbget
}

BuildDist()
{
    echo "Building dist"

    cd $BUILDDIR
    ./configure --disable-dependency-tracking --disable-cpp-check
    make dist
    cp nzbget-$VERSION.tar.gz $OUTPUTDIR/$BASENAME.tar.gz
}

BuildConfig()
{
    ConstructSuffix

    for OUTPUT in $OUTPUTS; do
        if [ $OUTPUT == "bin" ]; then
            for TARGET in $TARGETS; do
                if [ $TARGET == "dist" ]; then
                    if [ ! -f $OUTPUTDIR/$BASENAME.tar.gz ]; then
                        BuildDist
                    fi
                else
                    BuildTarget
                fi
            done
        elif [ $OUTPUT == "installer" ]; then
            BuildInstaller
        fi
    done
}

FilterTargets()
{
    TARGETS=""
    for TARGET in $BUILDTARGETS
    do
        if [[ $TARGET == *-bsd ]]; then
            PLAT="freebsd"
            TARGET="${TARGET%-bsd}"
        elif [[ $TARGET == *-ndk ]]; then
            PLAT="android"
            TARGET="${TARGET%-ndk}"
        else
            PLAT="linux"
        fi
        if [ $PLATFORM == $PLAT ]; then
              TARGETS=`echo "$TARGETS $TARGET" | xargs`
        fi
    done
}

CleanupIfNecessary()
{
    if [ $CLEAN == "yes" ]; then
        rm -r -f $OUTPUTDIR/*
    fi
}

##### Main script body #####

# Parsing command line
PrintUsage
ParseCommandLine $@
PrintConfig

# Checkout and update from git
UpdateFromRepository

# File name format for output files
DetectVersionRevision
ConstructOutputFilename

# Building
mkdir -p $OUTPUTDIR
CleanupIfNecessary

BUILDTARGETS=$TARGETS

for PLATFORM in $PLATFORMS; do
    FilterTargets
    for CONFIG in $CONFIGS; do
        BuildConfig
    done
done