build-ubunturepo

#!/bin/bash
# Update GSA Ubuntu Repositories or create a local repository
#
# Author: Leonardo Tonetto (tonetto@linux.vnet.ibm.com)
# Revisor: Arif Ali (aali@ocf.co.uk)
#
#
# Getting Started:
#  - Clone the xcat-core git repository under a directory named "xcat-core/src"
#  - make sure reprepro is installed on the build machine
#  - Run this script from the local git repository you just created.
#       ./build-ubunturepo -c BUILDALL=1

# Usage:  attr=value attr=value ... ./build-ubunturepo { -c | -d }
#               PROMOTE=1 - if the attribute "PROMOTE" is specified, means an official dot release.  This does not
#                           actually build xcat, just uploads the most recent snap build to http://xcat.org/files/xcat/ .
#                           If not specified, a snap build is assumed, which uploads to https://xcat.org/files/xcat/
#               PREGA=1 - use this option with PROMOTE=1 on a branch that already has a released dot release, but this
#                           build is a GA candidate build, not to be released yet.  This will result in the tarball
#                           being uploaded to http://xcat.org/files/xcat/repos/apt
#                           (but the tarball file name will be like a released tarball, not a snap build).
#                           When you are ready to release this build, use PROMOTE=1 without PREGA
#               BUILDALL=1 - build all rpms, whether they changed or not.  Should be used for snap builds that are in
#                            prep for a release.
#               GPGSIGN=0 -  Do not sign the repo in the end of the build. The repo will be signed by default
#
#               LOCAL_KEY=1  Use local keys to sign repo instead of WGET from GSA. By default use GSA.
#
#               SETUP=1      Setup environment for build. By default do not setup environment.
#
#               LOG=<filename> - provide an LOG file option to redirect some output into log file
#
#               DEST=<directory> - provide a directory to contains the build result
#
# For the dependency packages 1. All the xcat dependency deb packages should be uploaded to
#                                "pokgsa/projects/x/xcat/build/ubuntu/xcat-dep/debs/" on GSA
#                             2. run ./build-ubunturepo -d
#
#                             3. the built xcat-dep deb packages tarball can be found in "../../xcat-dep"
#                                related to the path of this script
############################
printusage()
{
    printf "Usage: %s {-c | -d} \n" $(basename $0) >&2
    echo "    -c : Build the xcat-core packages and create the repo"
    echo "    -d : Create the xcat-dep repo."
}
# For the purpose of getting the distribution name
if [[ ! -f /etc/lsb-release ]]; then
    echo "ERROR: Could not find /etc/lsb-release, is this script executed on a Ubuntu machine?"
    exit 1
fi
. /etc/lsb-release

export HOME=/root


# Process cmd line variable assignments, assigning each attr=val pair to a variable of same name
for i in $*; do
        echo $i | grep '=' -q
        if [ $? != 0 ];then
            continue
        fi
        # upper case the variable name
        varstring=`echo "$i"|cut -d '=' -f 1|tr '[a-z]' '[A-Z]'`=`echo "$i"|cut -d '=' -f 2`
        export $varstring
done

#Setup environment so the xcat-deps can be built on a FVT test machine
if [ "$SETUP" = "1" ];then
   #Mount GSA
   POKGSA="/gsa/pokgsa"
   POKGSA2="/gsa/pokgsa-p2"
   POKGSAIBM="pokgsa.ibm.com"
   if [ ! -d $POKGSA ];then
      mkdir -p $POKGSA
      mount ${POKGSAIBM}:${POKGSA} ${POKGSA}
   fi
   if [ ! -d $POKGSA2 ];then
      mkdir -p $POKGSA2
      mount ${POKGSAIBM}:${POKGSA2} ${POKGSA2}
   fi

   # Verify needed packages installed
   REPREPO="reprepro"
   DEVSCRIPTS="devscripts"
   DEBHELPER="debhelper"
   QUILT="quilt"

   apt-get -y install $REPREPO $DEVSCRIPTS $DEBHELPER $QUILT

   echo "Finised setup for xcat-dep build. Rerun this script with SETUP=0 LOCAL_KEY=1 flags"
   exit 1
fi

# Check the necessary packages before starting the build
declare -a packages=( "reprepro" "devscripts" "debhelper" "libsoap-lite-perl" "libdbi-perl" "quilt" "git")

for package in ${packages[@]}; do
    RC=`dpkg -l | grep $package >> /dev/null 2>&1; echo $?`
    if [[ ${RC} != 0 ]]; then
        echo "ERROR: Could not find $package, install using 'apt-get install $package' to continue"
        exit 1
    fi
done

# Supported distributions
dists="saucy trusty utopic xenial bionic focal jammy"

c_flag=            # xcat-core (trunk-delvel) path
d_flag=            # xcat-dep (trunk) path
r_flag=            #genesis base rpm package path

while getopts 'cdr:' OPTION
do
    case $OPTION in
	c)    c_flag=1
	        ;;
	d)    d_flag=1
	        ;;
        r)    r_flag=1
                genesis_rpm_path="$OPTARG"
                ;;
	?)  printusage
	    exit 2
	    ;;
    esac
done
shift $(($OPTIND - 1))

if [ -z "$c_flag" -a -z "$d_flag" ];then
    printusage
    exit 2
fi

if [ "$c_flag" -a "$d_flag" ];then
    printusage
    exit 2
fi

if [ -z "$BUILDALL" ]; then
    BUILDALL=1
fi

# Find where this script is located to set some build variables
old_pwd=`pwd`
cd `dirname $0`
curdir=`pwd`

# Use flock to only one person build at the same time
# Get a lock, so can not do 2 builds at once
exec 8>/var/lock/xcatbld.lock
if ! flock -n 8; then
    echo "ERROR: Can't get lock /var/lock/xcatbld.lock.  Someone else must be doing a build right now.  Exiting...."
    exit 1
fi

# for the git case, query the current branch and set REL (changing master to devel if necessary)
function setbranch {
    # Get the current branch name
    branch=`git rev-parse --abbrev-ref HEAD`
    if [ "$branch" = "master" ]; then
        REL="devel"
    elif [ "$branch" = "HEAD" ]; then
        # Special handling when in a 'detached HEAD' state
        branch=`git describe --abbrev=0 HEAD`
        [[ -n "$branch" ]] && REL=`echo $branch|cut -d. -f 1,2`
    else
        REL=$branch
    fi
}

WGET_CMD="wget"
if [ ! -z ${LOG} ]; then
    WGET_CMD="wget -o ${LOG}"
fi

if [ "$GPGSIGN" = "0" ];then
    echo "GPGSIGN=$GPGSIGN specified, skip gnupg key downloading"
else
    #sync the gpg key to the build machine local
    gsa_url=http://pokgsa.ibm.com/projects/x/xcat/build/linux
    mkdir -p $HOME/.gnupg
    for key_name in pubring.gpg secring.gpg trustdb.gpg; do
        if [ "$LOCAL_KEY" = "1" ];then
            # Keys are already in the local $HOME/.gnupg directory
            chmod 600 $HOME/.gnupg/$key_name
        else
            # Need to download keys from GSA
            if [ ! -f $HOME/.gnupg/$key_name ] || [ `wc -c $HOME/.gnupg/$key_name|cut -f 1 -d' '` == 0 ]; then
                rm -f $HOME/.gnupg/$key_name
                ${WGET_CMD} -P $HOME/.gnupg $gsa_url/keys/$key_name
                chmod 600 $HOME/.gnupg/$key_name
            fi
        fi
    done
fi

REL=xcat-core
if [ "$c_flag" ]
then
    setbranch
    package_dir_name=debs$REL

    #define the dep source code path, core build target path and dep build target path
    if [ -z "$DEST" ]; then
        local_core_repo_path="$curdir/../../xcat-core"
        PKGDIR="../../$package_dir_name"
    else
        local_core_repo_path="$DEST/$package_dir_name/xcat-core"
        PKGDIR="$DEST/$package_dir_name/$package_dir_name"
    fi
    if [ ! -d "$PKGDIR" ];then
        mkdir -p "$PKGDIR"
    fi

    echo "#############################################################"
    echo "Building xcat-core on branch ($REL) to $local_core_repo_path"
    echo "#############################################################"
    if [ "$PROMOTE" != 1 ]; then
        code_change=0
        update_log=''

        if [ -z "$GITUP" ];then
            update_log=../coregitup
            echo "git pull > $update_log"
            git pull > $update_log
        else
            update_log=$GITUP
        fi

        if ! grep -q  'Already up-to-date' $update_log; then
            code_change=1
        fi
        ver=`cat Version`
        short_ver=`cat Version|cut -d. -f 1,2`
        short_short_ver=`cat Version|cut -d. -f 1`
        build_time=`date`
        build_machine=`hostname`
        commit_id_long=`git rev-parse HEAD`
        commit_id="${commit_id_long:0:7}"

        if [ $code_change == 0 -a "$UP" != 1 -a "$BUILDALL" != 1 ]; then
            echo "Nothing new detected.  Exiting...."
            exit 0
        fi

        echo "###############################"
        echo "# Building xcat-core packages #"
        echo "###############################"

        #the package type:  local | snap | alpha
        #the build introduce string
        build_string="Snap_Build"
        xcat_release="snap$(date '+%Y%m%d%H%M')"
        pkg_version="${ver}-${xcat_release}"

        packages="xCAT-client xCAT-genesis-scripts perl-xCAT xCAT-server xCAT xCATsn xCAT-test xCAT-buildkit xCAT-vlan xCAT-confluent xCAT-probe"
        target_archs=(amd64 ppc64el)
        for file in $packages
        do
            file_low=`echo $file | tr '[A-Z]' '[a-z]'`
            if [ "$file" = "xCAT" -o "$file" = "xCAT-genesis-scripts" -o "$file" = "xCATsn" ]; then
                target_archs="amd64 ppc64el"
            else
                target_archs="all"
            fi
            for target_arch in $target_archs
            do
                if grep -q $file $update_log || [ "$BUILDALL" == 1 -o "$file" = "perl-xCAT" ]; then
                    rm -f $PKGDIR/${file_low}_*.$target_arch.deb
                    cd $file
                    CURDIR=$(pwd)
                    dch -v $pkg_version -b -c debian/changelog $build_string
                    if [ "$target_arch" = "all" ]; then
                        #xcat probe use some functions shipped by xCAT, for below reasons we need to copy files to xCAT-probe directory
                        #1 make xcat probe code to be self-contained
                        #2 don't maintain two files for each script
                        #3 symbolic link can't work during package
                        if [ $file_low = "xcat-probe" ]; then
                            mkdir -p ${CURDIR}/lib/perl/xCAT/
                            cp -f ${CURDIR}/../perl-xCAT/xCAT/NetworkUtils.pm ${CURDIR}/lib/perl/xCAT/
                            cp -f ${CURDIR}/../perl-xCAT/xCAT/GlobalDef.pm ${CURDIR}/lib/perl/xCAT/
                            cp -f ${CURDIR}/../perl-xCAT/xCAT/ServiceNodeUtils.pm ${CURDIR}/lib/perl/xCAT/
                        fi
                        CURDIR=$(pwd)
                        cp ${CURDIR}/debian/control ${CURDIR}/debian/control.save.998
                        # Magic string used here
                        sed -i -e "s#>= 2.13-snap000000000000#= ${pkg_version}#g" ${CURDIR}/debian/control
                        dpkg-buildpackage -rfakeroot -uc -us
                        mv ${CURDIR}/debian/control.save.998 ${CURDIR}/debian/control
                    else
                        if [ "$file" = "xCAT-genesis-scripts" ]; then
                            echo "Rename control file to build pkg: mv ${CURDIR}/debian/control-${target_arch} ${CURDIR}/debian/control"
                            cp ${CURDIR}/debian/control-${target_arch} ${CURDIR}/debian/control
                        elif [ "$file" = "xCAT" ]; then
                            # shipping bmcsetup and getipmi scripts as part of postscripts
                            files=("bmcsetup" "getipmi")
                            for f in "${files[@]}"; do
                                 cp ${CURDIR}/../xCAT-genesis-scripts/usr/bin/$f ${CURDIR}/postscripts/$f
                                 sed -i "s/xcat.genesis.$f/$f/g" ${CURDIR}/postscripts/$f
                            done
                        fi
                        CURDIR=$(pwd)
                        cp ${CURDIR}/debian/control ${CURDIR}/debian/control.save.998
                        # Magic string used here
                        sed -i -e "s#>= 2.13-snap000000000000#= ${pkg_version}#g" ${CURDIR}/debian/control
                        dpkg-buildpackage -rfakeroot -uc -us -a$target_arch
                        mv ${CURDIR}/debian/control.save.998 ${CURDIR}/debian/control
                        if [ "$file" = "xCAT-genesis-scripts" ]; then
                            echo "Move control file back: mv ${CURDIR}/debian/control ${CURDIR}/debian/control-${target_arch}"
                            rm ${CURDIR}/debian/control
                        elif [ "$file" = "xCAT" ]; then
                            files=("bmcsetup" "getipmi")
                            for f in "${files[@]}"; do
                                rm -f ${CURDIR}/postscripts/$f
                            done
                        fi
                    fi
                    rc=$?
                    if [ $rc -gt 0 ]; then
                        echo "Error: $file build package failed exit code $rc"
                        exit $rc
                    fi
                    cd -
                    find $file -maxdepth 3 -type d -name "${file_low}*" | grep debian | xargs rm -rf
                    find $file -maxdepth 3 -type f -name "files" | grep debian | xargs rm -rf
                    mv ${file_low}* $PKGDIR/
                fi
            done
        done

        find $PKGDIR/* ! -name *.deb | xargs rm -f
    fi

    if [ "$PROMOTE" = 1 ]; then
        upload_dir="xcat-core"
        tar_name="xcat-core-$ver.tar.bz2"
    else
        upload_dir="core-snap"
        tar_name="core-debs-snap.tar.bz2"
    fi

    echo "#################################"
    echo "# Creating xcat-core repository #"
    echo "#################################"

    #clean the repo directory
    if [ -e $local_core_repo_path ]; then
    	rm -rf $local_core_repo_path
    fi
    mkdir -p $local_core_repo_path
    cd $local_core_repo_path
    mkdir conf

    for dist in $dists; do
        # for all releases moving forward, support amd64 and ppc64el
        tmp_out_arch="amd64 ppc64el"
        if [ "$dist" = "saucy" ]; then
            # for older releases of Ubuntu that does not support ppc64el
            tmp_out_arch="amd64"
        fi
        cat << __EOF__ >> conf/distributions
Origin: xCAT internal repository
Label: xcat-core bazaar repository
Codename: $dist
Architectures: $tmp_out_arch
Components: main
Description: Repository automatically genereted conf
__EOF__

       if [ "$GPGSIGN" = "0" ];then
           #echo "GPGSIGN=$GPGSIGN specified, the repo will not be signed"
           echo "" >> conf/distributions
       else
           keyid=$(gpg --list-keys --keyid-format long "xCAT Automatic Signing Key" | grep '^pub' | sed -e 's/.*\///' -e 's/ .*//')
           echo "SignWith: $keyid" >> conf/distributions
           echo "" >> conf/distributions
       fi
    done

    cat << __EOF__ > conf/options
verbose
ask-passphrase
basedir .
__EOF__

    #import the deb packages into the repo
    amd_files=`ls ../$package_dir_name/*.deb | grep -v "ppc64el"`
    all_files=`ls ../$package_dir_name/*.deb`
    for dist in $dists; do
        deb_files=$all_files
        if [ "$dist" = "saucy" ]; then
            # for older releases of Ubuntu that does not support ppc64el
            deb_files=$amd_files
        fi
        for file in $deb_files; do
            reprepro -b ./ includedeb $dist $file;
        done
    done
    #create the mklocalrepo script
    cat << '__EOF__' > mklocalrepo.sh
. /etc/lsb-release
cd `dirname $0`
host_arch=`uname -m`
if [ "$host_arch" != "ppc64le" ];then
    host_arch="amd64"
else
    host_arch="ppc64el"
fi
echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-core.list
__EOF__

    chmod 775 mklocalrepo.sh

    #
    # Add a buildinfo file into the tar.bz2 file to track information about the build
    #
    BUILDINFO=$local_core_repo_path/buildinfo
    echo "VERSION=$ver" > $BUILDINFO
    echo "RELEASE=$xcat_release" >> $BUILDINFO
    echo "BUILD_TIME=$build_time" >> $BUILDINFO
    echo "BUILD_MACHINE=$build_machine" >> $BUILDINFO
    echo "COMMIT_ID=$commit_id" >> $BUILDINFO
    echo "COMMIT_ID_LONG=$commit_id_long" >> $BUILDINFO

    #create the xcat-core.list file

    cd ../
    if ! grep xcat /etc/group ; then
        groupadd xcat
    fi

    chgrp -R root xcat-core
    chmod -R g+w xcat-core

    #build the tar ball
    echo "Creating `pwd`/$tar_name ..."
    tar -hjcf $tar_name xcat-core
    chgrp root $tar_name
    chmod g+w $tar_name

    if [ -n "$DEST" ]; then
        ln -sf $(basename `pwd`)/$tar_name ../$tar_name
        if [ $? != 0 ]; then
            echo "ERROR: Failed to make symbol link $DEST/$tar_name"
        fi
    fi

    if [ ! -e core-snap ]; then
        ln -s xcat-core core-snap
    fi

    cd $old_pwd
    exit 0
fi

if [ "$d_flag" ]
then
    echo "################################"
    echo "# Creating xcat-dep repository #"
    echo "################################"

    #the path of ubuntu xcat-dep deb packages on GSA
    GSA="/gsa/pokgsa/projects/x/xcat/build/ubuntu/xcat-dep"
    if [ ! -d $GSA ]; then
       echo "build-ubunturepo:  It appears that you do not have GSA to access the xcat-dep pkgs."
       exit 1;
    fi

    #define the dep source code path, core build target path and dep build target path
    if [ -z "$DEST" ]; then
        local_dep_repo_path="$curdir/../../xcat-dep/xcat-dep"
    else
        local_dep_repo_path="$DEST/xcat-dep/xcat-dep"
    fi

    # Sync from the GSA master copy of the dep rpms
    echo "Creating directory $local_dep_repo_path"
    mkdir -p $local_dep_repo_path/

    echo "Syncing RPMs from $GSA/ to $local_dep_repo_path/../ ..."
    rsync -ilrtpu --delete $GSA/ $local_dep_repo_path/../
    if [ $? -ne 0 ]; then
        echo "Error from rsync, cannot continue!"
        exit 1
    fi

    #clean all old files
    if [ -e $local_dep_repo_path ];then
        rm -rf $local_dep_repo_path
    fi
    mkdir -p $local_dep_repo_path
    cd $local_dep_repo_path
    mkdir conf


    #create the conf/distributions file
    for dist in $dists; do
        tmp_out_arch="amd64 ppc64el"
        if [ "$dist" = "saucy" ]; then
            # for older releases of Ubuntu that does not support ppc64el
            tmp_out_arch="amd64"
        fi
        cat << __EOF__ >> conf/distributions
Origin: xCAT internal repository
Label: xcat-dep bazaar repository
Codename: $dist
Architectures: $tmp_out_arch
Components: main
Description: Repository automatically genereted conf
__EOF__

        if [ "$GPGSIGN" = "0" ];then
            echo "GPGSIGN=$GPGSIGN specified, the repo will not be signed"
            echo "" >> conf/distributions
        else
            keyid=$(gpg --list-keys --keyid-format long "xCAT Automatic Signing Key" | grep '^pub' | sed -e 's/.*\///' -e 's/ .*//')
            echo "SignWith: $keyid" >> conf/distributions
            echo "" >> conf/distributions
        fi

    done



    cat << __EOF__ > conf/options
verbose
ask-passphrase
basedir .
__EOF__

    #import the deb packages into the repo
    amd_files=`ls ../debs/*.deb | grep -v "ppc64el"`
    all_files=`ls ../debs/*.deb`
    for dist in $dists; do
        deb_files=$all_files
        if [ "$dist" = "saucy" ]; then
            # for older releases of Ubuntu that does not support ppc64el
            deb_files=$amd_files
        fi
        for file in $deb_files; do
            reprepro -b ./ includedeb $dist $file;
        done
    done

    cat << '__EOF__' > mklocalrepo.sh
. /etc/lsb-release
cd `dirname $0`
host_arch=`uname -m`
if [ "$host_arch" != "ppc64le" ];then
    host_arch="amd64"
else
    host_arch="ppc64el"
fi
echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-dep.list
__EOF__

    chmod 775 mklocalrepo.sh

    cd ..
    if ! grep xcat /etc/group ; then
        groupadd xcat
    fi

    chgrp -R root xcat-dep
    chmod -R g+w xcat-dep

    #create the tar ball
    dep_tar_name=xcat-dep-ubuntu-`date +%Y%m%d%H%M`.tar.bz2
    tar -hjcf $dep_tar_name xcat-dep
    chgrp root $dep_tar_name
    chmod g+w $dep_tar_name


    USER="xcat"
    SERVER="xcat.org"
    FILES_PATH="files"
    FRS="/var/www/${SERVER}/${FILES_PATH}"
    APT_DIR="${FRS}/xcat"
    APT_REPO_DIR="${APT_DIR}/repos/apt/devel"

    # Decide whether to upload the xcat-dep package or NOT (default is to NOT upload xcat-dep
    if [ "$UP" != "1" ]; then
        echo "Upload not specified, Done! (rerun with UP=1, to upload)"
        cd $old_pwd
        exit 0
    fi

    #upload the dep packages
    i=0
    echo "Uploading debs from xcat-dep to ${APT_REPO_DIR}/xcat-dep/ ..."
    while [ $((i+=1)) -le 5 ] && ! rsync -urLv --delete xcat-dep $USER@${SERVER}:${APT_REPO_DIR}/
    do : ; done

    #upload the tarball
    i=0
    echo "Uploading $dep_tar_name to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..."
    while [ $((i+=1)) -le 5 ] && ! rsync -v --force $dep_tar_name $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/
    do : ; done

    #upload the README file
    cd debs
    i=0
    echo "Uploading README to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..."
    while [ $((i+=1)) -le 5 ] && ! rsync -v --force README $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/
    do : ; done

fi

cd $old_pwd
exit 0