path: root/build_tools/bin/mp.py

#!/usr/bin/env python2
# Version 0.6.9

import os
import sys
import re
import subprocess
import gettext
from git import Repo,Git
import ConfigParser
import glob
import shutil
import optparse
import fileinput

PKGHOME = "/data/pkg_repo/packages"
SRCPKGHOME = "/data/pkg_repo/src_packages"
mydir = os.getcwd()
repolist = ['core', 'extra']
variables = ['pkgbase','pkgname','pkgver','pkgrel','arch']
cli_dict = {}
cli_list = []
makepkg_cmd = ['makepkg']
REPO = "none"
SFIX = ""
DOCROOT = ""
pkgfile = "none"
pkgbase = ""
pkgname = ""
pkgver = ""
pkgrel = ""
arch = ""
CARCH = ""
PKGDEST = ""
TOTALPKG = ""
GZPKG = ""
XZPKG = ""

# See what git branch we're working under
git_repo = Git()
branches = git_repo.branch()
git_branch = branches.split('*')[1].lstrip(' ')

if git_branch == "testing":
    SFIX = "-testing"
elif git_branch == "master":
    SFIX = ""
else:
    print "Can't determine which git branch is in use!"
    sys.exit(2)
print "Git Branch suffix is:",SFIX

# Color stuff -- does this work? Looks to bash-y to me.
ALL_OFF = "$(tput sgr0)"
BOLD = "$(tput bold)"
BLUE = "${BOLD}$(tput setaf 4)"
GREEN = "${BOLD}$(tput setaf 2)"
RED = "${BOLD}$(tput setaf 1)"
YELLOW = "${BOLD}$(tput setaf 3)"

def plain():
    mesg = list.pop()
    print "${BOLD}    "+mesg+ "${ALL_OFF}\n"+cmdline

def msg():
    mesg = list.pop()
    print "${GREEN}==>${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline

def msg():
    mesg = list.pop()
    print "${BLUE}  ->${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline

def warning():
    warning = gettext(WARNING)
    mesg = list.pop()
    print "${YELLOW}==>"+ warning+ "${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline

def error():
    error = gettext(ERROR)
    mesg = list.pop()
    print "${RED}==>"+ error+ "${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline

def commandline():
    global makepkg_cmd
    global cli_list
    global cli_dict

    clparser = optparse.OptionParser()
    clparser.add_option("--asroot", action="store_true", default=True, help="Allow makepkg to run as root.")
    clparser.add_option("-A", "--ignorearch", action="store_true", default=False, help="Ignore a missing or incomplete arch field in the build script.")
    clparser.add_option("-b", "--bump", action="store_true", default=False, help="Increase package release one unit.")
    clparser.add_option("-c", "--clean", action="store_true", default=False, help="Clean up leftover work files and directories after a successful build. ")
    clparser.add_option("-C", "--cleancache", action="store_true", default=False, help="Removes all cached source files from the directory specified in SRCDEST in makepkg.conf")
    clparser.add_option("--config", action="store", help="Use an alternate config file instead of the /etc/makepkg.conf default.")
    clparser.add_option("-d", "--nodeps", action="store_true", default=False, help="Do not perform any dependency checks.")
    clparser.add_option("-e", "--noextract", action="store_true", default=False, help="Do not extract source files; use whatever source already exists in the src/ directory.")
    clparser.add_option("-f", "--force", action="store_true", default=False, help="This allows a built package to be overwritten.")
    clparser.add_option("--forcever", action="store_true", default=False, help="This is a hidden option that should not be used unless you really know what you are doing.")
    clparser.add_option("-g", "--geninteg", action="store_true", default=False, help="For each source file in the source array of PKGBUILD, download the file if required and generate integrity checks.")
    clparser.add_option("--skipinteg", action="store_true", default=False, help="Do not perform any integrity checks, just print a warning instead.")
    clparser.add_option("--holdver", action="store_true", default=True, help="Prevents makepkg from automatically bumping the pkgver to the latest revision number in the package's development tree.")
    clparser.add_option("--nohold", action="store_false", dest="holdver", help="Automatically bump the pkgver to the latest revision number in the package's development tree.")
    clparser.add_option("-i", "--install", action="store_true", default=False, help="Install or upgrade the package after a successful build.")
    clparser.add_option("-L", "--log", action="store_true", default=True, help="Enable makepkg build logging.")
    clparser.add_option("-m", "--nocolor", action="store_true", default=False, help="Disable color in output messages.")
    clparser.add_option("-o", "--nobuild", action="store_true", default=False, help="Download and extract files only, but do not build them.")
    clparser.add_option("-p", action="store", help="Read the package script buildscript instead of the PKGBUILD default.", metavar='/path/to/buildscript')
    clparser.add_option("-r", "--rmdeps", action="store_true", default=False, help="Upon successful build, remove any dependencies installed by makepkg during dependency auto-resolution and installation when using -s.")
    clparser.add_option("-R", "--repackage", action="store_true", default=False, help="Repackage contents of the package without rebuilding the package.")
    clparser.add_option("-s", "--syncdeps", action="store_true", default=False, help="Install missing dependencies using pacman.")
    clparser.add_option("--allsource", action="store_true", default=False, help="Do not actually build the package, but build a source-only tarball that includes all sources, including those that are normally download via makepkg.")
    clparser.add_option("--source", action="store_true", default=False, help="Do not actually build the package, but build a source-only tarball that does not include sources that can be fetched via a download URL.")
    clparser.add_option("--pkg", action="store", help="Only build listed packages from a split package.")
    clparser.add_option("--noconfirm", action="store_true", default=False, help="(Passed to pacman) Prevent pacman from waiting for user input before proceeding with operations.")
    clparser.add_option("--noprogressbar", action="store_true", default=False, help="(Passed to pacman) Prevent pacman from displaying a progress bar.")
    clparser.add_option("--rmold", action="store_true", default=False, help="BETA: Remove old src and software packages from repos. Use with caution. False positives may occur (i.e. nvidia pkgs)!")

    (options, args) = clparser.parse_args()

    options1 = ['config', 'p', 'pkg']
    options2 = ['asroot', 'ignorearch', 'bump', 'clean', 'cleancache', 'nodeps',
               'noextract', 'force', 'forcever', 'geninteg', 'skipinteg', 'holdver',
               'install', 'log', 'nocolor', 'nobuild', 'rmdeps', 'repackage',
               'syncdeps', 'allsource', 'source', 'noconfirm', 'noprogressbar',
               'rmold']

    for o in options1:
        cmd1 = eval('options.'+o)
        if o is not 'p':
            if cmd1 is not None:
                cli_dict['--'+o] = cmd1
        elif cmd1 is not None:
                cli_dict['-'+o] = cmd1
    for o in options2:
        cmd2 = eval('options.'+o)
        if cmd2 is True:
            cli_list.append('--'+o)

    # Create makepkg command
    makepkg_cmd.extend(cli_list)
    for k in cli_dict.keys():
        v = cli_dict.get(k)
        makepkg_cmd.append(k)
        makepkg_cmd.append(v)

    # Remove bump option from makepkg command if it exists
    if "--bump" in makepkg_cmd:
        makepkg_cmd.remove("--bump")
    # Remove rmold option from makepkg command if it exists
    if "--rmold" in makepkg_cmd:
        makepkg_cmd.remove("--rmold")
    # Remove "--geninteg" option (if it exists) from makepkg command.
    # Checking for *sums is done automaticly by mp.py
    if "--geninteg" in makepkg_cmd:
        makepkg_cmd.remove("--geninteg")
    print "Makepkg Command:",makepkg_cmd

def bump_pkg():
    # We need pkgrel to live beyond this function
    global SRCPKG
    global pkgrel

    # Backup the original pkgfile
    shutil.copy2(pkgfile, pkgfile + ".old")
    # Let's increase the pkgrel
    for line in fileinput.input(pkgfile, inplace=1):
        if line.strip().startswith("pkgrel"):
            pkgrel = line.partition("=")[2].strip('\n')
            # Add 1 to pkgrel
            new_pkgrel = int(pkgrel) + 1
            line = line.replace("pkgrel=" + pkgrel, "pkgrel=" + str(new_pkgrel))
            pkgrel = str(new_pkgrel)
        sys.stdout.write(line)
    print "Bumped " + pkgname + " release to " + str(new_pkgrel)

    if pkgbase:
        SRCPKG = pkgbase + "-" + pkgver + "-" + str(new_pkgrel) + ".src.tar.gz"
    else:
        SRCPKG = pkgname + "-" + pkgver + "-" + str(new_pkgrel) + ".src.tar.gz"
    print "Source package will be: ",SRCPKG

def find_repo():
    global REPO
    global DOCROOT

    # Get all our possible repos into one list
    for item in ("chroot-devel", "mv-core", "xmpl", "local"):
        repolist.append(item)
    print "Repo List:",repolist

    # Create a list with the directory names of our current directory tree
    dir_tree = os.path.dirname(mydir).split("/")
    print "Dir Tree:",dir_tree
    # Loop through the dir_tree to see if we can find our repo
    for item in repolist: 
        if item not in dir_tree:
            continue
        else:
            repo_name = item
        if repo_name == "extra":
            REPO = "extra" + SFIX
        elif repo_name == "core":
            REPO = "core" + SFIX
        elif repo_name == "xmpl":
            REPO = "local"
        elif repo_name not in repolist:
            print "ERROR in function find_repo: Cannot determine repository!"
            sys.exit(2)
        else:
            REPO = repo_name
    # Ensure our DOCROOT exists and if not, create it
    DOCROOT = "/data/pkg_repo/" + CARCH + "/" + REPO
    print "DOCROOT:",DOCROOT
    if os.path.exists(DOCROOT):
        print "INFO: Repository is",REPO
    else:
        os.mkdir(DOCROOT,0755)
        print "INFO: ",DOCROOT,"directory created. Repository is",REPO

def update_repo():
    # pkgname could be a list of several pkgs. Since bash array format is
    # loose, let bash parse the pkgname(s) first, then return a list for us.
    os.chdir(DOCROOT)

    print
    print "INFO: Changed working dir to",DOCROOT
    print

    for i in pkglist:
        print "Package name:",i

        GZPKG = i + "-" + pkgver + "-" + pkgrel + "-" + CARCH + ".pkg.tar.gz"
        XZPKG = i + "-" + pkgver + "-" + pkgrel + "-" + CARCH + ".pkg.tar.xz"
        if os.path.isfile(PKGDEST + "/" + XZPKG):
            TOTALPKG = XZPKG
        elif os.path.isfile(PKGDEST + "/" + GZPKG):
            TOTALPKG = GZPKG
        else:
            print "ERROR in function update_repo: Couldn't find the new package",PKGDEST + "/" + TOTALPKG
            sys.exit(2)

        print "Package file:",TOTALPKG
        # Remove old package(s) from local copy
        pv = re.compile('[\d]+[\w.]+')
        dirlist = os.listdir(DOCROOT)
        for n in dirlist:
            if n.startswith(i):
                num = len(i) + 1
                print "Bytes to seek ahead:",num
                OLDPKG = glob.glob(i + "-" + pv.search(n,num).group() + "-[0-9]*.pkg.tar.?z")
                print "OLDPKG =",OLDPKG
                if OLDPKG:
                    for DELPKG in OLDPKG:
                        if "--rmold" in cli_list:
                            print "Deleting old package:",DELPKG
                            os.remove(DELPKG)
                    # Remove any symlinks to old packages
                    # We make it conditional on "--force" because force will overwrite
                    # an existing package and we want the symlink to stay, pointing to
                    # the newly built package with the same pkgrel.
                        if "--force" not in makepkg_cmd:
                            if os.path.islink(mydir + "/" + DELPKG):
                                os.remove(mydir + "/" + DELPKG)
        # Copy in new package
        print "############################################"
        print "Updating " + DOCROOT + " with " + TOTALPKG
        print "Copying " + PKGDEST + "/" + TOTALPKG
        shutil.copy2(PKGDEST + "/" + TOTALPKG, DOCROOT)
        subprocess.call(["repo-add", DOCROOT+ "/" + REPO + ".db.tar.gz", DOCROOT + "/" + TOTALPKG])
        print "############################################"

def update_src_pkg():
    print "---------------------------------SRC------------------------------"
    print "SRCPKG:",SRCPKG
    OLDSRCPKG = []
    os.chdir(SRCPKGHOME + "/" + REPO)
    dirlist = os.listdir(SRCPKGHOME + "/" + REPO)
    #pv = re.compile('[\d]+[.]*[\d]*[.]*[\d]*[.]*[\d]*[.]*[\d]*')
    pv = re.compile('[\d]+[\w.]+')
    if pkgbase:
        # Remove old src package(s) from local copy
        if "--rmold" in cli_list:
            for n in dirlist:
                if n.startswith(pkgbase):
                    num = len(pkgbase) + 1
                    print "Bytes to seek ahead:",num
                    OLDSRCPKG = glob.glob(pkgbase + "-" + pv.search(n,num).group() + "-*.src.tar.?z")
    else:
        # Remove old src package(s) from local copy
        if "--rmold" in cli_list:
            for n in dirlist:
                if n.startswith(pkgname):
                    num = len(pkgname) + 1
                    print "Bytes to seek ahead:",num
                    OLDSRCPKG = glob.glob(pkgname + "-" + pv.search(n,num).group() + "-*.src.tar.?z")
    if OLDSRCPKG:
        print "OLDSRCPKG =",OLDSRCPKG
        for DELSRCPKG in OLDSRCPKG:
            print "Deleting old source package",DELSRCPKG
            os.remove(DELSRCPKG)
    print "Copying new source package to",SRCPKGHOME + "/" + REPO + "/" + SRCPKG
    shutil.copy2(PKGHOME + "/" + SRCPKG, SRCPKGHOME + "/" + REPO + "/")

def dup_check():
    if REPO in ("local", "mv-core", "xmpl", "chroot-devel"):
        return
    for tmp_repo in ("core", "extra"):
        if tmp_repo + SFIX != REPO:
            p1 = subprocess.Popen(["pacman", "-Sl", tmp_repo + SFIX], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,)
            p2 = subprocess.Popen(["cut", "-d", " ", "-f", "2"], stdin=p1.stdout, stdout=subprocess.PIPE,)
            p3 = subprocess.Popen(["grep", pkgname], stdin=p2.stdout, stdout = subprocess.PIPE,)
            output = p3.communicate()[0].strip('\n')
            if output == 0:
                if "--force" not in cli_list:
                    print "#######################################"
                    print pkgname," already exists in ",tmp_repo + SFIX
                    print 
                    print "Use --force to overwite"
                    print "#######################################"
                    sys.exit(2)
                else:
                    print "Force detected! Making package regardless!"
                    print "#######################################"
            else:
                if output != 0:
                    return

def updateMD5():
    # Do we have integrity checks?
    sums = []

    # Open pkgfile for reading
    f = open(pkgfile, 'r')
    pkgfile_contents = f.readlines()
    f.close()

    # Iterate through the lines looking for a match pattern
    for line in pkgfile_contents:
        check = re.compile('sha1sums|sha256sums|sha384sums|sha512sums|md5sums')
        sums = check.match(line)
        if sums:
            break
    if not sums:
        # If no matches are found, append md5sums to the end of the pkgfile
        p = open(pkgfile, 'a')
        md5gen = subprocess.Popen(["makepkg", "--asroot", "-g"], stdout = subprocess.PIPE,).communicate()[0]
        p.writelines(md5gen)
        p.close()

def config_file():
    global pkgfile
    global pkgbase
    global pkgname
    global pkglist
    global pkgver
    global pkgrel
    global arch
    global SRCPKG
    global CARCH
    global PKGDEST

    # Check what file will be used for our PKGBUILD
    if "-p" in cli_dict:
        pkgfile = cli_dict["-p"]
    else:
        pkgfile = "PKGBUILD"
    print "Config File:",pkgfile

    # Does the file exist?
    if not os.path.isfile(pkgfile):
        print "ERROR in function config_file: Can't find ",pkgfile,"!"
        sys.exit(2)
    else:
        # Loop over contents to get our variables
        # Use bash to do it because PKGBUILDs are very loose with their format
        for item in variables:
            v = subprocess.Popen(['/bin/bash','-c', 'source ' + pkgfile + '; echo ${' + item + '[@]}'], stdout = subprocess.PIPE,)
            value = v.communicate()[0].strip('\n')
            if item == "pkgbase":
                pkgbase = value
                print "pkgbase:",pkgbase
            elif item == "pkgname":
                pkgname = value
                print "pkgname:",pkgname
                pkglist = list(value.split())
                print "pkglist:",pkglist
            elif item == "pkgver":
                pkgver = value
                print "pkgver:",pkgver
            elif item == "pkgrel":
                pkgrel = value
                print "pkgrel:",pkgrel
            elif item == "arch":
                arch = value
                print "arch:",arch

    if pkgbase:
        SRCPKG = pkgbase + "-" + pkgver + "-" + pkgrel + ".src.tar.gz"
    else:
        SRCPKG = pkgname + "-" + pkgver + "-" + pkgrel + ".src.tar.gz"
    print "Source package will be:",SRCPKG

    # Get needed makepkg.conf variables
    mpkg="/etc/makepkg.conf"
    f=open(mpkg,"r")
    # Read file contents into memory
    makepkg_contents=f.readlines()
    f.close()

    # Loop over contents to get our variables
    for mp_lines in makepkg_contents:
        if mp_lines.strip().startswith("CARCH"):
            CARCH = mp_lines.partition("=")[2].strip('\n').replace('\"','')
            if arch != 'any':
                arch = CARCH
        if mp_lines.strip().startswith("PKGDEST"):
            PKGDEST = mp_lines.partition("=")[2].strip('\n')
    print "CARCH is:",CARCH
    print "Package destination is:",PKGDEST
    print "Architecture is:",arch

def make_package():
    global arch
    global CARCH
    # Make the package!
    retcode = subprocess.call(makepkg_cmd)
    if retcode != 0:
        print "ERROR: makepkg failed with return code ",retcode
        if "--bump" in cli_list:
            if os.path.isfile(pkgfile + '.old'):
                shutil.move(pkgfile + '.old', pkgfile)
                print "Reverted pkgrel bump."
        sys.exit(2)
    elif "--nobuild" in makepkg_cmd:
        sys.exit(0)
    else:
        print "=============FINSHED CREATING PKG=================="
        if arch == 'any':
            CARCH = 'any'

# MAIN PROGRAM
commandline()
config_file()
find_repo()
dup_check()

# If "bump" was called, run the bump_pkg function
if "--bump" in cli_list:
    bump_pkg()

updateMD5()

# Everything look good? Go for it!
make_package()

print "------------updating database-------------"
update_repo()

print "----------creating source package---------"
os.chdir(mydir)
retcode = subprocess.call(["makepkg", "--force", "--holdver", "--asroot", "--source"])
if retcode != 0:
    print "ERROR: Source package creation failed with return code",retcode
    sys.exit(2)
else:
    update_src_pkg()

# Clean up old PKGBUILD if it exists
os.chdir(mydir)
if os.path.isfile(pkgfile + '.old'):
    os.remove(pkgfile + '.old')
    print "Removed temporary backup file",pkgfile + '.old'

# Remove any old -build and -package log files
os.chdir(mydir)
build_log = pkgname + "-" + pkgver + "-" + pkgrel + "-" + CARCH + "-build.log"
pkg_log = pkgname + "-" + pkgver + "-" + pkgrel + "-" + CARCH + "-package.log"
all_logs = glob.glob("*.log*")
#print "All Logs:",all_logs
saved_logs = []
if os.path.isfile(build_log):
    saved_logs.append(build_log)
if os.path.isfile(pkg_log):
    saved_logs.append(pkg_log)
#print "Saved Logs:",saved_logs
for log in all_logs:
    if log not in saved_logs:
        os.remove(log)