mp.py: Initial inclusion.Test away.

1 files changed, 418 insertions, 0 deletions

diff --git a/build_tools/bin/mp.py b/build_tools/bin/mp.py
new file mode 100755
index 0000000..77f73c8
--- /dev/null
+++ b/build_tools/bin/mp.py
@@ -0,0 +1,418 @@
+#!/usr/bin/env python2
+
+import os
+import sys
+import re
+import subprocess
+import gettext
+from git import Repo,Git
+import ConfigParser
+import glob
+import shutil
+import optparse
+import fileinput
+
+PKGHOME = "/data/pkg_repo/packages"
+SRCPKGHOME = "/data/pkg_repo/src_packages"
+mydir = os.getcwd()
+repolist = ['core', 'extra']
+cli_dict = {}
+cli_list = []
+makepkg_cmd = ['makepkg']
+REPO = "none"
+SFIX = ""
+DOCROOT = ""
+pkgfile = "none"
+pkgbase = ""
+pkgname = ""
+pkgver = ""
+pkgrel = ""
+arch = ""
+CARCH = ""
+PKGDEST = ""
+
+# See what git branch we're working under
+git_repo = Git()
+branches = git_repo.branch()
+git_branch = branches.split('*')[1].lstrip(' ')
+
+if git_branch == "testing":
+    SFIX = "-testing"
+elif git_branch == "master":
+    SFIX = ""
+else:
+    print "Can't determine which git branch is in use!"
+    sys.exit(2)
+print "Git Branch suffix is:",SFIX
+
+# Color stuff -- does this work? Looks to bash-y to me.
+ALL_OFF = "$(tput sgr0)"
+BOLD = "$(tput bold)"
+BLUE = "${BOLD}$(tput setaf 4)"
+GREEN = "${BOLD}$(tput setaf 2)"
+RED = "${BOLD}$(tput setaf 1)"
+YELLOW = "${BOLD}$(tput setaf 3)"
+
+def plain():
+    mesg = list.pop()
+    print "${BOLD}    "+mesg+ "${ALL_OFF}\n"+cmdline
+
+def msg():
+    mesg = list.pop()
+    print "${GREEN}==>${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline
+
+def msg():
+    mesg = list.pop()
+    print "${BLUE}  ->${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline
+
+def warning():
+    warning = gettext(WARNING)
+    mesg = list.pop()
+    print "${YELLOW}==>"+ warning+ "${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline
+
+def error():
+    error = gettext(ERROR)
+    mesg = list.pop()
+    print "${RED}==>"+ error+ "${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline
+
+def commandline():
+    global makepkg_cmd
+    global cli_list
+    global cli_dict
+
+    clparser = optparse.OptionParser()
+    clparser.add_option("--asroot", action="store_true", default=True, help="Allow makepkg to run as root.")
+    clparser.add_option("-A", "--ignorearch", action="store_true", default=False, help="Ignore a missing or incomplete arch field in the build script.")
+    clparser.add_option("-b", "--bump", action="store_true", default=False, help="Increase package release one unit.")
+    clparser.add_option("-c", "--clean", action="store_true", default=False, help="Clean up leftover work files and directories after a successful build. ")
+    clparser.add_option("-C", "--cleancache", action="store_true", default=False, help="Removes all cached source files from the directory specified in SRCDEST in makepkg.conf")
+    clparser.add_option("--config", action="store", help="Use an alternate config file instead of the /etc/makepkg.conf default.")
+    clparser.add_option("-d", "--nodeps", action="store_true", default=False, help="Do not perform any dependency checks.")
+    clparser.add_option("-e", "--noextract", action="store_true", default=False, help="Do not extract source files; use whatever source already exists in the src/ directory.")
+    clparser.add_option("-f", "--force", action="store_true", default=False, help="This allows a built package to be overwritten.")
+    clparser.add_option("--forcever", action="store_true", default=False, help="This is a hidden option that should not be used unless you really know what you are doing.")
+    clparser.add_option("-g", "--geninteg", action="store_true", default=False, help="For each source file in the source array of PKGBUILD, download the file if required and generate integrity checks.")
+    clparser.add_option("--skipinteg", action="store_true", default=False, help="Do not perform any integrity checks, just print a warning instead.")
+    clparser.add_option("--holdver", action="store_true", default=False, help="Prevents makepkg from automatically bumping the pkgver to the latest revision number in the package's development tree.")
+    clparser.add_option("-i", "--install", action="store_true", default=False, help="Install or upgrade the package after a successful build.")
+    clparser.add_option("-L", "--log", action="store_true", default=False, help="Enable makepkg build logging.")
+    clparser.add_option("-m", "--nocolor", action="store_true", default=False, help="Disable color in output messages.")
+    clparser.add_option("-o", "--nobuild", action="store_true", default=False, help="Download and extract files only, but do not build them.")
+    clparser.add_option("-p", action="store", help="Read the package script buildscript instead of the PKGBUILD default.", metavar='/path/to/buildscript')
+    clparser.add_option("-r", "--rmdeps", action="store_true", default=False, help="Upon successful build, remove any dependencies installed by makepkg during dependency auto-resolution and installation when using -s.")
+    clparser.add_option("-R", "--repackage", action="store_true", default=False, help="Repackage contents of the package without rebuilding the package.")
+    clparser.add_option("-s", "--syncdeps", action="store_true", default=False, help="Install missing dependencies using pacman.")
+    clparser.add_option("--allsource", action="store_true", default=False, help="Do not actually build the package, but build a source-only tarball that includes all sources, including those that are normally download via makepkg.")
+    clparser.add_option("--source", action="store_true", default=False, help="Do not actually build the package, but build a source-only tarball that does not include sources that can be fetched via a download URL.")
+    clparser.add_option("--pkg", action="store", help="Only build listed packages from a split package.")
+    clparser.add_option("--noconfirm", action="store_true", default=False, help="(Passed to pacman) Prevent pacman from waiting for user input before proceeding with operations.")
+    clparser.add_option("--noprogressbar", action="store_true", default=False, help="(Passed to pacman) Prevent pacman from displaying a progress bar.")
+
+    (options, args) = clparser.parse_args()
+
+    options1 = ['config', 'p', 'pkg']
+    options2 = ['asroot', 'ignorearch', 'bump', 'clean', 'cleancache', 'nodeps',
+               'noextract', 'force', 'forcever', 'geninteg', 'skipinteg', 'holdver',
+               'install', 'log', 'nocolor', 'nobuild', 'rmdeps', 'repackage',
+               'syncdeps', 'allsource', 'source', 'noconfirm', 'noprogressbar']
+
+    for o in options1:
+        cmd1 = eval('options.'+o)
+        if o is not 'p':
+            if cmd1 is not None:
+                cli_dict['--'+o] = cmd1
+        elif cmd1 is not None:
+                cli_dict['-'+o] = cmd1
+    for o in options2:
+        cmd2 = eval('options.'+o)
+        if cmd2 is True:
+            cli_list.append('--'+o)
+
+    # Convert key=value to a single string
+    k = cli_dict.keys()
+    v = cli_dict.values()
+    final_dict = ' '.join ([ "%s %s" % (k, v) for k, v in cli_dict.items()])
+    print "final_dict=",final_dict
+
+    # Create makepkg command
+    makepkg_cmd.extend(cli_list)
+    makepkg_cmd.extend(final_dict)
+
+    # Remove bump option if it exists
+    if "--bump" in makepkg_cmd:
+        makepkg_cmd.remove("--bump")
+    print "Makepkg Command:",makepkg_cmd
+
+def bump_pkg():
+    # We need pkgrel to live beyond this function
+    global SRCPKG
+    global pkgrel
+
+    # Backup the original pkgfile
+    shutil.copy2(pkgfile, pkgfile + ".old")
+    # Let's increase the pkgrel
+    for line in fileinput.input(pkgfile, inplace=1):
+        if line.strip().startswith("pkgrel"):
+            pkgrel = line.partition("=")[2].strip('\n')
+            # Add 1 to pkgrel
+            new_pkgrel = int(pkgrel) + 1
+            line = line.replace("pkgrel=" + pkgrel, "pkgrel=" + str(new_pkgrel))
+            pkgrel = str(new_pkgrel)
+        sys.stdout.write(line)
+    print "Bumped " + pkgname + " release to " + str(new_pkgrel)
+
+    if pkgbase:
+        SRCPKG = pkgbase + "-" + pkgver + "-" + str(new_pkgrel) + ".src.tar.gz"
+    else:
+        SRCPKG = pkgname + "-" + pkgver + "-" + str(new_pkgrel) + ".src.tar.gz"
+    print "Source package will be: ",SRCPKG
+
+def find_repo():
+    global REPO
+    global DOCROOT
+
+    # Get all our possible repos into one list
+    for item in ("chroot-devel", "mv-core", "xmpl", "local"):
+        repolist.append(item)
+    print "Repo List:",repolist
+
+    # Create a list with the directory names of our current directory tree
+    dir_tree = os.path.dirname(mydir).split("/")
+    print "Dir Tree:",dir_tree
+    # Loop through the dir_tree to see if we can find our repo
+    for item in repolist: 
+        if item not in dir_tree:
+            continue
+        else:
+            repo_name = item
+        if repo_name == "extra":
+            REPO = "extra" + SFIX
+        elif repo_name == "core":
+            REPO = "core" + SFIX
+        elif repo_name == "xmpl":
+            REPO = "local"
+        elif repo_name not in repolist:
+            print "ERROR in function find_repo: Cannot determine repository!"
+            sys.exit(2)
+        else:
+            REPO = repo_name
+    # Ensure our DOCROOT exists and if not, create it
+    DOCROOT = "/data/pkg_repo/" + CARCH + "/" + REPO
+    print "DOCROOT:",DOCROOT
+    if os.path.exists(DOCROOT):
+        print "INFO: Repository is",REPO
+    else:
+        os.mkdir(DOCROOT,0755)
+        print "INFO: ",DOCROOT,"directory created. Repository is",REPO
+
+def update_repo():
+    # pkgname could be a list of several pkgs. Since bash array format is
+    # loose, let bash parse the pkgname(s) first, then return a list for us.
+    b = subprocess.Popen(['/bin/bash','-c', 'source ' + pkgfile + '; echo ${pkgname[@]}'], stdout = subprocess.PIPE,)
+    output = b.communicate()[0].strip('\n')
+    print "Pkglist:",output
+    pkglist = list(output.split())
+    os.chdir(DOCROOT)
+
+    print
+    print "INFO: Changed working dir to ",DOCROOT
+    print
+
+    for i in pkglist:
+        print "Package name:",i
+        TOTALPKG = i + "-" + pkgver + "-" + pkgrel + "-" + CARCH + ".pkg.tar.gz"
+        if not os.path.isfile(PKGDEST + "/" + TOTALPKG):
+            TOTALPKG = i + "-" + pkgver + "-" + pkgrel + "-" + CARCH + ".pkg.tar.xz"
+        else:
+            print "ERROR in function update_repo: Couldn't find the new package ",TOTALPKG
+            sys.exit(2)
+        # Remove old package from local copy
+    	OLDPKG = glob.glob(i + "-" + pkgver + "-*-" + CARCH + ".pkg.tar.*z")
+        if OLDPKG:
+            DELPKG = OLDPKG.pop()
+            print "DELPKG:",DELPKG
+            os.remove(DELPKG)
+        # Copy in new package
+        print "############################################"
+        print "Updating " + DOCROOT + "/" + CARCH + "/" + REPO + " with " + TOTALPKG
+        shutil.copy2(PKGDEST + "/" + TOTALPKG, DOCROOT)
+        subprocess.call(["repo-add", DOCROOT+ "/" + REPO + ".db.tar.gz", DOCROOT + "/" + TOTALPKG])
+        print "############################################"
+
+def update_src_pkg():
+    print "---------------------------------SRC------------------------------"
+    print "SRCPKG:",SRCPKG
+    os.chdir(SRCPKGHOME + "/" + REPO)
+    OLDSRCPKG = glob.glob(pkgname + "-" + pkgver + "-*.src.tar.*z")
+    if OLDSRCPKG:
+        for member in OLDSRCPKG:
+            print "Removing old source package",member
+            os.remove(SRCPKGHOME + "/" + REPO + "/" + member)
+    print "Copying source package to",SRCPKGHOME + "/" + REPO + "/" + SRCPKG
+    shutil.copy2(PKGHOME + "/" + SRCPKG, SRCPKGHOME + "/" + REPO + "/")
+
+def dup_check():
+    if REPO in ("local", "mv-core", "xmpl", "chroot-devel"):
+        return
+    for tmp_repo in ("core", "extra"):
+        if tmp_repo + SFIX != REPO:
+            p1 = subprocess.Popen(["pacman", "-Sl", tmp_repo + SFIX], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,)
+            p2 = subprocess.Popen(["cut", "-d", " ", "-f", "2"], stdin=p1.stdout, stdout=subprocess.PIPE,)
+            p3 = subprocess.Popen(["grep", pkgname], stdin=p2.stdout, stdout = subprocess.PIPE,)
+            output = p3.communicate()[0].strip('\n')
+            if output == 0:
+                if "--force" not in cli_list:
+                    print "#######################################"
+                    print pkgname," already exists in ",tmp_repo + SFIX
+                    print 
+                    print "Use --force to overwite"
+                    print "#######################################"
+                    sys.exit(2)
+                else:
+                    print "Force detected! Making package regardless!"
+                    print "#######################################"
+            else:
+                if output != 0:
+                    return
+
+def updateMD5():
+    # Do we have integrity checks?
+    sums = []
+
+    # Open pkgfile for reading
+    f = open(pkgfile, 'r')
+    pkgfile_contents = f.readlines()
+    f.close()
+
+    # Iterate through the lines looking for a match pattern
+    for line in pkgfile_contents:
+        check = re.compile('sha1sums|sha256sums|sha384sums|sha512sums|md5sums')
+        sums = check.match(line)
+        if sums:
+            break
+    if not sums:
+        # If no matches are found, append md5sums to the end of the pkgfile
+        p = open(pkgfile, 'a')
+        md5gen = subprocess.Popen(["makepkg", "--asroot", "-g"], stdout = subprocess.PIPE,).communicate()[0]
+        p.writelines(md5gen)
+        p.close()
+
+def config_file():
+    global pkgfile
+    global pkgbase
+    global pkgname
+    global pkgver
+    global pkgrel
+    global SRCPKG
+    global CARCH
+    global PKGDEST
+
+    # Check what file will be used for our PKGBUILD
+    if "--config" in cli_dict:
+        pkgfile = cli_dict["--config"]
+    else:
+        pkgfile = "PKGBUILD"
+    print "Config File:",pkgfile
+
+    # Does the file exist?
+    if not os.path.isfile(pkgfile):
+        print "ERROR in function config_file: Can't find ",pkgfile,"!"
+        sys.exit(2)
+    else:
+        f=open(pkgfile,"r")
+        # Read file contents to memory
+        pkgbuild_contents=f.readlines()
+        f.close()
+
+    # Loop over contents to get our variables
+    for line in pkgbuild_contents:
+        if line.strip().startswith("pkgbase"):
+            pkgbase = line.partition("=")[2].strip('\n')
+            print "Package Base Name:",pkgbase
+        if line.strip().startswith("pkgname"):
+            pkgname = line.partition("=")[2].strip('\n')
+            print "Package Name:",pkgname
+        if line.strip().startswith("pkgver"):
+            pkgver = line.partition("=")[2].strip('\n')
+            print "Package Version:",pkgver
+        if line.strip().startswith("pkgrel"):
+            pkgrel = line.partition("=")[2].strip('\n')
+            print "Package Release:",pkgrel
+        if line.strip().startswith("arch"):
+            arch = line.partition("=")[2].strip('\n')
+
+    if pkgbase:
+        SRCPKG = pkgbase + "-" + pkgver + "-" + pkgrel + ".src.tar.gz"
+    else:
+        SRCPKG = pkgname + "-" + pkgver + "-" + pkgrel + ".src.tar.gz"
+    print "Source package will be: ",SRCPKG
+
+    # Get needed makepkg.conf variables
+    mpkg="/etc/makepkg.conf"
+    f=open(mpkg,"r")
+    # Read file contents into memory
+    makepkg_contents=f.readlines()
+    f.close()
+
+    # Loop over contents to get our variables
+    for mp_lines in makepkg_contents:
+        if mp_lines.strip().startswith("CARCH"):
+            CARCH = mp_lines.partition("=")[2].strip('\n').replace('\"','')
+            if arch != 'any':
+                arch = CARCH
+        if mp_lines.strip().startswith("PKGDEST"):
+            PKGDEST = mp_lines.partition("=")[2].strip('\n')
+    print "CARCH is:",CARCH
+    print "Package desitination is:",PKGDEST
+    print "Architecture is:",arch
+
+def make_package():
+    # Make the package!
+    retcode = subprocess.call(makepkg_cmd)
+    if retcode != 0:
+        print "ERROR: makepkg failed with return code ",retcode
+        if "--bump" in cli_list:
+            if os.path.isfile(pkgfile + '.old'):
+                shutil.move(pkgfile + '.old', pkgfile)
+                print "Reverted pkgrel bump."
+        sys.exit(2)
+    else:
+        print "=============FINSHED CREATING PKG=================="
+
+        if arch == 'any':
+            CARCH = 'any'
+
+# MAIN PROGRAM
+commandline()
+config_file()
+find_repo()
+dup_check()
+
+# If "bump" was called, run the bump_pkg function
+if "--bump" in cli_list:
+    bump_pkg()
+
+updateMD5()
+
+# Everything look good? Go for it!
+make_package()
+
+print "------------updating database-------------"
+
+update_repo()
+
+print "----------creating source package---------"
+
+os.chdir(mydir)
+retcode = subprocess.call(["makepkg", "--force", "--holdver", "--asroot", "--source"])
+if retcode != 0:
+    print "ERROR: Source package creation failed with return code",retcode
+    sys.exit(2)
+else:
+    update_src_pkg()
+
+# Clean up old PKGBUILD if it exists
+os.chdir(mydir)
+if os.path.isfile(pkgfile + '.old'):
+    os.remove(pkgfile + '.old')
+    print "Removed temporary backup file",pkgfile + '.old'