diff options
Diffstat (limited to 'build_tools')
| -rwxr-xr-x | build_tools/bin/mp.py | 418 |
1 files changed, 418 insertions, 0 deletions
diff --git a/build_tools/bin/mp.py b/build_tools/bin/mp.py new file mode 100755 index 0000000..77f73c8 --- /dev/null +++ b/build_tools/bin/mp.py @@ -0,0 +1,418 @@ +#!/usr/bin/env python2 + +import os +import sys +import re +import subprocess +import gettext +from git import Repo,Git +import ConfigParser +import glob +import shutil +import optparse +import fileinput + +PKGHOME = "/data/pkg_repo/packages" +SRCPKGHOME = "/data/pkg_repo/src_packages" +mydir = os.getcwd() +repolist = ['core', 'extra'] +cli_dict = {} +cli_list = [] +makepkg_cmd = ['makepkg'] +REPO = "none" +SFIX = "" +DOCROOT = "" +pkgfile = "none" +pkgbase = "" +pkgname = "" +pkgver = "" +pkgrel = "" +arch = "" +CARCH = "" +PKGDEST = "" + +# See what git branch we're working under +git_repo = Git() +branches = git_repo.branch() +git_branch = branches.split('*')[1].lstrip(' ') + +if git_branch == "testing": + SFIX = "-testing" +elif git_branch == "master": + SFIX = "" +else: + print "Can't determine which git branch is in use!" + sys.exit(2) +print "Git Branch suffix is:",SFIX + +# Color stuff -- does this work? Looks to bash-y to me. +ALL_OFF = "$(tput sgr0)" +BOLD = "$(tput bold)" +BLUE = "${BOLD}$(tput setaf 4)" +GREEN = "${BOLD}$(tput setaf 2)" +RED = "${BOLD}$(tput setaf 1)" +YELLOW = "${BOLD}$(tput setaf 3)" + +def plain(): + mesg = list.pop() + print "${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline + +def msg(): + mesg = list.pop() + print "${GREEN}==>${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline + +def msg(): + mesg = list.pop() + print "${BLUE} ->${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline + +def warning(): + warning = gettext(WARNING) + mesg = list.pop() + print "${YELLOW}==>"+ warning+ "${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline + +def error(): + error = gettext(ERROR) + mesg = list.pop() + print "${RED}==>"+ error+ "${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline + +def commandline(): + global makepkg_cmd + global cli_list + global cli_dict + + clparser = optparse.OptionParser() + clparser.add_option("--asroot", action="store_true", default=True, help="Allow makepkg to run as root.") + clparser.add_option("-A", "--ignorearch", action="store_true", default=False, help="Ignore a missing or incomplete arch field in the build script.") + clparser.add_option("-b", "--bump", action="store_true", default=False, help="Increase package release one unit.") + clparser.add_option("-c", "--clean", action="store_true", default=False, help="Clean up leftover work files and directories after a successful build. ") + clparser.add_option("-C", "--cleancache", action="store_true", default=False, help="Removes all cached source files from the directory specified in SRCDEST in makepkg.conf") + clparser.add_option("--config", action="store", help="Use an alternate config file instead of the /etc/makepkg.conf default.") + clparser.add_option("-d", "--nodeps", action="store_true", default=False, help="Do not perform any dependency checks.") + clparser.add_option("-e", "--noextract", action="store_true", default=False, help="Do not extract source files; use whatever source already exists in the src/ directory.") + clparser.add_option("-f", "--force", action="store_true", default=False, help="This allows a built package to be overwritten.") + clparser.add_option("--forcever", action="store_true", default=False, help="This is a hidden option that should not be used unless you really know what you are doing.") + clparser.add_option("-g", "--geninteg", action="store_true", default=False, help="For each source file in the source array of PKGBUILD, download the file if required and generate integrity checks.") + clparser.add_option("--skipinteg", action="store_true", default=False, help="Do not perform any integrity checks, just print a warning instead.") + clparser.add_option("--holdver", action="store_true", default=False, help="Prevents makepkg from automatically bumping the pkgver to the latest revision number in the package's development tree.") + clparser.add_option("-i", "--install", action="store_true", default=False, help="Install or upgrade the package after a successful build.") + clparser.add_option("-L", "--log", action="store_true", default=False, help="Enable makepkg build logging.") + clparser.add_option("-m", "--nocolor", action="store_true", default=False, help="Disable color in output messages.") + clparser.add_option("-o", "--nobuild", action="store_true", default=False, help="Download and extract files only, but do not build them.") + clparser.add_option("-p", action="store", help="Read the package script buildscript instead of the PKGBUILD default.", metavar='/path/to/buildscript') + clparser.add_option("-r", "--rmdeps", action="store_true", default=False, help="Upon successful build, remove any dependencies installed by makepkg during dependency auto-resolution and installation when using -s.") + clparser.add_option("-R", "--repackage", action="store_true", default=False, help="Repackage contents of the package without rebuilding the package.") + clparser.add_option("-s", "--syncdeps", action="store_true", default=False, help="Install missing dependencies using pacman.") + clparser.add_option("--allsource", action="store_true", default=False, help="Do not actually build the package, but build a source-only tarball that includes all sources, including those that are normally download via makepkg.") + clparser.add_option("--source", action="store_true", default=False, help="Do not actually build the package, but build a source-only tarball that does not include sources that can be fetched via a download URL.") + clparser.add_option("--pkg", action="store", help="Only build listed packages from a split package.") + clparser.add_option("--noconfirm", action="store_true", default=False, help="(Passed to pacman) Prevent pacman from waiting for user input before proceeding with operations.") + clparser.add_option("--noprogressbar", action="store_true", default=False, help="(Passed to pacman) Prevent pacman from displaying a progress bar.") + + (options, args) = clparser.parse_args() + + options1 = ['config', 'p', 'pkg'] + options2 = ['asroot', 'ignorearch', 'bump', 'clean', 'cleancache', 'nodeps', + 'noextract', 'force', 'forcever', 'geninteg', 'skipinteg', 'holdver', + 'install', 'log', 'nocolor', 'nobuild', 'rmdeps', 'repackage', + 'syncdeps', 'allsource', 'source', 'noconfirm', 'noprogressbar'] + + for o in options1: + cmd1 = eval('options.'+o) + if o is not 'p': + if cmd1 is not None: + cli_dict['--'+o] = cmd1 + elif cmd1 is not None: + cli_dict['-'+o] = cmd1 + for o in options2: + cmd2 = eval('options.'+o) + if cmd2 is True: + cli_list.append('--'+o) + + # Convert key=value to a single string + k = cli_dict.keys() + v = cli_dict.values() + final_dict = ' '.join ([ "%s %s" % (k, v) for k, v in cli_dict.items()]) + print "final_dict=",final_dict + + # Create makepkg command + makepkg_cmd.extend(cli_list) + makepkg_cmd.extend(final_dict) + + # Remove bump option if it exists + if "--bump" in makepkg_cmd: + makepkg_cmd.remove("--bump") + print "Makepkg Command:",makepkg_cmd + +def bump_pkg(): + # We need pkgrel to live beyond this function + global SRCPKG + global pkgrel + + # Backup the original pkgfile + shutil.copy2(pkgfile, pkgfile + ".old") + # Let's increase the pkgrel + for line in fileinput.input(pkgfile, inplace=1): + if line.strip().startswith("pkgrel"): + pkgrel = line.partition("=")[2].strip('\n') + # Add 1 to pkgrel + new_pkgrel = int(pkgrel) + 1 + line = line.replace("pkgrel=" + pkgrel, "pkgrel=" + str(new_pkgrel)) + pkgrel = str(new_pkgrel) + sys.stdout.write(line) + print "Bumped " + pkgname + " release to " + str(new_pkgrel) + + if pkgbase: + SRCPKG = pkgbase + "-" + pkgver + "-" + str(new_pkgrel) + ".src.tar.gz" + else: + SRCPKG = pkgname + "-" + pkgver + "-" + str(new_pkgrel) + ".src.tar.gz" + print "Source package will be: ",SRCPKG + +def find_repo(): + global REPO + global DOCROOT + + # Get all our possible repos into one list + for item in ("chroot-devel", "mv-core", "xmpl", "local"): + repolist.append(item) + print "Repo List:",repolist + + # Create a list with the directory names of our current directory tree + dir_tree = os.path.dirname(mydir).split("/") + print "Dir Tree:",dir_tree + # Loop through the dir_tree to see if we can find our repo + for item in repolist: + if item not in dir_tree: + continue + else: + repo_name = item + if repo_name == "extra": + REPO = "extra" + SFIX + elif repo_name == "core": + REPO = "core" + SFIX + elif repo_name == "xmpl": + REPO = "local" + elif repo_name not in repolist: + print "ERROR in function find_repo: Cannot determine repository!" + sys.exit(2) + else: + REPO = repo_name + # Ensure our DOCROOT exists and if not, create it + DOCROOT = "/data/pkg_repo/" + CARCH + "/" + REPO + print "DOCROOT:",DOCROOT + if os.path.exists(DOCROOT): + print "INFO: Repository is",REPO + else: + os.mkdir(DOCROOT,0755) + print "INFO: ",DOCROOT,"directory created. Repository is",REPO + +def update_repo(): + # pkgname could be a list of several pkgs. Since bash array format is + # loose, let bash parse the pkgname(s) first, then return a list for us. + b = subprocess.Popen(['/bin/bash','-c', 'source ' + pkgfile + '; echo ${pkgname[@]}'], stdout = subprocess.PIPE,) + output = b.communicate()[0].strip('\n') + print "Pkglist:",output + pkglist = list(output.split()) + os.chdir(DOCROOT) + + print + print "INFO: Changed working dir to ",DOCROOT + print + + for i in pkglist: + print "Package name:",i + TOTALPKG = i + "-" + pkgver + "-" + pkgrel + "-" + CARCH + ".pkg.tar.gz" + if not os.path.isfile(PKGDEST + "/" + TOTALPKG): + TOTALPKG = i + "-" + pkgver + "-" + pkgrel + "-" + CARCH + ".pkg.tar.xz" + else: + print "ERROR in function update_repo: Couldn't find the new package ",TOTALPKG + sys.exit(2) + # Remove old package from local copy + OLDPKG = glob.glob(i + "-" + pkgver + "-*-" + CARCH + ".pkg.tar.*z") + if OLDPKG: + DELPKG = OLDPKG.pop() + print "DELPKG:",DELPKG + os.remove(DELPKG) + # Copy in new package + print "############################################" + print "Updating " + DOCROOT + "/" + CARCH + "/" + REPO + " with " + TOTALPKG + shutil.copy2(PKGDEST + "/" + TOTALPKG, DOCROOT) + subprocess.call(["repo-add", DOCROOT+ "/" + REPO + ".db.tar.gz", DOCROOT + "/" + TOTALPKG]) + print "############################################" + +def update_src_pkg(): + print "---------------------------------SRC------------------------------" + print "SRCPKG:",SRCPKG + os.chdir(SRCPKGHOME + "/" + REPO) + OLDSRCPKG = glob.glob(pkgname + "-" + pkgver + "-*.src.tar.*z") + if OLDSRCPKG: + for member in OLDSRCPKG: + print "Removing old source package",member + os.remove(SRCPKGHOME + "/" + REPO + "/" + member) + print "Copying source package to",SRCPKGHOME + "/" + REPO + "/" + SRCPKG + shutil.copy2(PKGHOME + "/" + SRCPKG, SRCPKGHOME + "/" + REPO + "/") + +def dup_check(): + if REPO in ("local", "mv-core", "xmpl", "chroot-devel"): + return + for tmp_repo in ("core", "extra"): + if tmp_repo + SFIX != REPO: + p1 = subprocess.Popen(["pacman", "-Sl", tmp_repo + SFIX], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,) + p2 = subprocess.Popen(["cut", "-d", " ", "-f", "2"], stdin=p1.stdout, stdout=subprocess.PIPE,) + p3 = subprocess.Popen(["grep", pkgname], stdin=p2.stdout, stdout = subprocess.PIPE,) + output = p3.communicate()[0].strip('\n') + if output == 0: + if "--force" not in cli_list: + print "#######################################" + print pkgname," already exists in ",tmp_repo + SFIX + print + print "Use --force to overwite" + print "#######################################" + sys.exit(2) + else: + print "Force detected! Making package regardless!" + print "#######################################" + else: + if output != 0: + return + +def updateMD5(): + # Do we have integrity checks? + sums = [] + + # Open pkgfile for reading + f = open(pkgfile, 'r') + pkgfile_contents = f.readlines() + f.close() + + # Iterate through the lines looking for a match pattern + for line in pkgfile_contents: + check = re.compile('sha1sums|sha256sums|sha384sums|sha512sums|md5sums') + sums = check.match(line) + if sums: + break + if not sums: + # If no matches are found, append md5sums to the end of the pkgfile + p = open(pkgfile, 'a') + md5gen = subprocess.Popen(["makepkg", "--asroot", "-g"], stdout = subprocess.PIPE,).communicate()[0] + p.writelines(md5gen) + p.close() + +def config_file(): + global pkgfile + global pkgbase + global pkgname + global pkgver + global pkgrel + global SRCPKG + global CARCH + global PKGDEST + + # Check what file will be used for our PKGBUILD + if "--config" in cli_dict: + pkgfile = cli_dict["--config"] + else: + pkgfile = "PKGBUILD" + print "Config File:",pkgfile + + # Does the file exist? + if not os.path.isfile(pkgfile): + print "ERROR in function config_file: Can't find ",pkgfile,"!" + sys.exit(2) + else: + f=open(pkgfile,"r") + # Read file contents to memory + pkgbuild_contents=f.readlines() + f.close() + + # Loop over contents to get our variables + for line in pkgbuild_contents: + if line.strip().startswith("pkgbase"): + pkgbase = line.partition("=")[2].strip('\n') + print "Package Base Name:",pkgbase + if line.strip().startswith("pkgname"): + pkgname = line.partition("=")[2].strip('\n') + print "Package Name:",pkgname + if line.strip().startswith("pkgver"): + pkgver = line.partition("=")[2].strip('\n') + print "Package Version:",pkgver + if line.strip().startswith("pkgrel"): + pkgrel = line.partition("=")[2].strip('\n') + print "Package Release:",pkgrel + if line.strip().startswith("arch"): + arch = line.partition("=")[2].strip('\n') + + if pkgbase: + SRCPKG = pkgbase + "-" + pkgver + "-" + pkgrel + ".src.tar.gz" + else: + SRCPKG = pkgname + "-" + pkgver + "-" + pkgrel + ".src.tar.gz" + print "Source package will be: ",SRCPKG + + # Get needed makepkg.conf variables + mpkg="/etc/makepkg.conf" + f=open(mpkg,"r") + # Read file contents into memory + makepkg_contents=f.readlines() + f.close() + + # Loop over contents to get our variables + for mp_lines in makepkg_contents: + if mp_lines.strip().startswith("CARCH"): + CARCH = mp_lines.partition("=")[2].strip('\n').replace('\"','') + if arch != 'any': + arch = CARCH + if mp_lines.strip().startswith("PKGDEST"): + PKGDEST = mp_lines.partition("=")[2].strip('\n') + print "CARCH is:",CARCH + print "Package desitination is:",PKGDEST + print "Architecture is:",arch + +def make_package(): + # Make the package! + retcode = subprocess.call(makepkg_cmd) + if retcode != 0: + print "ERROR: makepkg failed with return code ",retcode + if "--bump" in cli_list: + if os.path.isfile(pkgfile + '.old'): + shutil.move(pkgfile + '.old', pkgfile) + print "Reverted pkgrel bump." + sys.exit(2) + else: + print "=============FINSHED CREATING PKG==================" + + if arch == 'any': + CARCH = 'any' + +# MAIN PROGRAM +commandline() +config_file() +find_repo() +dup_check() + +# If "bump" was called, run the bump_pkg function +if "--bump" in cli_list: + bump_pkg() + +updateMD5() + +# Everything look good? Go for it! +make_package() + +print "------------updating database-------------" + +update_repo() + +print "----------creating source package---------" + +os.chdir(mydir) +retcode = subprocess.call(["makepkg", "--force", "--holdver", "--asroot", "--source"]) +if retcode != 0: + print "ERROR: Source package creation failed with return code",retcode + sys.exit(2) +else: + update_src_pkg() + +# Clean up old PKGBUILD if it exists +os.chdir(mydir) +if os.path.isfile(pkgfile + '.old'): + os.remove(pkgfile + '.old') + print "Removed temporary backup file",pkgfile + '.old' |