#!/usr/bin/env python2 # Version 0.6.9 import os import sys import re import subprocess import gettext from git import Repo,Git import ConfigParser import glob import shutil import optparse import fileinput PKGHOME = "/data/pkg_repo/packages" SRCPKGHOME = "/data/pkg_repo/src_packages" mydir = os.getcwd() repolist = ['core', 'extra'] variables = ['pkgbase','pkgname','pkgver','pkgrel','arch','epoch'] cli_dict = {} cli_list = [] makepkg_cmd = ['makepkg'] REPO = "none" SFIX = "" DOCROOT = "" pkgfile = "none" pkgbase = "" pkgname = "" pkgver = "" pkgrel = "" epoch = "" arch = "" CARCH = "" PKGDEST = "" TOTALPKG = "" GZPKG = "" XZPKG = "" # See what git branch we're working under git_repo = Git() branches = git_repo.branch() git_branch = branches.split('*')[1].lstrip(' ') if git_branch == "testing": SFIX = "-testing" elif git_branch == "master": SFIX = "" else: print "Can't determine which git branch is in use!" sys.exit(2) print "Git Branch suffix is:",SFIX # Color stuff -- does this work? Looks to bash-y to me. ALL_OFF = "$(tput sgr0)" BOLD = "$(tput bold)" BLUE = "${BOLD}$(tput setaf 4)" GREEN = "${BOLD}$(tput setaf 2)" RED = "${BOLD}$(tput setaf 1)" YELLOW = "${BOLD}$(tput setaf 3)" def plain(): mesg = list.pop() print "${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline def msg(): mesg = list.pop() print "${GREEN}==>${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline def msg(): mesg = list.pop() print "${BLUE} ->${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline def warning(): warning = gettext(WARNING) mesg = list.pop() print "${YELLOW}==>"+ warning+ "${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline def error(): error = gettext(ERROR) mesg = list.pop() print "${RED}==>"+ error+ "${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline def commandline(): global makepkg_cmd global cli_list global cli_dict clparser = optparse.OptionParser() clparser.add_option("--asroot", action="store_true", default=True, help="Allow makepkg to run as root.") clparser.add_option("-A", "--ignorearch", action="store_true", default=False, help="Ignore a missing or incomplete arch field in the build script.") clparser.add_option("-b", "--bump", action="store_true", default=False, help="Increase package release one unit.") clparser.add_option("-c", "--clean", action="store_true", default=False, help="Clean up leftover work files and directories after a successful build. ") clparser.add_option("-C", "--cleancache", action="store_true", default=False, help="Removes all cached source files from the directory specified in SRCDEST in makepkg.conf") clparser.add_option("--config", action="store", help="Use an alternate config file instead of the /etc/makepkg.conf default.") clparser.add_option("-d", "--nodeps", action="store_true", default=False, help="Do not perform any dependency checks.") clparser.add_option("-e", "--noextract", action="store_true", default=False, help="Do not extract source files; use whatever source already exists in the src/ directory.") clparser.add_option("-f", "--force", action="store_true", default=False, help="This allows a built package to be overwritten.") clparser.add_option("--forcever", action="store_true", default=False, help="This is a hidden option that should not be used unless you really know what you are doing.") clparser.add_option("-g", "--geninteg", action="store_true", default=False, help="For each source file in the source array of PKGBUILD, download the file if required and generate integrity checks.") clparser.add_option("--skipinteg", action="store_true", default=False, help="Do not perform any integrity checks, just print a warning instead.") clparser.add_option("--holdver", action="store_true", default=True, help="Prevents makepkg from automatically bumping the pkgver to the latest revision number in the package's development tree.") clparser.add_option("--nohold", action="store_false", dest="holdver", help="Automatically bump the pkgver to the latest revision number in the package's development tree.") clparser.add_option("-i", "--install", action="store_true", default=False, help="Install or upgrade the package after a successful build.") clparser.add_option("-L", "--log", action="store_true", default=True, help="Enable makepkg build logging.") clparser.add_option("-m", "--nocolor", action="store_true", default=False, help="Disable color in output messages.") clparser.add_option("-o", "--nobuild", action="store_true", default=False, help="Download and extract files only, but do not build them.") clparser.add_option("-p", action="store", help="Read the package script buildscript instead of the PKGBUILD default.", metavar='/path/to/buildscript') clparser.add_option("-r", "--rmdeps", action="store_true", default=False, help="Upon successful build, remove any dependencies installed by makepkg during dependency auto-resolution and installation when using -s.") clparser.add_option("-R", "--repackage", action="store_true", default=False, help="Repackage contents of the package without rebuilding the package.") clparser.add_option("-s", "--syncdeps", action="store_true", default=False, help="Install missing dependencies using pacman.") clparser.add_option("--allsource", action="store_true", default=False, help="Do not actually build the package, but build a source-only tarball that includes all sources, including those that are normally download via makepkg.") clparser.add_option("--source", action="store_true", default=False, help="Do not actually build the package, but build a source-only tarball that does not include sources that can be fetched via a download URL.") clparser.add_option("--pkg", action="store", help="Only build listed packages from a split package.") clparser.add_option("--noconfirm", action="store_true", default=False, help="(Passed to pacman) Prevent pacman from waiting for user input before proceeding with operations.") clparser.add_option("--noprogressbar", action="store_true", default=False, help="(Passed to pacman) Prevent pacman from displaying a progress bar.") clparser.add_option("--rmold", action="store_true", default=False, help="BETA: Remove old src and software packages from repos. Use with caution. False positives may occur (i.e. nvidia pkgs)!") (options, args) = clparser.parse_args() options1 = ['config', 'p', 'pkg'] options2 = ['asroot', 'ignorearch', 'bump', 'clean', 'cleancache', 'nodeps', 'noextract', 'force', 'forcever', 'geninteg', 'skipinteg', 'holdver', 'install', 'log', 'nocolor', 'nobuild', 'rmdeps', 'repackage', 'syncdeps', 'allsource', 'source', 'noconfirm', 'noprogressbar', 'rmold'] for o in options1: cmd1 = eval('options.'+o) if o is not 'p': if cmd1 is not None: cli_dict['--'+o] = cmd1 elif cmd1 is not None: cli_dict['-'+o] = cmd1 for o in options2: cmd2 = eval('options.'+o) if cmd2 is True: cli_list.append('--'+o) # Create makepkg command makepkg_cmd.extend(cli_list) for k in cli_dict.keys(): v = cli_dict.get(k) makepkg_cmd.append(k) makepkg_cmd.append(v) # Remove bump option from makepkg command if it exists if "--bump" in makepkg_cmd: makepkg_cmd.remove("--bump") # Remove rmold option from makepkg command if it exists if "--rmold" in makepkg_cmd: makepkg_cmd.remove("--rmold") # Remove "--geninteg" option (if it exists) from makepkg command. # Checking for *sums is done automaticly by mp.py if "--geninteg" in makepkg_cmd: makepkg_cmd.remove("--geninteg") print "Makepkg Command:",makepkg_cmd def bump_pkg(): # We need pkgrel to live beyond this function global SRCPKG global pkgrel # Backup the original pkgfile shutil.copy2(pkgfile, pkgfile + ".old") # Let's increase the pkgrel for line in fileinput.input(pkgfile, inplace=1): if line.strip().startswith("pkgrel"): pkgrel = line.partition("=")[2].strip('\n') # Add 1 to pkgrel new_pkgrel = int(pkgrel) + 1 line = line.replace("pkgrel=" + pkgrel, "pkgrel=" + str(new_pkgrel)) pkgrel = str(new_pkgrel) sys.stdout.write(line) print "Bumped " + pkgname + " release to " + str(new_pkgrel) if pkgbase: SRCPKG = pkgbase + "-" + pkgver + "-" + str(new_pkgrel) + ".src.tar.gz" else: SRCPKG = pkgname + "-" + pkgver + "-" + str(new_pkgrel) + ".src.tar.gz" print "Source package will be: ",SRCPKG def find_repo(): global REPO global DOCROOT # Get all our possible repos into one list for item in ("chroot-devel", "mv-core", "xmpl", "local"): repolist.append(item) print "Repo List:",repolist # Create a list with the directory names of our current directory tree dir_tree = os.path.dirname(mydir).split("/") print "Dir Tree:",dir_tree # Loop through the dir_tree to see if we can find our repo for item in repolist: if item not in dir_tree: continue else: repo_name = item if repo_name == "extra": REPO = "extra" + SFIX elif repo_name == "core": REPO = "core" + SFIX elif repo_name == "xmpl": REPO = "local" elif repo_name not in repolist: print "ERROR in function find_repo: Cannot determine repository!" sys.exit(2) else: REPO = repo_name # Ensure our DOCROOT exists and if not, create it DOCROOT = "/data/pkg_repo/" + CARCH + "/" + REPO print "DOCROOT:",DOCROOT if os.path.exists(DOCROOT): print "INFO: Repository is",REPO else: os.mkdir(DOCROOT,0755) print "INFO: ",DOCROOT,"directory created. Repository is",REPO def update_repo(): # pkgname could be a list of several pkgs. Since bash array format is # loose, let bash parse the pkgname(s) first, then return a list for us. os.chdir(DOCROOT) print print "INFO: Changed working dir to",DOCROOT print for i in pkglist: print "Package name:",i GZPKG = i + "-" + epoch + pkgver + "-" + pkgrel + "-" + CARCH + ".pkg.tar.gz" XZPKG = i + "-" + epoch + pkgver + "-" + pkgrel + "-" + CARCH + ".pkg.tar.xz" if os.path.isfile(PKGDEST + "/" + XZPKG): TOTALPKG = XZPKG elif os.path.isfile(PKGDEST + "/" + GZPKG): TOTALPKG = GZPKG else: print "ERROR in function update_repo: Couldn't find the new package",PKGDEST + "/" + TOTALPKG sys.exit(2) print "Package file:",TOTALPKG # Remove old package(s) from local copy pv = re.compile('[\d]+[\w.]+') dirlist = os.listdir(DOCROOT) for n in dirlist: if n.startswith(i): num = len(i) + 1 print "Bytes to seek ahead:",num OLDPKG = glob.glob(i + "-" + pv.search(n,num).group() + "-[0-9]*.pkg.tar.?z") print "OLDPKG =",OLDPKG if OLDPKG: for DELPKG in OLDPKG: if "--rmold" in cli_list: print "Deleting old package:",DELPKG os.remove(DELPKG) # Remove any symlinks to old packages # We make it conditional on "--force" because force will overwrite # an existing package and we want the symlink to stay, pointing to # the newly built package with the same pkgrel. if "--force" not in makepkg_cmd: if os.path.islink(mydir + "/" + DELPKG): os.remove(mydir + "/" + DELPKG) # Copy in new package print "############################################" print "Updating " + DOCROOT + " with " + TOTALPKG print "Copying " + PKGDEST + "/" + TOTALPKG shutil.copy2(PKGDEST + "/" + TOTALPKG, DOCROOT) subprocess.call(["repo-add", DOCROOT+ "/" + REPO + ".db.tar.gz", DOCROOT + "/" + TOTALPKG]) print "############################################" def update_src_pkg(): print "---------------------------------SRC------------------------------" print "SRCPKG:",SRCPKG OLDSRCPKG = [] os.chdir(SRCPKGHOME + "/" + REPO) dirlist = os.listdir(SRCPKGHOME + "/" + REPO) #pv = re.compile('[\d]+[.]*[\d]*[.]*[\d]*[.]*[\d]*[.]*[\d]*') pv = re.compile('[\d]+[\w.]+') if pkgbase: # Remove old src package(s) from local copy if "--rmold" in cli_list: for n in dirlist: if n.startswith(pkgbase): num = len(pkgbase) + 1 print "Bytes to seek ahead:",num OLDSRCPKG = glob.glob(pkgbase + "-" + pv.search(n,num).group() + "-*.src.tar.?z") else: # Remove old src package(s) from local copy if "--rmold" in cli_list: for n in dirlist: if n.startswith(pkgname): num = len(pkgname) + 1 print "Bytes to seek ahead:",num OLDSRCPKG = glob.glob(pkgname + "-" + pv.search(n,num).group() + "-*.src.tar.?z") if OLDSRCPKG: print "OLDSRCPKG =",OLDSRCPKG for DELSRCPKG in OLDSRCPKG: print "Deleting old source package",DELSRCPKG os.remove(DELSRCPKG) print "Copying new source package to",SRCPKGHOME + "/" + REPO + "/" + SRCPKG shutil.copy2(PKGHOME + "/" + SRCPKG, SRCPKGHOME + "/" + REPO + "/") def dup_check(): if REPO in ("local", "mv-core", "xmpl", "chroot-devel"): return for tmp_repo in ("core", "extra"): if tmp_repo + SFIX != REPO: p1 = subprocess.Popen(["pacman", "-Sl", tmp_repo + SFIX], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,) p2 = subprocess.Popen(["cut", "-d", " ", "-f", "2"], stdin=p1.stdout, stdout=subprocess.PIPE,) p3 = subprocess.Popen(["grep", pkgname], stdin=p2.stdout, stdout = subprocess.PIPE,) output = p3.communicate()[0].strip('\n') if output == 0: if "--force" not in cli_list: print "#######################################" print pkgname," already exists in ",tmp_repo + SFIX print print "Use --force to overwite" print "#######################################" sys.exit(2) else: print "Force detected! Making package regardless!" print "#######################################" else: if output != 0: return def updateMD5(): # Do we have integrity checks? sums = [] # Open pkgfile for reading f = open(pkgfile, 'r') pkgfile_contents = f.readlines() f.close() # Iterate through the lines looking for a match pattern for line in pkgfile_contents: check = re.compile('sha1sums|sha256sums|sha384sums|sha512sums|md5sums') sums = check.match(line) if sums: break if not sums: # If no matches are found, append md5sums to the end of the pkgfile p = open(pkgfile, 'a') md5gen = subprocess.Popen(["makepkg", "--asroot", "-g"], stdout = subprocess.PIPE,).communicate()[0] p.writelines(md5gen) p.close() def config_file(): global pkgfile global pkgbase global pkgname global pkglist global pkgver global pkgrel global epoch global arch global SRCPKG global CARCH global PKGDEST # Check what file will be used for our PKGBUILD if "-p" in cli_dict: pkgfile = cli_dict["-p"] else: pkgfile = "PKGBUILD" print "Config File:",pkgfile # Does the file exist? if not os.path.isfile(pkgfile): print "ERROR in function config_file: Can't find ",pkgfile,"!" sys.exit(2) else: # Loop over contents to get our variables # Use bash to do it because PKGBUILDs are very loose with their format for item in variables: v = subprocess.Popen(['/bin/bash','-c', 'source ' + pkgfile + '; echo ${' + item + '[@]}'], stdout = subprocess.PIPE,) value = v.communicate()[0].strip('\n') if item == "pkgbase": pkgbase = value print "pkgbase:",pkgbase elif item == "pkgname": pkgname = value print "pkgname:",pkgname pkglist = list(value.split()) print "pkglist:",pkglist elif item == "pkgver": pkgver = value print "pkgver:",pkgver elif item == "pkgrel": pkgrel = value print "pkgrel:",pkgrel elif item == "epoch": if value: epoch = "%s:" %value print "epoch" , epoch elif item == "arch": arch = value print "arch:",arch if pkgbase: SRCPKG = pkgbase + "-" + epoch + pkgver + "-" + pkgrel + ".src.tar.gz" else: SRCPKG = pkgname + "-" + epoch + pkgver + "-" + pkgrel + ".src.tar.gz" print "Source package will be:",SRCPKG # Get needed makepkg.conf variables mpkg="/etc/makepkg.conf" f=open(mpkg,"r") # Read file contents into memory makepkg_contents=f.readlines() f.close() # Loop over contents to get our variables for mp_lines in makepkg_contents: if mp_lines.strip().startswith("CARCH"): CARCH = mp_lines.partition("=")[2].strip('\n').replace('\"','') if arch != 'any': arch = CARCH if mp_lines.strip().startswith("PKGDEST"): PKGDEST = mp_lines.partition("=")[2].strip('\n') print "CARCH is:",CARCH print "Package destination is:",PKGDEST print "Architecture is:",arch def make_package(): global arch global CARCH # Make the package! retcode = subprocess.call(makepkg_cmd) if retcode != 0: print "ERROR: makepkg failed with return code ",retcode if "--bump" in cli_list: if os.path.isfile(pkgfile + '.old'): shutil.move(pkgfile + '.old', pkgfile) print "Reverted pkgrel bump." sys.exit(2) elif "--nobuild" in makepkg_cmd: sys.exit(0) else: print "=============FINSHED CREATING PKG==================" if arch == 'any': CARCH = 'any' # MAIN PROGRAM commandline() config_file() find_repo() dup_check() # If "bump" was called, run the bump_pkg function if "--bump" in cli_list: bump_pkg() updateMD5() # Everything look good? Go for it! make_package() print "------------updating database-------------" update_repo() print "----------creating source package---------" os.chdir(mydir) retcode = subprocess.call(["makepkg", "--force", "--holdver", "--asroot", "--source"]) if retcode != 0: print "ERROR: Source package creation failed with return code",retcode sys.exit(2) else: update_src_pkg() # Clean up old PKGBUILD if it exists os.chdir(mydir) if os.path.isfile(pkgfile + '.old'): os.remove(pkgfile + '.old') print "Removed temporary backup file",pkgfile + '.old' #remove src.tar.gz so it stops cluttering up git if os.path.islink(SRCPKG): os.unlink(SRCPKG) # Remove any old -build and -package log files os.chdir(mydir) build_log = pkgname + "-" + pkgver + "-" + pkgrel + "-" + CARCH + "-build.log" pkg_log = pkgname + "-" + pkgver + "-" + pkgrel + "-" + CARCH + "-package.log" all_logs = glob.glob("*.log*") #print "All Logs:",all_logs saved_logs = [] if os.path.isfile(build_log): saved_logs.append(build_log) if os.path.isfile(pkg_log): saved_logs.append(pkg_log) #print "Saved Logs:",saved_logs for log in all_logs: if log not in saved_logs: os.remove(log)