#!/usr/bin/env python # Version 3.0.4 import os import stat import sys import re import subprocess import gettext from git import Repo,Git import configparser import glob import shutil import optparse import fileinput import time PKGHOME = "/data/dev/pkg_repo/packages" SRCHOME = "/data/dev/pkg_repo/sources" SRCPKGHOME = "/data/dev/pkg_repo/src_packages" # See what git branch we're working under git_repo = Git() branches = git_repo.branch() git_branch = branches.split('*')[1].lstrip(' ') if git_branch == "testing": SFIX = "-testing" elif git_branch == "master": SFIX = "" else: print("Can't determine which git branch is in use!") sys.exit(2) #print("Git Branch suffix is:",SFIX) def commandline(makepkg_cmd): #print makepkg_cmd cli_dict = {} cli_list = [] clparser = optparse.OptionParser() clparser.add_option("-b", "--bump", action="store_true", default=False, help="Increase package release one unit.") clparser.add_option("--pkglist", action="store", help="List of packages to compile") clparser.add_option("--config", action="store", help="Use an alternate config file instead of the /etc/makepkg.conf default.") clparser.add_option("--rmold", action="store_true", default=False, help="BETA: Remove old src and software packages from repos. Use with caution. False positives may occur (i.e. nvidia pkgs)!") (options, args) = clparser.parse_args() options1 = ['config', 'pkglist'] options2 = ['bump', 'rmold'] for o in options1: cmd1 = eval('options.'+o) if o != 'p': if cmd1 is not None: cli_dict['--'+o] = cmd1 elif cmd1 is not None: cli_dict['-'+o] = cmd1 for o in options2: cmd2 = eval('options.'+o) if cmd2 is True: cli_list.append('--'+o) # Create makepkg command makepkg_cmd.extend(cli_list) for k in list(cli_dict.keys()): v = cli_dict.get(k) makepkg_cmd.append(k) makepkg_cmd.append(v) # Remove bump option from makepkg command if it exists if "--bump" in makepkg_cmd: makepkg_cmd.remove("--bump") # Remove rmold option from makepkg command if it exists if "--rmold" in makepkg_cmd: makepkg_cmd.remove("--rmold") #print("Makepkg Command:",makepkg_cmd) return makepkg_cmd, cli_list, cli_dict class Packagefile(object): def __init__(self,cli_dict,cli_list,makepkg_cmd): self.failure=0 self.attempt=0 self.REPO = "none" self.DOCROOT = "" self.pkgfile = "none" self.pkgbase = "" self.pkgname = "" self.pkgver = "" self.pkgrel = "" self.pkglist=[] self.epoch = "" self.arch = "" self.CARCH = "" self.PKGDEST = "" self.SRCDEST = "" self.SRCPKGDEST = "" self.TOTALPKG = "" self.GZPKG = "" self.XZPKG = "" self.ZSTPKG = "" self.repolist=["core", "extra", "linhes", "chroot-devel", "mv-core", "xmpl", "local"] self.mydir = os.getcwd() self.variables = ['pkgbase','pkgname','pkgver','pkgrel','arch','epoch','depends','makedepends','makedepends_x86_64','checkdepends'] self.pkgrel_incremented = False self.makepkg_cmd = makepkg_cmd self.cli_list = cli_list self.cli_dict = cli_dict self.pkg_dependencies = [] self.makepkg_dependencies = [] self.check_dependencies = [] self.backupdir="/tmp/backup/" self.compile_status = "Pending" self.backup_pkgdir = "" self.pkg_dep_available = [] self.pkg_dep_not_in_repo = [] if "-p" in cli_dict: self.pkgfile = cli_dict["-p"] else: self.pkgfile = "PKGBUILD" self.updateINIT() def set_srcpkg(self): if self.pkgbase: self.SRCPKG = self.pkgbase + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + ".src.tar.gz" else: self.SRCPKG = self.pkglist[0] + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + ".src.tar.gz" def updateINIT(self): # Check what file will be used for our PKGBUILD # Does the file exist? if not os.path.isfile(self.pkgfile): print("ERROR in function config_file: Can't find ",self.pkgfile,"!") sys.exit(2) else: # Loop over contents to get our variables # Use bash to do it because PKGBUILDs are very loose with their format for item in self.variables: v = subprocess.Popen(['/bin/bash','-c', 'source ' + self.pkgfile + '; echo ${' + item + '[@]}'], stdout = subprocess.PIPE,) value = v.communicate()[0].strip(b'\n') value = value.decode('utf-8') if item == "pkgbase": self.pkgbase = value elif item == "pkgname": self.pkgname = value self.pkglist = list(value.split()) elif item == "pkgver": self.pkgver = value elif item == "pkgrel": self.pkgrel = value elif item == "epoch": if value == "0": value = "" if value: self.epoch = "%s:" %value elif item == "arch": self.arch = value elif item == "depends": self.pkg_dependencies = value.split() elif item == "makedepends": self.makepkg_dependencies = self.makepkg_dependencies + value.split() elif item == "makedepends_x86_64": self.makepkg_dependencies = self.makepkg_dependencies + value.split() elif item == "checkdepends": self.check_dependencies = value.split() self.set_srcpkg() # Get needed makepkg.conf variables mpkg="/etc/makepkg.conf" f=open(mpkg,"r") # Read file contents into memory makepkg_contents=f.readlines() f.close() self.PKGDEST = PKGHOME self.SRCDEST = SRCHOME self.SRCPKGDEST = SRCPKGHOME # Loop over contents to get our variables for mp_lines in makepkg_contents: if mp_lines.strip().startswith("CARCH"): self.CARCH = mp_lines.partition("=")[2].strip('\n').replace('\"','') if self.arch != 'any': self.arch = self.CARCH if mp_lines.strip().startswith("PKGDEST"): self.PKGDEST = mp_lines.partition("=")[2].strip('\n') def print_vars(self): print("####################################################") print("") print(" Config File:",self.pkgfile) print(" pkgbase:",self.pkgbase) print(" pkgname:",self.pkgname) print(" pkglist:",self.pkglist) print(" epoch:" ,self.epoch) print(" pkgver:",self.pkgver) print(" pkgrel:",self.pkgrel) print(" arch:",self.arch) print(" Source package:",self.SRCPKG) print(" CARCH is:",self.CARCH) print(" Architecture is:",self.arch) print(" Repository : ", self.REPO) print(" DOCROOT : ", self.DOCROOT) print(" Current directory: ", self.mydir) #print(" Backup directory: ", self.get_backupdir()) print(" Package destination:",self.PKGDEST) print(" Source destination:",self.SRCDEST) print(" Dependencies: ", self.pkg_dependencies) print(" Make Dependencies: ", self.makepkg_dependencies) print(" Package compile status: ", self.get_compile_status()) print("") print("####################################################") def getname(self): return self.pkgname def getfailure(self): return self.failure def getattempts(self): return self.attempt def getdir(self): return self.mydir def get_unmet_compile(self): return self.pkg_dep_not_in_repo def get_unmet_install(self): return self.pkg_dep_available def incrementcompile(self): self.attempt = self.attempt + 1 def get_backupdir(self): return self.backup_pkgdir def get_compile_status(self): return self.compile_status def find_repo(self): # Create a list with the directory names of our current directory tree dir_tree = os.path.dirname(self.mydir).split("/") # Loop through the dir_tree to see if we can find our repo for item in self.repolist: if item not in dir_tree: continue else: repo_name = item if repo_name == "extra": self.REPO = "extra" + SFIX elif repo_name == "core": self.REPO = "core" + SFIX elif repo_name == "linhes": self.REPO = "linhes" + SFIX elif repo_name == "xmpl": self.REPO = "local" elif repo_name not in repolist: print("ERROR in function find_repo: Cannot determine repository!") sys.exit(2) else: self.REPO = repo_name # Ensure our DOCROOT exists and if not, create it self.DOCROOT = "/data/dev/pkg_repo/" + self.CARCH + "/" + self.REPO #print("DOCROOT:",self.DOCROOT) if os.path.exists(self.DOCROOT): print("") #print("INFO: Repository is",self.REPO) else: try: os.mkdir(self.DOCROOT,0o755) print("INFO: ",self.DOCROOT,"directory created. Repository is",self.REPO) except Exception as e: print("ERROR: Creating DOCROOT failed: ", e) pass def increase_pkgrel(self): if not "--bump" in self.cli_list: return print("- Incrementing pkgrel for %s" %self.pkgname) # Backup the original pkgfile shutil.copy2(self.pkgfile, self.pkgfile + ".old") # Let's increase the pkgrel for line in fileinput.input(self.pkgfile, inplace=1): if line.strip().startswith("pkgrel"): self.pkgrel = line.partition("=")[2].strip('\n') # Add 1 to pkgrel new_pkgrel = int(self.pkgrel) + 1 line = line.replace("pkgrel=" + self.pkgrel, "pkgrel=" + str(new_pkgrel)) self.pkgrel = str(new_pkgrel) sys.stdout.write(line) print(" pkgrel:",self.pkgrel) self.set_srcpkg() self.pkgrel_incremented = True def runPreFlight(self): if os.path.isfile("mp_preflight"): print("- Running PreFlight...") os.chmod("mp_preflight", 0o755) retcode = subprocess.call(["./mp_preflight"]) if retcode != 0: print(" ERROR: PreFlight Error") print("") def updateSUMS(self): print("- Updating checksums...") retcode = subprocess.call(["updpkgsums"]) if retcode != 0: self.compile_status = "Failed: %s" %retcode self.failure = self.failure + 1 def make_package(self): #creates both binary and source package print(" Removing old packages...") for oldpkgs in self.pkglist: pkglinks = glob.glob(oldpkgs + "*.pkg.tar.*") print(" Old Packages:",pkglinks) if pkglinks: for DELPKG in pkglinks: os.remove(DELPKG) print("- Making package...") retcode = subprocess.call(["sudo", "ccm", "s"]) if retcode != 0: self.compile_status = "Failed: %s" %retcode self.failure = self.failure + 1 print(" ERROR: ccm failed with return code ",retcode) if self.pkgrel_incremented: if os.path.isfile(self.pkgfile + '.old'): shutil.move(self.pkgfile + '.old', self.pkgfile) print(" Reverted pkgrel increment.") return False else: print("-------------Making source package-------------") retcode = subprocess.call(["makepkg", "--force", "--holdver", "--source"]) if retcode != 0: self.compile_status = "Failed source: %s" %retcode self.failure = self.failure + 1 print("ERROR: Source package creation failed with return code",retcode) sys.exit(2) print("=============FINISHED CREATING PKG=============") self.compile_status = "Success" return True def update_database(self): print("- Copying files and updating database") # pkgname could be a list of several pkgs. Since bash array format is # loose, let bash parse the pkgname(s) first, then return a list for us. self.updateINIT() #if self.arch == 'any': # self.CARCH = 'any' for i in self.pkglist: print(" Package name:",i) self.GZPKG = i + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + "-" + self.arch + ".pkg.tar.gz" self.XZPKG = i + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + "-" + self.arch + ".pkg.tar.xz" self.ZSTPKG = i + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + "-" + self.arch + ".pkg.tar.zst" #print(self.GZPKG) #print(self.XZPKG) #print(self.ZSTPKG) if os.path.isfile(self.ZSTPKG): self.TOTALPKG = self.ZSTPKG elif os.path.isfile(self.XZPKG): self.TOTALPKG = self.XZPKG elif os.path.isfile(self.GZPKG): self.TOTALPKG = self.GZPKG else: print("") print(" ERROR in function update_database: Couldn't find the new package:",self.TOTALPKG) sys.exit(2) # Remove old package(s) from local copy print(" Looking for old packages") oldfilelist = glob.glob(self.DOCROOT + "/" + i + "-*-*-*.pkg.tar.*") print(" Old Packages:",oldfilelist) if oldfilelist: for DELPKG in oldfilelist: if "--rmold" in self.cli_list: print(" Deleting old package:",DELPKG) os.remove(DELPKG) # Copy in new package print(" Copying " + self.TOTALPKG + " to " + self.PKGDEST) if shutil.copy2(self.TOTALPKG, self.PKGDEST): os.remove(self.mydir + "/" + self.TOTALPKG) print(" Copying " + self.TOTALPKG + " to " + self.DOCROOT) shutil.copy2(self.PKGDEST + "/" + self.TOTALPKG, self.DOCROOT) print(" Creating symlink to " + self.PKGDEST + "/" + self.TOTALPKG) if os.path.islink(self.mydir + "/" + self.TOTALPKG): os.remove(self.mydir + "/" + self.TOTALPKG) os.symlink(self.DOCROOT + "/" + self.TOTALPKG, self.mydir + "/" + self.TOTALPKG) print(" Adding package to repo") subprocess.call(["repo-add", self.DOCROOT+ "/" + self.REPO + ".db.tar.gz", self.DOCROOT + "/" + self.TOTALPKG]) def update_srcrepo(self): print("- Updating source file repository") print(" SRCPKG:",self.SRCPKG) OLDSRCPKG="" if not os.path.isfile(self.SRCPKGDEST + "/" +self.SRCPKG): print("ERROR in function update_srcrepo: Couldn't find the new package",self.SRCPKG) sys.exit(2) i = self.pkglist[0] print(" Looking for old sources") oldfilelist = glob.glob(self.SRCPKGDEST + "/" + self.REPO + "/" + i + "-*-*.src.tar.gz") print(" Old Sources:",oldfilelist) if oldfilelist: for DELPKG in oldfilelist: if "--rmold" in self.cli_list: print(" Deleting old source:",DELPKG) os.remove(DELPKG) print(" Copying " + self.SRCPKG + " to " + self.SRCPKGDEST + "/" + self.REPO) if shutil.copy2(self.SRCPKGDEST + "/" + self.SRCPKG, self.SRCPKGDEST + "/" + self.REPO + "/"): os.remove(self.SRCPKGDEST + "/" + self.SRCPKG) def cleanup(self): os.chdir(self.mydir) if os.path.isfile(self.pkgfile + '.old'): os.remove(self.pkgfile + '.old') print("Removed temporary backup file",self.pkgfile + '.old') #remove src.tar.gz so it stops cluttering up git if os.path.islink(self.SRCPKG): os.unlink(self.SRCPKG) #remove src dir if empty if os.path.exists(self.mydir + "/src") and not os.path.isfile(self.mydir + "/src"): if not os.listdir(self.mydir + "/src"): os.rmdir(self.mydir + "/src") # Remove any old -build and -package log files os.chdir(self.mydir) build_log = self.pkgname + "-" + self.pkgver + "-" + self.pkgrel + "-" + self.arch + "-build.log" pkg_log = self.pkgname + "-" + self.pkgver + "-" + self.pkgrel + "-" + self.arch + "-package.log" all_logs = glob.glob("*.log*") #print "All Logs:",all_logs saved_logs = [] #save the logrotate files saved_logs = glob.glob("*.logrotate*") #if os.path.isfile(build_log): # saved_logs.append(build_log) #if os.path.isfile(pkg_log): # saved_logs.append(pkg_log) #print ("Saved Logs:",saved_logs) for log in all_logs: if log not in saved_logs: os.remove(log) pass def check_for_changelog(self, pkgdir): filename = "%s/__changelog" %pkgdir return os.path.exists(filename) def formatTD(time): # convert seconds to day, hour, minutes and seconds days = int(time // (24 * 3600)) time = time % (24 * 3600) hours = int(time // 3600) time %= 3600 minutes = int(time // 60) time %= 60 seconds = int(time) if days == 0: day_string = "" elif days > 1: day_string = "%s days, " %days else: day_string = "%s day, " %days if hours == 0: hour_string = "" elif hours > 1: hour_string = "%s hours, " %hours else: hour_string = "%s hour, " %hours if minutes == 0: minute_string = "" elif minutes > 1: minute_string = "%s minutes, " %minutes else: minute_string = "%s minute, " %minutes if seconds > 1: second_string = "%s seconds" %seconds else: second_string = "%s second" %seconds return_string = '%s%s%s%s' % (day_string, hour_string, minute_string, second_string) return return_string def main(): pkg_not_found=[] pkg_objdict={} pkg_list=[] global makepkg_cmd global cli_list global cli_dict makepkg_cmd = ['makepkg'] makepkg_cmd, cli_list, cli_dict = commandline(makepkg_cmd) try: pkg_list = cli_dict['--pkglist'].split(',') except: packagefile = None packagefile = Packagefile(cli_dict,cli_list,makepkg_cmd) pkg_objdict[packagefile.getname()]=packagefile pkg_list=[packagefile.getname()] packagefile.find_repo() packagefile.increase_pkgrel() packagefile.runPreFlight() packagefile.print_vars() packagefile.updateSUMS() if packagefile.make_package(): packagefile.update_database() packagefile.update_srcrepo() packagefile.cleanup() pkg_objdict[packagefile.getname()]=packagefile update_pkg = [] success_pkg = [] failed_compile = [] failed_update = [] #create lists for k, v in pkg_objdict.items(): #print k #print v.print_vars() if v.get_compile_status().strip() == "Success": success_pkg.append(k) else: temp = "%s --- %s (%s)" %(k,v.get_compile_status(),v.getattempts()) failed_compile.append(temp) print("\n\n\n\n") print("#######################################################") print("\n") print("Couldn't find these packages:") print("---------------------------------") print(pkg_not_found) print("\n") print("Failed to compile these packages:") print("---------------------------------") print(failed_compile) print("\n") print("Successful compiled :") print("---------------------------------") print(success_pkg) print("\n") if __name__ == "__main__": start = time.time() main() end = time.time() print("Build Time:",formatTD(end - start)) print("#######################################################")