path: root/linhes/linhes-dev/mp.py

#!/usr/bin/env python
# Version 3.0.4
import os
import stat
import sys
import re
import subprocess
import gettext
from git import Repo,Git
import configparser
import glob
import shutil
import optparse
import fileinput
import time

PKGHOME = "/data/dev/pkg_repo/packages"
SRCHOME = "/data/dev/pkg_repo/sources"
SRCPKGHOME = "/data/dev/pkg_repo/src_packages"

# See what git branch we're working under
git_repo = Git()
branches = git_repo.branch()
git_branch = branches.split('*')[1].lstrip(' ')

if git_branch == "testing":
    SFIX = "-testing"
elif git_branch == "master":
    SFIX = ""
else:
    print("Can't determine which git branch is in use!")
    sys.exit(2)
#print("Git Branch suffix is:",SFIX)


def commandline(makepkg_cmd):
    #print makepkg_cmd
    cli_dict = {}
    cli_list = []

    clparser = optparse.OptionParser()
    clparser.add_option("-b", "--bump", action="store_true", default=False, help="Increase package release one unit.")
    clparser.add_option("--pkglist", action="store", help="List of packages to compile")
    clparser.add_option("--config", action="store", help="Use an alternate config file instead of the /etc/makepkg.conf default.")
    clparser.add_option("--rmold", action="store_true", default=False, help="BETA: Remove old src and software packages from repos. Use with caution. False positives may occur (i.e. nvidia pkgs)!")

    (options, args) = clparser.parse_args()

    options1 = ['config', 'pkglist']
    options2 = ['bump', 'rmold']

    for o in options1:
        cmd1 = eval('options.'+o)
        if o != 'p':
            if cmd1 is not None:
                cli_dict['--'+o] = cmd1
        elif cmd1 is not None:
                cli_dict['-'+o] = cmd1
    for o in options2:
        cmd2 = eval('options.'+o)
        if cmd2 is True:
            cli_list.append('--'+o)

    # Create makepkg command
    makepkg_cmd.extend(cli_list)
    for k in list(cli_dict.keys()):
        v = cli_dict.get(k)
        makepkg_cmd.append(k)
        makepkg_cmd.append(v)

    # Remove bump option from makepkg command if it exists
    if "--bump" in makepkg_cmd:
        makepkg_cmd.remove("--bump")
    # Remove rmold option from makepkg command if it exists
    if "--rmold" in makepkg_cmd:
        makepkg_cmd.remove("--rmold")

    #print("Makepkg Command:",makepkg_cmd)
    return makepkg_cmd, cli_list, cli_dict



class Packagefile(object):
    def __init__(self,cli_dict,cli_list,makepkg_cmd):
        self.failure=0
        self.attempt=0
        self.REPO = "none"
        self.DOCROOT = ""
        self.pkgfile = "none"
        self.pkgbase = ""
        self.pkgname = ""
        self.pkgver = ""
        self.pkgrel = ""
        self.pkglist=[]
        self.epoch = ""
        self.arch = ""
        self.CARCH = ""
        self.PKGDEST = ""
        self.SRCDEST = ""
        self.SRCPKGDEST = ""
        self.TOTALPKG = ""
        self.GZPKG = ""
        self.XZPKG = ""
        self.ZSTPKG = ""
        self.repolist=["core", "extra", "linhes",
                        "chroot-devel", "mv-core", "xmpl", "local"]
        self.mydir = os.getcwd()
        self.variables = ['pkgbase','pkgname','pkgver','pkgrel','arch','epoch','depends','makedepends','makedepends_x86_64','checkdepends']
        self.pkgrel_incremented = False
        self.makepkg_cmd = makepkg_cmd
        self.cli_list = cli_list
        self.cli_dict = cli_dict
        self.pkg_dependencies = []
        self.makepkg_dependencies = []
        self.check_dependencies = []
        self.backupdir="/tmp/backup/"
        self.compile_status = "Pending"
        self.backup_pkgdir = ""
        self.pkg_dep_available = []
        self.pkg_dep_not_in_repo = []
        if "-p" in cli_dict:
            self.pkgfile = cli_dict["-p"]
        else:
            self.pkgfile = "PKGBUILD"

        self.updateINIT()


    def set_srcpkg(self):
        if self.pkgbase:
            self.SRCPKG = self.pkgbase + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + ".src.tar.gz"
        else:
            self.SRCPKG = self.pkglist[0] + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + ".src.tar.gz"


    def updateINIT(self):
        # Check what file will be used for our PKGBUILD

        # Does the file exist?
        if not os.path.isfile(self.pkgfile):
            print("ERROR in function config_file: Can't find ",self.pkgfile,"!")
            sys.exit(2)
        else:
            # Loop over contents to get our variables
            # Use bash to do it because PKGBUILDs are very loose with their format
            for item in self.variables:
                v = subprocess.Popen(['/bin/bash','-c', 'source ' +
                                     self.pkgfile +
                                     '; echo ${' + item + '[@]}'],
                                     stdout = subprocess.PIPE,)
                value = v.communicate()[0].strip(b'\n')
                value = value.decode('utf-8')
                if item == "pkgbase":
                    self.pkgbase = value
                elif item == "pkgname":
                    self.pkgname = value
                    self.pkglist = list(value.split())
                elif item == "pkgver":
                    self.pkgver = value
                elif item == "pkgrel":
                    self.pkgrel = value
                elif item == "epoch":
                    if value == "0":
                        value = ""
                    if value:
                        self.epoch = "%s:" %value
                elif item == "arch":
                    self.arch = value
                elif item == "depends":
                    self.pkg_dependencies = value.split()
                elif item == "makedepends":
                        self.makepkg_dependencies = self.makepkg_dependencies + value.split()
                elif item == "makedepends_x86_64":
                        self.makepkg_dependencies = self.makepkg_dependencies + value.split()
                elif item == "checkdepends":
                            self.check_dependencies = value.split()

        self.set_srcpkg()
        # Get needed makepkg.conf variables
        mpkg="/etc/makepkg.conf"
        f=open(mpkg,"r")
        # Read file contents into memory
        makepkg_contents=f.readlines()
        f.close()

        self.PKGDEST = PKGHOME
        self.SRCDEST = SRCHOME
        self.SRCPKGDEST = SRCPKGHOME
        # Loop over contents to get our variables
        for mp_lines in makepkg_contents:
            if mp_lines.strip().startswith("CARCH"):
                self.CARCH = mp_lines.partition("=")[2].strip('\n').replace('\"','')
                if self.arch != 'any':
                    self.arch = self.CARCH
            if mp_lines.strip().startswith("PKGDEST"):
                self.PKGDEST = mp_lines.partition("=")[2].strip('\n')



    def print_vars(self):
        print("####################################################")
        print("")
        print("  Config File:",self.pkgfile)
        print("  pkgbase:",self.pkgbase)
        print("  pkgname:",self.pkgname)
        print("  pkglist:",self.pkglist)
        print("  epoch:" ,self.epoch)
        print("  pkgver:",self.pkgver)
        print("  pkgrel:",self.pkgrel)
        print("  arch:",self.arch)
        print("  Source package:",self.SRCPKG)
        print("  CARCH is:",self.CARCH)
        print("  Architecture is:",self.arch)
        print("  Repository : ", self.REPO)
        print("  DOCROOT : ", self.DOCROOT)
        print("  Current directory: ", self.mydir)
        #print("  Backup directory: ", self.get_backupdir())
        print("  Package destination:",self.PKGDEST)
        print("  Source destination:",self.SRCDEST)
        print("  Dependencies: ", self.pkg_dependencies)
        print("  Make Dependencies: ", self.makepkg_dependencies)
        print("  Package compile status: ", self.get_compile_status())
        print("")
        print("####################################################")

    def getname(self):
        return self.pkgname

    def getfailure(self):
        return self.failure

    def getattempts(self):
        return self.attempt

    def getdir(self):
        return self.mydir

    def get_unmet_compile(self):
        return self.pkg_dep_not_in_repo

    def get_unmet_install(self):
        return self.pkg_dep_available

    def incrementcompile(self):
        self.attempt = self.attempt + 1

    def get_backupdir(self):
        return self.backup_pkgdir

    def get_compile_status(self):
            return self.compile_status

    def find_repo(self):
        # Create a list with the directory names of our current directory tree
        dir_tree = os.path.dirname(self.mydir).split("/")
        # Loop through the dir_tree to see if we can find our repo
        for item in self.repolist:
            if item not in dir_tree:
                continue
            else:
                repo_name = item
            if repo_name == "extra":
                self.REPO = "extra" + SFIX
            elif repo_name == "core":
                self.REPO = "core" + SFIX
            elif repo_name == "linhes":
                self.REPO = "linhes" + SFIX
            elif repo_name == "xmpl":
                self.REPO = "local"
            elif repo_name not in repolist:
                print("ERROR in function find_repo: Cannot determine repository!")
                sys.exit(2)
            else:
                self.REPO = repo_name
        # Ensure our DOCROOT exists and if not, create it
        self.DOCROOT = "/data/dev/pkg_repo/" + self.CARCH + "/" + self.REPO
        #print("DOCROOT:",self.DOCROOT)
        if os.path.exists(self.DOCROOT):
            print("")
            #print("INFO: Repository is",self.REPO)
        else:
            try:
                os.mkdir(self.DOCROOT,0o755)
                print("INFO: ",self.DOCROOT,"directory created. Repository is",self.REPO)
            except Exception as e:
                print("ERROR: Creating DOCROOT failed: ", e)
                pass

    def increase_pkgrel(self):
        if not "--bump" in self.cli_list:
            return
        print("-  Incrementing pkgrel for %s" %self.pkgname)
        # Backup the original pkgfile
        shutil.copy2(self.pkgfile, self.pkgfile + ".old")
        # Let's increase the pkgrel
        for line in fileinput.input(self.pkgfile, inplace=1):
            if line.strip().startswith("pkgrel"):
                self.pkgrel = line.partition("=")[2].strip('\n')
                # Add 1 to pkgrel
                new_pkgrel = int(self.pkgrel) + 1
                line = line.replace("pkgrel=" + self.pkgrel, "pkgrel=" + str(new_pkgrel))
                self.pkgrel = str(new_pkgrel)
            sys.stdout.write(line)
        print("    pkgrel:",self.pkgrel)
        self.set_srcpkg()
        self.pkgrel_incremented = True

    def runPreFlight(self):
        if os.path.isfile("mp_preflight"):
            print("-  Running PreFlight...")
            os.chmod("mp_preflight", 0o755)
            retcode = subprocess.call(["./mp_preflight"])
            if retcode != 0:
                print("    ERROR: PreFlight Error")
            print("")

    def updateSUMS(self):
        print("-  Updating checksums...")
        retcode = subprocess.call(["updpkgsums"])
        if retcode != 0:
            self.compile_status = "Failed: %s" %retcode
            self.failure = self.failure  + 1

    def make_package(self):
        #creates both binary and source package
        print("   Removing old packages...")
        for oldpkgs in self.pkglist:
            pkglinks = glob.glob(oldpkgs + "*.pkg.tar.*")
            print("    Old Packages:",pkglinks)
            if pkglinks:
                for DELPKG in pkglinks:
                    os.remove(DELPKG)

        print("-  Making package...")
        retcode = subprocess.call(["sudo", "ccm", "s"])
        if retcode != 0:
            self.compile_status = "Failed: %s" %retcode
            self.failure = self.failure  + 1
            print("    ERROR: ccm failed with return code ",retcode)
            if self.pkgrel_incremented:
                if os.path.isfile(self.pkgfile + '.old'):
                    shutil.move(self.pkgfile + '.old', self.pkgfile)
                    print("        Reverted pkgrel increment.")
            return False
        else:
            print("-------------Making source package-------------")
            retcode = subprocess.call(["makepkg", "--force", "--holdver", "--source"])
            if retcode != 0:
                self.compile_status = "Failed source: %s" %retcode
                self.failure = self.failure  + 1
                print("ERROR: Source package creation failed with return code",retcode)
                sys.exit(2)
            print("=============FINISHED CREATING PKG=============")
            self.compile_status = "Success"
            return True

    def update_database(self):
        print("-   Copying files and updating database")
        # pkgname could be a list of several pkgs. Since bash array format is
        # loose, let bash parse the pkgname(s) first, then return a list for us.
        self.updateINIT()
        #if self.arch == 'any':
        #    self.CARCH = 'any'

        for i in self.pkglist:
            print("    Package name:",i)

            self.GZPKG = i + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + "-" + self.arch + ".pkg.tar.gz"
            self.XZPKG = i + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + "-" + self.arch + ".pkg.tar.xz"
            self.ZSTPKG = i + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + "-" + self.arch + ".pkg.tar.zst"
            #print(self.GZPKG)
            #print(self.XZPKG)
            #print(self.ZSTPKG)

            if os.path.isfile(self.ZSTPKG):
                self.TOTALPKG = self.ZSTPKG
            elif os.path.isfile(self.XZPKG):
                self.TOTALPKG = self.XZPKG
            elif os.path.isfile(self.GZPKG):
                self.TOTALPKG = self.GZPKG
            else:
                print("")
                print("    ERROR in function update_database: Couldn't find the new package:",self.TOTALPKG)
                sys.exit(2)

            # Remove old package(s) from local copy
            print("    Looking for old packages")

            oldfilelist = glob.glob(self.DOCROOT + "/" + i + "-*-*-*.pkg.tar.*")
            print("    Old Packages:",oldfilelist)
            if oldfilelist:
                for DELPKG in oldfilelist:
                    if "--rmold" in self.cli_list:
                        print("    Deleting old package:",DELPKG)
                        os.remove(DELPKG)

            # Copy in new package
            print("    Copying " + self.TOTALPKG + " to " + self.PKGDEST)
            if shutil.copy2(self.TOTALPKG, self.PKGDEST):
                os.remove(self.mydir + "/" + self.TOTALPKG)
            print("    Copying " + self.TOTALPKG + " to " + self.DOCROOT)
            shutil.copy2(self.PKGDEST + "/" + self.TOTALPKG, self.DOCROOT)
            print("    Creating symlink to " + self.PKGDEST + "/" + self.TOTALPKG)
            if os.path.islink(self.mydir + "/" + self.TOTALPKG):
                os.remove(self.mydir + "/" + self.TOTALPKG)
            os.symlink(self.DOCROOT + "/" + self.TOTALPKG, self.mydir + "/" + self.TOTALPKG)
            print("    Adding package to repo")
            subprocess.call(["repo-add", self.DOCROOT+ "/" + self.REPO + ".db.tar.gz", self.DOCROOT + "/" + self.TOTALPKG])

    def update_srcrepo(self):
        print("-   Updating source file repository")
        print("    SRCPKG:",self.SRCPKG)
        OLDSRCPKG=""

        if not os.path.isfile(self.SRCPKGDEST + "/" +self.SRCPKG):
            print("ERROR in function update_srcrepo: Couldn't find the new package",self.SRCPKG)
            sys.exit(2)

        i = self.pkglist[0]

        print("    Looking for old sources")
        oldfilelist = glob.glob(self.SRCPKGDEST + "/" + self.REPO + "/" + i + "-*-*.src.tar.gz")
        print("    Old Sources:",oldfilelist)
        if oldfilelist:
            for DELPKG in oldfilelist:
                if "--rmold" in self.cli_list:
                    print("    Deleting old source:",DELPKG)
                    os.remove(DELPKG)

        print("    Copying " + self.SRCPKG + " to " + self.SRCPKGDEST + "/" + self.REPO)
        if shutil.copy2(self.SRCPKGDEST + "/" + self.SRCPKG, self.SRCPKGDEST + "/" + self.REPO + "/"):
            os.remove(self.SRCPKGDEST + "/" + self.SRCPKG)


    def cleanup(self):
        os.chdir(self.mydir)
        if os.path.isfile(self.pkgfile + '.old'):
            os.remove(self.pkgfile + '.old')
            print("Removed temporary backup file",self.pkgfile + '.old')
        #remove src.tar.gz so it stops cluttering up git
        if os.path.islink(self.SRCPKG):
            os.unlink(self.SRCPKG)
        #remove src dir if empty
        if os.path.exists(self.mydir + "/src") and not os.path.isfile(self.mydir + "/src"):
            if not os.listdir(self.mydir + "/src"):
                os.rmdir(self.mydir + "/src")
        # Remove any old -build and -package log files
        os.chdir(self.mydir)
        build_log = self.pkgname + "-" + self.pkgver + "-" + self.pkgrel + "-" + self.arch + "-build.log"
        pkg_log = self.pkgname + "-" + self.pkgver + "-" + self.pkgrel + "-" + self.arch + "-package.log"
        all_logs = glob.glob("*.log*")
        #print "All Logs:",all_logs
        saved_logs = []
        #save the logrotate files
        saved_logs = glob.glob("*.logrotate*")
        #if os.path.isfile(build_log):
        #    saved_logs.append(build_log)
        #if os.path.isfile(pkg_log):
        #    saved_logs.append(pkg_log)
        #print ("Saved Logs:",saved_logs)
        for log in all_logs:
            if log not in saved_logs:
                os.remove(log)
            pass


    def check_for_changelog(self, pkgdir):
        filename = "%s/__changelog" %pkgdir
        return os.path.exists(filename)

def formatTD(time):
    # convert seconds to day, hour, minutes and seconds
    days = int(time // (24 * 3600))
    time = time % (24 * 3600)
    hours = int(time // 3600)
    time %= 3600
    minutes = int(time // 60)
    time %= 60
    seconds = int(time)

    if days == 0:
      day_string = ""
    elif days > 1:
      day_string = "%s days, " %days
    else:
      day_string = "%s day, " %days

    if hours == 0:
      hour_string = ""
    elif hours > 1:
      hour_string = "%s hours, " %hours
    else:
      hour_string = "%s hour, " %hours

    if minutes == 0:
      minute_string = ""
    elif minutes > 1:
      minute_string = "%s minutes, " %minutes
    else:
      minute_string = "%s minute, " %minutes

    if seconds > 1:
      second_string = "%s seconds" %seconds
    else:
      second_string = "%s second" %seconds

    return_string = '%s%s%s%s' % (day_string, hour_string, minute_string, second_string)
    return return_string

def main():
    pkg_not_found=[]
    pkg_objdict={}
    pkg_list=[]

    global makepkg_cmd
    global cli_list
    global cli_dict

    makepkg_cmd = ['makepkg']
    makepkg_cmd, cli_list, cli_dict = commandline(makepkg_cmd)

    try:
        pkg_list = cli_dict['--pkglist'].split(',')
    except:
        packagefile = None
        packagefile = Packagefile(cli_dict,cli_list,makepkg_cmd)
        pkg_objdict[packagefile.getname()]=packagefile
        pkg_list=[packagefile.getname()]


    packagefile.find_repo()
    packagefile.increase_pkgrel()
    packagefile.runPreFlight()
    packagefile.print_vars()
    packagefile.updateSUMS()
    if packagefile.make_package():
        packagefile.update_database()
        packagefile.update_srcrepo()
        packagefile.cleanup()
        pkg_objdict[packagefile.getname()]=packagefile

    update_pkg = []
    success_pkg = []
    failed_compile = []
    failed_update = []

    #create lists
    for k, v in pkg_objdict.items():
        #print k
        #print v.print_vars()
        if v.get_compile_status().strip() == "Success":
            success_pkg.append(k)
        else:
            temp = "%s --- %s (%s)" %(k,v.get_compile_status(),v.getattempts())
            failed_compile.append(temp)

    print("\n\n\n\n")
    print("#######################################################")

    print("\n")
    print("Couldn't find these packages:")
    print("---------------------------------")
    print(pkg_not_found)

    print("\n")
    print("Failed to compile these packages:")
    print("---------------------------------")
    print(failed_compile)

    print("\n")
    print("Successful compiled :")
    print("---------------------------------")
    print(success_pkg)
    print("\n")

if __name__ == "__main__":
    start = time.time()
    main()
    end = time.time()
    print("Build Time:",formatTD(end - start))
    print("#######################################################")