summaryrefslogtreecommitdiffstats
path: root/build_tools/bin/mp.py
blob: 77444501d2c35189b972b2ae9f883e78f666aae3 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
#!/usr/bin/env python2
# Version 0.6.9

import os
import sys
import re
import subprocess
import gettext
from git import Repo,Git
import ConfigParser
import glob
import shutil
import optparse
import fileinput

PKGHOME = "/data/pkg_repo/packages"
SRCPKGHOME = "/data/pkg_repo/src_packages"
mydir = os.getcwd()
repolist = ['core', 'extra']
variables = ['pkgbase','pkgname','pkgver','pkgrel','arch']
cli_dict = {}
cli_list = []
makepkg_cmd = ['makepkg']
REPO = "none"
SFIX = ""
DOCROOT = ""
pkgfile = "none"
pkgbase = ""
pkgname = ""
pkgver = ""
pkgrel = ""
arch = ""
CARCH = ""
PKGDEST = ""
TOTALPKG = ""
GZPKG = ""
XZPKG = ""

# See what git branch we're working under
git_repo = Git()
branches = git_repo.branch()
git_branch = branches.split('*')[1].lstrip(' ')

if git_branch == "testing":
    SFIX = "-testing"
elif git_branch == "master":
    SFIX = ""
else:
    print "Can't determine which git branch is in use!"
    sys.exit(2)
print "Git Branch suffix is:",SFIX

# Color stuff -- does this work? Looks to bash-y to me.
ALL_OFF = "$(tput sgr0)"
BOLD = "$(tput bold)"
BLUE = "${BOLD}$(tput setaf 4)"
GREEN = "${BOLD}$(tput setaf 2)"
RED = "${BOLD}$(tput setaf 1)"
YELLOW = "${BOLD}$(tput setaf 3)"

def plain():
    mesg = list.pop()
    print "${BOLD}    "+mesg+ "${ALL_OFF}\n"+cmdline

def msg():
    mesg = list.pop()
    print "${GREEN}==>${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline

def msg():
    mesg = list.pop()
    print "${BLUE}  ->${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline

def warning():
    warning = gettext(WARNING)
    mesg = list.pop()
    print "${YELLOW}==>"+ warning+ "${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline

def error():
    error = gettext(ERROR)
    mesg = list.pop()
    print "${RED}==>"+ error+ "${ALL_OFF}${BOLD} "+mesg+ "${ALL_OFF}\n"+cmdline

def commandline():
    global makepkg_cmd
    global cli_list
    global cli_dict

    clparser = optparse.OptionParser()
    clparser.add_option("--asroot", action="store_true", default=True, help="Allow makepkg to run as root.")
    clparser.add_option("-A", "--ignorearch", action="store_true", default=False, help="Ignore a missing or incomplete arch field in the build script.")
    clparser.add_option("-b", "--bump", action="store_true", default=False, help="Increase package release one unit.")
    clparser.add_option("-c", "--clean", action="store_true", default=False, help="Clean up leftover work files and directories after a successful build. ")
    clparser.add_option("-C", "--cleancache", action="store_true", default=False, help="Removes all cached source files from the directory specified in SRCDEST in makepkg.conf")
    clparser.add_option("--config", action="store", help="Use an alternate config file instead of the /etc/makepkg.conf default.")
    clparser.add_option("-d", "--nodeps", action="store_true", default=False, help="Do not perform any dependency checks.")
    clparser.add_option("-e", "--noextract", action="store_true", default=False, help="Do not extract source files; use whatever source already exists in the src/ directory.")
    clparser.add_option("-f", "--force", action="store_true", default=False, help="This allows a built package to be overwritten.")
    clparser.add_option("--forcever", action="store_true", default=False, help="This is a hidden option that should not be used unless you really know what you are doing.")
    clparser.add_option("-g", "--geninteg", action="store_true", default=False, help="For each source file in the source array of PKGBUILD, download the file if required and generate integrity checks.")
    clparser.add_option("--skipinteg", action="store_true", default=False, help="Do not perform any integrity checks, just print a warning instead.")
    clparser.add_option("--holdver", action="store_true", default=True, help="Prevents makepkg from automatically bumping the pkgver to the latest revision number in the package's development tree.")
    clparser.add_option("--nohold", action="store_false", dest="holdver", help="Automatically bump the pkgver to the latest revision number in the package's development tree.")
    clparser.add_option("-i", "--install", action="store_true", default=False, help="Install or upgrade the package after a successful build.")
    clparser.add_option("-L", "--log", action="store_true", default=True, help="Enable makepkg build logging.")
    clparser.add_option("-m", "--nocolor", action="store_true", default=False, help="Disable color in output messages.")
    clparser.add_option("-o", "--nobuild", action="store_true", default=False, help="Download and extract files only, but do not build them.")
    clparser.add_option("-p", action="store", help="Read the package script buildscript instead of the PKGBUILD default.", metavar='/path/to/buildscript')
    clparser.add_option("-r", "--rmdeps", action="store_true", default=False, help="Upon successful build, remove any dependencies installed by makepkg during dependency auto-resolution and installation when using -s.")
    clparser.add_option("-R", "--repackage", action="store_true", default=False, help="Repackage contents of the package without rebuilding the package.")
    clparser.add_option("-s", "--syncdeps", action="store_true", default=False, help="Install missing dependencies using pacman.")
    clparser.add_option("--allsource", action="store_true", default=False, help="Do not actually build the package, but build a source-only tarball that includes all sources, including those that are normally download via makepkg.")
    clparser.add_option("--source", action="store_true", default=False, help="Do not actually build the package, but build a source-only tarball that does not include sources that can be fetched via a download URL.")
    clparser.add_option("--pkg", action="store", help="Only build listed packages from a split package.")
    clparser.add_option("--noconfirm", action="store_true", default=False, help="(Passed to pacman) Prevent pacman from waiting for user input before proceeding with operations.")
    clparser.add_option("--noprogressbar", action="store_true", default=False, help="(Passed to pacman) Prevent pacman from displaying a progress bar.")
    clparser.add_option("--rmold", action="store_true", default=False, help="BETA: Remove old src and software packages from repos. Use with caution. False positives may occur (i.e. nvidia pkgs)!")

    (options, args) = clparser.parse_args()

    options1 = ['config', 'p', 'pkg']
    options2 = ['asroot', 'ignorearch', 'bump', 'clean', 'cleancache', 'nodeps',
               'noextract', 'force', 'forcever', 'geninteg', 'skipinteg', 'holdver',
               'install', 'log', 'nocolor', 'nobuild', 'rmdeps', 'repackage',
               'syncdeps', 'allsource', 'source', 'noconfirm', 'noprogressbar',
               'rmold']

    for o in options1:
        cmd1 = eval('options.'+o)
        if o is not 'p':
            if cmd1 is not None:
                cli_dict['--'+o] = cmd1
        elif cmd1 is not None:
                cli_dict['-'+o] = cmd1
    for o in options2:
        cmd2 = eval('options.'+o)
        if cmd2 is True:
            cli_list.append('--'+o)

    # Create makepkg command
    makepkg_cmd.extend(cli_list)
    for k in cli_dict.keys():
        v = cli_dict.get(k)
        makepkg_cmd.append(k)
        makepkg_cmd.append(v)

    # Remove bump option from makepkg command if it exists
    if "--bump" in makepkg_cmd:
        makepkg_cmd.remove("--bump")
    # Remove rmold option from makepkg command if it exists
    if "--rmold" in makepkg_cmd:
        makepkg_cmd.remove("--rmold")
    # Remove "--geninteg" option (if it exists) from makepkg command.
    # Checking for *sums is done automaticly by mp.py
    if "--geninteg" in makepkg_cmd:
        makepkg_cmd.remove("--geninteg")
    print "Makepkg Command:",makepkg_cmd

def bump_pkg():
    # We need pkgrel to live beyond this function
    global SRCPKG
    global pkgrel

    # Backup the original pkgfile
    shutil.copy2(pkgfile, pkgfile + ".old")
    # Let's increase the pkgrel
    for line in fileinput.input(pkgfile, inplace=1):
        if line.strip().startswith("pkgrel"):
            pkgrel = line.partition("=")[2].strip('\n')
            # Add 1 to pkgrel
            new_pkgrel = int(pkgrel) + 1
            line = line.replace("pkgrel=" + pkgrel, "pkgrel=" + str(new_pkgrel))
            pkgrel = str(new_pkgrel)
        sys.stdout.write(line)
    print "Bumped " + pkgname + " release to " + str(new_pkgrel)

    if pkgbase:
        SRCPKG = pkgbase + "-" + pkgver + "-" + str(new_pkgrel) + ".src.tar.gz"
    else:
        SRCPKG = pkgname + "-" + pkgver + "-" + str(new_pkgrel) + ".src.tar.gz"
    print "Source package will be: ",SRCPKG

def find_repo():
    global REPO
    global DOCROOT

    # Get all our possible repos into one list
    for item in ("chroot-devel", "mv-core", "xmpl", "local"):
        repolist.append(item)
    print "Repo List:",repolist

    # Create a list with the directory names of our current directory tree
    dir_tree = os.path.dirname(mydir).split("/")
    print "Dir Tree:",dir_tree
    # Loop through the dir_tree to see if we can find our repo
    for item in repolist: 
        if item not in dir_tree:
            continue
        else:
            repo_name = item
        if repo_name == "extra":
            REPO = "extra" + SFIX
        elif repo_name == "core":
            REPO = "core" + SFIX
        elif repo_name == "xmpl":
            REPO = "local"
        elif repo_name not in repolist:
            print "ERROR in function find_repo: Cannot determine repository!"
            sys.exit(2)
        else:
            REPO = repo_name
    # Ensure our DOCROOT exists and if not, create it
    DOCROOT = "/data/pkg_repo/" + CARCH + "/" + REPO
    print "DOCROOT:",DOCROOT
    if os.path.exists(DOCROOT):
        print "INFO: Repository is",REPO
    else:
        os.mkdir(DOCROOT,0755)
        print "INFO: ",DOCROOT,"directory created. Repository is",REPO

def update_repo():
    # pkgname could be a list of several pkgs. Since bash array format is
    # loose, let bash parse the pkgname(s) first, then return a list for us.
    os.chdir(DOCROOT)

    print
    print "INFO: Changed working dir to",DOCROOT
    print

    for i in pkglist:
        print "Package name:",i

        GZPKG = i + "-" + pkgver + "-" + pkgrel + "-" + CARCH + ".pkg.tar.gz"
        XZPKG = i + "-" + pkgver + "-" + pkgrel + "-" + CARCH + ".pkg.tar.xz"
        if os.path.isfile(PKGDEST + "/" + XZPKG):
            TOTALPKG = XZPKG
        elif os.path.isfile(PKGDEST + "/" + GZPKG):
            TOTALPKG = GZPKG
        else:
            print "ERROR in function update_repo: Couldn't find the new package",PKGDEST + "/" + TOTALPKG
            sys.exit(2)

        print "Package file:",TOTALPKG
        # Remove old package(s) from local copy
        pv = re.compile('[\d]+[\w.]+')
        dirlist = os.listdir(DOCROOT)
        for n in dirlist:
            if n.startswith(i):
                num = len(i) + 1
                print "Bytes to seek ahead:",num
                OLDPKG = glob.glob(i + "-" + pv.search(n,num).group() + "-[0-9]*.pkg.tar.?z")
                print "OLDPKG =",OLDPKG
                if OLDPKG:
                    for DELPKG in OLDPKG:
                        if "--rmold" in cli_list:
                            print "Deleting old package:",DELPKG
                            os.remove(DELPKG)
                    # Remove any symlinks to old packages
                    # We make it conditional on "--force" because force will overwrite
                    # an existing package and we want the symlink to stay, pointing to
                    # the newly built package with the same pkgrel.
                        if "--force" not in makepkg_cmd:
                            if os.path.islink(mydir + "/" + DELPKG):
                                os.remove(mydir + "/" + DELPKG)
        # Copy in new package
        print "############################################"
        print "Updating " + DOCROOT + " with " + TOTALPKG
        print "Copying " + PKGDEST + "/" + TOTALPKG
        shutil.copy2(PKGDEST + "/" + TOTALPKG, DOCROOT)
        subprocess.call(["repo-add", DOCROOT+ "/" + REPO + ".db.tar.gz", DOCROOT + "/" + TOTALPKG])
        print "############################################"

def update_src_pkg():
    print "---------------------------------SRC------------------------------"
    print "SRCPKG:",SRCPKG
    OLDSRCPKG = []
    os.chdir(SRCPKGHOME + "/" + REPO)
    dirlist = os.listdir(SRCPKGHOME + "/" + REPO)
    #pv = re.compile('[\d]+[.]*[\d]*[.]*[\d]*[.]*[\d]*[.]*[\d]*')
    pv = re.compile('[\d]+[\w.]+')
    if pkgbase:
        # Remove old src package(s) from local copy
        if "--rmold" in cli_list:
            for n in dirlist:
                if n.startswith(pkgbase):
                    num = len(pkgbase) + 1
                    print "Bytes to seek ahead:",num
                    OLDSRCPKG = glob.glob(pkgbase + "-" + pv.search(n,num).group() + "-*.src.tar.?z")
    else:
        # Remove old src package(s) from local copy
        if "--rmold" in cli_list:
            for n in dirlist:
                if n.startswith(pkgname):
                    num = len(pkgname) + 1
                    print "Bytes to seek ahead:",num
                    OLDSRCPKG = glob.glob(pkgname + "-" + pv.search(n,num).group() + "-*.src.tar.?z")
    if OLDSRCPKG:
        print "OLDSRCPKG =",OLDSRCPKG
        for DELSRCPKG in OLDSRCPKG:
            print "Deleting old source package",DELSRCPKG
            os.remove(DELSRCPKG)
    print "Copying new source package to",SRCPKGHOME + "/" + REPO + "/" + SRCPKG
    shutil.copy2(PKGHOME + "/" + SRCPKG, SRCPKGHOME + "/" + REPO + "/")

def dup_check():
    if REPO in ("local", "mv-core", "xmpl", "chroot-devel"):
        return
    for tmp_repo in ("core", "extra"):
        if tmp_repo + SFIX != REPO:
            p1 = subprocess.Popen(["pacman", "-Sl", tmp_repo + SFIX], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,)
            p2 = subprocess.Popen(["cut", "-d", " ", "-f", "2"], stdin=p1.stdout, stdout=subprocess.PIPE,)
            p3 = subprocess.Popen(["grep", pkgname], stdin=p2.stdout, stdout = subprocess.PIPE,)
            output = p3.communicate()[0].strip('\n')
            if output == 0:
                if "--force" not in cli_list:
                    print "#######################################"
                    print pkgname," already exists in ",tmp_repo + SFIX
                    print 
                    print "Use --force to overwite"
                    print "#######################################"
                    sys.exit(2)
                else:
                    print "Force detected! Making package regardless!"
                    print "#######################################"
            else:
                if output != 0:
                    return

def updateMD5():
    # Do we have integrity checks?
    sums = []

    # Open pkgfile for reading
    f = open(pkgfile, 'r')
    pkgfile_contents = f.readlines()
    f.close()

    # Iterate through the lines looking for a match pattern
    for line in pkgfile_contents:
        check = re.compile('sha1sums|sha256sums|sha384sums|sha512sums|md5sums')
        sums = check.match(line)
        if sums:
            break
    if not sums:
        # If no matches are found, append md5sums to the end of the pkgfile
        p = open(pkgfile, 'a')
        md5gen = subprocess.Popen(["makepkg", "--asroot", "-g"], stdout = subprocess.PIPE,).communicate()[0]
        p.writelines(md5gen)
        p.close()

def config_file():
    global pkgfile
    global pkgbase
    global pkgname
    global pkglist
    global pkgver
    global pkgrel
    global arch
    global SRCPKG
    global CARCH
    global PKGDEST

    # Check what file will be used for our PKGBUILD
    if "-p" in cli_dict:
        pkgfile = cli_dict["-p"]
    else:
        pkgfile = "PKGBUILD"
    print "Config File:",pkgfile

    # Does the file exist?
    if not os.path.isfile(pkgfile):
        print "ERROR in function config_file: Can't find ",pkgfile,"!"
        sys.exit(2)
    else:
        # Loop over contents to get our variables
        # Use bash to do it because PKGBUILDs are very loose with their format
        for item in variables:
            v = subprocess.Popen(['/bin/bash','-c', 'source ' + pkgfile + '; echo ${' + item + '[@]}'], stdout = subprocess.PIPE,)
            value = v.communicate()[0].strip('\n')
            if item == "pkgbase":
                pkgbase = value
                print "pkgbase:",pkgbase
            elif item == "pkgname":
                pkgname = value
                print "pkgname:",pkgname
                pkglist = list(value.split())
                print "pkglist:",pkglist
            elif item == "pkgver":
                pkgver = value
                print "pkgver:",pkgver
            elif item == "pkgrel":
                pkgrel = value
                print "pkgrel:",pkgrel
            elif item == "arch":
                arch = value
                print "arch:",arch

    if pkgbase:
        SRCPKG = pkgbase + "-" + pkgver + "-" + pkgrel + ".src.tar.gz"
    else:
        SRCPKG = pkgname + "-" + pkgver + "-" + pkgrel + ".src.tar.gz"
    print "Source package will be:",SRCPKG

    # Get needed makepkg.conf variables
    mpkg="/etc/makepkg.conf"
    f=open(mpkg,"r")
    # Read file contents into memory
    makepkg_contents=f.readlines()
    f.close()

    # Loop over contents to get our variables
    for mp_lines in makepkg_contents:
        if mp_lines.strip().startswith("CARCH"):
            CARCH = mp_lines.partition("=")[2].strip('\n').replace('\"','')
            if arch != 'any':
                arch = CARCH
        if mp_lines.strip().startswith("PKGDEST"):
            PKGDEST = mp_lines.partition("=")[2].strip('\n')
    print "CARCH is:",CARCH
    print "Package destination is:",PKGDEST
    print "Architecture is:",arch

def make_package():
    global arch
    global CARCH
    # Make the package!
    retcode = subprocess.call(makepkg_cmd)
    if retcode != 0:
        print "ERROR: makepkg failed with return code ",retcode
        if "--bump" in cli_list:
            if os.path.isfile(pkgfile + '.old'):
                shutil.move(pkgfile + '.old', pkgfile)
                print "Reverted pkgrel bump."
        sys.exit(2)
    elif "--nobuild" in makepkg_cmd:
        sys.exit(0)
    else:
        print "=============FINSHED CREATING PKG=================="
        if arch == 'any':
            CARCH = 'any'

# MAIN PROGRAM
commandline()
config_file()
find_repo()
dup_check()

# If "bump" was called, run the bump_pkg function
if "--bump" in cli_list:
    bump_pkg()

updateMD5()

# Everything look good? Go for it!
make_package()

print "------------updating database-------------"
update_repo()

print "----------creating source package---------"
os.chdir(mydir)
retcode = subprocess.call(["makepkg", "--force", "--holdver", "--asroot", "--source"])
if retcode != 0:
    print "ERROR: Source package creation failed with return code",retcode
    sys.exit(2)
else:
    update_src_pkg()

# Clean up old PKGBUILD if it exists
os.chdir(mydir)
if os.path.isfile(pkgfile + '.old'):
    os.remove(pkgfile + '.old')
    print "Removed temporary backup file",pkgfile + '.old'

# Remove any old -build and -package log files
os.chdir(mydir)
build_log = pkgname + "-" + pkgver + "-" + pkgrel + "-" + CARCH + "-build.log"
pkg_log = pkgname + "-" + pkgver + "-" + pkgrel + "-" + CARCH + "-package.log"
all_logs = glob.glob("*.log*")
#print "All Logs:",all_logs
saved_logs = []
if os.path.isfile(build_log):
    saved_logs.append(build_log)
if os.path.isfile(pkg_log):
    saved_logs.append(pkg_log)
#print "Saved Logs:",saved_logs
for log in all_logs:
    if log not in saved_logs:
        os.remove(log)