diff options
Diffstat (limited to 'build_tools')
| -rw-r--r-- | build_tools/bin/bld_resource.py | 576 | ||||
| -rwxr-xr-x | build_tools/bin/check_repo.sh | 6 | ||||
| -rw-r--r-- | build_tools/bin/check_repo/README | 8 | ||||
| -rw-r--r-- | build_tools/bin/check_repo/alpm.c | 40 | ||||
| -rwxr-xr-x | build_tools/bin/check_repo/alpm.so | bin | 0 -> 14742 bytes | |||
| -rwxr-xr-x | build_tools/bin/check_repo/build/lib.linux-x86_64-2.6/alpm.so | bin | 0 -> 14742 bytes | |||
| -rwxr-xr-x | build_tools/bin/check_repo/check_packages.py | 506 | ||||
| -rwxr-xr-x | build_tools/bin/check_repo/parse_pkgbuilds.sh | 149 | ||||
| -rw-r--r-- | build_tools/bin/check_repo/setup.py | 10 | ||||
| -rwxr-xr-x | build_tools/bin/repo-cleanup.sh | 28 | ||||
| -rwxr-xr-x | build_tools/go.sh | 28 |
11 files changed, 1133 insertions, 218 deletions
diff --git a/build_tools/bin/bld_resource.py b/build_tools/bin/bld_resource.py index afcb1d6..3ce9ef7 100644 --- a/build_tools/bin/bld_resource.py +++ b/build_tools/bin/bld_resource.py @@ -1,3 +1,4 @@ +#!/bin/python # -*- coding: utf-8 -*- # Form implementation generated from reading ui file 'bldtest.ui' @@ -21,7 +22,11 @@ from PyQt4.QtGui import * import os, sys, time, subprocess, shutil import shelve, thread, tarfile, tempfile import distutils.dir_util - +import pprint +import operator +import itertools +import pprint, commands +pp = pprint.PrettyPrinter(indent=4) try: import pexpect except: @@ -41,10 +46,10 @@ sepLine="\n-------------------------------------------------" global bld_data homedir = os.path.expanduser("~") bld_data="%s/.bld_session_data" %homedir - -print bld_data - - +global masterPkgList +global masterListPopulated +masterPkgList=[] +masterListPopulated=False class OutLog: def __init__(self, edit, out=None, color=None): @@ -105,6 +110,7 @@ class TableWidget(QTableWidget): menu.addSeparator() resetStatus = menu.addAction("Reset status") resetBuildOrder = menu.addAction("Reset build order") + resetSize = menu.addAction("Resize to width") action = menu.exec_(self.mapToGlobal(event.pos())) #if action == quitAction: @@ -139,6 +145,8 @@ class TableWidget(QTableWidget): self.resetstatus() if action == resetBuildOrder: self.resetBuildOrder() + if action == resetSize: + self.resetSize() def resetstatus(self): for row in range(self.rowCount()): @@ -149,6 +157,11 @@ class TableWidget(QTableWidget): for row in range(self.rowCount()): self.item(row, 0).setText("") return + def resetSize(self): + columnlist=[1,2,4,6,7] + for i in columnlist: + self.resizeColumnToContents(i) + return def keep_col_in_list(self,itemList,col): new_itemList=[] @@ -161,7 +174,7 @@ class TableWidget(QTableWidget): def pkglocation(self,item): row = item.row() - locitem = self.item(row,4) + locitem = self.item(row,5) loc = str(locitem.text()) return loc @@ -226,16 +239,11 @@ class MyThread(QtCore.QThread): self.emit(SIGNAL("update(QString)"),outmsg) self.running = 0 - - # self.close() - def runcmd(self,cmd_in,itemText,itemrow,builddir): arglist = " ".join(self.mparglist) cwd=os.getcwd() os.chdir(builddir) - #print builddir cmd="%s %s %s" %(cmd_in, arglist, itemText) - # print cmd retcode = -1 try: logfile = "%s/%s.build.log" %(builddir.strip(),itemText.strip()) @@ -254,28 +262,13 @@ class MyThread(QtCore.QThread): self.emit(SIGNAL("update(QString)"),line) pass return retcode - #os.chdir(cwd) - #process = subprocess.Popen( - #cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - #while True: - #outitemText = (process.stdout.readline()) - #if outitemText == '' and process.poll() != None: - #break - #if outitemText != '': - #self.emit(SIGNAL("update(QString)"),outitemText) - ## sys.stdout.write(itemText) - ##sys.stdout.flush() - #self.msleep(5) - - #retcode = process.poll() def render(self,cmd_in,itemlist,mparglist): self.itemlist = itemlist self.mparglist = mparglist self.cmd_in = cmd_in self.running = 0 - self.TIMEOUT=300 + self.TIMEOUT=300 self.start() def run(self): @@ -330,6 +323,123 @@ class MyThread(QtCore.QThread): self.alive +class create_pkglist_thread(QtCore.QThread): + def __init__(self, parent = None ): + QtCore.QThread.__init__(self, parent ) + self.alive = 1 + self.running = 0 + self.n = 0 + def value_init(self,filename,repo_location,repo_type,group_location): + self.group_filename_list = filename + self.repo_location = repo_location + self.repo_type = repo_type + self.group_location=group_location + global masterPkgList + masterPkgList = [] + global masterListPopulated + masterListPopulated=False + self.start() + + def run(self): + total_packages=0 + for group_filename in self.group_filename_list: + group_file="%s/%s.group" %(self.group_location,group_filename) + try: + f = open(group_file,"r") + file_contents = f.readlines() + total_packages_temp=len(file_contents) + f.close() + except: + print "couldn't open file: %s" %group_file + #outText = "couldn't open file to populate pkg list: %s " %filename + #self.update_ouputwindow(outText) + #self.comboBox.setEnabled(True) + total_packages+=total_packages_temp + + process_pkg_num = 0 + pkggroup_list=[] + for group_filename in self.group_filename_list: + group_file="%s/%s.group" %(self.group_location,group_filename) + try: + f = open(group_file,"r") + file_contents = f.readlines() + f.close() + except: + print "couldn't open file: %s" %group_file + #outText = "couldn't open file to populate pkg list: %s " %filename + #self.update_ouputwindow(outText) + #self.comboBox.setEnabled(True) + skipped=[] + + + for index, line in enumerate(file_contents): + process_pkg_num+=1 + progress = ((float(index+1) / float(total_packages)) * 100) + currentfile = "%s/%s/PKGBUILD" %(self.repo_location,line.strip()) + pkglocation = "%s/%s" %(self.repo_location,line.strip()) + if os.path.isfile(currentfile): + pkgname="unknown" + pkgversion="unknown" + pkgdescription="unknown" + pkggroup=[] + try: + f= open(currentfile,"r") + file_contents= f.readlines() + f.close() + for line in file_contents: + if line.strip().startswith("pkgname"): + pkgname=line.partition("=")[2] + + elif line.strip().startswith("pkgver"): + pkgversion=line.partition("=")[2] + + elif line.strip().startswith("pkgdesc"): + pkgdescription=line.partition("=")[2] + + elif line.strip().startswith("groups"): + l=line.partition("=")[2].strip() + for c in ['(',')',"'"]: + l = l.replace( c, '' ) + pkggroup=l.split() + + except Exception, e: + print "Couldn't parse file: %s\n \treason: %s" %(currentfile,e) + skipped.append(currentfile) + continue + for i in pkggroup: + tmp_pkggroup = group_filename + " " + i + pkggroup_list.append(tmp_pkggroup) + #pkggroup_list.append(i) + itemTextList=[pkgname,pkgversion,"-",pkglocation,pkgdescription,pkggroup] + itemDict={"pkgname" : pkgname, + "pkgversion" : pkgversion, + "pkglocation" : pkglocation, + "pkgdescription" : pkgdescription, + "pkggroup" : pkggroup, + "pkggroupfile": group_filename } + masterPkgList.append(itemDict) + time.sleep(0.005) + progress = ((float(process_pkg_num) / float(total_packages)) * 100) + self.emit(SIGNAL("update_progress_bar(float)"),progress) + + self.emit(SIGNAL("pkg_load_list_finished(bool)"),True) + if len(skipped) > 0: + self.emit(SIGNAL("pkg_load_skipped(PyQt_PyObject)"),skipped) + + grpset = set(pkggroup_list) + for i in grpset: + groupname="*%s" %i + self.emit(SIGNAL("insert_group(QString)"),groupname) + + def toggle(self): + if self.running: + self.running = 0 + else: + self.running = 1 + def stop(self): + self.alive = 0 + + class load_pkg_thread(QtCore.QThread): def __init__(self, parent = None ): @@ -343,58 +453,120 @@ class load_pkg_thread(QtCore.QThread): self.repo_type = repo_type self.start() + def run(self): - try: - f = open(self.group_filename,"r") - file_contents = f.readlines() - total_packages=len(file_contents) - f.close() - except: - print "couldn't open file: %s" %self.group_filename - #outText = "couldn't open file to populate pkg list: %s " %filename - #self.update_ouputwindow(outText) - #self.comboBox.setEnabled(True) skipped=[] - for index, line in enumerate(file_contents): - - progress = ((float(index+1) / float(total_packages)) * 100) - currentfile = "%s/%s/PKGBUILD" %(self.repo_location,line.strip()) - pkglocation = "%s/%s" %(self.repo_location,line.strip()) - if os.path.isfile(currentfile): - pkgname="unknown" - pkgversion="unknown" - pkgdescription="unknown" - try: - f= open(currentfile,"r") - file_contents= f.readlines() - f.close() - for line in file_contents: - if line.strip().startswith("pkgname"): - pkgname=line.partition("=")[2] + selected_pkggroup='' + selected_groupfilename=[] + size = 0 + if masterListPopulated and self.repo_type != "ABS": + if self.group_filename == "All-testing": + selected_groupfilename=["core-testing","extra-testing"] + elif self.group_filename == "All-stable": + selected_groupfilename=["core","extra"] + elif str(self.group_filename).startswith("*"): + selected_groupfilename=[ str(self.group_filename.replace("*",'')).split()[0] ] + selected_pkggroup=str(self.group_filename.replace("*",'')).partition(" ")[2] + else: + selected_groupfilename=[str(self.group_filename)] + + list1 = [] + add_pkglist=[] + + for key, items in itertools.groupby(masterPkgList, operator.itemgetter('pkggroupfile')): + list1.append(list(items)) + #find total size + for item in list1: + use_len = False + group_file = item[0]['pkggroupfile'] + if group_file in selected_groupfilename : + if selected_pkggroup: + for index,pkg in enumerate(item): + #loop over list to see if pkggroup matches + #if found increase size + pkggroup = pkg['pkggroup'] + if selected_pkggroup in pkggroup: + size = size + 1 + add_pkglist.append(pkg) + else: + use_len = True + + if use_len: + size += len(item) + totalprocessed=0 + #print size + templist=[] + if add_pkglist: + templist = add_pkglist + else: + for item in list1: + group_file = item[0]['pkggroupfile'] + if group_file in selected_groupfilename: + for a in item: + templist.append(a) - elif line.strip().startswith("pkgver"): - pkgversion=line.partition("=")[2] - elif line.strip().startswith("pkgdesc"): - pkgdescription=line.partition("=")[2] + for index,pkg in enumerate(templist): + totalprocessed+=1 + progress = ((float(totalprocessed) / float(size)) * 100) + itemTextList=[pkg["pkgname"],pkg["pkgversion"],"-",pkg["pkglocation"],pkg["pkgdescription"],pkg["pkggroup"]] + self.emit(SIGNAL("insertItem(PyQt_PyObject)"),itemTextList) + self.emit(SIGNAL("update_progress_bar(float)"),progress) - #package = parched.PKGBUILD(currentfile) - #pkgname = str(package.name) - #pkgversion = str(package.version) - #pkgdescription = str(package.description) + #loading from ABS + else: + try: + f = open(self.group_filename,"r") + file_contents = f.readlines() + total_packages=len(file_contents) + f.close() + except: + print "couldn't open file: %s" %self.group_filename + #outText = "couldn't open file to populate pkg list: %s " %filename + #self.update_ouputwindow(outText) + #self.comboBox.setEnabled(True) + + for index, line in enumerate(file_contents): + progress = ((float(index+1) / float(total_packages)) * 100) + currentfile = "%s/%s/PKGBUILD" %(self.repo_location,line.strip()) + pkglocation = "%s/%s" %(self.repo_location,line.strip()) + if os.path.isfile(currentfile): + pkgname="unknown" + pkgversion="unknown" + pkgdescription="unknown" + pkggroup=[] + try: + f= open(currentfile,"r") + file_contents= f.readlines() + f.close() + for line in file_contents: + if line.strip().startswith("pkgname"): + pkgname=line.partition("=")[2] + + elif line.strip().startswith("pkgver"): + pkgversion=line.partition("=")[2] + + elif line.strip().startswith("pkgdesc"): + pkgdescription=line.partition("=")[2] + + elif line.strip().startswith("groups"): + l=line.partition("=")[2].strip() + for c in ['(',')',"'"]: + l = l.replace( c, '' ) + pkggroup=l.split() + except Exception, e: + print "Couldn't parse file: %s\n \treason: %s" %(currentfile,e) + skipped.append(currentfile) + continue + itemTextList=[pkgname,pkgversion,"-",pkglocation,pkgdescription,pkggroup] + if self.repo_type == "ABS": + self.emit(SIGNAL("insertItem_ABS(PyQt_PyObject)"),itemTextList) + self.emit(SIGNAL("update_progress_bar_ABS(float)"),progress) + else: + self.emit(SIGNAL("insertItem(PyQt_PyObject)"),itemTextList) + self.emit(SIGNAL("update_progress_bar(float)"),progress) + time.sleep(0.005) - except Exception, e: - print "Couldn't parse file: %s\n \treason: %s" %(currentfile,e) - skipped.append(currentfile) - continue - itemTextList=[pkgname,pkgversion,"-",pkglocation,pkgdescription] - if self.repo_type == "ABS": - self.emit(SIGNAL("insertItem_ABS(PyQt_PyObject)"),itemTextList) - self.emit(SIGNAL("update_progress_bar_ABS(float)"),progress) - else: - self.emit(SIGNAL("insertItem(PyQt_PyObject)"),itemTextList) - self.emit(SIGNAL("update_progress_bar(float)"),progress) - time.sleep(0.005) self.emit(SIGNAL("pkg_load_finished(bool)"),True) if len(skipped) > 0: self.emit(SIGNAL("pkg_load_skipped(PyQt_PyObject)"),skipped) @@ -417,7 +589,7 @@ class run_cmd_thread(QtCore.QThread): def value_init(self,runcmd,args): self.runcmd = runcmd self.args = args - self.TIMEOUT=300 + self.TIMEOUT=300 self.start() def run(self): @@ -436,7 +608,7 @@ class run_cmd_thread(QtCore.QThread): for line in child: f.write(line) self.emit(SIGNAL("update_build(QString)"),line) - self.msleep(20) + self.msleep(50) child.close() retcode=child.exitstatus f.write(str(retcode)) @@ -448,8 +620,6 @@ class run_cmd_thread(QtCore.QThread): pass self.emit(SIGNAL("enable_tab2(bool)"),True) - #return retcode - def toggle(self): if self.running: self.running = 0 @@ -459,12 +629,11 @@ class run_cmd_thread(QtCore.QThread): self.alive = 0 - class Ui_MainWindow(object): def setupUi(self, MainWindow): MainWindow.setObjectName("MainWindow") MainWindow.setEnabled(True) - MainWindow.resize(923, 757) + MainWindow.resize(982, 870) MainWindow.setLayoutDirection(QtCore.Qt.LeftToRight) self.centralwidget = QtGui.QWidget(MainWindow) self.centralwidget.setObjectName("centralwidget") @@ -545,7 +714,7 @@ class Ui_MainWindow(object): self.horizontalLayout.addWidget(self.progressBar) self.comboBox = QtGui.QComboBox(self.groupBox_2) self.comboBox.setEnabled(True) - self.comboBox.setMinimumSize(QtCore.QSize(200, 0)) + self.comboBox.setMinimumSize(QtCore.QSize(250, 0)) self.comboBox.setObjectName("comboBox") self.horizontalLayout.addWidget(self.comboBox) self.pushButton_8 = QtGui.QPushButton(self.groupBox_2) @@ -580,7 +749,7 @@ class Ui_MainWindow(object): self.tableWidget.setGridStyle(QtCore.Qt.SolidLine) self.tableWidget.setCornerButtonEnabled(False) self.tableWidget.setObjectName("tableWidget") - self.tableWidget.setColumnCount(6) + self.tableWidget.setColumnCount(7) self.tableWidget.setRowCount(0) item = QtGui.QTableWidgetItem() self.tableWidget.setHorizontalHeaderItem(0, item) @@ -594,6 +763,8 @@ class Ui_MainWindow(object): self.tableWidget.setHorizontalHeaderItem(4, item) item = QtGui.QTableWidgetItem() self.tableWidget.setHorizontalHeaderItem(5, item) + item = QtGui.QTableWidgetItem() + self.tableWidget.setHorizontalHeaderItem(6, item) self.tableWidget.horizontalHeader().setCascadingSectionResizes(True) self.tableWidget.horizontalHeader().setStretchLastSection(True) self.horizontalLayout_5.addWidget(self.tableWidget) @@ -840,7 +1011,7 @@ class Ui_MainWindow(object): self.tableWidget_2.setVerticalScrollMode(QtGui.QAbstractItemView.ScrollPerItem) self.tableWidget_2.setCornerButtonEnabled(False) self.tableWidget_2.setObjectName("tableWidget_2") - self.tableWidget_2.setColumnCount(6) + self.tableWidget_2.setColumnCount(7) self.tableWidget_2.setRowCount(0) item = QtGui.QTableWidgetItem() self.tableWidget_2.setHorizontalHeaderItem(0, item) @@ -854,6 +1025,8 @@ class Ui_MainWindow(object): self.tableWidget_2.setHorizontalHeaderItem(4, item) item = QtGui.QTableWidgetItem() self.tableWidget_2.setHorizontalHeaderItem(5, item) + item = QtGui.QTableWidgetItem() + self.tableWidget_2.setHorizontalHeaderItem(6, item) self.tableWidget_2.horizontalHeader().setStretchLastSection(True) self.verticalLayout_6.addWidget(self.tableWidget_2) self.verticalLayout_9.addWidget(self.groupBox) @@ -1053,7 +1226,7 @@ class Ui_MainWindow(object): self.dockWidget_3.setWidget(self.dockWidgetContents_6) MainWindow.addDockWidget(QtCore.Qt.DockWidgetArea(2), self.dockWidget_3) self.menubar = QtGui.QMenuBar(MainWindow) - self.menubar.setGeometry(QtCore.QRect(0, 0, 923, 24)) + self.menubar.setGeometry(QtCore.QRect(0, 0, 982, 24)) self.menubar.setObjectName("menubar") self.menuWelcome = QtGui.QMenu(self.menubar) self.menuWelcome.setEnabled(True) @@ -1179,17 +1352,16 @@ class Ui_MainWindow(object): self.menubar.addAction(self.menuSystem.menuAction()) self.retranslateUi(MainWindow) - self.tabWidget.setCurrentIndex(4) + self.tabWidget.setCurrentIndex(0) self.tabWidget_2.setCurrentIndex(0) QtCore.QObject.connect(self.actionExit, QtCore.SIGNAL("triggered()"), MainWindow.close) QtCore.QMetaObject.connectSlotsByName(MainWindow) - #------------- self.tabWidget.setCurrentIndex(0) #set location column to 5, this is enough to hide it but still accessiable. - self.tableWidget.horizontalHeader().resizeSection(4,5) - self.tableWidget_2.horizontalHeader().resizeSection(4,5) + self.tableWidget.horizontalHeader().resizeSection(5,5) + self.tableWidget_2.horizontalHeader().resizeSection(5,5) #set order column to 20 wide self.tableWidget.horizontalHeader().resizeSection(0,20) self.tableWidget_2.horizontalHeader().resizeSection(0,20) @@ -1199,16 +1371,9 @@ class Ui_MainWindow(object): self.thread_pkg_load = load_pkg_thread() self.run_cmd_thread = run_cmd_thread() self.upload_file_thread = run_cmd_thread() - + self.createPkgListThread = create_pkglist_thread() sys.stdout = OutLog(self.textEdit,out=sys.__stdout__) sys.stderr = OutLog(self.textEdit,out=sys.__stdout__) - #self.DLG = search.BookmarksDlg(['uno', 'dos'], None) - #self.DLG.hide() - - - - - #enable/disable t QtCore.QObject.connect(self.run_cmd_thread, QtCore.SIGNAL("enable_tab2(bool)"), self.enable_tab2) @@ -1216,6 +1381,8 @@ class Ui_MainWindow(object): QtCore.QObject.connect(self.run_cmd_thread, SIGNAL("finished()"), self.run_cmd_thread_finished) QtCore.QObject.connect(self.upload_file_thread, SIGNAL("finished()"), self.upload_thread_finished) + QtCore.QObject.connect(self.createPkgListThread,SIGNAL("pkg_load_list_finished(bool)"), self.pkg_load_finished) + QtCore.QObject.connect(self.createPkgListThread,SIGNAL("insert_group(QString)"), self.insert_group) #connect pkgload thread to insert item QtCore.QObject.connect(self.thread_pkg_load, QtCore.SIGNAL("insertItem(PyQt_PyObject)"), self.insert_item) @@ -1269,6 +1436,7 @@ class Ui_MainWindow(object): QtCore.QObject.connect(self.thread,QtCore.SIGNAL("update_progress_bar(int)"),self.update_progress_bar) QtCore.QObject.connect(self.thread_pkg_load, QtCore.SIGNAL("update_progress_bar_ABS(float)"),self.update_progress_bar_abs) QtCore.QObject.connect(self.thread_pkg_load, QtCore.SIGNAL("update_progress_bar(float)"),self.update_progress_bar) + QtCore.QObject.connect(self.createPkgListThread, QtCore.SIGNAL("update_progress_bar(float)"),self.update_progress_bar) QtCore.QObject.connect(self.thread_pkg_load, QtCore.SIGNAL("pkg_load_skipped(PyQt_PyObject)"), self.print_skipped_pkg) @@ -1296,8 +1464,6 @@ class Ui_MainWindow(object): QtCore.QObject.connect(self.toolButton_7, QtCore.SIGNAL("clicked()"), self.open_copy_location) QtCore.QObject.connect(self.toolButton_8, QtCore.SIGNAL("clicked()"), self.select_profile_dir) - - #Top menu actions QtCore.QObject.connect(self.actionSave_output_window_to_file, QtCore.SIGNAL("triggered()"), self.save_text_to_file) QtCore.QObject.connect(self.actionLoad_configuration_file, QtCore.SIGNAL("triggered()"), self.load_settings_from_file) @@ -1343,18 +1509,27 @@ class Ui_MainWindow(object): QtCore.QObject.connect(self.buttonBox, QtCore.SIGNAL("clicked(QAbstractButton*)"), self.show_which_button) QtCore.QObject.connect(self.toolButton_4, QtCore.SIGNAL("clicked()"), self.select_newpkg_dir) - #misc screen QtCore.QObject.connect(self.pushButton_sync, QtCore.SIGNAL("clicked()"), self.run_sync) QtCore.QObject.connect(self.create_cd, QtCore.SIGNAL("clicked()"), self.createcd) QtCore.QObject.connect(self.pushbutton_upload, QtCore.SIGNAL("clicked()"), self.upload_files) + def insert_group(self,g): + self.comboBox.addItem(g) + + def pkg_load_finished(self): + global masterListPopulated + masterListPopulated=True + self.comboBox.addItem("All-stable") + self.comboBox.addItem("All-testing") + self.enable_table() + def clear_searchbox(self): self.lineEdit_21.setText('') + def clear_searchbox_ABS(self): self.lineEdit_20.setText('') - def searchRepoPkglist(self,textValue): # print textValue if textValue=='': @@ -1391,8 +1566,6 @@ class Ui_MainWindow(object): if self.tableWidget_2.horizontalScrollBar().value > 5: self.tableWidget_2.horizontalScrollBar().setValue (0) - - def show_search(self): print "show" self.DLG.show() @@ -1412,7 +1585,10 @@ class Ui_MainWindow(object): self.showbox("Reset interface", msg,True) def repo_check_for_problems(self): - print "no problems to report, but I didn't run anything" + cmd = "%s" %(self.all_settings['repocheck']) + (status,msg) = commands.getstatusoutput(cmd) + self.update_ouputwindow(msg) + def show_which_button(self, clicked_button): role = QtGui.QDialogButtonBox.buttonRole(self.buttonBox, clicked_button) @@ -1463,7 +1639,6 @@ class Ui_MainWindow(object): self.update_ouputwindow(line,PrintBoth) def update_ouputwindow(self,line,PrintBoth=True): - self.textEdit.append(line) if PrintBoth: self.textEdit_dock.append(line) @@ -1479,20 +1654,17 @@ class Ui_MainWindow(object): self.update_ouputwindow(itemText) self.showbox("Command", "Finished!",True) - def upload_thread_finished(self): itemText="File upload finished" self.update_ouputwindow(itemText) self.showbox("File upload", "Finished!",True) - - def print_finished_msg(self,Value): outText="Finished Loading Packages!!" self.update_ouputwindow(outText) self.tableWidget.setSortingEnabled(True) self.tableWidget_2.setSortingEnabled(True) - columnlist=[1,2,3,5] + columnlist=[1,2,3,6] for i in columnlist: self.tableWidget.resizeColumnToContents(i) self.tableWidget_2.resizeColumnToContents(i) @@ -1513,7 +1685,6 @@ class Ui_MainWindow(object): self.tabWidget_2.setEnabled(Value) def disable_table(self): - self.tableWidget.setEnabled(False) self.comboBox.setEnabled(False) self.pushButton_8.setEnabled(False) @@ -1521,7 +1692,6 @@ class Ui_MainWindow(object): self.disable_go_button() def enable_table(self): - self.tableWidget.setEnabled(True) self.comboBox.setEnabled(True) self.pushButton_8.setEnabled(True) @@ -1536,7 +1706,6 @@ class Ui_MainWindow(object): self.lineEdit_20.setEnabled(Value) self.pushButton_2.setEnabled(Value) - def set_repo_box(self,Value): self.comboBox.setEnabled(Value) self.comboBox_2.setEnabled(Value) @@ -1553,7 +1722,6 @@ class Ui_MainWindow(object): self.tableWidget_2.setEnabled(True) self.comboBox_2.setEnabled(True) - def update_status_cell(self,itemrow,itemText,status,itemcol): item = self.tableWidget.item(itemrow,itemcol) self.tableWidget.setItemSelected (item, False) @@ -1571,9 +1739,29 @@ class Ui_MainWindow(object): pkgversion=itemTextList[1] pkglocation = itemTextList[3] pkgdescription=itemTextList[4] + pkggroup=itemTextList[5] + pkgtooltip=pkgdescription + if pkggroup: + pkgtooltip+="Groups: " + pkgloc=[] + pkglist=[] + pkgloc.append(pkgname.strip()) + pkgloc.append(pkglocation) + #print pkgloc + #for i in pkggroup: + #pkglist=[] + #pkg_group_name="%s/pkg_group_%s.group" %(self.all_settings['group_file_location'],i) + #pkgtuple=(pkgname.strip(),pkglocation) + #pkglist.append(pkgtuple) + ##pkglist.append(pkgname) + #pkglist.append(pkgloc) + #self.addPkgGroup(pkg_group_name,pkglist) + #pkgtooltip+=" " + #pkgtooltip+= i +# print pkggroup itr = self.tableWidget.rowCount() self.tableWidget.insertRow(itr) - for i in range(0,6): + for i in range(0,7): item = QtGui.QTableWidgetItem() item.setToolTip(pkgdescription) item.setFlags(Qt.ItemIsUserCheckable | @@ -1591,9 +1779,10 @@ class Ui_MainWindow(object): self.tableWidget.item(itr,1).setText(pkgname) self.tableWidget.item(itr,2).setText(pkgversion) self.tableWidget.item(itr,3).setText("-") - self.tableWidget.item(itr,4).setText(pkglocation) - self.tableWidget.item(itr,5).setText(pkgdescription) - self.tableWidget.item(itr,5).setToolTip(pkgdescription) + self.tableWidget.item(itr,4).setText(",".join(pkggroup)) + self.tableWidget.item(itr,5).setText(pkglocation) + self.tableWidget.item(itr,6).setText(pkgdescription) + self.tableWidget.item(itr,6).setToolTip(pkgtooltip) self.updateLCDtotal_pkg() def insert_item_ABS(self,itemTextList): @@ -1601,9 +1790,19 @@ class Ui_MainWindow(object): pkgversion=itemTextList[1] pkglocation = itemTextList[3] pkgdescription=itemTextList[4] + pkggroup=itemTextList[5] + pkgtooltip=pkgdescription + if pkggroup: + pkgtooltip+="Groups: " + pkgloc=[] + pkglist=[] + pkgloc.append(pkgname.strip()) + pkgloc.append(pkglocation) + + itr = self.tableWidget_2.rowCount() self.tableWidget_2.insertRow(itr) - for i in range(0,6): + for i in range(0,7): item = QtGui.QTableWidgetItem() item.setToolTip(pkgdescription) item.setFlags(Qt.ItemIsUserCheckable | @@ -1619,8 +1818,9 @@ class Ui_MainWindow(object): self.tableWidget_2.item(itr,1).setText(pkgname) self.tableWidget_2.item(itr,2).setText(pkgversion) self.tableWidget_2.item(itr,3).setText("-") - self.tableWidget_2.item(itr,4).setText(pkglocation) - self.tableWidget_2.item(itr,5).setText(pkgdescription) + self.tableWidget_2.item(itr,4).setText(",".join(pkggroup)) + self.tableWidget_2.item(itr,5).setText(pkglocation) + self.tableWidget_2.item(itr,6).setText(pkgdescription) self.updateLCDtotal_ABS() def populate_pkg_list(self,repo_value): @@ -1633,11 +1833,17 @@ class Ui_MainWindow(object): self.tableWidget.clearContents() self.tableWidget.setSortingEnabled(False) self.tableWidget.setRowCount(0) - filename = self.all_settings['group_file_location'] + "/" + repo_value + ".group" - outtext="Loading package list from %s" %filename - self.update_ouputwindow(outtext) - self.thread_pkg_load.value_init(filename,self.all_settings['repo_location'],"mainrepo") - headers=[".","Name", "Version", "Status" , "Location", "Description"] + + if masterListPopulated: + outtext="Loading package list from memory" + self.update_ouputwindow(outtext) + self.thread_pkg_load.value_init(repo_value,self.all_settings['repo_location'],"mainrepo") + else: + filename = self.all_settings['group_file_location'] + "/" + repo_value + ".group" + outtext="Loading package list from %s" %filename + self.update_ouputwindow(outtext) + self.thread_pkg_load.value_init(filename,self.all_settings['repo_location'],"mainrepo") + headers=[".","Name", "Version", "Status" ,"Group", "Location", "Description"] self.tableWidget.setHorizontalHeaderLabels(headers) def populate_abs_pkg_list(self,abs_repo_value): @@ -1655,7 +1861,7 @@ class Ui_MainWindow(object): outtext="Loading package list from %s" %filename self.update_ouputwindow(outtext) self.thread_pkg_load.value_init(filename,self.all_settings['abs_location'],"ABS") - headers=[".","Name", "Version", "Status" , "Location", "Description"] + headers=[".","Name", "Version", "Status" ,"Group", "Location", "Description"] self.tableWidget_2.setHorizontalHeaderLabels(headers) def select_all(self): @@ -1670,14 +1876,11 @@ class Ui_MainWindow(object): total_rows = self.tableWidget_2.rowCount() self.tableWidget_2.setRangeSelected(QtGui.QTableWidgetSelectionRange(0, 1, total_rows-1, 1), False) - def onIndexChanged(self, index): global repo_value repo_value = index self.tableWidget.clear() if repo_value: - t=repo_value, - #thread.start_new_thread(self.populate_pkg_list, (t)) self.populate_pkg_list(repo_value) def onIndexChanged_ABS(self, index): @@ -1685,8 +1888,6 @@ class Ui_MainWindow(object): ABS_repo_value = index self.tableWidget_2.clear() if ABS_repo_value: - t=ABS_repo_value, - #thread.start_new_thread(self.populate_abs_pkg_list, (t)) self.populate_abs_pkg_list(ABS_repo_value) def showbox(self,title,message,Modal=True): @@ -1696,15 +1897,7 @@ class Ui_MainWindow(object): self.msgbox.setText(message) self.msgbox.setModal(Modal) self.msgbox.show() - #ialog.setModal (self, bool modal) - #QMessageBox.information(None,title,message) - #Dialog.setModal (self, bool modal) - #msgbox = QMessageBox - #msgBox.setText("The document has been modified.") - #msgBox.setInformativeText("Do you want to save your changes?") - #msgBox.setStandardButtons(QMessageBox.Save | QMessageBox.Discard | QMessageBox.Cancel) - ##msgBox.setDefaultButton(QMessageBox.Save); - #msgBox.exec + def show_yes_no_box(self,title,message,Modal=True): self.Modal=Modal self.msgyesbox = QtGui.QMessageBox() @@ -1713,15 +1906,11 @@ class Ui_MainWindow(object): self.msgyesbox.setModal(Modal) self.msgyesbox.setStandardButtons(QMessageBox.Yes | QMessageBox.No); self.msgyesbox.setDefaultButton(QMessageBox.Save); - #int ret = self.msgyesbox.exec(); - if self.msgyesbox.exec_() == 16384 : return True else: return False - - def buttonpushed(self): itemlist=self.tableWidget.selectedItems() makepkg_list=[] @@ -1743,17 +1932,26 @@ class Ui_MainWindow(object): self.thread.toggle() def populate_group_box(self): + self.disable_table() postfix=".group" + grouplist=[] dir_list = os.listdir(self.all_settings['group_file_location']) self.comboBox.clear() + self.comboBox.setInsertPolicy(6) self.comboBox.setEnabled(False) for file in dir_list: if os.path.isfile(self.all_settings['group_file_location']+"/"+file): if file.endswith(postfix) and not file.startswith("ABS"): groupname = file[:-len(postfix)] - self.comboBox.addItem(groupname) + grouplist.append(groupname) #self.comboBox.setCurrentIndex(1) - self.comboBox.setEnabled(True) + grouplist.sort() + for g in grouplist: + self.comboBox.addItem(g) + #self.comboBox.setEnabled(True) + #kick of thread to create list of stuff + self.disable_table() + self.createPkgListThread.value_init(grouplist,self.all_settings['repo_location'],"mainrepo",self.all_settings['group_file_location']) def populate_abs_group_box(self): postfix=".group" @@ -1849,9 +2047,6 @@ class Ui_MainWindow(object): os.chdir("..") self.populate_abs_group_box() - - - def disable_go_button(self): self.pushButton_go.setEnabled(False) self.pushButton_stop.setEnabled(True) @@ -1860,7 +2055,7 @@ class Ui_MainWindow(object): def enable_go_button(self): self.pushButton_go.setEnabled(True) self.pushButton_stop.setEnabled(False) - self.comboBox.setEnabled(True) + #self.comboBox.setEnabled(True) def find_pkgdir(self): #return list of selected pkg locations @@ -1916,35 +2111,6 @@ class Ui_MainWindow(object): pass return False - #def search_group_file(self,groupfile,spackage): - ##""" - ## Searches group file for matching spackage name. - ## Returns False if the file was not found. - ##""" - #filename = bld_config.repo_location + "/" + groupfile + ".group" - #try: - #f = open(filename,"r") - #file_contents = f.readlines() - #f.close() - #except: - #outText = "couldn't open file to search: %s " %filename - #self.update_ouputwindow(outText) - #return False - #for line in file_contents: - #currentdir = "%s/%s" %(bld_config.repo_location,line.strip()) - #currentfile = "%s/PKGBUILD" %(currentdir) - #if os.path.isfile(currentfile): - #try: - #package = parched.PKGBUILD(currentfile) - #pkgname = str(package.name) - #pkgversion = str(package.version) - #pkgdescription = str(package.description) - #if pkgname == spackage: - #return currentdir - #except: - #continue - #return False - def runcmd_thread(self,cmd): os.system(cmd) @@ -2077,9 +2243,10 @@ class Ui_MainWindow(object): group = "%s/adHoc.group" %self.all_settings['group_file_location'] self.addPkgGroup(group) - def addPkgGroup(self,group=None): + def addPkgGroup(self,group=None,dir_list=None): groupname = group - dir_list = self.find_pkgdir() + if dir_list == None: + dir_list = self.find_pkgdir() if groupname == None: options = QtGui.QFileDialog.Options() options |= QtGui.QFileDialog.DontUseNativeDialog @@ -2094,16 +2261,19 @@ class Ui_MainWindow(object): self.update_ouputwindow(outtext) return filename = groupname + print filename try: f = open(filename,"a") except: outtext="Couldn't open file for writing: %s" %filename print outtext return False - print filename + for item,dir in dir_list: - itemText= str(item.text()) - print dir + if not type(item) == str: + itemText= str(item.text()) + else: + itemText=item.strip() location = dir.replace(self.all_settings["repo_location"]+"/", "") outtext = "Adding %s to %s" %(location,group) self.update_ouputwindow(outtext) @@ -2111,12 +2281,6 @@ class Ui_MainWindow(object): f.write("\n") f.close() - - - - - - def actionChangeLog(self): dir_list = self.find_pkgdir() for item,dir in dir_list: @@ -2158,7 +2322,6 @@ class Ui_MainWindow(object): cmd = cmd, thread.start_new_thread(self.runcmd_thread, cmd) - def backupAction(self): current_time=time.strftime("%Y-%m-%d_%H-%M-%S") dir_list = self.find_pkgdir() @@ -2202,7 +2365,6 @@ class Ui_MainWindow(object): def updatepkg(self,item,dir): #updates our pkg with the contents from abs self.update_ouputwindow(sepLine) - pkgname = item.text() itemrow = item.row() itemcol = item.column() @@ -2378,7 +2540,6 @@ class Ui_MainWindow(object): def promote(self,prunedlist,dir): - for item in prunedlist: itemText= str(item.text()) itemrow = item.row() @@ -2426,8 +2587,6 @@ class Ui_MainWindow(object): self.update_ouputwindow(outtext) return True - - def promoteAction(self): self.update_ouputwindow(sepLine) self.disable_table() @@ -2504,9 +2663,6 @@ class Ui_MainWindow(object): if dir: self.lineEdit_19.setText(dir) - - - def select_newpkg_dir(self): dir = self.getExistingDirectory("Location to create new packages.",self.all_settings['repo_location']) if dir: @@ -2659,8 +2815,6 @@ class Ui_MainWindow(object): outtext = "Could not create group: %s" %(text) self.update_ouputwindow(outtext) - - def updateSystem(self): runcmd="pacman" cmdargs=["-Su","--noconfirm"] @@ -2668,11 +2822,15 @@ class Ui_MainWindow(object): def switch_pacman_to_loca(self): print "switch to local" + cmd='''sed -i "s/Server = http\:.*knoppmyth.net.*$/ Server = file:\/\/data\/pkg_repo\/i686\/\$repo/g" /etc/pacman.conf''' + os.system(cmd) + self.updatePacmanDB() def switch_pacman_to_remote(self): print "switch to remote" - - + cmd='''sed -i "s/Server = file.*$/ Server = http:\/\/knoppmyth.net\/repo\/i686\/\$repo/g" /etc/pacman.conf''' + os.system(cmd) + self.updatePacmanDB() def save_config_settings(self): from __main__ import settings_dict @@ -2750,13 +2908,11 @@ class Ui_MainWindow(object): if self.checkBox_ignore.isChecked(): mp_arg_list.append("-d") return mp_arg_list - -#--- - +#-------------------------------------------------------------- def retranslateUi(self, MainWindow): MainWindow.setWindowTitle(QtGui.QApplication.translate("MainWindow", "BLD", None, QtGui.QApplication.UnicodeUTF8)) - self.label_22.setText(QtGui.QApplication.translate("MainWindow", "Search Repo", None, QtGui.QApplication.UnicodeUTF8)) + self.label_22.setText(QtGui.QApplication.translate("MainWindow", "Search Group", None, QtGui.QApplication.UnicodeUTF8)) self.pushButton.setText(QtGui.QApplication.translate("MainWindow", "Search", None, QtGui.QApplication.UnicodeUTF8)) self.pushButton_5.setText(QtGui.QApplication.translate("MainWindow", "clear", None, QtGui.QApplication.UnicodeUTF8)) self.checkBox_3.setText(QtGui.QApplication.translate("MainWindow", "Search All", None, QtGui.QApplication.UnicodeUTF8)) @@ -2772,8 +2928,9 @@ class Ui_MainWindow(object): self.tableWidget.horizontalHeaderItem(1).setText(QtGui.QApplication.translate("MainWindow", "Name", None, QtGui.QApplication.UnicodeUTF8)) self.tableWidget.horizontalHeaderItem(2).setText(QtGui.QApplication.translate("MainWindow", "Version", None, QtGui.QApplication.UnicodeUTF8)) self.tableWidget.horizontalHeaderItem(3).setText(QtGui.QApplication.translate("MainWindow", "Status", None, QtGui.QApplication.UnicodeUTF8)) - self.tableWidget.horizontalHeaderItem(4).setText(QtGui.QApplication.translate("MainWindow", "Location", None, QtGui.QApplication.UnicodeUTF8)) - self.tableWidget.horizontalHeaderItem(5).setText(QtGui.QApplication.translate("MainWindow", "Description", None, QtGui.QApplication.UnicodeUTF8)) + self.tableWidget.horizontalHeaderItem(4).setText(QtGui.QApplication.translate("MainWindow", "Group", None, QtGui.QApplication.UnicodeUTF8)) + self.tableWidget.horizontalHeaderItem(5).setText(QtGui.QApplication.translate("MainWindow", "Location", None, QtGui.QApplication.UnicodeUTF8)) + self.tableWidget.horizontalHeaderItem(6).setText(QtGui.QApplication.translate("MainWindow", "Description", None, QtGui.QApplication.UnicodeUTF8)) self.tabWidget.setTabText(self.tabWidget.indexOf(self.tab), QtGui.QApplication.translate("MainWindow", "Packages", None, QtGui.QApplication.UnicodeUTF8)) self.groupBox_14.setTitle(QtGui.QApplication.translate("MainWindow", "Profiles", None, QtGui.QApplication.UnicodeUTF8)) self.create_cd.setText(QtGui.QApplication.translate("MainWindow", "Create CD", None, QtGui.QApplication.UnicodeUTF8)) @@ -2819,8 +2976,9 @@ class Ui_MainWindow(object): self.tableWidget_2.horizontalHeaderItem(1).setText(QtGui.QApplication.translate("MainWindow", "Name", None, QtGui.QApplication.UnicodeUTF8)) self.tableWidget_2.horizontalHeaderItem(2).setText(QtGui.QApplication.translate("MainWindow", "Version", None, QtGui.QApplication.UnicodeUTF8)) self.tableWidget_2.horizontalHeaderItem(3).setText(QtGui.QApplication.translate("MainWindow", "Status", None, QtGui.QApplication.UnicodeUTF8)) - self.tableWidget_2.horizontalHeaderItem(4).setText(QtGui.QApplication.translate("MainWindow", "Location", None, QtGui.QApplication.UnicodeUTF8)) - self.tableWidget_2.horizontalHeaderItem(5).setText(QtGui.QApplication.translate("MainWindow", "Description", None, QtGui.QApplication.UnicodeUTF8)) + self.tableWidget_2.horizontalHeaderItem(4).setText(QtGui.QApplication.translate("MainWindow", "Group", None, QtGui.QApplication.UnicodeUTF8)) + self.tableWidget_2.horizontalHeaderItem(5).setText(QtGui.QApplication.translate("MainWindow", "Location", None, QtGui.QApplication.UnicodeUTF8)) + self.tableWidget_2.horizontalHeaderItem(6).setText(QtGui.QApplication.translate("MainWindow", "Description", None, QtGui.QApplication.UnicodeUTF8)) self.tabWidget.setTabText(self.tabWidget.indexOf(self.tab_5), QtGui.QApplication.translate("MainWindow", "Copy from ABS", None, QtGui.QApplication.UnicodeUTF8)) self.toolButton_3.setText(QtGui.QApplication.translate("MainWindow", "...", None, QtGui.QApplication.UnicodeUTF8)) self.toolButton_2.setText(QtGui.QApplication.translate("MainWindow", "...", None, QtGui.QApplication.UnicodeUTF8)) @@ -2922,6 +3080,6 @@ class Ui_MainWindow(object): self.actionInstall_all_updates.setText(QtGui.QApplication.translate("MainWindow", "System update", None, QtGui.QApplication.UnicodeUTF8)) self.actionSwitch_to_local.setText(QtGui.QApplication.translate("MainWindow", "Switch to local", None, QtGui.QApplication.UnicodeUTF8)) self.actionSwitch_to_remote.setText(QtGui.QApplication.translate("MainWindow", "Switch to remote", None, QtGui.QApplication.UnicodeUTF8)) - self.actionClear_Adhoc_Group.setText(QtGui.QApplication.translate("MainWindow", "Clear Adhoc Group", None, QtGui.QApplication.UnicodeUTF8)) + self.actionClear_Adhoc_Group.setText(QtGui.QApplication.translate("MainWindow", "Clear Adhoc Groupe", None, QtGui.QApplication.UnicodeUTF8)) self.actionCreate_Group.setText(QtGui.QApplication.translate("MainWindow", "Create Group", None, QtGui.QApplication.UnicodeUTF8)) diff --git a/build_tools/bin/check_repo.sh b/build_tools/bin/check_repo.sh new file mode 100755 index 0000000..50ebbab --- /dev/null +++ b/build_tools/bin/check_repo.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cd /build_tools/bin +#extra-testing will do both core and extra -testing +check_repo/check_packages.py --abs-tree=/data/LinHES-PKGBUILD/abs --repos=extra --repo-dir=/data/pkg_repo/i686 +check_repo/check_packages.py --abs-tree=/data/LinHES-PKGBUILD/abs --repos=extra-testing --repo-dir=/data/pkg_repo/i686 +cd - diff --git a/build_tools/bin/check_repo/README b/build_tools/bin/check_repo/README new file mode 100644 index 0000000..f3a1b90 --- /dev/null +++ b/build_tools/bin/check_repo/README @@ -0,0 +1,8 @@ +1) Build the python module +$ python setup.py build + +2) copy it back to the current working directory +$ cp build/lib.*/alpm.* . + +3) run the script +$ ./check_packages.py -h diff --git a/build_tools/bin/check_repo/alpm.c b/build_tools/bin/check_repo/alpm.c new file mode 100644 index 0000000..0b7cd2c --- /dev/null +++ b/build_tools/bin/check_repo/alpm.c @@ -0,0 +1,40 @@ +#include <Python.h> +#include <alpm.h> + +static PyObject * +alpm_vercmp(PyObject *self, PyObject *args) +{ + const char *v1, *v2; + int ret; + + if (!PyArg_ParseTuple(args, "ss", &v1, &v2)) + return NULL; + ret = alpm_pkg_vercmp(v1, v2); + return Py_BuildValue("i", ret); +} + +static PyMethodDef AlpmMethods[] = { + {"vercmp", alpm_vercmp, METH_VARARGS, + "Execute vercmp."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +PyMODINIT_FUNC +initalpm(void) +{ + (void) Py_InitModule("alpm", AlpmMethods); +} + +int +main(int argc, char *argv[]) +{ + /* Pass argv[0] to the Python interpreter */ + Py_SetProgramName(argv[0]); + + /* Initialize the Python interpreter. Required. */ + Py_Initialize(); + + /* Add a static module */ + initalpm(); + return 0; +} diff --git a/build_tools/bin/check_repo/alpm.so b/build_tools/bin/check_repo/alpm.so Binary files differnew file mode 100755 index 0000000..f6c4b3d --- /dev/null +++ b/build_tools/bin/check_repo/alpm.so diff --git a/build_tools/bin/check_repo/build/lib.linux-x86_64-2.6/alpm.so b/build_tools/bin/check_repo/build/lib.linux-x86_64-2.6/alpm.so Binary files differnew file mode 100755 index 0000000..f6c4b3d --- /dev/null +++ b/build_tools/bin/check_repo/build/lib.linux-x86_64-2.6/alpm.so diff --git a/build_tools/bin/check_repo/check_packages.py b/build_tools/bin/check_repo/check_packages.py new file mode 100755 index 0000000..39ac4fb --- /dev/null +++ b/build_tools/bin/check_repo/check_packages.py @@ -0,0 +1,506 @@ +#!/usr/bin/python +# +# check_archlinux.py +# +# Original script by Scott Horowitz <stonecrest@gmail.com> +# Rewritten by Xavier Chantry <shiningxc@gmail.com> +# +# This script currently checks for a number of issues in your ABS tree: +# 1. Directories with missing PKGBUILDS +# 2. Invalid PKGBUILDs (bash syntax error for instance) +# 3. PKGBUILD names that don't match their directory +# 4. Duplicate PKGBUILDs +# 5. Valid arch's in PKGBUILDS +# 6. Missing (make-)dependencies +# 7. Hierarchy of repos (e.g., that a core package doesn't depend on +# a non-core package) +# 8. Circular dependencies + +import os,re,commands,getopt,sys,tarfile,alpm +import pdb + +DBEXT='.db.tar.gz' + +packages = {} # pkgname : PacmanPackage +repopkgs = {} # pkgname : PacmanPackage +provisions = {} # provision : PacmanPackage +pkgdeps,makepkgdeps = {},{} # PacmanPackage : list of the PacmanPackage dependencies +invalid_pkgbuilds = [] +missing_pkgbuilds = [] +dups = [] + +dbonly = [] +absonly = [] + +mismatches = [] +missing_deps = [] +missing_makedeps = [] +invalid_archs = [] +dep_hierarchy = [] +makedep_hierarchy = [] +circular_deps = [] # pkgname>dep1>dep2>...>pkgname +checked_deps = [] + +class PacmanPackage: + def __init__(self): + self.name,self.version = "","" + self.base = "" + self.path,self.repo = "","" + self.deps,self.makedeps = [],[] + self.provides,self.conflicts = [],[] + self.archs = [] + +class Depend: + def __init__(self,name,version,mod): + self.name = name + self.version = version + self.mod = mod + +def parse_pkgbuilds(repos,arch): + for absroot in absroots: + for repo in repos: + cmd = os.path.dirname(os.path.realpath(sys.argv[0])) + '/parse_pkgbuilds.sh ' + cmd += arch + ' ' + absroot + '/' + repo + print cmd + (status,output) = commands.getstatusoutput(cmd) + if status != 0: + print "Error : failed to run '%s'" % cmd + sys.exit() + parse_data(repo,output) + +def parse_data(repo,data): + attrname = None + + for line in data.split('\n'): + if line.startswith('%'): + attrname = line.strip('%').lower() + elif line.strip() == '': + attrname = None + elif attrname == "invalid": + if repo in repos: + invalid_pkgbuilds.append(line) + elif attrname == "missing": + if repo in repos: + missing_pkgbuilds.append(line) + elif attrname == "name": + pkg = PacmanPackage() + pkg.name = line + pkg.repo = repo + dup = None + if packages.has_key(pkg.name): + dup = packages[pkg.name] + else: + packages[pkg.name] = pkg + elif attrname == "base": + pkg.base = line + elif attrname == "version": + pkg.version = line + elif attrname == "path": + pkg.path = line + if dup != None and (pkg.repo in repos or dup.repo in repos): + dups.append(pkg.path + " vs. " + dup.path) + elif attrname == "arch": + pkg.archs.append(line) + elif attrname == "depends": + pkg.deps.append(line) + elif attrname == "makedepends": + pkg.makedeps.append(line) + elif attrname == "conflicts": + pkg.conflicts.append(line) + elif attrname == "provides": + pkg.provides.append(line) + +def parse_dbs(repos,arch): + dbpkgs = {} + for repo in repos: + pkgs = set([]) + db = tarfile.open(os.path.join(repodir,repo,repo + DBEXT)) + for line in db.getnames(): + if not '/' in line: + pkgs.add(line.rsplit('-',2)[0]) + dbpkgs[repo] = pkgs + return(dbpkgs) + +def splitdep(dep): + name = dep + version = "" + mod = "" + for char in (">=", "<=", "=", ">", "<"): + pos = dep.find(char) + if pos > -1: + name = dep[:pos] + version = dep[pos:].replace(char, "") + mod = char + break + return Depend(name,version,mod) + +def splitprov(prov): + name = prov + version = "" + pos = prov.find("=") + if pos > -1: + name = prov[:pos] + version = prov[pos:].replace("=", "") + return (name,version) + +def vercmp(v1,mod,v2): + res = alpm.vercmp(v1,v2) + if res == 0: + return (mod.find("=") > -1) + elif res < 0: + return (mod.find("<") > -1) + elif res > 0: + return (mod.find(">") > -1) + return False + + +def depcmp(name,version,dep): + if name != dep.name: + return False + if dep.version == "" or dep.mod == "": + return True + if version == "": + return False + return vercmp(version,dep.mod,dep.version) + +def provcmp(pkg,dep): + for prov in pkg.provides: + (provname,provver) = splitprov(prov) + if depcmp(provname,provver,dep): + return True + return False + +def verify_dep(dep): + dep = splitdep(dep) + if packages.has_key(dep.name): + pkg = packages[dep.name] + if depcmp(pkg.name,pkg.version,dep): + return [pkg] + if provisions.has_key(dep.name): + provlist = provisions[dep.name] + results = [] + for prov in provlist: + if provcmp(prov,dep): + results.append(prov) + return results + return [] + +def verify_deps(name,repo,deps): + pkg_deps = [] + missdeps = [] + hierarchy = [] + for dep in deps: + pkglist = verify_dep(dep) + if pkglist == []: + missdeps.append(repo + "/" + name + " --> '" + dep + "'") + else: + valid_repos = get_repo_hierarchy(repo) + pkgdep = None + for pkg in pkglist: + if pkg.repo in valid_repos: + pkgdep = pkg + break + if not pkgdep: + pkgdep = pkglist[0] + hierarchy.append((repo,name,pkgdep)) + + pkg_deps.append(pkgdep) + + return (pkg_deps,missdeps,hierarchy) + +def compute_deplist(pkg): + list = [] + stack = [pkg] + while stack != []: + dep = stack.pop() + if pkgdeps.has_key(dep): + for dep2 in pkgdeps[dep]: + if dep2 not in list: + list.append(dep2) + stack.append(dep2) + if makepkgdeps.has_key(dep): + for dep2 in makepkgdeps[dep]: + if dep2 not in list: + list.append(dep2) + stack.append(dep2) + return list + +def check_hierarchy(deph): + hierarchy = [] + for (repo,name,pkgdep) in deph: + deplist = compute_deplist(pkgdep) + valid_repos = get_repo_hierarchy(repo) + extdeps = [] + for dep in deplist: + if dep.repo not in valid_repos: + extdeps.append(dep.name) + string = repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name + " (" + string += "%s extra (make)deps to pull" % len(extdeps) + if 0 < len(extdeps) < 10: + string += " : " + ' '.join(extdeps) + string += ")" + hierarchy.append(string) + return hierarchy + +def get_repo_hierarchy(repo): + repo_hierarchy = {'core': ['core'], \ + 'extra': ['core', 'extra'], \ + 'extra-testing': ['core-testing', 'extra-testing'], \ + 'core-testing': ['core-testing'], \ + 'mv-core': ['mv-core'], \ + 'community': ['core', 'extra', 'community']} + if repo_hierarchy.has_key(repo): + return repo_hierarchy[repo] + else: + return ['core','extra'] + #return ['core','extra','community'] + #return ['core','extra','community'] + +def verify_archs(name,repo,archs): + valid_archs = ['any', 'i686', 'x86_64'] + invalid_archs = [] + for arch in archs: + if arch not in valid_archs: + invalid_archs.append(repo + "/" + name + " --> " + arch) + return invalid_archs + +def find_scc(packages): + # reset all variables + global index,S,pkgindex,pkglowlink + index = 0 + S = [] + pkgindex = {} + pkglowlink = {} + cycles = [] + for pkg in packages: + tarjan(pkg) + +def tarjan(pkg): + global index,S,pkgindex,pkglowlink,cycles + pkgindex[pkg] = index + pkglowlink[pkg] = index + index += 1 + checked_deps.append(pkg) + S.append(pkg) + deps = [] + if pkgdeps.has_key(pkg): + deps = pkgdeps[pkg] + for dep in deps: + if not pkgindex.has_key(dep): + tarjan(dep) + pkglowlink[pkg] = min(pkglowlink[pkg],pkglowlink[dep]) + elif dep in S: + pkglowlink[pkg] = min(pkglowlink[pkg],pkgindex[dep]) + if pkglowlink[pkg] == pkgindex[pkg]: + dep = S.pop() + if pkg == dep: + return + path = pkg.name + while pkg != dep: + path = dep.repo + "/" + dep.name + ">" + path + dep = S.pop() + path = dep.name + ">" + path + if pkg.repo in repos: + circular_deps.append(path) + +def print_heading(heading): + print "" + print "=" * (len(heading) + 4) + print "= " + heading + " =" + print "=" * (len(heading) + 4) + +def print_subheading(subheading): + print "" + print subheading + print "-" * (len(subheading) + 2) + +def print_missdeps(pkgname,missdeps) : + for d in missdeps: + print pkgname + " : " + d + +def print_result(list, subheading): + if len(list) > 0: + list.sort() + print_subheading(subheading) + for item in list: + print item + +def print_results(): + print_result(missing_pkgbuilds, "Missing PKGBUILDs") + print_result(invalid_pkgbuilds, "Invalid PKGBUILDs") + print_result(mismatches, "Mismatched Pkgnames") + print_result(dups, "Duplicate PKGBUILDs") + print_result(invalid_archs, "Invalid Archs") + print_result(missing_deps, "Missing Dependencies") + print_result(missing_makedeps, "Missing Makedepends") + print_result(dep_hierarchy, "Repo Hierarchy for Dependencies") + print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends") + print_result(circular_deps, "Circular Dependencies") + print_result(dbonly, "Packages found in db, but not in tree") + print_result(absonly,"Packages found in tree, but not in db") + print_subheading("Summary") + print "Missing PKGBUILDs: ", len(missing_pkgbuilds) + print "Invalid PKGBUILDs: ", len(invalid_pkgbuilds) + print "Mismatching PKGBUILD names: ", len(mismatches) + print "Duplicate PKGBUILDs: ", len(dups) + print "Invalid archs: ", len(invalid_archs) + print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps) + print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy) + print "Circular dependencies: ", len(circular_deps) + print "In db, but not in tree: ", len(dbonly) + print "In tree, but not in db: ", len(absonly) + print "" + +def print_usage(): + print "" + print "Usage: ./check_packages.py [OPTION]" + print "" + print "Options:" + print " --abs-tree=<path[,path]> Check the specified tree(s) (default : /var/abs)" + print " --repos=<r1,r2,...> Check the specified repos (default : core,extra)" + print " --arch=<i686|x86_64> Check the specified arch (default : i686)" + print " --repo-dir=<path> Check the dbs at the specified path (default : /srv/ftp)" + print " -h, --help Show this help and exit" + print "" + print "Examples:" + print "\n Check core and extra in existing abs tree:" + print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra --arch=i686" + print "\n Check community:" + print " ./check_packages.py --abs-tree=/var/abs --repos=community --arch=i686" + print "" + +## Default path to the abs root directory +absroots = ["/var/abs"] +## Default list of repos to check +repos = ['core', 'extra'] +## Default arch +arch = "i686" +## Default repodir +repodir = "/srv/ftp" + +try: + opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", + "arch=", "repo-dir="]) +except getopt.GetoptError: + print_usage() + sys.exit() +if opts != []: + for o, a in opts: + if o in ("--abs-tree"): + absroots = a.split(',') + elif o in ("--repos"): + repos = a.split(",") + elif o in ("--arch"): + arch = a + elif o in ("--repo-dir"): + repodir = a + else: + print_usage() + sys.exit() + if args != []: + print_usage() + sys.exit() + +for absroot in absroots: + if not os.path.isdir(absroot): + print "Error : the abs tree " + absroot + " does not exist" + sys.exit() + for repo in repos: + repopath = absroot + "/" + repo + if not os.path.isdir(repopath): + print "Error : the repository " + repo + " does not exist in " + absroot + sys.exit() +if not os.path.isdir(repodir): + print "Error: the repository directory %s does not exist" % repodir + sys.exit() +for repo in repos: + #path = os.path.join(repodir,repo,repo + DBEXT) + path = os.path.join(repodir,repo,repo + DBEXT) + if not os.path.isfile(path): + print "Error : repo DB %s : File not found" % path + sys.exit() + if not tarfile.is_tarfile(path): + print "Error : Cant open repo DB %s, not a valid tar file" % path + sys.exit() +# repos which need to be loaded +loadrepos = set([]) +for repo in repos: + loadrepos = loadrepos | set(get_repo_hierarchy(repo)) + +print_heading("Integrity Check " + arch + " of " + ",".join(repos)) +print "\nPerforming integrity checks..." + +print "==> parsing pkgbuilds" +parse_pkgbuilds(loadrepos,arch) + +# fill provisions +for name,pkg in packages.iteritems(): + for prov in pkg.provides: + provname=prov.split("=")[0] + if not provisions.has_key(provname): + provisions[provname] = [] + provisions[provname].append(pkg) + +# fill repopkgs +for name,pkg in packages.iteritems(): + if pkg.repo in repos: + repopkgs[name] = pkg + +print "==> parsing db files" +dbpkgs = parse_dbs(repos,arch) + +print "==> checking mismatches" +for name,pkg in repopkgs.iteritems(): + pkgdirname = pkg.path.split("/")[-1] + if name != pkgdirname and pkg.base != pkgdirname: + mismatches.append(name + " vs. " + pkg.path) + +print "==> checking archs" +for name,pkg in repopkgs.iteritems(): + archs = verify_archs(name,pkg.repo,pkg.archs) + invalid_archs.extend(archs) + +# ugly hack to strip the weird kblic- deps +for name,pkg in packages.iteritems(): + p = re.compile('klibc-[\w\-]{27}|klibc-\*') + pkg.deps = [dep for dep in pkg.deps if not p.match(dep)] + pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)] + +deph,makedeph = [],[] + +print "==> checking dependencies" +for name,pkg in repopkgs.iteritems(): + (deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps) + pkgdeps[pkg] = deps + missing_deps.extend(missdeps) + deph.extend(hierarchy) + +print "==> checking makedepends" +for name,pkg in repopkgs.iteritems(): + (makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps) + makepkgdeps[pkg] = makedeps + missing_makedeps.extend(missdeps) + makedeph.extend(hierarchy) + +print "==> checking hierarchy" +dep_hierarchy = check_hierarchy(deph) +makedep_hierarchy = check_hierarchy(makedeph) + +print "==> checking for circular dependencies" +# make sure pkgdeps is filled for every package +for name,pkg in packages.iteritems(): + if not pkgdeps.has_key(pkg): + (deps,missdeps,_) = verify_deps(name,pkg.repo,pkg.deps) + pkgdeps[pkg] = deps +find_scc(repopkgs.values()) + +print "==> checking for differences between db files and pkgbuilds" +for repo in repos: + for pkg in dbpkgs[repo]: + if not (pkg in repopkgs.keys() and repopkgs[pkg].repo == repo): + dbonly.append("%s/%s" % (repo,pkg)) +for name,pkg in repopkgs.iteritems(): + if not name in dbpkgs[pkg.repo]: + absonly.append("%s/%s" % (pkg.repo,name)) + +print_results() diff --git a/build_tools/bin/check_repo/parse_pkgbuilds.sh b/build_tools/bin/check_repo/parse_pkgbuilds.sh new file mode 100755 index 0000000..5cd17e4 --- /dev/null +++ b/build_tools/bin/check_repo/parse_pkgbuilds.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +# Usage : parse_pkgbuilds.sh arch <pkgbuilds_dir1,dir2,...> +# Example : parse_pkgbuilds.sh i686 /var/abs/core /var/abs/extra + +exit() { return; } + +splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts') +variables=('pkgname' 'pkgbase' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]}) +readonly -a variables splitpkg_overrides + +backup_package_variables() { + for var in ${splitpkg_overrides[@]}; do + indirect="${var}_backup" + eval "${indirect}=(\${$var[@]})" + done +} + +restore_package_variables() { + for var in ${splitpkg_overrides[@]}; do + indirect="${var}_backup" + if [ -n "${!indirect}" ]; then + eval "${var}=(\${$indirect[@]})" + else + unset ${var} + fi + done +} + +print_info() { + echo -e "%NAME%\n$pkgname\n" + echo -e "%VERSION%\n$pkgver-$pkgrel\n" + echo -e "%PATH%\n$dir\n" + + if [ -n "$pkgbase" ]; then + echo -e "%BASE%\n$pkgbase\n" + fi + + if [ -n "$arch" ]; then + echo "%ARCH%" + for i in ${arch[@]}; do echo $i; done + echo "" + fi + if [ -n "$depends" ]; then + echo "%DEPENDS%" + for i in ${depends[@]}; do + echo $i + done + echo "" + fi + if [ -n "$makedepends" ]; then + echo "%MAKEDEPENDS%" + for i in ${makedepends[@]}; do + echo $i + done + echo "" + fi + if [ -n "$conflicts" ]; then + echo "%CONFLICTS%" + for i in ${conflicts[@]}; do echo $i; done + echo "" + fi + if [ -n "$provides" ]; then + echo "%PROVIDES%" + for i in ${provides[@]}; do echo $i; done + echo "" + fi +} + +source_pkgbuild() { + ret=0 + dir=$1 + pkgbuild=$dir/PKGBUILD + for var in ${variables[@]}; do + unset ${var} + done + source $pkgbuild &>/dev/null || ret=$? + + # ensure $pkgname and $pkgver variables were found + if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + fi + + if [ "${#pkgname[@]}" -gt "1" ]; then + pkgbase=${pkgbase:-${pkgname[0]}} + for pkg in ${pkgname[@]}; do + if [ "$(type -t package_${pkg})" != "function" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + else + backup_package_variables + pkgname=$pkg + while IFS= read -r line; do + var=${line%%=*} + var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters + for realvar in ${variables[@]}; do + if [ "$var" == "$realvar" ]; then + eval $line + break + fi + done + done < <(type package_${pkg}) + print_info + restore_package_variables + fi + done + else + echo + print_info + fi + + return 0 +} + +find_pkgbuilds() { + #Skip over some dirs + local d="${1##*/}" + if [ "$d" = "CVS" -o "$d" = ".svn" ]; then + return + fi + + if [ -f $1/PKGBUILD ]; then + source_pkgbuild $1 + return + fi + empty=1 + for dir in $1/*; do + if [ -d $dir ]; then + find_pkgbuilds $dir + unset empty + fi + done + if [ -n "$empty" ]; then + echo -e "%MISSING%\n$1\n" + fi +} + +if [ -z "$1" -o -z "$2" ]; then + exit 1 +fi + +CARCH=$1 +shift +for dir in "$@"; do + find_pkgbuilds $dir +done + +exit 0 diff --git a/build_tools/bin/check_repo/setup.py b/build_tools/bin/check_repo/setup.py new file mode 100644 index 0000000..b172752 --- /dev/null +++ b/build_tools/bin/check_repo/setup.py @@ -0,0 +1,10 @@ +from distutils.core import setup, Extension + +alpm = Extension('alpm', + libraries = ['alpm'], + sources = ['alpm.c']) + +setup (name = 'Alpm', + version = '1.0', + description = 'Alpm bindings', + ext_modules = [alpm]) diff --git a/build_tools/bin/repo-cleanup.sh b/build_tools/bin/repo-cleanup.sh new file mode 100755 index 0000000..bb4c369 --- /dev/null +++ b/build_tools/bin/repo-cleanup.sh @@ -0,0 +1,28 @@ +#!/bin/bash +currentrepo=$1 +echo $currentrepo +if [ x$currentrepo = "x" ] +then + echo "./repo_cleanup.sh $repo [core,core-testing,extra,extra-testing]" + exit 1 +fi +REPO=/data/pkg_repo/i686/$currentrepo +pacman -Sl $repo |cut -d" " -f 2- |tr " " - > /tmp/$currentrepo-list +cd $REPO >/dev/null +ls *-i686.pkg.tar.gz|sed -e "s/-i686.pkg.tar.gz//" > /tmp/pkglist-$currentrepo +cd - > /dev/null +if [ ! -e /tmp/$currentrepo_dups ] +then + mkdir /tmp/$currentrepo_dups/ +fi +while read LINE +do +# echo $LINE + grep -q $LINE /tmp/$currentrepo-list + if [ $? = 1 ] + then + echo "removing file $LINE" + mv $REPO/$LINE-i686.pkg.tar.gz /tmp/$currentrepo_dups/ + fi +done < /tmp/pkglist-$currentrepo + diff --git a/build_tools/go.sh b/build_tools/go.sh index 8d93db8..2d2f685 100755 --- a/build_tools/go.sh +++ b/build_tools/go.sh @@ -1,5 +1,7 @@ #!/bin/bash # +OLD=`pwd` +cd /build_tools . /etc/makepkg.conf clear cd clarch || exit 1 @@ -10,7 +12,11 @@ echo ${array1[@]} element_count=${#menuarray[*]} element_count=`expr $element_count - 1` -if [ ! x$1 = xmv ] + + + + +if [ x$1 = x ] then for i in `seq 0 $element_count` do @@ -20,17 +26,20 @@ then echo "Select the profile to use:" read a profile=${menuarray[$a]} - pwd - ls + else - profile=MV-local + if [ x$1 = xmv ] + then + profile=MV-local + else + profile=$1 + fi fi echo $profile -if [ x$a = x ] -then - - profile=${menuarray[3]} -fi +#if [ x$a = x ] +#then +# profile=${menuarray[3]} +#fi echo "Running ./mklarch -f -p larch/profiles/$profile" time ./mklarch -f -c /tmp/$profile -p larch/profiles/$profile @@ -44,3 +53,4 @@ else mv /home/larchroot/.larch/mylivecd.iso /tmp/testcd-2.iso fi date +cd $OLD |