diff options
22 files changed, 2347 insertions, 228 deletions
diff --git a/build_tools/archiso/archiso/Makefile b/build_tools/archiso/archiso/Makefile new file mode 100644 index 0000000..28868bf --- /dev/null +++ b/build_tools/archiso/archiso/Makefile @@ -0,0 +1,62 @@ +all: + +install: install-program install-hooks install-examples install-doc + +install-program: + # install to sbin since script only usable by root + install -D -m 755 mkarchiso $(DESTDIR)/usr/sbin/mkarchiso + # testiso can be used by anyone + install -D -m 755 testiso $(DESTDIR)/usr/bin/testiso + +install-hooks: + # hooks/install are needed by mkinitcpio + install -D -m 644 hooks/archiso $(DESTDIR)/usr/lib/initcpio/hooks/archiso + install -D -m 644 install/archiso $(DESTDIR)/usr/lib/initcpio/install/archiso + install -D -m 755 archiso_shutdown $(DESTDIR)/usr/lib/initcpio/archiso_shutdown + install -D -m 644 hooks/archiso_shutdown $(DESTDIR)/usr/lib/initcpio/hooks/archiso_shutdown + install -D -m 644 install/archiso_shutdown $(DESTDIR)/usr/lib/initcpio/install/archiso_shutdown + install -D -m 644 archiso_pxe_nbd $(DESTDIR)/usr/lib/initcpio/archiso_pxe_nbd + install -D -m 644 hooks/archiso_pxe_common $(DESTDIR)/usr/lib/initcpio/hooks/archiso_pxe_common + install -D -m 644 install/archiso_pxe_common $(DESTDIR)/usr/lib/initcpio/install/archiso_pxe_common + install -D -m 644 hooks/archiso_pxe_nbd $(DESTDIR)/usr/lib/initcpio/hooks/archiso_pxe_nbd + install -D -m 644 install/archiso_pxe_nbd $(DESTDIR)/usr/lib/initcpio/install/archiso_pxe_nbd + install -D -m 644 hooks/archiso_pxe_http $(DESTDIR)/usr/lib/initcpio/hooks/archiso_pxe_http + install -D -m 644 install/archiso_pxe_http $(DESTDIR)/usr/lib/initcpio/install/archiso_pxe_http + install -D -m 644 hooks/archiso_pxe_nfs $(DESTDIR)/usr/lib/initcpio/hooks/archiso_pxe_nfs + install -D -m 644 install/archiso_pxe_nfs $(DESTDIR)/usr/lib/initcpio/install/archiso_pxe_nfs + install -D -m 644 hooks/archiso_loop_mnt $(DESTDIR)/usr/lib/initcpio/hooks/archiso_loop_mnt + install -D -m 644 install/archiso_loop_mnt $(DESTDIR)/usr/lib/initcpio/install/archiso_loop_mnt + install -D -m 644 install/archiso_kms $(DESTDIR)/usr/lib/initcpio/install/archiso_kms + +install-examples: + # install examples + install -d -m 755 $(DESTDIR)/usr/share/archiso/ + cp -r ../configs $(DESTDIR)/usr/share/archiso/ + +install-doc: + install -d -m 755 $(DESTDIR)/usr/share/archiso/ + install -D -m 644 ../README $(DESTDIR)/usr/share/doc/archiso/README + +uninstall: + rm -f $(DESTDIR)/usr/sbin/mkarchiso + rm -f $(DESTDIR)/usr/bin/testiso + rm -f $(DESTDIR)/usr/lib/initcpio/hooks/archiso + rm -f $(DESTDIR)/usr/lib/initcpio/install/archiso + rm -f $(DESTDIR)/usr/lib/initcpio/archiso_shutdown + rm -f $(DESTDIR)/usr/lib/initcpio/hooks/archiso_shutdown + rm -f $(DESTDIR)/usr/lib/initcpio/install/archiso_shutdown + rm -f $(DESTDIR)/usr/lib/initcpio/archiso_pxe_nbd + rm -f $(DESTDIR)/usr/lib/initcpio/hooks/archiso_pxe_common + rm -f $(DESTDIR)/usr/lib/initcpio/install/archiso_pxe_common + rm -f $(DESTDIR)/usr/lib/initcpio/hooks/archiso_pxe_nbd + rm -f $(DESTDIR)/usr/lib/initcpio/install/archiso_pxe_nbd + rm -f $(DESTDIR)/usr/lib/initcpio/hooks/archiso_pxe_http + rm -f $(DESTDIR)/usr/lib/initcpio/install/archiso_pxe_http + rm -f $(DESTDIR)/usr/lib/initcpio/hooks/archiso_pxe_nfs + rm -f $(DESTDIR)/usr/lib/initcpio/install/archiso_pxe_nfs + rm -f $(DESTDIR)/usr/lib/initcpio/hooks/archiso_loop_mnt + rm -f $(DESTDIR)/usr/lib/initcpio/install/archiso_loop_mnt + rm -f $(DESTDIR)/usr/lib/initcpio/install/archiso_kms + rm -rf $(DESTDIR)/usr/share/archiso/ + +.PHONY: install install-program install-hooks install-examples install-doc uninstall diff --git a/build_tools/archiso/archiso/archiso_pxe_nbd b/build_tools/archiso/archiso/archiso_pxe_nbd new file mode 100644 index 0000000..e6b9872 --- /dev/null +++ b/build_tools/archiso/archiso/archiso_pxe_nbd @@ -0,0 +1,6 @@ +omit_kill_nbd_client() { + add_omit_pids $(< /run/archiso/nbd_client.pid) +} + +add_hook shutdown_prekillall omit_kill_nbd_client +add_hook single_prekillall omit_kill_nbd_client diff --git a/build_tools/archiso/archiso/archiso_shutdown b/build_tools/archiso/archiso/archiso_shutdown new file mode 100644 index 0000000..85cce59 --- /dev/null +++ b/build_tools/archiso/archiso/archiso_shutdown @@ -0,0 +1,40 @@ +#!/bin/ash + +# /oldroot depends on things inside /oldroot/run/archiso... +mkdir /oldrun +mount -n --move /oldroot/run /oldrun + +# Unmount all mounts now. +umount $(mount | awk '$3 ~/^\/oldroot/ {print $3}' | sort -r) + +# Remove all dm-snapshot devices. +dmsetup remove_all + +# Remove all loopback devices. +for _lup in $(grep ^/dev/loop /oldrun/archiso/used_block_devices | tac); do + if ! losetup -d ${_lup} 2> /dev/null; then + umount -d ${_lup} + fi +done + +# Unmount the space used to store *.cow. +umount /oldrun/archiso/cowspace + +# Unmount boot device if needed (no copytoram=y used) +if [[ ! -d /oldrun/archiso/copytoram ]]; then + if [[ -d /oldrun/archiso/img_dev ]]; then + umount /oldrun/archiso/img_dev + else + umount /oldrun/archiso/bootmnt + fi + if [[ -f /oldrun/archiso/nbd_client.pid ]]; then + nbd-client -d /dev/nbd0 + fi +fi + +# reboot / poweroff / halt, depending on the argument passed by init +# if something invalid is passed, we halt +case "$1" in + reboot|poweroff|halt) "$1" -f ;; + *) halt -f;; +esac diff --git a/build_tools/archiso/archiso/hooks/archiso b/build_tools/archiso/archiso/hooks/archiso new file mode 100644 index 0000000..9445498 --- /dev/null +++ b/build_tools/archiso/archiso/hooks/archiso @@ -0,0 +1,198 @@ +# args: source, newroot, mountpoint +_mnt_fs() { + local img="${1}" + local newroot="${2}" + local mnt="${3}" + local img_fullname="${img##*/}"; + local img_name="${img_fullname%%.*}" + local dm_snap_name="${dm_snap_prefix}_${img_name}" + local ro_dev ro_dev_size rw_dev + + ro_dev=$(losetup --find --show --read-only "${img}") + echo ${ro_dev} >> /run/archiso/used_block_devices + ro_dev_size=$(blockdev --getsz ${ro_dev}) + + if [[ "${cow_persistent}" == "P" ]]; then + if [[ -f "/run/archiso/cowspace/${cow_directory}/${img_name}.cow" ]]; then + msg ":: Found '/run/archiso/cowspace/${cow_directory}/${img_name}.cow', using as persistent." + else + msg ":: Creating '/run/archiso/cowspace/${cow_directory}/${img_name}.cow' as persistent." + dd of="/run/archiso/cowspace/${cow_directory}/${img_name}.cow" count=0 seek=${ro_dev_size} &> /dev/null + fi + else + if [[ -f "/run/archiso/cowspace/${cow_directory}/${img_name}.cow" ]]; then + msg ":: Found '/run/archiso/cowspace/${cow_directory}/${img_name}.cow' but non-persistent requested, removing." + rm -f "/run/archiso/cowspace/${cow_directory}/${img_name}.cow" + fi + msg ":: Creating '/run/archiso/cowspace/${cow_directory}/${img_name}.cow' as non-persistent." + dd of="/run/archiso/cowspace/${cow_directory}/${img_name}.cow" count=0 seek=${ro_dev_size} &> /dev/null + fi + + rw_dev=$(losetup --find --show "/run/archiso/cowspace/${cow_directory}/${img_name}.cow") + echo ${rw_dev} >> /run/archiso/used_block_devices + + echo "0 ${ro_dev_size} snapshot ${ro_dev} ${rw_dev} ${cow_persistent} 8" | dmsetup create ${dm_snap_name} + + _mnt_dev "/dev/mapper/${dm_snap_name}" "${newroot}${mnt}" "-w" + echo $(readlink -f /dev/mapper/${dm_snap_name}) >> /run/archiso/used_block_devices + echo "/dev/mapper/${dm_snap_name} ${mnt} auto defaults 0 0" >> ${newroot}/etc/fstab +} + +# args: /path/to/image_file, mountpoint +_mnt_sfs() { + local img="${1}" + local mnt="${2}" + local img_fullname="${img##*/}" + local sfs_dev + + if [[ "${copytoram}" == "y" ]]; then + msg -n ":: Copying squashfs image to RAM..." + if ! cp "${img}" "/run/archiso/copytoram/${img_fullname}" ; then + echo "ERROR: while copy '${img}' to '/run/archiso/copytoram/${img_fullname}'" + launch_interactive_shell + fi + img="/run/archiso/copytoram/${img_fullname}" + msg "done." + fi + sfs_dev=$(losetup --find --show --read-only "${img}") + echo ${sfs_dev} >> /run/archiso/used_block_devices + _mnt_dev "${sfs_dev}" "${mnt}" "-r" +} + +# args: device, mountpoint, flags +_mnt_dev() { + local dev="${1}" + local mnt="${2}" + local flg="${3}" + + mkdir -p "${mnt}" + + msg ":: Mounting '${dev}' to '${mnt}'" + + while ! poll_device "${dev}" 30; do + echo "ERROR: '${dev}' device did not show up after 30 seconds..." + echo " Falling back to interactive prompt" + echo " You can try to fix the problem manually, log out when you are finished" + launch_interactive_shell + done + + if mount "${flg}" "${dev}" "${mnt}"; then + msg ":: Device '${dev}' mounted successfully." + else + echo "ERROR; Failed to mount '${dev}'" + echo " Falling back to interactive prompt" + echo " You can try to fix the problem manually, log out when you are finished" + launch_interactive_shell + fi +} + +_verify_checksum() { + local _status + cd "/run/archiso/bootmnt/${archisobasedir}" + md5sum -c checksum.${arch}.md5 > /tmp/checksum.${arch}.log 2>&1 + _status=$? + cd "${OLDPWD}" + return ${_status} +} + +run_hook() { + [[ -z "${arch}" ]] && arch="$(uname -m)" + [[ -z "${cowspace_size}" ]] && cowspace_size="75%" + [[ -z "${copytoram_size}" ]] && copytoram_size="75%" + [[ -z "${archisobasedir}" ]] && archisobasedir="arch" + [[ -z "${dm_snap_prefix}" ]] && dm_snap_prefix="arch" + [[ -z "${archisodevice}" ]] && archisodevice="/dev/disk/by-label/${archisolabel}" + + if [[ -z "${aitab}" ]]; then + aitab="/run/archiso/bootmnt/${archisobasedir}/aitab" + else + aitab="/run/archiso/bootmnt/${archisobasedir}/${aitab}" + fi + + if [[ -n "${cow_label}" ]]; then + cow_device="/dev/disk/by-label/${cow_label}" + [[ -z "${cow_persistent}" ]] && cow_persistent="P" + elif [[ -n "${cow_device}" ]]; then + [[ -z "${cow_persistent}" ]] && cow_persistent="P" + else + cow_persistent="N" + fi + + [[ -z "${cow_directory}" ]] && cow_directory="persistent_${archisolabel}/${arch}" + + # set mount handler for archiso + mount_handler="archiso_mount_handler" +} + +# This function is called normally from init script, but it can be called +# as chain from other mount handlers. +# args: /path/to/newroot +archiso_mount_handler() { + local newroot="${1}" + + if ! mountpoint -q "/run/archiso/bootmnt"; then + _mnt_dev "${archisodevice}" "/run/archiso/bootmnt" "-r" + if [[ "${copytoram}" != "y" ]]; then + echo $(readlink -f ${archisodevice}) >> /run/archiso/used_block_devices + fi + fi + + + if [[ ! -f "${aitab}" ]]; then + echo "ERROR: '${aitab}' file does not exist." + echo " Falling back to interactive prompt" + echo " You can try to fix the problem manually, log out when you are finished" + launch_interactive_shell + fi + + if [[ "${checksum}" == "y" ]]; then + if [[ -f "/run/archiso/bootmnt/${archisobasedir}/checksum.${arch}.md5" ]]; then + msg -n ":: Self-test requested, please wait..." + if _verify_checksum; then + msg "done. Checksum is OK, continue booting." + else + echo "ERROR: one or more files are corrupted" + echo "see /tmp/checksum.${arch}.log for details" + launch_interactive_shell + fi + else + echo "ERROR: checksum=y option specified but checksum.${arch}.md5 not found" + launch_interactive_shell + fi + fi + + if [[ "${copytoram}" == "y" ]]; then + msg ":: Mounting /run/archiso/copytoram (tmpfs) filesystem, size=${copytoram_size}" + mkdir -p /run/archiso/copytoram + mount -t tmpfs -o "size=${copytoram_size}",mode=0755 copytoram /run/archiso/copytoram + fi + + if [[ -n "${cow_device}" ]]; then + _mnt_dev "${cow_device}" "/run/archiso/cowspace" "-r" + echo $(readlink -f ${cow_device}) >> /run/archiso/used_block_devices + mount -o remount,rw "/run/archiso/cowspace" + else + msg ":: Mounting /run/archiso/cowspace (tmpfs) filesystem, size=${cowspace_size}..." + mkdir -p /run/archiso/cowspace + mount -t tmpfs -o "size=${cowspace_size}",mode=0755 cowspace /run/archiso/cowspace + fi + mkdir -p "/run/archiso/cowspace/${cow_directory}" + + local aitab_img aitab_mnt aitab_arch aitab_sfs_comp aitab_fs_type aitab_fs_size + while read aitab_img aitab_mnt aitab_arch aitab_sfs_comp aitab_fs_type aitab_fs_size; do + [[ "${aitab_img#\#}" != "${aitab_img}" ]] && continue + [[ "${aitab_arch}" != "any" && "${aitab_arch}" != "${arch}" ]] && continue + if [[ "${aitab_fs_type}" != "none" ]]; then + _mnt_sfs "/run/archiso/bootmnt/${archisobasedir}/${aitab_arch}/${aitab_img}.fs.sfs" "/run/archiso/sfs/${aitab_img}" + _mnt_fs "/run/archiso/sfs/${aitab_img}/${aitab_img}.fs" "${newroot}" "${aitab_mnt}" + else + _mnt_sfs "/run/archiso/bootmnt/${archisobasedir}/${aitab_arch}/${aitab_img}.sfs" "${newroot}${aitab_mnt}" + fi + done < "${aitab}" + + if [[ "${copytoram}" == "y" ]]; then + umount /run/archiso/bootmnt + fi +} + +# vim:ft=sh:ts=4:sw=4:et: diff --git a/build_tools/archiso/archiso/hooks/archiso_loop_mnt b/build_tools/archiso/archiso/hooks/archiso_loop_mnt new file mode 100644 index 0000000..a02cfac --- /dev/null +++ b/build_tools/archiso/archiso/hooks/archiso_loop_mnt @@ -0,0 +1,32 @@ +# vim: set ft=sh: + +run_hook () { + [[ -n "${img_label}" ]] && img_dev="/dev/disk/by-label/${img_label}" + if [[ -n "${img_dev}" && -n "${img_loop}" ]]; then + mount_handler="archiso_loop_mount_handler" + fi +} + +archiso_loop_mount_handler () { + newroot="${1}" + + local _dev_loop + + msg ":: Setup a loop device from ${img_loop} located at device ${img_dev}" + _mnt_dev "${img_dev}" "/run/archiso/img_dev" "-r" + if [[ "${copytoram}" != "y" ]]; then + echo $(readlink -f ${img_dev}) >> /run/archiso/used_block_devices + fi + + if ! _dev_loop=$(losetup --find --show --read-only "/run/archiso/img_dev/${img_loop}"); then + echo "ERROR: Setting loopback device for file '/run/archiso/img_dev/${img_loop}'" + launch_interactive_shell + fi + + archiso_mount_handler ${newroot} + + if [[ "${copytoram}" == "y" ]]; then + losetup -d ${_dev_loop} + umount /run/archiso/img_dev + fi +} diff --git a/build_tools/archiso/archiso/hooks/archiso_pxe_common b/build_tools/archiso/archiso/hooks/archiso_pxe_common new file mode 100644 index 0000000..d8ac709 --- /dev/null +++ b/build_tools/archiso/archiso/hooks/archiso_pxe_common @@ -0,0 +1,50 @@ +# vim: set ft=sh: + +run_hook () { + local i net_mac bootif_mac bootif_dev + # These variables will be parsed from /tmp/net-*.conf generated by ipconfig + local DEVICE + local IPV4ADDR IPV4BROADCAST IPV4NETMASK IPV4GATEWAY IPV4DNS0 IPV4DNS1 + local HOSTNAME DNSDOMAIN NISDOMAIN ROOTSERVER ROOTPATH + local filename + # /tmp/net-*.conf + + if [[ -n "${ip}" ]]; then + if [[ -n "${BOOTIF}" ]]; then + bootif_mac=${BOOTIF#01-} + bootif_mac=${bootif_mac//-/:} + for i in /sys/class/net/*/address; do + read net_mac < ${i} + if [[ "${bootif_mac}" == "${net_mac}" ]]; then + bootif_dev=${i#/sys/class/net/} + bootif_dev=${bootif_dev%/address} + break + fi + done + ip="${ip}::${bootif_dev}" + fi + + # setup network and save some values + ipconfig "ip=${ip}" + + . /tmp/net-*.conf + + pxeserver=${ROOTSERVER} + + # setup DNS resolver + if [[ "${IPV4DNS0}" != "0.0.0.0" ]]; then + echo "nameserver ${IPV4DNS0}" > /etc/resolv.conf + fi + if [[ "${IPV4DNS1}" != "0.0.0.0" ]]; then + echo "nameserver ${IPV4DNS1}" >> /etc/resolv.conf + fi + fi +} + +run_latehook () { + [[ -z "${copy_resolvconf}" ]] && copy_resolvconf="y" + + if [[ "${copy_resolvconf}" != "n" && -f /etc/resolv.conf ]]; then + cp /etc/resolv.conf /new_root/etc/resolv.conf + fi +} diff --git a/build_tools/archiso/archiso/hooks/archiso_pxe_http b/build_tools/archiso/archiso/hooks/archiso_pxe_http new file mode 100644 index 0000000..ae56210 --- /dev/null +++ b/build_tools/archiso/archiso/hooks/archiso_pxe_http @@ -0,0 +1,61 @@ +# vim: set ft=sh: + +run_hook() { + if [[ -n "${ip}" && -n "${archiso_http_srv}" ]]; then + + archiso_http_srv=$(eval echo ${archiso_http_srv}) + [[ -z "${archiso_http_spc}" ]] && archiso_http_spc="75%" + + mount_handler="archiso_pxe_http_mount_handler" + fi +} + +# Fetch a file with CURL +# +# $1 URL +# $2 Destination directory inside httpspace/${archisobasedir} +_curl_get() { + local _url="${1}" + local _dst="${2}" + + msg ":: Downloading '${_url}'" + if ! curl -f -o "/run/archiso/httpspace/${archisobasedir}${_dst}/${_url##*/}" --create-dirs "${_url}"; then + echo "ERROR: Downloading '${_url}'" + echo " Falling back to interactive prompt" + echo " You can try to fix the problem manually, log out when you are finished" + launch_interactive_shell + fi +} + +archiso_pxe_http_mount_handler () { + newroot="${1}" + + msg ":: Mounting /run/archiso/httpspace (tmpfs) filesystem, size='${archiso_http_spc}'" + mkdir -p "/run/archiso/httpspace" + mount -t tmpfs -o size="${archiso_http_spc}",mode=0755 httpspace "/run/archiso/httpspace" + + local _aitab_url="${archiso_http_srv}${aitab#/run/archiso/bootmnt/}" + local _aitab_file="/run/archiso/httpspace/${aitab#/run/archiso/bootmnt/}" + + _curl_get "${_aitab_url}" "/" + + local aitab_img aitab_mnt aitab_arch aitab_sfs_comp aitab_fs_type aitab_fs_size + while read aitab_img aitab_mnt aitab_arch aitab_sfs_comp aitab_fs_type aitab_fs_size; do + [[ "${aitab_img#\#}" != "${aitab_img}" ]] && continue + [[ "${aitab_arch}" != "any" && "${aitab_arch}" != "${arch}" ]] && continue + if [[ "${aitab_fs_type}" != "none" ]]; then + _curl_get "${archiso_http_srv}${archisobasedir}/${aitab_arch}/${aitab_img}.fs.sfs" "/${aitab_arch}" + else + _curl_get "${archiso_http_srv}${archisobasedir}/${aitab_arch}/${aitab_img}.sfs" "/${aitab_arch}" + fi + done < "${_aitab_file}" + + if [[ "${checksum}" == "y" ]]; then + _curl_get "${archiso_http_srv}${archisobasedir}/checksum.${arch}.md5" "/" + fi + + mkdir -p "/run/archiso/bootmnt" + mount -o bind /run/archiso/httpspace /run/archiso/bootmnt + + archiso_mount_handler ${newroot} +} diff --git a/build_tools/archiso/archiso/hooks/archiso_pxe_nbd b/build_tools/archiso/archiso/hooks/archiso_pxe_nbd new file mode 100644 index 0000000..6147d06 --- /dev/null +++ b/build_tools/archiso/archiso/hooks/archiso_pxe_nbd @@ -0,0 +1,45 @@ +# vim: set ft=sh: + +run_hook() { + if [[ -n "${ip}" && -n "${archiso_nbd_srv}" ]]; then + + archiso_nbd_srv=$(eval echo ${archiso_nbd_srv}) + [[ -z "${archiso_nbd_name}" ]] && archiso_nbd_name="archiso" + + mount_handler="archiso_pxe_nbd_mount_handler" + fi +} + +archiso_pxe_nbd_mount_handler () { + newroot="${1}" + + # Module autoloading like with loop devices does not work, doing manually... + modprobe nbd 2> /dev/null + msg ":: Waiting for boot device..." + while ! poll_device /dev/nbd0 30; do + echo "ERROR: boot device didn't show up after 30 seconds..." + echo " Falling back to interactive prompt" + echo " You can try to fix the problem manually, log out when you are finished" + launch_interactive_shell + done + + msg ":: Setup NBD from ${archiso_nbd_srv} at /dev/nbd0" + if [[ "${copytoram}" = "y" ]]; then + nbd-client ${archiso_nbd_srv} -N ${archiso_nbd_name} /dev/nbd0 + else + nbd-client ${archiso_nbd_srv} -N ${archiso_nbd_name} /dev/nbd0 -persist + fi + + archisodevice=/dev/nbd0 + + archiso_mount_handler ${newroot} + + if [[ "${copytoram}" = "y" ]]; then + msg ":: Disconnect NBD from ${archiso_nbd_srv} at /dev/nbd0" + nbd-client -d /dev/nbd0 + else + mkdir -p /run/archiso + pidof nbd-client > /run/archiso/nbd_client.pid + cp /archiso_pxe_nbd ${newroot}/etc/rc.d/functions.d/ + fi +} diff --git a/build_tools/archiso/archiso/hooks/archiso_pxe_nfs b/build_tools/archiso/archiso/hooks/archiso_pxe_nfs new file mode 100644 index 0000000..a053672 --- /dev/null +++ b/build_tools/archiso/archiso/hooks/archiso_pxe_nfs @@ -0,0 +1,26 @@ +# vim: set ft=sh: + +run_hook() { + if [[ -n "${ip}" && -n "${archiso_nfs_srv}" ]]; then + + archiso_nfs_srv=$(eval echo ${archiso_nfs_srv}) + [[ -n "${archiso_nfs_opt}" ]] && archiso_nfs_opt="-o ${archiso_nfs_opt}" + + mount_handler="archiso_nfs_mount_handler" + fi +} + +archiso_nfs_mount_handler() { + newroot="${1}" + mkdir -p "/run/archiso/bootmnt" + msg ":: Mounting '${archiso_nfs_srv}'" + # Do not put "${archiso_nfs_opt}" nfsmount fails! + if ! nfsmount ${archiso_nfs_opt} "${archiso_nfs_srv}" "/run/archiso/bootmnt"; then + echo "ERROR: Mounting '${archiso_nfs_srv}'" + echo " Falling back to interactive prompt" + echo " You can try to fix the problem manually, log out when you are finished" + launch_interactive_shell + fi + + archiso_mount_handler ${newroot} +} diff --git a/build_tools/archiso/archiso/hooks/archiso_shutdown b/build_tools/archiso/archiso/hooks/archiso_shutdown new file mode 100644 index 0000000..24e4103 --- /dev/null +++ b/build_tools/archiso/archiso/hooks/archiso_shutdown @@ -0,0 +1,23 @@ +run_hook () +{ + msg -n ":: Creating shutdown ramfs..." + + for _dir in bin lib sbin usr/bin usr/lib usr/sbin; do + mkdir -p /run/initramfs/${_dir} + done + cp /bin/busybox /run/initramfs/bin/ + cp /lib/ld-* /run/initramfs/lib/ + cp /lib/lib* /run/initramfs/lib/ + cp /usr/lib/lib* /run/initramfs/usr/lib/ + cp /bin/mount /run/initramfs/bin/ + cp /sbin/dmsetup /run/initramfs/sbin/ + cp /sbin/losetup /run/initramfs/sbin/ + if [[ -x /bin/nbd-client ]]; then + cp /bin/nbd-client /run/initramfs/bin/ + fi + + chroot /run/initramfs /bin/busybox --install + cp /shutdown /run/initramfs/ + + msg "done." +} diff --git a/build_tools/archiso/archiso/install/archiso b/build_tools/archiso/archiso/install/archiso new file mode 100644 index 0000000..8893667 --- /dev/null +++ b/build_tools/archiso/archiso/install/archiso @@ -0,0 +1,22 @@ +#!/bin/bash + +build() { + add_module "cdrom" + add_module "loop" + add_module "dm-snapshot" + + add_runscript + + add_binary /usr/lib/udev/cdrom_id + add_binary blockdev + add_binary dmsetup + add_binary losetup + add_binary mountpoint + + add_file /usr/lib/udev/rules.d/60-cdrom_id.rules + add_file /usr/lib/udev/rules.d/10-dm.rules + add_file /usr/lib/udev/rules.d/95-dm-notify.rules + add_file /usr/lib/initcpio/udev/11-dm-initramfs.rules /usr/lib/udev/rules.d/11-dm-initramfs.rules +} + +# vim: set ft=sh ts=4 sw=4 et: diff --git a/build_tools/archiso/archiso/install/archiso_kms b/build_tools/archiso/archiso/install/archiso_kms new file mode 100644 index 0000000..3ff31f3 --- /dev/null +++ b/build_tools/archiso/archiso/install/archiso_kms @@ -0,0 +1,26 @@ +#!/bin/bash + +build() { + add_module "radeon" + add_module "nouveau" + add_module "i915" + add_module "via-agp" + add_module "sis-agp" + add_module "intel-agp" + + if [[ $(uname -m) == i686 ]]; then + add_module "amd64-agp" + add_module "ati-agp" + add_module "sworks-agp" + add_module "ali-agp" + add_module "amd-k7-agp" + add_module "nvidia-agp" + add_module "efficeon-agp" + fi +} + +help() { + cat << HELPEOF +Adds all common KMS drivers to the initramfs image. +HELPEOF +} diff --git a/build_tools/archiso/archiso/install/archiso_loop_mnt b/build_tools/archiso/archiso/install/archiso_loop_mnt new file mode 100644 index 0000000..59f1d94 --- /dev/null +++ b/build_tools/archiso/archiso/install/archiso_loop_mnt @@ -0,0 +1,13 @@ +#!/bin/bash + +build() { + add_runscript +} + +help() { +cat<<HELPEOF + This hook loads the necessary modules for boot via loop device. +HELPEOF +} + +# vim: set ft=sh ts=4 sw=4 et: diff --git a/build_tools/archiso/archiso/install/archiso_pxe_common b/build_tools/archiso/archiso/install/archiso_pxe_common new file mode 100644 index 0000000..77d0bb6 --- /dev/null +++ b/build_tools/archiso/archiso/install/archiso_pxe_common @@ -0,0 +1,26 @@ +#!/bin/bash + +build() { + add_checked_modules -f "(irda|phy|wimax|wireless|ppp_|plip|pppoe)" "/drivers/net/" + + add_runscript + + add_binary /usr/lib/initcpio/ipconfig /bin/ipconfig + + # Add hosts support files+dns + add_symlink /lib/libnss_files.so.2 $(readlink /lib/libnss_files.so.2) + add_binary $(readlink -f /lib/libnss_files.so.2) + add_symlink /lib/libnss_dns.so.2 $(readlink /lib/libnss_dns.so.2) + add_binary $(readlink -f /lib/libnss_dns.so.2) + + add_dir /etc + echo "hosts: files dns" > $BUILDROOT/etc/nsswitch.conf +} + +help() { +cat<<HELPEOF + This hook loads the necessary modules for boot via PXE. +HELPEOF +} + +# vim: set ft=sh ts=4 sw=4 et: diff --git a/build_tools/archiso/archiso/install/archiso_pxe_http b/build_tools/archiso/archiso/install/archiso_pxe_http new file mode 100644 index 0000000..4e02f98 --- /dev/null +++ b/build_tools/archiso/archiso/install/archiso_pxe_http @@ -0,0 +1,15 @@ +#!/bin/bash + +build() { + add_runscript + + add_binary curl +} + +help() { +cat<<HELPEOF + This hook loads the necessary modules for boot via PXE and HTTP. +HELPEOF +} + +# vim: set ft=sh ts=4 sw=4 et: diff --git a/build_tools/archiso/archiso/install/archiso_pxe_nbd b/build_tools/archiso/archiso/install/archiso_pxe_nbd new file mode 100644 index 0000000..8d9552a --- /dev/null +++ b/build_tools/archiso/archiso/install/archiso_pxe_nbd @@ -0,0 +1,19 @@ +#!/bin/bash + +build() { + add_module "nbd" + + add_runscript + + add_binary nbd-client + + add_file /usr/lib/initcpio/archiso_pxe_nbd /archiso_pxe_nbd +} + +help() { +cat<<HELPEOF + This hook loads the necessary modules for boot via PXE and NBD. +HELPEOF +} + +# vim: set ft=sh ts=4 sw=4 et: diff --git a/build_tools/archiso/archiso/install/archiso_pxe_nfs b/build_tools/archiso/archiso/install/archiso_pxe_nfs new file mode 100644 index 0000000..f8226e5 --- /dev/null +++ b/build_tools/archiso/archiso/install/archiso_pxe_nfs @@ -0,0 +1,17 @@ +#!/bin/bash + +build() { + add_module "nfs" + + add_runscript + + add_binary /usr/lib/initcpio/nfsmount /bin/nfsmount +} + +help() { + cat <<HELPEOF + This hook loads the necessary modules for boot via PXE and NFS. +HELPEOF +} + +# vim: set ft=sh ts=4 sw=4 et: diff --git a/build_tools/archiso/archiso/install/archiso_shutdown b/build_tools/archiso/archiso/install/archiso_shutdown new file mode 100644 index 0000000..962be95 --- /dev/null +++ b/build_tools/archiso/archiso/install/archiso_shutdown @@ -0,0 +1,18 @@ +#!/bin/bash + +build() { + add_runscript + + add_file /usr/lib/initcpio/archiso_shutdown /shutdown +} + +help() { + cat <<HELPEOF +This hook will create a shutdown initramfs in /run/initramfs +that we can pivot to on shutdown in order to unmount / and +and others mount points, dm-snapshot devices and loopback devices. +Mostly usefull for dm-snapshot persistent. +HELPEOF +} + +# vim: set ft=sh ts=4 sw=4 et: diff --git a/build_tools/archiso/archiso/mkarchiso b/build_tools/archiso/archiso/mkarchiso new file mode 100755 index 0000000..1b2a024 --- /dev/null +++ b/build_tools/archiso/archiso/mkarchiso @@ -0,0 +1,578 @@ +#!/bin/bash + +set -e -u + +export LANG=C + +app_name=${0##*/} +arch=$(uname -m) +pkg_list="" +run_cmd="" +quiet="y" +pacman_conf="/etc/pacman.conf" +export iso_label="ARCH_$(date +%Y%m)" +iso_publisher="Arch Linux <http://www.archlinux.org>" +iso_application="Arch Linux Live/Rescue CD" +install_dir="arch" +work_dir="work" +out_dir="out" + +# Show an INFO message +# $1: message string +_msg_info() { + local _msg="${1}" + echo "[mkarchiso] INFO: ${_msg}" +} + +# Show an ERROR message then exit with status +# $1: message string +# $2: exit code number (with 0 does not exit) +_msg_error() { + local _msg="${1}" + local _error=${2} + echo + echo "[mkarchiso] ERROR: ${_msg}" + echo + if [[ ${_error} -gt 0 ]]; then + exit ${_error} + fi +} + +# Show space usage similar to df, but better formatted. +# $1: mount-point or mounted device. +_show_space_usage () { + local _where="${1}" + local _fs _total _used _avail _pct_u=0 _mnt + read _fs _total _used _avail _pct_u _mnt < <(df -m "${_where}" | tail -1) &> /dev/null + _msg_info "Total: ${_total} MiB (100%) | Used: ${_used} MiB (${_pct_u}) | Avail: ${_avail} MiB ($((100 - ${_pct_u%\%}))%)" +} + +_chroot_mount () { + mount -t devtmpfs dev "${work_dir}/root-image/dev" + mount -t devpts devpts "${work_dir}/root-image/dev/pts" + mount -t tmpfs devshm "${work_dir}/root-image/dev/shm" + mount -t proc proc "${work_dir}/root-image/proc" + mount -t tmpfs run "${work_dir}/root-image/run" + mount -t sysfs sys "${work_dir}/root-image/sys" + mount -t tmpfs tmp "${work_dir}/root-image/tmp" + + trap '_chroot_umount' EXIT HUP INT TERM +} + +_chroot_umount () { + umount "${work_dir}/root-image/tmp" + umount "${work_dir}/root-image/sys" + umount "${work_dir}/root-image/run" + umount "${work_dir}/root-image/proc" + umount "${work_dir}/root-image/dev/shm" + umount "${work_dir}/root-image/dev/pts" + umount "${work_dir}/root-image/dev" + + trap - EXIT HUP INT TERM +} + +_chroot_init() { + if [[ ! -d ${work_dir}/root-image/var/cache/pacman ]]; then + mkdir -p ${work_dir}/root-image/{dev,proc,run,sys,tmp,var/lib/pacman} + _pacman "base" + _pacman "syslinux" + fi +} + +_chroot_run() { + _chroot_mount + eval chroot ${work_dir}/root-image "${run_cmd}" + _chroot_umount +} + +# Mount a filesystem (trap signals in case of error for unmounting it +# $1: source image +# $2: mount-point +_mount_fs() { + local _src="${1}" + local _dst="${2}" + trap "_umount_fs ${_src}" EXIT HUP INT TERM + mkdir -p "${_dst}" + _msg_info "Mounting '${_src}' on '${_dst}'" + mount "${_src}" "${_dst}" + _show_space_usage "${_dst}" +} + +# Unmount a filesystem (and untrap signals) +# $1: mount-point or device/image +_umount_fs() { + local _dst="${1}" + _show_space_usage "${_dst}" + _msg_info "Unmounting '${_dst}'" + umount "${_dst}" + rmdir "${_dst}" + trap - EXIT HUP INT TERM +} + +# Compare if a file/directory (source) is newer than other file (target) +# $1: source file/directory +# $2: target file +# return: 0 if target does not exists or if target is older than source. +# 1 if target is newer than source +_is_directory_changed() { + local _src="${1}" + local _dst="${2}" + + if [ -e "${_dst}" ]; then + if [[ $(find ${_src} -newer ${_dst} | wc -l) -gt 0 ]]; then + _msg_info "Target '${_dst}' is older than '${_src}', updating." + rm -f "${_dst}" + return 0 + else + _msg_info "Target '${_dst}' is up to date with '${_src}', skipping." + return 1 + fi + else + _msg_info "Target '${_dst}' does not exist, making it from '${_src}'" + return 0 + fi +} + +# Show help usage, with an exit status. +# $1: exit status number. +_usage () +{ + echo "usage ${app_name} [options] command <command options>" + echo " general options:" + echo " -p PACKAGE(S) Package(s) to install, can be used multiple times" + echo " -r <command> Run <command> inside root-image" + echo " -C <file> Config file for pacman." + echo " Default: '${pacman_conf}'" + echo " -L <label> Set a label for the disk" + echo " Default: '${iso_label}'" + echo " -P <publisher> Set a publisher for the disk" + echo " Default: '${iso_publisher}'" + echo " -A <application> Set an application name for the disk" + echo " Default: '${iso_application}'" + echo " -D <install_dir> Set an install_dir. All files will by located here." + echo " Default: '${install_dir}'" + echo " NOTE: Max 8 characters, use only [a-z0-9]" + echo " -w <work_dir> Set the working directory" + echo " Default: '${work_dir}'" + echo " -o <out_dir> Set the output directory" + echo " Default: '${out_dir}'" + echo " -v Enable verbose output" + echo " -h This message" + echo " commands:" + echo " init" + echo " Make base layout and install base group" + echo " install" + echo " Install all specified packages (-p)" + echo " run" + echo " run command specified by -r" + echo " prepare" + echo " build all images" + echo " checksum" + echo " make a checksum.md5 for self-test" + echo " pkglist" + echo " make a pkglist.txt of packages installed on root-image" + echo " iso <image name>" + echo " build an iso image from the working dir" + exit ${1} +} + +# Shows configuration according to command mode. +# $1: init | install | run | prepare | checksum | iso +_show_config () { + local _mode="$1" + echo + _msg_info "Configuration settings" + _msg_info " Command: ${command_name}" + _msg_info " Architecture: ${arch}" + _msg_info " Working directory: ${work_dir}" + _msg_info " Installation directory: ${install_dir}" + case "${_mode}" in + init) + _msg_info " Pacman config file: ${pacman_conf}" + ;; + install) + _msg_info " Pacman config file: ${pacman_conf}" + _msg_info " Packages: ${pkg_list}" + ;; + run) + _msg_info " Run command: ${run_cmd}" + ;; + prepare) + ;; + checksum) + ;; + pkglist) + ;; + iso) + _msg_info " Image name: ${img_name}" + _msg_info " Disk label: ${iso_label}" + _msg_info " Disk publisher: ${iso_publisher}" + _msg_info " Disk application: ${iso_application}" + ;; + esac + echo +} + +# Install desired packages to root-image +_pacman () +{ + _msg_info "Installing packages to '${work_dir}/root-image/'..." + + _chroot_mount + + if [[ "${quiet}" = "y" ]]; then + pacman -Sy -r "${work_dir}/root-image" --config "${pacman_conf}" --needed --noconfirm $* &> /dev/null + else + pacman -Sy -r "${work_dir}/root-image" --config "${pacman_conf}" --needed --noconfirm $* + fi + + _chroot_umount + + _msg_info "Packages installed successfully!" +} + +# Cleanup root-image +_cleanup () { + _msg_info "Cleaning up what we can on root-image..." + + # remove the initcpio images that were generated for the host system + if [[ -d "${work_dir}/root-image/boot" ]]; then + find "${work_dir}/root-image/boot" -name '*.img' -delete + fi + # Delete pacman database sync cache files (*.tar.gz) + if [[ -d "${work_dir}/root-image/var/lib/pacman" ]]; then + find "${work_dir}/root-image/var/lib/pacman" -maxdepth 1 -type f -delete + fi + # Delete pacman database sync cache + if [[ -d "${work_dir}/root-image/var/lib/pacman/sync" ]]; then + find "${work_dir}/root-image/var/lib/pacman/sync" -delete + fi + # Delete pacman package cache + if [[ -d "${work_dir}/root-image/var/cache/pacman/pkg" ]]; then + find "${work_dir}/root-image/var/cache/pacman/pkg" -type f -delete + fi + # Delete all log files, keeps empty dirs. + if [[ -d "${work_dir}/root-image/var/log" ]]; then + find "${work_dir}/root-image/var/log" -type f -delete + fi + # Delete all temporary files and dirs + if [[ -d "${work_dir}/root-image/var/tmp" ]]; then + find "${work_dir}/root-image/var/tmp" -mindepth 1 -delete + fi + # Delete all temporary files and dirs + if [[ -d "${work_dir}/root-image/tmp" ]]; then + find "${work_dir}/root-image/tmp" -mindepth 1 -delete + fi + # Delete package pacman related files. + find "${work_dir}" -name "*.pacnew" -name "*.pacsave" -name "*.pacorig" -delete + # Create etc/mtab if not is a symlink. + if [[ ! -L "${work_dir}/root-image/etc/mtab" ]]; then + ln -sf "/proc/self/mounts" "${work_dir}/root-image/etc/mtab" + fi + _msg_info "Done!" +} + +# Makes a SquashFS filesystem image of file/directory passes as argument with desired compression. +# $1: Source file/directory +# $2: SquashFS compression type (gzip | lzo | xz) +_mksfs () { + local _src="${1}" + local _sfs_comp="${2}" + + if [[ ! -e "${work_dir}/${_src}" ]]; then + _msg_error "The path '${work_dir}/${_src}' does not exist" 1 + fi + + local _sfs_img="${work_dir}/${_src}.sfs" + + _msg_info "Creating SquashFS image for '${work_dir}/${_src}', This may take some time..." + local _seconds=${SECONDS} + if [[ "${quiet}" = "y" ]]; then + mksquashfs "${work_dir}/${_src}" "${_sfs_img}" -noappend -comp "${_sfs_comp}" -no-progress &> /dev/null + else + mksquashfs "${work_dir}/${_src}" "${_sfs_img}" -noappend -comp "${_sfs_comp}" -no-progress + fi + _seconds=$((SECONDS - _seconds)) + printf "[mkarchiso] INFO: Image creation done in %02d:%02d minutes\n" $((_seconds / 60)) $((_seconds % 60)) +} + +# Makes a filesystem from a source directory. +# $1: Source directory +# $2: Target filesystem type (ext4 | ext3 | ext2 | xfs) +# $3: Size of target filesystem. Can be an absolute value in MiB, or relative value of desired free space (1% - 99%) +_mkfs () { + local _src="${1}" + local _fs_type="${2}" + local _fs_size="${3}" + + local _fs_src="${work_dir}/${_src}" + local _fs_img="${work_dir}/${_src}.fs" + + if [[ ! -e "${_fs_src}" ]]; then + _msg_error "The path '${_fs_src}' does not exist" 1 + fi + + local _spc_used + _spc_used=$(du -sxm "${_fs_src}" | awk '{print $1}') + + # Caculate FS size with desired % of free space, adds 10% overhead to used space. + if [[ ${_fs_size} != ${_fs_size%\%} ]]; then + if [[ ${_fs_size%\%} -le 0 || ${_fs_size%\%} -ge 100 ]]; then + _msg_error "Invalid percentage of free space specified '${_fs_size}' on '${_src}', should be 0% < x < 100%" 1 + fi + _fs_size=$((_spc_used * 110 / (100 - ${_fs_size%\%}))) + else + local _spc_used_over=$((_spc_used * 11 / 10)) + if [[ ${_fs_size} -lt ${_spc_used_over} ]]; then + _msg_error "Filesystem size specified '${_fs_size}' MiB for '${_src}' is too small, must be at least '${_spc_used_over}' MiB" 1 + fi + fi + + _msg_info "Creating ${_fs_type} image of ${_fs_size} MiB..." + rm -f "${_fs_img}" + dd of="${_fs_img}" count=0 bs=1M seek=${_fs_size} &> /dev/null + local _qflag="" + if [[ ${quiet} == "y" ]]; then + _qflag="-q" + fi + case "${_fs_type}" in + ext4) + mkfs.ext4 ${_qflag} -O ^has_journal -m 0 -F "${_fs_img}" + tune2fs -c 0 -i 0 "${_fs_img}" &> /dev/null + ;; + ext3) + mkfs.ext3 ${_qflag} -m 0 -F "${_fs_img}" + tune2fs -c 0 -i 0 "${_fs_img}" &> /dev/null + ;; + ext2) + mkfs.ext2 ${_qflag} -m 0 -F "${_fs_img}" + tune2fs -c 0 -i 0 "${_fs_img}" &> /dev/null + ;; + xfs) + mkfs.xfs ${_qflag} "${_fs_img}" + ;; + *) + _msg_error "Invalid filesystem: ${_fs_type}" 1 + ;; + esac + _msg_info "Done!" + _mount_fs "${_fs_img}" "${work_dir}/mnt/${_src}" + _msg_info "Copying '${_fs_src}/' to '${work_dir}/mnt/${_src}/'..." + rsync -aH "${_fs_src}/" "${work_dir}/mnt/${_src}/" + _msg_info "Done!" + _umount_fs "${work_dir}/mnt/${_src}" +} + +command_checksum () { + _show_config checksum + + local _chk_arch + + for _chk_arch in i686 x86_64; do + if _is_directory_changed "${work_dir}/iso/${install_dir}" "${work_dir}/iso/${install_dir}/checksum.${_chk_arch}.md5"; then + _msg_info "Creating checksum file for self-test (${_chk_arch})..." + cd "${work_dir}/iso/${install_dir}" + if [[ -d "${_chk_arch}" ]]; then + md5sum aitab > checksum.${_chk_arch}.md5 + find ${_chk_arch} -type f -print0 | xargs -0 md5sum >> checksum.${_chk_arch}.md5 + if [[ -d "any" ]]; then + find any -type f -print0 | xargs -0 md5sum >> checksum.${_chk_arch}.md5 + fi + fi + cd ${OLDPWD} + _msg_info "Done!" + fi + done +} + +command_pkglist () { + _show_config pkglist + + if _is_directory_changed "${work_dir}/root-image/var/lib/pacman/local" "${work_dir}/iso/${install_dir}/pkglist.${arch}.txt"; then + _msg_info "Creating a list of installed packages on live-enviroment..." + pacman -Sl -r "${work_dir}/root-image" --config "${pacman_conf}" | \ + awk '/\[installed\]$/ {print $1 "/" $2 "-" $3}' > \ + "${work_dir}/iso/${install_dir}/pkglist.${arch}.txt" + _msg_info "Done!" + fi + +} + +# Create an ISO9660 filesystem from "iso" directory. +command_iso () { + local _iso_efi_boot_args="" + + if [[ ! -f "${work_dir}/iso/isolinux/isolinux.bin" ]]; then + _msg_error "The file '${work_dir}/iso/isolinux/isolinux.bin' does not exist." 1 + fi + if [[ ! -f "${work_dir}/iso/isolinux/isohdpfx.bin" ]]; then + _msg_error "The file '${work_dir}/iso/isolinux/isohdpfx.bin' does not exist." 1 + fi + + # If exists, add an EFI "El Torito" boot image (FAT filesystem) to ISO-9660 image. + if [[ -f "${work_dir}/iso/EFI/archiso/efiboot.img" ]]; then + _iso_efi_boot_args="--efi-boot EFI/archiso/efiboot.img" + fi + + _show_config iso + + if _is_directory_changed "${work_dir}/iso" "${out_dir}/${img_name}"; then + mkdir -p ${out_dir} + _msg_info "Creating ISO image..." + local _qflag="" + if [[ ${quiet} == "y" ]]; then + _qflag="-quiet" + fi + xorriso -as mkisofs ${_qflag} \ + -iso-level 3 \ + -full-iso9660-filenames \ + -volid "${iso_label}" \ + -appid "${iso_application}" \ + -publisher "${iso_publisher}" \ + -preparer "prepared by mkarchiso" \ + -eltorito-boot isolinux/isolinux.bin \ + -eltorito-catalog isolinux/boot.cat \ + -no-emul-boot -boot-load-size 4 -boot-info-table \ + ${_iso_efi_boot_args} \ + -isohybrid-mbr ${work_dir}/iso/isolinux/isohdpfx.bin \ + -output "${out_dir}/${img_name}" \ + "${work_dir}/iso/" + _msg_info "Done! | $(ls -sh ${out_dir}/${img_name})" + fi +} + +# Parse aitab and create each filesystem specified on that, and push it in "iso" directory. +command_prepare () { + if [[ ! -f "${work_dir}/iso/${install_dir}/aitab" ]]; then + _msg_error "The file '${work_dir}/iso/${install_dir}/aitab' does not exist." 1 + fi + _show_config prepare + + _cleanup + local _aitab_img _aitab_mnt _aitab_arch _aitab_sfs_comp _aitab_fs_type _aitab_fs_size + while read _aitab_img _aitab_mnt _aitab_arch _aitab_sfs_comp _aitab_fs_type _aitab_fs_size ; do + if [[ ${_aitab_img} =~ ^# ]]; then + continue + fi + local _src="${work_dir}/${_aitab_img}" + local _dst="${work_dir}/iso/${install_dir}/${_aitab_arch}" + mkdir -p "${_dst}" + if [[ ${_aitab_fs_type} != "none" ]]; then + if _is_directory_changed "${_src}" "${_dst}/${_aitab_img}.fs.sfs"; then + _mkfs ${_aitab_img} ${_aitab_fs_type} ${_aitab_fs_size} + _mksfs ${_aitab_img}.fs ${_aitab_sfs_comp} + mv "${_src}.fs.sfs" "${_dst}" + rm "${_src}.fs" + fi + else + if _is_directory_changed "${_src}" "${_dst}/${_aitab_img}.sfs"; then + _mksfs ${_aitab_img} ${_aitab_sfs_comp} + mv "${work_dir}/${_aitab_img}.sfs" "${_dst}" + fi + fi + done < "${work_dir}/iso/${install_dir}/aitab" +} + +# Install packages on root-image. +# A basic check to avoid double execution/reinstallation is done via hashing package names. +command_install () { + if [[ ! -f "${pacman_conf}" ]]; then + _msg_error "Pacman config file '${pacman_conf}' does not exist" 1 + fi + + #trim spaces + pkg_list="$(echo ${pkg_list})" + + if [[ -z ${pkg_list} ]]; then + _msg_error "Packages must be specified" 0 + _usage 1 + fi + + _show_config install + + local _pkg_list_hash + _pkg_list_hash=$(echo ${pkg_list} | sort -u | md5sum | cut -c1-32) + if [[ -f "${work_dir}/install.${_pkg_list_hash}" ]]; then + _msg_info "These packages are already installed, skipping." + else + _pacman "${pkg_list}" + : > "${work_dir}/install.${_pkg_list_hash}" + fi +} + +command_init() { + _show_config init + _chroot_init +} + +command_run() { + _show_config run + _chroot_run +} + +if [[ ${EUID} -ne 0 ]]; then + _msg_error "This script must be run as root." 1 +fi + +while getopts 'p:r:C:L:P:A:D:w:o:vh' arg; do + case "${arg}" in + p) pkg_list="${pkg_list} ${OPTARG}" ;; + r) run_cmd="${OPTARG}" ;; + C) pacman_conf="${OPTARG}" ;; + L) iso_label="${OPTARG}" ;; + P) iso_publisher="${OPTARG}" ;; + A) iso_application="${OPTARG}" ;; + D) install_dir="${OPTARG}" ;; + w) work_dir="${OPTARG}" ;; + o) out_dir="${OPTARG}" ;; + v) quiet="n" ;; + h|?) _usage 0 ;; + *) + _msg_error "Invalid argument '${arg}'" 0 + _usage 1 + ;; + esac +done + +shift $((OPTIND - 1)) + +if [[ $# -lt 1 ]]; then + _msg_error "No command specified" 0 + _usage 1 +fi +command_name="${1}" + +case "${command_name}" in + init) + command_init + ;; + install) + command_install + ;; + run) + command_run + ;; + prepare) + command_prepare + ;; + checksum) + command_checksum + ;; + pkglist) + command_pkglist + ;; + iso) + if [[ $# -lt 2 ]]; then + _msg_error "No image specified" 0 + _usage 1 + fi + img_name="${2}" + command_iso + ;; + *) + _msg_error "Invalid command name '${command_name}'" 0 + _usage 1 + ;; +esac + +# vim:ts=4:sw=4:et: diff --git a/build_tools/archiso/archiso/testiso b/build_tools/archiso/archiso/testiso new file mode 100755 index 0000000..71e85e1 --- /dev/null +++ b/build_tools/archiso/archiso/testiso @@ -0,0 +1,26 @@ +#!/bin/bash + +if [[ $# -eq 0 ]]; then + echo "usage: testiso <isofile> [id|ic|sd|sc|vd|vc]" + echo + echo "id boot .iso as IDE disk" + echo "ic boot .iso as IDE cd-rom" + echo "sd boot .iso as SCSI disk" + echo "sc boot .iso as SCSI cd-rom" + echo "vd boot .iso as virtio disk" +# echo "vc boot .iso as virtio cd-rom" + exit 1 +fi + +case "${2}" in + id) IF="ide"; MEDIA="disk";; + ic) IF="ide"; MEDIA="cdrom";; + sd) IF="scsi"; MEDIA="disk";; + sc) IF="scsi"; MEDIA="cdrom";; + vd) IF="virtio"; MEDIA="disk";; +# vc) IF="virtio"; MEDIA="cdrom";; + *) IF="scsi"; MEDIA="cdrom";; +esac + +echo qemu-system-x86_64 -m 256 -drive file=${1},if=${IF},media=${MEDIA},boot=on +qemu-system-x86_64 -m 256 -drive file=${1},if=${IF},media=${MEDIA},boot=on diff --git a/build_tools/bin/mp b/build_tools/bin/mp deleted file mode 100755 index 08d7b89..0000000 --- a/build_tools/bin/mp +++ /dev/null @@ -1,228 +0,0 @@ -#!/bin/bash -# Used to compile a package and copy it to the correct repository location. -# all arguments are passed to makepkg -# -# -. /etc/makepkg.conf -. PKGBUILD -PKGHOME=/data/pkg_repo/packages -SRCPKGHOME=/data/pkg_repo/src_packages -mydir=`pwd` -repolist="core extra" -cmdline=$@ - -git_branch=`git branch --no-color 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/\1/'` - -case $git_branch in - "testing") PFIX="-testing" ;; - "master" ) PFIX="" ;; - * ) echo "can't determine which is in use";; - esac - - -#if [ x`basename $0` = xmpr ] -#then -# PFIX="" -#else -# PFIX="-testing" -#fi - ALL_OFF="$(tput sgr0)" - BOLD="$(tput bold)" - BLUE="${BOLD}$(tput setaf 4)" - GREEN="${BOLD}$(tput setaf 2)" - RED="${BOLD}$(tput setaf 1)" - YELLOW="${BOLD}$(tput setaf 3)" - -plain() { - local mesg=$1; shift - printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg() { - local mesg=$1; shift - printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg() { - local mesg=$1; shift - printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -warning() { - local mesg=$1; shift - printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - - -function bump { - msg x$cmdline | grep -q \\-b - if [ $? = 0 ] - then - cmdline=`msg $cmdline|sed -e "s/-b//g"` - i="PKGBUILD" - #getting pkgrel number after = - package_pkgrel=`cat $i | grep pkgrel= | cut -d'=' -f 2` - - #extract number before dot in pkgrel - beforedot=`msg $package_pkgrel | cut -n -d'.' -f 1` - - #add 1 to pkgrel - new_pkgrel=$[$beforedot+1] - - #write it back to PKGBUILD - sed -i "s:pkgrel=${package_pkgrel}:pkgrel=${new_pkgrel}:g" $i - msg "bumped $pkgname release from $package_pkgrel to $new_pkgrel" - . PKGBUILD - fi - #defined below -# TOTALPKG="$pkgname"-"$pkgver"-"$pkgrel"-"$CARCH".pkg.tar.gz - SRCPKG="$pkgname"-"$pkgver"-"$pkgrel".src.tar.gz - -} - - - - -function find_repo { - dirname=`dirname $mydir` - - echo $dirname|grep -q "extra" - if [ $? = 0 ] - then - REPO=extra${PFIX} - else - REPO=core${PFIX} - fi - echo $dirname|grep -q "chroot-devel" - if [ $? = 0 ] - then - REPO=chroot-devel - fi - - echo $dirname|grep -q "mv-core" - if [ $? = 0 ] - then - REPO=mv-core - fi - - if [ x`basename $0` = xmpl ] - then - REPO="local" - fi - DOCROOT=/data/pkg_repo/$CARCH/$REPO - if [ ! -e $DOCROOT ] - then - mkdir -p $DOCROOT - fi - msg $REPO -} - - -function update-repo { - cd $DOCROOT - msg - msg "DOCROOT=$DOCROOT" - msg - for i in ${pkgname[@]} - do - msg $i - TOTALPKG="$i"-"$pkgver"-"$pkgrel"-"$CARCH".pkg.tar.gz - if [ ! -e $PKGDEST/$TOTALPKG ] - then - TOTALPKG="$i"-"$pkgver"-"$pkgrel"-"$CARCH".pkg.tar.xz - fi - if [ ! -e $PKGDEST/$TOTALPKG ] - then - msg $PKGDEST/$TOTALPKG - msg "Couldn't find the new package" - exit 0 - fi - - #remove old package from local copy - DELPKG="$i"-"$pkgver"-*-"$CARCH".pkg.tar.*z - rm -fvI $DELPKG - #copy in new package - msg "############################################" - msg "Updating $CARCH/$REPO with $TOTALPKG" - cp -f $PKGDEST/$TOTALPKG . - repo-add $DOCROOT/$REPO.db.tar.gz $DOCROOT/$TOTALPKG - msg "############################################" - done - cd - - -} - - -function update-src-pkg { - msg "---------------------------------SRC-------------------------" - DELPKG="$pkgname"-"$pkgver"-*.src.tar.*z - msg "Removing old src pkg" - if [ ! -d $PKGHOME/$REPO/$SRCPKG ] - then - msg "Creating src_pkg directory" - mkdir -p $SRCPKGHOME/$REPO - fi - rm -vf $SRCPKGHOME/$REPO/$DELPKG - msg "copying src_pkg to $SRCPKGHOME/$REPO" - cp -f $PKGHOME/$SRCPKG $SRCPKGHOME/$REPO -} - -function dup_check { - - if [ $REPO = "local" -o $REPO = "mv-core" ] - then - return - fi - - for i in $repolist - do - if [ ! $i$PFIX = $REPO ] - then - out=`pacman -Sl $i$PFIX|cut -d" " -f2 |grep ^pkgname$` - if [ $? = 0 ] - then - msg "#######################################" - msg "$pkgname already exists in $i$PFIX" - msg " "$out - msg - - msg x$cmdline|grep -q \\-f - if [ $? = 0 ] - then - msg "Force detected, making pkg anyhow" - msg "#######################################" - return - else - msg "#######################################" - exit 2 - fi - fi - fi - done - -} - -#MAIN PROGRAM -find_repo -dup_check -bump - -makepkg --asroot $cmdline || exit 1 -msg "=============FINSHED CREATING PKG==================" -if [ $arch = "any" ] -then - CARCH="any" -fi -msg "---------updating database ----------" -update-repo #&& pacman -Sy -msg "----------creating source package---------" -makepkg -f --holdver --asroot --source --skipinteg && update-src-pkg - - - - diff --git a/build_tools/bin/mpv2.py b/build_tools/bin/mpv2.py new file mode 100755 index 0000000..bcbbfec --- /dev/null +++ b/build_tools/bin/mpv2.py @@ -0,0 +1,1044 @@ +#!/usr/bin/env python2 +# Version 0.8.0 +import pprint +import os +import sys +import re +import subprocess +import gettext +from git import Repo,Git +import ConfigParser +import glob +import shutil +import optparse +import fileinput +import parched +import time + +PKGHOME = "/data/pkg_repo/packages" +SRCPKGHOME = "/data/pkg_repo/src_packages" +ARCHABS = "/var/abs" + +# See what git branch we're working under +git_repo = Git() +branches = git_repo.branch() +git_branch = branches.split('*')[1].lstrip(' ') + +if git_branch == "testing": + SFIX = "-testing" +elif git_branch == "master": + SFIX = "" +else: + print "Can't determine which git branch is in use!" + sys.exit(2) +print "Git Branch suffix is:",SFIX + + +def commandline(makepkg_cmd): + #print makepkg_cmd + cli_dict = {} + cli_list = [] + + clparser = optparse.OptionParser() + clparser.add_option("--asroot", action="store_true", default=True, help="Allow makepkg to run as root.") + clparser.add_option("-A", "--ignorearch", action="store_true", default=False, help="Ignore a missing or incomplete arch field in the build script.") + clparser.add_option("-b", "--bump", action="store_true", default=False, help="Increase package release one unit.") + clparser.add_option("-5", "--md5", action="store_true", default=False, help="Clear out old md5 sums") + clparser.add_option("--abs", action="store_true", default=False, help="update packages from abs") + clparser.add_option("--recurse", action="store_true", default=False, help="Recursively search for all dependencies") + clparser.add_option("--pkgdir", action="store", help="pkg dir to cd into.") + clparser.add_option("--pkglist", action="store", help="List of packages to compile") + clparser.add_option("-c", "--clean", action="store_true", default=False, help="Clean up leftover work files and directories after a successful build. ") + clparser.add_option("-C", "--cleancache", action="store_true", default=False, help="Removes all cached source files from the directory specified in SRCDEST in makepkg.conf") + clparser.add_option("--config", action="store", help="Use an alternate config file instead of the /etc/makepkg.conf default.") + clparser.add_option("-d", "--nodeps", action="store_true", default=False, help="Do not perform any dependency checks.") + clparser.add_option("-e", "--noextract", action="store_true", default=False, help="Do not extract source files; use whatever source already exists in the src/ directory.") + clparser.add_option("-f", "--force", action="store_true", default=False, help="This allows a built package to be overwritten.") + clparser.add_option("--forcever", action="store_true", default=False, help="This is a hidden option that should not be used unless you really know what you are doing.") + clparser.add_option("-g", "--geninteg", action="store_true", default=False, help="For each source file in the source array of PKGBUILD, download the file if required and generate integrity checks.") + clparser.add_option("--skipinteg", action="store_true", default=False, help="Do not perform any integrity checks, just print a warning instead.") + clparser.add_option("--holdver", action="store_true", default=True, help="Prevents makepkg from automatically bumping the pkgver to the latest revision number in the package's development tree.") + clparser.add_option("--nohold", action="store_false", dest="holdver", help="Automatically bump the pkgver to the latest revision number in the package's development tree.") + clparser.add_option("-i", "--install", action="store_true", default=False, help="Install or upgrade the package after a successful build.") + clparser.add_option("-L", "--log", action="store_true", default=True, help="Enable makepkg build logging.") + clparser.add_option("-m", "--nocolor", action="store_true", default=False, help="Disable color in output messages.") + clparser.add_option("-o", "--nobuild", action="store_true", default=False, help="Download and extract files only, but do not build them.") + clparser.add_option("-p", action="store", help="Read the package script buildscript instead of the PKGBUILD default.", metavar='/path/to/buildscript') + clparser.add_option("-r", "--rmdeps", action="store_true", default=False, help="Upon successful build, remove any dependencies installed by makepkg during dependency auto-resolution and installation when using -s.") + clparser.add_option("-R", "--repackage", action="store_true", default=False, help="Repackage contents of the package without rebuilding the package.") + clparser.add_option("-s", "--syncdeps", action="store_true", default=False, help="Install missing dependencies using pacman.") + clparser.add_option("--allsource", action="store_true", default=False, help="Do not actually build the package, but build a source-only tarball that includes all sources, including those that are normally download via makepkg.") + clparser.add_option("--source", action="store_true", default=False, help="Do not actually build the package, but build a source-only tarball that does not include sources that can be fetched via a download URL.") + clparser.add_option("--pkg", action="store", help="Only build listed packages from a split package.") + clparser.add_option("--noconfirm", action="store_true", default=False, help="(Passed to pacman) Prevent pacman from waiting for user input before proceeding with operations.") + clparser.add_option("--noprogressbar", action="store_true", default=False, help="(Passed to pacman) Prevent pacman from displaying a progress bar.") + clparser.add_option("--rmold", action="store_true", default=False, help="BETA: Remove old src and software packages from repos. Use with caution. False positives may occur (i.e. nvidia pkgs)!") + + (options, args) = clparser.parse_args() + + options1 = ['config', 'p', 'pkg', 'pkgdir', 'pkglist'] + options2 = ['asroot', 'ignorearch', 'bump', 'clean', 'cleancache', 'nodeps', + 'noextract', 'force', 'forcever', 'geninteg', 'skipinteg', 'holdver', + 'install', 'log', 'nocolor', 'nobuild', 'rmdeps', 'repackage', + 'syncdeps', 'allsource', 'source', 'noconfirm', 'noprogressbar', + 'rmold', 'md5','recurse','abs' ] + + for o in options1: + cmd1 = eval('options.'+o) + if o is not 'p': + if cmd1 is not None: + cli_dict['--'+o] = cmd1 + elif cmd1 is not None: + cli_dict['-'+o] = cmd1 + for o in options2: + cmd2 = eval('options.'+o) + if cmd2 is True: + cli_list.append('--'+o) + + # Create makepkg command + makepkg_cmd.extend(cli_list) + for k in cli_dict.keys(): + v = cli_dict.get(k) + makepkg_cmd.append(k) + makepkg_cmd.append(v) + + # Remove bump option from makepkg command if it exists + if "--bump" in makepkg_cmd: + makepkg_cmd.remove("--bump") + # Remove rmold option from makepkg command if it exists + if "--rmold" in makepkg_cmd: + makepkg_cmd.remove("--rmold") + # Remove "--geninteg" option (if it exists) from makepkg command. + # Checking for *sums is done automaticly by mp.py + if "--geninteg" in makepkg_cmd: + makepkg_cmd.remove("--geninteg") + + if "--md5" in makepkg_cmd: + makepkg_cmd.remove("--md5") + + if "--recurse" in makepkg_cmd: + makepkg_cmd.remove("--recurse") + + if "--pkgdir" in makepkg_cmd: + makepkg_cmd.remove("--pkgdir") + makepkg_cmd.remove(cli_dict["--pkgdir"]) + + if "--pkglist" in makepkg_cmd: + makepkg_cmd.remove("--pkglist") + makepkg_cmd.remove(cli_dict["--pkglist"]) + + if "--abs" in makepkg_cmd: + makepkg_cmd.remove("--abs") + + + print "Makepkg Command:",makepkg_cmd + return makepkg_cmd, cli_list, cli_dict + + + +class Packagefile(object): + def __init__(self,cli_dict,cli_list,makepkg_cmd): + self.failure=0 + self.attempt=0 + self.REPO = "none" + self.DOCROOT = "" + self.pkgfile = "none" + self.pkgbase = "" + self.pkgname = "" + self.pkgver = "" + self.pkgrel = "" + self.pkglist=[] + self.epoch = "" + self.arch = "" + self.CARCH = "" + self.PKGDEST = "" + self.TOTALPKG = "" + self.GZPKG = "" + self.XZPKG = "" + self.repolist=["core", "extra", + "chroot-devel", "mv-core", "xmpl", "local"] + self.mydir = os.getcwd() + self.variables = ['pkgbase','pkgname','pkgver','pkgrel','arch','epoch','depends','makedepends','checkdepends'] + self.pkgrel_incremented = False + self.makepkg_cmd = makepkg_cmd + self.cli_list = cli_list + self.cli_dict = cli_dict + self.pkg_dependencies = [] + self.makepkg_dependencies = [] + self.check_dependencies = [] + self.backupdir="/tmp/backup/" + self.updateABS = "Update waiting or not requested" + self.compile_status = "Pending" + self.backup_pkgdir = "" + self.pkg_dep_available = [] + self.pkg_dep_not_in_repo = [] + if "-p" in cli_dict: + self.pkgfile = cli_dict["-p"] + else: + self.pkgfile = "PKGBUILD" + + self.updateINIT() + + + + def set_srcpkg(self): + if self.pkgbase: + self.SRCPKG = self.pkgbase + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + ".src.tar.gz" + else: + self.SRCPKG = self.pkglist[0] + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + ".src.tar.gz" + #print "Changed " + self.pkgname + " release to " + str(self.pkgrel) + + def updateINIT(self): + # Check what file will be used for our PKGBUILD + + # Does the file exist? + if not os.path.isfile(self.pkgfile): + print "ERROR in function config_file: Can't find ",self.pkgfile,"!" + sys.exit(2) + else: + # Loop over contents to get our variables + # Use bash to do it because PKGBUILDs are very loose with their format + for item in self.variables: + v = subprocess.Popen(['/bin/bash','-c', 'source ' + + self.pkgfile + + '; echo ${' + item + '[@]}'], + stdout = subprocess.PIPE,) + value = v.communicate()[0].strip('\n') + if item == "pkgbase": + self.pkgbase = value + elif item == "pkgname": + self.pkgname = value + self.pkglist = list(value.split()) + elif item == "pkgver": + self.pkgver = value + elif item == "pkgrel": + self.pkgrel = value + elif item == "epoch": + if value: + self.epoch = "%s:" %value + elif item == "arch": + self.arch = value + elif item == "depends": + self.pkg_dependencies = value.split() + + elif item == "makedepends": + self.makepkg_dependencies = value.split() + elif item == "checkdepends": + self.check_dependencies = value.split() + + self.set_srcpkg() + # Get needed makepkg.conf variables + mpkg="/etc/makepkg.conf" + f=open(mpkg,"r") + # Read file contents into memory + makepkg_contents=f.readlines() + f.close() + # Loop over contents to get our variables + for mp_lines in makepkg_contents: + if mp_lines.strip().startswith("CARCH"): + self.CARCH = mp_lines.partition("=")[2].strip('\n').replace('\"','') + if self.arch != 'any': + self.arch = self.CARCH + if mp_lines.strip().startswith("PKGDEST"): + self.PKGDEST = mp_lines.partition("=")[2].strip('\n') + + + + def print_vars(self): + print "####################################################" + print "" + print " Config File:",self.pkgfile + print " pkgbase:",self.pkgbase + print " pkgname:",self.pkgname + print " pkglist:",self.pkglist + print " pkgver:",self.pkgver + print " pkgrel:",self.pkgrel + print " epoch:" ,self.epoch + print " arch:",self.arch + print " Source package:",self.SRCPKG + print " CARCH is:",self.CARCH + print " Package destination:",self.PKGDEST + print " Architecture is:",self.arch + print " Repository : ", self.REPO + print " DOCROOT : ", self.DOCROOT + print " Current directory: ", self.mydir + print " Backup direcotory: ", self.get_backupdir() + print " Dependencies: ", self.pkg_dependencies + print " Make Dependencies: ", self.makepkg_dependencies + print " Unmet Dependencies - compile:" , self.pkg_dep_not_in_repo + print " Unmet Dependencies - available:" , self.pkg_dep_available + print " Update ABS Status: " ,self.get_ABS_status() + print " Package compile status: ", self.get_compile_status() + print "" + print "####################################################" + + def getname(self): + return self.pkgname + + def getfailure(self): + return self.failure + + def getattempts(self): + return self.attempt + + def getdir(self): + return self.mydir + + def get_unmet_compile(self): + return self.pkg_dep_not_in_repo + + def get_unmet_install(self): + return self.pkg_dep_available + + def incrementcompile(self): + self.attempt = self.attempt + 1 + + def calculate_unmet_depends(self): + pkg_not_installed = [] + pkg_not_in_repo = [] + pkg_available = [] + val='' + command = [ "pacman" , "-T" ] + pkg_list= self.pkg_dependencies + self.makepkg_dependencies + self.check_dependencies + + if len(pkg_list) > 0: + command = command + pkg_list + print " Checking for dependencies against installed packages" + #print command + pkg_not_installed = subprocess.Popen(command, stdout = subprocess.PIPE,).communicate()[0].strip().split('\n') + while val in pkg_not_installed: + pkg_not_installed.remove(val) + + for i in pkg_list: + split_package = i.replace('>',' ').replace('<',' ').replace('=',' ').split() + + if len(split_package) > 1: + split_package_name = split_package[0] + package_version = split_package[1] + else: + split_package_name = split_package[0] + package_version = None + + pkgregext="^%s$" %split_package_name + command = ["pacman" , "-Ss" , pkgregext ] + #print " Checking for dependencies in repository: %s" %i + p = subprocess.Popen(command, stdout = subprocess.PIPE,) + p.wait() + rc = p.returncode + if rc == 0 : + if i in pkg_not_installed: + print "- Dependency found in repository, but not installed: %s" %i + pkg_available.append(split_package_name) + else: + print "- Dependency not found, it must be built: %s" %i + pkg_not_in_repo.append(split_package_name) + + self.pkg_dep_available = pkg_available + self.pkg_dep_not_in_repo = pkg_not_in_repo + + + return + + def get_ABS_status(self): + return self.updateABS + + def get_backupdir(self): + return self.backup_pkgdir + + def get_compile_status(self): + return self.compile_status + + def find_repo(self): + # Create a list with the directory names of our current directory tree + dir_tree = os.path.dirname(self.mydir).split("/") + # Loop through the dir_tree to see if we can find our repo + for item in self.repolist: + if item not in dir_tree: + continue + else: + repo_name = item + if repo_name == "extra": + self.REPO = "extra" + SFIX + elif repo_name == "core": + self.REPO = "core" + SFIX + elif repo_name == "xmpl": + self.REPO = "local" + elif repo_name not in repolist: + print "ERROR in function find_repo: Cannot determine repository!" + sys.exit(2) + else: + self.REPO = repo_name + # Ensure our DOCROOT exists and if not, create it + self.DOCROOT = "/data/pkg_repo/" + self.CARCH + "/" + self.REPO + print "DOCROOT:",self.DOCROOT + if os.path.exists(self.DOCROOT): + print "INFO: Repository is",self.REPO + else: + try: + os.mkdir(DOCROOT,0755) + print "INFO: ",self.DOCROOT,"directory created. Repository is",self.REPO + except: + print " Creating DOCROOT failed" + pass + + def dup_check(self): + print "- Checking if package is a duplicate in core or extra:" + #if REPO in ("local", "mv-core", "xmpl", "chroot-devel"): + # return + for tmp_repo in ("core", "extra"): + if tmp_repo + SFIX != self.REPO: + p1 = subprocess.Popen(["pacman", "-Sl", tmp_repo + SFIX], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,) + p2 = subprocess.Popen(["cut", "-d", " ", "-f", "2"], stdin=p1.stdout, stdout=subprocess.PIPE,) + p3 = subprocess.Popen(["grep","-w" , self.pkgname], stdin=p2.stdout, stdout = subprocess.PIPE,) + #output = p3.communicate()[0].strip('\n') + output = p3.communicate()[0].strip('\n') + #print tmp_repo + SFIX + #print output + #print "###############" + + if output == self.pkgname: + if "--force" not in self.cli_list: + print "#######################################" + print self.pkgname," already exists in ",tmp_repo + SFIX + print + print "Use --force to overwite" + print "#######################################" + sys.exit(2) + else: + print "Force detected! Making package regardless!" + print "#######################################" + else: + if output != 0: + print " Package is not a duplicate" + return + def increase_pkgrel(self): + if not "--bump" in self.cli_list: + return + print "- Incrementing pkgrel for %s" %self.pkgname + # Backup the original pkgfile + shutil.copy2(self.pkgfile, self.pkgfile + ".old") + # Let's increase the pkgrel + for line in fileinput.input(self.pkgfile, inplace=1): + if line.strip().startswith("pkgrel"): + self.pkgrel = line.partition("=")[2].strip('\n') + # Add 1 to pkgrel + new_pkgrel = int(self.pkgrel) + 1 + line = line.replace("pkgrel=" + self.pkgrel, "pkgrel=" + str(new_pkgrel)) + self.pkgrel = str(new_pkgrel) + sys.stdout.write(line) + print " pkgrel:",self.pkgrel + self.set_srcpkg() + self.pkgrel_incremented = True + + def clearMD5(self): + print " Clearing out old md5sums" + sumlist=['sha1sums', 'sha256sums', 'sha384sums', 'sha512sums', 'md5sums'] + filename=self.pkgfile + file_contents=[] + f = open(filename,"r") + lineIter= iter(f) + for aLine in lineIter: + if aLine.strip().startswith("md5sum"): + if aLine.strip().endswith(")"): + continue + else: + for aModule in lineIter: + if aModule.strip().endswith(")"): + break + else: + file_contents.append(aLine) + f.close() + try: + f = open(filename,"w") + except: + outtext="Couldn't open file for writing: %s" %filename + print outtext + return False + for line in file_contents: + f.write(line) + f.close() + + def updateMD5(self): + print "- Checking MD5 sums" + if "--md5" in self.cli_list: + self.clearMD5() + + sums = [] + pkgfile_contents=[] + + # Open pkgfile for reading + f = open(self.pkgfile, 'r') + pkgfile_contents = f.readlines() + f.close() + + + # Iterate through the lines looking for a match pattern + for line in pkgfile_contents: + check = re.compile('sha1sums|sha256sums|sha384sums|sha512sums|md5sums') + sums = check.match(line) + if sums: + break + if not sums: + print " Adding md5 sums" + # If no matches are found, append md5sums to the end of the pkgfile + p = open(self.pkgfile, 'a') + md5gen = subprocess.Popen(["makepkg", "--asroot", "-g"], stdout = subprocess.PIPE,).communicate()[0] + p.writelines(md5gen) + p.close() + + def make_package(self): + if self.arch == 'any': + self.CARCH = 'any' + #creates both binary and source package + print "- Making package" + retcode = subprocess.call(self.makepkg_cmd) + if retcode != 0: + self.compile_status = "Failed: %s" %retcode + self.failure = self.failure + 1 + print " ERROR: makepkg failed with return code ",retcode + if self.pkgrel_incremented: + if os.path.isfile(self.pkgfile + '.old'): + shutil.move(self.pkgfile + '.old', self.pkgfile) + print " Reverted pkgrel increment." + return False + elif "--nobuild" in self.makepkg_cmd: + self.compile_status = "Failed: nobuild" + return False + else: + print "- Making source package" + retcode = subprocess.call(["makepkg", "--force", "--holdver", "--asroot", "--source"]) + if retcode != 0: + self.compile_status = "Failed source: %s" %retcode + self.failure = self.failure + 1 + print "ERROR: Source package creation failed with return code",retcode + sys.exit(2) + print "=============FINSHED CREATING PKG==================" + self.compile_status = "Success" + return True + + def update_database(self): + print "- Copying files and updating database" + # pkgname could be a list of several pkgs. Since bash array format is + # loose, let bash parse the pkgname(s) first, then return a list for us. + os.chdir(self.DOCROOT) + + print + print " INFO: Changed working dir to",self.DOCROOT + print + + for i in self.pkglist: + print " Package name:",i + + self.GZPKG = i + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + "-" + self.CARCH + ".pkg.tar.gz" + self.XZPKG = i + "-" + self.epoch + self.pkgver + "-" + self.pkgrel + "-" + self.CARCH + ".pkg.tar.xz" + print self.GZPKG + print self.XZPKG + + if os.path.isfile(self.PKGDEST + "/" + self.XZPKG): + self.TOTALPKG = self.XZPKG + elif os.path.isfile(self.PKGDEST + "/" + self.GZPKG): + self.TOTALPKG = GZPKG + else: + print "" + print "!!!!!!!!!!!!!!!!!" + print " ERROR in function update_repo: Couldn't find the new package",self.PKGDEST + "/" + self.TOTALPKG + sys.exit(2) + + print " Package file:",self.TOTALPKG + # Remove old package(s) from local copy + oldpkgcheck = re.compile( i + "-" + self.epoch + '(pkgver|[\d.]+)' + "-" + '(?!pkgrel)' ) + dirlist = os.listdir(self.DOCROOT) + for n in dirlist: + if n.startswith(i): + if not oldpkgcheck.search(n): + print n,"does not match" + continue + else: + OLDPKG = glob.glob(oldpkgcheck.search(n).group() + "*.pkg.tar.?z") + print "OLDPKG =",OLDPKG + if OLDPKG: + for DELPKG in OLDPKG: + if "--rmold" in self.cli_list: + print "Deleting old package:",DELPKG + os.remove(DELPKG) + # Remove any symlinks to old packages + # We make it conditional on "--force" because force will overwrite + # an existing package and we want the symlink to stay, pointing to + # the newly built package with the same pkgrel. + if "--force" not in self.makepkg_cmd: + if os.path.islink(self.mydir + "/" + DELPKG): + os.remove(self.mydir + "/" + DELPKG) + # Copy in new package + print " Updating " + self.DOCROOT + " with " + self.TOTALPKG + print " Copying " + self.PKGDEST + "/" + self.TOTALPKG + shutil.copy2(self.PKGDEST + "/" + self.TOTALPKG, self.DOCROOT) + subprocess.call(["repo-add", self.DOCROOT+ "/" + self.REPO + ".db.tar.gz", self.DOCROOT + "/" + self.TOTALPKG]) + print " Updating pacman database" + subprocess.call(["pacman","-Syyyyy" ]) + + def update_srcrepo(self): + print "- Updating source file repoistory" + print " SRCPKG:",self.SRCPKG + OLDSRCPKG="" + os.chdir(SRCPKGHOME + "/" + self.REPO) + print " INFO: Changed working dir to",SRCPKGHOME + "/" + self.REPO + + if not SRCPKGHOME + "/" "/" + self.SRCPKG: + print "ERROR in function update_repo: Couldn't find the new package",SRCPKGHOME + "/" + self.REPO + "/" + self.SRCPKG + sys.exit(2) + + i = self.pkglist[0] + oldpkgcheck = re.compile( i + "-" + self.epoch + '(pkgver|[\d.]+)' + "-" + '(?!pkgrel)' ) + dirlist = os.listdir(SRCPKGHOME + "/" ) + if self.pkgbase: + # Remove old src package(s) from local copy + if "--rmold" in self.cli_list: + for n in dirlist: + if n.startswith(self.pkgbase): + if not oldpkgcheck.search(n): + print n,"does not match" + continue + else: + OLDSRCPKG = glob.glob(oldpkgcheck.search(n).group() + "*.src.tar.gz") + else: + # Remove old src package(s) from local copy + if "--rmold" in self.cli_list: + for n in dirlist: + if n.startswith(self.pkgname): + if not oldpkgcheck.search(n): + print n,"does not match" + continue + else: + OLDSRCPKG = glob.glob(oldpkgcheck.search(n).group() + "*.src.tar.gz") + if OLDSRCPKG: + print "OLDSRCPKG =",OLDSRCPKG + for DELSRCPKG in OLDSRCPKG: + print "Deleting old source package",DELSRCPKG + os.remove(DELSRCPKG) + print "Copying new source package to",SRCPKGHOME + "/" + self.REPO + "/" + self.SRCPKG + shutil.copy2(SRCPKGHOME + "/" + self.SRCPKG, SRCPKGHOME + "/" + self.REPO + "/") + + + def cleanup(self): + os.chdir(self.mydir) + if os.path.isfile(self.pkgfile + '.old'): + os.remove(self.pkgfile + '.old') + print "Removed temporary backup file",self.pkgfile + '.old' + #remove src.tar.gz so it stops cluttering up git + if os.path.islink(self.SRCPKG): + os.unlink(self.SRCPKG) + # Remove any old -build and -package log files + os.chdir(self.mydir) + build_log = self.pkgname + "-" + self.pkgver + "-" + self.pkgrel + "-" + self.CARCH + "-build.log" + pkg_log = self.pkgname + "-" + self.pkgver + "-" + self.pkgrel + "-" + self.CARCH + "-package.log" + all_logs = glob.glob("*.log*") + #print "All Logs:",all_logs + saved_logs = [] + #save the logrotate files + saved_logs = glob.glob("*.logrotate.d") + if os.path.isfile(build_log): + saved_logs.append(build_log) + if os.path.isfile(pkg_log): + saved_logs.append(pkg_log) + #print "Saved Logs:",saved_logs + for log in all_logs: + if log not in saved_logs: + os.remove(log) + pass + + def findPKGdir(self,directory, spackage_list): + #""" + # Searches for PKGBUILDS with name recursively in directory. + # Returns False if the file was not found. + # This is used by promote and update + #""" + print "\n Searching for %s in %s" %(spackage_list,directory) + for root, dirs, files in os.walk(directory): + if "PKGBUILD" in files: + try: + currentfile=root+"/PKGBUILD" + package = parched.PKGBUILD(currentfile) + pkgname = package.name + pkgversion = str(package.version) + for spackage in spackage_list: + if type(pkgname) == str: + pkgname = package.name.strip() + if pkgname == spackage: + print " Found package in %s" %root + return root + if type(pkgname) == list: + pkgname = package.name + if spackage in pkgname: + print " Found package in %s" %root + return root + + except Exception, reason: + #print reason + pass + return False + + def check_for_changelog(self, pkgdir): + filename = "%s/__changelog" %pkgdir + return os.path.exists(filename) + + + def rsync_files(self,src_dir,dest_dir,rsync_flags="ar"): + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + if os.path.exists(dest_dir): + try: + cmd="rsync -%s --exclude src --exclude pkg %s/ %s" %(rsync_flags,src_dir,dest_dir ) + # print cmd + retcode = subprocess.call(cmd, shell=True) + if retcode == 0 : + outtext=" Copy was a succeses" + print outtext + else: + outtext="- Copying Failed" + print outtext + return False + except OSError, e: + outtext="- Copying Failed: %s " %(e) + print outtext + return False + return True + else: + outtext = "- Dest dir %s does not exist" %(dest_dir) + print outtext + return False + + + def backupAction(self): + current_time=time.strftime("%Y-%m-%d_%H-%M-%S") + src_dir=self.mydir + last_bit_of_path=src_dir.split("/")[-1] + dest_dir="%s/%s-%s" %(self.backupdir,last_bit_of_path,current_time) + self.backup_pkgdir = dest_dir + if self.rsync_files(src_dir,dest_dir): + outText=" Backup of %s was a success: %s" %(last_bit_of_path,dest_dir) + else: + outText="- Backup failed" + print outText + + def update_pkg_from_archabs(self): + if "--abs" in self.cli_list: + print " Searching if ABS has an updated PKGBUILD" + dest_dir = self.mydir + has_changelog = self.check_for_changelog(dest_dir) + if has_changelog : + outText = " Will not update, because changelog was detected: %s" %self.pkgname + print outText + self.updateABS="Failed, Changelog detected" + return False + #search /var/abs for pkg + src_dir = self.findPKGdir(ARCHABS, self.pkglist) + if src_dir: + #Check for diff in PKGBUILD HERE + diffpkg1 = "%s/PKGBUILD" %dest_dir + diffpkg2 = "%s/PKGBUILD" %src_dir + command = ["diff" , diffpkg1, diffpkg2 ] + p = subprocess.Popen(command, stdout = subprocess.PIPE,) + p.wait() + rc = p.returncode + if rc == 0 : + print " Will not update, PKGBUILDS look to be the same" + self.updateABS=" Failed, PKGBUILD are the same" + return False + self.backupAction() + print " Updated pkg found, started copy from %s" %src_dir + rsync_flags="ar --delete" + if self.rsync_files(src_dir,dest_dir,rsync_flags): + outText=" Update of %s was a success: %s" %(self.pkgname,src_dir) + self.updateABS = "Success" + self.updateINIT() + else: + outText="- Update failed" + self.updateABS = "Failed" + print outText + else: + outtext="- Search did not package %s in ABS" %self.pkgname + print outtext + self.updateABS = "Failed, did not find package" + return False + return True + else: + outtext = "- ABS update not requested" + print outtext + self.updateABS = "Update not requested" + return False + + def install_available_deps(self): + print " Installing available dependencies from repo" + self.calculate_unmet_depends() + for pkg in self.pkg_dep_available: + print " " + pkg + command=["pacman", "-S" ,"--noconfirm" , pkg] + p = subprocess.Popen(command, stdout = subprocess.PIPE,) + p.wait() + + + def uninstall_available_deps(self): + print " Removing installed deps" + for pkg in self.pkg_dep_available: + print " " + pkg + command=["pacman", "-R" ,"--noconfirm" , pkg] + p = subprocess.Popen(command, stdout = subprocess.PIPE,) + p.wait() + + +def search_forpkg(directory, spackage): + #""" + # Searches for PKGBUILDS with name recursively in directory. + # Returns False if the file was not found. + # This is used by promote and update + #""" + pkgname = None + pkgname_list = None + pkgversion = None + pkgFound=False + + split_package = spackage.replace('>',' ').replace('<',' ').replace('=',' ').split() + + if len(split_package) > 1: + split_package_name = split_package[0] + package_version = split_package[1] + else: + split_package_name = split_package[0] + package_version = None + #directory="/data/linhes_pkgbuild/abs/core/ddcxinfo" + print " Searching for %s in %s" %(spackage,directory) + for root, dirs, files in os.walk(directory): + if "PKGBUILD" in files: + try: + currentfile=root+"/PKGBUILD" + package = parched.PKGBUILD(currentfile) + if type(package.name) is str: + pkgname = str(package.name).strip() + if type(package.name) is list: + pkgname_list = package.name + + pkgname = str(package.name).strip() + #pkgbase = package.pkgbase + pkgversion = str(package.version) + + if ( pkgname == split_package_name ) : + pkgFound = True + elif ( split_package_name in pkgname_list ): + pkgFound = True + + if pkgFound: + return root + + except Exception, reason: + #print reason + pass + print "- Failed to find %s" %spackage + return False + + +def process_pkg(pkg_list,pkg_objdict,pkg_not_found,calling_pkg,recursive_failure): + LHABS="/data/linhes_pkgbuild/abs" + + #makepkg_cmd = ['makepkg'] + #makepkg_cmd, cli_list, cli_dict = commandline(makepkg_cmd) + + if recursive_failure == True : + return pkg_list,pkg_objdict,pkg_not_found,calling_pkg,recursive_failure + + + print " Updating pacman database" + subprocess.call(["pacman","-Syyyyy" ]) + for pkg in pkg_list: + #skip over blank ones + + if pkg == '': + continue + pkgdir='' + packagefile = None + if pkg in pkg_not_found: + print "***********" + print " pkg not found, recursive_failure detected" + print "***********" + recursive_failure = True + break + + for k, v in pkg_objdict.iteritems(): + #print k + #print v + if k == pkg: + print " Found package in working list." + pkgdir = v.getdir() + packagefile = v + + if pkgdir == '' : + pkgdir = search_forpkg(LHABS, pkg) + + if pkgdir: + print "\n\n" + print "$-------------------start of new pkg----------------------$" + print " %s , called by %s " %(pkg,calling_pkg) + cwd = os.getcwd() + os.chdir(pkgdir) + #print os.getcwd() + if packagefile == None: + packagefile = Packagefile(cli_dict,cli_list,makepkg_cmd) + else: + packagefile.incrementcompile() + print "##########################################" + print "* Number of times failed compile %s" %packagefile.getfailure() + print "* Number of times seen for compile %s" %packagefile.getattempts() + print "##########################################" + + if packagefile.getfailure() > 1: + recursive_failure = True + print "recursive failure detected, breaking out" + return pkg_list , pkg_objdict, pkg_not_found , recursive_failure + + if packagefile.getattempts() > 3: + recursive_failure = True + print "recursive attempts detected, breaking out" + return pkg_list , pkg_objdict, pkg_not_found , recursive_failure + + + #do update here + packagefile.update_pkg_from_archabs() + packagefile.calculate_unmet_depends() + list_of_unmet_to_compile = packagefile.get_unmet_compile() + list_of_unmet_to_install = packagefile.get_unmet_install() + print "* compile these" + print list_of_unmet_to_compile + + print "* install these" + print list_of_unmet_to_install + #print len(list_of_unmet_to_compile) + if len(list_of_unmet_to_compile) > 0 : + + #check if recursive right here + if "--recurse" in cli_list: + print " Couldn't compile not all dependencies are met" + pkg_objdict[packagefile.getname()]=packagefile + recursive_failure=process_pkg(list_of_unmet_to_compile,pkg_objdict,pkg_not_found,pkg,recursive_failure)[3] + if recursive_failure: + break + print "* continue_working_on_pkg all deps found" + else: + print " Skipping recursive dependencies build" + packagefile.find_repo() + packagefile.dup_check() + packagefile.increase_pkgrel() + packagefile.updateMD5() + packagefile.print_vars() + packagefile.install_available_deps() + if packagefile.make_package(): + packagefile.update_database() + packagefile.update_srcrepo() + packagefile.cleanup() + packagefile.uninstall_available_deps() + pkg_objdict[packagefile.getname()]=packagefile + os.chdir(cwd) + #pass + + else: + print " Couldn't find package in LinHES ABS: %s" %pkg + pkg_not_found.append(pkg) + + return pkg_list , pkg_objdict, pkg_not_found , recursive_failure + +def main(): + #pkg_list=['bac', 'system-templates', 'xscreensaver'] + pkg_list=['bac', 'system-templates', 'udisks' ] + pkg_list=['imagemagick'] + pkg_list=['fluxbox'] + #pkg_list=['libarchive'] + pkg_list=['gnupg'] + + + + pkg_not_found=[] + pkg_objdict={} + pkg_list=[] + + + global makepkg_cmd + global cli_list + global cli_dict + + makepkg_cmd = ['makepkg'] + makepkg_cmd, cli_list, cli_dict = commandline(makepkg_cmd) + + try: + pkg_list = cli_dict['--pkglist'].split(',') + except: + packagefile = None + packagefile = Packagefile(cli_dict,cli_list,makepkg_cmd) + pkg_objdict[packagefile.getname()]=packagefile + pkg_list=[packagefile.getname()] + + #send the list of packages to compile and watch the magic happen + #print cli_list + #print cli_dict + #sys.exit(1) + pkg_list, pkg_objdict, pkg_not_found,recursive_failure = process_pkg(pkg_list, + pkg_objdict, + pkg_not_found, + 'Initialloop',False) + + + update_pkg = [] + success_pkg = [] + failed_compile = [] + failed_update = [] + + #create lists + for k, v in pkg_objdict.iteritems(): + #print k + #print v.print_vars() + if v.get_compile_status().strip() == "Success": + success_pkg.append(k) + else: + temp = "%s --- %s (%s)" %(k,v.get_compile_status(),v.getattempts()) + failed_compile.append(temp) + + if v.get_ABS_status() == "Success": + temp = "%s --- Backup location: %s" %(k,v.get_backupdir()) + update_pkg.append(temp) + elif v.get_ABS_status() != 'Update waiting or not requested': + temp = "%s --- %s" %(k,v.get_ABS_status()) + failed_update.append(temp) + + print "\n\n\n\n" + print "#######################################################" + + print "\n" + print "Successful updated from ABS:" + print "----------------------------" + pprint.pprint(update_pkg) + + print "\n" + print "Failed updated from ABS:" + print "----------------------------" + pprint.pprint(failed_update) + + + print "\n" + print "Couldn't find these packages:" + print "----------------------------" + pprint.pprint(pkg_not_found) + + print "\n" + print "Successful compiled :" + print "---------------------------" + pprint.pprint(success_pkg) + + print "\n" + print "Failed to compile these packages:" + print "----------------------------" + pprint.pprint( failed_compile) + +if __name__ == "__main__": + main() + print "--------------------------" + +#libsig +#libxml++ dep search and compile +#
\ No newline at end of file |