path: root/build_tools/larch7/larch0/cli/larchify.py

#!/usr/bin/env python
#
# larchify.py
#
# (c) Copyright 2009-2010 Michael Towers (larch42 at googlemail dot com)
#
# This file is part of the larch project.
#
#    larch is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    larch is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with larch; if not, write to the Free Software Foundation, Inc.,
#    51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
#
#----------------------------------------------------------------------------
# 2010.07.13

# This is a command line script to prepare a larch live system from an
# Arch Linux installation. All needed parameters are passed as options.

import os, sys
from config import *
from backend import *
from userinfo import Userinfo
from glob import glob
import random, crypt
from subprocess import Popen, PIPE, STDOUT


class Builder:
    """This class manages 'larchifying' an Arch Linux installation.
    """
    def __init__(self, options):
        self.installation_dir = get_installation_dir()
        self.installation0 = (self.installation_dir
                if self.installation_dir != "/" else "")
        testfile = self.installation0 + '/etc/pacman.conf'
        if not os.path.isfile(testfile):
            errout(_("File '%s' doesn't exist:\n"
                    "  '%s' not an Arch installation?")
                    % (testfile, self.installation_dir))

        self.profile_dir = get_profile()


    def build(self, options):
        if not (self.installation0 or query_yn(_(
                    "Building a larch live medium from the running system is"
                    "\nan error prone process. Changes to the running system"
                    "\nmade while running this function may be only partially"
                    "\nincorporated into the compressed system images."
                    "\n\nDo you wish to continue?")), True):
                return False

        # Define the working area - it must be inside the installation
        # because of the use of chroot for some functions
        self.larchify_dir = self.installation0 + CHROOT_DIR_LARCHIFY
        # Location for the live medium image
        self.medium_dir = self.installation0 + CHROOT_DIR_MEDIUM
        # And potentially a saved system.sqf
        self.system_sqf = self.installation0 + CHROOT_SYSTEMSQF
        # Needed for a potentially saved locales directory
        self.locales_base = self.installation0 + CHROOT_DIR_BUILD
        # For building the (mods.sqf) overlay
        self.overlay = self.installation0 + CHROOT_DIR_OVERLAY
        comment("Initializing larchify process")

        if options.oldsqf:
            if os.path.isfile(self.medium_dir + "/larch/system.sqf"):
                runcmd("mv %s/larch/system.sqf %s" %
                        (self.medium_dir, self.system_sqf))
        else:
            runcmd("rm -f %s" % self.system_sqf)

        # Clean out larchify area and create overlay directory
        runcmd('rm -rf %s' % self.larchify_dir)
        runcmd('mkdir -p %s' % self.overlay)

        if not self.find_kernel():
            return False

        if not self.system_check():
            return False

        comment("Beginning to build larch medium files")
        # Clear out the directory
        runcmd('rm -rf %s' % self.medium_dir)
        # The base medium boot directory, bootloader independent.
        runcmd('mkdir -p %s/boot' % self.medium_dir)
        # The main larch direcory
        runcmd('mkdir -p %s/larch' % self.medium_dir)

        # kernel
        runcmd("cp -f %s/boot/%s %s/boot/larch.kernel" %
                (self.installation0, self.kname, self.medium_dir))
        # Remember file name (to ease update handling)
        runcmd('bash -c "echo \'%s\' > %s/larch/kernelname"'
                % (self.kname, self.medium_dir))

        # if no saved system.sqf, squash the Arch installation at self.installation_dir
        if not os.path.isfile(self.system_sqf):
            comment("Generating system.sqf")
            # root directories which are not included in the squashed system.sqf
            #ignoredirs = "boot dev mnt media proc sys tmp .livesys "
            ignoredirs = "dev mnt media proc sys tmp .livesys "
            ignoredirs += CHROOT_DIR_BUILD.lstrip("/")
            # /var stuff
            ignoredirs += " var/log var/tmp var/lock"
            # others
            ignoredirs += " usr/lib/locale"

            # Additional directories to ignore can also be specified in the
            # profile. This is a nasty option. It was requested, and might
            # be useful under certain special circumstances, but I recommend
            # not using it unless you are really sure what you are doing.
            veto_file = self.profile_dir + '/vetodirs'
            if os.path.isfile(veto_file):
                fh = open(veto_file)
                for line in fh:
                    line = line.strip()
                    if line and (line[0] != '#'):
                        ignoredirs += ' ' + line.lstrip('/')
                fh.close()

            if not chroot(self.installation0,
                    "/sbin/mksquashfs '/' '%s' -e %s"
                    % (CHROOT_SYSTEMSQF, ignoredirs),
                    filter=mksquashfs_filter_gen()):
                errout(_("Squashing system.sqf failed"))
            # remove execute attrib
            runcmd("chmod oga-x %s" % self.system_sqf)

        # move system.sqf to medium directory
        runcmd("mv %s %s/larch" % (self.system_sqf, self.medium_dir))

        # prepare overlay
        comment("Generating larch overlay")
        # Copy over the overlay from the selected profile
        if os.path.isdir("%s/rootoverlay" % self.profile_dir):
            runcmd('bash -c "cp -rf %s/rootoverlay/* %s"'
                    % (self.profile_dir, self.overlay))
        # Ensure there is an /etc directory in the overlay
        runcmd("mkdir -p %s/etc" % self.overlay)
        # fix sudoers if any
        if os.path.isfile("%s/etc/sudoers" % self.overlay):
            runcmd("chmod 0440  %s/etc/sudoers" % self.overlay)
            runcmd("chown root:root %s/etc/sudoers" % self.overlay)

        # Prepare inittab
        inittab = self.overlay + "/etc/inittab"
        itsave = inittab + ".larchsave"
        it0 = self.installation0 + "/etc/inittab"
        itl = self.overlay + "/etc/inittab.larch"
        if not os.path.isfile(itl):
            itl = self.installation0 + "/etc/inittab.larch"
            if not os.path.isfile(itl):
                itl = None
        # Save the original inittab if there is an inittab.larch file,
        #   ... if there isn't already a saved one
        if itl:
            if ((not os.path.isfile(it0 + ".larchsave"))
                    and (not os.path.isfile(itsave))):
                runcmd("cp %s %s" % (it0, itsave))
            # Use the .larch version in the live system
            runcmd("cp -f %s %s" % (itl, inittab))

        comment("Generating larch initcpio")
        if not self.gen_initramfs():
            return False

        lpath = self.locales_base + '/locale'
        if self.installation0:
            if options.oldlocales and os.path.isdir(lpath):
                comment("Copying saved glibc locales")
                runcmd('rm -rf %s/usr/lib/locale' % self.overlay)
                runcmd('mkdir -p %s/usr/lib' % self.overlay)
                runcmd('cp -a %s %s/usr/lib' % (lpath, self.overlay))
            else:
                comment("Generating glibc locales")
                runcmd('rm -rf %s' % lpath)
                script('larch-locales "%s" "%s"' % (self.installation0,
                        self.overlay))
                # Save the generated locales for possible reuse
                runcmd('cp -a %s/usr/lib/locale %s' % (self.overlay,
                        self.locales_base))

        if (os.path.isfile(self.installation0 + '/usr/bin/ssh-keygen')
                and not os.path.isfile(self.profile_dir + '/nosshkeys')):
            # ssh initialisation - done here so that it doesn't need to
            # be done when the live system boots
            comment("Generating ssh keys to overlay")
            sshdir = CHROOT_DIR_OVERLAY + "/etc/ssh"
            runcmd("mkdir -p %s" % (self.installation0 + sshdir))
            for k, f in [("rsa1", "ssh_host_key"), ("rsa", "ssh_host_rsa_key"),
                    ("dsa", "ssh_host_dsa_key")]:
                chroot(self.installation0,
                        "ssh-keygen -t %s -N '' -f %s/%s"
                        % (k, sshdir, f), ["dev"])

        # Ensure the hostname is in /etc/hosts
        script("larch-hosts %s %s" % (self.installation0, self.overlay))

        # Handle /mnt
        runcmd("mkdir -p %s/mnt" % self.overlay)
        for d in os.listdir("%s/mnt" % self.installation0):
            if os.path.isdir("%s/mnt/%s" % (self.installation0, d)):
                runcmd("mkdir %s/mnt/%s" % (self.overlay, d))

        # Ensure there is a /boot directory
        runcmd("mkdir -p %s/boot" % self.overlay)

        # Run customization script
        tweak = self.profile_dir + '/build-tweak'
        if os.path.isfile(tweak):
            comment("(WARNING): Running user's build customization script")
            if runcmd(tweak + ' %s %s' % (self.installation0,
                    self.overlay))[0]:
                comment("Customization script completed")
            else:
                errout(_("Build customization script failed"))

        # Get root password
        rootpwf = self.profile_dir + '/rootpw'
        if os.path.isfile(rootpwf):
            rootpw = readfile(rootpwf).strip()
            if rootpw == '!':
                # Lock the password
                rootcmd = 'usermod -L'
            else:
                rootcmd = "usermod -p '%s'" % encryptPW(rootpw)
        else:
            rootcmd = None

        # Add users and set root password
        if self.installation0 and not self.add_users(rootcmd):
            return False

        comment("Squashing mods.sqf")
        if not chroot(self.installation0,
                "/sbin/mksquashfs '%s' '%s/larch/mods.sqf'"
                % (CHROOT_DIR_OVERLAY, CHROOT_DIR_MEDIUM),
                filter=mksquashfs_filter_gen()):
            errout(_("Squashing mods.sqf failed"))
        # remove execute attrib
        runcmd("chmod oga-x %s/larch/mods.sqf" % self.medium_dir)

        runcmd("rm -rf %s" % self.overlay)

        comment(" *** %s ***" % _("larchify-process completed"))
        return True


    def add_users(self, rootcmd):
        userinfo = Userinfo(self.profile_dir)
        userlist = []
        for user in userinfo.allusers():
            # Only include if the user does not yet exist
            if runcmd('bash -c "grep \"^%s\" %s/etc/passwd || echo -"'
                    % (user, self.installation_dir))[1][0] != '-':
                comment("(WARNING): User '%s' exists already" % user)
            else:
                userlist.append(user)

        # Only continue if there are new users in the list
        if rootcmd:
            clist = [('root', rootcmd + ' %s')]
        else:
            if userlist == []:
                return True
            clist = []

        # Save system files and replace them by the overlay versions
        savedir = self.larchify_dir + '/save_etc'
        runcmd('rm -rf %s' % savedir)
        runcmd('mkdir -p %s/default' % savedir)
        savelist = 'group,gshadow,passwd,shadow,login.defs,skel'
        runcmd('bash -c "cp -a %s/etc/{%s} %s"'
                % (self.installation0, savelist, savedir))
        runcmd('cp -a %s/etc/default/useradd %s/default'
                % (self.installation0, savedir))
        for f in ('group', 'gshadow', 'passwd', 'shadow', 'login.defs'):
            if os.path.isfile(self.overlay + '/etc/%s'):
                runcmd('cp %s/etc/%s %s/etc'
                        % (self.overlay, f, self.installation0))
        if os.path.isfile(self.overlay + '/etc/default/useradd'):
            runcmd('cp %s/etc/default/useradd %s/etc/default'
                    % (self.overlay, self.installation0))
        if os.path.isdir(self.overlay + '/etc/skel'):
            runcmd('cp -r %s/etc/skel %s/etc'
                    % (self.overlay, self.installation0))

        # Build the useradd command
        userdir0 = '/users'
        userdir = self.larchify_dir + userdir0
        userdirs = []
        runcmd('mkdir -p %s/home' % self.overlay)
        for u in userlist:
            cline = 'useradd -m'
            pgroup = userinfo.get(u, 'maingroup')
            if pgroup:
                cline += ' -g ' + pgroup
            uid = userinfo.get(u, 'uid')
            if uid:
                cline += ' -u ' + uid
            pw = userinfo.get(u, 'pw')
            if (pw == ''):
                # Passwordless login
                pwcrypt = ''
            else:
                # Normal MD5 password
                pwcrypt = encryptPW(pw)
            cline += " -p '%s'" % pwcrypt
            skeldir = userinfo.get(u, 'skel')
            if skeldir:
                # Custom home initialization directories in the profile
                # always start with 'skel_'
                skel = 'skel_' + skeldir
                if skel not in userdirs:
                    userdirs.append(skel)
                cline += ' -k %s/%s' % (CHROOT_DIR_LARCHIFY + userdir0,
                        skel)
            # Allow for expert tweaking
            cline += ' ' + userinfo.get(u, 'expert')
            # The user and the command to be run
            clist.append((u, cline + ' %s'))
            xgroups = userinfo.get(u, 'xgroups')
            if xgroups:
                xgl = []
                for g in xgroups.split(','):
                    clist.append((u, 'usermod -a -G %s %%s' % g))

        if userdirs:
            # Copy custom 'skel' directories to build space
            runcmd('rm -rf %s' % userdir)
            runcmd('mkdir -p %s' % userdir)
            for ud in userdirs:
                runcmd('cp -r %s/%s %s/%s' %
                        (self.profile_dir, ud, userdir, ud))

        nfail = 0
        ok = True
        for u, cmd in clist:
            if not chroot(self.installation0, cmd % u):
                nfail += 1
                # Errors adding users to groups are not fatal:
                if not cmd.startswith('usermod -a -G'):
                    ok = False
            if os.path.isdir('%s/home/%s' % (self.installation0, u)):
                runcmd('mv %s/home/%s %s/home'
                        % (self.installation0, u, self.overlay))

        if nfail > 0:
            errout(_("%d user account operation(s) failed") % nfail, 0)
        # Move changed /etc/{group,gshadow,passwd,shadow} to overlay
        runcmd('bash -c "mv %s/etc/{group,gshadow,passwd,shadow} %s/etc"'
                % (self.installation0, self.overlay))
        # Restore system files in base installation
        runcmd('rm -rf %s/etc/skel' % self.installation0)
        runcmd('bash -c "cp -a %s/* %s/etc"'
                % (savedir, self.installation0))
        return ok


    def system_check(self):
        comment("Testing for necessary packages and kernel modules")
        fail = ""
        warn = ""
        nplist = ["larch-live"]

        mdep = (self.installation0 +
                "/lib/modules/%s/modules.dep" % self.kversion)
        if Popen(["grep", "/squashfs.ko", mdep], stdout=PIPE,
                stderr=STDOUT).wait() != 0:
            fail += _("No squashfs module found\n")

        if Popen(["grep", "/aufs.ko", mdep], stdout=PIPE,
                stderr=STDOUT).wait() == 0:
            self.ufs='_aufs'
            nplist.append("aufs2-util")

        elif Popen(["grep", "/unionfs.ko", mdep], stdout=PIPE,
                stderr=STDOUT).wait() == 0:
            self.ufs='_unionfs'

        else:
            fail += _("No aufs or unionfs module found\n")

        for p in nplist:
            if not self.haspack(p):
                fail += _("Package '%s' is needed by larch systems\n") % p

        if not self.haspack("syslinux"):
            warn += _("Without package 'syslinux' you will not be able\n"
                    "to create syslinux or isolinux booting media\n")

        if (not self.haspack("cdrkit")) and (not self.haspack("cdrtools")):
            warn += _("Without package 'cdrkit' (or 'cdrtools') you will\n"
                    "not be able to create CD/DVD media\n")

        if not self.haspack("eject"):
            warn += _("Without package 'eject' you will have problems\n"
                    "using CD/DVD media\n")

        if warn:
            cont = query_yn(_("WARNING:\n%s"
                    "\n    Continue building?") % warn)
        else:
            cont = True

        if fail:
            errout(_("ERROR:\n%s") % fail)

        return cont


    def haspack(self, package):
        """Check whether the given package is installed.
        """
        for p in os.listdir(self.installation0 + '/var/lib/pacman/local'):
            if p.rsplit("-", 2)[0] == package:
                return True
        return False


    def find_kernel(self):
        # The uncomfortable length of this function is deceptive,
        # most of it is for dealing with errors.
        comment("Seeking kernel information")
        kscript = "%s/kernel" % self.profile_dir
        if os.path.isfile(kscript):
            p = Popen([kscript], stdout=PIPE, stderr=STDOUT)
            r = p.communicate()[0]
            if p.returncode == 0:
                self.kname, self.kversion = r.split()

            else:
                errout(_("Problem running %s:\n  %s") % (kscript, r))
        else:
            kernels = glob(self.installation0 + '/boot/vmlinuz*')
            if len(kernels) > 1:
                errout(_("More than one kernel found:\n  %s") %
                        "\n  ".join(kernels))
            elif not kernels:
                errout(_("No kernel found"))
            self.kname = os.path.basename(kernels[0])

            self.kversion = None
            for kv in os.listdir(self.installation0 + '/lib/modules'):
                if os.path.isfile(self.installation0
                        + ('/lib/modules/%s/modules.dep' % kv)):
                    if self.kversion:
                        errout(_("More than one set of kernel modules in %s")
                                % (self.installation0 + '/lib/modules'))
                    self.kversion = kv
                else:
                    kmpath = self.installation0 + ('/lib/modules/%s' % kv)
                    comment("Unexpected kernel files at %s" % kmpath)
                    # Try to find packages concerned
                    p = Popen(["find", ".", "-name", "*.ko"], cwd=kmpath,
                            stdout=PIPE, stderr=STDOUT)
                    r = p.communicate()[0]
                    if p.returncode == 0:
                        packs = []
                        for km in r.split():
                            a = chroot(self.installation0,
                                    'pacman -Qoq /lib/modules/%s/%s'
                                    % (kv, km))

                            if a:
                                pack = "-".join(a[0].split())
                                if pack not in packs:
                                    packs.append(pack)
                                    comment(" Package: %s" % pack)

                    else:
                        comment("Couldn't determine guilty packages")

                    if not query_yn(_("WARNING:"
                            "\n  You seem to have installed a package containing modules"
                            "\nwhich aren't compatible with your kernel (see log)."
                            "\nPlease check that this won't cause problems."
                            "\nMaybe you need the corresponding package for your kernel?"
                            "\n\n    Continue building?")):
                        return False

        if not self.kversion:
            errout(_("Couldn't find kernel modules"))

        comment("Kernel: %s   -   version: %s" % (self.kname, self.kversion))
        chroot(self.installation0, "depmod %s" % self.kversion)
        return True


    def gen_initramfs(self):
        # Fix up larch mkinitcpio.conf for unionfs/aufs
        conf = self.overlay + "/etc/mkinitcpio.conf.larch"
        if os.path.isfile(conf + "0"):
            conf0 = conf + "0"
        else:
            conf0 = self.installation0 + "/etc/mkinitcpio.conf.larch0"
        runcmd('bash -c "sed \'s|___aufs___|%s|g\' <%s >%s"' % (self.ufs, conf0, conf))

        presets = [os.path.basename(f) for f in glob(
                self.installation0 + "/etc/mkinitcpio.d/kernel26*.preset")]
        if len(presets) != 1:
            errout(_("Couldn't find usable mkinitcpio preset: %s") %
                    self.installation0 + "/etc/mkinitcpio.d/kernel26*.preset")

        # Save original preset file (unless a '*.larchsave' is already present)
        idir = self.installation0 + "/etc/mkinitcpio.d"
        oldir = self.overlay + "/etc/mkinitcpio.d"
        if not os.path.isfile("%s/%s.larchsave" % (idir, presets[0])):
            runcmd("mkdir -p %s" % oldir)
            runcmd("cp %s/%s %s/%s.larchsave" %
                    (idir, presets[0], oldir, presets[0]))

        # Adjust larch.preset file for custom kernels
        runcmd('bash -c "sed \'s|___|%s|\' <%s/larch.preset0 >%s/larch.preset"'
                % (presets[0].rsplit(".", 1)[0], idir, oldir))

        # Replace 'normal' preset in overlay
        runcmd("cp %s/larch.preset %s/%s" % (oldir, oldir, presets[0]))

        # Generate initramfs
        return chroot(self.installation0,
                "mkinitcpio -k %s -c %s -g %s" %
                (self.kversion,
                 CHROOT_DIR_OVERLAY + "/etc/mkinitcpio.conf.larch",
                 CHROOT_DIR_MEDIUM + "/boot/larch.img"))


def encryptPW(pw):
    """Encrypt a password - needed for user account generation.
    """
    salt = '$1$'
    for i in range(8):
        salt += random.choice("./0123456789abcdefghijklmnopqrstuvwxyz"
                "ABCDEFGHIJKLMNOPQRSTUVWXYZ")
    return crypt.crypt(pw, salt)



if __name__ == "__main__":
    start_translator()

    from optparse import OptionParser, OptionGroup
    parser = OptionParser(usage=_("usage: %prog [options]"))

    parser.add_option("-p", "--profile", action="store", type="string",
            default="", dest="profile",
            help=_("Profile: 'user:profile-name' or path to profile directory"))
    parser.add_option("-i", "--installation-dir", action="store", type="string",
            default="", dest="idir",
            help=_("Path to directory to be larchified (default %s)")
                    % INSTALLATION)
    parser.add_option("-s", "--slave", action="store_true", dest="slave",
            default=False, help=_("Run as a slave from a controlling program"
                    " (e.g. from a gui)"))
    parser.add_option("-q", "--quiet", action="store_true", dest="quiet",
            default=False, help=_("Suppress output messages, except errors"
                    " (no effect if -s specified)"))
    parser.add_option("-o", "--oldsqf", action="store_true", dest="oldsqf",
            default=False, help=_("Reuse previously generated system.sqf"))
    parser.add_option("-l", "--oldlocales", action="store_true",
            dest="oldlocales", default=False,
            help=_("Reuse previously generated locales"))
    parser.add_option("-f", "--force", action="store_true", dest="force",
            default=False, help=_("Don't ask for confirmation"))

    (options, args) = parser.parse_args()
# Should there be arguments?

    if os.getuid() != 0:
        print _("This application must be run as root")
        sys.exit(1)
    init('larchify', options)
    builder = Builder(options)
    if builder.build(options):
        sys.exit(0)
    else:
        sys.exit(1)