diff --git a/share/composer/live-iso.ks b/share/composer/live-iso.ks new file mode 100644 index 000000000..f56ecc8aa --- /dev/null +++ b/share/composer/live-iso.ks @@ -0,0 +1,371 @@ +# Lorax Composer Live ISO output kickstart template + +# +sshpw --username=root --plaintext randOmStrinGhERE +# Firewall configuration +firewall --enabled --service=mdns + +# X Window System configuration information +xconfig --startxonboot +# Root password +rootpw --plaintext removethispw +# Network information +network --bootproto=dhcp --onboot=on --activate +# System authorization information +auth --useshadow --enablemd5 +# System keyboard +keyboard --xlayouts=us --vckeymap=us +# System language +lang en_US.UTF-8 +# SELinux configuration +selinux --enforcing +# Installation logging level +logging --level=info +# Shutdown after installation +shutdown +# System services +services --disabled="network,sshd" --enabled="NetworkManager" +# System timezone +timezone US/Eastern +# System bootloader configuration +bootloader --location=mbr +# Clear the Master Boot Record +zerombr +# Partition clearing information +clearpart --all +# Disk partitioning information +part biosboot --size=1 +part / --fstype="ext4" --size=5000 +part swap --size=1000 + +%post +# FIXME: it'd be better to get this installed from a package +cat > /etc/rc.d/init.d/livesys << EOF +#!/bin/bash +# +# live: Init script for live image +# +# chkconfig: 345 00 99 +# description: Init script for live image. + +. /etc/init.d/functions + +if ! strstr "\`cat /proc/cmdline\`" rd.live.image || [ "\$1" != "start" ]; then + exit 0 +fi + +if [ -e /.liveimg-configured ] ; then + configdone=1 +fi + +exists() { + which \$1 >/dev/null 2>&1 || return + \$* +} + +touch /.liveimg-configured + +# mount live image +if [ -b \`readlink -f /dev/live\` ]; then + mkdir -p /mnt/live + mount -o ro /dev/live /mnt/live 2>/dev/null || mount /dev/live /mnt/live +fi + +livedir="LiveOS" +for arg in \`cat /proc/cmdline\` ; do + if [ "\${arg##live_dir=}" != "\${arg}" ]; then + livedir=\${arg##live_dir=} + return + fi +done + +# enable swaps unless requested otherwise +swaps=\`blkid -t TYPE=swap -o device\` +if ! strstr "\`cat /proc/cmdline\`" noswap && [ -n "\$swaps" ] ; then + for s in \$swaps ; do + action "Enabling swap partition \$s" swapon \$s + done +fi +if ! strstr "\`cat /proc/cmdline\`" noswap && [ -f /mnt/live/\${livedir}/swap.img ] ; then + action "Enabling swap file" swapon /mnt/live/\${livedir}/swap.img +fi + +mountPersistentHome() { + # support label/uuid + if [ "\${homedev##LABEL=}" != "\${homedev}" -o "\${homedev##UUID=}" != "\${homedev}" ]; then + homedev=\`/sbin/blkid -o device -t "\$homedev"\` + fi + + # if we're given a file rather than a blockdev, loopback it + if [ "\${homedev##mtd}" != "\${homedev}" ]; then + # mtd devs don't have a block device but get magic-mounted with -t jffs2 + mountopts="-t jffs2" + elif [ ! -b "\$homedev" ]; then + loopdev=\`losetup -f\` + if [ "\${homedev##/mnt/live}" != "\${homedev}" ]; then + action "Remounting live store r/w" mount -o remount,rw /mnt/live + fi + losetup \$loopdev \$homedev + homedev=\$loopdev + fi + + # if it's encrypted, we need to unlock it + if [ "\$(/sbin/blkid -s TYPE -o value \$homedev 2>/dev/null)" = "crypto_LUKS" ]; then + echo + echo "Setting up encrypted /home device" + plymouth ask-for-password --command="cryptsetup luksOpen \$homedev EncHome" + homedev=/dev/mapper/EncHome + fi + + # and finally do the mount + mount \$mountopts \$homedev /home + # if we have /home under what's passed for persistent home, then + # we should make that the real /home. useful for mtd device on olpc + if [ -d /home/home ]; then mount --bind /home/home /home ; fi + [ -x /sbin/restorecon ] && /sbin/restorecon /home + if [ -d /home/liveuser ]; then USERADDARGS="-M" ; fi +} + +findPersistentHome() { + for arg in \`cat /proc/cmdline\` ; do + if [ "\${arg##persistenthome=}" != "\${arg}" ]; then + homedev=\${arg##persistenthome=} + return + fi + done +} + +if strstr "\`cat /proc/cmdline\`" persistenthome= ; then + findPersistentHome +elif [ -e /mnt/live/\${livedir}/home.img ]; then + homedev=/mnt/live/\${livedir}/home.img +fi + +# if we have a persistent /home, then we want to go ahead and mount it +if ! strstr "\`cat /proc/cmdline\`" nopersistenthome && [ -n "\$homedev" ] ; then + action "Mounting persistent /home" mountPersistentHome +fi + +# make it so that we don't do writing to the overlay for things which +# are just tmpdirs/caches +mount -t tmpfs -o mode=0755 varcacheyum /var/cache/yum +mount -t tmpfs tmp /tmp +mount -t tmpfs vartmp /var/tmp +[ -x /sbin/restorecon ] && /sbin/restorecon /var/cache/yum /tmp /var/tmp >/dev/null 2>&1 + +if [ -n "\$configdone" ]; then + exit 0 +fi + +# add fedora user with no passwd +action "Adding live user" useradd \$USERADDARGS -c "Live System User" liveuser +passwd -d liveuser > /dev/null + +# turn off firstboot for livecd boots +chkconfig --level 345 firstboot off 2>/dev/null +# We made firstboot a native systemd service, so it can no longer be turned +# off with chkconfig. It should be possible to turn it off with systemctl, but +# that doesn't work right either. For now, this is good enough: the firstboot +# service will start up, but this tells it not to run firstboot. I suspect the +# other services 'disabled' below are not actually getting disabled properly, +# with systemd, but we can look into that later. - AdamW 2010/08 F14Alpha +echo "RUN_FIRSTBOOT=NO" > /etc/sysconfig/firstboot + +# don't start yum-updatesd for livecd boots +chkconfig --level 345 yum-updatesd off 2>/dev/null + +# turn off mdmonitor by default +chkconfig --level 345 mdmonitor off 2>/dev/null + +# turn off setroubleshoot on the live image to preserve resources +chkconfig --level 345 setroubleshoot off 2>/dev/null + +# don't do packagekit checking by default +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t string /apps/gnome-packagekit/update-icon/frequency_get_updates never >/dev/null +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t string /apps/gnome-packagekit/update-icon/frequency_get_upgrades never >/dev/null +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t string /apps/gnome-packagekit/update-icon/frequency_refresh_cache never >/dev/null +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t bool /apps/gnome-packagekit/update-icon/notify_available false >/dev/null +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t bool /apps/gnome-packagekit/update-icon/notify_distro_upgrades false >/dev/null +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t bool /apps/gnome-packagekit/enable_check_firmware false >/dev/null +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t bool /apps/gnome-packagekit/enable_check_hardware false >/dev/null +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t bool /apps/gnome-packagekit/enable_codec_helper false >/dev/null +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t bool /apps/gnome-packagekit/enable_font_helper false >/dev/null +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t bool /apps/gnome-packagekit/enable_mime_type_helper false >/dev/null + + +# don't start cron/at as they tend to spawn things which are +# disk intensive that are painful on a live image +chkconfig --level 345 crond off 2>/dev/null +chkconfig --level 345 atd off 2>/dev/null +chkconfig --level 345 anacron off 2>/dev/null +chkconfig --level 345 readahead_early off 2>/dev/null +chkconfig --level 345 readahead_later off 2>/dev/null + +# Stopgap fix for RH #217966; should be fixed in HAL instead +touch /media/.hal-mtab + +# workaround clock syncing on shutdown that we don't want (#297421) +sed -i -e 's/hwclock/no-such-hwclock/g' /etc/rc.d/init.d/halt + +# and hack so that we eject the cd on shutdown if we're using a CD... +if strstr "\`cat /proc/cmdline\`" CDLABEL= ; then + cat >> /sbin/halt.local << FOE +#!/bin/bash +# XXX: This often gets stuck during shutdown because /etc/init.d/halt +# (or something else still running) wants to read files from the block\ +# device that was ejected. Disable for now. Bug #531924 +# we want to eject the cd on halt, but let's also try to avoid +# io errors due to not being able to get files... +#cat /sbin/halt > /dev/null +#cat /sbin/reboot > /dev/null +#/usr/sbin/eject -p -m \$(readlink -f /dev/live) >/dev/null 2>&1 +#echo "Please remove the CD from your drive and press Enter to finish restarting" +#read -t 30 < /dev/console +FOE +chmod +x /sbin/halt.local +fi + +EOF + +# bah, hal starts way too late +cat > /etc/rc.d/init.d/livesys-late << EOF +#!/bin/bash +# +# live: Late init script for live image +# +# chkconfig: 345 99 01 +# description: Late init script for live image. + +. /etc/init.d/functions + +if ! strstr "\`cat /proc/cmdline\`" rd.live.image || [ "\$1" != "start" ] || [ -e /.liveimg-late-configured ] ; then + exit 0 +fi + +exists() { + which \$1 >/dev/null 2>&1 || return + \$* +} + +touch /.liveimg-late-configured + +# read some variables out of /proc/cmdline +for o in \`cat /proc/cmdline\` ; do + case \$o in + ks=*) + ks="--kickstart=\${o#ks=}" + ;; + xdriver=*) + xdriver="\${o#xdriver=}" + ;; + esac +done + +# if liveinst or textinst is given, start anaconda +if strstr "\`cat /proc/cmdline\`" liveinst ; then + plymouth --quit + /usr/sbin/liveinst \$ks +fi +if strstr "\`cat /proc/cmdline\`" textinst ; then + plymouth --quit + /usr/sbin/liveinst --text \$ks +fi + +# configure X, allowing user to override xdriver +if [ -n "\$xdriver" ]; then + cat > /etc/X11/xorg.conf.d/00-xdriver.conf <> /etc/rc.d/init.d/livesys << EOF +# disable screensaver locking +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t bool /apps/gnome-screensaver/lock_enabled false >/dev/null +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t bool /desktop/gnome/lockdown/disable_lock_screen true >/dev/null + +# set up timed auto-login for after 60 seconds +cat >> /etc/gdm/custom.conf << FOE +[daemon] +AutomaticLoginEnable=True +AutomaticLogin=liveuser +FOE + +# Show harddisk install on the desktop +sed -i -e 's/NoDisplay=true/NoDisplay=false/' /usr/share/applications/liveinst.desktop +mkdir /home/liveuser/Desktop +cp /usr/share/applications/liveinst.desktop /home/liveuser/Desktop +chown -R liveuser.liveuser /home/liveuser/Desktop +chmod a+x /home/liveuser/Desktop/liveinst.desktop + +# But not trash and home +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t bool /apps/nautilus/desktop/trash_icon_visible false >/dev/null +gconftool-2 --direct --config-source=xml:readwrite:/etc/gconf/gconf.xml.defaults -s -t bool /apps/nautilus/desktop/home_icon_visible false >/dev/null + +# Turn off PackageKit-command-not-found while uninstalled +sed -i -e 's/^SoftwareSourceSearch=true/SoftwareSourceSearch=false/' /etc/PackageKit/CommandNotFound.conf + +EOF + +# Remove root password +passwd -d root > /dev/null + +# fstab from the install won't match anything. remove it and let dracut +# handle mounting. +cat /dev/null > /etc/fstab + +%end + +# NOTE Do NOT add any other sections after %packages +%packages +# Packages requires to support this output format go here +isomd5sum +kernel +memtest86+ +syslinux +-dracut-config-rescue + +# This package is needed to boot the iso on UEFI +shim +shim-ia32 +grub2 +grub2-efi +grub2-efi-*-cdboot +grub2-efi-ia32 +efibootmgr + + +# NOTE lorax-composer will add the recipe packages below here, including the final %end%packages diff --git a/src/pylorax/api/compose.py b/src/pylorax/api/compose.py index 25ad0da09..bbfe7b5ad 100644 --- a/src/pylorax/api/compose.py +++ b/src/pylorax/api/compose.py @@ -39,6 +39,8 @@ import shutil from uuid import uuid4 +from pyanaconda.simpleconfig import SimpleConfigFile + from pylorax.api.projects import projects_depsolve, dep_nevra from pylorax.api.projects import ProjectsError from pylorax.api.recipes import read_recipe_and_id @@ -178,18 +180,26 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m # Setup the config to pass to novirt_install log_dir = joinpaths(results_dir, "logs/") cfg_args = compose_args(compose_type) + + # Get the title, project, and release version from the host + if not os.path.exists("/etc/os-release"): + log.error("/etc/os-release is missing, cannot determine product or release version") + os_release = SimpleConfigFile("/etc/os-release") + os_release.read() + + log.debug("os_release = %s", os_release) + + cfg_args["title"] = os_release.get("PRETTY_NAME") + cfg_args["project"] = os_release.get("NAME") + cfg_args["releasever"] = os_release.get("VERSION_ID") + cfg_args["volid"] = "" + cfg_args.update({ "compression": "xz", - #"compress_args": ["-9"], "compress_args": [], "ks": [ks_path], - "anaconda_args": "", - "proxy": "", - "armplatform": "", - "project": "Red Hat Enterprise Linux", "releasever": "7", - "logfile": log_dir }) with open(joinpaths(results_dir, "config.toml"), "w") as f: @@ -216,12 +226,73 @@ def compose_types(share_dir): return [os.path.basename(ks)[:-3] for ks in glob(joinpaths(share_dir, "composer/*.ks"))] def compose_args(compose_type): - """ Returns the settings to pass to novirt_install for the compose type""" - _MAP = {"tar": {"make_tar": True, - "make_iso": False, - "make_fsimage": False, - "qcow2": False, - "image_name": default_image_name("xz", "root.tar")}, - } + """ Returns the settings to pass to novirt_install for the compose type + :param compose_type: The type of compose to create, from `compose_types()` + :type compose_type: str + + This will return a dict of options that match the ArgumentParser options for livemedia-creator. + These are the ones the define the type of output, it's filename, etc. + Other options will be filled in by `make_compose()` + """ + _MAP = {"tar": {"make_iso": False, + "make_disk": False, + "make_fsimage": False, + "make_appliance": False, + "make_ami": False, + "make_tar": True, + "make_pxe_live": False, + "make_ostree_live": False, + "ostree": False, + "live_rootfs_keep_size": False, + "live_rootfs_size": 0, + "qcow2": False, + "qcow2_arg": [], + "image_name": default_image_name("xz", "root.tar"), + "image_only": True, + "app_name": None, + "app_template": None, + "app_file": None + }, + "live-iso": {"make_iso": True, + "make_disk": False, + "make_fsimage": False, + "make_appliance": False, + "make_ami": False, + "make_tar": False, + "make_pxe_live": False, + "make_ostree_live": False, + "ostree": False, + "live_rootfs_keep_size": False, + "live_rootfs_size": 0, + "qcow2": False, + "qcow2_arg": [], + "image_name": "live.iso", + "fs_label": "Anaconda", # Live booting may expect this to be 'Anaconda' + "image_only": False, + "app_name": None, + "app_template": None, + "app_file": None + }, + } return _MAP[compose_type] + +def move_compose_results(cfg, results_dir): + """Move the final image to the results_dir and cleanup the unneeded compose files + + :param cfg: Build configuration + :type cfg: DataHolder + :param results_dir: Directory to put the results into + :type results_dir: str + """ + if cfg["make_tar"]: + shutil.move(joinpaths(cfg["result_dir"], cfg["image_name"]), results_dir) + elif cfg["make_iso"]: + # Output from live iso is always a boot.iso under images/, move and rename it + shutil.move(joinpaths(cfg["result_dir"], "images/boot.iso"), joinpaths(results_dir, cfg["image_name"])) + + # Cleanup the compose directory, but only if it looks like a compose directory + if os.path.basename(cfg["result_dir"]) == "compose": + shutil.rmtree(cfg["result_dir"]) + else: + log.error("Incorrect compose directory, not cleaning up") diff --git a/src/pylorax/api/queue.py b/src/pylorax/api/queue.py index ecea6d178..875e0f35a 100644 --- a/src/pylorax/api/queue.py +++ b/src/pylorax/api/queue.py @@ -27,12 +27,11 @@ import subprocess from subprocess import Popen, PIPE import time -from pykickstart.version import makeVersion, RHEL7 -from pykickstart.parser import KickstartParser +from pylorax.api.compose import move_compose_results from pylorax.api.recipes import recipe_from_file from pylorax.base import DataHolder -from pylorax.installer import novirt_install +from pylorax.creator import run_creator from pylorax.sysutils import joinpaths def start_queue_monitor(cfg, uid, gid): @@ -47,7 +46,8 @@ def start_queue_monitor(cfg, uid, gid): :returns: None """ lib_dir = cfg.get("composer", "lib_dir") - monitor_cfg = DataHolder(composer_dir=lib_dir, uid=uid, gid=gid) + share_dir = cfg.get("composer", "share_dir") + monitor_cfg = DataHolder(composer_dir=lib_dir, share_dir=share_dir, uid=uid, gid=gid) p = mp.Process(target=monitor, args=(monitor_cfg,)) p.daemon = True p.start() @@ -56,7 +56,7 @@ def monitor(cfg): """Monitor the queue for new compose requests :param cfg: Configuration settings - :type cfg: ComposerConfig + :type cfg: DataHolder :returns: Does not return The queue has 2 subdirectories, new and run. When a compose is ready to be run @@ -106,15 +106,20 @@ def queue_sort(uuid): log.info("Finished building %s, results are in %s", dst, os.path.realpath(dst)) open(joinpaths(dst, "STATUS"), "w").write("FINISHED\n") except Exception as e: - log.error("Error running compose: %s", e) + import traceback + log.error("traceback: %s", traceback.format_exc()) + +# TODO - Write the error message to an ERROR-LOG file to include with the status +# log.error("Error running compose: %s", e) open(joinpaths(dst, "STATUS"), "w").write("FAILED\n") + os.unlink(dst) def make_compose(cfg, results_dir): """Run anaconda with the final-kickstart.ks from results_dir :param cfg: Configuration settings - :type cfg: ComposerConfig + :type cfg: DataHolder :param results_dir: The directory containing the metadata and results for the build :type results_dir: str :returns: Nothing @@ -129,7 +134,7 @@ def make_compose(cfg, results_dir): object. """ - # Check on the ks's presense + # Check on the ks's presence ks_path = joinpaths(results_dir, "final-kickstart.ks") if not os.path.exists(ks_path): raise RuntimeError("Missing kickstart file at %s" % ks_path) @@ -139,24 +144,38 @@ def make_compose(cfg, results_dir): if not os.path.exists(log_dir): os.makedirs(log_dir) - ks_version = makeVersion(RHEL7) - ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) - ks.readKickstart(ks_path) - # anaconda can only handle a url, it cannot use a mirrorlist or metalink for the primary repository. - if not ks.handler.method.url: - raise RuntimeError("The kickstart is missing a valid url line") - - repo_url = ks.handler.method.url - # Load the compose configuration cfg_path = joinpaths(results_dir, "config.toml") if not os.path.exists(cfg_path): raise RuntimeError("Missing config.toml for %s" % results_dir) cfg_dict = toml.loads(open(cfg_path, "r").read()) + # The keys in cfg_dict correspond to the arguments setup in livemedia-creator + # keys that define what to build should be setup in compose_args, and keys with + # defaults should be setup here. + # Make sure that image_name contains no path components cfg_dict["image_name"] = os.path.basename(cfg_dict["image_name"]) + # Only support novirt installation, set some other defaults + cfg_dict["no_virt"] = True + cfg_dict["disk_image"] = None + cfg_dict["fs_image"] = None + cfg_dict["keep_image"] = False + cfg_dict["domacboot"] = False + cfg_dict["anaconda_args"] = "" + cfg_dict["proxy"] = "" + cfg_dict["armplatform"] = "" + cfg_dict["squashfs_args"] = None + + cfg_dict["lorax_templates"] = cfg.share_dir + cfg_dict["tmp"] = "/var/tmp/" + cfg_dict["dracut_args"] = None # Use default args for dracut + + # Compose things in a temporary directory inside the results directory + cfg_dict["result_dir"] = joinpaths(results_dir, "compose") + os.makedirs(cfg_dict["result_dir"]) + install_cfg = DataHolder(**cfg_dict) # Some kludges for the 99-copy-logs %post, failure in it will crash the build @@ -167,7 +186,7 @@ def make_compose(cfg, results_dir): def cancel_build(): return os.path.exists(joinpaths(results_dir, "CANCEL")) - log.debug("repo_url = %s, cfg = %s", repo_url, install_cfg) + log.debug("cfg = %s", install_cfg) try: test_path = joinpaths(results_dir, "TEST") if os.path.exists(test_path): @@ -182,8 +201,10 @@ def cancel_build(): else: open(joinpaths(results_dir, install_cfg.image_name), "w").write("TEST IMAGE") else: - novirt_install(install_cfg, joinpaths(results_dir, install_cfg.image_name), None, repo_url, - callback_func=cancel_build) + run_creator(install_cfg, callback_func=cancel_build) + + # Extract the results of the compose into results_dir and cleanup the compose directory + move_compose_results(install_cfg, results_dir) finally: # Make sure that everything under the results directory is owned by the user user = pwd.getpwuid(cfg.uid).pw_name