Add script to construct a new helper VM image

The scripts runs debootstrap and creates a new disk image

host/snf-image-update-helper.in

+#!/bin/bash
+
+set -e
+set -x
+
+. @osdir@/common.sh
+
+CACHE_FILE="$HELPER_DIR/cache.tar"
+ARCH=amd64
+EXTRA_PKGS="linux-image-amd64,e2fsprogs,ntfs-3g,xmlstarlet"
+
+helper_img=$(mktemp $HELPER_DIR/image.XXXXXXXX)
+
+dd if=/dev/zero of=$helper_img bs=1k count=400000
+blockdev=$(losetup -sf $helper_img)
+CLEANUP+=("losetup -d $blockdev")
+
+format_disk0 $blockdev "extdump"
+
+root_dev=$(map_disk0 $blockdev)-1
+CLEANUP+=("unmap_disk0 $blockdev")
+
+mkfs.ext3 $root_dev
+
+TMPDIR=$(mktemp -d)
+CLEANUP+=("rmdir $TMPDIR")
+
+mount $root_dev $TMPDIR
+CLEANUP+=("umount $root_dev")
+
+if [  -f "$CACHE_FILE" ]; then
+    tar xf "$CACHE_FILE" -C $TMPDIR
+else
+    debootstrap --arch "$ARCH" --include $EXTRA_PKGS --variant=minbase \
+	    squeeze $TMPDIR
+
+    # remove the downloaded debs, as they are no longer needed
+    find "$TMPDIR/var/cache/apt/archives" -type f -name '*.deb' -print0 | \
+        xargs -r0 rm -f
+
+    TMP_CACHE=$(mktemp "${CACHE_FILE}.XXXXXX")
+    tar cf "$TMP_CACHE" -C $TMPDIR .
+    mv -f "$TMP_CACHE" "$CACHE_FILE"
+fi
+
+echo helper > $TMPDIR/etc/hostname
+
+cat > $TMPDIR/etc/fstab <<EOF
+# /etc/fstab: static file system information.
+#
+# <file system>   <mount point>   <type>  <options>       <dump>  <pass>
+/dev/sda1         /               ext3    defaults        0       1
+proc              /proc           proc    defaults        0       0
+EOF
+
+if [ ! -L "$TMPDIR/vmlinuz" -o ! -L "$TMPDIR/vmlinuz" ]; then
+
+	log_error "vmlinuz or initrd.img link in root is missing."
+	log_error "I don't know how to find a usable kernel/initrd pair."
+	exit 1
+fi
+
+kernel=$(readlink -en $TMPDIR/vmlinuz)
+initrd=$(readlink -en $TMPDIR/initrd.img)
+
+echo "Moving $(basename $kernel) and $(basename $initrd) to $HELPER_DIR"
+mv $kernel $initrd $HELPER_DIR
+
+kernel=$(basename $kernel)
+initrd=$(basename $initrd)
+
+(cd $HELPER_DIR; ln -fs $kernel kernel; ln -fs $initrd initrd)
+
+rm $TMPDIR/vmlinuz $TMPDIR/initrd.img
+
+cleanup
+
+mv $helper_img $HELPER_DIR/image
+
+trap - EXIT
+
+exit 0
+
+# vim: set sta sts=4 shiftwidth=4 sw=4 et ai :

snf-image-host/Makefile.am

@@ -15,6 +15,8 @@ dist_os_SCRIPTS = ${srcdir}/create ${srcdir}/import ${srcdir}/export \
 dist_os_DATA = ${srcdir}/ganeti_api_version ${srcdir}/parameters.list \
                ${srcdir}/variants.list
 
+dist_bin_SCRIPTS = snf-image-update-helper
+
 os_DATA = common.sh
 
 edit = sed \
@@ -23,9 +25,11 @@ edit = sed \
 	   -e 's|@osdir[@]|$(osdir)|g' \
 	   -e 's|@osname[@]|$(osname)|g' \
 	   -e 's|@defaultdir[@]|$(defaultdir)|g' \
-	   -e 's|@HELPER[@]|$(HELPER)|g' \
+	   -e 's|@HELPER_DIR[@]|$(HELPER_DIR)|g' \
+	   -e 's|@HELPER_IMG[@]|$(HELPER_IMG)|g' \
 	   -e 's|@HELPER_KERNEL[@]|$(HELPER_KERNEL)|g' \
 	   -e 's|@HELPER_INITRD[@]|$(HELPER_INITRD)|g' \
+	   -e 's|@HELPER_PKG[@]|$(HELPER_PKG)|g' \
 	   -e 's|@AWK[@]|$(AWK)|g' \
 	   -e 's|@MKDIR_P[@]|$(MKDIR_P)|g' \
 	   -e 's|@LOSETUP[@]|$(LOSETUP)|g' \
@@ -35,7 +39,7 @@ edit = sed \
 	   -e 's|@progress_monitor_support[@]|$(progress_monitor_support)|g' \
 	   -e 's|@PROGRESS_MONITOR[@]|$(PROGRESS_MONITOR)|g'
 
-common.sh: Makefile
+%:%.in Makefile
 	rm -f $@ $@.tmp
 	srcdir=''; \
 		   test -f ./$@.in || srcdir=$(srcdir)/; \
@@ -48,4 +52,4 @@ install-exec-local:
 	touch "$(DESTDIR)$(variantsdir)/default.conf"
 	
 
-CLEANFILES = $(os_DATA)
+CLEANFILES = $(os_DATA) $(dist_bin_SCRIPTS)

snf-image-host/common.sh.in

@@ -183,14 +183,77 @@ create_floppy() {
     umount $target
 }
 
+# this one is only to be called by create
+ganeti_os_main() {
+    if [ -z "$OS_API_VERSION" -o "$OS_API_VERSION" = "5" ]; then
+        OS_API_VERSION=5
+        GETOPT_RESULT=`getopt -o o:n:i:b:s: -n '$0' -- "$@"`
+        if [ $? != 0 ] ; then log_error "Terminating..."; exit 1 ; fi
+        get_api5_arguments $GETOPT_RESULT
+    elif [ "$OS_API_VERSION" = "10" -o "$OS_API_VERSION" = "15" ]; then
+        get_api10_arguments
+    elif [ "$OS_API_VERSION" = "20" ]; then
+        get_api20_arguments
+        IMAGE_NAME=$IMG_ID
+        IMAGE_TYPE=$IMG_FORMAT
+    else
+        log_error "Unknown OS API VERSION $OS_API_VERSION"
+        exit 1
+    fi
+    
+    if [ -n "$OS_VARIANT" ]; then
+        if [ ! -d "$VARIANTS_DIR" ]; then
+            log_error "OS Variants directory $VARIANTS_DIR doesn't exist"
+            exit 1
+        fi
+        VARIANT_CONFIG="$VARIANTS_DIR/$OS_VARIANT.conf"
+        if [ -f "$VARIANT_CONFIG" ]; then
+            . "$VARIANT_CONFIG"
+        else
+            if grep -qxF "$OS_VARIANT" variants.list; then
+                log_error "ERROR: instance-image configuration error"
+                log_error "  Published variant $OS_VARIANT is missing its config" \
+                          "file"
+                log_error "  Please create $VARIANT_CONFIG or unpublish the variant"
+                log_error "  (by removing $OS_VARIANT from variants.list)"
+            else
+                log_error "Unofficial variant $OS_VARIANT is unsupported"
+                log_error "Most probably this is a user error, forcing a wrong name"
+                log_error "To support this variant please create file" \
+                            "$VARIANT_CONFIG"
+            fi
+            exit 1
+        fi
+    fi
+
+}
+
 cleanup() {
+# if something fails here, it souldn't call cleanup again...
+    trap - EXIT
     if [ ${#CLEANUP[*]} -gt 0 ]; then
         LAST_ELEMENT=$((${#CLEANUP[*]}-1))
         REVERSE_INDEXES=$(seq ${LAST_ELEMENT} -1 0)
         for i in $REVERSE_INDEXES; do
-            ${CLEANUP[$i]}
+            # If something fails here, it's better to retry it for a few times
+            # before we give up with an error. This is needed for kpartx when
+            # dealing with ntfs partitions mounted through fuse. umount is not
+            # synchronous and may return while the partition is still busy. A
+            # premature attempt to delete partition mappings through kpartx on a
+            # device that hosts previously mounted ntfs partition may fail with
+            # an  `device-mapper: remove ioctl failed: Device or resource busy'
+            # error. A sensible workaround for this is to wait for a while and
+            # then try again.
+            local cmd=${CLEANUP[$i]}
+            $cmd || for interval in 0.25 0.5 1 2 4; do
+            echo "Command $cmd failed!"
+            echo "I'll wait for $interval secs and will retry..."
+            sleep $interval
+            $cmd && break
         done
-    fi
+        test $? -eq 1 && { echo "Giving Up..."; exit 1; }
+    done
+  fi
 }
 
 trap cleanup EXIT
@@ -203,12 +266,13 @@ fi
 : ${ARCH:="x86_64"}
 : ${VARIANTS_DIR:="@sysconfdir@/ganeti/snf-image/variants"}
 : ${IMAGE_DIR:="@localstatedir@/lib/snf-image"}
-: ${HELPER:="@HELPER@"}
+: ${HELPER_DIR:="@HELPER_DIR@"}
+: ${HELPER_IMG:="@HELPER_IMG@"}
 : ${HELPER_KERNEL:="@HELPER_KERNEL@"}
 : ${HELPER_INITRD:="@HELPER_INITRD@"}
 : ${HELPER_SOFT_TIMEOUT=120}
 : ${HELPER_HARD_TIMEOUT=15}
-: ${HELPER_USR="nobody"}
+: ${HELPER_USER="nobody"}
 
 SCRIPT_NAME=$(basename $0)
 
@@ -229,46 +293,4 @@ if [ -z "$VOL_ID" ]; then
     exit 1
 fi
 
-
-if [ -z "$OS_API_VERSION" -o "$OS_API_VERSION" = "5" ]; then
-    OS_API_VERSION=5
-    GETOPT_RESULT=`getopt -o o:n:i:b:s: -n '$0' -- "$@"`
-    if [ $? != 0 ] ; then log_error "Terminating..."; exit 1 ; fi
-    get_api5_arguments $GETOPT_RESULT
-elif [ "$OS_API_VERSION" = "10" -o "$OS_API_VERSION" = "15" ]; then
-    get_api10_arguments
-elif [ "$OS_API_VERSION" = "20" ]; then
-    get_api20_arguments
-    IMAGE_NAME=$IMG_ID
-    IMAGE_TYPE=$IMG_FORMAT
-else
-    log_error "Unknown OS API VERSION $OS_API_VERSION"
-    exit 1
-fi
-
-if [ -n "$OS_VARIANT" ]; then
-    if [ ! -d "$VARIANTS_DIR" ]; then
-        log_error "OS Variants directory $VARIANTS_DIR doesn't exist"
-        exit 1
-    fi
-    VARIANT_CONFIG="$VARIANTS_DIR/$OS_VARIANT.conf"
-    if [ -f "$VARIANT_CONFIG" ]; then
-        . "$VARIANT_CONFIG"
-    else
-        if grep -qxF "$OS_VARIANT" variants.list; then
-            log_error "ERROR: instance-image configuration error"
-            log_error "  Published variant $OS_VARIANT is missing its config" \
-                      "file"
-            log_error "  Please create $VARIANT_CONFIG or unpublish the variant"
-            log_error "  (by removing $OS_VARIANT from variants.list)"
-        else
-            log_error "Unofficial variant $OS_VARIANT is unsupported"
-            log_error "Most probably this is a user error, forcing a wrong name"
-            log_error "To support this variant please create file" \
-                        "$VARIANT_CONFIG"
-        fi
-        exit 1
-    fi
-fi
-
 # vim: set sta sts=4 shiftwidth=4 sw=4 et ai :

snf-image-host/configure.ac

@@ -25,33 +25,41 @@ AC_ARG_WITH([progress-monitor],
 
 AM_CONDITIONAL(PROGMONSUPPORT, [test -n "$PROGRESS_MONITOR"])
 
-# --with-helper..
-AC_ARG_WITH([helper],
-    [AS_HELP_STRING([--with-helper=IMG_PATH],
-        [Path to helper VM image [LOCALSTATEDIR/lib/snf-image/helper.img]]
+# --with-helper-dir
+AC_ARG_WITH([helper-dir],
+    [AS_HELP_STRING([--with-helper-dir=DIR],
+        [top-level directory to host the helper VM
+        [LOCALSTATEDIR/lib/snf-image/helper]]
     )],
-    [helper="$withval"],
-    [helper="$localstatedir/lib/snf-image/helper.img"])
-AC_SUBST(HELPER, $helper)
+    [helper_dir="$withval"],
+    [helper_dir="$localstatedir/lib/snf-image/helper"])
+AC_SUBST(HELPER_DIR, $helper_dir)
+
+# --with-helper-img
+AC_ARG_WITH([helper-img],
+    [AS_HELP_STRING([--with-helper-img=IMG_PATH],
+        [Path to helper VM image [HELPERDIR/image]]
+    )],
+    [helper_img="$withval"],
+    [helper_img="$helper_dir/image"])
+AC_SUBST(HELPER_IMG, $helper_img)
 
 # --with-helper-kernel
-AC_ARG_WITH([helper_kernel],
+AC_ARG_WITH([helper-kernel],
     [AS_HELP_STRING([--with-helper-kernel=KERNEL_PATH],
-        [Path to a kernel to use to boot the helper VM image 
-        [LOCALSTATEDIR/lib/snf-image/helper-kernel]]
+        [Path to the helper VM kernel [HELPERDIR/kernel]]
     )],
     [helper_ernel="$withval"],
-    [helper_kernel="$localstatedir/lib/snf-image/helper-kernel"])
+    [helper_kernel="$helper_dir/kernel"])
 AC_SUBST(HELPER_KERNEL, ${helper_kernel})
 
 # --with-helper-initrd..
-AC_ARG_WITH([helper_initrd],
+AC_ARG_WITH([helper-initrd],
     [AS_HELP_STRING([--with-helper-initrd=INITRD_PATH],
-        [Path to an initial ramdisk to use to boot the helper VM image
-        [LOCALSTATEDIR/lib/snf-image/helper-initrd]]
+        [Path to the helper VM initial ramdist [HELPERDIR/initrd]]
     )],
     [helper_initrd="$withval"],
-    [helper_initrd="$localstatedir/lib/snf-image/helper-initrd"])
+    [helper_initrd="$helper_dir/initrd"])
 AC_SUBST(HELPER_INITRD, ${helper_initrd})
 
 # --with-os-dir=...

snf-image-host/create

@@ -33,6 +33,8 @@ set -e
 
 . common.sh
 
+ganeti_os_main
+
 case "$IMAGE_TYPE" in
     extdump)
 	IMAGE_FILE="${IMAGE_DIR}/${IMAGE_NAME}-${ARCH}.extdump";;
@@ -94,7 +96,7 @@ result_file=$(mktemp --tmpdir result.XXXXXXXX)
 CLEANUP+=("rm -f $result_file")
 
 $TIMELIMIT -t $HELPER_SOFT_TIMEOUT -T $HELPER_HARD_TIMEOUT \
-    kvm -runas $HELPER_USR -drive file=${HELPER},snapshot=on \
+    kvm -runas $HELPER_USER -drive file=${HELPER_IMG},snapshot=on \
     -drive file=$root_dev,format=raw,if=virtio,cache=none \
     -boot c -serial stdio -serial file:$result_file -fda $floppy \
     -vga none -nographic -parallel none -monitor null -nographic \

snf-image-host/defaults

@@ -17,14 +17,20 @@
 # IMAGE_DEBUG: turn on debugging output for the scripts
 # IMAGE_DEBUG=0
 
-# HELPER: path to the image file of the helper vm
-# HELPER="var/lib/snf-image/helper.img"
+# HELPER_DIR: Directory hosting the helper files
+# HELPER_DIR: /var/lib/snf-image/helper/
 
-# HELPER_KERNEL: path to the kernel, kvm should use to run the helper image
-# HELPER_KERNEL="/var/lib/snf-image/helper-kernel"
+# HELPER_IMG: Path to the helper VM image
+# HELPER_IMG="$(HELPER_DIR)/image"
 
-# HELPER_INITRD: path to the init ramdisk the helper vm should use.
-# HELPER_INITRD="/var/lib/snf-image/helper-initrd"
+# HELPER_KERNEL: Path to the helper VM kernel
+# HELPER_KERNEL="$(HELPER_DIR)/kernel"
+
+# HELPER_PKG: Path to the snf-image-helper package
+# HELPER_PKG="$(HELPER_DIR)/snf-image-helper.deb"
+
+# HELPER_INITRD: Path to the helper VM initial ramdisk
+# HELPER_INITRD="$(HELPER_DIR)/initrd"
 
 # HELPER_TIMOUT: Soft and hard timeout limits for helper instance.
 # Helper instance will be terminated after a given time if it hasn't exited by
@@ -36,7 +42,7 @@
 # HELPER_USR: For security reasons, the helper vm is not adviced to to run as
 # root. It should drop privileges just before the guest execution starts. The
 # user the helper vm should run as is specifies by HELPER_USR variable.
-# HELPER_USR="nobody"
+# HELPER_USER="nobody"
 
 # Paths for needed programs. Uncommend and change the variables below, if you
 # don't want to use the default one.