Commit 72052003 authored by Dimitris Aragiorgis's avatar Dimitris Aragiorgis
Browse files

Xen support

Support launching helper domain on top of Xen hypervisor under PV mode.

Helper VM becomes hypervisor aware by passing hypervisor=$HYPERVISOR
in the kernel command line and thus acts accordingly (which logging
devices to use, what disk/floppy devices to expect, etc)

We separate all hypervisor specific functions and vars and source them
according to $HYPERVISOR env var.

Due to the fact that multiconsole for PV domains is supported only after
linux kernel 3.2 (Stefano Stabellini tree - branch 3.2-multiconsole-2
git://xenbits.xen.org/people/sstabellini/linux-pvhvm.git

) all
output/error report and logging is done via hvc0.

Helper domain is debootstraped with 2 kernels (linux-image-amd64,
linux-image-xen-amd64) and is booted accordingly.

Floppy is not supported in PV so floppy is passed as another disk.

The images created can run on top of xen-hvm and xen-pvm hypervisor.

Change logging to start with HELPER_MONITOR_ in order to parse it when
having only one console.
Signed-off-by: default avatarDimitris Aragiorgis <dimara@grnet.gr>
parent eafa1728
......@@ -16,10 +16,6 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
RESULT=/dev/ttyS1
MONITOR=/dev/ttyS2
FLOPPY_DEV=/dev/fd0
PROGNAME=$(basename $0)
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
......@@ -45,6 +41,38 @@ MSG_TYPE_TASK_END="TASK_END"
STDERR_LINE_SIZE=10
# hypervisor param is passed in /proc/cmdline
case "$hypervisor" in
kvm)
FLOPPY_DEV=/dev/fd0
ROOT_FSTAB_ENTRY=/dev/sda1
IMG_DEV=/dev/vda
RESULT=/dev/ttyS1
MONITOR=/dev/ttyS2
;;
xen-hvm|xen-pvm)
FLOPPY_DEV=/dev/xvdc1
ROOT_FSTAB_ENTRY=/dev/xvda1
IMG_DEV=/dev/xvdb
RESULT=/dev/hvc0
MONITOR=/dev/hvc0
;;
esac
to_monitor() {
echo "HELPER_MONITOR_$@" > "$MONITOR"
}
to_result() {
echo "HELPER_RESULT_$@" > "$RESULT"
}
add_cleanup() {
local cmd=""
for arg; do cmd+=$(printf "%q " "$arg"); done
......@@ -53,33 +81,36 @@ add_cleanup() {
log_error() {
ERRORS+=("$@")
echo "ERROR: $@" | tee $RESULT >&2
echo "ERROR: $@" >&2
to_result "ERROR: $@"
exit 1
}
warn() {
echo "Warning: $@" >&2
echo "WARNING:$@" > "$MONITOR"
to_monitor "WARNING:$@"
}
report_task_start() {
echo "$MSG_TYPE_TASK_START:${PROGNAME:2}" > "$MONITOR"
to_monitor "$MSG_TYPE_TASK_START:${PROGNAME:2}"
}
report_task_end() {
echo "$MSG_TYPE_TASK_END:${PROGNAME:2}" > "$MONITOR"
to_monitor "$MSG_TYPE_TASK_END:${PROGNAME:2}"
}
report_error() {
if [ ${#ERRORS[*]} -eq 0 ]; then
# No error message. Print stderr
local lines=$(tail --lines=${STDERR_LINE_SIZE} "$STDERR_FILE" | wc -l)
echo -n "STDERR:${lines}:" > "$MONITOR"
tail --lines=$lines "$STDERR_FILE" > "$MONITOR"
lines="$(tail --lines=${STDERR_LINE_SIZE} "$STDERR_FILE")"
cnt=$(echo $lines | wc -l)
for line in lines; do
to_monitor "$STDERR:$cnt: $line"
let cnt--
done
else
echo -n "ERROR:" > "$MONITOR"
for line in "${ERRORS[@]}"; do
echo "$line" > "$MONITOR"
to_monitor "ERROR: $line"
done
fi
}
......
......@@ -12,6 +12,7 @@ variantsdir=${sysconfdir}/ganeti/snf-image/variants
dist_os_SCRIPTS = ${srcdir}/create ${srcdir}/import ${srcdir}/export \
${srcdir}/rename ${srcdir}/verify ${srcdir}/pithcat \
${srcdir}/copy-monitor.py ${srcdir}/helper-monitor.py \
${srcdir}/xen-common.sh ${srcdir}/kvm-common.sh \
${srcdir}/host-monitor.py
dist_os_DATA = ${srcdir}/ganeti_api_version ${srcdir}/parameters.list \
......
......@@ -37,6 +37,7 @@ MSG_TYPE_INFO="image-info"
CLEANUP=( )
ERROR_MSGS=( )
add_cleanup() {
local cmd=""
for arg; do cmd+=$(printf "%q " "$arg"); done
......@@ -387,13 +388,20 @@ fi
: ${HELPER_USER:="nobody"}
: ${HELPER_CACHE_FILE:="@HELPER_DIR@/cache.tar"}
: ${HELPER_CACHE_PKGS:="@HELPER_DIR@/packages"}
: ${HELPER_EXTRA_PKGS:="linux-image-amd64,e2fsprogs,ntfs-3g,ntfsprogs,xmlstarlet,python,parted,reglookup,chntpw,util-linux"}
: ${HELPER_EXTRA_PKGS:="e2fsprogs,ntfs-3g,ntfsprogs,xmlstarlet,python,parted,reglookup,chntpw,util-linux"}
: ${HELPER_LINUX_IMAGE_PKG:="linux-image-amd64"}
: ${HELPER_LINUX_IMAGE_XEN_PKG:="linux-image-xen-amd64"}
: ${HELPER_MIRROR:=""}
: ${PITHOS_DB:="sqlite:////@localstatedir@/lib/pithos/backend.db"}
: ${PITHOS_DATA:="@localstatedir@/lib/pithos/data/"}
: ${PROGRESS_MONITOR:="@PROGRESS_MONITOR@"}
: ${UNATTEND:="@UNATTEND@"}
case $HYPERVISOR in
xen-hvm|xen-pvm) . xen-common.sh ;;
kvm) . kvm-common.sh ;;
esac
SCRIPT_NAME=$(basename $0)
# vim: set sta sts=4 shiftwidth=4 sw=4 et ai :
......@@ -131,7 +131,6 @@ log_info "Image copy finished."
floppy=$(mktemp --tmpdir floppy.XXXXXX)
add_cleanup rm "$floppy"
snf_export_DEV=/dev/vda
snf_export_TYPE="$IMG_FORMAT"
snf_export_PASSWORD="$IMG_PASSWD"
snf_export_HOSTNAME="$instance"
......@@ -148,7 +147,7 @@ create_floppy "$floppy"
jail=$(mktemp -d --tmpdir tmpfsXXXXXXX)
add_cleanup rmdir "$jail"
mount tmpfs -t tmpfs "$jail" -o size=50M
mount tmpfs -t tmpfs "$jail" -o size=1G
add_cleanup umount "$jail"
result_file=$(mktemp --tmpdir="$jail" result.XXXXXX)
......@@ -157,20 +156,14 @@ add_cleanup rm "$result_file"
snapshot=$(mktemp --tmpdir="$jail" helperXXXXXX.img)
add_cleanup rm "$snapshot"
"$QEMU_IMG" create -f qcow2 -b "$HELPER_IMG" "$snapshot"
mk_snapshot
echo -n "$(date +%Y:%m:%d-%H:%M:%S.%N) " >&2
log_info "Starting customization VM..."
set +e
$TIMEOUT -k "$HELPER_HARD_TIMEOUT" "$HELPER_SOFT_TIMEOUT" \
kvm -runas "$HELPER_USER" -drive file="$snapshot" \
-drive file="$blockdev",format=raw,if=virtio,cache=none \
-boot c -serial stdio -serial "file:$(printf "%q" "$result_file")" \
-serial file:>(./helper-monitor.py ${MONITOR_FD}) \
-fda "$floppy" -vga none -nographic -parallel none -monitor null \
-kernel "$HELPER_KERNEL" -initrd "$HELPER_INITRD" \
-append "quiet ro root=/dev/sda1 console=ttyS0,9600n8 snf_image_activate_helper" \
2>&1 | sed -u 's|^|HELPER: |g'
launch_helper
rc=$?
set -e
if [ $rc -ne 0 ]; then
......@@ -193,8 +186,7 @@ else
log_info "Customization VM finished."
fi
# Read the first line. This will remove \r and \n chars
result=$(sed 's|\r||g' "$result_file" | head -1)
get_helper_result
if [ "x$result" != "xSUCCESS" ]; then
log_error "Image customization failed."
......
......@@ -53,7 +53,9 @@
# HELPER_EXTRA_PKGS: Extra packages that will need to be supplied
# to debootstrap to make the resulting helper image workable.
# DO NOT OVERWRITE THIS UNLESS YOU KNOW WHAT YOU ARE DOING
# HELPER_EXTRA_PKGS="linux-image-amd64,e2fsprogs,ntfs-3g,ntfsprogs,xmlstarlet,python,parted,reglookup,chntpw,util-linux"
# HELPER_EXTRA_PKGS="e2fsprogs,ntfs-3g,ntfsprogs,xmlstarlet,python,parted,reglookup,chntpw,util-linux"
# HELPER_LINUX_IMAGE_PKG="linux-image-amd64"
# HELPER_LINUX_XEN_IMAGE_PKG="linux-image-xen-amd64"
# HELPER_MIRROR: Debian mirror to use with debootstrap. Using a mirror close to
# you will speed up the whole debootstraping process. By default we do not
......
......@@ -23,19 +23,20 @@ import time
import json
import re
LINESIZE = 512
BUFSIZE = 512
# add HELPER_MONITOR_
LINESIZE = 512+16
BUFSIZE = 512+16
PROGNAME = os.path.basename(sys.argv[0])
STDERR_MAXLINES = 10
MAXLINES = 100
MSG_TYPE = 'image-helper'
PROTOCOL = {
'TASK_START': ('task-start', 'task'),
'TASK_END': ('task-end', 'task'),
'WARNING': ('warning', 'messages'),
'STDERR': ('error', 'stderr'),
'ERROR': ('error', 'messages')}
'HELPER_MONITOR_TASK_START': ('task-start', 'task'),
'HELPER_MONITOR_TASK_END': ('task-end', 'task'),
'HELPER_MONITOR_WARNING': ('warning', 'messages'),
'HELPER_MONITOR_STDERR': ('error', 'stderr'),
'HELPER_MONITOR_ERROR': ('error', 'messages')}
def error(msg):
......@@ -104,7 +105,7 @@ if __name__ == "__main__":
stderr += "%s\n" % line
lines_left -= 1
if lines_left == 0:
send(fd, "STDERR", stderr)
send(fd, "HELPER_MONITROR_STDERR", stderr)
stderr = ""
line = ""
continue
......@@ -113,8 +114,8 @@ if __name__ == "__main__":
if len(line) == 0:
continue
if line.startswith("STDERR:"):
m = re.match("STDERR:(\d+):(.*)", line)
if line.startswith("HELPER_MONITOR_STDERR:"):
m = re.match("HELPER_MONITOR_STDERR:(\d+):(.*)", line)
if not m:
error("Invalid syntax for STDERR line")
try:
......@@ -132,15 +133,16 @@ if __name__ == "__main__":
lines_left -= 1
if lines_left == 0:
send(fd, "STDERR", stderr)
send(fd, "HELPER_MONITOR_STDERR", stderr)
stderr = ""
elif line.startswith("TASK_START:") \
or line.startswith("TASK_END:") \
or line.startswith("WARNING:") \
or line.startswith("ERROR:"):
elif line.startswith("HELPER_MONITOR_TASK_START:") \
or line.startswith("HELPER_MONITOR_TASK_END:") \
or line.startswith("HELPER_MONITOR_WARNING:") \
or line.startswith("HELPER_MONITOR_ERROR:"):
(msg_type, _, value) = line.partition(':')
if line.startswith("WARNING:") or line.startswith("ERROR:"):
if line.startswith("HELPER_MONITOR_WARNING:") \
or line.startswith("HELPER_MONITOR_ERROR:"):
value = [value]
send(fd, msg_type, value)
else:
......
snf_export_DEV=/dev/vda
mk_snapshot() {
"$QEMU_IMG" create -f qcow2 -b "$HELPER_IMG" "$snapshot"
}
launch_helper() {
$TIMEOUT -k "$HELPER_HARD_TIMEOUT" "$HELPER_SOFT_TIMEOUT" \
kvm -runas "$HELPER_USER" -drive file="$snapshot" \
-drive file="$blockdev",format=raw,if=virtio,cache=none \
-boot c -serial stdio -serial "file:$(printf "%q" "$result_file")" \
-serial file:>(./helper-monitor.py ${MONITOR_FD}) \
-fda "$floppy" -vga none -nographic -parallel none -monitor null \
-kernel "$HELPER_KERNEL" -initrd "$HELPER_INITRD" \
-append "quiet ro root=/dev/sda1 console=ttyS0,9600n8 \
hypervisor=$HYPERVISOR snf_image_activate_helper" \
2>&1 | sed -u 's|^|HELPER: |g'
}
get_helper_result() {
result=$(sed 's|\r||g' "$result_file" | head -1)
}
......@@ -123,7 +123,7 @@ rm -f "$HELPER_DIR/initrd" "$HELPER_DIR/kernel" "$HELPER_DIR/image"
echo -n "Allocating space for helper disk image..."
helper_img=$(mktemp "$HELPER_DIR/image.XXXXXX")
dd if=/dev/zero of="$helper_img" bs=1k count=400000 &> /dev/null
dd if=/dev/zero of="$helper_img" bs=1k count=800000 &> /dev/null
echo "done"
echo "Creating partitions..."
......@@ -142,6 +142,8 @@ mkfs.ext3 "$root_dev" 2>&1 | sed -e 's/^/MKFS.EXT3: /g'
# The helper vm should never do filesystem checks...
tune2fs -i 0 -c 0 "$root_dev" 2>&1 | sed -e 's/^/TUNE2FS: /g'
blkid=$(blkid -s UUID -o value $root_dev)
target=$(mktemp -d)
add_cleanup rmdir "$target"
......@@ -210,32 +212,52 @@ cat > "$target/etc/fstab" <<EOF
# /etc/fstab: static file system information.
#
# <file system> <mount point> <type> <options> <dump> <pass>
/dev/sda1 / ext3 defaults 0 1
UUID=$blkid / ext3 defaults 0 1
proc /proc proc defaults 0 0
EOF
echo "done"
echo -n "Extracting kernel..."
if [ ! -L "$target/vmlinuz" -o ! -L "$target/vmlinuz" ]; then
echo -e "\033[1;31mfailed\033[0m"
log_error "vmlinuz or initrd.img link in root is missing."
log_error "I don't know how to find a usable kernel/initrd pair."
exit 1
fi
echo "done"
kernel=$(readlink -en "$target/vmlinuz")
initrd=$(readlink -en "$target/initrd.img")
echo "Moving $(basename "$kernel") and $(basename "$initrd") to \`$HELPER_DIR'"
mv "$kernel" "$initrd" "$HELPER_DIR"
kernel=$(basename "$kernel")
initrd=$(basename "$initrd")
setup_image() {
# setup_image pkg suffix
echo "Getting linux image $1"
chroot "$target" apt-get install -y --force-yes -qq "$1" &>/dev/null
echo -n "Extracting kernel..."
if [ ! -L "$target/vmlinuz" -o ! -L "$target/vmlinuz" ]; then
echo -e "\033[1;31mfailed\033[0m"
log_error "vmlinuz or initrd.img link in root is missing."
log_error "I don't know how to find a usable kernel/initrd pair."
exit 1
fi
echo "done"
kernel=$(readlink -en "$target/vmlinuz")
initrd=$(readlink -en "$target/initrd.img")
echo "Moving $(basename "$kernel") and $(basename "$initrd") to \`$HELPER_DIR'"
mv "$kernel" "$initrd" "$HELPER_DIR"
kernel=$(basename "$kernel")
initrd=$(basename "$initrd")
(
cd "$HELPER_DIR";
ln -fs "$kernel" kernel$2;
ln -fs "$initrd" initrd$2;
)
(cd "$HELPER_DIR"; ln -fs "$kernel" kernel; ln -fs "$initrd" initrd)
rm "$target/vmlinuz" "$target/initrd.img"
}
setup_image $HELPER_LINUX_IMAGE_PKG ""
setup_image $HELPER_LINUX_IMAGE_XEN_PKG "-xen"
rm "$target/vmlinuz" "$target/initrd.img"
echo "Installing snf-image-helper pkg in the new image..."
cp "$HELPER_PKG" "$target/tmp/"
......@@ -261,6 +283,12 @@ if ! grep -q snf_image_activate_helper /proc/cmdline; then
echo "WARNING: NOT calling snf-image-helper, add snf_image_activate_helper"
echo "to the kernel command line if you want to do so."
else
for x in \$(cat /proc/cmdline); do
case x in
hypervisor*) eval \$x
;;
esac
done
/usr/bin/snf-image-helper --force
fi
......
snf_export_DEV=/dev/xvdb
HELPER_KERNEL=${HELPER_KERNEL}-xen
HELPER_INITRD=${HELPER_INITRD}-xen
mk_snapshot() {
cp "$HELPER_IMG" "$snapshot"
}
launch_helper() {
tail -f --pid=$$ "$result_file" | sed -u 's|^|HELPER: |' &
$TIMEOUT -k $HELPER_HARD_TIMEOUT $HELPER_SOFT_TIMEOUT \
screen -D -m -c /etc/screenrc bash -c 'xm create /dev/null \
kernel="'$HELPER_KERNEL'" ramdisk="'$HELPER_INITRD'" \
disk="file:'$snapshot',xvda,w" \
disk="phy:'$blockdev',xvdb,w" \
disk="file:'$floppy',xvdc1,w" \
extra="console=hvc0 hypervisor='$HYPERVISOR' snf_image_activate_helper" \
memory="256" root="/dev/xvda1 quiet ro boot=local" boot="c" vcpus=1 \
name="snf-image-helper" -c > '$result_file''
}
get_helper_result() {
result=$(sed 's|\r||g' "$result_file" | grep ^SUCCESS$)
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment