Commit be739023 authored by Nikos Skalkotos's avatar Nikos Skalkotos
Browse files

Merge branch 'develop'

parents 5b2ee8c2 ffc64d7c
......@@ -16,6 +16,12 @@ itself.
Options
-------
-a URL, --authentication-url=URL
use this authentication URL when uploading/registering images
-c CLOUD, --cloud=CLOUD
use this saved cloud account to authenticate against a cloud when
uploading/registering images
--disable-sysprep=SYSPREP
prevent SYSPREP operation from running on the input media
......@@ -42,27 +48,26 @@ Options
dump image to FILE
--public
register image with cyclades as public
register image with the storage service as public
--print-sysprep
print the enabled and disabled system preparation operations for this
input media
-r IMAGENAME, --register=IMAGENAME
register the image with cyclades as IMAGENAME
register the image with the compute service with name IMAGENAME
-s, --silent
output only errors
-t TOKEN, --token=TOKEN
use this token when uploading/registering images to a Synnefo
deployment
use this token when uploading/registering images
--tmpdir=DIR
create large temporary image files under DIR
-u FILENAME, --upload=FILENAME
upload the image to pithos with name FILENAME
save the image to the storage service with remote name FILENAME
--version
show program's version number and exit
......
docs/snapshots/confirm.png

19.6 KB | W: | H:

docs/snapshots/confirm.png

19.1 KB | W: | H:

docs/snapshots/confirm.png
docs/snapshots/confirm.png
docs/snapshots/confirm.png
docs/snapshots/confirm.png
  • 2-up
  • Swipe
  • Onion skin
docs/snapshots/main_menu.png

18.3 KB | W: | H:

docs/snapshots/main_menu.png

23.8 KB | W: | H:

docs/snapshots/main_menu.png
docs/snapshots/main_menu.png
docs/snapshots/main_menu.png
docs/snapshots/main_menu.png
  • 2-up
  • Swipe
  • Onion skin
docs/snapshots/wizard.png

17.7 KB | W: | H:

docs/snapshots/wizard.png

17.6 KB | W: | H:

docs/snapshots/wizard.png
docs/snapshots/wizard.png
docs/snapshots/wizard.png
docs/snapshots/wizard.png
  • 2-up
  • Swipe
  • Onion skin
......@@ -17,42 +17,50 @@ snf-image-creator receives the following options:
.. code-block:: console
$ snf-image-creator --help
Usage: snf-image-creator [options] <input_media>
Options:
--version show program's version number and exit
-h, --help show this help message and exit
-o FILE, --outfile=FILE
dump image to FILE
-f, --force overwrite output files if they exist
-s, --silent output only errors
-u FILENAME, --upload=FILENAME
upload the image to pithos with name FILENAME
-r IMAGENAME, --register=IMAGENAME
register the image with ~okeanos as IMAGENAME
-m KEY=VALUE, --metadata=KEY=VALUE
add custom KEY=VALUE metadata to the image
-t TOKEN, --token=TOKEN
use this token when uploading/registering images
[Default: None]
--print-sysprep print the available enabled and disabled system
preparation operations for this input media
--enable-sysprep=SYSPREP
run SYSPREP operation on the input media
--disable-sysprep=SYSPREP
prevent SYSPREP operation from running on the input
media
--no-sysprep don't perform any system preparation operation
--no-shrink don't shrink the image
--public register image with cyclades as public
--tmpdir=DIR create large temporary image files under DIR
$ snf-image-creator --help
Usage: snf-image-creator [options] <input_media>
Options:
--version show program's version number and exit
-h, --help show this help message and exit
-o FILE, --outfile=FILE
dump image to FILE
-f, --force overwrite output files if they exist
-s, --silent output only errors
-u FILENAME, --upload=FILENAME
upload the image to the storage service with name FILENAME
-r IMAGENAME, --register=IMAGENAME
register the image with the compute service as IMAGENAME
-m KEY=VALUE, --metadata=KEY=VALUE
add custom KEY=VALUE metadata to the image
-t TOKEN, --token=TOKEN
use this authentication token when
uploading/registering images
-a URL, --authentication-url=URL
use this authentication URL when uploading/registering
images
-c CLOUD, --cloud=CLOUD
use this saved cloud account to authenticate against a
cloud when uploading/registering images
--print-sysprep print the enabled and disabled system preparation
operations for this input media
--enable-sysprep=SYSPREP
run SYSPREP operation on the input media
--disable-sysprep=SYSPREP
prevent SYSPREP operation from running on the input
media
--no-sysprep don't perform any system preparation operation
--no-shrink don't shrink any partition
--public register image with the compute service as public
--tmpdir=DIR create large temporary image files under DIR
Most input options are self-describing. If you want to save a local copy of
the image you create, provide a filename using the *-o* option. To upload the
image to *pithos+*, provide a valid authentication token using *-t* and a
filename using *-u*. If you also want to register the image with *~okeanos*, in
addition to *-u* provide a registration name using *-r*. All images are
image to the storage service of a cloud, provide valid cloud API access info
(by either using a token and a URL with *-t* and *-a* respectively, or a cloud
name with *-c*) and a remote filename using *-u*. If you also want to register
the image with the compute service of the cloud, in addition to *-u* provide a
registration name using *-r*. All images are
registered as *private*. Only the user that registers the image can create
VM's out of it. If you want the image to be visible by other user too, use the
*--public* option.
......@@ -75,61 +83,62 @@ debian system, we print the following output:
.. code-block:: console
$ snf-image-creator --print-sysprep debian_desktop.img
snf-image-creator 0.1
$ snf-image-creator --print-sysprep ubuntu.raw
snf-image-creator 0.3
=====================
Examining source media `debian_desktop.img'... looks like an image file
Snapshotting media source... done
Examining source media `ubuntu_hd.raw' ... looks like an image file
Snapshotting media source ... done
Enabling recovery proc
Launching helper VM... done
Inspecting Operating System... found a(n) debian system
Mounting the media read-only... done
Launching helper VM (may take a while) ... done
Inspecting Operating System ... ubuntu
Mounting the media read-only ... done
Collecting image metadata ... done
Umounting the media ... done
Enabled system preparation operations:
cleanup-cache:
Remove all regular files under /var/cache
Remove all regular files under /var/cache
cleanup-log:
Empty all files under /var/log
Empty all files under /var/log
cleanup-passwords:
Remove all passwords and lock all user accounts
Remove all passwords and lock all user accounts
cleanup-tmp:
Remove all files under /tmp and /var/tmp
Remove all files under /tmp and /var/tmp
cleanup-userdata:
Delete sensitive userdata
Delete sensitive userdata
fix-acpid:
Replace acpid powerdown action scripts to immediately shutdown the
system without checking if a GUI is running.
Replace acpid powerdown action scripts to immediately shutdown the
system without checking if a GUI is running.
remove-persistent-net-rules:
Remove udev rules that will keep network interface names persistent
after hardware changes and reboots. Those rules will be created again
the next time the image runs.
Remove udev rules that will keep network interface names persistent
after hardware changes and reboots. Those rules will be created again
the next time the image runs.
remove-swap-entry:
Remove swap entry from /etc/fstab. If swap is the last partition
then the partition will be removed when shrinking is performed. If the
swap partition is not the last partition in the disk or if you are not
going to shrink the image you should probably disable this.
Remove swap entry from /etc/fstab. If swap is the last partition
then the partition will be removed when shrinking is performed. If the
swap partition is not the last partition in the disk or if you are not
going to shrink the image you should probably disable this.
use-persistent-block-device-names:
Scan fstab & grub configuration files and replace all non-persistent
device references with UUIDs.
Scan fstab & grub configuration files and replace all non-persistent
device references with UUIDs.
Disabled system preparation operations:
cleanup-mail:
Remove all files under /var/mail and /var/spool/mail
Remove all files under /var/mail and /var/spool/mail
remove-user-accounts:
Remove all user accounts with id greater than 1000
cleaning up...
Remove all user accounts with id greater than 1000
cleaning up ...
If you want the image to have all normal user accounts and all mail files
removed, you should use *--enable-sysprep* option like this:
......@@ -173,15 +182,15 @@ Wizard mode
When *snf-mkimage* runs in *wizard* mode, the user is just asked to provide the
following basic information:
* Cloud: The cloud account to use to upload and register the resulting image
* Name: A short name for the image (ex. "Slackware")
* Description: An one-line description for the image
(ex. "Slackware Linux 14.0 with KDE")
* Registration Type: Private or Public
* Account: The authentication token for an *~okeanos* account
After confirming, the image will be extracted, uploaded to *pithos+* and
registered with *~okeanos*. The user will also be given the choice to keep a
local copy of it.
After confirming, the image will be extracted, uploaded to the storage service
and registered with the compute service of the selected cloud. The user will
also be given the choice to keep a local copy of it.
For most users the functionality this mode provides should be sufficient.
......@@ -202,11 +211,10 @@ In the *Customize* sub-menu the user can control:
In the *Register* sub-menu the user can provide:
* The credentials (authentication token) to use when authenticating
to *~okeanos*
* A *pithos+* filename for the uploaded *diskdump* image
* A name for the image to use when registering it with *~okeanos*, as well as
the registration type (*private* or *public*)
* Which cloud account to use
* A filename for the uploaded *diskdump* image
* A name for the image to use when registering it with the storage service of
the cloud, as well as the registration type (*private* or *public*)
By choosing the *Extract* menu entry, the user can dump the image to the local
file system. Finally, if the user selects *Reset*, the system will ignore
......@@ -244,13 +252,13 @@ Create a 2G sparse file to host the new system:
.. code-block:: console
$ truncate -s 2G ubuntu_hd.raw
$ truncate -s 2G ubuntu.raw
And install the Ubuntu system on this file:
.. code-block:: console
$ sudo kvm -boot d -drive file=ubuntu_hd.raw,format=raw,cache=none,if=virtio \
$ sudo kvm -boot d -drive file=ubuntu.raw,format=raw,cache=none,if=virtio \
-m 1G -cdrom ubuntu-12.04.2-server-amd64.iso
.. warning::
......@@ -261,7 +269,7 @@ And install the Ubuntu system on this file:
You will be able to boot your installed OS and make any changes you want
(e.g. install openssh-server) using the following command::
$ sudo kvm -m 1G -boot c -drive file=ubuntu_hd.raw,format=raw,cache=none,if=virtio
$ sudo kvm -m 1G -boot c -drive file=ubuntu.raw,format=raw,cache=none,if=virtio
After you're done, you may use *snf-mkimage* as root to create and upload the
image:
......@@ -269,20 +277,20 @@ image:
.. code-block:: console
$ sudo -s
$ snf-mkimage ubuntu_hd.raw
$ snf-mkimage ubuntu.raw
In the first screen you will be asked to choose if you want to run the program
in *Wizard* or *Expert* mode. Choose *Wizard*.
.. image:: /snapshots/wizard.png
Then you will be asked to provide a name, a description, a registration type
(*private* or *public*) and the authentication token corresponding to your
*~okeanos* account. Finally, you'll be asked to confirm the provided data.
Then you will be asked to select a cloud and provide a name, a description and
a registration type (*private* or *public*). Finally, you'll be asked to
confirm the provided data.
.. image:: /snapshots/confirm.png
Choosing *YES* will create and upload the image to your *~okeanos* account.
Choosing *YES* will create and upload the image to your cloud account.
Limitations
===========
......@@ -304,13 +312,13 @@ contain primary or logical partitions.
Para-virtualized drivers
------------------------
*~Okeanos* uses the *VirtIO* framework. The disk I/O controller and the
Ethernet cards on the VM instances are para-virtualized and need special
*VirtIO* drivers. Those drivers are included in the Linux Kernel mainline since
version 2.6.25 and are shipped with all the popular Linux distributions. The
problem is that if the driver for the para-virtualized disk I/O controller is
built as module, it needs to be preloaded using an initial ramdisk, otherwise
the VM won't be able to boot.
Most synnefo deployments uses the *VirtIO* framework. The disk I/O controller
and the Ethernet cards on the VM instances are para-virtualized and need
special *VirtIO* drivers. Those drivers are included in the Linux Kernel
mainline since version 2.6.25 and are shipped with all the popular Linux
distributions. The problem is that if the driver for the para-virtualized disk
I/O controller is built as module, it needs to be preloaded using an initial
ramdisk, otherwise the VM won't be able to boot.
Many popular Linux distributions, like Ubuntu and Debian, will automatically
create a generic initial ramdisk file that contains many different modules,
......
# -*- coding: utf-8 -*-
#
# Copyright 2012 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
......@@ -31,6 +33,10 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
"""Package for creating images to be used with Synnefo open source cloud
software.
"""
from image_creator.version import __version__
# vim: set sta sts=4 shiftwidth=4 sw=4 et ai :
# -*- coding: utf-8 -*-
#
# Copyright 2012 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
......@@ -31,6 +33,11 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
"""This module hosts the code that performes the host bundling operation. By
using the create_image method of the BundleVolume class the user can create an
image out of the running system.
"""
import os
import re
import tempfile
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2012 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
......@@ -33,6 +34,11 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
"""This module is the entrance point for the dialog-based version of the
snf-image-creator program. The main function will create a dialog where the
user is asked if he wants to use the program in expert or wizard mode.
"""
import dialog
import sys
import os
......@@ -40,6 +46,7 @@ import stat
import textwrap
import signal
import optparse
import types
from image_creator import __version__ as version
from image_creator.util import FatalError
......@@ -157,6 +164,39 @@ def select_file(d, media):
return media
def _dialog_form(self, text, height=20, width=60, form_height=15, fields=[],
**kwargs):
"""Display a form box.
fields is in the form: [(label1, item1, item_length1), ...]
"""
cmd = ["--form", text, str(height), str(width), str(form_height)]
label_len = 0
for field in fields:
if len(field[0]) > label_len:
label_len = len(field[0])
input_len = width - label_len - 1
line = 1
for field in fields:
label = field[0]
item = field[1]
item_len = field[2]
cmd.extend((label, str(line), str(1), item, str(line),
str(label_len + 1), str(input_len), str(item_len)))
line += 1
code, output = self._perform(*(cmd,), **kwargs)
if not output:
return (code, [])
return (code, output.splitlines())
def main():
d = dialog.Dialog(dialog="dialog")
......@@ -175,6 +215,10 @@ def main():
dialog._common_args_syntax["no_label"] = \
lambda string: ("--no-label", string)
# Monkey-patch pythondialog to include support for form dialog boxes
if not hasattr(dialog, 'form'):
d.form = types.MethodType(_dialog_form, d)
usage = "Usage: %prog [options] [<input_media>]"
parser = optparse.OptionParser(version=version, usage=usage)
parser.add_option("-l", "--logfile", type="string", dest="logfile",
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2012 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
......@@ -33,18 +33,23 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
"""This module implements the "expert" mode of the dialog-based version of
snf-image-creator.
"""
import os
import textwrap
import StringIO
import json
from image_creator import __version__ as version
from image_creator.util import MD5
from image_creator.util import MD5, FatalError
from image_creator.output.dialog import GaugeOutput, InfoBoxOutput
from image_creator.kamaki_wrapper import Kamaki, ClientError
from image_creator.help import get_help_file
from image_creator.dialog_util import SMALL_WIDTH, WIDTH, \
update_background_title, confirm_reset, confirm_exit, Reset, \
extract_image, extract_metadata_string
extract_image, extract_metadata_string, add_cloud, edit_cloud
CONFIGURATION_TASKS = [
("Partition table manipulation", ["FixPartitionTable"],
......@@ -108,15 +113,15 @@ class MetadataMonitor(object):
def upload_image(session):
"""Upload the image to pithos+"""
"""Upload the image to the storage service"""
d = session["dialog"]
image = session['image']
meta = session['metadata']
size = image.size
if "account" not in session:
d.msgbox("You need to provide your ~okeanos credentials before you "
"can upload images to pithos+", width=SMALL_WIDTH)
d.msgbox("You need to select a valid cloud before you can upload "
"images to it", width=SMALL_WIDTH)
return False
while 1:
......@@ -144,8 +149,8 @@ def upload_image(session):
overwrite.append(f)
if len(overwrite) > 0:
if d.yesno("The following pithos object(s) already exist(s):\n"
"%s\nDo you want to overwrite them?" %
if d.yesno("The following storage service object(s) already "
"exist(s):\n%s\nDo you want to overwrite them?" %
"\n".join(overwrite), width=WIDTH, defaultno=1):
continue
......@@ -177,8 +182,9 @@ def upload_image(session):
out.success("done")
except ClientError as e:
d.msgbox("Error in pithos+ client: %s" % e.message,
title="Pithos+ Client Error", width=SMALL_WIDTH)
d.msgbox(
"Error in storage service client: %s" % e.message,
title="Storage Service Client Error", width=SMALL_WIDTH)
if 'pithos_uri' in session:
del session['pithos_uri']
return False
......@@ -187,26 +193,26 @@ def upload_image(session):
finally:
gauge.cleanup()
d.msgbox("Image file `%s' was successfully uploaded to pithos+" % filename,
d.msgbox("Image file `%s' was successfully uploaded" % filename,
width=SMALL_WIDTH)
return True
def register_image(session):
"""Register image with cyclades"""
"""Register image with the compute service"""
d = session["dialog"]
is_public = False
if "account" not in session:
d.msgbox("You need to provide your ~okeanos credentians before you "
"can register an images with cyclades", width=SMALL_WIDTH)
d.msgbox("You need to select a valid cloud before you "
"can register an images with it", width=SMALL_WIDTH)
return False
if "pithos_uri" not in session:
d.msgbox("You need to upload the image to pithos+ before you can "
"register it with cyclades", width=SMALL_WIDTH)
d.msgbox("You need to upload the image to the cloud before you can "
"register it", width=SMALL_WIDTH)
return False
while 1:
......@@ -243,14 +249,14 @@ def register_image(session):
out.add(gauge)
try:
try:
out.output("Registering %s image with Cyclades..." % img_type)
out.output("Registering %s image with the cloud..." % img_type)
kamaki = Kamaki(session['account'], out)
kamaki.register(name, session['pithos_uri'], metadata,
is_public)
result = kamaki.register(name, session['pithos_uri'], metadata,
is_public)
out.success('done')
# Upload metadata file
out.output("Uploading metadata file...")
metastring = extract_metadata_string(session)
metastring = unicode(json.dumps(result, ensure_ascii=False))
kamaki.upload(StringIO.StringIO(metastring),
size=len(metastring),
remote_path="%s.meta" % session['upload'])
......@@ -261,39 +267,116 @@ def register_image(session):
kamaki.share("%s.md5sum" % session['upload'])
out.success('done')
except ClientError as e:
d.msgbox("Error in pithos+ client: %s" % e.message)
d.msgbox("Error in storage service client: %s" % e.message)
return False
finally:
out.remove(gauge)
finally:
gauge.cleanup()
d.msgbox("%s image `%s' was successfully registered with Cyclades as `%s'"
d.msgbox("%s image `%s' was successfully registered with the cloud as `%s'"
% (img_type.title(), session['upload'], name), width=SMALL_WIDTH)
return True
def modify_clouds(session):
"""Modify existing cloud accounts"""
d = session['dialog']
while 1:
clouds = Kamaki.get_clouds()
if not len(clouds):
if not add_cloud(session):
break
continue
choices = []
for (name, cloud) in clouds.items():
descr = cloud['description'] if 'description' in cloud else ''
choices.append((name, descr))
(code, choice) = d.menu(
"In this menu you can edit existing cloud accounts or add new "
" ones. Press <Edit> to edit an existing account or <Add> to add "
" a new one. Press <Back> or hit <ESC> when done.", height=18,
width=WIDTH, choices=choices, menu_height=10, ok_label="Edit",
extra_button=1, extra_label="Add", cancel="Back", help_button=1,
title="Clouds")
if code in (d.DIALOG_CANCEL, d.DIALOG_ESC):
return True
elif code == d.DIALOG_OK: # Edit button
edit_cloud(session, choice)
elif code == d.DIALOG_EXTRA: # Add button
add_cloud(session)
def delete_clouds(session):
"""Delete existing cloud accounts"""
d = session['dialog']