Commit f5174d2c authored by Nikos Skalkotos's avatar Nikos Skalkotos
Browse files

Rename DiskDevice class to Image

Rename DiskDevice class to Image and move it to a seperate module.
Also, don't let the user access the os_type/* modules directly. In
Image class create the `os' member variable that will host an
appropriate instance of one of the OSBase classes.
parent d3f3bfbf
......@@ -48,14 +48,13 @@ from image_creator.output.cli import SimpleOutput
from image_creator.output.dialog import GaugeOutput
from image_creator.output.composite import CompositeOutput
from image_creator.disk import Disk
from image_creator.os_type import os_cls
from image_creator.dialog_wizard import wizard
from image_creator.dialog_menu import main_menu
from image_creator.dialog_util import SMALL_WIDTH, WIDTH, confirm_exit, \
Reset, update_background_title
def image_creator(d, media, out, tmp):
def create_image(d, media, out, tmp):
d.setBackgroundTitle('snf-image-creator')
......@@ -71,19 +70,14 @@ def image_creator(d, media, out, tmp):
signal.signal(signal.SIGTERM, signal_handler)
try:
snapshot = disk.snapshot()
dev = disk.get_device(snapshot)
image = disk.get_image(snapshot)
out.output("Collecting image metadata...")
metadata = {}
for (key, value) in dev.meta.items():
for (key, value) in image.meta.items():
metadata[str(key)] = str(value)
dev.mount(readonly=True)
out.output("Collecting image metadata...")
cls = os_cls(dev.distro, dev.ostype)
image_os = cls(dev.root, dev.g, out)
dev.umount()
for (key, value) in image_os.meta.items():
for (key, value) in image.os.meta.items():
metadata[str(key)] = str(value)
out.success("done")
......@@ -97,9 +91,7 @@ def image_creator(d, media, out, tmp):
session = {"dialog": d,
"disk": disk,
"snapshot": snapshot,
"device": dev,
"image_os": image_os,
"image": image,
"metadata": metadata}
msg = "snf-image-creator detected a %s system on the input media. " \
......@@ -107,8 +99,8 @@ def image_creator(d, media, out, tmp):
"image creation process?\n\nChoose <Wizard> to run the wizard," \
" <Expert> to run the snf-image-creator in expert mode or " \
"press ESC to quit the program." \
% (dev.ostype if dev.ostype == dev.distro else "%s (%s)" %
(dev.ostype, dev.distro))
% (image.ostype if image.ostype == image.distro else "%s (%s)" %
(image.ostype, image.distro))
update_background_title(session)
......@@ -226,7 +218,7 @@ def main():
out = CompositeOutput([log])
out.output("Starting %s v%s..." %
(parser.get_prog_name(), version))
ret = image_creator(d, media, out, options.tmp)
ret = create_image(d, media, out, options.tmp)
sys.exit(ret)
except Reset:
log.output("Resetting everything...")
......
......@@ -108,9 +108,9 @@ class MetadataMonitor(object):
def upload_image(session):
d = session["dialog"]
dev = session['device']
image = session['image']
meta = session['metadata']
size = dev.size
size = image.size
if "account" not in session:
d.msgbox("You need to provide your ~okeanos credentials before you "
......@@ -139,17 +139,17 @@ def upload_image(session):
gauge = GaugeOutput(d, "Image Upload", "Uploading...")
try:
out = dev.out
out = image.out
out.add(gauge)
try:
if 'checksum' not in session:
md5 = MD5(out)
session['checksum'] = md5.compute(session['snapshot'], size)
session['checksum'] = md5.compute(image.device, size)
kamaki = Kamaki(session['account'], out)
try:
# Upload image file
with open(session['snapshot'], 'rb') as f:
with open(image.device, 'rb') as f:
session["pithos_uri"] = \
kamaki.upload(f, size, filename,
"Calculating block hashes",
......@@ -188,14 +188,12 @@ def upload_image(session):
def register_image(session):
d = session["dialog"]
dev = session['device']
is_public = False
if "account" not in session:
d.msgbox("You need to provide your ~okeanos credentians before you "
"can register an images with cyclades",
width=SMALL_WIDTH)
"can register an images with cyclades", width=SMALL_WIDTH)
return False
if "pithos_uri" not in session:
......@@ -233,7 +231,7 @@ def register_image(session):
img_type = "public" if is_public else "private"
gauge = GaugeOutput(d, "Image Registration", "Registering image...")
try:
out = dev.out
out = session['image'].out
out.add(gauge)
try:
out.output("Registering %s image with Cyclades..." % img_type)
......@@ -481,7 +479,7 @@ def exclude_tasks(session):
def sysprep(session):
d = session['dialog']
image_os = session['image_os']
image = session['image']
# Is the image already shrinked?
if 'shrinked' in session and session['shrinked']:
......@@ -500,7 +498,7 @@ def sysprep(session):
if 'exec_syspreps' not in session:
session['exec_syspreps'] = []
all_syspreps = image_os.list_syspreps()
all_syspreps = image.os.list_syspreps()
# Only give the user the choice between syspreps that have not ran yet
syspreps = [s for s in all_syspreps if s not in session['exec_syspreps']]
......@@ -513,7 +511,7 @@ def sysprep(session):
choices = []
index = 0
for sysprep in syspreps:
name, descr = image_os.sysprep_info(sysprep)
name, descr = image.os.sysprep_info(sysprep)
display_name = name.replace('-', ' ').capitalize()
sysprep_help += "%s\n" % display_name
sysprep_help += "%s\n" % ('-' * len(display_name))
......@@ -536,34 +534,33 @@ def sysprep(session):
# Enable selected syspreps and disable the rest
for i in range(len(syspreps)):
if str(i + 1) in tags:
image_os.enable_sysprep(syspreps[i])
image.os.enable_sysprep(syspreps[i])
session['exec_syspreps'].append(syspreps[i])
else:
image_os.disable_sysprep(syspreps[i])
image.os.disable_sysprep(syspreps[i])
infobox = InfoBoxOutput(d, "Image Configuration")
try:
dev = session['device']
dev.out.add(infobox)
image.out.add(infobox)
try:
dev.mount(readonly=False)
image.mount(readonly=False)
try:
# The checksum is invalid. We have mounted the image rw
if 'checksum' in session:
del session['checksum']
# Monitor the metadata changes during syspreps
with MetadataMonitor(session, image_os.meta):
image_os.do_sysprep()
with MetadataMonitor(session, image.os.meta):
image.os.do_sysprep()
infobox.finalize()
# Disable syspreps that have ran
for sysprep in session['exec_syspreps']:
image_os.disable_sysprep(sysprep)
image.os.disable_sysprep(sysprep)
finally:
dev.umount()
image.umount()
finally:
dev.out.remove(infobox)
image.out.remove(infobox)
finally:
infobox.cleanup()
break
......@@ -572,7 +569,7 @@ def sysprep(session):
def shrink(session):
d = session['dialog']
dev = session['device']
image = session['image']
shrinked = 'shrinked' in session and session['shrinked']
......@@ -589,14 +586,14 @@ def shrink(session):
if not d.yesno("%s\n\nDo you want to continue?" % msg, width=WIDTH,
height=12, title="Image Shrinking"):
with MetadataMonitor(session, dev.meta):
with MetadataMonitor(session, image.meta):
infobox = InfoBoxOutput(d, "Image Shrinking", height=4)
dev.out.add(infobox)
image.out.add(infobox)
try:
dev.shrink()
image.shrink()
infobox.finalize()
finally:
dev.out.remove(infobox)
image.out.remove(infobox)
session['shrinked'] = True
update_background_title(session)
......
......@@ -43,17 +43,17 @@ WIDTH = 70
def update_background_title(session):
d = session['dialog']
dev = session['device']
disk = session['disk']
image = session['image']
MB = 2 ** 20
size = (dev.size + MB - 1) // MB
size = (image.size + MB - 1) // MB
shrinked = 'shrinked' in session and session['shrinked']
postfix = " (shrinked)" if shrinked else ''
title = "OS: %s, Distro: %s, Size: %dMB%s, Source: %s" % \
(dev.ostype, dev.distro, size, postfix,
(image.ostype, image.distro, size, postfix,
os.path.abspath(disk.source))
d.setBackgroundTitle(title)
......@@ -128,18 +128,16 @@ def extract_image(session):
gauge = GaugeOutput(d, "Image Extraction", "Extracting image...")
try:
dev = session['device']
out = dev.out
image = session['image']
out = image.out
out.add(gauge)
try:
if "checksum" not in session:
size = dev.size
md5 = MD5(out)
session['checksum'] = md5.compute(session['snapshot'],
size)
session['checksum'] = md5.compute(image.device, image.size)
# Extract image file
dev.dump(path)
image.dump(path)
# Extract metadata file
out.output("Extracting metadata file...")
......
......@@ -180,7 +180,7 @@ def wizard(session):
name = WizardInputPage(
"ImageName", "Image Name", "Please provide a name for the image:",
title="Image Name", init=session['device'].distro)
title="Image Name", init=session['image'].distro)
descr = WizardInputPage(
"ImageDescription", "Image Description",
......@@ -232,38 +232,35 @@ def wizard(session):
def create_image(session):
d = session['dialog']
disk = session['disk']
device = session['device']
snapshot = session['snapshot']
image_os = session['image_os']
image = session['image']
wizard = session['wizard']
# Save Kamaki credentials
Kamaki.save_token(wizard['Account']['auth_token'])
with_progress = OutputWthProgress(True)
out = disk.out
out = image.out
out.add(with_progress)
try:
out.clear()
#Sysprep
device.mount(False)
image_os.do_sysprep()
metadata = image_os.meta
device.umount()
image.mount(False)
image.os.do_sysprep()
metadata = image.os.meta
image.umount()
#Shrink
size = device.shrink()
size = image.shrink()
session['shrinked'] = True
update_background_title(session)
metadata.update(device.meta)
metadata.update(image.meta)
metadata['DESCRIPTION'] = wizard['ImageDescription']
#MD5
md5 = MD5(out)
session['checksum'] = md5.compute(snapshot, size)
session['checksum'] = md5.compute(image.device, size)
#Metadata
metastring = '\n'.join(
......@@ -278,7 +275,7 @@ def create_image(session):
name = "%s-%s.diskdump" % (wizard['ImageName'],
time.strftime("%Y%m%d%H%M"))
pithos_file = ""
with open(snapshot, 'rb') as f:
with open(image.device, 'rb') as f:
pithos_file = kamaki.upload(f, size, name,
"(1/4) Calculating block hashes",
"(2/4) Uploading missing blocks")
......
......@@ -32,21 +32,17 @@
# or implied, of GRNET S.A.
from image_creator.util import get_command
from image_creator.util import FatalError
from image_creator.util import try_fail_repeat
from image_creator.util import free_space
from image_creator.gpt import GPTPartitionTable
from image_creator.util import FatalError
from image_creator.bundle_volume import BundleVolume
from image_creator.image import Image
import stat
import os
import tempfile
import uuid
import re
import guestfs
import shutil
from sendfile import sendfile
dd = get_command('dd')
dmsetup = get_command('dmsetup')
......@@ -70,7 +66,7 @@ class Disk(object):
media can be an image file, a block device or a directory.
"""
self._cleanup_jobs = []
self._devices = []
self._images = []
self.source = source
self.out = output
self.meta = {}
......@@ -123,9 +119,9 @@ class Disk(object):
program ends.
"""
try:
while len(self._devices):
device = self._devices.pop()
device.destroy()
while len(self._images):
image = self._images.pop()
image.destroy()
finally:
# Make sure those are executed even if one of the device.destroy
# methods throws exeptions.
......@@ -176,322 +172,19 @@ class Disk(object):
self.out.success('done')
return "/dev/mapper/%s" % snapshot
def get_device(self, media):
"""Returns a newly created DiskDevice instance."""
def get_image(self, media):
"""Returns a newly created ImageCreator instance."""
new_device = DiskDevice(media, self.out)
self._devices.append(new_device)
new_device.enable()
return new_device
image = Image(media, self.out)
self._images.append(image)
image.enable()
return image
def destroy_device(self, device):
"""Destroys a DiskDevice instance previously created by
get_device method.
def destroy_image(self, image):
"""Destroys an ImageCreator instance previously created by
get_image_creator method.
"""
self._devices.remove(device)
device.destroy()
class DiskDevice(object):
"""This class represents a block device hosting an Operating System
as created by the device-mapper.
"""
def __init__(self, device, output, bootable=True, meta={}):
"""Create a new DiskDevice."""
self.real_device = device
self.out = output
self.bootable = bootable
self.meta = meta
self.progress_bar = None
self.guestfs_device = None
self.size = 0
self.g = guestfs.GuestFS()
self.g.add_drive_opts(self.real_device, readonly=0, format="raw")
# Before version 1.17.14 the recovery process, which is a fork of the
# original process that called libguestfs, did not close its inherited
# file descriptors. This can cause problems especially if the parent
# process has opened pipes. Since the recovery process is an optional
# feature of libguestfs, it's better to disable it.
self.g.set_recovery_proc(0)
version = self.g.version()
if version['major'] > 1 or \
(version['major'] == 1 and (version['minor'] >= 18 or
(version['minor'] == 17 and
version['release'] >= 14))):
self.g.set_recovery_proc(1)
self.out.output("Enabling recovery proc")
#self.g.set_trace(1)
#self.g.set_verbose(1)
self.guestfs_enabled = False
def enable(self):
"""Enable a newly created DiskDevice"""
self.out.output('Launching helper VM (may take a while) ...', False)
# self.progressbar = self.out.Progress(100, "Launching helper VM",
# "percent")
# eh = self.g.set_event_callback(self.progress_callback,
# guestfs.EVENT_PROGRESS)
self.g.launch()
self.guestfs_enabled = True
# self.g.delete_event_callback(eh)
# self.progressbar.success('done')
# self.progressbar = None
self.out.success('done')
self.out.output('Inspecting Operating System ...', False)
roots = self.g.inspect_os()
if len(roots) == 0:
raise FatalError("No operating system found")
if len(roots) > 1:
raise FatalError("Multiple operating systems found."
"We only support images with one OS.")
self.root = roots[0]
self.guestfs_device = self.g.part_to_dev(self.root)
self.size = self.g.blockdev_getsize64(self.guestfs_device)
self.meta['PARTITION_TABLE'] = \
self.g.part_get_parttype(self.guestfs_device)
self.ostype = self.g.inspect_get_type(self.root)
self.distro = self.g.inspect_get_distro(self.root)
self.out.success('found a(n) %s system' % self.distro)
def destroy(self):
"""Destroy this DiskDevice instance."""
# In new guestfs versions, there is a handy shutdown method for this
try:
if self.guestfs_enabled:
self.g.umount_all()
self.g.sync()
finally:
# Close the guestfs handler if open
self.g.close()
# def progress_callback(self, ev, eh, buf, array):
# position = array[2]
# total = array[3]
#
# self.progressbar.goto((position * 100) // total)
def mount(self, readonly=False):
"""Mount all disk partitions in a correct order."""
mount = self.g.mount_ro if readonly else self.g.mount
msg = " read-only" if readonly else ""
self.out.output("Mounting the media%s ..." % msg, False)
mps = self.g.inspect_get_mountpoints(self.root)
# Sort the keys to mount the fs in a correct order.
# / should be mounted befor /boot, etc
def compare(a, b):
if len(a[0]) > len(b[0]):
return 1
elif len(a[0]) == len(b[0]):
return 0
else:
return -1
mps.sort(compare)
for mp, dev in mps:
try:
mount(dev, mp)
except RuntimeError as msg:
self.out.warn("%s (ignored)" % msg)
self.out.success("done")
def umount(self):
"""Umount all mounted filesystems."""
self.g.umount_all()
def _last_partition(self):
if self.meta['PARTITION_TABLE'] not in 'msdos' 'gpt':
msg = "Unsupported partition table: %s. Only msdos and gpt " \
"partition tables are supported" % self.meta['PARTITION_TABLE']
raise FatalError(msg)
is_extended = lambda p: \
self.g.part_get_mbr_id(self.guestfs_device, p['part_num']) \
in (0x5, 0xf)
is_logical = lambda p: \
self.meta['PARTITION_TABLE'] == 'msdos' and p['part_num'] > 4
partitions = self.g.part_list(self.guestfs_device)
last_partition = partitions[-1]
if is_logical(last_partition):
# The disk contains extended and logical partitions....
extended = filter(is_extended, partitions)[0]
last_primary = [p for p in partitions if p['part_num'] <= 4][-1]
# check if extended is the last primary partition
if last_primary['part_num'] > extended['part_num']:
last_partition = last_primary
return last_partition
def shrink(self):
"""Shrink the disk.
This is accomplished by shrinking the last filesystem in the
disk and then updating the partition table. The new disk size
(in bytes) is returned.
ATTENTION: make sure unmount is called before shrink
"""
get_fstype = lambda p: \
self.g.vfs_type("%s%d" % (self.guestfs_device, p['part_num']))
is_logical = lambda p: \
self.meta['PARTITION_TABLE'] == 'msdos' and p['part_num'] > 4
is_extended = lambda p: \
self.meta['PARTITION_TABLE'] == 'msdos' and \
self.g.part_get_mbr_id(self.guestfs_device, p['part_num']) \
in (0x5, 0xf)
part_add = lambda ptype, start, stop: \
self.g.part_add(self.guestfs_device, ptype, start, stop)
part_del = lambda p: self.g.part_del(self.guestfs_device, p)
part_get_id = lambda p: self.g.part_get_mbr_id(self.guestfs_device, p)
part_set_id = lambda p, id: \
self.g.part_set_mbr_id(self.guestfs_device, p, id)
part_get_bootable = lambda p: \
self.g.part_get_bootable(self.guestfs_device, p)
part_set_bootable = lambda p, bootable: \
self.g.part_set_bootable(self.guestfs_device, p, bootable)
MB = 2 ** 20
self.out.output("Shrinking image (this may take a while) ...", False)
sector_size = self.g.blockdev_getss(self.guestfs_device)
last_part = None
fstype = None
while True:
last_part = self._last_partition()
fstype = get_fstype(last_part)
if fstype == 'swap':
self.meta['SWAP'] = "%d:%s" % \
(last_part['part_num'],
(last_part['part_size'] + MB - 1) // MB)
part_del(last_part['part_num'])
continue
elif is_extended(last_part):
part_del(last_part['part_num'])
continue
# Most disk manipulation programs leave 2048 sectors after the last
# partition
new_size = last_part['part_end'] + 1 + 2048 * sector_size
self.size = min(self.size, new_size)
break
if not re.match("ext[234]", fstype):
self.out.warn("Don't know how to resize %s partitions." % fstype)
return self.size
part_dev =