diff --git a/ChangeLog b/ChangeLog index 6dcd994750eb3cee223e01c04277a9f87e17dc91..51cf16707d0e9e3f546e615200b3b153043544f6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,10 @@ +2013-05-27, v0.3 + * Support media hosting FreeBSD systems + * Check if remote files exist when uploading images to pithos + * Make the md5sum and metadate files public if image gets registered as + public + * Fix minor bugs and typos + 2013-05-01, v0.2.10 * Fix a bug where acl and user_xattr mount options where not respected in host bundling operation diff --git a/docs/conf.py b/docs/conf.py index c72f4e98f303068defc797edd90fdee3a2d24f58..2e40c86c509fb13fbf0d8871f1b19308ea6b5e77 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -50,9 +50,9 @@ copyright = u'2012, 2013 GRNET S.A. All rights reserved' # built documents. # # The short X.Y version. -version = '0.2.10' +version = '0.3' # The full version, including alpha/beta/rc tags. -release = '0.2.10' +release = '0.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/image_creator/__init__.py b/image_creator/__init__.py index 62ea6d9cae4c5187bc5896305bd422aa390d9cd3..67775ed29ce7044660aeaf62fbdae13d86dd2b6b 100644 --- a/image_creator/__init__.py +++ b/image_creator/__init__.py @@ -31,6 +31,6 @@ # interpreted as representing official policies, either expressed # or implied, of GRNET S.A. -__version__ = '0.2.10' +__version__ = '0.3' # vim: set sta sts=4 shiftwidth=4 sw=4 et ai : diff --git a/image_creator/bundle_volume.py b/image_creator/bundle_volume.py index 40acf53edae7302970fba95d4895b5286b9202c7..18d0245e29334f63d2fd882b7f5d50e87673da41 100644 --- a/image_creator/bundle_volume.py +++ b/image_creator/bundle_volume.py @@ -413,7 +413,7 @@ class BundleVolume(object): mopts = filter( lambda p: p.startswith('Default mount options:'), tune2fs('-l', orig_dev[i]).splitlines() - )[0].split(':')[1].strip().split() + )[0].split(':')[1].strip().split() if not (len(mopts) == 1 and mopts[0] == '(none)'): for opt in mopts: diff --git a/image_creator/dialog_main.py b/image_creator/dialog_main.py index 7e6b33dcf30e523320a50c968406fe848358752b..024c37f4af21d35e476fb9551acfcb2134532825 100644 --- a/image_creator/dialog_main.py +++ b/image_creator/dialog_main.py @@ -72,7 +72,7 @@ def create_image(d, media, out, tmp): snapshot = disk.snapshot() image = disk.get_image(snapshot) - out.output("Collecting image metadata...") + out.output("Collecting image metadata ...") metadata = {} for (key, value) in image.meta.items(): metadata[str(key)] = str(value) @@ -217,12 +217,12 @@ def main(): while 1: try: out = CompositeOutput([log]) - out.output("Starting %s v%s..." % + out.output("Starting %s v%s ..." % (parser.get_prog_name(), version)) ret = create_image(d, media, out, options.tmp) sys.exit(ret) except Reset: - log.output("Resetting everything...") + log.output("Resetting everything ...") continue finally: if logfile is not None: diff --git a/image_creator/dialog_menu.py b/image_creator/dialog_menu.py index 0e32167ca8b579e50193bebf9930d26462070314..b6d31624a7c68724db80d36d0c4f0e6f14dc107f 100644 --- a/image_creator/dialog_menu.py +++ b/image_creator/dialog_menu.py @@ -136,6 +136,19 @@ def upload_image(session): if len(filename) == 0: d.msgbox("Filename cannot be empty", width=SMALL_WIDTH) continue + + kamaki = Kamaki(session['account'], None) + overwrite = [] + for f in (filename, "%s.md5sum" % filename, "%s.meta" % filename): + if kamaki.object_exists(f): + overwrite.append(f) + + if len(overwrite) > 0: + if d.yesno("The following pithos object(s) already exist(s):\n" + "%s\nDo you want to overwrite them?" % + "\n".join(overwrite), width=WIDTH, defaultno=1): + continue + session['upload'] = filename break @@ -143,12 +156,12 @@ def upload_image(session): try: out = image.out out.add(gauge) + kamaki.out = out try: if 'checksum' not in session: md5 = MD5(out) session['checksum'] = md5.compute(image.device, size) - kamaki = Kamaki(session['account'], out) try: # Upload image file with open(image.device, 'rb') as f: @@ -156,14 +169,6 @@ def upload_image(session): kamaki.upload(f, size, filename, "Calculating block hashes", "Uploading missing blocks") - # Upload metadata file - out.output("Uploading metadata file...") - metastring = extract_metadata_string(session) - kamaki.upload(StringIO.StringIO(metastring), - size=len(metastring), - remote_path="%s.meta" % filename) - out.success("done") - # Upload md5sum file out.output("Uploading md5sum file...") md5str = "%s %s\n" % (session['checksum'], filename) @@ -237,12 +242,24 @@ def register_image(session): out = session['image'].out out.add(gauge) try: - out.output("Registering %s image with Cyclades..." % img_type) try: + out.output("Registering %s image with Cyclades..." % img_type) kamaki = Kamaki(session['account'], out) kamaki.register(name, session['pithos_uri'], metadata, is_public) out.success('done') + # Upload metadata file + out.output("Uploading metadata file...") + metastring = extract_metadata_string(session) + kamaki.upload(StringIO.StringIO(metastring), + size=len(metastring), + remote_path="%s.meta" % session['upload']) + out.success("done") + if is_public: + out.output("Sharing metadata and md5sum files...") + kamaki.share("%s.meta" % session['upload']) + kamaki.share("%s.md5sum" % session['upload']) + out.success('done') except ClientError as e: d.msgbox("Error in pithos+ client: %s" % e.message) return False @@ -504,12 +521,7 @@ def sysprep(session): help_title = "System Preperation Tasks" sysprep_help = "%s\n%s\n\n" % (help_title, '=' * len(help_title)) - if 'exec_syspreps' not in session: - session['exec_syspreps'] = [] - - all_syspreps = image.os.list_syspreps() - # Only give the user the choice between syspreps that have not ran yet - syspreps = [s for s in all_syspreps if s not in session['exec_syspreps']] + syspreps = image.os.list_syspreps() if len(syspreps) == 0: d.msgbox("No system preparation task available to run!", @@ -544,16 +556,31 @@ def sysprep(session): for i in range(len(syspreps)): if str(i + 1) in tags: image.os.enable_sysprep(syspreps[i]) - session['exec_syspreps'].append(syspreps[i]) else: image.os.disable_sysprep(syspreps[i]) + if len([s for s in image.os.list_syspreps() if s.enabled]) == 0: + d.msgbox("No system preperation task is selected!", + title="System Preperation", width=SMALL_WIDTH) + continue + infobox = InfoBoxOutput(d, "Image Configuration") try: image.out.add(infobox) try: image.mount(readonly=False) try: + err = "Unable to execute the system preparation " \ + "tasks. Couldn't mount the media%s." + title = "System Preparation" + if not image.mounted: + d.msgbox(err % "", title=title, width=SMALL_WIDTH) + return + elif image.mounted_ro: + d.msgbox(err % " read-write", title=title, + width=SMALL_WIDTH) + return + # The checksum is invalid. We have mounted the image rw if 'checksum' in session: del session['checksum'] @@ -563,9 +590,6 @@ def sysprep(session): image.os.do_sysprep() infobox.finalize() - # Disable syspreps that have ran - for sysprep in session['exec_syspreps']: - image.os.disable_sysprep(sysprep) finally: image.umount() finally: diff --git a/image_creator/dialog_util.py b/image_creator/dialog_util.py index e18fb89dc6ef0c60b7b75546e21e095c9a840d81..046ee3ee1505fbc138513960a16f26395b9005c9 100644 --- a/image_creator/dialog_util.py +++ b/image_creator/dialog_util.py @@ -146,13 +146,13 @@ def extract_image(session): image.dump(path) # Extract metadata file - out.output("Extracting metadata file...") + out.output("Extracting metadata file ...") with open('%s.meta' % path, 'w') as f: f.write(extract_metadata_string(session)) out.success('done') # Extract md5sum file - out.output("Extracting md5sum file...") + out.output("Extracting md5sum file ...") md5str = "%s %s\n" % (session['checksum'], name) with open('%s.md5sum' % path, 'w') as f: f.write(md5str) diff --git a/image_creator/dialog_wizard.py b/image_creator/dialog_wizard.py index 5605cc756e7f8b40b8a52d42de7a2a2e67d96678..30dc7c622b10dbe704b853e54527e241a97040ff 100644 --- a/image_creator/dialog_wizard.py +++ b/image_creator/dialog_wizard.py @@ -195,9 +195,11 @@ def start_wizard(session): if init_token is None: init_token = "" + distro = session['image'].distro + ostype = session['image'].ostype name = WizardInputPage( "ImageName", "Image Name", "Please provide a name for the image:", - title="Image Name", init=session['image'].distro) + title="Image Name", init=ostype if distro == "unknown" else distro) descr = WizardInputPage( "ImageDescription", "Image Description", @@ -265,6 +267,12 @@ def create_image(session): #Sysprep image.mount(False) + err_msg = "Unable to execute the system preparation tasks." + if not image.mounted: + raise FatalError("%s Couldn't mount the media." % err_msg) + elif image.mounted_ro: + raise FatalError("%s Couldn't mount the media read-write." + % err_msg) image.os.do_sysprep() metadata = image.os.meta image.umount() @@ -317,6 +325,14 @@ def create_image(session): kamaki.register(wizard['ImageName'], pithos_file, metadata, is_public) out.success('done') + if is_public: + out.output("Sharing md5sum file ...", False) + kamaki.share("%s.md5sum" % name) + out.success('done') + out.output("Sharing metadata file ...", False) + kamaki.share("%s.meta" % name) + out.success('done') + out.output() except ClientError as e: diff --git a/image_creator/disk.py b/image_creator/disk.py index b4311f8fa44ce3091a7db77d3549c021ccfd2471..38372c52fc3f155ca34232d133fabd88178f0f07 100644 --- a/image_creator/disk.py +++ b/image_creator/disk.py @@ -158,7 +158,7 @@ class Disk(object): self.out.success('looks like a block device') # Take a snapshot and return it to the user - self.out.output("Snapshotting media source...", False) + self.out.output("Snapshotting media source ...", False) size = blockdev('--getsz', sourcedev) cowfd, cow = tempfile.mkstemp(dir=self.tmp) os.close(cowfd) diff --git a/image_creator/gpt.py b/image_creator/gpt.py index 3be594e6a459eccf98094654460af914142b7d65..968a918e5c5013272b3ad5062e317abad7c9be20 100644 --- a/image_creator/gpt.py +++ b/image_creator/gpt.py @@ -216,7 +216,7 @@ class GPTPartitionTable(object): return struct.calcsize(GPTPartitionTable.GPTHeader.format) def __str__(self): - """Print a GPTHeader""" + """Print a GPTHeader""" return "Signature: %s\n" % self.signature + \ "Revision: %r\n" % self.revision + \ "Header Size: %d\n" % self.hdr_size + \ diff --git a/image_creator/image.py b/image_creator/image.py index 7d1ddfeef98f6c529bf631923ff2a727ab6e6c6d..de287568d697464b93f37c4df719ac37e53e979e 100644 --- a/image_creator/image.py +++ b/image_creator/image.py @@ -54,6 +54,7 @@ class Image(object): self.guestfs_device = None self.size = 0 self.mounted = False + self.mounted_ro = False self.g = guestfs.GuestFS() self.g.add_drive_opts(self.device, readonly=0, format="raw") @@ -158,10 +159,12 @@ class Image(object): def mount(self, readonly=False): """Mount all disk partitions in a correct order.""" - mount = self.g.mount_ro if readonly else self.g.mount - msg = " read-only" if readonly else "" - self.out.output("Mounting the media%s ..." % msg, False) - mps = self.g.inspect_get_mountpoints(self.root) + msg = "Mounting the media%s ..." % (" read-only" if readonly else "") + self.out.output(msg, False) + + #If something goes wrong when mounting rw, remount the filesystem ro + remount_ro = False + rw_mpoints = ('/', '/etc', '/root', '/home', '/var') # Sort the keys to mount the fs in a correct order. # / should be mounted befor /boot, etc @@ -172,15 +175,45 @@ class Image(object): return 0 else: return -1 + mps = self.g.inspect_get_mountpoints(self.root) mps.sort(compare) + + mopts = 'ro' if readonly else 'rw' for mp, dev in mps: + if self.ostype == 'freebsd': + # libguestfs can't handle correct freebsd partitions on GUID + # Partition Table. We have to do the translation to linux + # device names ourselves + m = re.match('^/dev/((?:ada)|(?:vtbd))(\d+)p(\d+)$', dev) + if m: + m2 = int(m.group(2)) + m3 = int(m.group(3)) + dev = '/dev/sd%c%d' % (chr(ord('a') + m2), m3) try: - mount(dev, mp) + self.g.mount_options(mopts, dev, mp) except RuntimeError as msg: - self.out.warn("%s (ignored)" % msg) - - self.mounted = True - self.out.success("done") + if self.ostype == 'freebsd': + freebsd_mopts = "ufstype=ufs2,%s" % mopts + try: + self.g.mount_vfs(freebsd_mopts, 'ufs', dev, mp) + except RuntimeError as msg: + if readonly is False and mp in rw_mpoints: + remount_ro = True + break + elif readonly is False and mp in rw_mpoints: + remount_ro = True + break + else: + self.out.warn("%s (ignored)" % msg) + if remount_ro: + self.out.warn("Unable to mount %s read-write. " + "Remounting everything read-only..." % mp) + self.umount() + self.mount(True) + else: + self.mounted = True + self.mounted_ro = readonly + self.out.success("done") def umount(self): """Umount all mounted filesystems.""" @@ -272,7 +305,7 @@ class Image(object): break if not re.match("ext[234]", fstype): - self.out.warn("Don't know how to resize %s partitions." % fstype) + self.out.warn("Don't know how to shrink %s partitions." % fstype) return self.size part_dev = "%s%d" % (self.guestfs_device, last_part['part_num']) diff --git a/image_creator/kamaki_wrapper.py b/image_creator/kamaki_wrapper.py index 3f809d0f4d37a9f230e0d5b79e6c4637e1f5a1b0..e2012d27c84228bb794b3cdea7b554b5a50e5f2c 100644 --- a/image_creator/kamaki_wrapper.py +++ b/image_creator/kamaki_wrapper.py @@ -117,4 +117,21 @@ class Kamaki(object): params = {'is_public': is_public, 'disk_format': 'diskdump'} self.image_client.register(name, location, params, str_metadata) + def share(self, location): + """Share this file with all the users""" + + self.pithos_client.set_object_sharing(location, "*") + + def object_exists(self, location): + """Check if an object exists in pythos""" + + try: + self.pithos_client.get_object_info(location) + except ClientError as e: + if e.status == 404: # Object not found error + return False + else: + raise + return True + # vim: set sta sts=4 shiftwidth=4 sw=4 et ai : diff --git a/image_creator/main.py b/image_creator/main.py index 4b86f7c7d42fffdbb54b6b67ee661077778b4467..10fc0b5e50818b71e7186a9f615425154e80ccf6 100644 --- a/image_creator/main.py +++ b/image_creator/main.py @@ -181,8 +181,8 @@ def image_creator(): for extension in ('', '.meta', '.md5sum'): filename = "%s%s" % (options.outfile, extension) if os.path.exists(filename): - raise FatalError("Output file %s exists " - "(use --force to overwrite it)" % filename) + raise FatalError("Output file `%s' exists " + "(use --force to overwrite it)." % filename) # Check if the authentication token is valid. The earlier the better if options.token is not None: @@ -191,9 +191,24 @@ def image_creator(): if account is None: raise FatalError("The authentication token you provided is not" " valid!") + else: + kamaki = Kamaki(account, out) except ClientError as e: raise FatalError("Astakos client: %d %s" % (e.status, e.message)) + if options.upload and not options.force: + if kamaki.object_exists(options.upload): + raise FatalError("Remote pithos object `%s' exists " + "(use --force to overwrite it)." % options.upload) + if kamaki.object_exists("%s.md5sum" % options.upload): + raise FatalError("Remote pithos object `%s.md5sum' exists " + "(use --force to overwrite it)." % options.upload) + + if options.register and not options.force: + if kamaki.object_exists("%s.meta" % options.upload): + raise FatalError("Remote pithos object `%s.meta' exists " + "(use --force to overwrite it)." % options.upload) + disk = Disk(options.source, out, options.tmp) def signal_handler(signum, frame): @@ -206,7 +221,7 @@ def image_creator(): image = disk.get_image(snapshot) - # If no customization is to be applied, the image should be mounted ro + # If no customization is to be done, the image should be mounted ro ro = (not (options.sysprep or options.shrink) or options.print_sysprep) image.mount(ro) try: @@ -224,6 +239,13 @@ def image_creator(): return 0 if options.sysprep: + err_msg = "Unable to perform the system preparation tasks. " \ + "Couldn't mount the media%s. Use --no-sysprep if you " \ + "don't won't to perform any system preparation task." + if not image.mounted: + raise FatalError(err_msg % "") + elif image.mounted_ro: + raise FatalError(err_msg % " read-write") image.os.do_sysprep() metadata = image.os.meta @@ -265,19 +287,12 @@ def image_creator(): uploaded_obj = "" if options.upload: out.output("Uploading image to pithos:") - kamaki = Kamaki(account, out) with open(snapshot, 'rb') as f: uploaded_obj = kamaki.upload( f, size, options.upload, - "(1/4) Calculating block hashes", - "(2/4) Uploading missing blocks") - - out.output("(3/4) Uploading metadata file ...", False) - kamaki.upload(StringIO.StringIO(metastring), - size=len(metastring), - remote_path="%s.%s" % (options.upload, 'meta')) - out.success('done') - out.output("(4/4) Uploading md5sum file ...", False) + "(1/3) Calculating block hashes", + "(2/3) Uploading missing blocks") + out.output("(3/3) Uploading md5sum file ...", False) md5sumstr = '%s %s\n' % (checksum, os.path.basename(options.upload)) kamaki.upload(StringIO.StringIO(md5sumstr), @@ -293,6 +308,19 @@ def image_creator(): kamaki.register(options.register, uploaded_obj, metadata, options.public) out.success('done') + out.output("Uploading metadata file ...", False) + kamaki.upload(StringIO.StringIO(metastring), + size=len(metastring), + remote_path="%s.%s" % (options.upload, 'meta')) + out.success('done') + if options.public: + out.output("Sharing md5sum file ...", False) + kamaki.share("%s.md5sum" % options.upload) + out.success('done') + out.output("Sharing metadata file ...", False) + kamaki.share("%s.meta" % options.upload) + out.success('done') + out.output() except ClientError as e: raise FatalError("Pithos client: %d %s" % (e.status, e.message)) diff --git a/image_creator/os_type/__init__.py b/image_creator/os_type/__init__.py index 92dde0da4a03b230a5a704a896d623493b80c21b..cea08a32a8ac43575f4c0e167a858674ca5e8fc7 100644 --- a/image_creator/os_type/__init__.py +++ b/image_creator/os_type/__init__.py @@ -65,6 +65,7 @@ def sysprep(enabled=True): def wrapper(func): func.sysprep = True func.enabled = enabled + func.executed = False return func return wrapper @@ -82,6 +83,8 @@ class OSBase(object): self.meta['ROOT_PARTITION'] = "%d" % self.g.part_to_partnum(self.root) self.meta['OSFAMILY'] = self.g.inspect_get_type(self.root) self.meta['OS'] = self.g.inspect_get_distro(self.root) + if self.meta['OS'] == "unknown": + self.meta['OS'] = self.meta['OSFAMILY'] self.meta['DESCRIPTION'] = self.g.inspect_get_product_name(self.root) def _is_sysprep(self, obj): @@ -92,7 +95,7 @@ class OSBase(object): objs = [getattr(self, name) for name in dir(self) if not name.startswith('_')] - return [x for x in objs if self._is_sysprep(x)] + return [x for x in objs if self._is_sysprep(x) and x.executed is False] def sysprep_info(self, obj): assert self._is_sysprep(obj), "Object is not a sysprep" @@ -220,6 +223,7 @@ class OSBase(object): cnt += 1 self.out.output(('(%d/%d)' % (cnt, size)).ljust(7), False) task() + setattr(task.im_func, 'executed', True) self.out.output() # vim: set sta sts=4 shiftwidth=4 sw=4 et ai : diff --git a/image_creator/os_type/freebsd.py b/image_creator/os_type/freebsd.py index 5feb9fbe2bb080dc208c53ee56189ec886dd2707..d0ed78b20b1e6e6ac31c3174d0573e75f643a7d3 100644 --- a/image_creator/os_type/freebsd.py +++ b/image_creator/os_type/freebsd.py @@ -31,11 +31,73 @@ # interpreted as representing official policies, either expressed # or implied, of GRNET S.A. -from image_creator.os_type.unix import Unix +from image_creator.os_type.unix import Unix, sysprep + +import re class Freebsd(Unix): """OS class for FreeBSD Unix-like os""" - pass + def __init__(self, rootdev, ghandler, output): + super(Freebsd, self).__init__(rootdev, ghandler, output) + + self.meta["USERS"] = " ".join(self._get_passworded_users()) + + #The original product name key is long and ugly + self.meta['DESCRIPTION'] = \ + self.meta['DESCRIPTION'].split('#')[0].strip() + + # Delete the USERS metadata if empty + if not len(self.meta['USERS']): + self.out.warn("No passworded users found!") + del self.meta['USERS'] + + def _get_passworded_users(self): + users = [] + regexp = re.compile( + '^([^:]+):((?:![^:]+)|(?:[^!*][^:]+)|):(?:[^:]*:){7}(?:[^:]*)' + ) + + for line in self.g.cat('/etc/master.passwd').splitlines(): + line = line.split('#')[0] + match = regexp.match(line) + if not match: + continue + + user, passwd = match.groups() + if len(passwd) > 0 and passwd[0] == '!': + self.out.warn("Ignoring locked %s account." % user) + else: + users.append(user) + + return users + + @sysprep() + def cleanup_password(self, print_header=True): + """Remove all passwords and lock all user accounts""" + + if print_header: + self.out.output("Cleaning up passwords & locking all user " + "accounts") + + master_passwd = [] + + for line in self.g.cat('/etc/master.passwd').splitlines(): + + # Check for empty or comment lines + if len(line.split('#')[0]) == 0: + master_passwd.append(line) + continue + + fields = line.split(':') + if fields[1] not in ('*', '!'): + fields[1] = '!' + + master_passwd.append(":".join(fields)) + + self.g.write('/etc/master.passwd', "\n".join(master_passwd) + '\n') + + # Make sure no one can login on the system + self.g.rm_rf('/etc/spwd.db') # vim: set sta sts=4 shiftwidth=4 sw=4 et ai : diff --git a/image_creator/os_type/unix.py b/image_creator/os_type/unix.py index 8489d4dec8ce6017f5c77f0bc9efd0ddc56efb9b..b0b3ae62e56ebe40d609048467e2cd5ca100ccfb 100644 --- a/image_creator/os_type/unix.py +++ b/image_creator/os_type/unix.py @@ -85,14 +85,18 @@ class Unix(OSBase): if print_header: self.out.output('Removing files under /var/mail & /var/spool/mail') - self.foreach_file('/var/spool/mail', self.g.rm_rf, maxdepth=1) + if self.g.is_dir('/var/spool/mail'): + self.foreach_file('/var/spool/mail', self.g.rm_rf, maxdepth=1) + self.foreach_file('/var/mail', self.g.rm_rf, maxdepth=1) @sysprep() def cleanup_userdata(self, print_header=True): """Delete sensitive userdata""" - homedirs = ['/root'] + self.ls('/home/') + homedirs = ['/root'] + if self.g.is_dir('/home/'): + homedirs += self.ls('/home/') if print_header: self.out.output("Removing sensitive user data under %s" % diff --git a/image_creator/output/__init__.py b/image_creator/output/__init__.py index 7beceac59080f1457b85e9a43562c8880d670ca5..e421c75aec020d1bf260562b0681cb7ff8d62d1e 100644 --- a/image_creator/output/__init__.py +++ b/image_creator/output/__init__.py @@ -71,7 +71,7 @@ class Output(object): def __init__(self, size, title, bar_type='default'): self.size = size self.bar_type = bar_type - self.output.output("%s..." % title, False) + self.output.output("%s ..." % title, False) def goto(self, dest): """Move progress to a specific position"""