Commit 77722043 authored by Sofia Papagiannaki's avatar Sofia Papagiannaki
Browse files

Merge branch 'master' of https://code.grnet.gr/git/synnefo

parents 3e0f4b7b a6c51362
......@@ -245,9 +245,34 @@ Scaling up to multiple nodes
============================
Here we will describe how to deploy all services, interconnected with each
other, on multiple physical nodes. For now, if you installed successfully using
the quick installation guide and need more details, please refer to each
component's own documentation.
other, on multiple physical nodes.
synnefo components
------------------
You need to install the appropriate synnefo software components on each node,
depending on its type, see :ref:`Architecture <cyclades-architecture>`.
Please see the page of each synnefo software component for specific
installation instructions, where applicable.
Install the following synnefo components:
Nodes of type :ref:`APISERVER <APISERVER_NODE>`
Components
:ref:`snf-common <snf-common>`,
:ref:`snf-webproject <snf-webproject>`,
:ref:`snf-cyclades-app <snf-cyclades-app>`
Nodes of type :ref:`GANETI-MASTER <GANETI_MASTER>` and :ref:`GANETI-NODE <GANETI_NODE>`
Components
:ref:`snf-common <snf-common>`,
:ref:`snf-cyclades-gtools <snf-cyclades-gtools>`
Nodes of type :ref:`LOGIC <LOGIC_NODE>`
Components
:ref:`snf-common <snf-common>`,
:ref:`snf-webproject <snf-webproject>`,
:ref:`snf-cyclades-app <snf-cyclades-app>`.
Upgrade Notes
=============
......
This diff is collapsed.
Changelog
---------
v0.9.13
*******
* Restore plankton backends from invalid commit
v0.9.12
*******
......
......@@ -86,12 +86,12 @@ class BackendException(Exception):
pass
class PithosImageBackend(object):
class ImageBackend(object):
"""A wrapper arround the pithos backend to simplify image handling."""
def __init__(self, user):
self.user = user
original_filters = warnings.filters
warnings.simplefilter('ignore') # Suppress SQLAlchemy warnings
db_connection = settings.BACKEND_DB_CONNECTION
......@@ -99,21 +99,21 @@ class PithosImageBackend(object):
self.backend = connect_backend(db_connection=db_connection,
block_path=block_path)
warnings.filters = original_filters # Restore warnings
def _get_image(self, location):
def format_timestamp(t):
return strftime('%Y-%m-%d %H:%M:%S', gmtime(t))
account, container, object = split_location(location)
try:
versions = self.backend.list_versions(self.user, account,
container, object)
except NameError:
return None
image = {}
meta = self._get_meta(location)
if meta:
image['deleted_at'] = ''
......@@ -122,12 +122,12 @@ class PithosImageBackend(object):
version, timestamp = versions[-1]
meta = self._get_meta(location, version)
image['deleted_at'] = format_timestamp(timestamp)
if PLANKTON_PREFIX + 'name' not in meta:
return None # Not a Plankton image
permissions = self._get_permissions(location)
image['checksum'] = meta['hash']
image['created_at'] = format_timestamp(versions[0][1])
image['id'] = meta['uuid']
......@@ -138,7 +138,7 @@ class PithosImageBackend(object):
image['store'] = 'pithos'
image['updated_at'] = format_timestamp(meta['modified'])
image['properties'] = {}
for key, val in meta.items():
if not key.startswith(PLANKTON_PREFIX):
continue
......@@ -147,9 +147,9 @@ class PithosImageBackend(object):
val = json.loads(val)
if key in PLANKTON_META:
image[key] = val
return image
def _get_meta(self, location, version=None):
account, container, object = split_location(location)
try:
......@@ -157,77 +157,77 @@ class PithosImageBackend(object):
object, PLANKTON_DOMAIN, version)
except NameError:
return None
def _get_permissions(self, location):
account, container, object = split_location(location)
action, path, permissions = self.backend.get_object_permissions(
self.user, account, container, object)
return permissions
def _store(self, f, size=None):
"""Breaks data into blocks and stores them in the backend"""
bytes = 0
hashmap = []
backend = self.backend
blocksize = backend.block_size
data = f.read(blocksize)
while data:
hash = backend.put_block(data)
hashmap.append(hash)
bytes += len(data)
data = f.read(blocksize)
if size and size != bytes:
raise BackendException("Invalid size")
return hashmap, bytes
def _update(self, location, size, hashmap, meta, permissions):
account, container, object = split_location(location)
self.backend.update_object_hashmap(self.user, account, container,
object, size, hashmap, '', PLANKTON_DOMAIN,
permissions=permissions)
self._update_meta(location, meta, replace=True)
def _update_meta(self, location, meta, replace=False):
account, container, object = split_location(location)
prefixed = {}
for key, val in meta.items():
if key == 'properties':
val = json.dumps(val)
if key in PLANKTON_META:
prefixed[PLANKTON_PREFIX + key] = val
self.backend.update_object_meta(self.user, account, container, object,
PLANKTON_DOMAIN, prefixed, replace)
def _update_permissions(self, location, permissions):
account, container, object = split_location(location)
self.backend.update_object_permissions(self.user, account, container,
object, permissions)
def add_user(self, image_id, user):
image = self.get_image(image_id)
assert image, "Image not found"
location = image['location']
permissions = self._get_permissions(location)
read = set(permissions.get('read', []))
read.add(user)
permissions['read'] = list(read)
self._update_permissions(location, permissions)
def close(self):
self.backend.close()
def delete(self, image_id):
image = self.get_image(image_id)
account, container, object = split_location(image['location'])
self.backend.delete_object(self.user, account, container, object)
def get_data(self, location):
account, container, object = split_location(location)
size, hashmap = self.backend.get_object_hashmap(self.user, account,
......@@ -235,20 +235,20 @@ class PithosImageBackend(object):
data = ''.join(self.backend.get_block(hash) for hash in hashmap)
assert len(data) == size
return data
def get_image(self, image_id):
try:
account, container, object = self.backend.get_uuid(self.user,
image_id)
except NameError:
return None
location = get_location(account, container, object)
return self._get_image(location)
def iter(self):
"""Iter over all images available to the user"""
backend = self.backend
for account in backend.list_accounts(self.user):
for container in backend.list_containers(self.user, account,
......@@ -259,14 +259,14 @@ class PithosImageBackend(object):
image = self._get_image(location)
if image:
yield image
def iter_public(self, filters=None):
filters = filters or {}
backend = self.backend
keys = [PLANKTON_PREFIX + 'name']
size_range = (None, None)
for key, val in filters.items():
if key == 'size_min':
size_range = (int(val), size_range[1])
......@@ -274,7 +274,7 @@ class PithosImageBackend(object):
size_range = (size_range[0], int(val))
else:
keys.append('%s = %s' % (PLANKTON_PREFIX + key, val))
for account in backend.list_accounts(None):
for container in backend.list_containers(None, account,
shared=True):
......@@ -285,12 +285,12 @@ class PithosImageBackend(object):
image = self._get_image(location)
if image:
yield image
def iter_shared(self, member):
"""Iterate over image ids shared to this member"""
backend = self.backend
# To get the list we connect as member and get the list shared by us
for container in backend.list_containers(member, self.user):
for object, version_id in backend.list_objects(member, self.user,
......@@ -303,26 +303,26 @@ class PithosImageBackend(object):
yield meta['uuid']
except (NameError, NotAllowedError):
continue
def list(self):
"""Iter over all images available to the user"""
return list(self.iter())
def list_public(self, filters, params):
images = list(self.iter_public(filters))
key = itemgetter(params.get('sort_key', 'created_at'))
reverse = params.get('sort_dir', 'desc') == 'desc'
images.sort(key=key, reverse=reverse)
return images
def list_users(self, image_id):
image = self.get_image(image_id)
assert image, "Image not found"
permissions = self._get_permissions(image['location'])
return [user for user in permissions.get('read', []) if user != '*']
def put(self, name, f, params):
assert 'checksum' not in params, "Passing a checksum is not supported"
assert 'id' not in params, "Passing an ID is not supported"
......@@ -333,23 +333,23 @@ class PithosImageBackend(object):
assert params.setdefault('container_format',
settings.DEFAULT_CONTAINER_FORMAT) in \
settings.ALLOWED_CONTAINER_FORMATS, "Invalid container_format"
container = settings.DEFAULT_PLANKTON_CONTAINER
filename = params.pop('filename', name)
location = 'pithos://%s/%s/%s' % (self.user, container, filename)
is_public = params.pop('is_public', False)
permissions = {'read': ['*']} if is_public else {}
size = params.pop('size', None)
hashmap, size = self._store(f, size)
meta = {}
meta['properties'] = params.pop('properties', {})
meta.update(name=name, status='available', **params)
self._update(location, size, hashmap, meta, permissions)
return self._get_image(location)
def register(self, name, location, params):
assert 'id' not in params, "Passing an ID is not supported"
assert location.startswith('pithos://'), "Invalid location"
......@@ -360,36 +360,36 @@ class PithosImageBackend(object):
assert params.setdefault('container_format',
settings.DEFAULT_CONTAINER_FORMAT) in \
settings.ALLOWED_CONTAINER_FORMATS, "Invalid container_format"
user = self.user
account, container, object = split_location(location)
meta = self._get_meta(location)
assert meta, "File not found"
size = int(params.pop('size', meta['bytes']))
if size != meta['bytes']:
raise BackendException("Invalid size")
checksum = params.pop('checksum', meta['hash'])
if checksum != meta['hash']:
raise BackendException("Invalid checksum")
is_public = params.pop('is_public', False)
permissions = {'read': ['*']} if is_public else {}
meta = {}
meta['properties'] = params.pop('properties', {})
meta.update(name=name, status='available', **params)
self._update_meta(location, meta)
self._update_permissions(location, permissions)
return self._get_image(location)
def remove_user(self, image_id, user):
image = self.get_image(image_id)
assert image, "Image not found"
location = image['location']
permissions = self._get_permissions(location)
try:
......@@ -397,22 +397,22 @@ class PithosImageBackend(object):
except ValueError:
return # User did not have access anyway
self._update_permissions(location, permissions)
def replace_users(self, image_id, users):
image = self.get_image(image_id)
assert image, "Image not found"
location = image['location']
permissions = self._get_permissions(location)
permissions['read'] = users
if image.get('is_public', False):
permissions['read'].append('*')
self._update_permissions(location, permissions)
def update(self, image_id, params):
image = self.get_image(image_id)
assert image, "Image not found"
location = image['location']
is_public = params.pop('is_public', None)
if is_public is not None:
......@@ -424,38 +424,10 @@ class PithosImageBackend(object):
read.discard('*')
permissions['read'] = list(read)
self.backend._update_permissions(location, permissions)
meta = {}
meta['properties'] = params.pop('properties', {})
meta.update(**params)
self._update_meta(location, meta)
return self.get_image(image_id)
class DummyImageBackend():
def __init__(self, user, images=None):
self.user = user
self.images = images or [{"status":"available","name":"Windows","checksum":"c4524d6f6e4f04ecf0212e50bb36f70114f459b223f932f1a3fa6a995e1334f7","created_at":"2012-03-28 14:39:43","disk_format":"diskdump","updated_at":"2012-03-28 16:56:34","properties":{"kernel":"Windows NT Kernel","osfamily":"windows","users":"Administrator","gui":"Windows, Aero Theme","sortorder":"7","size":"10537","os":"windows","root_partition":"2","description":"Windows 2008 R2, Aero Desktop Experience","OS":"windows"},"location":"pithos://images@okeanos.grnet.gr/pithos/windows-2008R2-7-x86_64.diskdump","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"34138811-299e-45db-9a53-acd5f1f7693c","size":11037310976,"metadata":{"values":{"kernel":"Windows NT Kernel","osfamily":"windows","users":"Administrator","gui":"Windows, Aero Theme","sortorder":"7","size":"10537","os":"windows","root_partition":"2","description":"Windows 2008 R2, Aero Desktop Experience","OS":"windows"}},"OS":"windows","description":"Windows 2008 R2, Aero Desktop Experience","kernel":"Windows NT Kernel","GUI":""},{"status":"available","name":"CentOS","checksum":"a7517d876f2387527f35e9d8c19cba4a37419279005354ca03bd625db1cf410e","created_at":"2012-03-28 13:37:28","disk_format":"diskdump","updated_at":"2012-03-28 16:56:33","properties":{"kernel":"2.6.32","osfamily":"linux","users":"root","gui":"No GUI","sortorder":"6","size":"601","os":"centos","root_partition":"1","description":"CentOS 6.0","OS":"centos"},"location":"pithos://images@okeanos.grnet.gr/pithos/centos-6.0-8-x86_64.diskdump","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"8f06e736-d890-4aed-8448-aa0e978695a9","size":628834304,"metadata":{"values":{"kernel":"2.6.32","osfamily":"linux","users":"root","gui":"No GUI","sortorder":"6","size":"601","os":"centos","root_partition":"1","description":"CentOS 6.0","OS":"centos"}},"OS":"centos","description":"CentOS 6.0","kernel":"2.6.32","GUI":""},{"status":"available","name":"Fedora","checksum":"407c7fa05c14937e213503783149aa74ad9bcfc8783ccb653a419fb86bffe0d9","created_at":"2012-03-28 13:52:45","disk_format":"diskdump","updated_at":"2012-03-28 16:56:32","properties":{"kernel":"3.1.9","osfamily":"linux","users":"root user","gui":"GNOME 3.2","sortorder":"5","size":"2641","os":"fedora","root_partition":"1","description":"Fedora 16 Desktop Edition","OS":"fedora"},"location":"debian_base-6.0-7-x86_64","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"1ba3666b-6e57-4d52-813b-e2a33185d12d","size":2765684736,"metadata":{"values":{"kernel":"3.1.9","osfamily":"linux","users":"root user","gui":"GNOME 3.2","sortorder":"5","size":"2641","os":"fedora","root_partition":"1","description":"Fedora 16 Desktop Edition","OS":"fedora"}},"OS":"fedora","description":"Fedora 16 Desktop Edition","kernel":"3.1.9","GUI":""},{"status":"available","name":"Kubuntu","checksum":"a149289f512d70c8f9f6acb0636d2ea9a5b5c3ec0b83e4398aed4a5678da6848","created_at":"2012-03-28 15:05:52","disk_format":"diskdump","updated_at":"2012-03-28 16:56:31","properties":{"kernel":"3.0.0","osfamily":"linux","users":"user","gui":"KDE 4.7.4","sortorder":"4","size":"2850","os":"kubuntu","root_partition":"1","description":"Kubuntu 11.10","OS":"kubuntu"},"location":"pithos://images@okeanos.grnet.gr/pithos/kubuntu-11.10-1-x86_64.diskdump","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"79d24739-af8f-436b-8f6e-eb2d908e0b7e","size":2985041920,"metadata":{"values":{"kernel":"3.0.0","osfamily":"linux","users":"user","gui":"KDE 4.7.4","sortorder":"4","size":"2850","os":"kubuntu","root_partition":"1","description":"Kubuntu 11.10","OS":"kubuntu"}},"OS":"kubuntu","description":"Kubuntu 11.10","kernel":"3.0.0","GUI":""},{"status":"available","name":"Ubuntu","checksum":"f508e1fc8d9cbbd360a9cfc3a68e475933063c77691dac652cb7a5c824791e1b","created_at":"2012-03-28 14:08:35","disk_format":"diskdump","updated_at":"2012-03-28 16:58:10","properties":{"kernel":"3.0.0","osfamily":"linux","users":"user","gui":"Unity 4.22","sortorder":"3","size":"2540","os":"ubuntu","root_partition":"1","description":"Ubuntu 11.10","OS":"ubuntu"},"location":"pithos://images@okeanos.grnet.gr/pithos/ubuntu-11.10-1-x86_64.diskdump","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"d8317451-820a-4c41-917b-665492ab0f81","size":2660171776,"metadata":{"values":{"kernel":"3.0.0","osfamily":"linux","users":"user","gui":"Unity 4.22","sortorder":"3","size":"2540","os":"ubuntu","root_partition":"1","description":"Ubuntu 11.10","OS":"ubuntu"}},"OS":"ubuntu","description":"Ubuntu 11.10","kernel":"3.0.0","GUI":""},{"status":"available","name":"Debian Desktop","checksum":"a7bea0bf6815168a281b505454cd9b2f07ffb9cad0d92e08e950a633c0f05bd2","created_at":"2012-03-28 14:55:37","disk_format":"diskdump","updated_at":"2012-04-03 15:52:09","properties":{"kernel":"2.6.32","osfamily":"linux","users":"root user","gui":"GNOME 2.30","sortorder":"2","size":"3314","os":"debian","root_partition":"1","description":"Debian Squeeze Desktop","OS":"debian"},"location":"pithos://images@okeanos.grnet.gr/pithos/debian_desktop-6.0-6-x86_64.diskdump","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"d7970a84-99b9-40be-b790-ce61001a5b9a","size":3482439680,"metadata":{"values":{"kernel":"2.6.32","osfamily":"linux","users":"root user","gui":"GNOME 2.30","sortorder":"2","size":"3314","os":"debian","root_partition":"1","description":"Debian Squeeze Desktop","OS":"debian"}},"OS":"debian","description":"Debian Squeeze Desktop","kernel":"2.6.32","GUI":""},{"status":"available","name":"Debian Base","checksum":"65352163c9842d6fbc5717437811495f163da2338c807819a1d4d7a50766e56c","created_at":"2012-03-28 13:39:38","disk_format":"diskdump","updated_at":"2012-03-28 16:56:29","properties":{"kernel":"2.6.32","osfamily":"linux","users":"root","gui":"No GUI","sortorder":"1","size":"451","os":"debian","root_partition":"1","description":"Debian Squeeze Base System","OS":"debian"},"location":"pithos://images@okeanos.grnet.gr/pithos/debian_base-6.0-7-x86_64.diskdump","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"49971ade-3bbc-4700-9a84-b3ca00133850","size":471891968,"metadata":{"values":{"kernel":"2.6.32","osfamily":"linux","users":"root","gui":"No GUI","sortorder":"1","size":"451","os":"debian","root_partition":"1","description":"Debian Squeeze Base System","OS":"debian"}},"OS":"debian","description":"Debian Squeeze Base System","kernel":"2.6.32","GUI":""},{"id":"20","name":"(deleted image)","size":-1,"progress":100,"status":"DELETED"},{"id":"21","name":"(deleted image)","size":-1,"progress":100,"status":"DELETED"}]
def iter(self):
return self.images
def get_image(self, image_id):
for i in self.images:
if i['id'] == image_id:
return i
return None
def close(self):
pass
def list_public(self, filters, params):
return self.images
ImageBackend = PithosImageBackend
ImageBackend = DummyImageBackend
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment