Merge branch 'hotfix-0.14.3' into develop

Conflicts:
	Changelog
	snf-astakos-app/astakos/im/messages.py
	snf-pithos-app/pithos/api/manage_accounts/__init__.py
	snf-pithos-app/pithos/api/management/commands/reconcile-resources-pithos.py
	snf-pithos-app/pithos/api/settings.py
	snf-pithos-app/pithos/api/util.py
	snf-pithos-backend/pithos/backends/lib/sqlite/node.py
	version

Changelog

@@ -13,7 +13,6 @@ v0.14next
 
 Released: UNRELEASED
 
-
 Synnefo-wide
 ------------
 
@@ -50,6 +49,24 @@ objects by domain attribute. This is used by Plankton for listing VM images.
 
 * Enforce container-level atomicity in (most) Pithos API calls.
 
+
+.. _Changelog-0.14.3:
+
+v0.14.3
+=======
+
+Released: Thu Jul 25 12:22:47 EEST 2013
+
+Synnefo-wide
+------------
+
+* Use the SYNNEFO_TRACE environmental variable to control whether the greenlet
+  tracing code will get loaded or not.
+* Split the HIDDEN_COOKIES setting in HIDDEN_HEADERS and HIDDEN_COOKIES, and
+  add the MAIL_MAX_LEN setting, to limit the mail size for unhandled
+  exceptions.
+
+
 .. _Changelog-0.14.2:
 
 Released: Fri Jul 12 13:13:32 EEST 2013

NEWS

@@ -5,6 +5,12 @@ Unified NEWS file for Synnefo versions >= 0.13
 
 Since v0.13 all Synnefo components have been merged into a single repository.
 
+.. _NEWS-0.14.3:
+
+v0.14.3
+=======
+
+Released: Thu Jul 25 12:22:47 EEST 2013
 
 .. _NEWS-0.14.2:
 

docs/admin-guide.rst

@@ -1809,6 +1809,7 @@ Changelog, NEWS
 ===============
 
 
+* v0.14.3 :ref:`Changelog <Changelog-0.14.3>`, :ref:`NEWS <NEWS-0.14.3>`
 * v0.14.2 :ref:`Changelog <Changelog-0.14.2>`, :ref:`NEWS <NEWS-0.14.2>`
 * v0.14 :ref:`Changelog <Changelog-0.14>`, :ref:`NEWS <NEWS-0.14>`
 * v0.13 :ref:`Changelog <Changelog-0.13>`, :ref:`NEWS <NEWS-0.13>`

snf-astakos-app/astakos/api/quotas.py

@@ -83,8 +83,7 @@ def commissions(request):
         return get_pending_commissions(request)
     elif method == 'POST':
         return issue_commission(request)
-    else:
-        raise BadRequest('Method not allowed.')
+    return api.api_method_not_allowed(request)
 
 
 @api.api_method(http_method='GET', token_required=True, user_required=False)

snf-astakos-app/astakos/im/functions.py

@@ -371,7 +371,7 @@ def checkAllowed(entity, request_user, admin_only=False):
 
 def checkAlive(project):
     if not project.is_alive:
-        m = _(astakos_messages.NOT_ALIVE_PROJECT) % project.__dict__
+        m = _(astakos_messages.NOT_ALIVE_PROJECT) % project.id
         raise PermissionDenied(m)
 
 
@@ -805,7 +805,7 @@ def resume(project_id, request_user=None):
     checkAllowed(project, request_user, admin_only=True)
 
     if not project.is_suspended:
-        m = _(astakos_messages.NOT_SUSPENDED_PROJECT) % project.__dict__
+        m = _(astakos_messages.NOT_SUSPENDED_PROJECT) % project.id
         raise PermissionDenied(m)
 
     project.resume()

snf-astakos-app/astakos/im/messages.py

@@ -224,9 +224,8 @@ QH_SYNC_ERROR = 'Failed to get synchronized with quotaholder.'
 UNIQUE_PROJECT_NAME_CONSTRAIN_ERR = (
     'The project name (as specified in its application\'s definition) must '
     'be unique among all active projects.')
-INVALID_PROJECT = 'Project %(id)s is invalid.'
-NOT_ALIVE_PROJECT = 'Project %(id)s is not alive.'
-NOT_SUSPENDED_PROJECT = 'Project %(id)s is not suspended.'
+NOT_ALIVE_PROJECT = 'Project %s is not alive.'
+NOT_SUSPENDED_PROJECT = 'Project %s is not suspended.'
 NOT_ALLOWED = 'You do not have the permissions to perform this action.'
 MEMBER_NUMBER_LIMIT_REACHED = (
     'You have reached the maximum number of members for this Project.')
@@ -269,7 +268,6 @@ APPLICATION_CANNOT_DENY = "Cannot deny application %s in state '%s'"
 APPLICATION_CANNOT_DISMISS = "Cannot dismiss application %s in state '%s'"
 APPLICATION_CANNOT_CANCEL = "Cannot cancel application %s in state '%s'"
 APPLICATION_CANCELLED = "Your project application has been cancelled."
-
 REACHED_PENDING_APPLICATION_LIMIT = ("You have reached the maximum number "
                                      "of pending project applications: %s.")
 

snf-astakos-app/astakos/im/models.py

@@ -1695,6 +1695,8 @@ class ProjectApplication(models.Model):
         project.application = self
         project.last_approval_date = now
         project.save()
+        if project.is_deactivated():
+            project.resume()
         return project
 
     @property

snf-astakos-app/astakos/im/project_notif.py

@@ -113,11 +113,12 @@ def application_approve_notify(application):
 
 
 def project_termination_notify(project):
+    app = project.application
     try:
         notification = build_notification(
             SENDER,
             [project.application.owner.email],
-            _(messages.PROJECT_TERMINATION_SUBJECT) % project.__dict__,
+            _(messages.PROJECT_TERMINATION_SUBJECT) % app.__dict__,
             template='im/projects/project_termination_notification.txt',
             dictionary={'object': project}
         ).send()

snf-astakos-app/astakos/im/templates/im/projects/project_suspension_notification.txt

 {% extends "im/email.txt" %}
 
 {% block content %}
-Your project application request ({{object.name}}) has been suspended.
+Your project ({{object.name}}) has been suspended.
 {% endblock content %}

snf-astakos-app/astakos/im/templates/im/projects/project_termination_notification.txt

 {% extends "im/email.txt" %}
   
 {% block content %}
-Your project application request ({{object.name}}) has been terminated.
-{% endblock content %}
\ No newline at end of file
+Your project ({{object.application.name}}) has been terminated.
+{% endblock content %}

snf-astakos-app/astakos/im/tests/api.py

@@ -380,6 +380,10 @@ class QuotaAPITest(TestCase):
         self.assertEqual(r11['usage'], 102)
         self.assertEqual(r11['pending'], 101)
 
+        # Bad Request
+        r = client.head(u('commissions'))
+        self.assertEqual(r.status_code, 400)
+
 
 class TokensApiTest(TestCase):
     def setUp(self):

snf-astakos-app/astakos/im/tests/projects.py

@@ -230,3 +230,8 @@ class TestProjects(TestCase):
         newlimit = user_quotas[self.member.uuid]['system'][resource]['limit']
         # 200 - 100 from project
         self.assertEqual(newlimit, 100)
+
+        # support email gets rendered in emails content
+        for mail in get_mailbox('user@synnefo.org'):
+            self.assertTrue(settings.CONTACT_EMAIL in
+                            mail.message().as_string())

snf-branding/synnefo_branding/utils.py

+# Copyright 2013 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+from django.conf import settings as django_settings
 from synnefo_branding import settings
 from django.template.loader import render_to_string as django_render_to_string
 
 
 def get_branding_dict(prepend=None):
-    dct = {}
+    # CONTACT_EMAIL may not be a branding setting. We include it here though
+    # for practial reasons.
+    dct = {'support': django_settings.CONTACT_EMAIL}
     for key in dir(settings):
         if key == key.upper():
             newkey = key.lower()
@@ -22,9 +58,11 @@ def brand_message(msg, **extra_args):
 def render_to_string(template_name, dictionary=None, context_instance=None):
     if not dictionary:
         dictionary = {}
+
     if isinstance(dictionary, dict):
         newdict = get_branding_dict("BRANDING")
         newdict.update(dictionary)
     else:
         newdict = dictionary
+
     return django_render_to_string(template_name, newdict, context_instance)

snf-common/synnefo/lib/middleware/cleanse.py

@@ -39,27 +39,64 @@ from django.views import debug
 
 import re
 
+HIDDEN_ALL = settings.HIDDEN_COOKIES + settings.HIDDEN_HEADERS
+
 
 def mail_admins_safe(subject, message, fail_silently=False, connection=None):
     '''
     Wrapper function to cleanse email body from sensitive content before
     sending it
     '''
+    new_msg = ""
+
+    if len(message) > settings.MAIL_MAX_LEN:
+        new_msg += "Mail size over limit (truncated)\n\n"
+        message = message[:settings.MAIL_MAX_LEN]
+
+    for line in message.splitlines():
+        # Lines of interest in the mail are in the form of
+        # key:value.
+        try:
+            (key, value) = line.split(':', 1)
+        except ValueError:
+            new_msg += line + '\n'
+            continue
+
+        new_msg += key + ':'
+
+        # Special case when the first header / cookie printed
+        # (prefixed by 'META:{' or 'COOKIES:{') needs to be hidden.
+        if value.startswith('{'):
+            try:
+                (newkey, newval) = value.split(':', 1)
+            except ValueError:
+                new_msg += value + '\n'
+                continue
 
-    HIDDEN_ALL = settings.HIDDEN_SETTINGS + "|" + settings.HIDDEN_COOKIES
-    message = re.sub("((\S+)?(%s)(\S+)?(:|\=)( )?)('|\"?)\S+('|\"?)"
-                     % HIDDEN_ALL, r"\1*******", message)
+            new_msg += newkey + ':'
+            key = newkey.lstrip('{')
+            value = newval
 
-    return mail.mail_admins_plain(subject, message, fail_silently, connection)
+        if key.strip(" '") not in HIDDEN_ALL:
+            new_msg += value + '\n'
+            continue
+
+        # Append value[-1] to the clensed string, so that commas / closing
+        # brackets are printed correctly.
+        # (it will 'eat up' the closing bracket if the header is the last one
+        # printed)
+        new_msg += ' ' + '*'*8 + value[-1] + '\n'
+
+    return mail.mail_admins_plain(subject, new_msg, fail_silently, connection)
 
 
 class CleanseSettingsMiddleware(object):
+    '''
+    Prevent django from printing sensitive information (paswords, tokens
+    etc), when handling server errors (for both DEBUG and no-DEBUG
+    deployments.
+    '''
     def __init__(self):
-        '''
-        Prevent django from printing sensitive information (paswords, tokens
-        etc), when handling server errors (for both DEBUG and no-DEBUG
-        deployments.
-        '''
         debug.HIDDEN_SETTINGS = re.compile(settings.HIDDEN_SETTINGS)
 
         if not hasattr(mail, 'mail_admins_plain'):

snf-common/synnefo/lib/trace.py

+# Copyright 2013 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+
+def set_signal_trap():
+    from os import getpid
+    from traceback import format_stack, print_exc
+    from signal import signal, SIGTRAP
+    from sys import stderr
+    import gc
+
+    def greenlet_trace(arg):
+        i = 0
+        stderr.write("--- Greenlet trace: %s\n" % arg)
+        for ob in gc.get_objects():
+            if not isinstance(ob, greenlet):
+                continue
+            if not ob:
+                continue
+            i = i + 1
+            stderr.write(("--- > Greenlet %d:\n" % i +
+                          "".join(format_stack(ob.gr_frame)) + "\n\n"))
+        stderr.write("--- End of trace: %s\n" % arg)
+
+    try:
+        from greenlet import greenlet
+    except ImportError:
+
+        def greenlet_trace(arg):
+            return
+
+    def handle_trap(*args):
+        try:
+            import trap_inject
+            reload(trap_inject)
+            trap_inject.inject()
+        except ImportError:
+            pass
+        except:
+            print_exc()
+
+        msg = ('=== pid: %s' % getpid()) + '\n'.join(format_stack()) + '\n'
+        stderr.write(msg)
+        greenlet_trace('TRAP')
+
+    signal(SIGTRAP, handle_trap)

snf-common/synnefo/settings/__init__.py

@@ -57,14 +57,23 @@ if os.path.exists(SYNNEFO_SETTINGS_DIR):
         conffiles = [f for f in entries if os.path.isfile(f) and
                      f.endswith(".conf")]
     except Exception as e:
-        print >>sys.stderr, "Failed to list *.conf files under %s" % \
-                            SYNNEFO_SETTINGS_DIR
+        print >> sys.stderr, "Failed to list *.conf files under %s" % \
+                             SYNNEFO_SETTINGS_DIR
         raise SystemExit(1)
     conffiles.sort()
     for f in conffiles:
         try:
             execfile(os.path.abspath(f))
         except Exception as e:
-            print >>sys.stderr, "Failed to read settings file: %s [%r]" % \
-                                (os.path.abspath(f), e)
+            print >> sys.stderr, "Failed to read settings file: %s [%r]" % \
+                                 (os.path.abspath(f), e)
             raise SystemExit(1)
+
+
+from os import environ
+# The tracing code is enabled by an environmental variable and not a synnefo
+# setting, on purpose, so that you can easily control whether it'll get loaded
+# or not, based on context (eg enable it for gunicorn but not for eventd).
+if environ.get('SYNNEFO_TRACE'):
+    from synnefo.lib import trace
+    trace.set_signal_trap()

snf-cyclades-app/synnefo/ui/templates/synnefo-windows.rdp

@@ -9,7 +9,7 @@ audiomode:i:0
 redirectprinters:i:1
 redirectcomports:i:0
 redirectsmartcards:i:0
-redirectclipboard:i:0
+redirectclipboard:i:1
 redirectposdevices:i:0
 drivestoredirect:s:
 displayconnectionbar:i:1

snf-pithos-app/pithos/api/manage_accounts/__init__.py

@@ -164,13 +164,22 @@ class ManageAccounts():
         if dest_account not in self._existing_accounts():
             raise NameError('%s does not exist' % dest_account)
 
-        self._copy_object(src_account, src_container, src_name,
-                          dest_account, move=True)
-        if not silent:
+        trans = self.backend.wrapper.conn.begin()
+        try:
+            self._copy_object(src_account, src_container, src_name,
+                              dest_account, move=True)
+
             if dry:
-                print "Database commit skipped."
+                if not silent:
+                    print "Skipping database commit."
+                trans.rollback()
             else:
-                print "%s is deleted" % src_account
+                trans.commit()
+                if not silent:
+                    print "%s is deleted." % src_account
+        except:
+            trans.rollback()
+            raise
 
     def _copy_object(self, src_account, src_container, src_name,
                      dest_account, move=False):
@@ -287,12 +296,22 @@ class ManageAccounts():
             return
         self._merge_account(src_account, dest_account, delete_src)
 
-        if not silent:
+        trans = self.backend.wrapper.conn.begin()
+        try:
+            self._merge_account(src_account, dest_account, delete_src)
+
             if dry:
-                print "Database commit skipped."
+                if not silent:
+                    print "Skipping database commit."
+                trans.rollback()
             else:
-                print "%s has been merged into %s." % (src_account,
-                                                       dest_account)
+                trans.commit()
+                if not silent:
+                    msg = "%s merged into %s."
+                    print msg % (src_account, dest_account)
+        except:
+            trans.rollback()
+            raise
 
     def _delete_container_contents(self, account, container):
         self.backend.delete_container(account, account, container,
@@ -328,11 +347,21 @@ class ManageAccounts():
             return
         self._delete_account(account)
 
-        if not silent:
+        trans = self.backend.wrapper.conn.begin()
+        try:
+            self._delete_account(account)
+
             if dry:
-                print "Database commit skipped."
+                if not silent:
+                    print "Skipping database commit."
+                trans.rollback()
             else:
-                print "%s has been deleted." % account
+                trans.commit()
+                if not silent:
+                    print "%s is deleted." % account
+        except:
+            trans.rollback()
+            raise
 
     @manage_transactions(lock_container_path=True)
     def create_account(self, account):

snf-pithos-app/pithos/api/manage_accounts/cli/__init__.py

@@ -138,16 +138,16 @@ def set_container_quota(args):
         failed = []
 
         def update_container_policy(account):
-            utils.backend.wrapper.execute()
+            trans = utils.backend.wrapper.conn.begin()
             try:
                 utils.backend.update_container_policy(
                     account, account, args.container, {'quota': quota}
                 )
                 if args.dry:
                     print "Skipping database commit."
-                    utils.backend.wrapper.rollback()
+                    trans.rollback()
                 else:
-                    utils.backend.wrapper.commit()
+                    trans.commit()
             except Exception, e:
                 failed.append((account, e))
 

snf-pithos-app/pithos/api/management/commands/reconcile-resources-pithos.py

@@ -31,17 +31,17 @@
 # interpreted as representing official policies, either expressed
 # or implied, of GRNET S.A.
 
-from django.core.management.base import NoArgsCommand, CommandError
+from django.core.management.base import NoArgsCommand
 
 from optparse import make_option
 
 from pithos.api.util import get_backend
 from pithos.api.resources import resources
-from pithos.backends.modular import CLUSTER_NORMAL, DEFAULT_SOURCE
+from pithos.backends.modular import DEFAULT_SOURCE
 
 from synnefo.webproject.management import utils
 
-from astakosclient.errors import QuotaLimit
+from astakosclient.errors import QuotaLimit, NotFound
 
 backend = get_backend()
 
@@ -71,21 +71,26 @@ class Command(NoArgsCommand):
     def handle_noargs(self, **options):
         try:
             backend.pre_exec()
-            qh_result = backend.astakosclient.service_get_quotas(
-                backend.service_token)
+            userid = options['userid']
 
-            users = (options['userid'],) if options['userid'] else None
-            account_nodes = backend.node.node_accounts(users)
-            if not account_nodes:
-                raise CommandError('No users found.')
+            # Get holding from Pithos DB
+            db_usage = backend.node.node_account_usage(userid)
 
-            db_usage = {}
-            for path, node in account_nodes:
-                size = backend.node.node_account_usage(node, CLUSTER_NORMAL)
-                db_usage[path] = size or 0
+            users = set(db_usage.keys())
+            if userid and userid not in users:
+                self.stdout.write("User '%s' does not exist in DB!\n" % userid)
+                return
 
-            users = set(qh_result.keys())
-            users.update(db_usage.keys())
+            # Get holding from Quotaholder
+            try:
+                qh_result = backend.astakosclient.service_get_quotas(
+                    backend.service_token, userid)
+            except NotFound:
+                self.stdout.write(
+                    "User '%s' does not exist in Quotaholder!\n" % userid)
+                return
+
+            users.update(qh_result.keys())
 
             pending_exists = False
             unknown_user_exists = False
@@ -96,8 +101,7 @@ class Command(NoArgsCommand):
                     qh_all = qh_result[uuid]
                 except KeyError:
                     self.stdout.write(
-                        "User '%s' does not exist in Quotaholder!\n" % uuid
-                    )
+                        "User '%s' does not exist in Quotaholder!\n" % uuid)
                     unknown_user_exists = True
                     continue
                 else:
@@ -115,14 +119,13 @@ class Command(NoArgsCommand):
                             self.stdout.write(
                                 "Pending commission. "
                                 "User '%s', resource '%s'.\n" %
-                                (uuid, resource)
-                            )
+                                (uuid, resource))
                             pending_exists = True
                             continue
 
                         qh_value = qh_resource['usage']
 
-                        if  db_value != qh_value: