Merge branch 'release-0.18'

.gitignore

@@ -51,3 +51,5 @@ snf-deploy/files/root/.ssh
 snf-deploy/files/root/ddns
 *.egg
 *.tar.gz
+snf-admin-app/synnefo_admin/admin/static/min-css
+snf-admin-app/synnefo_admin/admin/static/css

Changelog

@@ -6,6 +6,84 @@ Unified Changelog file for Synnefo versions >= 0.13
 Since v0.13 most of the Synnefo components have been merged into a single
 repository and have aligned versions.
 
+.. _Changelog-0.18:
+
+v0.18rc3
+========
+
+Released: UNRELEASED
+
+Cyclades
+--------
+
+* Fix minor logging issues
+
+Astakos
+-------
+
+* Add option to control whether to send e-mail to the user when suspending them
+
+v0.18rc2
+========
+
+Released: UNRELEASED
+
+Documentation
+-------------
+
+* Reword Administrator's guide
+
+v0.18rc1
+========
+
+Released: UNRELEASED
+
+Astakos
+-------
+
+* Introduce 'overquota' state on project memberships, which can be used by
+  third-party plugins in order to implement a quota reclamation policy.
+  Setting `QUOTA_POLICY_MODULE` specifies the plugin that updates the
+  overquota state on project actions.
+* Support suspending and unsuspending project memberships.
+* User deactivation now automatically suspends user's system project, owned
+  projects, and project memberships. Reactivation unsuspends them.
+* Add command `user-check`. It supports suspending projects for previously
+  deactivated users.
+* Send an informative email to the user's current email address when they
+  request to change their email.
+
+Cyclades
+--------
+
+* Command `enforce-resources-cyclades` now provides an option to "soft"
+  enforce dangerous resources. There is now no default list of resources to
+  check; the administrator must provide one.
+* Make dispatcher use DISPATCHER_LOGGING_SETUP from synnefo settings to setup
+  its logging. Remove previous settings.
+* Add a raven processor to filter sensitive information sent to Sentry using
+  the Sentry logging handler.
+* Imporove SynnefoExceptionReporterFilter to better cleanse request body.
+* Make eventd detect Ganeti Master failovers and allow having multiple eventd
+  instances running in the Ganeti cluster.
+
+Admin
+-----
+
+* Add mechanism to generate css files on packaging
+* Improve the displayed data in the tables
+* Display more information regarding the enabled authentication providers
+* Display pending modifications of projects
+* Add the action 'modify user e-mail' in the Admin interface
+* Display data related with the modification of users' e-mails like 'e-mail
+  pending verification', 'e-mail change requested at', 'initially accepted
+  e-mail'
+
+Pithos
+------
+
+* Optimize object latest listing query
+
 .. _Changelog-0.17:
 
 v0.17

NEWS

@@ -5,6 +5,25 @@ Unified NEWS file for Synnefo versions >= 0.13
 
 Since v0.13 all Synnefo components have been merged into a single repository.
 
+.. _NEWS-0.18:
+
+v0.18rc1
+========
+
+Released: UNKNOWN
+
+The Synnefo 0.18 release brings significant bug fixes across Synnefo.
+
+The most notable changes are:
+
+* Improved project management and quota policy enforcement
+* Performance optimizations of Pithos object listing queries
+* Support for modifying user e-mails from the Admin Panel
+* Various Admin panel enhancements
+* Support for multiple eventd instances and automatic ganeti master failover
+  detection
+* Support for Sentry
+
 .. _NEWS-0.17:
 
 v0.17

README.md

@@ -47,7 +47,7 @@ for more information on the Synnefo users and developers lists.
 Copyright and license
 =====================
 
-Copyright (C) 2010-2015 GRNET S.A. and individual contributors
+Copyright (C) 2010-2016 GRNET S.A. and individual contributors
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

astakosclient/docs/conf.py

@@ -5,7 +5,7 @@ sys.path.insert(0, os.path.abspath('..'))
 from astakosclient.version import __version__
 
 project = u'nnefo'
-copyright = u'2012-2013, GRNET'
+copyright = u'2012-2016, GRNET'
 version = __version__
 release = __version__
 html_title = 'synnefo ' + version

ci/ci_wheezy.conf

@@ -76,6 +76,11 @@ ssh_port =
 # installation of custom packages (e.g. Ganeti, Archipelago).
 local_package_dir =
 
+# Set this to a wildcard dns service (e.g. xip.io) for snf-deploy to use it 
+# as the base domain of the deployed services. Leave it empty to use the 
+# default synnefo.live domain.
+wildcard_dns =
+
 
 [Burnin]
 # Maybe add some burnin options

ci/snf-ci

@@ -201,12 +201,12 @@ def main():  # pylint: disable=too-many-statements, too-many-branches
                                  image=options.image,
                                  ssh_keys=options.ssh_keys,
                                  server_name=options.server_name)
+    if getattr(options, BUILD_SYNNEFO_CMD, False):
         synnefo_ci.clone_repo(
             synnefo_repo=options.synnefo_repo,
             synnefo_branch=options.synnefo_branch,
             local_repo=options.local_repo,
             pull_request=options.pull_request)
-    if getattr(options, BUILD_SYNNEFO_CMD, False):
         synnefo_ci.build_packages()
         if options.fetch_packages:
             dest = os.path.abspath(options.fetch_packages)

ci/utils.py

@@ -483,7 +483,7 @@ class SynnefoCI(object):
         echo 'deb https://deb.nodesource.com/node_0.12 wheezy main' >> /etc/apt/sources.list.d/nodejs.list
         echo 'deb-src https://deb.nodesource.com/node_0.12 wheezy main' >> /etc/apt/sources.list.d/nodejs.list
         apt-get update
-        apt-get install -q=2 --force-yes nodejs
+        apt-get install -q=2 --force-yes nodejs ruby ruby-dev
         """
         _run(cmd, False)
 
@@ -1156,6 +1156,21 @@ class SynnefoCI(object):
         """.format(fabric.env.password)
         _run(cmd, False)
 
+        wildcard_dns = self.get_config(
+                'Deployment', 'wildcard_dns', False, '').strip()
+        if wildcard_dns:
+            address = self.temp_config.get(str(self.build_id), 'server_ip')
+            domain = "{0}.{1}".format(address, wildcard_dns)
+            self.logger.debug("Setting domain to {0}".format(domain))
+            cmd = """
+            sed -i 's/^domain.*=.*/domain = {0}/' /etc/snf-deploy/nodes.conf
+            """.format(domain)
+            _run(cmd, False)
+            cmd = """
+            sed -i 's/^domain.*=.*/domain = {0}/' /etc/snf-deploy/ganeti.conf
+            """.format(domain)
+            _run(cmd, False)
+
         self.logger.debug("Run snf-deploy")
         cmd = """
         snf-deploy --disable-colors --autoconf synnefo

docs/admin-guide.rst

@@ -430,6 +430,22 @@ add up quota from different projects. Note also that if allocating an entity
 requires multiple resources (e.g. cpu and ram for a Cyclades VM) these must
 be all assigned to a single project.
 
+Reclaiming resources
+````````````````````
+
+When a project is deactivated or a user is removed from a project, the quota
+that have been granted to the user are revoked. If the user still owns
+resources assigned to the project, the user quota appear overlimit on that
+project. The services are responsible to inspect the overquota state of
+users and reclaim their resources. For instance, cyclades provides
+the management command ``enforce-resources-cyclades`` to reclaim VMs,
+volumes, and floating IPs.
+
+When a user is deactivated, their system project, owned projects and project
+memberships are suspended. Subsequently, the user's resources can be
+reclaimed as explained above.
+
+
 Control projects
 ````````````````
 
@@ -1461,20 +1477,30 @@ quota limits, dependent on the overlimit resource:
 * `cyclades.cpu`: Shutdown VMs
 * `cyclades.total_ram`: Delete VMs
 * `cyclades.ram`: Shutdown VMs
-* `cyclades.disk`: Delete VMs
-* `cyclades.floating_ip`: Detach and remove IPs
+* `cyclades.disk`: Delete volumes (may also trigger VM deletion)
+* `cyclades.floating_ip`: Detach and delete IPs
 
 VMs to be deleted/shutdown are chosen first by state in the following order:
 ERROR, BUILD, STOPPED, STARTED or RESIZE and then by decreasing ID. When
 needing to remove IPs, we first choose IPs that are free, then those
 attached to VMs, using the same VM ordering.
 
-By default, the command checks only the following resources: `cyclades.cpu`,
-`cyclades.ram`, and `cyclades.floating_ip`; that is, the less dangerous
-ones, those that do not result in *deleting* any VM. One can change the
-default behavior by specifying the desired resources with option
-``--resources``. It is also possible to specify users to be checked or
-excluded.
+You need to specify the resources to be checked, using the option
+``--resources``. A safe first attempt would be to specify
+``cyclades.cpu,cyclades.ram``, that is, to check the less dangerous resources,
+those that do not result in *deleting* any VM, volume, or IP.
+
+If you want to handle overlimit quota in a safer way for resources that
+would normally trigger a deletion, you can use the option
+``--soft-resources``. Enforcing e.g. `cyclades.vm` in a "soft" way will
+shutdown the VMs rather than deleting them. This is useful as an initial
+warning for a user who is overquota; but notice that the user may restart
+their shutdown VMs, if the resources that control starting VMs allows them
+to do so.
+
+With option ``--list-resources`` you can inspect the available resources
+along with the related standard and soft enforce actions. It is also
+possible to specify users and projects to be checked or excluded.
 
 Actual enforcement is done with option ``--fix``. In order to control the
 load that quota enforcement may cause on Cyclades, one can limit the number
@@ -3051,6 +3077,7 @@ Upgrade Notes
    v0.15 -> v0.16 <upgrade/upgrade-0.16>
    v0.16.1 -> v0.16.2 <upgrade/upgrade-0.16.2>
    v0.16.2 -> v0.17 <upgrade/upgrade-0.17>
+   v0.17 -> v0.18 <upgrade/upgrade-0.18>
 
 
 .. _changelog-news:
@@ -3059,6 +3086,7 @@ Changelog, NEWS
 ===============
 
 
+* v0.18 :ref:`Changelog <Changelog-0.18>`, :ref:`NEWS <NEWS-0.18>`
 * v0.17 :ref:`Changelog <Changelog-0.17>`, :ref:`NEWS <NEWS-0.17>`
 * v0.16.2 :ref:`Changelog <Changelog-0.16.2>`, :ref:`NEWS <NEWS-0.16.2>`
 * v0.16.1 :ref:`Changelog <Changelog-0.16.1>`, :ref:`NEWS <NEWS-0.16.1>`

docs/conf.py

@@ -8,7 +8,7 @@ reload(synnefo.versions)
 from synnefo.versions.app import __version__
 
 project = u'synnefo'
-copyright = u'2012-2015, GRNET'
+copyright = u'2012-2016, GRNET'
 version = __version__
 release = __version__
 html_title = 'synnefo ' + version

docs/upgrade/upgrade-0.18.rst

+Upgrade to Synnefo v0.18
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Upgrade Steps
+=============
+
+The upgrade to v0.18 consists of the following steps:
+
+#. Stop gunicorn in all nodes
+
+   .. code-block:: console
+
+      # service gunicorn stop
+
+#. Upgrade Synnefo on all nodes to the latest version (0.18)
+
+   .. code-block:: console
+
+      # apt-get update
+      # apt-get upgrade
+
+#. Run migrations on Astakos.
+
+   .. code-block:: console
+
+      astakos.host$ snf-manage migrate
+
+
+   From this version on, user deactivation triggers suspension of all projects
+   and project memberships related to the user. To apply this new policy to
+   users that have already been deactivated, run:
+
+   .. code-block:: console
+
+      astakos.host$ snf-manage user-check --all-users --suspend-deactivated --noemail --fix
+
+#. Start gunicorn
+
+  .. code-block:: console
+
+     # service gunicorn start
+
+
+New configuration options
+=========================
+
+On the admin app, there is a new access control option regarding the new modify
+email action. The action setting is named 'modify_email'. The list of user
+groups defined in this have access on the modify email action.
+
+The following line (modified accordingly) should be added on 'ADMIN_RBAC'
+setting under the 'user' dictionary:
+
+.. code-block:: console
+
+   'modify_email': [ADMIN_HELPDESK_GROUP, ADMIN_GROUP],

snf-admin-app/setup.py

-# Copyright (C) 2010-2014 GRNET S.A.
+# Copyright (C) 2010-2016 GRNET S.A.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -25,6 +25,7 @@ from fnmatch import fnmatchcase
 from distutils.util import convert_path
 
 HERE = os.path.abspath(os.path.normpath(os.path.dirname(__file__)))
+os.chdir(HERE)
 
 from synnefo_admin.version import __version__
 
@@ -137,6 +138,41 @@ def find_package_data(
                 out.setdefault(package, []).append(prefix+name)
     return out
 
+trigger_build = ["sdist", "build", "develop", "install"]
+
+
+def compile_sass():
+    import subprocess
+    from distutils.spawn import find_executable
+
+    css_dir = os.path.join(".", "synnefo_admin", "admin", "static")
+    css_dir = os.path.abspath(css_dir)
+
+    if not find_executable("gem"):
+        raise Exception("gem not found, please install ruby and gem")
+
+    os.environ["PATH"] += ":/usr/local/bin"
+    if not find_executable("compass"):
+        print "Install compass"
+        ret = subprocess.call(["gem", "install", "compass"])
+        if ret == 1:
+            raise Exception("gem install failed")
+
+    environment = "development" if "develop" in sys.argv else "production"
+    compass_bin = find_executable("compass")
+    compass_cmd = [compass_bin, "compile", css_dir,
+                   "-e", environment, "--force"]
+    ret = subprocess.call(compass_cmd)
+    if ret == 1:
+        raise Exception("compass compile failed")
+
+
+if any(x in sys.argv for x in trigger_build):
+    if os.environ.get('SNFADMIN_AUTO_COMPILE', True) not in \
+            ['False', 'false', '0']:
+        compile_sass()
+
+
 setup(
     name='snf-admin-app',
     version=VERSION,

snf-admin-app/synnefo_admin/admin/actions.py

-# Copyright (C) 2010-2014 GRNET S.A.
+# Copyright (C) 2010-2016 GRNET S.A.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -34,6 +34,8 @@ class AdminAction(object):
         caution_level:  Indication of how much careful the user should be:
                         Accepted values: none, warning, dangerous.
         description:    A short text that describes an action
+        data_keys:      A list with the extra data dict keys required for the
+                        action
 
     Methods:
         f:              The function that will trigger once an action is
@@ -43,7 +45,8 @@ class AdminAction(object):
     """
 
     def __init__(self, name, target, f, c=None, allowed_groups='admin',
-                 karma='neutral', caution_level='none', description=''):
+                 karma='neutral', caution_level='none', description='',
+                 data_keys=[]):
         """Initialize the AdminAction class."""
         self.name = name
         self.description = description
@@ -52,6 +55,7 @@ class AdminAction(object):
         self.caution_level = caution_level
         self.allowed_groups = allowed_groups
         self.f = f
+        self.data_keys = data_keys
         if c:
             self.check = c
 

snf-admin-app/synnefo_admin/admin/resources/groups/views.py

-# Copyright (C) 2010-2014 GRNET S.A.
+# Copyright (C) 2010-2016 GRNET S.A.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -54,7 +54,7 @@ class GroupJSONView(AdminJSONView):
 JSON_CLASS = GroupJSONView
 
 
-def do_action(request, op, id):
+def do_action(request, op, id, data):
     raise AdminHttp404("There are no actions for Groups")
 
 

snf-admin-app/synnefo_admin/admin/resources/ip_logs/utils.py

@@ -46,4 +46,9 @@ def get_network_details_href(ip_log):
 def get_user_details_href(ip_log):
     vm = VirtualMachine.objects.get(pk=ip_log.server_id)
     user = AstakosUser.objects.get(uuid=vm.userid)
-    return create_details_href('user', user.realname, user.email)
+    return create_details_href('user', user.realname, user.email, user.uuid)
+
+
+def get_user_uuid_from_server(server_id):
+    vm = VirtualMachine.objects.get(pk=server_id)
+    return vm.userid

snf-admin-app/synnefo_admin/admin/resources/ip_logs/views.py

@@ -24,7 +24,8 @@ from synnefo_admin.admin.utils import _filter_public_ip_log
 from synnefo_admin.admin.tables import AdminJSONView
 
 from .utils import (get_user_details_href, get_ip_details_href,
-                    get_vm_details_href, get_network_details_href)
+                    get_vm_details_href, get_network_details_href,
+                    get_user_uuid_from_server,)
 from .filters import IPLogFilterSet
 
 
@@ -35,7 +36,7 @@ templates = {
 
 class IPLogJSONView(AdminJSONView):
     model = IPAddressLog
-    fields = ('address', 'server_id', 'network_id', 'allocated_at',
+    fields = ('address', 'server_id', 'server_id', 'network_id', 'allocated_at',
               'released_at', 'active',)
     filters = IPLogFilterSet
 
@@ -48,15 +49,21 @@ class IPLogJSONView(AdminJSONView):
 
     def format_data_row(self, row):
         row = list(row)
-        row[3] = row[3].strftime("%Y-%m-%d %H:%M")
-        if row[4]:
-            row[4] = row[4].strftime("%Y-%m-%d %H:%M")
+        row[1] = get_user_uuid_from_server(row[1])
+        row[4] = row[4].strftime("%Y-%m-%d %H:%M")
+        if row[5]:
+            row[5] = row[5].strftime("%Y-%m-%d %H:%M")
         else:
-            row[4] = "-"
+            row[5] = "-"
         return row
 
     def get_extra_data_row(self, inst):
         extra_dict = OrderedDict()
+        extra_dict['user_info'] = {
+            'display_name': "Owner",
+            'value': get_user_details_href(inst),
+            'visible': True,
+        }
         extra_dict['id'] = {
             'display_name': "ID",
             'value': inst.pk,
@@ -77,12 +84,6 @@ class IPLogJSONView(AdminJSONView):
             'value': get_network_details_href(inst),
             'visible': True,
         }
-        extra_dict['user_info'] = {
-            'display_name': "User",
-            'value': get_user_details_href(inst),
-            'visible': True,
-        }
-
         return extra_dict
 
 
@@ -94,7 +95,7 @@ def catalog(request):
     context = {}
     context['action_dict'] = None
     context['filter_dict'] = IPLogFilterSet().filters.values()
-    context['columns'] = ["Address", "Server ID", "Network ID",
+    context['columns'] = ["Address", "Owner UUID", "Server ID", "Network ID",
                           "Allocation date", "Release date", "Active", ""]
     context['item_type'] = 'ip_log'
 

snf-admin-app/synnefo_admin/admin/resources/ips/actions.py

-# Copyright (C) 2010-2014 GRNET S.A.
+# Copyright (C) 2010-2016 GRNET S.A.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -60,7 +60,7 @@ def generate_actions():
     actions['reassign'] = IPAction(name='Reassign to project', f=noop,
                                    karma='neutral', caution_level='dangerous',)
 
-    actions['contact'] = IPAction(name='Send e-mail', f=send_admin_email,)
+    actions['contact'] = IPAction(name='Send e‑mail', f=send_admin_email,)
 
     update_actions_rbac(actions)
 

snf-admin-app/synnefo_admin/admin/resources/ips/utils.py

@@ -25,9 +25,11 @@ from synnefo_admin.admin.exceptions import AdminHttp404
 from synnefo_admin.admin.utils import create_details_href
 
 
-def get_ip_or_404(query):
+def get_ip_or_404(query, for_update=False):
+    ip_object = IPAddress.objects.select_for_update() if for_update\
+            else IPAddress.objects
     try:
-        return IPAddress.objects.get(address=query)
+        return ip_object.get(address=query)
     except ObjectDoesNotExist:
         pass
     except MultipleObjectsReturned:
@@ -35,7 +37,7 @@ def get_ip_or_404(query):
                            entries for this address: %s""" % query)
 
     try:
-        return IPAddress.objects.get(pk=int(query))
+        return ip_object.get(pk=int(query))
     except (ObjectDoesNotExist, ValueError):
         # Check the IPAddressLog and inform the user that the IP existed at
         # sometime.
@@ -62,7 +64,7 @@ def get_contact_name(inst):
 def get_user_details_href(ip):
     if ip.userid:
         user = AstakosUser.objects.get(uuid=ip.userid)
-        return create_details_href('user', user.realname, user.email)
+        return create_details_href('user', user.realname, user.email, user.uuid)
     else:
         return "-"
 

snf-admin-app/synnefo_admin/admin/resources/ips/views.py

-# Copyright (C) 2010-2014 GRNET S.A.
+# Copyright (C) 2010-2016 GRNET S.A.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,12 +20,14 @@ from collections import OrderedDict
 from django.core.urlresolvers import reverse
 from django.utils.html import escape
 
+from synnefo.db import transaction
 from synnefo.db.models import IPAddress, IPAddressLog
 from astakos.im.models import AstakosUser, Project
 
 from synnefo_admin.admin.actions import (has_permission_or_403,
                                          get_allowed_actions,
                                          get_permitted_actions,)
+from synnefo_admin.admin.resources.ips.utils import get_ip_or_404
 from synnefo_admin.admin.resources.users.utils import get_user_or_404
 from synnefo_admin.admin.tables import AdminJSONView
 from synnefo_admin.admin.associations import (
@@ -49,12 +51,12 @@ templates = {
 
 class IPJSONView(AdminJSONView):
     model = IPAddress
-    fields = ('pk', 'address', 'floating_ip', 'created', 'userid',)
+    fields = ('pk', 'userid', 'address', 'floating_ip', 'created',)
     filters = IPFilterSet
 
     def format_data_row(self, row):
         row = list(row)
-        row[3] = row[3].strftime("%Y-%m-%d %H:%M")
+        row[4] = row[4].strftime("%Y-%m-%d %H:%M")
         return row
 
     def get_extra_data(self, qs):
@@ -120,7 +122,7 @@ class IPJSONView(AdminJSONView):
     def add_verbose_data(self, inst):
         extra_dict = OrderedDict()
         extra_dict['user_info'] = {
-            'display_name': "User",
+            'display_name': "Owner",
             'value': get_user_details_href(inst),
             'visible': True,
         }
@@ -151,13 +153,14 @@ class IPJSONView(AdminJSONView):
 JSON_CLASS = IPJSONView
 
 
+@transaction.commit_on_success
 @has_permission_or_403(cached_actions)
-def do_action(request, op, id):
+def do_action(request, op, id, data):
     """Apply the requested action on the specified ip."""
     if op == "contact":
         user = get_user_or_404(id)
     else:
-        ip = IPAddress.objects.get(id=id)
+        ip = get_ip_or_404(id, for_update=True)
     actions = get_permitted_actions(cached_actions, request.user)