Commit df8a40a0 authored by Nikos Skalkotos's avatar Nikos Skalkotos

Merge branch 'release-0.18'

parents 75110bb6 182176ca
......@@ -51,3 +51,5 @@ snf-deploy/files/root/.ssh
snf-deploy/files/root/ddns
*.egg
*.tar.gz
snf-admin-app/synnefo_admin/admin/static/min-css
snf-admin-app/synnefo_admin/admin/static/css
......@@ -6,6 +6,84 @@ Unified Changelog file for Synnefo versions >= 0.13
Since v0.13 most of the Synnefo components have been merged into a single
repository and have aligned versions.
.. _Changelog-0.18:
v0.18rc3
========
Released: UNRELEASED
Cyclades
--------
* Fix minor logging issues
Astakos
-------
* Add option to control whether to send e-mail to the user when suspending them
v0.18rc2
========
Released: UNRELEASED
Documentation
-------------
* Reword Administrator's guide
v0.18rc1
========
Released: UNRELEASED
Astakos
-------
* Introduce 'overquota' state on project memberships, which can be used by
third-party plugins in order to implement a quota reclamation policy.
Setting `QUOTA_POLICY_MODULE` specifies the plugin that updates the
overquota state on project actions.
* Support suspending and unsuspending project memberships.
* User deactivation now automatically suspends user's system project, owned
projects, and project memberships. Reactivation unsuspends them.
* Add command `user-check`. It supports suspending projects for previously
deactivated users.
* Send an informative email to the user's current email address when they
request to change their email.
Cyclades
--------
* Command `enforce-resources-cyclades` now provides an option to "soft"
enforce dangerous resources. There is now no default list of resources to
check; the administrator must provide one.
* Make dispatcher use DISPATCHER_LOGGING_SETUP from synnefo settings to setup
its logging. Remove previous settings.
* Add a raven processor to filter sensitive information sent to Sentry using
the Sentry logging handler.
* Imporove SynnefoExceptionReporterFilter to better cleanse request body.
* Make eventd detect Ganeti Master failovers and allow having multiple eventd
instances running in the Ganeti cluster.
Admin
-----
* Add mechanism to generate css files on packaging
* Improve the displayed data in the tables
* Display more information regarding the enabled authentication providers
* Display pending modifications of projects
* Add the action 'modify user e-mail' in the Admin interface
* Display data related with the modification of users' e-mails like 'e-mail
pending verification', 'e-mail change requested at', 'initially accepted
e-mail'
Pithos
------
* Optimize object latest listing query
.. _Changelog-0.17:
v0.17
......
......@@ -5,6 +5,25 @@ Unified NEWS file for Synnefo versions >= 0.13
Since v0.13 all Synnefo components have been merged into a single repository.
.. _NEWS-0.18:
v0.18rc1
========
Released: UNKNOWN
The Synnefo 0.18 release brings significant bug fixes across Synnefo.
The most notable changes are:
* Improved project management and quota policy enforcement
* Performance optimizations of Pithos object listing queries
* Support for modifying user e-mails from the Admin Panel
* Various Admin panel enhancements
* Support for multiple eventd instances and automatic ganeti master failover
detection
* Support for Sentry
.. _NEWS-0.17:
v0.17
......
......@@ -47,7 +47,7 @@ for more information on the Synnefo users and developers lists.
Copyright and license
=====================
Copyright (C) 2010-2015 GRNET S.A. and individual contributors
Copyright (C) 2010-2016 GRNET S.A. and individual contributors
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
......
......@@ -5,7 +5,7 @@ sys.path.insert(0, os.path.abspath('..'))
from astakosclient.version import __version__
project = u'nnefo'
copyright = u'2012-2013, GRNET'
copyright = u'2012-2016, GRNET'
version = __version__
release = __version__
html_title = 'synnefo ' + version
......
......@@ -76,6 +76,11 @@ ssh_port =
# installation of custom packages (e.g. Ganeti, Archipelago).
local_package_dir =
# Set this to a wildcard dns service (e.g. xip.io) for snf-deploy to use it
# as the base domain of the deployed services. Leave it empty to use the
# default synnefo.live domain.
wildcard_dns =
[Burnin]
# Maybe add some burnin options
......
......@@ -201,12 +201,12 @@ def main(): # pylint: disable=too-many-statements, too-many-branches
image=options.image,
ssh_keys=options.ssh_keys,
server_name=options.server_name)
if getattr(options, BUILD_SYNNEFO_CMD, False):
synnefo_ci.clone_repo(
synnefo_repo=options.synnefo_repo,
synnefo_branch=options.synnefo_branch,
local_repo=options.local_repo,
pull_request=options.pull_request)
if getattr(options, BUILD_SYNNEFO_CMD, False):
synnefo_ci.build_packages()
if options.fetch_packages:
dest = os.path.abspath(options.fetch_packages)
......
......@@ -483,7 +483,7 @@ class SynnefoCI(object):
echo 'deb https://deb.nodesource.com/node_0.12 wheezy main' >> /etc/apt/sources.list.d/nodejs.list
echo 'deb-src https://deb.nodesource.com/node_0.12 wheezy main' >> /etc/apt/sources.list.d/nodejs.list
apt-get update
apt-get install -q=2 --force-yes nodejs
apt-get install -q=2 --force-yes nodejs ruby ruby-dev
"""
_run(cmd, False)
......@@ -1156,6 +1156,21 @@ class SynnefoCI(object):
""".format(fabric.env.password)
_run(cmd, False)
wildcard_dns = self.get_config(
'Deployment', 'wildcard_dns', False, '').strip()
if wildcard_dns:
address = self.temp_config.get(str(self.build_id), 'server_ip')
domain = "{0}.{1}".format(address, wildcard_dns)
self.logger.debug("Setting domain to {0}".format(domain))
cmd = """
sed -i 's/^domain.*=.*/domain = {0}/' /etc/snf-deploy/nodes.conf
""".format(domain)
_run(cmd, False)
cmd = """
sed -i 's/^domain.*=.*/domain = {0}/' /etc/snf-deploy/ganeti.conf
""".format(domain)
_run(cmd, False)
self.logger.debug("Run snf-deploy")
cmd = """
snf-deploy --disable-colors --autoconf synnefo
......
......@@ -430,6 +430,22 @@ add up quota from different projects. Note also that if allocating an entity
requires multiple resources (e.g. cpu and ram for a Cyclades VM) these must
be all assigned to a single project.
Reclaiming resources
````````````````````
When a project is deactivated or a user is removed from a project, the quota
that have been granted to the user are revoked. If the user still owns
resources assigned to the project, the user quota appear overlimit on that
project. The services are responsible to inspect the overquota state of
users and reclaim their resources. For instance, cyclades provides
the management command ``enforce-resources-cyclades`` to reclaim VMs,
volumes, and floating IPs.
When a user is deactivated, their system project, owned projects and project
memberships are suspended. Subsequently, the user's resources can be
reclaimed as explained above.
Control projects
````````````````
......@@ -1461,20 +1477,30 @@ quota limits, dependent on the overlimit resource:
* `cyclades.cpu`: Shutdown VMs
* `cyclades.total_ram`: Delete VMs
* `cyclades.ram`: Shutdown VMs
* `cyclades.disk`: Delete VMs
* `cyclades.floating_ip`: Detach and remove IPs
* `cyclades.disk`: Delete volumes (may also trigger VM deletion)
* `cyclades.floating_ip`: Detach and delete IPs
VMs to be deleted/shutdown are chosen first by state in the following order:
ERROR, BUILD, STOPPED, STARTED or RESIZE and then by decreasing ID. When
needing to remove IPs, we first choose IPs that are free, then those
attached to VMs, using the same VM ordering.
By default, the command checks only the following resources: `cyclades.cpu`,
`cyclades.ram`, and `cyclades.floating_ip`; that is, the less dangerous
ones, those that do not result in *deleting* any VM. One can change the
default behavior by specifying the desired resources with option
``--resources``. It is also possible to specify users to be checked or
excluded.
You need to specify the resources to be checked, using the option
``--resources``. A safe first attempt would be to specify
``cyclades.cpu,cyclades.ram``, that is, to check the less dangerous resources,
those that do not result in *deleting* any VM, volume, or IP.
If you want to handle overlimit quota in a safer way for resources that
would normally trigger a deletion, you can use the option
``--soft-resources``. Enforcing e.g. `cyclades.vm` in a "soft" way will
shutdown the VMs rather than deleting them. This is useful as an initial
warning for a user who is overquota; but notice that the user may restart
their shutdown VMs, if the resources that control starting VMs allows them
to do so.
With option ``--list-resources`` you can inspect the available resources
along with the related standard and soft enforce actions. It is also
possible to specify users and projects to be checked or excluded.
Actual enforcement is done with option ``--fix``. In order to control the
load that quota enforcement may cause on Cyclades, one can limit the number
......@@ -3051,6 +3077,7 @@ Upgrade Notes
v0.15 -> v0.16 <upgrade/upgrade-0.16>
v0.16.1 -> v0.16.2 <upgrade/upgrade-0.16.2>
v0.16.2 -> v0.17 <upgrade/upgrade-0.17>
v0.17 -> v0.18 <upgrade/upgrade-0.18>
.. _changelog-news:
......@@ -3059,6 +3086,7 @@ Changelog, NEWS
===============
* v0.18 :ref:`Changelog <Changelog-0.18>`, :ref:`NEWS <NEWS-0.18>`
* v0.17 :ref:`Changelog <Changelog-0.17>`, :ref:`NEWS <NEWS-0.17>`
* v0.16.2 :ref:`Changelog <Changelog-0.16.2>`, :ref:`NEWS <NEWS-0.16.2>`
* v0.16.1 :ref:`Changelog <Changelog-0.16.1>`, :ref:`NEWS <NEWS-0.16.1>`
......
......@@ -8,7 +8,7 @@ reload(synnefo.versions)
from synnefo.versions.app import __version__
project = u'synnefo'
copyright = u'2012-2015, GRNET'
copyright = u'2012-2016, GRNET'
version = __version__
release = __version__
html_title = 'synnefo ' + version
......
This diff is collapsed.
Upgrade to Synnefo v0.18
^^^^^^^^^^^^^^^^^^^^^^^^
Upgrade Steps
=============
The upgrade to v0.18 consists of the following steps:
#. Stop gunicorn in all nodes
.. code-block:: console
# service gunicorn stop
#. Upgrade Synnefo on all nodes to the latest version (0.18)
.. code-block:: console
# apt-get update
# apt-get upgrade
#. Run migrations on Astakos.
.. code-block:: console
astakos.host$ snf-manage migrate
From this version on, user deactivation triggers suspension of all projects
and project memberships related to the user. To apply this new policy to
users that have already been deactivated, run:
.. code-block:: console
astakos.host$ snf-manage user-check --all-users --suspend-deactivated --noemail --fix
#. Start gunicorn
.. code-block:: console
# service gunicorn start
New configuration options
=========================
On the admin app, there is a new access control option regarding the new modify
email action. The action setting is named 'modify_email'. The list of user
groups defined in this have access on the modify email action.
The following line (modified accordingly) should be added on 'ADMIN_RBAC'
setting under the 'user' dictionary:
.. code-block:: console
'modify_email': [ADMIN_HELPDESK_GROUP, ADMIN_GROUP],
# Copyright (C) 2010-2014 GRNET S.A.
# Copyright (C) 2010-2016 GRNET S.A.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
......@@ -25,6 +25,7 @@ from fnmatch import fnmatchcase
from distutils.util import convert_path
HERE = os.path.abspath(os.path.normpath(os.path.dirname(__file__)))
os.chdir(HERE)
from synnefo_admin.version import __version__
......@@ -137,6 +138,41 @@ def find_package_data(
out.setdefault(package, []).append(prefix+name)
return out
trigger_build = ["sdist", "build", "develop", "install"]
def compile_sass():
import subprocess
from distutils.spawn import find_executable
css_dir = os.path.join(".", "synnefo_admin", "admin", "static")
css_dir = os.path.abspath(css_dir)
if not find_executable("gem"):
raise Exception("gem not found, please install ruby and gem")
os.environ["PATH"] += ":/usr/local/bin"
if not find_executable("compass"):
print "Install compass"
ret = subprocess.call(["gem", "install", "compass"])
if ret == 1:
raise Exception("gem install failed")
environment = "development" if "develop" in sys.argv else "production"
compass_bin = find_executable("compass")
compass_cmd = [compass_bin, "compile", css_dir,
"-e", environment, "--force"]
ret = subprocess.call(compass_cmd)
if ret == 1:
raise Exception("compass compile failed")
if any(x in sys.argv for x in trigger_build):
if os.environ.get('SNFADMIN_AUTO_COMPILE', True) not in \
['False', 'false', '0']:
compile_sass()
setup(
name='snf-admin-app',
version=VERSION,
......
# Copyright (C) 2010-2014 GRNET S.A.
# Copyright (C) 2010-2016 GRNET S.A.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
......@@ -34,6 +34,8 @@ class AdminAction(object):
caution_level: Indication of how much careful the user should be:
Accepted values: none, warning, dangerous.
description: A short text that describes an action
data_keys: A list with the extra data dict keys required for the
action
Methods:
f: The function that will trigger once an action is
......@@ -43,7 +45,8 @@ class AdminAction(object):
"""
def __init__(self, name, target, f, c=None, allowed_groups='admin',
karma='neutral', caution_level='none', description=''):
karma='neutral', caution_level='none', description='',
data_keys=[]):
"""Initialize the AdminAction class."""
self.name = name
self.description = description
......@@ -52,6 +55,7 @@ class AdminAction(object):
self.caution_level = caution_level
self.allowed_groups = allowed_groups
self.f = f
self.data_keys = data_keys
if c:
self.check = c
......
# Copyright (C) 2010-2014 GRNET S.A.
# Copyright (C) 2010-2016 GRNET S.A.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
......@@ -54,7 +54,7 @@ class GroupJSONView(AdminJSONView):
JSON_CLASS = GroupJSONView
def do_action(request, op, id):
def do_action(request, op, id, data):
raise AdminHttp404("There are no actions for Groups")
......
......@@ -46,4 +46,9 @@ def get_network_details_href(ip_log):
def get_user_details_href(ip_log):
vm = VirtualMachine.objects.get(pk=ip_log.server_id)
user = AstakosUser.objects.get(uuid=vm.userid)
return create_details_href('user', user.realname, user.email)
return create_details_href('user', user.realname, user.email, user.uuid)
def get_user_uuid_from_server(server_id):
vm = VirtualMachine.objects.get(pk=server_id)
return vm.userid
......@@ -24,7 +24,8 @@ from synnefo_admin.admin.utils import _filter_public_ip_log
from synnefo_admin.admin.tables import AdminJSONView
from .utils import (get_user_details_href, get_ip_details_href,
get_vm_details_href, get_network_details_href)
get_vm_details_href, get_network_details_href,
get_user_uuid_from_server,)
from .filters import IPLogFilterSet
......@@ -35,7 +36,7 @@ templates = {
class IPLogJSONView(AdminJSONView):
model = IPAddressLog
fields = ('address', 'server_id', 'network_id', 'allocated_at',
fields = ('address', 'server_id', 'server_id', 'network_id', 'allocated_at',
'released_at', 'active',)
filters = IPLogFilterSet
......@@ -48,15 +49,21 @@ class IPLogJSONView(AdminJSONView):
def format_data_row(self, row):
row = list(row)
row[3] = row[3].strftime("%Y-%m-%d %H:%M")
if row[4]:
row[4] = row[4].strftime("%Y-%m-%d %H:%M")
row[1] = get_user_uuid_from_server(row[1])
row[4] = row[4].strftime("%Y-%m-%d %H:%M")
if row[5]:
row[5] = row[5].strftime("%Y-%m-%d %H:%M")
else:
row[4] = "-"
row[5] = "-"
return row
def get_extra_data_row(self, inst):
extra_dict = OrderedDict()
extra_dict['user_info'] = {
'display_name': "Owner",
'value': get_user_details_href(inst),
'visible': True,
}
extra_dict['id'] = {
'display_name': "ID",
'value': inst.pk,
......@@ -77,12 +84,6 @@ class IPLogJSONView(AdminJSONView):
'value': get_network_details_href(inst),
'visible': True,
}
extra_dict['user_info'] = {
'display_name': "User",
'value': get_user_details_href(inst),
'visible': True,
}
return extra_dict
......@@ -94,7 +95,7 @@ def catalog(request):
context = {}
context['action_dict'] = None
context['filter_dict'] = IPLogFilterSet().filters.values()
context['columns'] = ["Address", "Server ID", "Network ID",
context['columns'] = ["Address", "Owner UUID", "Server ID", "Network ID",
"Allocation date", "Release date", "Active", ""]
context['item_type'] = 'ip_log'
......
# Copyright (C) 2010-2014 GRNET S.A.
# Copyright (C) 2010-2016 GRNET S.A.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
......@@ -60,7 +60,7 @@ def generate_actions():
actions['reassign'] = IPAction(name='Reassign to project', f=noop,
karma='neutral', caution_level='dangerous',)
actions['contact'] = IPAction(name='Send e-mail', f=send_admin_email,)
actions['contact'] = IPAction(name='Send e&#8209;mail', f=send_admin_email,)
update_actions_rbac(actions)
......
......@@ -25,9 +25,11 @@ from synnefo_admin.admin.exceptions import AdminHttp404
from synnefo_admin.admin.utils import create_details_href
def get_ip_or_404(query):
def get_ip_or_404(query, for_update=False):
ip_object = IPAddress.objects.select_for_update() if for_update\
else IPAddress.objects
try:
return IPAddress.objects.get(address=query)
return ip_object.get(address=query)
except ObjectDoesNotExist:
pass
except MultipleObjectsReturned:
......@@ -35,7 +37,7 @@ def get_ip_or_404(query):
entries for this address: %s""" % query)
try:
return IPAddress.objects.get(pk=int(query))
return ip_object.get(pk=int(query))
except (ObjectDoesNotExist, ValueError):
# Check the IPAddressLog and inform the user that the IP existed at
# sometime.
......@@ -62,7 +64,7 @@ def get_contact_name(inst):
def get_user_details_href(ip):
if ip.userid:
user = AstakosUser.objects.get(uuid=ip.userid)
return create_details_href('user', user.realname, user.email)
return create_details_href('user', user.realname, user.email, user.uuid)
else:
return "-"
......
# Copyright (C) 2010-2014 GRNET S.A.
# Copyright (C) 2010-2016 GRNET S.A.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
......@@ -20,12 +20,14 @@ from collections import OrderedDict
from django.core.urlresolvers import reverse
from django.utils.html import escape
from synnefo.db import transaction
from synnefo.db.models import IPAddress, IPAddressLog
from astakos.im.models import AstakosUser, Project
from synnefo_admin.admin.actions import (has_permission_or_403,
get_allowed_actions,
get_permitted_actions,)
from synnefo_admin.admin.resources.ips.utils import get_ip_or_404
from synnefo_admin.admin.resources.users.utils import get_user_or_404
from synnefo_admin.admin.tables import AdminJSONView
from synnefo_admin.admin.associations import (
......@@ -49,12 +51,12 @@ templates = {
class IPJSONView(AdminJSONView):
model = IPAddress
fields = ('pk', 'address', 'floating_ip', 'created', 'userid',)
fields = ('pk', 'userid', 'address', 'floating_ip', 'created',)
filters = IPFilterSet
def format_data_row(self, row):
row = list(row)
row[3] = row[3].strftime("%Y-%m-%d %H:%M")
row[4] = row[4].strftime("%Y-%m-%d %H:%M")
return row
def get_extra_data(self, qs):
......@@ -120,7 +122,7 @@ class IPJSONView(AdminJSONView):
def add_verbose_data(self, inst):
extra_dict = OrderedDict()
extra_dict['user_info'] = {
'display_name': "User",
'display_name': "Owner",
'value': get_user_details_href(inst),
'visible': True,
}
......@@ -151,13 +153,14 @@ class IPJSONView(AdminJSONView):
JSON_CLASS = IPJSONView
@transaction.commit_on_success
@has_permission_or_403(cached_actions)
def do_action(request, op, id):
def do_action(request, op, id, data):
"""Apply the requested action on the specified ip."""
if op == "contact":
user = get_user_or_404(id)
else:
ip = IPAddress.objects.get(id=id)
ip = get_ip_or_404(id, for_update=True)
actions = get_permitted_actions(cached_actions, request.user)
if op == 'contact':
......@@ -172,8 +175,8 @@ def catalog(request):
context['action_dict'] = get_permitted_actions(cached_actions,
request.user)
context['filter_dict'] = IPFilterSet().filters.values()
context['columns'] = ["ID", "Address", "Floating",
"Creation date", "User ID", ""]
context['columns'] = ["ID", "Owner UUID", "Address", "Floating",
"Creation date", ""]
context['item_type'] = 'ip'
return context
......
# Copyright (C) 2010-2014 GRNET S.A.
# Copyright (C) 2010-2016 GRNET S.A.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
......@@ -85,7 +85,7 @@ def generate_actions():
karma='neutral',
caution_level='dangerous',)
actions['contact'] = NetworkAction(name='Send e-mail', f=send_admin_email,
actions['contact'] = NetworkAction(name='Send e&#8209;mail', f=send_admin_email,
c=check_network_action("CONTACT"),)
update_actions_rbac(actions)
......