Commit 4893503e authored by Christos Stavrakakis's avatar Christos Stavrakakis
Browse files

Merge branch 'feature-export-stats' into develop

parents 1bce4de3 e007807a
......@@ -52,6 +52,12 @@ Astakos
* Remove API call GET /account/v1.0/authenticate in favor of
POST /identity/v2.0/tokens.
* Export basic statistics about Astakos service from '/admin/stats/detail' API
endpoint. Access to this endpoint is only allowed to users that belong to
the Astakos groups that are defined in the
'ASTAKOS_ADMIN_STATS_PERMITTED_GROUPS' setting. Statistics are also availble
from 'snf-manage stats-astakos' management command.
* Management commands:
* Introduced new commands:
* component-show
......@@ -99,6 +105,11 @@ Cyclades
* Do not automatically release externally reserved IPs if they are released
from a Ganeti backend. Management of externally reserved IPs must be
performed from Cyclades with 'network-modify' command.
* Export basic statistics about Cyclades Service from '/admin/stats/detail'
API endpoint. Access to this endpoint is only allowed to users that belong
to the Astakos groups that are defined in the 'ADMIN_STATS_PERMITTED_GROUPS'
setting. Statistics are also availble from 'snf-manage stats-cyclades'
management command.
Pithos
------
......
# Copyright 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from django.conf.urls import url, patterns
from astakos.admin import views
from django.http import Http404
def index(request):
raise Http404
urlpatterns = patterns(
'',
url(r'^$', index),
url(r'^stats$', views.get_public_stats),
url(r'^stats/detail$', views.get_astakos_stats),
)
# Copyright 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
import datetime
from django.conf import settings
from django.db.models import Sum
from astakos.im.models import AstakosUser, Resource
from astakos.quotaholder_app.models import Holding
def get_public_stats():
users = AstakosUser.objects.all()
active = users.filter(is_active=True)
return {"users": {"total": users.count(),
"active": active.count()}}
def get_astakos_stats():
stats = {"datetime": datetime.datetime.now().strftime("%c")}
resources = Resource.objects.values_list("name", flat=True)
users = AstakosUser.objects.all()
verified = users.filter(email_verified=True)
active = users.filter(is_active=True)
user_stats = {}
user_stats["total"] = {"total": users.count(),
"verified": verified.count(),
"active": active.count(),
"usage": {}}
for resource in resources:
usage = Holding.objects.filter(resource=resource)\
.aggregate(summ=Sum("usage_max"))
user_stats["total"]["usage"][resource] = int(usage["summ"])
for provider in settings.ASTAKOS_IM_MODULES:
users = AstakosUser.objects.filter(auth_providers__module=provider)
verified = users.filter(email_verified=True)
active = users.filter(is_active=True)
user_stats[provider] = {"total": users.count(),
"verified": verified.count(),
"active": active.count(),
"usage": {}}
users_uuids = users.values_list("uuid", flat=True)
for resource in resources:
usage = Holding.objects\
.filter(holder__in=users_uuids, resource=resource)\
.aggregate(summ=Sum("usage_max"))
user_stats[provider]["usage"][resource] = int(usage["summ"])
stats["users"] = user_stats
return stats
# Copyright 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
import logging
from django import http
from django.utils import simplejson as json
from snf_django.lib import api
from astakos.im import settings
from astakos.admin import stats
logger = logging.getLogger(__name__)
PERMITTED_GROUPS = settings.ADMIN_STATS_PERMITTED_GROUPS
@api.api_method(http_method='GET', user_required=False, token_required=False,
logger=logger, serializations=['json'])
@api.allow_jsonp()
def get_public_stats(request):
_stats = stats.get_public_stats()
data = json.dumps(_stats)
return http.HttpResponse(data, status=200, content_type='application/json')
@api.api_method(http_method='GET', user_required=True, token_required=True,
logger=logger, serializations=['json'])
@api.user_in_groups(permitted_groups=PERMITTED_GROUPS,
logger=logger)
def get_astakos_stats(request):
_stats = stats.get_astakos_stats()
data = json.dumps(_stats)
return http.HttpResponse(data, status=200, content_type='application/json')
......@@ -85,4 +85,15 @@ astakos_services = {
'publicURL': None},
],
},
'astakos_admin': {
'type': 'astakos_admin',
'component': 'astakos',
'prefix': 'admin',
'public': False,
'endpoints': [
{'versionId': '',
'publicURL': None},
],
},
}
# Copyright 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
import json
import string
#from optparse import make_option
from snf_django.management.commands import SynnefoCommand, CommandError
from snf_django.management.utils import pprint_table
#from astakos.im.models import AstakosUser, Resource
#from astakos.quotaholder_app.models import Holding
from astakos.admin import stats as statistics
class Command(SynnefoCommand):
help = "Get available statistics of Astakos service"
can_import_settings = True
option_list = SynnefoCommand.option_list + (
)
def handle(self, *args, **options):
stats = statistics.get_astakos_stats()
output_format = options["output_format"]
if output_format == "json":
self.stdout.write(json.dumps(stats, indent=4) + "\n")
elif output_format == "pretty":
pretty_print_stats(stats, self.stdout)
else:
raise CommandError("Output format '%s' not supported." %
output_format)
def columns_from_fields(fields, values):
return zip(map(string.lower, fields), [values.get(f, 0) for f in fields])
def pretty_print_stats(stats, stdout):
newline = lambda: stdout.write("\n")
_datetime = stats.get("datetime")
stdout.write("datetime: %s\n" % _datetime)
newline()
users = stats.get("users", {})
all_providers = users.pop("total")
if all_providers is not None:
fields = ["total", "verified", "active"]
table = columns_from_fields(fields, all_providers)
usage = all_providers.get("usage", {})
for name, val in sorted(usage.items()):
table.append((name, val))
pprint_table(stdout, table, None,
title="Statistics for All Providers")
newline()
for provider_name, provider_info in sorted(users.items()):
fields = ["total", "verified", "active"]
table = columns_from_fields(fields, provider_info)
usage = provider_info.get("usage", {})
for name, val in sorted(usage.items()):
table.append((name, val))
pprint_table(stdout, table, None,
title="Statistics for Provider '%s'" % provider_name)
newline()
......@@ -21,6 +21,7 @@ ACCOUNTS_PREFIX = get_path(astakos_services, 'astakos_account.prefix')
VIEWS_PREFIX = get_path(astakos_services, 'astakos_ui.prefix')
KEYSTONE_PREFIX = get_path(astakos_services, 'astakos_identity.prefix')
WEBLOGIN_PREFIX = get_path(astakos_services, 'astakos_weblogin.prefix')
ADMIN_PREFIX = get_path(astakos_services, 'astakos_admin.prefix')
# Set the expiration time of newly created auth tokens
# to be this many hours after their creation time.
......@@ -207,3 +208,7 @@ KAMAKI_CONFIG_CLOUD_NAME = getattr(settings,
REDIRECT_ALLOWED_SCHEMES = getattr(settings,
'ASTAKOS_REDIRECT_ALLOWED_SCHEMES',
('pithos', 'pithosdev'))
ADMIN_STATS_PERMITTED_GROUPS = getattr(settings,
'ASTAKOS_ADMIN_STATS_PERMITTED_GROUPS',
['admin-stats'])
......@@ -34,7 +34,8 @@
from django.conf.urls import include, patterns
from astakos.im.settings import (
BASE_PATH, ACCOUNTS_PREFIX, VIEWS_PREFIX, KEYSTONE_PREFIX, WEBLOGIN_PREFIX)
BASE_PATH, ACCOUNTS_PREFIX, VIEWS_PREFIX, KEYSTONE_PREFIX, WEBLOGIN_PREFIX,
ADMIN_PREFIX)
from snf_django.lib.api.utils import prefix_pattern
from snf_django.utils.urls import \
extend_with_root_redirects, extend_endpoint_with_slash
......@@ -52,6 +53,7 @@ astakos_patterns = patterns(
(prefix_pattern(ACCOUNTS_PREFIX), include('astakos.api.urls')),
(prefix_pattern(KEYSTONE_PREFIX), include('astakos.api.keystone_urls')),
(prefix_pattern(WEBLOGIN_PREFIX), include('astakos.im.weblogin_urls')),
(prefix_pattern(ADMIN_PREFIX), include('astakos.admin.admin_urls')),
)
......
......@@ -136,3 +136,6 @@
# Migrate existing shibboleth user entries which where previously associated
# with EPPN instead of the provided value of REMOTE_ID mod_shib2 header.
# ASTAKOS_SHIBBOLETH_MIGRATE_EPPN = False
#
## Astakos groups that have access to '/admin' views.
# ASTAKOS_ADMIN_STATS_PERMITTED_GROUPS = ["admin-stats"]
......@@ -13,6 +13,9 @@
## parameter refers to a point in time more than POLL_LIMIT seconds ago.
#POLL_LIMIT = 3600
#
## Astakos groups that have access to '/admin' views.
#ADMIN_STATS_PERMITTED_GROUPS = ["admin-stats"]
#
##
## Network Configuration
##
......
# Copyright 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
import itertools
import operator
import datetime
from collections import defaultdict # , OrderedDict
from copy import copy
from django.conf import settings
from django.db.models import Count, Sum
from snf_django.lib.astakos import UserCache
from synnefo.db.models import VirtualMachine, Network, Backend
from synnefo.api.util import get_image
from synnefo.logic import backend as backend_mod
def get_cyclades_stats(backend=None, clusters=True, servers=True,
resources=True, networks=True, images=True):
stats = {"datetime": datetime.datetime.now().strftime("%c")}
if clusters:
stats["clusters"] = get_cluster_stats(backend=backend)
if servers:
stats["servers"] = get_servers_stats(backend=backend)
if resources:
stats["resources"] = get_resources_stats(backend=backend)
if networks:
stats["networks"] = get_networks_stats()
if images:
stats["images"] = get_images_stats(backend=None)
return stats
def get_cluster_stats(backend):
total = Backend.objects.all()
stats = {"total": total.count(),
"drained": total.filter(drained=True).count(),
"offline": total.filter(offline=True).count()}
return stats
def _get_total_servers(backend=None):
total_servers = VirtualMachine.objects.all()
if backend is not None:
total_servers = total_servers.filter(backend=backend)
return total_servers
def get_servers_stats(backend=None):
total_servers = _get_total_servers(backend=backend)
per_state = total_servers.values("operstate")\
.annotate(count=Count("operstate"))
stats = {"total": 0}
[stats.setdefault(s[0], 0) for s in VirtualMachine.OPER_STATES]
for x in per_state:
stats[x["operstate"]] = x["count"]
stats["total"] += x["count"]
return stats
def get_resources_stats(backend=None):
total_servers = _get_total_servers(backend=backend)
active_servers = total_servers.filter(deleted=False)
allocated = {}
server_count = {}
for res in ["cpu", "ram", "disk", "disk_template"]:
server_count[res] = {}
allocated[res] = 0
val = "flavor__%s" % res
results = active_servers.values(val).annotate(count=Count(val))
for result in results:
server_count[res][result[val]] = result["count"]
if res != "disk_template":
allocated[res] += result["count"]
resources_stats = get_backend_stats(backend=backend)
for res in ["cpu", "ram", "disk", "disk_template"]:
if res not in resources_stats:
resources_stats[res] = {}
resources_stats[res]["servers"] = server_count[res]
resources_stats[res]["allocated"] = allocated[res]
return resources_stats
def get_images_stats(backend=None):
total_servers = _get_total_servers(backend=backend)
active_servers = total_servers.filter(deleted=False)
active_servers_images = active_servers.values("imageid", "userid")\
.annotate(number=Count("imageid"))
image_cache = ImageCache()
image_stats = defaultdict(int)
for result in active_servers_images:
imageid = image_cache.get_image(result["imageid"], result["userid"])
image_stats[imageid] += result["number"]
return dict(image_stats)
def get_networks_stats():
total_networks = Network.objects.all()
stats = {"public_ips": get_ip_stats(),
"total": 0}
per_state = total_networks.values("state")\
.annotate(count=Count("state"))
[stats.setdefault(s[0], 0) for s in Network.OPER_STATES]
for x in per_state:
stats[x["state"]] = x["count"]
stats["total"] += x["count"]
return stats
def group_by_resource(objects, resource):
stats = {}
key = operator.attrgetter("flavor."+resource)
grouped = itertools.groupby(sorted(objects, key=key), key)
for val, group in grouped:
stats[val] = len(list(group))
return stats
def get_ip_stats():
total, free = 0, 0,
for network in Network.objects.filter(public=True, deleted=False):
try:
net_total, net_free = network.ip_count()
except AttributeError:
# TODO: Check that this works..
pool = network.get_pool(locked=False)
net_total = pool.pool_size
net_free = pool.count_available()
if not network.drained:
total += net_total
free += net_free
return {"total": total,
"free": free}
def get_backend_stats(backend=None):
if backend is None:
backends = Backend.objects.filter(offline=False)
else:
if backend.offline:
return {}
backends = [backend]
[backend_mod.update_backend_resources(b) for b in backends]
resources = {}
for attr in ("dfree", "dtotal", "mfree", "mtotal", "ctotal"):
resources[attr] = 0
for b in backends:
resources[attr] += getattr(b, attr)
return {"disk": {"free": resources["dfree"], "total": resources["dtotal"]},
"ram": {"free": resources["mfree"], "total": resources["mtotal"]},
"cpu": {"free": resources["ctotal"], "total": resources["ctotal"]},
"disk_template": {"free": 0, "total": 0}}
class ImageCache(object):
def __init__(self):
self.images = {}
usercache = UserCache(settings.ASTAKOS_BASE_URL,
settings.CYCLADES_SERVICE_TOKEN)
self.system_user_uuid = \
usercache.get_uuid(settings.SYSTEM_IMAGES_OWNER)
def get_image(self, imageid, userid):
if not imageid in self.images:
try:
image = get_image(imageid, userid)
owner = image["owner"]
owner = "system" if image["owner"] == self.system_user_uuid\