Merge branch 'feature-admin-release-0.16' of https://github.com/apyrgio/synnefo into release-0.16

.gitignore

@@ -9,6 +9,8 @@ share/
 build/
 include/
 !snf-cyclades-app/synnefo/ui/static/snf/extra/noVNC/include/
+_sass/
+.sass-cache/
 *.pt.py
 *.installed.cfg
 *.sqlite
@@ -21,7 +23,7 @@ selenium-server-standalone-2.0b2.jar
 .settings/
 settings.d/*-local.conf
 *.egg-info
-dist
+#dist
 _build
 .coverage
 
@@ -32,6 +34,7 @@ pithos/data
 */synnefo/versions/*.py
 */pithos/*/version.py
 !*/synnefo/versions/__init__.py
+snf-admin-app/synnefo_admin/version.py
 snf-astakos-app/astakos/version.py
 snf-tools/synnefo_tools/version.py
 snf-quotaholder-app/quotaholder_django/version.py

Changelog

@@ -6,7 +6,7 @@ Unified Changelog file for Synnefo versions >= 0.13
 Since v0.13 most of the Synnefo components have been merged into a single
 repository and have aligned versions.
 
-v0.15next
+v0.16
 =========
 
 Released: UNRELEASED
@@ -216,6 +216,17 @@ Pithos
   specifically requested or an invalid value is passed, the disposition type
   is set to 'inline'.
 
+Admin
+-----
+
+* Introduce Administrator Dashboard, which provides the following:
+
+	* Graphic access to the details of various Synnefo entities (users, VMs,
+	  Projects).
+	* Intuitive filtering.
+	* Multiple actions and notifications.
+	* Charts and statistics generation.
+
 .. _Changelog-0.15.2:
 
 v0.15.2

NEWS

@@ -5,6 +5,18 @@ Unified NEWS file for Synnefo versions >= 0.13
 
 Since v0.13 all Synnefo components have been merged into a single repository.
 
+.. _NEWS-0.16:
+
+v0.16
+=======
+
+Released: UNRELEASED
+
+The most notables changes are:
+
+* Addition of an Administrator Dashboard (Admin) to manage and view various
+  Synnefo entities.
+
 .. _NEWS-0.15.2:
 
 v0.15.2

ci/ci_wheezy.conf

@@ -74,7 +74,7 @@ cmd_options = --images "name:.*" --flavors "name:C1R512D2file" --no-ipv6
 
 
 [Unit Tests]
-component = astakos cyclades pithos astakosclient
+component = astakos cyclades pithos astakosclient admin
 
 
 [Repository]
@@ -89,5 +89,6 @@ projects =
     snf-pithos-backend
     snf-cyclades-gtools
     snf-cyclades-app
+    snf-admin-app
     snf-pithos-app
     snf-tools

ci/config

@@ -9,4 +9,5 @@ PROJECTS="\
  snf-cyclades-gtools\
  snf-cyclades-app\
  snf-pithos-app\
+ snf-admin-app\
  snf-tools"

ci/schemas/one_node_wheezy/packages.conf

@@ -7,5 +7,6 @@ curl = 7.26.0-1+wheezy9
 snf-image = unstable
 snf-ganeti = unstable
 ganeti2 = unstable
+python-django-eztables = 0.3.3-1~snf~0.2
 
 [ubuntu]

ci/schemas/one_node_wheezy/setups.conf

@@ -13,6 +13,7 @@ db = node
 mq = node
 astakos = node
 cyclades = node
+admin = node
 pithos = node
 cms = node
 stats = node

ci/tests.sh

@@ -4,7 +4,7 @@ set -e
 runAstakosTests () {
     if [ -z "$astakos_tests" ]; then return; fi
 
-    export SYNNEFO_EXCLUDE_PACKAGES="snf-cyclades-app"
+    export SYNNEFO_EXCLUDE_PACKAGES="snf-cyclades-app snf-admin-app snf-pithos-app"
     CURRENT_COMPONENT=astakos
     createSnfManageTest $astakos_tests
     runTest
@@ -13,16 +13,25 @@ runAstakosTests () {
 runCycladesTests () {
     if [ -z "$cyclades_tests" ]; then return; fi
 
-    export SYNNEFO_EXCLUDE_PACKAGES="snf-pithos-app snf-astakos-app"
+    export SYNNEFO_EXCLUDE_PACKAGES="snf-pithos-app snf-astakos-app snf-admin-app"
     CURRENT_COMPONENT=synnefo
     createSnfManageTest $cyclades_tests
     runTest
 }
 
+runAdminTests () {
+    if [ -z "$admin_tests" ]; then return; fi
+
+    export SYNNEFO_EXCLUDE_PACKAGES="snf-pithos-app"
+    CURRENT_COMPONENT=synnefo_admin
+    createSnfManageTest $admin_tests
+    runTest
+}
+
 runPithosTests () {
     if [ -z "$pithos_tests" ]; then return; fi
 
-    export SYNNEFO_EXCLUDE_PACKAGES="snf-cyclades-app"
+    export SYNNEFO_EXCLUDE_PACKAGES="snf-cyclades-app snf-astakos-app"
     CURRENT_COMPONENT=pithos
     createSnfManageTest $pithos_tests
     runTest
@@ -118,7 +127,7 @@ extract_tests () {
                 append "${c}_tests" "$(eval "echo \$"${c}"_all_tests")"
                 return
             elif contains $1 "$c."; then
-                append "${c}_tests" $(echo $1 | sed -e 's/[a-z]*\.//g')
+                append "${c}_tests" $(echo $1 | sed -e 's/^[a-z]*\.//g')
                 return
             fi
         fi
@@ -131,12 +140,14 @@ export SYNNEFO_SETTINGS_DIR=/tmp/snf-test-settings
 
 astakos_all_tests="im quotaholder_app oa2"
 cyclades_all_tests="api db logic plankton quotas vmapi helpdesk userdata volume"
+admin_all_tests="admin"
 pithos_all_tests="api"
 astakosclient_all_tests="astakosclient"
-ALL_COMPONENTS="astakos cyclades pithos astakosclient"
+ALL_COMPONENTS="astakos cyclades admin pithos astakosclient"
 
 astakos_tests=""
 cyclades_tests=""
+admin_tests=""
 pithos_tests=""
 astakosclient_tests=""
 
@@ -170,6 +181,7 @@ echo "| Component     | Tests"
 echo "|---------------|----------------------------"
 echo "| Astakos       | $astakos_tests"
 echo "| Cyclades      | $cyclades_tests"
+echo "| Admin         | $admin_tests"
 echo "| Pithos        | $pithos_tests"
 echo "| Astakosclient | $astakosclient_tests"
 echo "|===============|============================"
@@ -183,5 +195,6 @@ fi
 # For each component, run the specified tests.
 runAstakosTests
 runCycladesTests
+runAdminTests
 runPithosTests
 runAstakosclientTests

devflow.conf

@@ -7,6 +7,8 @@
       version_file = "snf-astakos-app/astakos/version.py"
   [[snf-cyclades-app]]
       version_file = "snf-cyclades-app/synnefo/versions/app.py"
+  [[snf-admin-app]]
+      version_file = "snf-admin-app/synnefo_admin/version.py"
   [[snf-cyclades-gtools]]
       version_file = "snf-cyclades-gtools/synnefo/versions/ganeti.py"
   [[snf-pithos-app]]

docs/admin-guide.rst

@@ -1651,6 +1651,124 @@ these messages and properly updates the state of the Cyclades DB. Subsequent
 requests to the Cyclades API, will retrieve the updated state from the DB.
 
 
+Admin Dashboard (Admin)
+=======================
+
+Introduction
+------------
+
+Admin is the Synnefo component that provides to trusted users the ability to
+manage and view various different Synnefo entities such as users, VMs, projects
+etc. Additionally, it automatically generates charts and statistics using data
+from the Astakos/Cyclades stats.
+
+Access and permissions
+----------------------
+
+The Admin dashboard can be accessed by default from the `/admin/` URL. Since
+there is no login form, the user must login on Astakos first and then visit the
+above URL. Access will be granted only to users that belong to a predefined
+list of Astakos groups. By default, there are three group categories that
+are mapped 1-to-1 to Astakos groups:
+
+* ADMIN_READONLY_GROUP: 'admin-readonly'
+* ADMIN_HELPDESK_GROUP: 'helpdesk'
+* ADMIN_GROUP:          'admin'
+
+The group categories can be changed using the `ADMIN_PERMITTED_GROUPS` setting.
+In order to change the Astakos group that a category corresponds to, the
+administrator can specify the group that he/she wants in the
+`ADMIN_READONLY_GROUP`, `ADMIN_HELPDESK_GROUP` or `ADMIN_GROUP` settings.
+
+Note that while any user that belongs to the `ADMIN_PERMITTED_GROUPS` has the
+same access to the administrator dashboard, the actions that are allowed for a
+group may differ. That's because Admin implements a Role-Based Access Control
+(RBAC) policy, which can be changed from the `ADMIN_RBAC` setting. By default,
+users in the `ADMIN_READONLY_GROUP` cannot perform any actions. On the other
+hand, users in the `ADMIN_GROUP` can perform all actions.  In the middle of the
+spectrum is the `ADMIN_HELPDESK_GROUP`, which by default performs a small
+subset of reversible actions.
+
+Seting up Admin
+---------------
+
+Admin is bundled by default with a list of sane settings. The most important
+one, `ADMIN_ENABLED`, is set to `True` and defines whether Admin will be used
+or not. Therefore, the administrator only needs to create the necessary Astakos
+groups and add trusted users in them. The following example will create an
+admin group and will add a user in it:
+
+.. code-block:: console
+
+ snf-manage group-add admin
+ snf-manage user-modify --add-group=admin <user_id>
+
+That's all that is required for a single-node setup. For a multi-node setup,
+please consult the following section:
+
+Multi-node Setup
+~~~~~~~~~~~~~~~~
+
+Admin by design does not use the Astakos/Cyclades API for any action. Instead,
+it requires direct access to the Astakos/Cyclades database as well as the
+settings of their nodes. As a result, when installing Admin in a node, the
+Astakos and Cyclades packages will also be installed.
+
+In order to disable the Astakos/Cyclades API in the Admin node, the
+administrator can add the following line in `99-local.conf`:
+
+.. code-block:: console
+
+    ROOT_URLCONF="synnefo_admin.admin.urls"
+
+As a result, Admin will be accessible by this URL: `<Admin_Node_URL>/`. If the
+administrator wishes to use the default URL (`<Admin_Node_URL>/admin/`), then
+the following line must be added instead.
+
+.. code-block:: console
+
+    ROOT_URLCONF="synnefo_admin.urls"
+
+Furthermore, if Astakos and Cyclades have separate databases, then they must be
+defined in the `DATABASES` setting of `10-snf-webproject-database.conf`. An
+example setup is the following:
+
+.. code-block:: console
+
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.postgresql_psycopg2',
+            'NAME': 'snf_apps_cyclades',
+            'HOST': <Cyclades host>,
+            <...snip..>
+        }, 'cyclades': {
+            'ENGINE': 'django.db.backends.postgresql_psycopg2',
+            'NAME': 'snf_apps_cyclades',
+            'HOST': <Cyclades host>,
+            <...snip..>
+        }, 'astakos': {
+            'ENGINE': 'django.db.backends.postgresql_psycopg2',
+            'NAME': 'snf_apps_astakos',
+            'HOST': <Cyclades host>,
+            <...snip..>
+        }
+    }
+
+    DATABASE_ROUTERS = ['synnefo_admin.admin.routers.AdminRouter']
+
+You may notice that there are three databases instead of two. That's because
+Django requires that every `DATABASES` setting has a *default* database. In our
+case, we suggest that you use as default the Cyclades database. Finally, you
+must not forget to add the `DATABASE_ROUTERS` setting in the above example that
+must always be used in multi-db setups.
+
+Disabling Admin
+---------------
+
+The easiest way to disable the Admin Dashboard is to set the `ADMIN_ENABLED`
+setting to False.
+
+
 List of all Synnefo components
 ==============================
 
@@ -1664,6 +1782,7 @@ They are also available from our apt repository: ``apt.dev.grnet.gr``
  * `snf-pithos-webclient <http://www.synnefo.org/docs/pithos-webclient/latest/index.html>`_
  * `snf-cyclades-app <http://www.synnefo.org/docs/snf-cyclades-app/latest/index.html>`_
  * `snf-cyclades-gtools <http://www.synnefo.org/docs/snf-cyclades-gtools/latest/index.html>`_
+ * `snf-admin-app <http://www.synnefo.org/docs/snf-admin-app/latest/index.html>`_
  * `astakosclient <http://www.synnefo.org/docs/astakosclient/latest/index.html>`_
  * `snf-vncauthproxy <https://github.com/grnet/snf-vncauthproxy>`_
  * `snf-image <http://www.synnefo.org/docs/snf-image/latest/index.html/>`_

docs/install-guide-debian.rst

@@ -2538,3 +2538,43 @@ SSH or RDP (for windows machines).
 
 Congratulations. You have successfully installed the whole Synnefo stack and
 connected all components.
+
+
+Installation of Admin on node1
+==============================
+
+This section describes the installation of Admin. Admin is a Synnefo component
+that provides to trusted users the ability to manage and view various different
+Synnefo entities such as users, VMs, projects etc.
+
+We will install Admin on node1. To do so, we install the corresponding
+package by running on node1 the following command:
+
+.. code-block:: console
+
+   # apt-get install snf-admin-app
+
+Once the package is installed, we are done. We can proceed with testing Admin.
+
+Testing of Admin
+================
+
+In order to test the Admin Dashboard, we need a user that belongs to the
+`admin` group. We will use the user that was created in `Testing of Astakos`_
+section:
+
+.. code-block:: console
+
+    root@node1:~ # snf-manage group-add admin
+    root@node1:~ # snf-manage user-modify 1 --add-group=admin
+
+Then, you need to login to the Astakos node by visiting the following URL:
+
+    ``http://node1.example.com/astakos``
+
+Once you login successfully, you can access the Admin Dashboard from this URL:
+
+    ``http://node1.example.com/admin/``
+
+This should redirect you to the **Users** table, where there should be an entry
+with this user.

snf-admin-app/MANIFEST.in

+recursive-include synnefo_admin/admin/static *
+recursive-include synnefo_admin/docs/ *.rst
+
+include distribute_setup.py

snf-admin-app/distribute_setup.py

+#!python
+"""Bootstrap distribute installation
+
+If you want to use setuptools in your package's setup.py, just include this
+file in the same directory with it, and add this to the top of your setup.py::
+
+    from distribute_setup import use_setuptools
+    use_setuptools()
+
+If you want to require a specific version of setuptools, set a download
+mirror, or use an alternate download directory, you can do so by supplying
+the appropriate options to ``use_setuptools()``.
+
+This file can also be run as a script to install or upgrade setuptools.
+"""
+import os
+import sys
+import time
+import fnmatch
+import tempfile
+import tarfile
+from distutils import log
+
+try:
+    from site import USER_SITE
+except ImportError:
+    USER_SITE = None
+
+try:
+    import subprocess
+
+    def _python_cmd(*args):
+        args = (sys.executable,) + args
+        return subprocess.call(args) == 0
+
+except ImportError:
+    # will be used for python 2.3
+    def _python_cmd(*args):
+        args = (sys.executable,) + args
+        # quoting arguments if windows
+        if sys.platform == 'win32':
+            def quote(arg):
+                if ' ' in arg:
+                    return '"%s"' % arg
+                return arg
+            args = [quote(arg) for arg in args]
+        return os.spawnl(os.P_WAIT, sys.executable, *args) == 0
+
+DEFAULT_VERSION = "0.6.10"
+DEFAULT_URL = "http://pypi.python.org/packages/source/d/distribute/"
+SETUPTOOLS_FAKED_VERSION = "0.6c11"
+
+SETUPTOOLS_PKG_INFO = """\
+Metadata-Version: 1.0
+Name: setuptools
+Version: %s
+Summary: xxxx
+Home-page: xxx
+Author: xxx
+Author-email: xxx
+License: xxx
+Description: xxx
+""" % SETUPTOOLS_FAKED_VERSION
+
+
+def _install(tarball):
+    # extracting the tarball
+    tmpdir = tempfile.mkdtemp()
+    log.warn('Extracting in %s', tmpdir)
+    old_wd = os.getcwd()
+    try:
+        os.chdir(tmpdir)
+        tar = tarfile.open(tarball)
+        _extractall(tar)
+        tar.close()
+
+        # going in the directory
+        subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
+        os.chdir(subdir)
+        log.warn('Now working in %s', subdir)
+
+        # installing
+        log.warn('Installing Distribute')
+        if not _python_cmd('setup.py', 'install'):
+            log.warn('Something went wrong during the installation.')
+            log.warn('See the error message above.')
+    finally:
+        os.chdir(old_wd)
+
+
+def _build_egg(egg, tarball, to_dir):
+    # extracting the tarball
+    tmpdir = tempfile.mkdtemp()
+    log.warn('Extracting in %s', tmpdir)
+    old_wd = os.getcwd()
+    try:
+        os.chdir(tmpdir)
+        tar = tarfile.open(tarball)
+        _extractall(tar)
+        tar.close()
+
+        # going in the directory
+        subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
+        os.chdir(subdir)
+        log.warn('Now working in %s', subdir)
+
+        # building an egg
+        log.warn('Building a Distribute egg in %s', to_dir)
+        _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
+
+    finally:
+        os.chdir(old_wd)
+    # returning the result
+    log.warn(egg)
+    if not os.path.exists(egg):
+        raise IOError('Could not build the egg.')
+
+
+def _do_download(version, download_base, to_dir, download_delay):
+    egg = os.path.join(to_dir, 'distribute-%s-py%d.%d.egg'
+                       % (version, sys.version_info[0], sys.version_info[1]))
+    if not os.path.exists(egg):
+        tarball = download_setuptools(version, download_base,
+                                      to_dir, download_delay)
+        _build_egg(egg, tarball, to_dir)
+    sys.path.insert(0, egg)
+    import setuptools
+    setuptools.bootstrap_install_from = egg
+
+
+def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
+                   to_dir=os.curdir, download_delay=15, no_fake=True):
+    # making sure we use the absolute path
+    to_dir = os.path.abspath(to_dir)
+    was_imported = 'pkg_resources' in sys.modules or \
+        'setuptools' in sys.modules
+    try:
+        try:
+            import pkg_resources
+            if not hasattr(pkg_resources, '_distribute'):
+                if not no_fake:
+                    _fake_setuptools()
+                raise ImportError
+        except ImportError:
+            return _do_download(version, download_base, to_dir, download_delay)
+        try:
+            pkg_resources.require("distribute>="+version)
+            return
+        except pkg_resources.VersionConflict:
+            e = sys.exc_info()[1]
+            if was_imported:
+                sys.stderr.write(
+                "The required version of distribute (>=%s) is not available,\n"
+                "and can't be installed while this script is running. Please\n"
+                "install a more recent version first, using\n"
+                "'easy_install -U distribute'."
+                "\n\n(Currently using %r)\n" % (version, e.args[0]))
+                sys.exit(2)
+            else:
+                del pkg_resources, sys.modules['pkg_resources']    # reload ok
+                return _do_download(version, download_base, to_dir,
+                                    download_delay)
+        except pkg_resources.DistributionNotFound:
+            return _do_download(version, download_base, to_dir,
+                                download_delay)
+    finally:
+        if not no_fake:
+            _create_fake_setuptools_pkg_info(to_dir)
+
+def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
+                        to_dir=os.curdir, delay=15):
+    """Download distribute from a specified location and return its filename
+
+    `version` should be a valid distribute version number that is available
+    as an egg for download under the `download_base` URL (which should end
+    with a '/'). `to_dir` is the directory where the egg will be downloaded.
+    `delay` is the number of seconds to pause before an actual download
+    attempt.
+    """
+    # making sure we use the absolute path
+    to_dir = os.path.abspath(to_dir)
+    try:
+        from urllib.request import urlopen
+    except ImportError:
+        from urllib2 import urlopen
+    tgz_name = "distribute-%s.tar.gz" % version
+    url = download_base + tgz_name
+    saveto = os.path.join(to_dir, tgz_name)
+    src = dst = None
+    if not os.path.exists(saveto):  # Avoid repeated downloads
+        try:
+            log.warn("Downloading %s", url)
+            src = urlopen(url)
+            # Read/write all in one block, so we don't create a corrupt file
+            # if the download is interrupted.
+            data = src.read()
+            dst = open(saveto, "wb")
+            dst.write(data)
+        finally:
+            if src:
+                src.close()
+            if dst:
+                dst.close()
+    return os.path.realpath(saveto)
+
+def _no_sandbox(function):
+    def __no_sandbox(*args, **kw):
+        try:
+            from setuptools.sandbox import DirectorySandbox
+            if not hasattr(DirectorySandbox, '_old'):
+                def violation(*args):
+                    pass
+                DirectorySandbox._old = DirectorySandbox._violation
+                DirectorySandbox._violation = violation
+                patched = True
+            else:
+                patched = False
+        except ImportError:
+            patched = False
+
+        try:
+            return function(*args, **kw)
+        finally:
+            if patched:
+                DirectorySandbox._violation = DirectorySandbox._old
+                del DirectorySandbox._old
+
+    return __no_sandbox
+
+def _patch_file(path, content):
+    """Will backup the file then patch it"""
+    existing_content = open(path).read()
+    if existing_content == content:
+        # already patched
+        log.warn('Already patched.')
+        return False
+    log.warn('Patching...')
+    _rename_path(path)
+    f = open(path, 'w')
+    try:
+        f.write(content)
+    finally:
+        f.close()
+    return True