snf-deploy: Add stats support

ci/schemas/one_node_wheezy/synnefo.conf

@@ -21,6 +21,7 @@ mq = node1
 ns = node1
 client = node1
 router = node1
+stats = node1
 
 
 [synnefo]

ci/schemas/one_node_wheezy/wheezy.conf

@@ -23,6 +23,7 @@ ganeti-instance-debootstrap =
 python-django-south =
 python-django =
 drbd8-utils =
+collectd =
 
 
 [synnefo]
@@ -37,6 +38,7 @@ snf-webproject = wheezy
 snf-pithos-app = wheezy
 snf-pithos-backend = wheezy
 snf-tools = wheezy
+snf-stats-app = wheezy
 
 
 [ganeti]

snf-deploy/conf/synnefo.conf

@@ -21,6 +21,7 @@ mq = node1
 ns = node1
 client = node1
 router = node1
+stats = node1
 
 
 [synnefo]

snf-deploy/conf/wheezy.conf

@@ -23,6 +23,7 @@ ganeti-instance-debootstrap =
 python-django-south =
 python-django =
 drbd8-utils =
+collectd =
 
 
 [synnefo]
@@ -37,6 +38,7 @@ snf-webproject = wheezy
 snf-pithos-app = wheezy
 snf-pithos-backend = wheezy
 snf-tools = wheezy
+snf-stats-app = wheezy
 
 
 [ganeti]

snf-deploy/files/etc/collectd/collectd.conf

+FQDNLookup true
+
+LoadPlugin syslog
+<Plugin syslog>
+	LogLevel info
+</Plugin>
+
+Include "/etc/collectd/synnefo-*.conf"
+Include "/etc/collectd/filters.conf"
+Include "/etc/collectd/thresholds.conf"

snf-deploy/files/etc/collectd/passwd

+user: pass

snf-deploy/files/etc/collectd/synnefo-ganeti.conf

+<LoadPlugin python>
+	Globals true
+</LoadPlugin>
+<Plugin python>
+	ModulePath "/usr/lib/snf-cyclades-gtools/collectd/"
+	LogTraces true
+	Interactive false
+	Import "ganeti-cpustats"
+	Import "ganeti-netstats"
+</Plugin>
+
+LoadPlugin network
+<Plugin network>
+    <Server "%STATS%" "25826">
+        SecurityLevel "Encrypt"
+        Username "user"
+        Password "secret"
+    </Server>
+    TimeToLive 128
+    ReportStats false
+    MaxPacketSize 65535
+</Plugin>

snf-deploy/files/etc/collectd/synnefo-stats.conf

+LoadPlugin network
+<Plugin network>
+    TimeToLive 128
+
+    <Listen "%STATS%" "25826">
+        SecurityLevel "Sign"
+        AuthFile "/etc/collectd/passwd"
+    </Listen>
+
+    ReportStats false
+    MaxPacketSize 65535
+</Plugin>
+
+LoadPlugin rrdtool
+<Plugin rrdtool>
+	DataDir "/var/lib/collectd/rrd"
+	CacheTimeout 120
+	CacheFlush 900
+	WritesPerSecond 30
+	RandomTimeout 0
+</Plugin>

snf-deploy/files/etc/synnefo/cyclades.conf

@@ -14,10 +14,10 @@ VMS_USER_QUOTA = {
 }
 MAX_NETWORKS_PER_USER = 3
 NETWORKS_USER_QUOTA = { 'psomas@grnet.gr': 1000 }
-CPU_BAR_GRAPH_URL = 'https://%HOST%/stats/%s/cpu-bar.png'
-CPU_TIMESERIES_GRAPH_URL = 'https://%HOST%/stats/%s/cpu-ts.png'
-NET_BAR_GRAPH_URL = 'https://%HOST%/stats/%s/net-bar.png'
-NET_TIMESERIES_GRAPH_URL = 'https://%HOST%/stats/%s/net-ts.png'
+CPU_BAR_GRAPH_URL = 'https://%STATS%/stats/v1.0/cpu-bar/%s'
+CPU_TIMESERIES_GRAPH_URL = 'https://%STATS%/stats/v1.0/cpu-ts/%s'
+NET_BAR_GRAPH_URL = 'https://%STATS%/stats/v1.0/net-bar/%s'
+NET_TIMESERIES_GRAPH_URL = 'https://%STATS%/stats/v1.0/net-ts/%s'
 GANETI_DISK_TEMPLATES = ('blockdev', 'diskless', 'drbd', 'file', 'plain',
                          'rbd',  'sharedfile', 'ext')
 ASTAKOS_AUTH_URL = 'https://%ACCOUNTS%/identity/v2.0'
@@ -86,3 +86,5 @@ CYCLADES_VNCAUTHPROXY_OPTS = {
     'auth_user': 'synnefo',
     'auth_password': 'synnefo_vnc_pass',
 }
+
+CYCLADES_STATS_SECRET_KEY = "random"

snf-deploy/files/etc/synnefo/stats.conf

+STATS_BASE_URL = "https://%STATS%/stats/"
+STATS_SECRET_KEY = "random"

snf-deploy/snfdeploy/__init__.py

@@ -80,10 +80,12 @@ Usage: snf-deploy run <action> [<action>...]
       setup_apt              add_rapi_user                 add_backend
       setup_astakos          add_nodes                     add_image_locally
       setup_cms              astakos_loaddata              add_network
+      setup_collectd
       setup_common           astakos_register_components   add_ns
       setup_cyclades         cms_loaddata                  add_user
       setup_db               cyclades_loaddata             connect_bridges
       setup_ganeti           enable_drbd                   create_bridges
+      setup_ganeti_collectd
       setup_gtools           init_cluster                  create_vlans
       setup_gunicorn         setup_nfs_clients             destroy_db
       setup_hosts            setup_nfs_server              \
@@ -100,6 +102,8 @@ get_auth_token_from_db
       setup_pithos
       setup_pithos_dir
       setup_router
+      setup_stats
+      setup_stats_collectd
       setup_vncauthproxy
       setup_webproject
 
@@ -445,7 +449,8 @@ def get_actions(*args):
             "setup_cyclades", "cyclades_loaddata", "add_pools",
             "export_services", "import_services", "set_user_quota",
             "setup_kamaki", "upload_image", "register_image",
-            "setup_burnin"
+            "setup_burnin",
+            "setup_stats"
         ],
         "supdate": [
             "apt_get_update", "setup_astakos",
@@ -480,6 +485,7 @@ def get_actions(*args):
             "setup_image_host", "setup_image_helper", "add_image_locally",
             "debootstrap", "setup_net_infra",
             "setup_lvm", "enable_lvm", "enable_drbd", "setup_drbd_dparams",
+            "setup_ganeti_collectd"
         ],
         "gupdate": ["setup_apt", "setup_ganeti"],
         "gdestroy": ["destroy_cluster"],

snf-deploy/snfdeploy/fabfile.py

@@ -62,6 +62,7 @@ def setup_env(args):
         "ns": [env.env.ns.ip],
         "client": [env.env.client.ip],
         "router": [env.env.router.ip],
+        "stats": [env.env.stats.ip],
     }
 
     env.enable_lvm = False
@@ -1129,6 +1130,70 @@ def cyclades_loaddata():
     #run("snf-manage loaddata flavors")
 
 
+@roles("ganeti", "stats")
+def setup_collectd():
+    install_package("collectd")
+    tmpl = "/etc/collectd/collectd.conf"
+    replace = {}
+    custom = customize_settings_from_tmpl(tmpl, replace)
+    try_put(custom, tmpl, mode=0644)
+
+
+@roles("ganeti")
+def setup_ganeti_collectd():
+    setup_collectd()
+
+    tmpl = "/etc/collectd/passwd"
+    replace = {}
+    custom = customize_settings_from_tmpl(tmpl, replace)
+    try_put(custom, tmpl, mode=0644)
+
+    tmpl = "/etc/collectd/synnefo-ganeti.conf"
+    replace = {
+        "STATS": env.env.stats.fqdn,
+        }
+    custom = customize_settings_from_tmpl(tmpl, replace)
+    try_put(custom, tmpl, mode=0644)
+
+    try_run("/etc/init.d/collectd restart")
+
+
+@roles("stats")
+def setup_stats_collectd():
+    setup_collectd()
+    tmpl = "/etc/collectd/synnefo-stats.conf"
+
+    replace = {
+        "STATS": env.env.stats.fqdn,
+        }
+    custom = customize_settings_from_tmpl(tmpl, replace)
+    try_put(custom, tmpl, mode=0644)
+    try_run("/etc/init.d/collectd restart")
+
+
+@roles("stats")
+def setup_stats():
+    debug(env.host, "Setting up snf-stats-app...")
+    setup_stats_collectd()
+    setup_gunicorn()
+    setup_apache()
+    setup_webproject()
+    install_package("snf-stats-app")
+    cmd = """
+    mkdir /var/cache/snf-stats-app/
+    chown www-data:www-data /var/cache/snf-stats-app/
+    """
+    try_run(cmd)
+    tmpl = "/etc/synnefo/stats.conf"
+
+    replace = {
+        "STATS": env.env.stats.fqdn,
+        }
+    custom = customize_settings_from_tmpl(tmpl, replace)
+    try_put(custom, tmpl, mode=0644)
+    try_run("/etc/init.d/gunicorn restart")
+
+
 @roles("cyclades")
 def setup_cyclades():
     debug(env.host, "Setting up snf-cyclades-app...")
@@ -1163,6 +1228,7 @@ def setup_cyclades():
         "HOST": env.env.cyclades.ip,
         "domain": env.env.domain,
         "CYCLADES_SERVICE_TOKEN": service_token,
+        'STATS': env.env.stats.fqdn,
         }
     custom = customize_settings_from_tmpl(tmpl, replace)
     try_put(custom, tmpl, mode=0644)