utils.py 9.74 KB
Newer Older
Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
1 2
import rrdtool
import time
Sergios Aftsidis's avatar
Sergios Aftsidis committed
3
import os
Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
4 5 6 7 8

from django.core.urlresolvers import reverse
from django.conf import settings

from network.models import Ifce
9
from rg.models import Graph, DataSource
10 11 12 13 14 15 16
# import gevent

from gevent.pool import Pool

CONCURRECY = 50
pool = Pool(CONCURRECY)

Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94


def last_x_rec(x):
    return range(2, x + 1)


def get_load(ifce, start, end):
    if not start:
        start = str(int(time.time()) - 10000)
    if not end:
        end = str(int(time.time()) - 1000)
    datasources = ifce.get('datasources')
    if datasources:
        rrdfile = datasources
        for i in last_x_rec(3):
            # rrdtool hates unicode
            try:
                latest = rrdtool.fetch(
                    rrdfile,
                    'AVERAGE',
                    '-s %s' % start.encode('ascii', 'ignore'),
                    '-e %s' % end.encode('ascii', 'ignore')
                )[2][-i]
            except IndexError:
                # no rrds found
                return (0, 0, start, end)
            if latest[0] is None or latest[1] is None:
                loadin = 0
                loadout = 0
                continue
            else:
                loadin = latest[0]
                loadout = latest[1]
                break
        # Making sure our tuple does not contain Nones
        if ifce.get('bandwidth') == 0:
            ifce_in = '0'
            ifce_out = '0'
        else:
            ifce_in = '%s' % round(loadin * 800 / ifce.get('bandwidth'), 2)
            ifce_out = '%s' % round(loadout * 800 / ifce.get('bandwidth'), 2)
    else:
        ifce_in = '0'
        ifce_out = '0'
    return (ifce_in, ifce_out, start, end)


def get_load_for_interfaces(ifces, key, start=None, end=None):
    traffic_in = 0
    traffic_out = 0
    bandwidth = 0
    response = {}
    if ifces:
        for ifce in ifces:
            bandwidth += ifce.get('bandwidth')
            ifce_in, ifce_out, start, end = get_load(ifce, start, end)
            traffic_in += float(ifce_in)
            traffic_out += float(ifce_out)
        if ifces:
            traffic_in = traffic_in / len(ifces)
            traffic_out = traffic_out / len(ifces)
        response = {
            '%s' % key: {
                'bandwidth': bandwidth,
                'load': {
                    'in': traffic_in,
                    'out': traffic_out
                },
                'start': start,
                'end': end
            }
        }
    return response


def get_load_for_links(ifces, start=None, end=None):
    response = {}
    threads = []
95 96

    spawned = 0
Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
97 98
    for key, val in ifces.iteritems():
        # response.update(get_load_for_interfaces(val, key, start, end))
99 100 101 102 103
        threads.append(pool.spawn(get_load_for_interfaces, val, key, start, end))
        spawned += 1
        if spawned == CONCURRECY:
            pool.join()
    pool.join()
Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
104 105 106 107 108 109
    for thread in threads:
        if thread.value:
            response.update(thread.value)
    return response


110 111
def graph_for_each_interface(graph, datasources, start='-1d', end='-300'):

112
    response = {'Total': graph.get_draw_url()}
113

Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
114 115 116 117
    for i in range(0, len(datasources) / 2):
        dss = datasources[i * 2:i * 2 + 2]
        if dss:
            ifce = Ifce.objects.get(pk=dss[0].object_id)
118 119 120 121 122
            try:
                png = dss[0].graph_set.filter(type='traffic')[0].get_draw_url()
                mon = dss[0].graph_set.filter(type='traffic')[0].get_absolute_url()
            except IndexError:
                continue
Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
123 124 125 126 127 128 129 130 131
            response.update({
                ifce.name: {
                    'png': png,
                    'link_to_mon': mon,
                }
            })
    return response


Sergios Aftsidis's avatar
Sergios Aftsidis committed
132 133 134 135 136 137
def create_graph_for_interfaces(datasources, start=None, end=None, **kwargs):
    if 'dryrun' in kwargs:
        dryrun = kwargs['dryrun']
    else:
        dryrun = False

Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
138 139 140 141 142
    if not start:
        start = '-1d'
    if not end:
        end = '-100'
    if datasources:
Sergios Aftsidis's avatar
Sergios Aftsidis committed
143
        name = os.path.basename(datasources[0].rrdfile.path).split('.')
Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
144 145 146 147 148 149 150
        if name[0]:
            name = name[0]
        else:
            name = name[1]
        title = '%s to %s' % (name.split('-')[0], name.split('-')[1])
        arguments = ['%s/%s%s%s.png' % (settings.RG_STATICPATH, name, start, end)]

Sergios Aftsidis's avatar
Sergios Aftsidis committed
151 152
        if hasattr(settings, 'RG_RRDCACHED'):
            arguments.append('--daemon=%s' % str(settings.RG_RRDCACHED))
Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
153 154 155 156
        arguments.append('-s %s' % start)
        arguments.append('-e %s' % end)
        arguments.append('-t %s' % title)
        arguments.append('--slope-mode')
Sergios Aftsidis's avatar
Sergios Aftsidis committed
157
        arguments.append('-l 0')
Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
        aggregate_dict = {'ds0': [], 'ds1': []}
        interfaces_dict = {
            'ds0': {
                'label': 'In ',
                'color': '#0000ff',
                'graph_type': 'AREA'
            },
            'ds1': {
                'label': 'Out',
                'color': '#00ff00',
                'graph_type': 'LINE'
            }
        }
        legend = []
        used_ds = []
        for d in datasources:
            if d.pk not in used_ds:
                arguments.append('DEF:d%s=%s:%s:AVERAGE' % (d.pk, d.rrdfile.path, d.name))
                arguments.append('DEF:%smax=%s:%s:MAX' % (d.pk, d.rrdfile.path, d.name))
                aggregate_dict[d.name].append(d.pk)
                arguments.append('CDEF:d%sdispAVG=d%s,8,*' % (d.pk, d.pk))
                used_ds.append(d.pk)
        arguments.append('COMMENT:\t\tMin\g')
        arguments.append('COMMENT:\tMax\g')
        arguments.append('COMMENT:\tAverage\g')
        arguments.append('COMMENT:\tCurrent\\n')
        for key, val in aggregate_dict.iteritems():
            arguments.append('CDEF:aggr%s=' % (key))
            legend.append('%s:aggr%s%s:%s' % (
                interfaces_dict.get(key).get('graph_type'), key, interfaces_dict.get(key).get('color'), interfaces_dict.get(key).get('label'))
            )
            for rrd in val:
                arguments[-1] += 'd%sdispAVG,' % rrd
                if rrd != val[0]:
Sergios Aftsidis's avatar
Sergios Aftsidis committed
192
                    arguments[-1] += 'ADDNAN,'
Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
193 194 195 196 197 198
            legend.append('%s:aggr%s:%s:%s' % ('GPRINT', key, 'MIN', '\t%4.2lf%s\g'))
            legend.append('%s:aggr%s:%s:%s' % ('GPRINT', key, 'MAX', '\t%4.2lf%s\g'))
            legend.append('%s:aggr%s:%s:%s' % ('GPRINT', key, 'AVERAGE', '\t%4.2lf%s\g'))
            legend.append('%s:aggr%s:%s:%s' % ('GPRINT', key, 'LAST', '\t%4.2lf%s\\n'))
        arguments.extend(legend)
        args = [str(val) for val in arguments]
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215

        if dryrun == True:
            import re
            for ri in range(len(args)):
                if args[ri].find('-') == 0 and \
                        len(args[ri].split()) > 2:
                    larg = args[ri].split()
                    larg[1] = "'" + larg[1]
                    larg[-1] = larg[-1] + "'"
                    args[ri] = ' '.join(larg)
                elif bool(re.search(r'^([CV]?DEF|AREA|GPRINT|COMMENT|HRULE):',
                                    args[ri])):
                    args[ri] = "'%s'" % args[ri]
            return ' \n'.join('rrdtool graph'.split() + args)
        else:
            rrdtool.graphv(*args)

Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
216 217 218 219 220 221 222 223 224 225 226 227
        url = reverse(
            'get-png-data',
            kwargs={
                'path': '%s%s%s.png' % (name, start, end)
            }
        )
        return url
    return False


def create_links_dict(links):
    ifces = {}
228 229
    ifce_pairs = {}
    # walk through links to get all interfaces
Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
230 231
    for link in links:
        try:
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268
            ifces.setdefault(link.local_ifce.pk, link.local_ifce)
            ifces.setdefault(link.remote_ifce.pk, link.remote_ifce)
        except AttributeError:
            continue

    # fetch all DataSource objects for our set of interfaces,
    # pre-loading related RrdFile objects
    dss = DataSource.objects.\
        filter(object_id__in=ifces.keys(),
               content_type__model='ifce',
               graph__type='traffic').\
        select_related('rrdfile')
    # force QuerySet to be evaluated, returning a list of tuples
    # indexed by ifce
    dss = [(x.object_id, x) for x in dss]

    # fill the dict for each ifce (previously: ifce.as_dict())
    for k, ifce in ifces.iteritems():
        datasources = [x[1] for x in dss if x[0] == k]
        if datasources:
            datasources = datasources[0].rrdfile.path.__str__()
        ifces[k] = {
            'ifce_id': ifce.pk,
            'bandwidth': ifce.bandwidth,
            'node': ifce.node.name,
            'datasources': datasources
        }

    # re-iterate links to create {ifce_pair: [ifce_dicts]}
    for link in links:
        try:
            localifce_as_dict = ifces.get(link.local_ifce.pk)
            ifce_pair_name = '%s_%s' % (link.local_ifce.node.name,
                                        link.remote_ifce.node.name)
            ifce_pair = ifce_pairs.get(ifce_pair_name)
            if ifce_pair:
                ifce_pair.append(localifce_as_dict)
Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
269
            else:
270 271
                ifce_pairs.update({ifce_pair_name: [localifce_as_dict]})
        except AttributeError:
Stauros Kroustouris's avatar
initial  
Stauros Kroustouris committed
272
            continue
273 274

    return ifce_pairs
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297


def get_graph_for_node_link(local, remote, separate=False):

    response = []

    try:
        graph = Graph.objects.get(
            type='n2ntraffic',
            description__contains='%s - %s' % (local, remote)
        )
    except:
        graph = None

    if graph:
        if separate:
            # '-1d', '-300' now need to be fixed values as we are not creating
            # the graphs, only getting the ones that rg creates with those
            # default values
            response = graph_for_each_interface(
                graph, graph.datasources.all(), '-1d', '-300')
        else:
            response = {
298
                'graph': graph.get_draw_url(),
299 300 301 302 303 304
                'links': [{"from_descr": graph.description}]
            }

    else:
        response = {'graph': False, 'links': []}
    return response