utils.py 40.4 KB
Newer Older
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
1
#!/usr/bin/env python
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
2
3
4
5
6
7

"""
Synnefo ci utils module
"""

import os
8
import re
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
9
10
11
12
import sys
import time
import logging
import fabric.api as fabric
13
import subprocess
14
import tempfile
15
from ConfigParser import ConfigParser, DuplicateSectionError
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
16

17
from kamaki.cli import config as kamaki_config
18
from kamaki.clients.astakos import AstakosClient, parse_endpoints
19
from kamaki.clients.cyclades import CycladesClient, CycladesNetworkClient
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
20
from kamaki.clients.image import ImageClient
21
from kamaki.clients.compute import ComputeClient
22
from kamaki.clients import ClientError
23
import filelocker
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
24

25
DEFAULT_CONFIG_FILE = "ci_wheezy.conf"
26
27
# Is our terminal a colorful one?
USE_COLORS = True
28
29
30
31
# UUID of owner of system images
DEFAULT_SYSTEM_IMAGES_UUID = [
    "25ecced9-bf53-4145-91ee-cf47377e9fb2",  # production (okeanos.grnet.gr)
    "04cbe33f-29b7-4ef1-94fb-015929e5fc06",  # testing (okeanos.io)
32
]
33

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
34
35
36
37
38
39
40

def _run(cmd, verbose):
    """Run fabric with verbose level"""
    if verbose:
        args = ('running',)
    else:
        args = ('running', 'stdout',)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
41
    with fabric.hide(*args):  # Used * or ** magic. pylint: disable-msg=W0142
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
42
43
44
        return fabric.run(cmd)


45
46
47
48
49
50
def _put(local, remote):
    """Run fabric put command without output"""
    with fabric.quiet():
        fabric.put(local, remote)


Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
51
52
def _red(msg):
    """Red color"""
53
54
    ret = "\x1b[31m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
    return ret
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
55
56
57
58


def _yellow(msg):
    """Yellow color"""
59
60
    ret = "\x1b[33m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
    return ret
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
61
62
63
64


def _green(msg):
    """Green color"""
65
66
    ret = "\x1b[32m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
    return ret
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
67
68
69
70


def _check_fabric(fun):
    """Check if fabric env has been set"""
71
    def wrapper(self, *args, **kwargs):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
72
73
74
        """wrapper function"""
        if not self.fabric_installed:
            self.setup_fabric()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
75
            self.fabric_installed = True
76
        return fun(self, *args, **kwargs)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
77
78
79
80
81
    return wrapper


def _check_kamaki(fun):
    """Check if kamaki has been initialized"""
82
    def wrapper(self, *args, **kwargs):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
83
84
85
        """wrapper function"""
        if not self.kamaki_installed:
            self.setup_kamaki()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
86
            self.kamaki_installed = True
87
        return fun(self, *args, **kwargs)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
88
89
90
91
92
93
94
95
    return wrapper


class _MyFormatter(logging.Formatter):
    """Logging Formatter"""
    def format(self, record):
        format_orig = self._fmt
        if record.levelno == logging.DEBUG:
96
            self._fmt = "  %(message)s"
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
97
        elif record.levelno == logging.INFO:
98
            self._fmt = "%(message)s"
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
99
        elif record.levelno == logging.WARNING:
100
            self._fmt = _yellow("[W] %(message)s")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
101
        elif record.levelno == logging.ERROR:
102
            self._fmt = _red("[E] %(message)s")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
103
104
105
106
107
        result = logging.Formatter.format(self, record)
        self._fmt = format_orig
        return result


108
109
110
111
112
113
114
115
116
# Too few public methods. pylint: disable-msg=R0903
class _InfoFilter(logging.Filter):
    """Logging Filter that allows DEBUG and INFO messages only"""
    def filter(self, rec):
        """The filter"""
        return rec.levelno in (logging.DEBUG, logging.INFO)


# Too many instance attributes. pylint: disable-msg=R0902
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
117
118
119
class SynnefoCI(object):
    """SynnefoCI python class"""

120
    def __init__(self, config_file=None, build_id=None, cloud=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
121
122
123
124
125
126
127
        """ Initialize SynnefoCI python class

        Setup logger, local_dir, config and kamaki
        """
        # Setup logger
        self.logger = logging.getLogger('synnefo-ci')
        self.logger.setLevel(logging.DEBUG)
128
129
130
131
132
133
134
135
136
137
138

        handler1 = logging.StreamHandler(sys.stdout)
        handler1.setLevel(logging.DEBUG)
        handler1.addFilter(_InfoFilter())
        handler1.setFormatter(_MyFormatter())
        handler2 = logging.StreamHandler(sys.stderr)
        handler2.setLevel(logging.WARNING)
        handler2.setFormatter(_MyFormatter())

        self.logger.addHandler(handler1)
        self.logger.addHandler(handler2)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
139
140
141
142
143
144

        # Get our local dir
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
        self.repo_dir = os.path.dirname(self.ci_dir)

        # Read config file
145
        if config_file is None:
146
147
            config_file = os.path.join(self.ci_dir, DEFAULT_CONFIG_FILE)
        config_file = os.path.abspath(config_file)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
148
149
        self.config = ConfigParser()
        self.config.optionxform = str
150
        self.config.read(config_file)
151
152

        # Read temporary_config file
153
154
        self.temp_config_file = \
            os.path.expanduser(self.config.get('Global', 'temporary_config'))
155
156
        self.temp_config = ConfigParser()
        self.temp_config.optionxform = str
157
        self.temp_config.read(self.temp_config_file)
158
        self.build_id = build_id
159
160
161
        if build_id is not None:
            self.logger.info("Will use \"%s\" as build id" %
                             _green(self.build_id))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
162

163
164
165
166
167
168
169
170
171
172
        # Set kamaki cloud
        if cloud is not None:
            self.kamaki_cloud = cloud
        elif self.config.has_option("Deployment", "kamaki_cloud"):
            kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
            if kamaki_cloud == "":
                self.kamaki_cloud = None
        else:
            self.kamaki_cloud = None

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
173
174
175
176
        # Initialize variables
        self.fabric_installed = False
        self.kamaki_installed = False
        self.cyclades_client = None
177
        self.network_client = None
178
        self.compute_client = None
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
179
        self.image_client = None
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
180
        self.astakos_client = None
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
181
182
183
184

    def setup_kamaki(self):
        """Initialize kamaki

185
        Setup cyclades_client, image_client and compute_client
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
186
        """
187
188
189

        config = kamaki_config.Config()
        if self.kamaki_cloud is None:
190
191
192
193
            try:
                self.kamaki_cloud = config.get("global", "default_cloud")
            except AttributeError:
                # Compatibility with kamaki version <=0.10
194
                self.kamaki_cloud = config.get("global", "default_cloud")
195
196
197
198

        self.logger.info("Setup kamaki client, using cloud '%s'.." %
                         self.kamaki_cloud)
        auth_url = config.get_cloud(self.kamaki_cloud, "url")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
199
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
200
        token = config.get_cloud(self.kamaki_cloud, "token")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
201
202
        #self.logger.debug("Token is %s" % _green(token))

203
204
        self.astakos_client = AstakosClient(auth_url, token)
        endpoints = self.astakos_client.authenticate()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
205

206
        cyclades_url = get_endpoint_url(endpoints, "compute")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
207
208
209
210
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
        self.cyclades_client = CycladesClient(cyclades_url, token)
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2

211
        network_url = get_endpoint_url(endpoints, "network")
212
213
214
215
        self.logger.debug("Network API url is %s" % _green(network_url))
        self.network_client = CycladesNetworkClient(network_url, token)
        self.network_client.CONNECTION_RETRY_LIMIT = 2

216
        image_url = get_endpoint_url(endpoints, "image")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
217
218
219
220
        self.logger.debug("Images API url is %s" % _green(image_url))
        self.image_client = ImageClient(cyclades_url, token)
        self.image_client.CONNECTION_RETRY_LIMIT = 2

221
        compute_url = get_endpoint_url(endpoints, "compute")
222
223
224
225
        self.logger.debug("Compute API url is %s" % _green(compute_url))
        self.compute_client = ComputeClient(compute_url, token)
        self.compute_client.CONNECTION_RETRY_LIMIT = 2

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
226
227
228
229
230
231
232
233
234
235
236
237
238
    def _wait_transition(self, server_id, current_status, new_status):
        """Wait for server to go from current_status to new_status"""
        self.logger.debug("Waiting for server to become %s" % new_status)
        timeout = self.config.getint('Global', 'build_timeout')
        sleep_time = 5
        while True:
            server = self.cyclades_client.get_server_details(server_id)
            if server['status'] == new_status:
                return server
            elif timeout < 0:
                self.logger.error(
                    "Waiting for server to become %s timed out" % new_status)
                self.destroy_server(False)
239
                sys.exit(1)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
240
241
242
243
244
245
246
247
            elif server['status'] == current_status:
                # Sleep for #n secs and continue
                timeout = timeout - sleep_time
                time.sleep(sleep_time)
            else:
                self.logger.error(
                    "Server failed with status %s" % server['status'])
                self.destroy_server(False)
248
                sys.exit(1)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
249
250
251
252

    @_check_kamaki
    def destroy_server(self, wait=True):
        """Destroy slave server"""
253
        server_id = int(self.read_temp_config('server_id'))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
254
255
        fips = [f for f in self.network_client.list_floatingips()
                if str(f['instance_id']) == str(server_id)]
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
256
257
258
259
        self.logger.info("Destoying server with id %s " % server_id)
        self.cyclades_client.delete_server(server_id)
        if wait:
            self._wait_transition(server_id, "ACTIVE", "DELETED")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
260
261
262
263
        for fip in fips:
            self.logger.info("Destroying floating ip %s",
                             fip['floating_ip_address'])
            self.network_client.delete_floatingip(fip['id'])
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
264

265
266
267
    def _create_floating_ip(self):
        """Create a new floating ip"""
        networks = self.network_client.list_networks(detail=True)
268
269
270
271
272
273
274
275
276
277
278
279
280
281
        pub_nets = [n for n in networks
                    if n['SNF:floating_ip_pool'] and n['public']]
        for pub_net in pub_nets:
            # Try until we find a public network that is not full
            try:
                fip = self.network_client.create_floatingip(pub_net['id'])
            except ClientError as err:
                self.logger.warning("%s: %s", err.message, err.details)
                continue
            self.logger.debug("Floating IP %s with id %s created",
                              fip['floating_ip_address'], fip['id'])
            return fip
        self.logger.error("No mor IP addresses available")
        sys.exit(1)
282
283
284
285
286
287
288
289
290
291

    def _create_port(self, floating_ip):
        """Create a new port for our floating IP"""
        net_id = floating_ip['floating_network_id']
        self.logger.debug("Creating a new port to network with id %s", net_id)
        fixed_ips = [{'ip_address': floating_ip['floating_ip_address']}]
        port = self.network_client.create_port(
            net_id, device_id=None, fixed_ips=fixed_ips)
        return port

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
292
    @_check_kamaki
293
    # Too many local variables. pylint: disable-msg=R0914
294
295
    def create_server(self, image=None, flavor=None, ssh_keys=None,
                      server_name=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
296
297
        """Create slave server"""
        self.logger.info("Create a new server..")
298
299

        # Find a build_id to use
300
        self._create_new_build_id()
301
302

        # Find an image to use
303
        image_id = self._find_image(image)
304
        # Find a flavor to use
305
306
307
        flavor_id = self._find_flavor(flavor)

        # Create Server
308
309
310
311
312
313
314
        networks = []
        if self.config.get("Deployment", "allocate_floating_ip") == "True":
            fip = self._create_floating_ip()
            port = self._create_port(fip)
            networks.append({'port': port['id']})
        private_networks = self.config.get('Deployment', 'private_networks')
        if private_networks:
315
316
            private_networks = [p.strip() for p in private_networks.split(",")]
            networks.extend([{"uuid": uuid} for uuid in private_networks])
317
318
319
        if server_name is None:
            server_name = self.config.get("Deployment", "server_name")
            server_name = "%s(BID: %s)" % (server_name, self.build_id)
320
321
        server = self.cyclades_client.create_server(
            server_name, flavor_id, image_id, networks=networks)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
322
        server_id = server['id']
323
        self.write_temp_config('server_id', server_id)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
324
325
        self.logger.debug("Server got id %s" % _green(server_id))
        server_user = server['metadata']['users']
326
        self.write_temp_config('server_user', server_user)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
327
328
        self.logger.debug("Server's admin user is %s" % _green(server_user))
        server_passwd = server['adminPass']
329
        self.write_temp_config('server_passwd', server_passwd)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
330
331

        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
332
        self._get_server_ip_and_port(server, private_networks)
333
        self._copy_ssh_keys(ssh_keys)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
334

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
335
        # Setup Firewall
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
336
337
        self.setup_fabric()
        self.logger.info("Setup firewall")
338
        accept_ssh_from = self.config.get('Global', 'accept_ssh_from')
339
340
341
342
343
344
345
346
347
348
349
        if accept_ssh_from != "":
            self.logger.debug("Block ssh except from %s" % accept_ssh_from)
            cmd = """
            local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
                cut -d':' -f2 | cut -d' ' -f1)
            iptables -A INPUT -s localhost -j ACCEPT
            iptables -A INPUT -s $local_ip -j ACCEPT
            iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
            iptables -A INPUT -p tcp --dport 22 -j DROP
            """.format(accept_ssh_from)
            _run(cmd, False)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
350

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
351
352
353
354
        # Setup apt, download packages
        self.logger.debug("Setup apt. Install x2goserver and firefox")
        cmd = """
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
355
        echo 'precedence ::ffff:0:0/96  100' >> /etc/gai.conf
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
356
        apt-get update
357
        apt-get install curl --yes --force-yes
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
358
359
360
        echo -e "\n\n{0}" >> /etc/apt/sources.list
        # Synnefo repo's key
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
361

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
362
363
        # X2GO Key
        apt-key adv --recv-keys --keyserver keys.gnupg.net E1F958385BFE2B6E
364
        apt-get install x2go-keyring --yes --force-yes
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
365
        apt-get update
366
367
        apt-get install x2goserver x2goserver-xsession \
                iceweasel --yes --force-yes
368
369
370
371
372
373
374
375
376
377
378
379
380

        # xterm published application
        echo '[Desktop Entry]' > /usr/share/applications/xterm.desktop
        echo 'Name=XTerm' >> /usr/share/applications/xterm.desktop
        echo 'Comment=standard terminal emulator for the X window system' >> \
            /usr/share/applications/xterm.desktop
        echo 'Exec=xterm' >> /usr/share/applications/xterm.desktop
        echo 'Terminal=false' >> /usr/share/applications/xterm.desktop
        echo 'Type=Application' >> /usr/share/applications/xterm.desktop
        echo 'Encoding=UTF-8' >> /usr/share/applications/xterm.desktop
        echo 'Icon=xterm-color_48x48' >> /usr/share/applications/xterm.desktop
        echo 'Categories=System;TerminalEmulator;' >> \
                /usr/share/applications/xterm.desktop
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
381
382
383
        """.format(self.config.get('Global', 'apt_repo'))
        _run(cmd, False)

384
385
386
387
388
389
390
391
    def _find_flavor(self, flavor=None):
        """Find a suitable flavor to use

        Search by name (reg expression) or by id
        """
        # Get a list of flavors from config file
        flavors = self.config.get('Deployment', 'flavors').split(",")
        if flavor is not None:
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
392
            # If we have a flavor_name to use, add it to our list
393
394
395
396
            flavors.insert(0, flavor)

        list_flavors = self.compute_client.list_flavors()
        for flv in flavors:
397
398
            flv_type, flv_value = parse_typed_option(option="flavor",
                                                     value=flv)
399
400
401
402
403
404
            if flv_type == "name":
                # Filter flavors by name
                self.logger.debug(
                    "Trying to find a flavor with name \"%s\"" % flv_value)
                list_flvs = \
                    [f for f in list_flavors
405
406
                     if re.search(flv_value, f['name'], flags=re.I)
                     is not None]
407
408
409
410
411
412
            elif flv_type == "id":
                # Filter flavors by id
                self.logger.debug(
                    "Trying to find a flavor with id \"%s\"" % flv_value)
                list_flvs = \
                    [f for f in list_flavors
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
413
                     if str(f['id']) == flv_value]
414
415
416
417
418
419
            else:
                self.logger.error("Unrecognized flavor type %s" % flv_type)

            # Check if we found one
            if list_flvs:
                self.logger.debug("Will use \"%s\" with id \"%s\""
420
421
                                  % (_green(list_flvs[0]['name']),
                                     _green(list_flvs[0]['id'])))
422
                return list_flvs[0]['id']
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
423
424
425

        self.logger.error("No matching flavor found.. aborting")
        sys.exit(1)
426

427
    def _find_image(self, image=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
428
429
        """Find a suitable image to use

430
431
432
        In case of search by name, the image has to belong to one
        of the `DEFAULT_SYSTEM_IMAGES_UUID' users.
        In case of search by id it only has to exist.
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
433
        """
434
435
436
437
438
439
        # Get a list of images from config file
        images = self.config.get('Deployment', 'images').split(",")
        if image is not None:
            # If we have an image from command line, add it to our list
            images.insert(0, image)

440
441
        auth = self.astakos_client.authenticate()
        user_uuid = auth["access"]["token"]["tenant"]["id"]
442
443
        list_images = self.image_client.list_public(detail=True)['images']
        for img in images:
444
            img_type, img_value = parse_typed_option(option="image", value=img)
445
446
447
448
            if img_type == "name":
                # Filter images by name
                self.logger.debug(
                    "Trying to find an image with name \"%s\"" % img_value)
449
                accepted_uuids = DEFAULT_SYSTEM_IMAGES_UUID + [user_uuid]
450
                list_imgs = \
451
452
453
                    [i for i in list_images if i['user_id'] in accepted_uuids
                     and
                     re.search(img_value, i['name'], flags=re.I) is not None]
454
455
456
457
458
459
460
461
462
463
464
465
466
467
            elif img_type == "id":
                # Filter images by id
                self.logger.debug(
                    "Trying to find an image with id \"%s\"" % img_value)
                list_imgs = \
                    [i for i in list_images
                     if i['id'].lower() == img_value.lower()]
            else:
                self.logger.error("Unrecognized image type %s" % img_type)
                sys.exit(1)

            # Check if we found one
            if list_imgs:
                self.logger.debug("Will use \"%s\" with id \"%s\""
468
469
                                  % (_green(list_imgs[0]['name']),
                                     _green(list_imgs[0]['id'])))
470
471
472
473
474
                return list_imgs[0]['id']

        # We didn't found one
        self.logger.error("No matching image found.. aborting")
        sys.exit(1)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
475

476
    def _get_server_ip_and_port(self, server, private_networks):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
477
478
        """Compute server's IPv4 and ssh port number"""
        self.logger.info("Get server connection details..")
479
480
481
482
483
484
        if private_networks:
            # Choose the networks that belong to private_networks
            networks = [n for n in server['attachments']
                        if n['network_id'] in private_networks]
        else:
            # Choose the networks that are public
485
486
487
            networks = [n for n in server['attachments']
                        if self.network_client.
                        get_network_details(n['network_id'])['public']]
488
489
490
491
492
        # Choose the networks with IPv4
        networks = [n for n in networks if n['ipv4']]
        # Use the first network as IPv4
        server_ip = networks[0]['ipv4']

493
494
        if (".okeanos.io" in self.cyclades_client.base_url or
           ".demo.synnefo.org" in self.cyclades_client.base_url):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
495
496
497
498
499
500
            tmp1 = int(server_ip.split(".")[2])
            tmp2 = int(server_ip.split(".")[3])
            server_ip = "gate.okeanos.io"
            server_port = 10000 + tmp1 * 256 + tmp2
        else:
            server_port = 22
501
        self.write_temp_config('server_ip', server_ip)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
502
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
503
        self.write_temp_config('server_port', server_port)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
504
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
505
506
507
508
        ssh_command = "ssh -p %s %s@%s" \
            % (server_port, server['metadata']['users'], server_ip)
        self.logger.debug("Access server using \"%s\"" %
                          (_green(ssh_command)))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
509

Christos Stavrakakis's avatar
Christos Stavrakakis committed
510
    @_check_fabric
511
    def _copy_ssh_keys(self, ssh_keys):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
512
        """Upload/Install ssh keys to server"""
513
        self.logger.debug("Check for authentication keys to use")
514
515
516
        if ssh_keys is None:
            ssh_keys = self.config.get("Deployment", "ssh_keys")

517
        if ssh_keys != "":
518
            ssh_keys = os.path.expanduser(ssh_keys)
519
520
            self.logger.debug("Will use \"%s\" authentication keys file" %
                              _green(ssh_keys))
Christos Stavrakakis's avatar
Christos Stavrakakis committed
521
522
            keyfile = '/tmp/%s.pub' % fabric.env.user
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
523
524
525
526
527
            if ssh_keys.startswith("http://") or \
                    ssh_keys.startswith("https://") or \
                    ssh_keys.startswith("ftp://"):
                cmd = """
                apt-get update
528
                apt-get install wget --yes --force-yes
529
530
531
532
533
534
535
                wget {0} -O {1} --no-check-certificate
                """.format(ssh_keys, keyfile)
                _run(cmd, False)
            elif os.path.exists(ssh_keys):
                _put(ssh_keys, keyfile)
            else:
                self.logger.debug("No ssh keys found")
536
                return
Christos Stavrakakis's avatar
Christos Stavrakakis committed
537
538
539
540
541
542
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
            _run('rm %s' % keyfile, False)
            self.logger.debug("Uploaded ssh authorized keys")
        else:
            self.logger.debug("No ssh keys found")

543
544
545
546
547
548
549
550
    def _create_new_build_id(self):
        """Find a uniq build_id to use"""
        with filelocker.lock("%s.lock" % self.temp_config_file,
                             filelocker.LOCK_EX):
            # Read temp_config again to get any new entries
            self.temp_config.read(self.temp_config_file)

            # Find a uniq build_id to use
551
552
553
554
555
556
557
558
            if self.build_id is None:
                ids = self.temp_config.sections()
                if ids:
                    max_id = int(max(self.temp_config.sections(), key=int))
                    self.build_id = max_id + 1
                else:
                    self.build_id = 1
            self.logger.debug("Will use \"%s\" as build id"
559
560
561
                              % _green(self.build_id))

            # Create a new section
562
563
564
565
566
567
568
569
            try:
                self.temp_config.add_section(str(self.build_id))
            except DuplicateSectionError:
                msg = ("Build id \"%s\" already in use. " +
                       "Please use a uniq one or cleanup \"%s\" file.\n") \
                    % (self.build_id, self.temp_config_file)
                self.logger.error(msg)
                sys.exit(1)
570
571
572
573
574
575
576
577
578
            creation_time = \
                time.strftime("%a, %d %b %Y %X", time.localtime())
            self.temp_config.set(str(self.build_id),
                                 "created", str(creation_time))

            # Write changes back to temp config file
            with open(self.temp_config_file, 'wb') as tcf:
                self.temp_config.write(tcf)

579
    def write_temp_config(self, option, value):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
580
        """Write changes back to config file"""
581
582
583
584
585
586
587
588
589
590
        # Acquire the lock to write to temp_config_file
        with filelocker.lock("%s.lock" % self.temp_config_file,
                             filelocker.LOCK_EX):

            # Read temp_config again to get any new entries
            self.temp_config.read(self.temp_config_file)

            self.temp_config.set(str(self.build_id), option, str(value))
            curr_time = time.strftime("%a, %d %b %Y %X", time.localtime())
            self.temp_config.set(str(self.build_id), "modified", curr_time)
591
592

            # Write changes back to temp config file
593
594
            with open(self.temp_config_file, 'wb') as tcf:
                self.temp_config.write(tcf)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
595

596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
    def read_temp_config(self, option):
        """Read from temporary_config file"""
        # If build_id is None use the latest one
        if self.build_id is None:
            ids = self.temp_config.sections()
            if ids:
                self.build_id = int(ids[-1])
            else:
                self.logger.error("No sections in temporary config file")
                sys.exit(1)
            self.logger.debug("Will use \"%s\" as build id"
                              % _green(self.build_id))
        # Read specified option
        return self.temp_config.get(str(self.build_id), option)

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
611
612
613
    def setup_fabric(self):
        """Setup fabric environment"""
        self.logger.info("Setup fabric parameters..")
614
615
616
617
        fabric.env.user = self.read_temp_config('server_user')
        fabric.env.host_string = self.read_temp_config('server_ip')
        fabric.env.port = int(self.read_temp_config('server_port'))
        fabric.env.password = self.read_temp_config('server_passwd')
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
        fabric.env.connection_attempts = 10
        fabric.env.shell = "/bin/bash -c"
        fabric.env.disable_known_hosts = True
        fabric.env.output_prefix = None

    def _check_hash_sum(self, localfile, remotefile):
        """Check hash sums of two files"""
        self.logger.debug("Check hash sum for local file %s" % localfile)
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
        self.logger.debug("Local file has sha256 hash %s" % hash1)
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
        hash2 = _run("sha256sum %s" % remotefile, False)
        hash2 = hash2.split(' ')[0]
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
        if hash1 != hash2:
            self.logger.error("Hashes differ.. aborting")
634
            sys.exit(1)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
635
636

    @_check_fabric
637
    def clone_repo(self, local_repo=False):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
638
639
        """Clone Synnefo repo from slave server"""
        self.logger.info("Configure repositories on remote server..")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
640
        self.logger.debug("Install/Setup git")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
641
        cmd = """
642
        apt-get install git --yes --force-yes
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
643
644
645
        git config --global user.name {0}
        git config --global user.email {1}
        """.format(self.config.get('Global', 'git_config_name'),
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
646
647
648
                   self.config.get('Global', 'git_config_mail'))
        _run(cmd, False)

649
650
651
652
653
654
655
656
        # Clone synnefo_repo
        synnefo_branch = self.clone_synnefo_repo(local_repo=local_repo)
        # Clone pithos-web-client
        self.clone_pithos_webclient_repo(synnefo_branch)

    @_check_fabric
    def clone_synnefo_repo(self, local_repo=False):
        """Clone Synnefo repo to remote server"""
657
        # Find synnefo_repo and synnefo_branch to use
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
658
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
659
660
        synnefo_branch = self.config.get("Global", "synnefo_branch")
        if synnefo_branch == "":
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
661
662
663
            synnefo_branch = \
                subprocess.Popen(
                    ["git", "rev-parse", "--abbrev-ref", "HEAD"],
664
665
666
                    stdout=subprocess.PIPE).communicate()[0].strip()
            if synnefo_branch == "HEAD":
                synnefo_branch = \
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
667
668
                    subprocess.Popen(
                        ["git", "rev-parse", "--short", "HEAD"],
669
                        stdout=subprocess.PIPE).communicate()[0].strip()
670
        self.logger.debug("Will use branch \"%s\"" % _green(synnefo_branch))
671

672
        if local_repo or synnefo_repo == "":
673
674
675
676
677
678
679
680
681
            # Use local_repo
            self.logger.debug("Push local repo to server")
            # Firstly create the remote repo
            _run("git init synnefo", False)
            # Then push our local repo over ssh
            # We have to pass some arguments to ssh command
            # namely to disable host checking.
            (temp_ssh_file_handle, temp_ssh_file) = tempfile.mkstemp()
            os.close(temp_ssh_file_handle)
682
            # XXX: git push doesn't read the password
683
684
685
686
687
688
            cmd = """
            echo 'exec ssh -o "StrictHostKeyChecking no" \
                           -o "UserKnownHostsFile /dev/null" \
                           -q "$@"' > {4}
            chmod u+x {4}
            export GIT_SSH="{4}"
689
            echo "{0}" | git push --quiet --mirror ssh://{1}@{2}:{3}/~/synnefo
690
691
692
693
694
695
696
697
698
            rm -f {4}
            """.format(fabric.env.password,
                       fabric.env.user,
                       fabric.env.host_string,
                       fabric.env.port,
                       temp_ssh_file)
            os.system(cmd)
        else:
            # Clone Synnefo from remote repo
699
700
            self.logger.debug("Clone synnefo from %s" % synnefo_repo)
            self._git_clone(synnefo_repo)
701
702

        # Checkout the desired synnefo_branch
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
703
        self.logger.debug("Checkout \"%s\" branch/commit" % synnefo_branch)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
704
        cmd = """
705
        cd synnefo
706
        for branch in `git branch -a | grep remotes | grep -v HEAD`; do
707
708
709
710
711
712
            git branch --track ${branch##*/} $branch
        done
        git checkout %s
        """ % (synnefo_branch)
        _run(cmd, False)

713
714
        return synnefo_branch

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
715
    @_check_fabric
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
    def clone_pithos_webclient_repo(self, synnefo_branch):
        """Clone Pithos WebClient repo to remote server"""
        # Find pithos_webclient_repo and pithos_webclient_branch to use
        pithos_webclient_repo = \
            self.config.get('Global', 'pithos_webclient_repo')
        pithos_webclient_branch = \
            self.config.get('Global', 'pithos_webclient_branch')

        # Clone pithos-webclient from remote repo
        self.logger.debug("Clone pithos-webclient from %s" %
                          pithos_webclient_repo)
        self._git_clone(pithos_webclient_repo)

        # Track all pithos-webclient branches
        cmd = """
        cd pithos-web-client
        for branch in `git branch -a | grep remotes | grep -v HEAD`; do
            git branch --track ${branch##*/} $branch > /dev/null 2>&1
        done
735
        git --no-pager branch --no-color
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
        """
        webclient_branches = _run(cmd, False)
        webclient_branches = webclient_branches.split()

        # If we have pithos_webclient_branch in config file use this one
        # else try to use the same branch as synnefo_branch
        # else use an appropriate one.
        if pithos_webclient_branch == "":
            if synnefo_branch in webclient_branches:
                pithos_webclient_branch = synnefo_branch
            else:
                # If synnefo_branch starts with one of
                # 'master', 'hotfix'; use the master branch
                if synnefo_branch.startswith('master') or \
                        synnefo_branch.startswith('hotfix'):
                    pithos_webclient_branch = "master"
                # If synnefo_branch starts with one of
                # 'develop', 'feature'; use the develop branch
                elif synnefo_branch.startswith('develop') or \
                        synnefo_branch.startswith('feature'):
                    pithos_webclient_branch = "develop"
                else:
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
758
                    self.logger.warning(
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
                        "Cannot determine which pithos-web-client branch to "
                        "use based on \"%s\" synnefo branch. "
                        "Will use develop." % synnefo_branch)
                    pithos_webclient_branch = "develop"
        # Checkout branch
        self.logger.debug("Checkout \"%s\" branch" %
                          _green(pithos_webclient_branch))
        cmd = """
        cd pithos-web-client
        git checkout {0}
        """.format(pithos_webclient_branch)
        _run(cmd, False)

    def _git_clone(self, repo):
        """Clone repo to remote server

        Currently clonning from code.grnet.gr can fail unexpectedly.
        So retry!!

        """
        cloned = False
        for i in range(1, 11):
            try:
                _run("git clone %s" % repo, False)
                cloned = True
                break
            except BaseException:
                self.logger.warning("Clonning failed.. retrying %s/10" % i)
        if not cloned:
            self.logger.error("Can not clone repo.")
            sys.exit(1)

    @_check_fabric
    def build_packages(self):
        """Build packages needed by Synnefo software"""
        self.logger.info("Install development packages")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
795
796
797
        cmd = """
        apt-get update
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
798
                python-dev python-all python-pip ant --yes --force-yes
799
800
        pip install -U devflow
        """
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
801
802
        _run(cmd, False)

803
        # Patch pydist bug
Christos Stavrakakis's avatar
Christos Stavrakakis committed
804
        if self.config.get('Global', 'patch_pydist') == "True":
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
805
806
807
808
809
810
811
            self.logger.debug("Patch pydist.py module")
            cmd = r"""
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
                /usr/share/python/debpython/pydist.py
            """
            _run(cmd, False)

812
        # Build synnefo packages
813
814
815
816
817
818
819
820
821
        self.build_synnefo()
        # Build pithos-web-client packages
        self.build_pithos_webclient()

    @_check_fabric
    def build_synnefo(self):
        """Build Synnefo packages"""
        self.logger.info("Build Synnefo packages..")

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
822
        cmd = """
823
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
824
        """
825
        with fabric.cd("synnefo"):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
826
827
            _run(cmd, True)

828
        # Install snf-deploy package
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
829
830
831
        self.logger.debug("Install snf-deploy package")
        cmd = """
        dpkg -i snf-deploy*.deb
832
        apt-get -f install --yes --force-yes
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
833
        """
834
        with fabric.cd("synnefo_build-area"):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
835
836
837
            with fabric.settings(warn_only=True):
                _run(cmd, True)

838
        # Setup synnefo packages for snf-deploy
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
839
840
841
842
843
844
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
        cmd = """
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
        """
        _run(cmd, False)

845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
    @_check_fabric
    def build_pithos_webclient(self):
        """Build pithos-web-client packages"""
        self.logger.info("Build pithos-web-client packages..")

        cmd = """
        devflow-autopkg snapshot -b ~/webclient_build-area --no-sign
        """
        with fabric.cd("pithos-web-client"):
            _run(cmd, True)

        # Setup pithos-web-client packages for snf-deploy
        self.logger.debug("Copy webclient debs to snf-deploy packages dir")
        cmd = """
        cp ~/webclient_build-area/*.deb /var/lib/snf-deploy/packages/
        """
        _run(cmd, False)

863
864
    @_check_fabric
    def build_documentation(self):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
865
        """Build Synnefo documentation"""
866
867
868
        self.logger.info("Build Synnefo documentation..")
        _run("pip install -U Sphinx", False)
        with fabric.cd("synnefo"):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
869
870
            _run("devflow-update-version; "
                 "./ci/make_docs.sh synnefo_documentation", False)
871
872

    def fetch_documentation(self, dest=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
873
874
        """Fetch Synnefo documentation"""
        self.logger.info("Fetch Synnefo documentation..")
875
876
877
878
879
880
881
882
883
        if dest is None:
            dest = "synnefo_documentation"
        dest = os.path.abspath(dest)
        if not os.path.exists(dest):
            os.makedirs(dest)
        self.fetch_compressed("synnefo/synnefo_documentation", dest)
        self.logger.info("Downloaded documentation to %s" %
                         _green(dest))

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
884
    @_check_fabric
885
    def deploy_synnefo(self, schema=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
886
887
        """Deploy Synnefo using snf-deploy"""
        self.logger.info("Deploy Synnefo..")
888
889
        if schema is None:
            schema = self.config.get('Global', 'schema')
890
        self.logger.debug("Will use \"%s\" schema" % _green(schema))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
891

892
893
894
895
        schema_dir = os.path.join(self.ci_dir, "schemas/%s" % schema)
        if not (os.path.exists(schema_dir) and os.path.isdir(schema_dir)):
            raise ValueError("Unknown schema: %s" % schema)

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
896
        self.logger.debug("Upload schema files to server")
897
        _put(os.path.join(schema_dir, "*"), "/etc/snf-deploy/")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
898
899
900
901
902
903
904
905
906

        self.logger.debug("Change password in nodes.conf file")
        cmd = """
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
        """.format(fabric.env.password)
        _run(cmd, False)

        self.logger.debug("Run snf-deploy")
        cmd = """
907
        snf-deploy keygen --force
908
        snf-deploy --disable-colors --autoconf all
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
909
910
911
912
913
914
915
916
917
918
919
        """
        _run(cmd, True)

    @_check_fabric
    def unit_test(self):
        """Run Synnefo unit test suite"""
        self.logger.info("Run Synnefo unit test suite")
        component = self.config.get('Unit Tests', 'component')

        self.logger.debug("Install needed packages")
        cmd = """
920
921
        pip install -U mock
        pip install -U factory_boy
922
        pip install -U nose
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
923
924
925
        """
        _run(cmd, False)

926
927
        self.logger.debug("Upload tests.sh file")
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
928
        _put(unit_tests_file, ".")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
929
930
931

        self.logger.debug("Run unit tests")
        cmd = """
932
        bash tests.sh {0}
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
933
934
935
936
937
938
939
940
941
942
943
944
        """.format(component)
        _run(cmd, True)

    @_check_fabric
    def run_burnin(self):
        """Run burnin functional test suite"""
        self.logger.info("Run Burnin functional test suite")
        cmd = """
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
        images_user=$(kamaki image list -l | grep owner | \
                      cut -d':' -f2 | tr -d ' ')
945
        snf-burnin --auth-url=$auth_url --token=$token {0}
946
947
        BurninExitStatus=$?
        exit $BurninExitStatus
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
948
949
950
951
        """.format(self.config.get('Burnin', 'cmd_options'))
        _run(cmd, True)

    @_check_fabric
952
    def fetch_compressed(self, src, dest=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
953
        """Create a tarball and fetch it locally"""
954
955
956
957
        self.logger.debug("Creating tarball of %s" % src)
        basename = os.path.basename(src)
        tar_file = basename + ".tgz"
        cmd = "tar czf %s %s" % (tar_file, src)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
958
        _run(cmd, False)
959
960
        if not os.path.exists(dest):
            os.makedirs(dest)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
961

962
963
        tmp_dir = tempfile.mkdtemp()
        fabric.get(tar_file, tmp_dir)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
964

965
966
967
        dest_file = os.path.join(tmp_dir, tar_file)
        self._check_hash_sum(dest_file, tar_file)
        self.logger.debug("Untar packages file %s" % dest_file)
968
969
        cmd = """
        cd %s
970
971
972
973
        tar xzf %s
        cp -r %s/* %s
        rm -r %s
        """ % (tmp_dir, tar_file, src, dest, tmp_dir)
974
        os.system(cmd)
975
976
977
978
979
        self.logger.info("Downloaded %s to %s" %
                         (src, _green(dest)))

    @_check_fabric
    def fetch_packages(self, dest=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
980
        """Fetch Synnefo packages"""
981
982
        if dest is None:
            dest = self.config.get('Global', 'pkgs_dir')
983
        dest = os.path.abspath(os.path.expanduser(dest))
984
985
986
        if not os.path.exists(dest):
            os.makedirs(dest)
        self.fetch_compressed("synnefo_build-area", dest)
987
        self.fetch_compressed("webclient_build-area", dest)
988
        self.logger.info("Downloaded debian packages to %s" %
989
                         _green(dest))
990

991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
    def x2go_plugin(self, dest=None):
        """Produce an html page which will use the x2goplugin

        Arguments:
          dest  -- The file where to save the page (String)

        """
        output_str = """
        <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
        <html>
        <head>
        <title>X2Go SynnefoCI Service</title>
        </head>
        <body onload="checkPlugin()">
        <div id="x2goplugin">
            <object
                src="location"
                type="application/x2go"
                name="x2goplugin"
                palette="background"
                height="100%"
                hspace="0"
                vspace="0"
                width="100%"
                x2goconfig="
                    session=X2Go-SynnefoCI-Session
                    server={0}
                    user={1}
                    sshport={2}
                    published=true
                    autologin=true
                ">
            </object>
        </div>
        </body>
        </html>
        """.format(self.read_temp_config('server_ip'),
                   self.read_temp_config('server_user'),
                   self.read_temp_config('server_port'))
        if dest is None:
            dest = self.config.get('Global', 'x2go_plugin_file')

        self.logger.info("Writting x2go plugin html file to %s" % dest)
        fid = open(dest, 'w')
        fid.write(output_str)
        fid.close()

1038
1039

def parse_typed_option(option, value):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
1040
    """Parsed typed options (flavors and images)"""
1041
1042
1043
1044
1045
1046
1047
1048
    try:
        [type_, val] = value.strip().split(':')
        if type_ not in ["id", "name"]:
            raise ValueError
        return type_, val
    except ValueError:
        msg = "Invalid %s format. Must be [id|name]:.+" % option
        raise ValueError(msg)
1049
1050
1051
1052
1053
1054
1055


def get_endpoint_url(endpoints, endpoint_type):
    """Get the publicURL for the specified endpoint"""

    service_catalog = parse_endpoints(endpoints, ep_type=endpoint_type)
    return service_catalog[0]['endpoints'][0]['publicURL']