utils.py 37.5 KB
Newer Older
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
1
#!/usr/bin/env python
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
2
3
4
5
6
7

"""
Synnefo ci utils module
"""

import os
8
import re
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
9
10
11
12
import sys
import time
import logging
import fabric.api as fabric
13
import subprocess
14
import tempfile
15
from ConfigParser import ConfigParser, DuplicateSectionError
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
16

17
from kamaki.cli import config as kamaki_config
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
18
19
20
from kamaki.clients.astakos import AstakosClient
from kamaki.clients.cyclades import CycladesClient
from kamaki.clients.image import ImageClient
21
from kamaki.clients.compute import ComputeClient
22
import filelocker
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
23

24
DEFAULT_CONFIG_FILE = "ci_wheezy.conf"
25
26
# Is our terminal a colorful one?
USE_COLORS = True
27
28
29
30
# UUID of owner of system images
DEFAULT_SYSTEM_IMAGES_UUID = [
    "25ecced9-bf53-4145-91ee-cf47377e9fb2",  # production (okeanos.grnet.gr)
    "04cbe33f-29b7-4ef1-94fb-015929e5fc06",  # testing (okeanos.io)
31
]
32

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
33
34
35
36
37
38
39

def _run(cmd, verbose):
    """Run fabric with verbose level"""
    if verbose:
        args = ('running',)
    else:
        args = ('running', 'stdout',)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
40
    with fabric.hide(*args):  # Used * or ** magic. pylint: disable-msg=W0142
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
41
42
43
        return fabric.run(cmd)


44
45
46
47
48
49
def _put(local, remote):
    """Run fabric put command without output"""
    with fabric.quiet():
        fabric.put(local, remote)


Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
50
51
def _red(msg):
    """Red color"""
52
53
    ret = "\x1b[31m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
    return ret
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
54
55
56
57


def _yellow(msg):
    """Yellow color"""
58
59
    ret = "\x1b[33m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
    return ret
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
60
61
62
63


def _green(msg):
    """Green color"""
64
65
    ret = "\x1b[32m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
    return ret
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
66
67
68
69


def _check_fabric(fun):
    """Check if fabric env has been set"""
70
    def wrapper(self, *args, **kwargs):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
71
72
73
        """wrapper function"""
        if not self.fabric_installed:
            self.setup_fabric()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
74
            self.fabric_installed = True
75
        return fun(self, *args, **kwargs)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
76
77
78
79
80
    return wrapper


def _check_kamaki(fun):
    """Check if kamaki has been initialized"""
81
    def wrapper(self, *args, **kwargs):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
82
83
84
        """wrapper function"""
        if not self.kamaki_installed:
            self.setup_kamaki()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
85
            self.kamaki_installed = True
86
        return fun(self, *args, **kwargs)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    return wrapper


class _MyFormatter(logging.Formatter):
    """Logging Formatter"""
    def format(self, record):
        format_orig = self._fmt
        if record.levelno == logging.DEBUG:
            self._fmt = "  %(msg)s"
        elif record.levelno == logging.INFO:
            self._fmt = "%(msg)s"
        elif record.levelno == logging.WARNING:
            self._fmt = _yellow("[W] %(msg)s")
        elif record.levelno == logging.ERROR:
            self._fmt = _red("[E] %(msg)s")
        result = logging.Formatter.format(self, record)
        self._fmt = format_orig
        return result


107
108
109
110
111
112
113
114
115
# Too few public methods. pylint: disable-msg=R0903
class _InfoFilter(logging.Filter):
    """Logging Filter that allows DEBUG and INFO messages only"""
    def filter(self, rec):
        """The filter"""
        return rec.levelno in (logging.DEBUG, logging.INFO)


# Too many instance attributes. pylint: disable-msg=R0902
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
116
117
118
class SynnefoCI(object):
    """SynnefoCI python class"""

119
    def __init__(self, config_file=None, build_id=None, cloud=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
120
121
122
123
124
125
126
        """ Initialize SynnefoCI python class

        Setup logger, local_dir, config and kamaki
        """
        # Setup logger
        self.logger = logging.getLogger('synnefo-ci')
        self.logger.setLevel(logging.DEBUG)
127
128
129
130
131
132
133
134
135
136
137

        handler1 = logging.StreamHandler(sys.stdout)
        handler1.setLevel(logging.DEBUG)
        handler1.addFilter(_InfoFilter())
        handler1.setFormatter(_MyFormatter())
        handler2 = logging.StreamHandler(sys.stderr)
        handler2.setLevel(logging.WARNING)
        handler2.setFormatter(_MyFormatter())

        self.logger.addHandler(handler1)
        self.logger.addHandler(handler2)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
138
139
140
141
142
143

        # Get our local dir
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
        self.repo_dir = os.path.dirname(self.ci_dir)

        # Read config file
144
        if config_file is None:
145
146
            config_file = os.path.join(self.ci_dir, DEFAULT_CONFIG_FILE)
        config_file = os.path.abspath(config_file)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
147
148
        self.config = ConfigParser()
        self.config.optionxform = str
149
        self.config.read(config_file)
150
151

        # Read temporary_config file
152
153
        self.temp_config_file = \
            os.path.expanduser(self.config.get('Global', 'temporary_config'))
154
155
        self.temp_config = ConfigParser()
        self.temp_config.optionxform = str
156
        self.temp_config.read(self.temp_config_file)
157
        self.build_id = build_id
158
159
160
        if build_id is not None:
            self.logger.info("Will use \"%s\" as build id" %
                             _green(self.build_id))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
161

162
163
164
165
166
167
168
169
170
171
        # Set kamaki cloud
        if cloud is not None:
            self.kamaki_cloud = cloud
        elif self.config.has_option("Deployment", "kamaki_cloud"):
            kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
            if kamaki_cloud == "":
                self.kamaki_cloud = None
        else:
            self.kamaki_cloud = None

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
172
173
174
175
        # Initialize variables
        self.fabric_installed = False
        self.kamaki_installed = False
        self.cyclades_client = None
176
        self.compute_client = None
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
177
        self.image_client = None
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
178
        self.astakos_client = None
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
179
180
181
182

    def setup_kamaki(self):
        """Initialize kamaki

183
        Setup cyclades_client, image_client and compute_client
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
184
        """
185
186
187

        config = kamaki_config.Config()
        if self.kamaki_cloud is None:
188
189
190
191
192
            try:
                self.kamaki_cloud = config.get("global", "default_cloud")
            except AttributeError:
                # Compatibility with kamaki version <=0.10
                self.kamaki_cloud = config.get_global("default_cloud")
193
194
195
196

        self.logger.info("Setup kamaki client, using cloud '%s'.." %
                         self.kamaki_cloud)
        auth_url = config.get_cloud(self.kamaki_cloud, "url")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
197
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
198
        token = config.get_cloud(self.kamaki_cloud, "token")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
199
200
        #self.logger.debug("Token is %s" % _green(token))

201
        self.astakos_client = AstakosClient(auth_url, token)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
202
203

        cyclades_url = \
204
            self.astakos_client.get_service_endpoints('compute')['publicURL']
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
205
206
207
208
209
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
        self.cyclades_client = CycladesClient(cyclades_url, token)
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2

        image_url = \
210
            self.astakos_client.get_service_endpoints('image')['publicURL']
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
211
212
213
214
        self.logger.debug("Images API url is %s" % _green(image_url))
        self.image_client = ImageClient(cyclades_url, token)
        self.image_client.CONNECTION_RETRY_LIMIT = 2

215
        compute_url = \
216
            self.astakos_client.get_service_endpoints('compute')['publicURL']
217
218
219
220
        self.logger.debug("Compute API url is %s" % _green(compute_url))
        self.compute_client = ComputeClient(compute_url, token)
        self.compute_client.CONNECTION_RETRY_LIMIT = 2

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
221
222
223
224
225
226
227
228
229
230
231
232
233
    def _wait_transition(self, server_id, current_status, new_status):
        """Wait for server to go from current_status to new_status"""
        self.logger.debug("Waiting for server to become %s" % new_status)
        timeout = self.config.getint('Global', 'build_timeout')
        sleep_time = 5
        while True:
            server = self.cyclades_client.get_server_details(server_id)
            if server['status'] == new_status:
                return server
            elif timeout < 0:
                self.logger.error(
                    "Waiting for server to become %s timed out" % new_status)
                self.destroy_server(False)
234
                sys.exit(1)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
235
236
237
238
239
240
241
242
            elif server['status'] == current_status:
                # Sleep for #n secs and continue
                timeout = timeout - sleep_time
                time.sleep(sleep_time)
            else:
                self.logger.error(
                    "Server failed with status %s" % server['status'])
                self.destroy_server(False)
243
                sys.exit(1)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
244
245
246
247

    @_check_kamaki
    def destroy_server(self, wait=True):
        """Destroy slave server"""
248
        server_id = int(self.read_temp_config('server_id'))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
249
250
251
252
253
254
        self.logger.info("Destoying server with id %s " % server_id)
        self.cyclades_client.delete_server(server_id)
        if wait:
            self._wait_transition(server_id, "ACTIVE", "DELETED")

    @_check_kamaki
255
256
    def create_server(self, image=None, flavor=None, ssh_keys=None,
                      server_name=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
257
258
        """Create slave server"""
        self.logger.info("Create a new server..")
259
260

        # Find a build_id to use
261
        self._create_new_build_id()
262
263

        # Find an image to use
264
        image_id = self._find_image(image)
265
        # Find a flavor to use
266
267
268
        flavor_id = self._find_flavor(flavor)

        # Create Server
269
270
271
272
273
        if server_name is None:
            server_name = self.config.get("Deployment", "server_name")
            server_name = "%s(BID: %s)" % (server_name, self.build_id)
        server = self.cyclades_client.create_server(server_name, flavor_id,
                                                    image_id)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
274
        server_id = server['id']
275
        self.write_temp_config('server_id', server_id)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
276
277
        self.logger.debug("Server got id %s" % _green(server_id))
        server_user = server['metadata']['users']
278
        self.write_temp_config('server_user', server_user)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
279
280
        self.logger.debug("Server's admin user is %s" % _green(server_user))
        server_passwd = server['adminPass']
281
        self.write_temp_config('server_passwd', server_passwd)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
282
283
284

        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
        self._get_server_ip_and_port(server)
285
        self._copy_ssh_keys(ssh_keys)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
286

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
287
        # Setup Firewall
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
288
289
        self.setup_fabric()
        self.logger.info("Setup firewall")
290
        accept_ssh_from = self.config.get('Global', 'accept_ssh_from')
291
292
293
294
295
296
297
298
299
300
301
        if accept_ssh_from != "":
            self.logger.debug("Block ssh except from %s" % accept_ssh_from)
            cmd = """
            local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
                cut -d':' -f2 | cut -d' ' -f1)
            iptables -A INPUT -s localhost -j ACCEPT
            iptables -A INPUT -s $local_ip -j ACCEPT
            iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
            iptables -A INPUT -p tcp --dport 22 -j DROP
            """.format(accept_ssh_from)
            _run(cmd, False)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
302

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
303
304
305
306
        # Setup apt, download packages
        self.logger.debug("Setup apt. Install x2goserver and firefox")
        cmd = """
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
307
        echo 'precedence ::ffff:0:0/96  100' >> /etc/gai.conf
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
308
        apt-get update
309
        apt-get install curl --yes --force-yes
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
310
311
312
        echo -e "\n\n{0}" >> /etc/apt/sources.list
        # Synnefo repo's key
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
313

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
314
315
        # X2GO Key
        apt-key adv --recv-keys --keyserver keys.gnupg.net E1F958385BFE2B6E
316
        apt-get install x2go-keyring --yes --force-yes
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
317
        apt-get update
318
319
        apt-get install x2goserver x2goserver-xsession \
                iceweasel --yes --force-yes
320
321
322
323
324
325
326
327
328
329
330
331
332

        # xterm published application
        echo '[Desktop Entry]' > /usr/share/applications/xterm.desktop
        echo 'Name=XTerm' >> /usr/share/applications/xterm.desktop
        echo 'Comment=standard terminal emulator for the X window system' >> \
            /usr/share/applications/xterm.desktop
        echo 'Exec=xterm' >> /usr/share/applications/xterm.desktop
        echo 'Terminal=false' >> /usr/share/applications/xterm.desktop
        echo 'Type=Application' >> /usr/share/applications/xterm.desktop
        echo 'Encoding=UTF-8' >> /usr/share/applications/xterm.desktop
        echo 'Icon=xterm-color_48x48' >> /usr/share/applications/xterm.desktop
        echo 'Categories=System;TerminalEmulator;' >> \
                /usr/share/applications/xterm.desktop
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
333
334
335
        """.format(self.config.get('Global', 'apt_repo'))
        _run(cmd, False)

336
337
338
339
340
341
342
343
    def _find_flavor(self, flavor=None):
        """Find a suitable flavor to use

        Search by name (reg expression) or by id
        """
        # Get a list of flavors from config file
        flavors = self.config.get('Deployment', 'flavors').split(",")
        if flavor is not None:
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
344
            # If we have a flavor_name to use, add it to our list
345
346
347
348
            flavors.insert(0, flavor)

        list_flavors = self.compute_client.list_flavors()
        for flv in flavors:
349
350
            flv_type, flv_value = parse_typed_option(option="flavor",
                                                     value=flv)
351
352
353
354
355
356
            if flv_type == "name":
                # Filter flavors by name
                self.logger.debug(
                    "Trying to find a flavor with name \"%s\"" % flv_value)
                list_flvs = \
                    [f for f in list_flavors
357
358
                     if re.search(flv_value, f['name'], flags=re.I)
                     is not None]
359
360
361
362
363
364
            elif flv_type == "id":
                # Filter flavors by id
                self.logger.debug(
                    "Trying to find a flavor with id \"%s\"" % flv_value)
                list_flvs = \
                    [f for f in list_flavors
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
365
                     if str(f['id']) == flv_value]
366
367
368
369
370
371
            else:
                self.logger.error("Unrecognized flavor type %s" % flv_type)

            # Check if we found one
            if list_flvs:
                self.logger.debug("Will use \"%s\" with id \"%s\""
372
373
                                  % (_green(list_flvs[0]['name']),
                                     _green(list_flvs[0]['id'])))
374
                return list_flvs[0]['id']
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
375
376
377

        self.logger.error("No matching flavor found.. aborting")
        sys.exit(1)
378

379
    def _find_image(self, image=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
380
381
        """Find a suitable image to use

382
383
384
        In case of search by name, the image has to belong to one
        of the `DEFAULT_SYSTEM_IMAGES_UUID' users.
        In case of search by id it only has to exist.
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
385
        """
386
387
388
389
390
391
        # Get a list of images from config file
        images = self.config.get('Deployment', 'images').split(",")
        if image is not None:
            # If we have an image from command line, add it to our list
            images.insert(0, image)

392
393
        auth = self.astakos_client.authenticate()
        user_uuid = auth["access"]["token"]["tenant"]["id"]
394
395
        list_images = self.image_client.list_public(detail=True)['images']
        for img in images:
396
            img_type, img_value = parse_typed_option(option="image", value=img)
397
398
399
400
            if img_type == "name":
                # Filter images by name
                self.logger.debug(
                    "Trying to find an image with name \"%s\"" % img_value)
401
                accepted_uuids = DEFAULT_SYSTEM_IMAGES_UUID + [user_uuid]
402
                list_imgs = \
403
404
405
                    [i for i in list_images if i['user_id'] in accepted_uuids
                     and
                     re.search(img_value, i['name'], flags=re.I) is not None]
406
407
408
409
410
411
412
413
414
415
416
417
418
419
            elif img_type == "id":
                # Filter images by id
                self.logger.debug(
                    "Trying to find an image with id \"%s\"" % img_value)
                list_imgs = \
                    [i for i in list_images
                     if i['id'].lower() == img_value.lower()]
            else:
                self.logger.error("Unrecognized image type %s" % img_type)
                sys.exit(1)

            # Check if we found one
            if list_imgs:
                self.logger.debug("Will use \"%s\" with id \"%s\""
420
421
                                  % (_green(list_imgs[0]['name']),
                                     _green(list_imgs[0]['id'])))
422
423
424
425
426
                return list_imgs[0]['id']

        # We didn't found one
        self.logger.error("No matching image found.. aborting")
        sys.exit(1)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
427
428
429
430
431

    def _get_server_ip_and_port(self, server):
        """Compute server's IPv4 and ssh port number"""
        self.logger.info("Get server connection details..")
        server_ip = server['attachments'][0]['ipv4']
432
433
        if (".okeanos.io" in self.cyclades_client.base_url or
           ".demo.synnefo.org" in self.cyclades_client.base_url):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
434
435
436
437
438
439
            tmp1 = int(server_ip.split(".")[2])
            tmp2 = int(server_ip.split(".")[3])
            server_ip = "gate.okeanos.io"
            server_port = 10000 + tmp1 * 256 + tmp2
        else:
            server_port = 22
440
        self.write_temp_config('server_ip', server_ip)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
441
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
442
        self.write_temp_config('server_port', server_port)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
443
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
444
445
446
447
        ssh_command = "ssh -p %s %s@%s" \
            % (server_port, server['metadata']['users'], server_ip)
        self.logger.debug("Access server using \"%s\"" %
                          (_green(ssh_command)))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
448

Christos Stavrakakis's avatar
Christos Stavrakakis committed
449
    @_check_fabric
450
    def _copy_ssh_keys(self, ssh_keys):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
451
        """Upload/Install ssh keys to server"""
452
        self.logger.debug("Check for authentication keys to use")
453
454
455
        if ssh_keys is None:
            ssh_keys = self.config.get("Deployment", "ssh_keys")

456
        if ssh_keys != "":
457
            ssh_keys = os.path.expanduser(ssh_keys)
458
459
            self.logger.debug("Will use \"%s\" authentication keys file" %
                              _green(ssh_keys))
Christos Stavrakakis's avatar
Christos Stavrakakis committed
460
461
            keyfile = '/tmp/%s.pub' % fabric.env.user
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
462
463
464
465
466
            if ssh_keys.startswith("http://") or \
                    ssh_keys.startswith("https://") or \
                    ssh_keys.startswith("ftp://"):
                cmd = """
                apt-get update
467
                apt-get install wget --yes --force-yes
468
469
470
471
472
473
474
                wget {0} -O {1} --no-check-certificate
                """.format(ssh_keys, keyfile)
                _run(cmd, False)
            elif os.path.exists(ssh_keys):
                _put(ssh_keys, keyfile)
            else:
                self.logger.debug("No ssh keys found")
475
                return
Christos Stavrakakis's avatar
Christos Stavrakakis committed
476
477
478
479
480
481
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
            _run('rm %s' % keyfile, False)
            self.logger.debug("Uploaded ssh authorized keys")
        else:
            self.logger.debug("No ssh keys found")

482
483
484
485
486
487
488
489
    def _create_new_build_id(self):
        """Find a uniq build_id to use"""
        with filelocker.lock("%s.lock" % self.temp_config_file,
                             filelocker.LOCK_EX):
            # Read temp_config again to get any new entries
            self.temp_config.read(self.temp_config_file)

            # Find a uniq build_id to use
490
491
492
493
494
495
496
497
            if self.build_id is None:
                ids = self.temp_config.sections()
                if ids:
                    max_id = int(max(self.temp_config.sections(), key=int))
                    self.build_id = max_id + 1
                else:
                    self.build_id = 1
            self.logger.debug("Will use \"%s\" as build id"
498
499
500
                              % _green(self.build_id))

            # Create a new section
501
502
503
504
505
506
507
508
            try:
                self.temp_config.add_section(str(self.build_id))
            except DuplicateSectionError:
                msg = ("Build id \"%s\" already in use. " +
                       "Please use a uniq one or cleanup \"%s\" file.\n") \
                    % (self.build_id, self.temp_config_file)
                self.logger.error(msg)
                sys.exit(1)
509
510
511
512
513
514
515
516
517
            creation_time = \
                time.strftime("%a, %d %b %Y %X", time.localtime())
            self.temp_config.set(str(self.build_id),
                                 "created", str(creation_time))

            # Write changes back to temp config file
            with open(self.temp_config_file, 'wb') as tcf:
                self.temp_config.write(tcf)

518
    def write_temp_config(self, option, value):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
519
        """Write changes back to config file"""
520
521
522
523
524
525
526
527
528
529
        # Acquire the lock to write to temp_config_file
        with filelocker.lock("%s.lock" % self.temp_config_file,
                             filelocker.LOCK_EX):

            # Read temp_config again to get any new entries
            self.temp_config.read(self.temp_config_file)

            self.temp_config.set(str(self.build_id), option, str(value))
            curr_time = time.strftime("%a, %d %b %Y %X", time.localtime())
            self.temp_config.set(str(self.build_id), "modified", curr_time)
530
531

            # Write changes back to temp config file
532
533
            with open(self.temp_config_file, 'wb') as tcf:
                self.temp_config.write(tcf)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
534

535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
    def read_temp_config(self, option):
        """Read from temporary_config file"""
        # If build_id is None use the latest one
        if self.build_id is None:
            ids = self.temp_config.sections()
            if ids:
                self.build_id = int(ids[-1])
            else:
                self.logger.error("No sections in temporary config file")
                sys.exit(1)
            self.logger.debug("Will use \"%s\" as build id"
                              % _green(self.build_id))
        # Read specified option
        return self.temp_config.get(str(self.build_id), option)

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
550
551
552
    def setup_fabric(self):
        """Setup fabric environment"""
        self.logger.info("Setup fabric parameters..")
553
554
555
556
        fabric.env.user = self.read_temp_config('server_user')
        fabric.env.host_string = self.read_temp_config('server_ip')
        fabric.env.port = int(self.read_temp_config('server_port'))
        fabric.env.password = self.read_temp_config('server_passwd')
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
        fabric.env.connection_attempts = 10
        fabric.env.shell = "/bin/bash -c"
        fabric.env.disable_known_hosts = True
        fabric.env.output_prefix = None

    def _check_hash_sum(self, localfile, remotefile):
        """Check hash sums of two files"""
        self.logger.debug("Check hash sum for local file %s" % localfile)
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
        self.logger.debug("Local file has sha256 hash %s" % hash1)
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
        hash2 = _run("sha256sum %s" % remotefile, False)
        hash2 = hash2.split(' ')[0]
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
        if hash1 != hash2:
            self.logger.error("Hashes differ.. aborting")
573
            sys.exit(1)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
574
575

    @_check_fabric
576
    def clone_repo(self, local_repo=False):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
577
578
        """Clone Synnefo repo from slave server"""
        self.logger.info("Configure repositories on remote server..")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
579
        self.logger.debug("Install/Setup git")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
580
        cmd = """
581
        apt-get install git --yes --force-yes
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
582
583
584
        git config --global user.name {0}
        git config --global user.email {1}
        """.format(self.config.get('Global', 'git_config_name'),
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
585
586
587
                   self.config.get('Global', 'git_config_mail'))
        _run(cmd, False)

588
589
590
591
592
593
594
595
        # Clone synnefo_repo
        synnefo_branch = self.clone_synnefo_repo(local_repo=local_repo)
        # Clone pithos-web-client
        self.clone_pithos_webclient_repo(synnefo_branch)

    @_check_fabric
    def clone_synnefo_repo(self, local_repo=False):
        """Clone Synnefo repo to remote server"""
596
        # Find synnefo_repo and synnefo_branch to use
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
597
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
598
599
        synnefo_branch = self.config.get("Global", "synnefo_branch")
        if synnefo_branch == "":
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
600
601
602
            synnefo_branch = \
                subprocess.Popen(
                    ["git", "rev-parse", "--abbrev-ref", "HEAD"],
603
604
605
                    stdout=subprocess.PIPE).communicate()[0].strip()
            if synnefo_branch == "HEAD":
                synnefo_branch = \
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
606
607
                    subprocess.Popen(
                        ["git", "rev-parse", "--short", "HEAD"],
608
                        stdout=subprocess.PIPE).communicate()[0].strip()
609
        self.logger.debug("Will use branch \"%s\"" % _green(synnefo_branch))
610

611
        if local_repo or synnefo_repo == "":
612
613
614
615
616
617
618
619
620
            # Use local_repo
            self.logger.debug("Push local repo to server")
            # Firstly create the remote repo
            _run("git init synnefo", False)
            # Then push our local repo over ssh
            # We have to pass some arguments to ssh command
            # namely to disable host checking.
            (temp_ssh_file_handle, temp_ssh_file) = tempfile.mkstemp()
            os.close(temp_ssh_file_handle)
621
            # XXX: git push doesn't read the password
622
623
624
625
626
627
            cmd = """
            echo 'exec ssh -o "StrictHostKeyChecking no" \
                           -o "UserKnownHostsFile /dev/null" \
                           -q "$@"' > {4}
            chmod u+x {4}
            export GIT_SSH="{4}"
628
            echo "{0}" | git push --quiet --mirror ssh://{1}@{2}:{3}/~/synnefo
629
630
631
632
633
634
635
636
637
            rm -f {4}
            """.format(fabric.env.password,
                       fabric.env.user,
                       fabric.env.host_string,
                       fabric.env.port,
                       temp_ssh_file)
            os.system(cmd)
        else:
            # Clone Synnefo from remote repo
638
639
            self.logger.debug("Clone synnefo from %s" % synnefo_repo)
            self._git_clone(synnefo_repo)
640
641

        # Checkout the desired synnefo_branch
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
642
        self.logger.debug("Checkout \"%s\" branch/commit" % synnefo_branch)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
643
        cmd = """
644
        cd synnefo
645
        for branch in `git branch -a | grep remotes | grep -v HEAD`; do
646
647
648
649
650
651
            git branch --track ${branch##*/} $branch
        done
        git checkout %s
        """ % (synnefo_branch)
        _run(cmd, False)

652
653
        return synnefo_branch

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
654
    @_check_fabric
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
    def clone_pithos_webclient_repo(self, synnefo_branch):
        """Clone Pithos WebClient repo to remote server"""
        # Find pithos_webclient_repo and pithos_webclient_branch to use
        pithos_webclient_repo = \
            self.config.get('Global', 'pithos_webclient_repo')
        pithos_webclient_branch = \
            self.config.get('Global', 'pithos_webclient_branch')

        # Clone pithos-webclient from remote repo
        self.logger.debug("Clone pithos-webclient from %s" %
                          pithos_webclient_repo)
        self._git_clone(pithos_webclient_repo)

        # Track all pithos-webclient branches
        cmd = """
        cd pithos-web-client
        for branch in `git branch -a | grep remotes | grep -v HEAD`; do
            git branch --track ${branch##*/} $branch > /dev/null 2>&1
        done
        git branch
        """
        webclient_branches = _run(cmd, False)
        webclient_branches = webclient_branches.split()

        # If we have pithos_webclient_branch in config file use this one
        # else try to use the same branch as synnefo_branch
        # else use an appropriate one.
        if pithos_webclient_branch == "":
            if synnefo_branch in webclient_branches:
                pithos_webclient_branch = synnefo_branch
            else:
                # If synnefo_branch starts with one of
                # 'master', 'hotfix'; use the master branch
                if synnefo_branch.startswith('master') or \
                        synnefo_branch.startswith('hotfix'):
                    pithos_webclient_branch = "master"
                # If synnefo_branch starts with one of
                # 'develop', 'feature'; use the develop branch
                elif synnefo_branch.startswith('develop') or \
                        synnefo_branch.startswith('feature'):
                    pithos_webclient_branch = "develop"
                else:
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
697
                    self.logger.warning(
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
                        "Cannot determine which pithos-web-client branch to "
                        "use based on \"%s\" synnefo branch. "
                        "Will use develop." % synnefo_branch)
                    pithos_webclient_branch = "develop"
        # Checkout branch
        self.logger.debug("Checkout \"%s\" branch" %
                          _green(pithos_webclient_branch))
        cmd = """
        cd pithos-web-client
        git checkout {0}
        """.format(pithos_webclient_branch)
        _run(cmd, False)

    def _git_clone(self, repo):
        """Clone repo to remote server

        Currently clonning from code.grnet.gr can fail unexpectedly.
        So retry!!

        """
        cloned = False
        for i in range(1, 11):
            try:
                _run("git clone %s" % repo, False)
                cloned = True
                break
            except BaseException:
                self.logger.warning("Clonning failed.. retrying %s/10" % i)
        if not cloned:
            self.logger.error("Can not clone repo.")
            sys.exit(1)

    @_check_fabric
    def build_packages(self):
        """Build packages needed by Synnefo software"""
        self.logger.info("Install development packages")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
734
735
736
        cmd = """
        apt-get update
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
737
                python-dev python-all python-pip ant --yes --force-yes
738
        pip install -U devflow
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
739
740
741
        """
        _run(cmd, False)

742
        # Patch pydist bug
Christos Stavrakakis's avatar
Christos Stavrakakis committed
743
        if self.config.get('Global', 'patch_pydist') == "True":
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
744
745
746
747
748
749
750
            self.logger.debug("Patch pydist.py module")
            cmd = r"""
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
                /usr/share/python/debpython/pydist.py
            """
            _run(cmd, False)

751
        # Build synnefo packages
752
753
754
755
756
757
758
759
760
        self.build_synnefo()
        # Build pithos-web-client packages
        self.build_pithos_webclient()

    @_check_fabric
    def build_synnefo(self):
        """Build Synnefo packages"""
        self.logger.info("Build Synnefo packages..")

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
761
        cmd = """
762
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
763
        """
764
        with fabric.cd("synnefo"):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
765
766
            _run(cmd, True)

767
        # Install snf-deploy package
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
768
769
770
        self.logger.debug("Install snf-deploy package")
        cmd = """
        dpkg -i snf-deploy*.deb
771
        apt-get -f install --yes --force-yes
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
772
        """
773
        with fabric.cd("synnefo_build-area"):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
774
775
776
            with fabric.settings(warn_only=True):
                _run(cmd, True)

777
        # Setup synnefo packages for snf-deploy
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
778
779
780
781
782
783
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
        cmd = """
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
        """
        _run(cmd, False)

784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
    @_check_fabric
    def build_pithos_webclient(self):
        """Build pithos-web-client packages"""
        self.logger.info("Build pithos-web-client packages..")

        cmd = """
        devflow-autopkg snapshot -b ~/webclient_build-area --no-sign
        """
        with fabric.cd("pithos-web-client"):
            _run(cmd, True)

        # Setup pithos-web-client packages for snf-deploy
        self.logger.debug("Copy webclient debs to snf-deploy packages dir")
        cmd = """
        cp ~/webclient_build-area/*.deb /var/lib/snf-deploy/packages/
        """
        _run(cmd, False)

802
803
    @_check_fabric
    def build_documentation(self):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
804
        """Build Synnefo documentation"""
805
806
807
        self.logger.info("Build Synnefo documentation..")
        _run("pip install -U Sphinx", False)
        with fabric.cd("synnefo"):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
808
809
            _run("devflow-update-version; "
                 "./ci/make_docs.sh synnefo_documentation", False)
810
811

    def fetch_documentation(self, dest=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
812
813
        """Fetch Synnefo documentation"""
        self.logger.info("Fetch Synnefo documentation..")
814
815
816
817
818
819
820
821
822
        if dest is None:
            dest = "synnefo_documentation"
        dest = os.path.abspath(dest)
        if not os.path.exists(dest):
            os.makedirs(dest)
        self.fetch_compressed("synnefo/synnefo_documentation", dest)
        self.logger.info("Downloaded documentation to %s" %
                         _green(dest))

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
823
    @_check_fabric
824
    def deploy_synnefo(self, schema=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
825
826
        """Deploy Synnefo using snf-deploy"""
        self.logger.info("Deploy Synnefo..")
827
828
        if schema is None:
            schema = self.config.get('Global', 'schema')
829
        self.logger.debug("Will use \"%s\" schema" % _green(schema))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
830

831
832
833
834
        schema_dir = os.path.join(self.ci_dir, "schemas/%s" % schema)
        if not (os.path.exists(schema_dir) and os.path.isdir(schema_dir)):
            raise ValueError("Unknown schema: %s" % schema)

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
835
        self.logger.debug("Upload schema files to server")
836
        _put(os.path.join(schema_dir, "*"), "/etc/snf-deploy/")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
837
838
839
840
841
842
843
844
845

        self.logger.debug("Change password in nodes.conf file")
        cmd = """
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
        """.format(fabric.env.password)
        _run(cmd, False)

        self.logger.debug("Run snf-deploy")
        cmd = """
846
        snf-deploy keygen --force
847
        snf-deploy --disable-colors --autoconf all
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
848
849
850
851
852
853
854
855
856
857
858
        """
        _run(cmd, True)

    @_check_fabric
    def unit_test(self):
        """Run Synnefo unit test suite"""
        self.logger.info("Run Synnefo unit test suite")
        component = self.config.get('Unit Tests', 'component')

        self.logger.debug("Install needed packages")
        cmd = """
859
860
        pip install -U mock
        pip install -U factory_boy
861
        pip install -U nose
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
862
863
864
        """
        _run(cmd, False)

865
866
        self.logger.debug("Upload tests.sh file")
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
867
        _put(unit_tests_file, ".")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
868
869
870

        self.logger.debug("Run unit tests")
        cmd = """
871
        bash tests.sh {0}
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
        """.format(component)
        _run(cmd, True)

    @_check_fabric
    def run_burnin(self):
        """Run burnin functional test suite"""
        self.logger.info("Run Burnin functional test suite")
        cmd = """
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
        images_user=$(kamaki image list -l | grep owner | \
                      cut -d':' -f2 | tr -d ' ')
        snf-burnin --auth-url=$auth_url --token=$token \
            --force-flavor=2 --image-id=all \
            --system-images-user=$images_user \
            {0}
888
        BurninExitStatus=$?
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
889
890
891
892
893
894
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
        for i in $(ls $log_folder/*/details*); do
            echo -e "\\n\\n"
            echo -e "***** $i\\n"
            cat $i
        done
895
        exit $BurninExitStatus
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
896
897
898
899
        """.format(self.config.get('Burnin', 'cmd_options'))
        _run(cmd, True)

    @_check_fabric
900
    def fetch_compressed(self, src, dest=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
901
        """Create a tarball and fetch it locally"""
902
903
904
905
        self.logger.debug("Creating tarball of %s" % src)
        basename = os.path.basename(src)
        tar_file = basename + ".tgz"
        cmd = "tar czf %s %s" % (tar_file, src)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
906
        _run(cmd, False)
907
908
        if not os.path.exists(dest):
            os.makedirs(dest)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
909

910
911
        tmp_dir = tempfile.mkdtemp()
        fabric.get(tar_file, tmp_dir)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
912

913
914
915
        dest_file = os.path.join(tmp_dir, tar_file)
        self._check_hash_sum(dest_file, tar_file)
        self.logger.debug("Untar packages file %s" % dest_file)
916
917
        cmd = """
        cd %s
918
919
920
921
        tar xzf %s
        cp -r %s/* %s
        rm -r %s
        """ % (tmp_dir, tar_file, src, dest, tmp_dir)
922
        os.system(cmd)
923
924
925
926
927
        self.logger.info("Downloaded %s to %s" %
                         (src, _green(dest)))

    @_check_fabric
    def fetch_packages(self, dest=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
928
        """Fetch Synnefo packages"""
929
930
        if dest is None:
            dest = self.config.get('Global', 'pkgs_dir')
931
        dest = os.path.abspath(os.path.expanduser(dest))
932
933
934
        if not os.path.exists(dest):
            os.makedirs(dest)
        self.fetch_compressed("synnefo_build-area", dest)
935
        self.fetch_compressed("webclient_build-area", dest)
936
        self.logger.info("Downloaded debian packages to %s" %
937
                         _green(dest))
938

939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
    def x2go_plugin(self, dest=None):
        """Produce an html page which will use the x2goplugin

        Arguments:
          dest  -- The file where to save the page (String)

        """
        output_str = """
        <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
        <html>
        <head>
        <title>X2Go SynnefoCI Service</title>
        </head>
        <body onload="checkPlugin()">
        <div id="x2goplugin">
            <object
                src="location"
                type="application/x2go"
                name="x2goplugin"
                palette="background"
                height="100%"
                hspace="0"
                vspace="0"
                width="100%"
                x2goconfig="
                    session=X2Go-SynnefoCI-Session
                    server={0}
                    user={1}
                    sshport={2}
                    published=true
                    autologin=true
                ">
            </object>
        </div>
        </body>
        </html>
        """.format(self.read_temp_config('server_ip'),
                   self.read_temp_config('server_user'),
                   self.read_temp_config('server_port'))
        if dest is None:
            dest = self.config.get('Global', 'x2go_plugin_file')

        self.logger.info("Writting x2go plugin html file to %s" % dest)
        fid = open(dest, 'w')
        fid.write(output_str)
        fid.close()

986
987

def parse_typed_option(option, value):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
988
    """Parsed typed options (flavors and images)"""
989
990
991
992
993
994
995
996
    try:
        [type_, val] = value.strip().split(':')
        if type_ not in ["id", "name"]:
            raise ValueError
        return type_, val
    except ValueError:
        msg = "Invalid %s format. Must be [id|name]:.+" % option
        raise ValueError(msg)