utils.py 28.7 KB
Newer Older
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
1
2
3
4
5
6
7
#!/usr/bin/env python

"""
Synnefo ci utils module
"""

import os
8
import re
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
9
10
11
12
import sys
import time
import logging
import fabric.api as fabric
13
import subprocess
14
import tempfile
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
15
16
from ConfigParser import ConfigParser, DuplicateSectionError

17
from kamaki.cli import config as kamaki_config
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
18
19
20
from kamaki.clients.astakos import AstakosClient
from kamaki.clients.cyclades import CycladesClient
from kamaki.clients.image import ImageClient
21
from kamaki.clients.compute import ComputeClient
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
22

23
DEFAULT_CONFIG_FILE = "new_config"
24
25
26
27
28
# UUID of owner of system images
DEFAULT_SYSTEM_IMAGES_UUID = [
    "25ecced9-bf53-4145-91ee-cf47377e9fb2",  # production (okeanos.grnet.gr)
    "04cbe33f-29b7-4ef1-94fb-015929e5fc06",  # testing (okeanos.io)
    ]
29

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
30
31
32
33
34
35
36

def _run(cmd, verbose):
    """Run fabric with verbose level"""
    if verbose:
        args = ('running',)
    else:
        args = ('running', 'stdout',)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
37
    with fabric.hide(*args):  # Used * or ** magic. pylint: disable-msg=W0142
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
38
39
40
        return fabric.run(cmd)


41
42
43
44
45
46
def _put(local, remote):
    """Run fabric put command without output"""
    with fabric.quiet():
        fabric.put(local, remote)


Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
def _red(msg):
    """Red color"""
    #return "\x1b[31m" + str(msg) + "\x1b[0m"
    return str(msg)


def _yellow(msg):
    """Yellow color"""
    #return "\x1b[33m" + str(msg) + "\x1b[0m"
    return str(msg)


def _green(msg):
    """Green color"""
    #return "\x1b[32m" + str(msg) + "\x1b[0m"
    return str(msg)


def _check_fabric(fun):
    """Check if fabric env has been set"""
67
    def wrapper(self, *args, **kwargs):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
68
69
70
        """wrapper function"""
        if not self.fabric_installed:
            self.setup_fabric()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
71
            self.fabric_installed = True
72
        return fun(self, *args, **kwargs)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
73
74
75
76
77
    return wrapper


def _check_kamaki(fun):
    """Check if kamaki has been initialized"""
78
    def wrapper(self, *args, **kwargs):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
79
80
81
        """wrapper function"""
        if not self.kamaki_installed:
            self.setup_kamaki()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
82
            self.kamaki_installed = True
83
        return fun(self, *args, **kwargs)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
    return wrapper


class _MyFormatter(logging.Formatter):
    """Logging Formatter"""
    def format(self, record):
        format_orig = self._fmt
        if record.levelno == logging.DEBUG:
            self._fmt = "  %(msg)s"
        elif record.levelno == logging.INFO:
            self._fmt = "%(msg)s"
        elif record.levelno == logging.WARNING:
            self._fmt = _yellow("[W] %(msg)s")
        elif record.levelno == logging.ERROR:
            self._fmt = _red("[E] %(msg)s")
        result = logging.Formatter.format(self, record)
        self._fmt = format_orig
        return result


104
105
106
107
108
109
110
111
112
# Too few public methods. pylint: disable-msg=R0903
class _InfoFilter(logging.Filter):
    """Logging Filter that allows DEBUG and INFO messages only"""
    def filter(self, rec):
        """The filter"""
        return rec.levelno in (logging.DEBUG, logging.INFO)


# Too many instance attributes. pylint: disable-msg=R0902
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
113
114
115
class SynnefoCI(object):
    """SynnefoCI python class"""

116
    def __init__(self, config_file=None, build_id=None, cloud=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
117
118
119
120
121
122
123
        """ Initialize SynnefoCI python class

        Setup logger, local_dir, config and kamaki
        """
        # Setup logger
        self.logger = logging.getLogger('synnefo-ci')
        self.logger.setLevel(logging.DEBUG)
124
125
126
127
128
129
130
131
132
133
134

        handler1 = logging.StreamHandler(sys.stdout)
        handler1.setLevel(logging.DEBUG)
        handler1.addFilter(_InfoFilter())
        handler1.setFormatter(_MyFormatter())
        handler2 = logging.StreamHandler(sys.stderr)
        handler2.setLevel(logging.WARNING)
        handler2.setFormatter(_MyFormatter())

        self.logger.addHandler(handler1)
        self.logger.addHandler(handler2)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
135
136
137
138
139
140

        # Get our local dir
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
        self.repo_dir = os.path.dirname(self.ci_dir)

        # Read config file
141
142
143
144
        if config_file is None:
            config_file = DEFAULT_CONFIG_FILE
        if not os.path.isabs(config_file):
            config_file = os.path.join(self.ci_dir, config_file)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
145
146
        self.config = ConfigParser()
        self.config.optionxform = str
147
        self.config.read(config_file)
148
149

        # Read temporary_config file
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
150
        temp_config = self.config.get('Global', 'temporary_config')
151
152
        self.temp_config = ConfigParser()
        self.temp_config.optionxform = str
153
        self.temp_config.read(os.path.expanduser(temp_config))
154
155
        self.build_id = build_id
        self.logger.info("Will use \"%s\" as build id" % _green(self.build_id))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
156

157
158
159
160
161
162
163
164
165
166
        # Set kamaki cloud
        if cloud is not None:
            self.kamaki_cloud = cloud
        elif self.config.has_option("Deployment", "kamaki_cloud"):
            kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
            if kamaki_cloud == "":
                self.kamaki_cloud = None
        else:
            self.kamaki_cloud = None

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
167
168
169
170
        # Initialize variables
        self.fabric_installed = False
        self.kamaki_installed = False
        self.cyclades_client = None
171
        self.compute_client = None
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
172
173
174
175
176
        self.image_client = None

    def setup_kamaki(self):
        """Initialize kamaki

177
        Setup cyclades_client, image_client and compute_client
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
178
        """
179
180
181
182
183
184
185
186

        config = kamaki_config.Config()
        if self.kamaki_cloud is None:
            self.kamaki_cloud = config.get_global("default_cloud")

        self.logger.info("Setup kamaki client, using cloud '%s'.." %
                         self.kamaki_cloud)
        auth_url = config.get_cloud(self.kamaki_cloud, "url")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
187
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
188
        token = config.get_cloud(self.kamaki_cloud, "token")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
        #self.logger.debug("Token is %s" % _green(token))

        astakos_client = AstakosClient(auth_url, token)

        cyclades_url = \
            astakos_client.get_service_endpoints('compute')['publicURL']
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
        self.cyclades_client = CycladesClient(cyclades_url, token)
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2

        image_url = \
            astakos_client.get_service_endpoints('image')['publicURL']
        self.logger.debug("Images API url is %s" % _green(image_url))
        self.image_client = ImageClient(cyclades_url, token)
        self.image_client.CONNECTION_RETRY_LIMIT = 2

205
206
207
208
209
210
        compute_url = \
            astakos_client.get_service_endpoints('compute')['publicURL']
        self.logger.debug("Compute API url is %s" % _green(compute_url))
        self.compute_client = ComputeClient(compute_url, token)
        self.compute_client.CONNECTION_RETRY_LIMIT = 2

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
    def _wait_transition(self, server_id, current_status, new_status):
        """Wait for server to go from current_status to new_status"""
        self.logger.debug("Waiting for server to become %s" % new_status)
        timeout = self.config.getint('Global', 'build_timeout')
        sleep_time = 5
        while True:
            server = self.cyclades_client.get_server_details(server_id)
            if server['status'] == new_status:
                return server
            elif timeout < 0:
                self.logger.error(
                    "Waiting for server to become %s timed out" % new_status)
                self.destroy_server(False)
                sys.exit(-1)
            elif server['status'] == current_status:
                # Sleep for #n secs and continue
                timeout = timeout - sleep_time
                time.sleep(sleep_time)
            else:
                self.logger.error(
                    "Server failed with status %s" % server['status'])
                self.destroy_server(False)
                sys.exit(-1)

    @_check_kamaki
    def destroy_server(self, wait=True):
        """Destroy slave server"""
238
        server_id = int(self.read_temp_config('server_id'))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
239
240
241
242
243
244
        self.logger.info("Destoying server with id %s " % server_id)
        self.cyclades_client.delete_server(server_id)
        if wait:
            self._wait_transition(server_id, "ACTIVE", "DELETED")

    @_check_kamaki
245
    def create_server(self, image=None, flavor=None, ssh_keys=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
246
247
        """Create slave server"""
        self.logger.info("Create a new server..")
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262

        # Find a build_id to use
        if self.build_id is None:
            # If build_id is given use this, else ..
            # Find a uniq build_id to use
            ids = self.temp_config.sections()
            if ids:
                max_id = int(max(self.temp_config.sections(), key=int))
                self.build_id = max_id + 1
            else:
                self.build_id = 1
        self.logger.debug("New build id \"%s\" was created"
                          % _green(self.build_id))

        # Find an image to use
263
        image_id = self._find_image(image)
264
        # Find a flavor to use
265
266
267
        flavor_id = self._find_flavor(flavor)

        # Create Server
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
268
269
        server = self.cyclades_client.create_server(
            self.config.get('Deployment', 'server_name'),
270
271
            flavor_id,
            image_id)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
272
        server_id = server['id']
273
        self.write_temp_config('server_id', server_id)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
274
275
        self.logger.debug("Server got id %s" % _green(server_id))
        server_user = server['metadata']['users']
276
        self.write_temp_config('server_user', server_user)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
277
278
        self.logger.debug("Server's admin user is %s" % _green(server_user))
        server_passwd = server['adminPass']
279
        self.write_temp_config('server_passwd', server_passwd)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
280
281
282

        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
        self._get_server_ip_and_port(server)
283
        self._copy_ssh_keys(ssh_keys)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
284

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
285
        # Setup Firewall
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
286
287
        self.setup_fabric()
        self.logger.info("Setup firewall")
288
        accept_ssh_from = self.config.get('Global', 'accept_ssh_from')
289
290
291
292
293
294
295
296
297
298
299
        if accept_ssh_from != "":
            self.logger.debug("Block ssh except from %s" % accept_ssh_from)
            cmd = """
            local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
                cut -d':' -f2 | cut -d' ' -f1)
            iptables -A INPUT -s localhost -j ACCEPT
            iptables -A INPUT -s $local_ip -j ACCEPT
            iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
            iptables -A INPUT -p tcp --dport 22 -j DROP
            """.format(accept_ssh_from)
            _run(cmd, False)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
300

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
301
302
303
304
305
306
307
308
309
310
311
        # Setup apt, download packages
        self.logger.debug("Setup apt. Install x2goserver and firefox")
        cmd = """
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
        apt-get update
        apt-get install curl --yes
        echo -e "\n\n{0}" >> /etc/apt/sources.list
        # Synnefo repo's key
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
        # X2GO Key
        apt-key adv --recv-keys --keyserver keys.gnupg.net E1F958385BFE2B6E
312
        apt-get install x2go-keyring --yes
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
313
        apt-get update
314
        apt-get install x2goserver x2goserver-xsession iceweasel --yes
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
315
316
317
        """.format(self.config.get('Global', 'apt_repo'))
        _run(cmd, False)

318
319
320
321
322
323
324
325
    def _find_flavor(self, flavor=None):
        """Find a suitable flavor to use

        Search by name (reg expression) or by id
        """
        # Get a list of flavors from config file
        flavors = self.config.get('Deployment', 'flavors').split(",")
        if flavor is not None:
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
326
            # If we have a flavor_name to use, add it to our list
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
            flavors.insert(0, flavor)

        list_flavors = self.compute_client.list_flavors()
        for flv in flavors:
            [flv_type, flv_value] = flv.strip().split(':')
            if flv_type == "name":
                # Filter flavors by name
                self.logger.debug(
                    "Trying to find a flavor with name \"%s\"" % flv_value)
                list_flvs = \
                    [f for f in list_flavors
                     if re.search(flv_value, f['name'], flags=re.I) is not None]
            elif flv_type == "id":
                # Filter flavors by id
                self.logger.debug(
                    "Trying to find a flavor with id \"%s\"" % flv_value)
                list_flvs = \
                    [f for f in list_flavors
                     if f['id'].lower() == flv_value.lower()]
            else:
                self.logger.error("Unrecognized flavor type %s" % flv_type)

            # Check if we found one
            if list_flvs:
                self.logger.debug("Will use \"%s\" with id \"%s\""
                                  % (list_flvs[0]['name'], list_flvs[0]['id']))
                return list_flvs[0]['id']
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
354
355
356

        self.logger.error("No matching flavor found.. aborting")
        sys.exit(1)
357

358
    def _find_image(self, image=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
359
360
        """Find a suitable image to use

361
362
363
        In case of search by name, the image has to belong to one
        of the `DEFAULT_SYSTEM_IMAGES_UUID' users.
        In case of search by id it only has to exist.
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
364
        """
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
        # Get a list of images from config file
        images = self.config.get('Deployment', 'images').split(",")
        if image is not None:
            # If we have an image from command line, add it to our list
            images.insert(0, image)

        list_images = self.image_client.list_public(detail=True)['images']
        for img in images:
            [img_type, img_value] = img.strip().split(':')
            if img_type == "name":
                # Filter images by name
                self.logger.debug(
                    "Trying to find an image with name \"%s\"" % img_value)
                list_imgs = \
                    [i for i in list_images
                     if i['user_id'] in DEFAULT_SYSTEM_IMAGES_UUID and
                        re.search(img_value, i['name'], flags=re.I) is not None]
            elif img_type == "id":
                # Filter images by id
                self.logger.debug(
                    "Trying to find an image with id \"%s\"" % img_value)
                list_imgs = \
                    [i for i in list_images
                     if i['id'].lower() == img_value.lower()]
            else:
                self.logger.error("Unrecognized image type %s" % img_type)
                sys.exit(1)

            # Check if we found one
            if list_imgs:
                self.logger.debug("Will use \"%s\" with id \"%s\""
                                  % (list_imgs[0]['name'], list_imgs[0]['id']))
                return list_imgs[0]['id']

        # We didn't found one
        self.logger.error("No matching image found.. aborting")
        sys.exit(1)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
402
403
404
405
406

    def _get_server_ip_and_port(self, server):
        """Compute server's IPv4 and ssh port number"""
        self.logger.info("Get server connection details..")
        server_ip = server['attachments'][0]['ipv4']
407
        if ".okeanos.io" in self.cyclades_client.base_url:
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
408
409
410
411
412
413
            tmp1 = int(server_ip.split(".")[2])
            tmp2 = int(server_ip.split(".")[3])
            server_ip = "gate.okeanos.io"
            server_port = 10000 + tmp1 * 256 + tmp2
        else:
            server_port = 22
414
        self.write_temp_config('server_ip', server_ip)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
415
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
416
        self.write_temp_config('server_port', server_port)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
417
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
418
        self.logger.debug("Access server using \"ssh -X -p %s %s@%s\"" %
419
                          (server_port, server['metadata']['users'], server_ip))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
420

Christos Stavrakakis's avatar
Christos Stavrakakis committed
421
    @_check_fabric
422
    def _copy_ssh_keys(self, ssh_keys):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
423
        """Upload/Install ssh keys to server"""
424
        self.logger.debug("Check for authentication keys to use")
425
426
427
        if ssh_keys is None:
            ssh_keys = self.config.get("Deployment", "ssh_keys")

428
        if ssh_keys != "":
429
            ssh_keys = os.path.expanduser(ssh_keys)
430
            self.logger.debug("Will use %s authentication keys file" % ssh_keys)
Christos Stavrakakis's avatar
Christos Stavrakakis committed
431
432
            keyfile = '/tmp/%s.pub' % fabric.env.user
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
433
434
435
436
437
438
439
440
441
442
443
444
445
            if ssh_keys.startswith("http://") or \
                    ssh_keys.startswith("https://") or \
                    ssh_keys.startswith("ftp://"):
                cmd = """
                apt-get update
                apt-get install wget --yes
                wget {0} -O {1} --no-check-certificate
                """.format(ssh_keys, keyfile)
                _run(cmd, False)
            elif os.path.exists(ssh_keys):
                _put(ssh_keys, keyfile)
            else:
                self.logger.debug("No ssh keys found")
446
                return
Christos Stavrakakis's avatar
Christos Stavrakakis committed
447
448
449
450
451
452
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
            _run('rm %s' % keyfile, False)
            self.logger.debug("Uploaded ssh authorized keys")
        else:
            self.logger.debug("No ssh keys found")

453
    def write_temp_config(self, option, value):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
454
        """Write changes back to config file"""
455
456
457
458
459
460
461
        # If build_id section doesn't exist create a new one
        try:
            self.temp_config.add_section(str(self.build_id))
            creation_time = time.strftime("%a, %d %b %Y %X", time.localtime())
            self.write_temp_config("created", creation_time)
        except DuplicateSectionError:
            pass
462
463
464
        self.temp_config.set(str(self.build_id), option, str(value))
        curr_time = time.strftime("%a, %d %b %Y %X", time.localtime())
        self.temp_config.set(str(self.build_id), "modified", curr_time)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
465
466
        temp_conf_file = self.config.get('Global', 'temporary_config')
        with open(temp_conf_file, 'wb') as tcf:
467
            self.temp_config.write(tcf)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
468

469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
    def read_temp_config(self, option):
        """Read from temporary_config file"""
        # If build_id is None use the latest one
        if self.build_id is None:
            ids = self.temp_config.sections()
            if ids:
                self.build_id = int(ids[-1])
            else:
                self.logger.error("No sections in temporary config file")
                sys.exit(1)
            self.logger.debug("Will use \"%s\" as build id"
                              % _green(self.build_id))
        # Read specified option
        return self.temp_config.get(str(self.build_id), option)

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
484
485
486
    def setup_fabric(self):
        """Setup fabric environment"""
        self.logger.info("Setup fabric parameters..")
487
488
489
490
        fabric.env.user = self.read_temp_config('server_user')
        fabric.env.host_string = self.read_temp_config('server_ip')
        fabric.env.port = int(self.read_temp_config('server_port'))
        fabric.env.password = self.read_temp_config('server_passwd')
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
        fabric.env.connection_attempts = 10
        fabric.env.shell = "/bin/bash -c"
        fabric.env.disable_known_hosts = True
        fabric.env.output_prefix = None

    def _check_hash_sum(self, localfile, remotefile):
        """Check hash sums of two files"""
        self.logger.debug("Check hash sum for local file %s" % localfile)
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
        self.logger.debug("Local file has sha256 hash %s" % hash1)
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
        hash2 = _run("sha256sum %s" % remotefile, False)
        hash2 = hash2.split(' ')[0]
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
        if hash1 != hash2:
            self.logger.error("Hashes differ.. aborting")
            sys.exit(-1)

    @_check_fabric
510
    def clone_repo(self, local_repo=False):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
511
512
        """Clone Synnefo repo from slave server"""
        self.logger.info("Configure repositories on remote server..")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
513
        self.logger.debug("Install/Setup git")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
514
        cmd = """
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
515
516
517
518
        apt-get install git --yes
        git config --global user.name {0}
        git config --global user.email {1}
        """.format(self.config.get('Global', 'git_config_name'),
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
519
520
521
                   self.config.get('Global', 'git_config_mail'))
        _run(cmd, False)

522
        # Find synnefo_repo and synnefo_branch to use
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
523
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
524
525
        synnefo_branch = self.config.get("Global", "synnefo_branch")
        if synnefo_branch == "":
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
526
527
528
            synnefo_branch = \
                subprocess.Popen(
                    ["git", "rev-parse", "--abbrev-ref", "HEAD"],
529
530
531
                    stdout=subprocess.PIPE).communicate()[0].strip()
            if synnefo_branch == "HEAD":
                synnefo_branch = \
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
532
533
                    subprocess.Popen(
                        ["git", "rev-parse", "--short", "HEAD"],
534
535
                        stdout=subprocess.PIPE).communicate()[0].strip()
        self.logger.info("Will use branch %s" % synnefo_branch)
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578

        if local_repo or synnefo_branch == "":
            # Use local_repo
            self.logger.debug("Push local repo to server")
            # Firstly create the remote repo
            _run("git init synnefo", False)
            # Then push our local repo over ssh
            # We have to pass some arguments to ssh command
            # namely to disable host checking.
            (temp_ssh_file_handle, temp_ssh_file) = tempfile.mkstemp()
            os.close(temp_ssh_file_handle)
            cmd = """
            echo 'exec ssh -o "StrictHostKeyChecking no" \
                           -o "UserKnownHostsFile /dev/null" \
                           -q "$@"' > {4}
            chmod u+x {4}
            export GIT_SSH="{4}"
            echo "{0}" | git push --mirror ssh://{1}@{2}:{3}/~/synnefo
            rm -f {4}
            """.format(fabric.env.password,
                       fabric.env.user,
                       fabric.env.host_string,
                       fabric.env.port,
                       temp_ssh_file)
            os.system(cmd)
        else:
            # Clone Synnefo from remote repo
            # Currently clonning synnefo can fail unexpectedly
            cloned = False
            for i in range(10):
                self.logger.debug("Clone synnefo from %s" % synnefo_repo)
                try:
                    _run("git clone %s synnefo" % synnefo_repo, False)
                    cloned = True
                    break
                except BaseException:
                    self.logger.warning(
                        "Clonning synnefo failed.. retrying %s" % i)
            if not cloned:
                self.logger.error("Can not clone Synnefo repo.")
                sys.exit(-1)

        # Checkout the desired synnefo_branch
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
579
        self.logger.debug("Checkout \"%s\" branch/commit" % synnefo_branch)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
580
        cmd = """
581
        cd synnefo
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
582
583
        for branch in `git branch -a | grep remotes | \
                       grep -v HEAD | grep -v master`; do
584
585
586
587
588
589
            git branch --track ${branch##*/} $branch
        done
        git checkout %s
        """ % (synnefo_branch)
        _run(cmd, False)

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
590
591
592
593
594
595
596
597
598
599
600
601
602
    @_check_fabric
    def build_synnefo(self):
        """Build Synnefo packages"""
        self.logger.info("Build Synnefo packages..")
        self.logger.debug("Install development packages")
        cmd = """
        apt-get update
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
                python-dev python-all python-pip --yes
        pip install devflow
        """
        _run(cmd, False)

Christos Stavrakakis's avatar
Christos Stavrakakis committed
603
        if self.config.get('Global', 'patch_pydist') == "True":
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
604
605
606
607
608
609
610
            self.logger.debug("Patch pydist.py module")
            cmd = r"""
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
                /usr/share/python/debpython/pydist.py
            """
            _run(cmd, False)

611
612
        # Build synnefo packages
        self.logger.debug("Build synnefo packages")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
613
        cmd = """
614
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
615
        """
616
        with fabric.cd("synnefo"):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
617
618
            _run(cmd, True)

619
        # Install snf-deploy package
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
620
621
622
623
624
        self.logger.debug("Install snf-deploy package")
        cmd = """
        dpkg -i snf-deploy*.deb
        apt-get -f install --yes
        """
625
        with fabric.cd("synnefo_build-area"):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
626
627
628
            with fabric.settings(warn_only=True):
                _run(cmd, True)

629
        # Setup synnefo packages for snf-deploy
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
630
631
632
633
634
635
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
        cmd = """
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
        """
        _run(cmd, False)

636
637
    @_check_fabric
    def build_documentation(self):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
638
        """Build Synnefo documentation"""
639
640
641
        self.logger.info("Build Synnefo documentation..")
        _run("pip install -U Sphinx", False)
        with fabric.cd("synnefo"):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
642
643
            _run("devflow-update-version; "
                 "./ci/make_docs.sh synnefo_documentation", False)
644
645

    def fetch_documentation(self, dest=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
646
647
        """Fetch Synnefo documentation"""
        self.logger.info("Fetch Synnefo documentation..")
648
649
650
651
652
653
654
655
656
        if dest is None:
            dest = "synnefo_documentation"
        dest = os.path.abspath(dest)
        if not os.path.exists(dest):
            os.makedirs(dest)
        self.fetch_compressed("synnefo/synnefo_documentation", dest)
        self.logger.info("Downloaded documentation to %s" %
                         _green(dest))

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
657
    @_check_fabric
658
    def deploy_synnefo(self, schema=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
659
660
        """Deploy Synnefo using snf-deploy"""
        self.logger.info("Deploy Synnefo..")
661
662
        if schema is None:
            schema = self.config.get('Global', 'schema')
663
        self.logger.debug("Will use \"%s\" schema" % schema)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
664

665
666
667
668
        schema_dir = os.path.join(self.ci_dir, "schemas/%s" % schema)
        if not (os.path.exists(schema_dir) and os.path.isdir(schema_dir)):
            raise ValueError("Unknown schema: %s" % schema)

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
669
        self.logger.debug("Upload schema files to server")
670
        _put(os.path.join(schema_dir, "*"), "/etc/snf-deploy/")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
671
672
673
674
675
676
677
678
679

        self.logger.debug("Change password in nodes.conf file")
        cmd = """
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
        """.format(fabric.env.password)
        _run(cmd, False)

        self.logger.debug("Run snf-deploy")
        cmd = """
680
        snf-deploy --disable-colors --autoconf all
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
        """
        _run(cmd, True)

    @_check_fabric
    def unit_test(self):
        """Run Synnefo unit test suite"""
        self.logger.info("Run Synnefo unit test suite")
        component = self.config.get('Unit Tests', 'component')

        self.logger.debug("Install needed packages")
        cmd = """
        pip install mock
        pip install factory_boy
        """
        _run(cmd, False)

697
698
        self.logger.debug("Upload tests.sh file")
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
699
        _put(unit_tests_file, ".")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
700
701
702

        self.logger.debug("Run unit tests")
        cmd = """
703
        bash tests.sh {0}
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
        """.format(component)
        _run(cmd, True)

    @_check_fabric
    def run_burnin(self):
        """Run burnin functional test suite"""
        self.logger.info("Run Burnin functional test suite")
        cmd = """
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
        images_user=$(kamaki image list -l | grep owner | \
                      cut -d':' -f2 | tr -d ' ')
        snf-burnin --auth-url=$auth_url --token=$token \
            --force-flavor=2 --image-id=all \
            --system-images-user=$images_user \
            {0}
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
        for i in $(ls $log_folder/*/details*); do
            echo -e "\\n\\n"
            echo -e "***** $i\\n"
            cat $i
        done
        """.format(self.config.get('Burnin', 'cmd_options'))
        _run(cmd, True)

    @_check_fabric
730
    def fetch_compressed(self, src, dest=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
731
        """Create a tarball and fetch it locally"""
732
733
734
735
        self.logger.debug("Creating tarball of %s" % src)
        basename = os.path.basename(src)
        tar_file = basename + ".tgz"
        cmd = "tar czf %s %s" % (tar_file, src)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
736
        _run(cmd, False)
737
738
        if not os.path.exists(dest):
            os.makedirs(dest)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
739

740
741
        tmp_dir = tempfile.mkdtemp()
        fabric.get(tar_file, tmp_dir)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
742

743
744
745
        dest_file = os.path.join(tmp_dir, tar_file)
        self._check_hash_sum(dest_file, tar_file)
        self.logger.debug("Untar packages file %s" % dest_file)
746
747
        cmd = """
        cd %s
748
749
750
751
        tar xzf %s
        cp -r %s/* %s
        rm -r %s
        """ % (tmp_dir, tar_file, src, dest, tmp_dir)
752
        os.system(cmd)
753
754
755
756
757
        self.logger.info("Downloaded %s to %s" %
                         (src, _green(dest)))

    @_check_fabric
    def fetch_packages(self, dest=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
758
        """Fetch Synnefo packages"""
759
760
        if dest is None:
            dest = self.config.get('Global', 'pkgs_dir')
761
        dest = os.path.abspath(os.path.expanduser(dest))
762
763
764
        if not os.path.exists(dest):
            os.makedirs(dest)
        self.fetch_compressed("synnefo_build-area", dest)
765
        self.logger.info("Downloaded debian packages to %s" %
766
                         _green(dest))