utils.py 47.9 KB
Newer Older
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
1
#!/usr/bin/env python
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
2
#
Vangelis Koukis's avatar
Vangelis Koukis committed
3
# Copyright (C) 2010-2014 GRNET S.A.
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
4
#
Vangelis Koukis's avatar
Vangelis Koukis committed
5 6 7 8
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
9
#
Vangelis Koukis's avatar
Vangelis Koukis committed
10 11 12 13
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
14
#
Vangelis Koukis's avatar
Vangelis Koukis committed
15 16
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
17 18 19 20 21 22

"""
Synnefo ci utils module
"""

import os
23
import re
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
24 25
import sys
import time
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
26
import httplib
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
27 28
import logging
import fabric.api as fabric
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
29
import simplejson as json
30
import subprocess
31
import tempfile
32
from ConfigParser import ConfigParser, DuplicateSectionError
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
33

34
from kamaki.cli import config as kamaki_config
35
from kamaki.clients.astakos import AstakosClient, parse_endpoints
36
from kamaki.clients.cyclades import CycladesClient, CycladesNetworkClient
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
37
from kamaki.clients.image import ImageClient
38
from kamaki.clients.compute import ComputeClient
39
from kamaki.clients.utils import https
40
from kamaki.clients import ClientError
41
import filelocker
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
42

43
DEFAULT_CONFIG_FILE = "ci_wheezy.conf"
44 45
# Is our terminal a colorful one?
USE_COLORS = True
46 47
# Ignore SSL verification
IGNORE_SSL = False
48 49 50 51
# UUID of owner of system images
DEFAULT_SYSTEM_IMAGES_UUID = [
    "25ecced9-bf53-4145-91ee-cf47377e9fb2",  # production (okeanos.grnet.gr)
    "04cbe33f-29b7-4ef1-94fb-015929e5fc06",  # testing (okeanos.io)
52
]
53

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
54 55 56 57 58 59 60

def _run(cmd, verbose):
    """Run fabric with verbose level"""
    if verbose:
        args = ('running',)
    else:
        args = ('running', 'stdout',)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
61
    with fabric.hide(*args):  # Used * or ** magic. pylint: disable-msg=W0142
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
62 63 64
        return fabric.run(cmd)


65 66 67 68 69 70
def _put(local, remote):
    """Run fabric put command without output"""
    with fabric.quiet():
        fabric.put(local, remote)


Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
71 72
def _red(msg):
    """Red color"""
73 74
    ret = "\x1b[31m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
    return ret
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
75 76 77 78


def _yellow(msg):
    """Yellow color"""
79 80
    ret = "\x1b[33m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
    return ret
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
81 82 83 84


def _green(msg):
    """Green color"""
85 86
    ret = "\x1b[32m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
    return ret
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
87 88 89 90


def _check_fabric(fun):
    """Check if fabric env has been set"""
91
    def wrapper(self, *args, **kwargs):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
92 93 94
        """wrapper function"""
        if not self.fabric_installed:
            self.setup_fabric()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
95
            self.fabric_installed = True
96
        return fun(self, *args, **kwargs)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
97 98 99
    return wrapper


100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
def _kamaki_ssl(ignore_ssl=None):
    """Patch kamaki to use the correct CA certificates

    Read kamaki's config file and decide if we are going to use
    CA certificates and patch kamaki clients accordingly.

    """
    config = kamaki_config.Config()
    if ignore_ssl is None:
        ignore_ssl = config.get("global", "ignore_ssl").lower() == "on"
    ca_file = config.get("global", "ca_certs")

    if ignore_ssl:
        # Skip SSL verification
        https.patch_ignore_ssl()
    else:
        # Use ca_certs path found in kamakirc
        https.patch_with_certs(ca_file)


Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
120 121
def _check_kamaki(fun):
    """Check if kamaki has been initialized"""
122
    def wrapper(self, *args, **kwargs):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
123 124 125
        """wrapper function"""
        if not self.kamaki_installed:
            self.setup_kamaki()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
126
            self.kamaki_installed = True
127
        return fun(self, *args, **kwargs)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
128 129 130 131 132 133 134 135
    return wrapper


class _MyFormatter(logging.Formatter):
    """Logging Formatter"""
    def format(self, record):
        format_orig = self._fmt
        if record.levelno == logging.DEBUG:
136
            self._fmt = "  %(message)s"
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
137
        elif record.levelno == logging.INFO:
138
            self._fmt = "%(message)s"
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
139
        elif record.levelno == logging.WARNING:
140
            self._fmt = _yellow("[W] %(message)s")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
141
        elif record.levelno == logging.ERROR:
142
            self._fmt = _red("[E] %(message)s")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
143 144 145 146 147
        result = logging.Formatter.format(self, record)
        self._fmt = format_orig
        return result


148 149 150 151 152 153 154 155 156
# Too few public methods. pylint: disable-msg=R0903
class _InfoFilter(logging.Filter):
    """Logging Filter that allows DEBUG and INFO messages only"""
    def filter(self, rec):
        """The filter"""
        return rec.levelno in (logging.DEBUG, logging.INFO)


# Too many instance attributes. pylint: disable-msg=R0902
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
157 158 159
class SynnefoCI(object):
    """SynnefoCI python class"""

160
    def __init__(self, config_file=None, build_id=None, cloud=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
161 162 163 164 165 166 167
        """ Initialize SynnefoCI python class

        Setup logger, local_dir, config and kamaki
        """
        # Setup logger
        self.logger = logging.getLogger('synnefo-ci')
        self.logger.setLevel(logging.DEBUG)
168 169 170 171 172 173 174 175 176 177 178

        handler1 = logging.StreamHandler(sys.stdout)
        handler1.setLevel(logging.DEBUG)
        handler1.addFilter(_InfoFilter())
        handler1.setFormatter(_MyFormatter())
        handler2 = logging.StreamHandler(sys.stderr)
        handler2.setLevel(logging.WARNING)
        handler2.setFormatter(_MyFormatter())

        self.logger.addHandler(handler1)
        self.logger.addHandler(handler2)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
179 180 181 182 183 184

        # Get our local dir
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
        self.repo_dir = os.path.dirname(self.ci_dir)

        # Read config file
185
        if config_file is None:
186 187
            config_file = os.path.join(self.ci_dir, DEFAULT_CONFIG_FILE)
        config_file = os.path.abspath(config_file)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
188 189
        self.config = ConfigParser()
        self.config.optionxform = str
190
        self.config.read(config_file)
191 192

        # Read temporary_config file
193 194
        self.temp_config_file = \
            os.path.expanduser(self.config.get('Global', 'temporary_config'))
195 196
        self.temp_config = ConfigParser()
        self.temp_config.optionxform = str
197
        self.temp_config.read(self.temp_config_file)
198
        self.build_id = build_id
199 200 201
        if build_id is not None:
            self.logger.info("Will use \"%s\" as build id" %
                             _green(self.build_id))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
202

203 204 205 206
        # Set kamaki cloud
        if cloud is not None:
            self.kamaki_cloud = cloud
        elif self.config.has_option("Deployment", "kamaki_cloud"):
207 208
            self.kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
            if self.kamaki_cloud == "":
209 210 211 212
                self.kamaki_cloud = None
        else:
            self.kamaki_cloud = None

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
213 214 215 216
        # Initialize variables
        self.fabric_installed = False
        self.kamaki_installed = False
        self.cyclades_client = None
217
        self.network_client = None
218
        self.compute_client = None
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
219
        self.image_client = None
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
220
        self.astakos_client = None
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
221 222 223 224

    def setup_kamaki(self):
        """Initialize kamaki

225
        Setup cyclades_client, image_client and compute_client
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
226
        """
227

228 229 230
        # Patch kamaki for SSL verification
        _kamaki_ssl(ignore_ssl=IGNORE_SSL)

231 232
        config = kamaki_config.Config()
        if self.kamaki_cloud is None:
233 234 235 236
            try:
                self.kamaki_cloud = config.get("global", "default_cloud")
            except AttributeError:
                # Compatibility with kamaki version <=0.10
237
                self.kamaki_cloud = config.get("global", "default_cloud")
238 239 240 241

        self.logger.info("Setup kamaki client, using cloud '%s'.." %
                         self.kamaki_cloud)
        auth_url = config.get_cloud(self.kamaki_cloud, "url")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
242
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
243
        token = config.get_cloud(self.kamaki_cloud, "token")
244
        # self.logger.debug("Token is %s" % _green(token))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
245

246 247
        self.astakos_client = AstakosClient(auth_url, token)
        endpoints = self.astakos_client.authenticate()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
248

249
        cyclades_url = get_endpoint_url(endpoints, "compute")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
250 251 252 253
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
        self.cyclades_client = CycladesClient(cyclades_url, token)
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2

254
        network_url = get_endpoint_url(endpoints, "network")
255 256 257 258
        self.logger.debug("Network API url is %s" % _green(network_url))
        self.network_client = CycladesNetworkClient(network_url, token)
        self.network_client.CONNECTION_RETRY_LIMIT = 2

259
        image_url = get_endpoint_url(endpoints, "image")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
260 261 262 263
        self.logger.debug("Images API url is %s" % _green(image_url))
        self.image_client = ImageClient(cyclades_url, token)
        self.image_client.CONNECTION_RETRY_LIMIT = 2

264
        compute_url = get_endpoint_url(endpoints, "compute")
265 266 267 268
        self.logger.debug("Compute API url is %s" % _green(compute_url))
        self.compute_client = ComputeClient(compute_url, token)
        self.compute_client.CONNECTION_RETRY_LIMIT = 2

269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313
    __quota_cache = None

    def _get_available_project(self, skip_config=False, **resources):
        self.project_uuid = None
        if self.config.has_option("Deployment", "project"):
            self.project_uuid = self.config.get("Deployment",
                                                "project").strip() or None

        # user requested explicit project
        if self.project_uuid and not skip_config:
            return self.project_uuid

        def _filter_projects(_project):
            uuid, project_quota = _project
            can_fit = False
            for resource, required in resources.iteritems():
                # transform dots in order to permit direct keyword
                # arguments to be used.
                # (cyclades__disk=1) -> 'cyclades.disk': 1
                resource = resource.replace("__", ".")
                project_resource = project_quota.get(resource)
                if not project_resource:
                    raise Exception("Requested resource does not exist %s" \
                                    % resource)

                plimit, ppending, pusage, musage, mlimit, mpending = \
                    project_resource.values()

                pavailable = plimit - ppending - pusage
                mavailable = mlimit - mpending - musage

                can_fit = (pavailable - required) >= 0 and \
                            (mavailable - required) >= 0
                if not can_fit:
                    return None
            return uuid

        self.__quota_cache = quota = self.__quota_cache or \
            self.astakos_client.get_quotas()
        projects = filter(bool, map(_filter_projects, quota.iteritems()))
        if not len(projects):
            raise Exception("No project available for %r" % resources)
        return projects[0]


Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
314 315 316 317 318 319 320 321 322 323 324 325 326
    def _wait_transition(self, server_id, current_status, new_status):
        """Wait for server to go from current_status to new_status"""
        self.logger.debug("Waiting for server to become %s" % new_status)
        timeout = self.config.getint('Global', 'build_timeout')
        sleep_time = 5
        while True:
            server = self.cyclades_client.get_server_details(server_id)
            if server['status'] == new_status:
                return server
            elif timeout < 0:
                self.logger.error(
                    "Waiting for server to become %s timed out" % new_status)
                self.destroy_server(False)
327
                sys.exit(1)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
328 329 330 331 332 333 334 335
            elif server['status'] == current_status:
                # Sleep for #n secs and continue
                timeout = timeout - sleep_time
                time.sleep(sleep_time)
            else:
                self.logger.error(
                    "Server failed with status %s" % server['status'])
                self.destroy_server(False)
336
                sys.exit(1)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
337 338 339 340

    @_check_kamaki
    def destroy_server(self, wait=True):
        """Destroy slave server"""
341
        server_id = int(self.read_temp_config('server_id'))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
342 343
        fips = [f for f in self.network_client.list_floatingips()
                if str(f['instance_id']) == str(server_id)]
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
344 345 346 347
        self.logger.info("Destoying server with id %s " % server_id)
        self.cyclades_client.delete_server(server_id)
        if wait:
            self._wait_transition(server_id, "ACTIVE", "DELETED")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
348 349 350 351
        for fip in fips:
            self.logger.info("Destroying floating ip %s",
                             fip['floating_ip_address'])
            self.network_client.delete_floatingip(fip['id'])
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
352

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
353 354 355 356 357 358
    # pylint: disable= no-self-use
    @_check_fabric
    def shell_connect(self):
        """Open shell to remote server"""
        fabric.open_shell("export TERM=xterm")

359 360
    def _create_floating_ip(self):
        """Create a new floating ip"""
361
        project_id = self._get_available_project(cyclades__floating_ip=1)
362
        networks = self.network_client.list_networks(detail=True)
363 364 365 366 367
        pub_nets = [n for n in networks
                    if n['SNF:floating_ip_pool'] and n['public']]
        for pub_net in pub_nets:
            # Try until we find a public network that is not full
            try:
368 369
                fip = self.network_client.create_floatingip(
                    pub_net['id'], project_id=project_id)
370
            except ClientError as err:
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
371
                self.logger.warning("%s", str(err.message).strip())
372 373 374 375
                continue
            self.logger.debug("Floating IP %s with id %s created",
                              fip['floating_ip_address'], fip['id'])
            return fip
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
376
        self.logger.error("No more IP addresses available")
377
        sys.exit(1)
378 379 380 381 382 383 384 385 386 387

    def _create_port(self, floating_ip):
        """Create a new port for our floating IP"""
        net_id = floating_ip['floating_network_id']
        self.logger.debug("Creating a new port to network with id %s", net_id)
        fixed_ips = [{'ip_address': floating_ip['floating_ip_address']}]
        port = self.network_client.create_port(
            net_id, device_id=None, fixed_ips=fixed_ips)
        return port

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
388
    @_check_kamaki
389
    # Too many local variables. pylint: disable-msg=R0914
390 391
    def create_server(self, image=None, flavor=None, ssh_keys=None,
                      server_name=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
392 393
        """Create slave server"""
        self.logger.info("Create a new server..")
394 395

        # Find a build_id to use
396
        self._create_new_build_id()
397 398

        # Find an image to use
399
        image_id = self._find_image(image)
400
        # Find a flavor to use
401 402
        flavor_id = self._find_flavor(flavor)

403 404 405 406 407 408 409 410 411 412
        # get available project
        flavor = self.cyclades_client.get_flavor_details(flavor_id)
        quota = {
            'cyclades.disk': flavor['disk'] * 1024 ** 3,
            'cyclades.ram': flavor['ram'] * 1024 ** 2,
            'cyclades.cpu': flavor['vcpus'],
            'cyclades.vm': 1
        }
        project_id = self._get_available_project(**quota)

413
        # Create Server
414 415 416 417 418 419 420
        networks = []
        if self.config.get("Deployment", "allocate_floating_ip") == "True":
            fip = self._create_floating_ip()
            port = self._create_port(fip)
            networks.append({'port': port['id']})
        private_networks = self.config.get('Deployment', 'private_networks')
        if private_networks:
421 422
            private_networks = [p.strip() for p in private_networks.split(",")]
            networks.extend([{"uuid": uuid} for uuid in private_networks])
423 424 425
        if server_name is None:
            server_name = self.config.get("Deployment", "server_name")
            server_name = "%s(BID: %s)" % (server_name, self.build_id)
426
        server = self.cyclades_client.create_server(
427 428
            server_name, flavor_id, image_id, networks=networks,
            project_id=project_id)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
429
        server_id = server['id']
430
        self.write_temp_config('server_id', server_id)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
431
        self.logger.debug("Server got id %s" % _green(server_id))
432 433 434 435

        # An image may have more than one user. Choose the first one.
        server_user = server['metadata']['users'].split(" ")[0]

436
        self.write_temp_config('server_user', server_user)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
437 438
        self.logger.debug("Server's admin user is %s" % _green(server_user))
        server_passwd = server['adminPass']
439
        self.write_temp_config('server_passwd', server_passwd)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
440 441

        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
442
        self._get_server_ip_and_port(server, private_networks)
443
        self._copy_ssh_keys(ssh_keys)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
444

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
445
        # Setup Firewall
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
446 447
        self.setup_fabric()
        self.logger.info("Setup firewall")
448
        accept_ssh_from = self.config.get('Global', 'accept_ssh_from')
449 450 451 452 453 454 455 456 457 458 459
        if accept_ssh_from != "":
            self.logger.debug("Block ssh except from %s" % accept_ssh_from)
            cmd = """
            local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
                cut -d':' -f2 | cut -d' ' -f1)
            iptables -A INPUT -s localhost -j ACCEPT
            iptables -A INPUT -s $local_ip -j ACCEPT
            iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
            iptables -A INPUT -p tcp --dport 22 -j DROP
            """.format(accept_ssh_from)
            _run(cmd, False)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
460

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
461
        # Setup apt, download packages
462
        self.logger.debug("Setup apt")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
463 464
        cmd = """
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
465
        echo 'precedence ::ffff:0:0/96  100' >> /etc/gai.conf
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
466
        apt-get update
467
        apt-get install -q=2 curl --yes --force-yes
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
468 469 470
        echo -e "\n\n{0}" >> /etc/apt/sources.list
        # Synnefo repo's key
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
471 472
        """.format(self.config.get('Global', 'apt_repo'))
        _run(cmd, False)
473

474
        cmd = """
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
475 476
        # X2GO Key
        apt-key adv --recv-keys --keyserver keys.gnupg.net E1F958385BFE2B6E
477
        apt-get install x2go-keyring --yes --force-yes
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
478
        apt-get update
479 480
        apt-get install x2goserver x2goserver-xsession \
                iceweasel --yes --force-yes
481 482 483 484 485 486 487 488 489 490 491 492

        # xterm published application
        echo '[Desktop Entry]' > /usr/share/applications/xterm.desktop
        echo 'Name=XTerm' >> /usr/share/applications/xterm.desktop
        echo 'Comment=standard terminal emulator for the X window system' >> \
            /usr/share/applications/xterm.desktop
        echo 'Exec=xterm' >> /usr/share/applications/xterm.desktop
        echo 'Terminal=false' >> /usr/share/applications/xterm.desktop
        echo 'Type=Application' >> /usr/share/applications/xterm.desktop
        echo 'Encoding=UTF-8' >> /usr/share/applications/xterm.desktop
        echo 'Icon=xterm-color_48x48' >> /usr/share/applications/xterm.desktop
        echo 'Categories=System;TerminalEmulator;' >> \
493 494 495 496
                /usr/share/applications/xterm.desktop"""
        if self.config.get("Global", "setup_x2go") == "True":
            self.logger.debug("Install x2goserver and firefox")
            _run(cmd, False)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
497

498 499 500 501 502
    def _find_flavor(self, flavor=None):
        """Find a suitable flavor to use

        Search by name (reg expression) or by id
        """
503 504 505 506 507 508 509 510 511 512
        def _is_true(value):
            """Boolean or string value that represents a bool"""
            if isinstance(value, bool):
                return value
            elif isinstance(value, str):
                return value in ["True", "true"]
            else:
                self.logger.error("Unrecognized boolean value %s" % value)
                return False

513 514 515
        # Get a list of flavors from config file
        flavors = self.config.get('Deployment', 'flavors').split(",")
        if flavor is not None:
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
516
            # If we have a flavor_name to use, add it to our list
517 518
            flavors.insert(0, flavor)

519
        list_flavors = self.compute_client.list_flavors(detail=True)
520
        for flv in flavors:
521 522
            flv_type, flv_value = parse_typed_option(option="flavor",
                                                     value=flv)
523 524 525 526 527 528
            if flv_type == "name":
                # Filter flavors by name
                self.logger.debug(
                    "Trying to find a flavor with name \"%s\"" % flv_value)
                list_flvs = \
                    [f for f in list_flavors
529 530
                     if re.search(flv_value, f['name'], flags=re.I)
                     is not None]
531 532 533 534 535 536
            elif flv_type == "id":
                # Filter flavors by id
                self.logger.debug(
                    "Trying to find a flavor with id \"%s\"" % flv_value)
                list_flvs = \
                    [f for f in list_flavors
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
537
                     if str(f['id']) == flv_value]
538 539 540 541
            else:
                self.logger.error("Unrecognized flavor type %s" % flv_type)

            # Check if we found one
542 543
            list_flvs = [f for f in list_flvs
                         if _is_true(f['SNF:allow_create'])]
544 545
            if list_flvs:
                self.logger.debug("Will use \"%s\" with id \"%s\""
546 547
                                  % (_green(list_flvs[0]['name']),
                                     _green(list_flvs[0]['id'])))
548
                return list_flvs[0]['id']
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
549 550 551

        self.logger.error("No matching flavor found.. aborting")
        sys.exit(1)
552

553
    def _find_image(self, image=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
554 555
        """Find a suitable image to use

556 557 558
        In case of search by name, the image has to belong to one
        of the `DEFAULT_SYSTEM_IMAGES_UUID' users.
        In case of search by id it only has to exist.
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
559
        """
560 561 562 563 564 565
        # Get a list of images from config file
        images = self.config.get('Deployment', 'images').split(",")
        if image is not None:
            # If we have an image from command line, add it to our list
            images.insert(0, image)

566 567
        auth = self.astakos_client.authenticate()
        user_uuid = auth["access"]["token"]["tenant"]["id"]
568 569
        list_images = self.image_client.list_public(detail=True)['images']
        for img in images:
570
            img_type, img_value = parse_typed_option(option="image", value=img)
571 572 573 574
            if img_type == "name":
                # Filter images by name
                self.logger.debug(
                    "Trying to find an image with name \"%s\"" % img_value)
575
                accepted_uuids = DEFAULT_SYSTEM_IMAGES_UUID + [user_uuid]
576
                list_imgs = \
577 578 579
                    [i for i in list_images if i['user_id'] in accepted_uuids
                     and
                     re.search(img_value, i['name'], flags=re.I) is not None]
580 581 582 583 584 585 586 587 588 589 590 591 592 593
            elif img_type == "id":
                # Filter images by id
                self.logger.debug(
                    "Trying to find an image with id \"%s\"" % img_value)
                list_imgs = \
                    [i for i in list_images
                     if i['id'].lower() == img_value.lower()]
            else:
                self.logger.error("Unrecognized image type %s" % img_type)
                sys.exit(1)

            # Check if we found one
            if list_imgs:
                self.logger.debug("Will use \"%s\" with id \"%s\""
594 595
                                  % (_green(list_imgs[0]['name']),
                                     _green(list_imgs[0]['id'])))
596 597 598 599 600
                return list_imgs[0]['id']

        # We didn't found one
        self.logger.error("No matching image found.. aborting")
        sys.exit(1)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
601

602
    def _get_server_ip_and_port(self, server, private_networks):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
603 604
        """Compute server's IPv4 and ssh port number"""
        self.logger.info("Get server connection details..")
605 606 607 608 609 610
        if private_networks:
            # Choose the networks that belong to private_networks
            networks = [n for n in server['attachments']
                        if n['network_id'] in private_networks]
        else:
            # Choose the networks that are public
611 612 613
            networks = [n for n in server['attachments']
                        if self.network_client.
                        get_network_details(n['network_id'])['public']]
614 615 616 617 618
        # Choose the networks with IPv4
        networks = [n for n in networks if n['ipv4']]
        # Use the first network as IPv4
        server_ip = networks[0]['ipv4']

Alex Pyrgiotis's avatar
Alex Pyrgiotis committed
619
        # Check if config has ssh_port option and if so, use that port.
620 621 622 623 624 625
        server_port = self.config.get("Deployment", "ssh_port")
        if not server_port:
            # No ssh port given. Get it from API (SNF:port_forwarding)
            if '22' in server['SNF:port_forwarding']:
                server_ip = server['SNF:port_forwarding']['22']['host']
                server_port = int(server['SNF:port_forwarding']['22']['port'])
Alex Pyrgiotis's avatar
Alex Pyrgiotis committed
626 627 628
            else:
                server_port = 22

629
        self.write_temp_config('server_ip', server_ip)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
630
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
631
        self.write_temp_config('server_port', server_port)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
632
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
633 634 635 636
        ssh_command = "ssh -p %s %s@%s" \
            % (server_port, server['metadata']['users'], server_ip)
        self.logger.debug("Access server using \"%s\"" %
                          (_green(ssh_command)))
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
637

Christos Stavrakakis's avatar
Christos Stavrakakis committed
638
    @_check_fabric
639
    def _copy_ssh_keys(self, ssh_keys):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
640
        """Upload/Install ssh keys to server"""
641
        self.logger.debug("Check for authentication keys to use")
642 643 644
        if ssh_keys is None:
            ssh_keys = self.config.get("Deployment", "ssh_keys")

645
        if ssh_keys != "":
646
            ssh_keys = os.path.expanduser(ssh_keys)
647 648
            self.logger.debug("Will use \"%s\" authentication keys file" %
                              _green(ssh_keys))
Christos Stavrakakis's avatar
Christos Stavrakakis committed
649 650
            keyfile = '/tmp/%s.pub' % fabric.env.user
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
651 652 653 654 655
            if ssh_keys.startswith("http://") or \
                    ssh_keys.startswith("https://") or \
                    ssh_keys.startswith("ftp://"):
                cmd = """
                apt-get update
656
                apt-get install wget --yes --force-yes
657 658 659 660 661 662 663
                wget {0} -O {1} --no-check-certificate
                """.format(ssh_keys, keyfile)
                _run(cmd, False)
            elif os.path.exists(ssh_keys):
                _put(ssh_keys, keyfile)
            else:
                self.logger.debug("No ssh keys found")
664
                return
Christos Stavrakakis's avatar
Christos Stavrakakis committed
665 666 667 668 669 670
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
            _run('rm %s' % keyfile, False)
            self.logger.debug("Uploaded ssh authorized keys")
        else:
            self.logger.debug("No ssh keys found")

671 672 673 674 675 676 677 678
    def _create_new_build_id(self):
        """Find a uniq build_id to use"""
        with filelocker.lock("%s.lock" % self.temp_config_file,
                             filelocker.LOCK_EX):
            # Read temp_config again to get any new entries
            self.temp_config.read(self.temp_config_file)

            # Find a uniq build_id to use
679 680 681 682 683 684 685 686
            if self.build_id is None:
                ids = self.temp_config.sections()
                if ids:
                    max_id = int(max(self.temp_config.sections(), key=int))
                    self.build_id = max_id + 1
                else:
                    self.build_id = 1
            self.logger.debug("Will use \"%s\" as build id"
687 688 689
                              % _green(self.build_id))

            # Create a new section
690 691 692 693 694 695 696 697
            try:
                self.temp_config.add_section(str(self.build_id))
            except DuplicateSectionError:
                msg = ("Build id \"%s\" already in use. " +
                       "Please use a uniq one or cleanup \"%s\" file.\n") \
                    % (self.build_id, self.temp_config_file)
                self.logger.error(msg)
                sys.exit(1)
698 699 700 701 702 703 704 705 706
            creation_time = \
                time.strftime("%a, %d %b %Y %X", time.localtime())
            self.temp_config.set(str(self.build_id),
                                 "created", str(creation_time))

            # Write changes back to temp config file
            with open(self.temp_config_file, 'wb') as tcf:
                self.temp_config.write(tcf)

707
    def write_temp_config(self, option, value):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
708
        """Write changes back to config file"""
709 710 711 712 713 714 715 716 717 718
        # Acquire the lock to write to temp_config_file
        with filelocker.lock("%s.lock" % self.temp_config_file,
                             filelocker.LOCK_EX):

            # Read temp_config again to get any new entries
            self.temp_config.read(self.temp_config_file)

            self.temp_config.set(str(self.build_id), option, str(value))
            curr_time = time.strftime("%a, %d %b %Y %X", time.localtime())
            self.temp_config.set(str(self.build_id), "modified", curr_time)
719 720

            # Write changes back to temp config file
721 722
            with open(self.temp_config_file, 'wb') as tcf:
                self.temp_config.write(tcf)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
723

724 725 726 727 728 729 730 731 732 733 734 735 736 737 738
    def read_temp_config(self, option):
        """Read from temporary_config file"""
        # If build_id is None use the latest one
        if self.build_id is None:
            ids = self.temp_config.sections()
            if ids:
                self.build_id = int(ids[-1])
            else:
                self.logger.error("No sections in temporary config file")
                sys.exit(1)
            self.logger.debug("Will use \"%s\" as build id"
                              % _green(self.build_id))
        # Read specified option
        return self.temp_config.get(str(self.build_id), option)

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
739 740 741
    def setup_fabric(self):
        """Setup fabric environment"""
        self.logger.info("Setup fabric parameters..")
742 743 744 745
        fabric.env.user = self.read_temp_config('server_user')
        fabric.env.host_string = self.read_temp_config('server_ip')
        fabric.env.port = int(self.read_temp_config('server_port'))
        fabric.env.password = self.read_temp_config('server_passwd')
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761
        fabric.env.connection_attempts = 10
        fabric.env.shell = "/bin/bash -c"
        fabric.env.disable_known_hosts = True
        fabric.env.output_prefix = None

    def _check_hash_sum(self, localfile, remotefile):
        """Check hash sums of two files"""
        self.logger.debug("Check hash sum for local file %s" % localfile)
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
        self.logger.debug("Local file has sha256 hash %s" % hash1)
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
        hash2 = _run("sha256sum %s" % remotefile, False)
        hash2 = hash2.split(' ')[0]
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
        if hash1 != hash2:
            self.logger.error("Hashes differ.. aborting")
762
            sys.exit(1)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
763 764

    @_check_fabric
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
765 766
    def clone_repo(self, synnefo_repo=None, synnefo_branch=None,
                   local_repo=False, pull_request=None):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
767 768
        """Clone Synnefo repo from slave server"""
        self.logger.info("Configure repositories on remote server..")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
769
        self.logger.debug("Install/Setup git")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
770
        cmd = """
771
        apt-get install git --yes --force-yes
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
772 773 774
        git config --global user.name {0}
        git config --global user.email {1}
        """.format(self.config.get('Global', 'git_config_name'),
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
775 776 777
                   self.config.get('Global', 'git_config_mail'))
        _run(cmd, False)

778
        # Clone synnefo_repo
779 780
        synnefo_branch = self.clone_synnefo_repo(
            synnefo_repo=synnefo_repo, synnefo_branch=synnefo_branch,
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
781
            local_repo=local_repo, pull_request=pull_request)
782
        # Clone pithos-web-client
783 784 785
        if self.config.get("Global", "build_pithos_webclient") == "True":
            # Clone pithos-web-client
            self.clone_pithos_webclient_repo(synnefo_branch)
786 787

    @_check_fabric
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
788 789
    def clone_synnefo_repo(self, synnefo_repo=None, synnefo_branch=None,
                           local_repo=False, pull_request=None):
790
        """Clone Synnefo repo to remote server"""
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806

        assert (pull_request is None or
                (synnefo_branch is None and synnefo_repo is None))

        pull_repo = None
        if pull_request is not None:
            # Get a Github pull request and run the testsuite in
            # a sophisticated way.
            # Sophisticated means that it will not just check the remote branch
            # from which the pull request originated. Instead it will checkout
            # the branch for which the pull request is indented (e.g.
            # grnet:develop) and apply the pull request over it. This way it
            # checks the pull request against the branch this pull request
            # targets.
            m = re.search("github.com/([^/]+)/([^/]+)/pull/(\d+)",
                          pull_request)
807 808 809 810 811
            if m is None:
                self.logger.error("Couldn't find a valid GitHub pull request"
                                  " URL")
                sys.exit(1)

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832
            group = m.group(1)
            repo = m.group(2)
            pull_number = m.group(3)

            # Construct api url
            api_url = "/repos/%s/%s/pulls/%s" % \
                (group, repo, pull_number)
            headers = {'User-Agent': "snf-ci"}
            # Get pull request info
            try:
                conn = httplib.HTTPSConnection("api.github.com")
                conn.request("GET", api_url, headers=headers)
                response = conn.getresponse()
                payload = json.load(response)
                synnefo_repo = payload['base']['repo']['html_url']
                synnefo_branch = payload['base']['ref']
                pull_repo = (payload['head']['repo']['html_url'],
                             payload['head']['ref'])
            finally:
                conn.close()

833
        # Find synnefo_repo and synnefo_branch to use
834 835 836 837
        if synnefo_repo is None:
            synnefo_repo = self.config.get('Global', 'synnefo_repo')
        if synnefo_branch is None:
            synnefo_branch = self.config.get("Global", "synnefo_branch")
838
        if synnefo_branch == "":
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
839 840 841
            synnefo_branch = \
                subprocess.Popen(
                    ["git", "rev-parse", "--abbrev-ref", "HEAD"],
842 843 844
                    stdout=subprocess.PIPE).communicate()[0].strip()
            if synnefo_branch == "HEAD":
                synnefo_branch = \
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
845 846
                    subprocess.Popen(
                        ["git", "rev-parse", "--short", "HEAD"],
847
                        stdout=subprocess.PIPE).communicate()[0].strip()
848
        self.logger.debug("Will use branch \"%s\"" % _green(synnefo_branch))
849

850
        if local_repo or synnefo_repo == "":
851 852 853 854 855 856 857 858 859
            # Use local_repo
            self.logger.debug("Push local repo to server")
            # Firstly create the remote repo
            _run("git init synnefo", False)
            # Then push our local repo over ssh
            # We have to pass some arguments to ssh command
            # namely to disable host checking.
            (temp_ssh_file_handle, temp_ssh_file) = tempfile.mkstemp()
            os.close(temp_ssh_file_handle)
860
            # XXX: git push doesn't read the password
861 862 863 864 865 866
            cmd = """
            echo 'exec ssh -o "StrictHostKeyChecking no" \
                           -o "UserKnownHostsFile /dev/null" \
                           -q "$@"' > {4}
            chmod u+x {4}
            export GIT_SSH="{4}"
867
            echo "{0}" | git push --quiet --mirror ssh://{1}@{2}:{3}/~/synnefo
868 869 870 871 872 873 874 875 876
            rm -f {4}
            """.format(fabric.env.password,
                       fabric.env.user,
                       fabric.env.host_string,
                       fabric.env.port,
                       temp_ssh_file)
            os.system(cmd)
        else:
            # Clone Synnefo from remote repo
877
            self.logger.debug("Clone synnefo from %s" % synnefo_repo)
878
            self._git_clone(synnefo_repo, directory="synnefo")
879 880

        # Checkout the desired synnefo_branch
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
881
        self.logger.debug("Checkout \"%s\" branch/commit" % synnefo_branch)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
882
        cmd = """
883
        cd synnefo
884
        for branch in `git branch -a | grep remotes | grep -v HEAD`; do
885 886 887 888 889
            git branch --track ${branch##*/} $branch
        done
        git checkout %s
        """ % (synnefo_branch)
        _run(cmd, False)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
890 891 892 893 894 895 896 897 898 899

        # Apply a Github pull request
        if pull_repo is not None:
            self.logger.debug("Apply patches from pull request %s",
                              pull_number)
            cmd = """
            cd synnefo
            git pull --no-edit --no-rebase {0} {1}
            """.format(pull_repo[0], pull_repo[1])
            _run(cmd, False)
900

901 902
        return synnefo_branch

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
903
    @_check_fabric
904 905 906 907 908 909 910 911 912 913 914
    def clone_pithos_webclient_repo(self, synnefo_branch):
        """Clone Pithos WebClient repo to remote server"""
        # Find pithos_webclient_repo and pithos_webclient_branch to use
        pithos_webclient_repo = \
            self.config.get('Global', 'pithos_webclient_repo')
        pithos_webclient_branch = \
            self.config.get('Global', 'pithos_webclient_branch')

        # Clone pithos-webclient from remote repo
        self.logger.debug("Clone pithos-webclient from %s" %
                          pithos_webclient_repo)
915
        self._git_clone(pithos_webclient_repo, directory="pithos-web-client")
916 917 918 919 920 921 922

        # Track all pithos-webclient branches
        cmd = """
        cd pithos-web-client
        for branch in `git branch -a | grep remotes | grep -v HEAD`; do
            git branch --track ${branch##*/} $branch > /dev/null 2>&1
        done
923
        git --no-pager branch --no-color
924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945
        """
        webclient_branches = _run(cmd, False)
        webclient_branches = webclient_branches.split()

        # If we have pithos_webclient_branch in config file use this one
        # else try to use the same branch as synnefo_branch
        # else use an appropriate one.
        if pithos_webclient_branch == "":
            if synnefo_branch in webclient_branches:
                pithos_webclient_branch = synnefo_branch
            else:
                # If synnefo_branch starts with one of
                # 'master', 'hotfix'; use the master branch
                if synnefo_branch.startswith('master') or \
                        synnefo_branch.startswith('hotfix'):
                    pithos_webclient_branch = "master"
                # If synnefo_branch starts with one of
                # 'develop', 'feature'; use the develop branch
                elif synnefo_branch.startswith('develop') or \
                        synnefo_branch.startswith('feature'):
                    pithos_webclient_branch = "develop"
                else:
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
946
                    self.logger.warning(
947 948 949 950 951 952 953 954 955 956 957 958 959
                        "Cannot determine which pithos-web-client branch to "
                        "use based on \"%s\" synnefo branch. "
                        "Will use develop." % synnefo_branch)
                    pithos_webclient_branch = "develop"
        # Checkout branch
        self.logger.debug("Checkout \"%s\" branch" %
                          _green(pithos_webclient_branch))
        cmd = """
        cd pithos-web-client
        git checkout {0}
        """.format(pithos_webclient_branch)
        _run(cmd, False)

960
    def _git_clone(self, repo, directory=""):
961 962 963 964 965 966 967 968 969
        """Clone repo to remote server

        Currently clonning from code.grnet.gr can fail unexpectedly.
        So retry!!

        """
        cloned = False
        for i in range(1, 11):
            try:
970
                _run("git clone %s %s" % (repo, directory), False)
971 972 973 974 975 976 977 978 979 980 981 982
                cloned = True
                break
            except BaseException:
                self.logger.warning("Clonning failed.. retrying %s/10" % i)
        if not cloned:
            self.logger.error("Can not clone repo.")
            sys.exit(1)

    @_check_fabric
    def build_packages(self):
        """Build packages needed by Synnefo software"""
        self.logger.info("Install development packages")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
983 984 985
        cmd = """
        apt-get update
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
986
                python-dev python-all python-pip ant --yes --force-yes
987 988
        pip install -U devflow
        """
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
989 990
        _run(cmd, False)

991
        # Patch pydist bug
Christos Stavrakakis's avatar
Christos Stavrakakis committed
992
        if self.config.get('Global', 'patch_pydist') == "True":
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
993 994 995 996 997 998 999
            self.logger.debug("Patch pydist.py module")
            cmd = r"""
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
                /usr/share/python/debpython/pydist.py
            """
            _run(cmd, False)

1000
        # Build synnefo packages
1001 1002
        self.build_synnefo()
        # Build pithos-web-client packages
1003 1004
        if self.config.get("Global", "build_pithos_webclient") == "True":
            self.build_pithos_webclient()
1005 1006 1007