components.py 51.9 KB
Newer Older
Vangelis Koukis's avatar
Vangelis Koukis committed
1
# Copyright (C) 2010-2014 GRNET S.A.
2
#
Vangelis Koukis's avatar
Vangelis Koukis committed
3 4 5 6
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
7
#
Vangelis Koukis's avatar
Vangelis Koukis committed
8 9 10 11
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
12
#
Vangelis Koukis's avatar
Vangelis Koukis committed
13 14
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
15

16
import re
17
import datetime
18
import simplejson
19 20 21 22 23 24
import copy
from snfdeploy import base
from snfdeploy import config
from snfdeploy import constants
from snfdeploy import context
from snfdeploy.lib import FQDN
25 26


27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
#
# Helper decorators that wrap methods of Component
# class and update its execution context (self, self.ctx, context):
#
def parse_user_info(fn):
    """ Parse the output of DB.get_user_info_from_db()

    For the given user email found in config,
    update user_id, user_auth_token and user_uuid attributes of context.

    """
    def wrapper(*args, **kwargs):
        user_email = config.user_email
        result = fn(*args, **kwargs)
        r = re.compile(r"(\d+)[ |]*(\S+)[ |]*(\S+)[ |]*" + user_email, re.M)
        match = r.search(result)
        if config.dry_run:
            context.user_id, context.user_auth_token, context.user_uuid = \
                ("dummy_uid", "dummy_user_auth_token", "dummy_user_uuid")
        elif match:
            context.user_id, context.user_auth_token, context.user_uuid = \
                match.groups()
        else:
            raise BaseException("Cannot parse info for user %s" % user_email)
        return result
    return wrapper


def parse_service_info(fn):
    """ Parse the output of Astakos.get_services()

    For the given service (found in ctx.admin_service)
    updates service_id and service_token attributes of context.

    """
    def wrapper(*args, **kwargs):
        cl = args[0]
        service = cl.ctx.admin_service
        result = fn(*args, **kwargs)
        r = re.compile(r"(\d+)[ ]*%s[ ]*(\S+)" % service, re.M)
        match = r.search(result)
        if config.dry_run:
            context.service_id, context.service_token = \
                ("dummy_service_id", "dummy_service_token")
        elif match:
            context.service_id, context.service_token = \
                match.groups()
        else:
            raise BaseException("Cannot parse info for service %s" % service)
        return result
    return wrapper


def parse_backend_info(fn):
    """ Parse the output of Cyclades.list_backends()

    For the given cluster (found in ctx.admin_cluster)
    updates the backend_id attributes of context.

    """
    def wrapper(*args, **kwargs):
        cl = args[0]
        fqdn = cl.ctx.admin_cluster.fqdn
        result = fn(*args, **kwargs)
        r = re.compile(r"(\d+)[ ]*%s.*" % fqdn, re.M)
        match = r.search(result)
        if config.dry_run:
            context.backend_id = "dummy_backend_id"
        elif match:
            context.backend_id, = match.groups()
        else:
            raise BaseException("Cannot parse info for backend %s" % fqdn)
        return result
    return wrapper


103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
def parse_volume_type_info(fn):
    """ Parse the output of Cyclades.list_volume_types()

    For the given volume_type (found in ctx.admin_volume_type)
    updates the volume_type_id attributes of context.

    """
    def wrapper(*args, **kwargs):
        cl = args[0]
        volume_type = cl.ctx.admin_template
        result = fn(*args, **kwargs)
        r = re.compile(r"(\d+)[ ]*%s.*" % volume_type, re.M)
        match = r.search(result)
        if config.dry_run:
            context.volume_type_id = "dummy_volume_type_id"
        elif match:
            context.volume_type_id, = match.groups()
        else:
            raise BaseException("Cannot parse info for volume type %s" %
                                volume_type)
        return result
    return wrapper


127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
def update_admin(fn):
    """ Initializes the admin roles for each component

    Initialize the admin roles (NS, Astakos, Cyclades, etc.) and make them
    available under self.NS, self.ASTAKOS, etc. These have the same execution
    context of the current components besides the target node which gets
    derived from the corresponding config.

    """
    def wrapper(*args, **kwargs):
        """If used as decorator of a class method first argument is self."""
        cl = args[0]
        ctx = copy.deepcopy(cl.ctx)
        ctx.admin_service = cl.service
        ctx.admin_cluster = cl.cluster
        ctx.admin_node = cl.node
        ctx.admin_fqdn = cl.fqdn
        cl.NS = NS(node=ctx.ns.node, ctx=ctx)
        cl.NFS = NFS(node=ctx.nfs.node, ctx=ctx)
        cl.DB = DB(node=ctx.db.node, ctx=ctx)
        cl.ASTAKOS = Astakos(node=ctx.astakos.node, ctx=ctx)
        cl.CYCLADES = Cyclades(node=ctx.cyclades.node, ctx=ctx)
149
        cl.CLIENT = Client(node=ctx.client.node, ctx=ctx)
150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
        return fn(*args, **kwargs)
    return wrapper


def update_cluster_admin(fn):
    """ Initializes the cluster admin roles for each component

    Finds the master role for the corresponding cluster

    """
    def wrapper(*args, **kwargs):
        """If used as decorator of a class method first argument is self."""
        cl = args[0]
        ctx = copy.deepcopy(cl.ctx)
        ctx.admin_cluster = cl.cluster
        cl.MASTER = Master(node=ctx.master.node, ctx=ctx)
        return fn(*args, **kwargs)
    return wrapper


def export_and_import_service(fn):
    """ Export and import synnefo service

    Used in Astakos, Pithos, and Cyclades service admin_post method

    """
    def wrapper(*args, **kwargs):
        cl = args[0]
        f = config.jsonfile
        cl.export_service()
        cl.get(f, f + ".local")
        cl.ASTAKOS.put(f + ".local", f)
        cl.ASTAKOS.import_service()
        return fn(*args, **kwargs)
    return wrapper


187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
def cert_override(fn):
    """ Create all needed entries for cert_override.txt file

    Append them in a tmp file and upload them to client node

    """
    def wrapper(*args, **kwargs):
        cl = args[0]
        f = "/tmp/" + constants.CERT_OVERRIDE + "_" + cl.service
        for domain in [cl.node.domain, cl.node.cname, cl.node.ip]:
            cmd = """
python /root/firefox_cert_override.py {0} {1}:443 >> {2}
""".format(constants.CERT_PATH, domain, f)
            cl.run(cmd)
        cl.get(f, f + ".local")
        cl.CLIENT.put(f + ".local", f)
        return fn(*args, **kwargs)
    return wrapper


207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
# ########################## Components ############################

# A Component() gets initialized with an execution context that is a
# configuration snapshot of the target setup, cluster and node. A
# component implements the following helper methods: check, install,
# prepare, configure, restart, initialize, and test. All those methods
# will be executed on the target node with this order during setup.
#
# Additionally each Component class implements admin_pre, and
# admin_post methods which invoke actions on different components on
# the same execution context before and after installation. For
# example before a backend gets installed, its FQDN must resolve to
# the master floating IP, so we have to run some actions on the ns
# node and after installation we must add it to cyclades (snf-manage
# backend-add on the cyclades node).
#
# Component() inherits ComponentRunner() which practically exports the
# setup() method. This will first check if the required components are
# installed, will install them if not and update the status of target
# node.
#
# ComponentRunner() inherits FabricRunner() which practically wraps
# basic fabric commands (put, get, run) with the correct execution
# environment.
#
# Each component gets initialized with an execution context and uses
# the config module for accessing global wide options. The context
# provides node, cluster, and setup related info.
235

236 237
class HW(base.Component):
    @base.run_cmds
238 239
    def test(self):
        return [
240
            "ping -c 1 %s" % self.node.ip,
241 242 243 244
            "ping -c 1 www.google.com",
            "apt-get update",
            ]

245

246 247
class SSH(base.Component):
    @base.run_cmds
248 249 250 251 252 253 254
    def prepare(self):
        return [
            "mkdir -p /root/.ssh",
            "for f in $(ls /root/.ssh/*); do cp $f $f.bak ; done",
            "echo StrictHostKeyChecking no >> /etc/ssh/ssh_config",
            ]

255
    def _configure(self):
256 257 258
        files = [
            "authorized_keys", "id_dsa", "id_dsa.pub", "id_rsa", "id_rsa.pub"
            ]
259
        ssh = [("/root/.ssh/%s" % f, {}, {"mode": 0600}) for f in files]
260 261
        return ssh

262
    @base.run_cmds
263 264 265
    def initialize(self):
        f = "/root/.ssh/authorized_keys"
        return [
266
            "test -e {0}.bak && cat {0}.bak >> {0} || true".format(f)
267 268
            ]

269
    @base.run_cmds
270
    def test(self):
271 272
        return ["ssh %s date" % self.node.ip]

273

274 275 276 277
class DNS(base.Component):
    @update_admin
    def admin_pre(self):
        self.NS.update_ns()
278

279
    @base.run_cmds
280 281 282 283
    def prepare(self):
        return [
            "chattr -i /etc/resolv.conf",
            "sed -i 's/^127.*$/127.0.0.1 localhost/g' /etc/hosts",
284 285
            "echo %s > /etc/hostname" % self.node.hostname,
            "hostname %s" % self.node.hostname
286 287
            ]

288
    def _configure(self):
289 290
        r1 = {
            "date": str(datetime.datetime.today()),
291 292
            "domain": self.node.domain,
            "ns_node_ip": self.ctx.ns.ip,
293 294 295 296 297 298
            }
        resolv = [
            ("/etc/resolv.conf", r1, {})
            ]
        return resolv

299
    @base.run_cmds
300 301 302 303
    def initialize(self):
        return ["chattr +i /etc/resolv.conf"]


304
class DDNS(base.Component):
305 306 307 308
    REQUIRED_PACKAGES = [
        "dnsutils",
        ]

309
    @base.run_cmds
310 311 312 313 314
    def prepare(self):
        return [
            "mkdir -p /root/ddns/"
            ]

315
    def _configure(self):
316
        return [
317
            ("/root/ddns/" + k, {}, {}) for k in config.ddns_keys
318 319 320
            ]


321
class NS(base.Component):
322 323 324 325
    REQUIRED_PACKAGES = [
        "bind9",
        ]

326 327 328 329 330 331
    alias = constants.NS

    def required_components(self):
        return [HW, SSH, DDNS]

    def _nsupdate(self, cmd):
332 333 334 335 336 337
        ret = """
nsupdate -k {0} > /dev/null <<EOF || true
server {1}
{2}
send
EOF
338
""".format(config.ddns_private_key, self.ctx.ns.ip, cmd)
339 340
        return ret

341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
    @base.run_cmds
    def update_ns(self, info=None):
        if not info:
            info = self.ctx.admin_fqdn
        return [
            self._nsupdate("update add %s" % info.arecord),
            self._nsupdate("update add %s" % info.ptrrecord),
            self._nsupdate("update add %s" % info.cnamerecord),
            ]

    def add_qa_instances(self):
        instances = [
            ("xen-test-inst1", "1.2.3.4"),
            ("xen-test-inst2", "1.2.3.5"),
            ("xen-test-inst3", "1.2.3.6"),
            ("xen-test-inst4", "1.2.3.7"),
            ]
        for name, ip in instances:
            info = {
                "name": name,
                "ip": ip,
                "domain": self.node.domain
                }
            node_info = FQDN(**info)
            self.update_ns(node_info)

    @base.run_cmds
368 369 370 371 372 373 374 375
    def prepare(self):
        return [
            "mkdir -p /etc/bind/zones",
            "chmod g+w /etc/bind/zones",
            "mkdir -p /etc/bind/rev",
            "chmod g+w /etc/bind/rev",
            ]

376 377 378
    def _configure(self):
        d = self.node.domain
        ip = self.node.ip
379 380 381 382 383 384 385 386 387 388 389
        return [
            ("/etc/bind/named.conf.local", {"domain": d}, {}),
            ("/etc/bind/zones/example.com",
             {"domain": d, "ns_node_ip": ip},
             {"remote": "/etc/bind/zones/%s" % d}),
            ("/etc/bind/zones/vm.example.com",
             {"domain": d, "ns_node_ip": ip},
             {"remote": "/etc/bind/zones/vm.%s" % d}),
            ("/etc/bind/rev/synnefo.in-addr.arpa.zone", {"domain": d}, {}),
            ("/etc/bind/rev/synnefo.ip6.arpa.zone", {"domain": d}, {}),
            ("/etc/bind/named.conf.options",
390
             {"node_ips": ";".join(config.all_ips)}, {}),
391 392 393
            ("/root/ddns/ddns.key", {}, {"remote": "/etc/bind/ddns.key"}),
            ]

394
    @base.run_cmds
395 396 397 398
    def restart(self):
        return ["/etc/init.d/bind9 restart"]


399
class APT(base.Component):
400 401 402
    """ Setup apt repos and check fqdns """
    REQUIRED_PACKAGES = ["curl"]

403
    @base.run_cmds
404 405 406
    def prepare(self):
        return [
            "echo 'APT::Install-Suggests \"false\";' >> /etc/apt/apt.conf",
407 408
            "curl -k https://dev.grnet.gr/files/apt-grnetdev.pub | \
                apt-key add -",
409 410
            ]

411
    def _configure(self):
412 413 414 415
        return [
            ("/etc/apt/sources.list.d/synnefo.wheezy.list", {}, {})
            ]

416
    @base.run_cmds
417 418 419 420 421 422
    def initialize(self):
        return [
            "apt-get update",
            ]


423
class MQ(base.Component):
424 425
    REQUIRED_PACKAGES = ["rabbitmq-server"]

426 427 428 429 430 431 432 433 434 435
    alias = constants.MQ

    def required_components(self):
        return [HW, SSH, DNS, APT]

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()

    @base.run_cmds
436
    def check(self):
437
        return ["ping -c 1 %s" % self.node.cname]
438

439
    @base.run_cmds
440
    def initialize(self):
441 442
        u = config.synnefo_user
        p = config.synnefo_rabbitmq_passwd
443 444 445 446 447 448 449 450
        return [
            "rabbitmqctl add_user %s %s" % (u, p),
            "rabbitmqctl set_permissions %s \".*\" \".*\" \".*\"" % u,
            "rabbitmqctl delete_user guest",
            "rabbitmqctl set_user_tags %s administrator" % u,
            ]


451
class DB(base.Component):
452 453
    REQUIRED_PACKAGES = ["postgresql"]

454 455 456 457 458 459 460 461 462 463
    alias = constants.DB

    def required_components(self):
        return [HW, SSH, DNS, APT]

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()

    @base.run_cmds
464
    def check(self):
465
        return ["ping -c 1 %s" % self.node.cname]
466

467 468
    @parse_user_info
    @base.run_cmds
469 470 471 472 473 474 475 476
    def get_user_info_from_db(self):
        cmd = """
cat > /tmp/psqlcmd <<EOF
select id, auth_token, uuid, email from auth_user, im_astakosuser \
where auth_user.id = im_astakosuser.user_ptr_id and auth_user.email = '{0}';
EOF

su - postgres -c  "psql -w -d snf_apps -f /tmp/psqlcmd"
477
""".format(config.user_email)
478 479 480

        return [cmd]

481 482 483 484 485
    @base.run_cmds
    def allow_db_access(self):
        user = "all"
        method = "md5"
        ip = self.ctx.admin_node.ip
486 487
        f = "/etc/postgresql/*/main/pg_hba.conf"
        cmd1 = "echo host all %s %s/32 %s >> %s" % \
488
            (user, ip, method, f)
489
        cmd2 = "sed -i 's/\(host.*127.0.0.1.*\)md5/\\1trust/' %s" % f
490
        return [cmd1, cmd2]
491

492 493 494
    def _configure(self):
        u = config.synnefo_user
        p = config.synnefo_db_passwd
495 496 497 498 499
        replace = {"synnefo_user": u, "synnefo_db_passwd": p}
        return [
            ("/tmp/db-init.psql", replace, {}),
            ]

500
    @base.check_if_testing
501 502 503 504 505
    def make_db_fast(self):
        f = "/etc/postgresql/*/main/postgresql.conf"
        opts = "fsync=off\nsynchronous_commit=off\nfull_page_writes=off\n"
        return ["""echo -e "%s" >> %s""" % (opts, f)]

506
    @base.run_cmds
507 508
    def prepare(self):
        f = "/etc/postgresql/*/main/postgresql.conf"
509 510
        ret = ["""echo "listen_addresses = '*'" >> %s""" % f]
        return ret + self.make_db_fast()
511

512
    @base.run_cmds
513 514 515 516 517
    def initialize(self):
        script = "/tmp/db-init.psql"
        cmd = "su - postgres -c \"psql -w -f %s\" " % script
        return [cmd]

518
    @base.run_cmds
519 520 521
    def restart(self):
        return ["/etc/init.d/postgresql restart"]

522
    @base.run_cmds
523 524 525 526 527 528 529
    def destroy_db(self):
        return [
            """su - postgres -c ' psql -w -c "drop database snf_apps" '""",
            """su - postgres -c ' psql -w -c "drop database snf_pithos" '"""
            ]


530 531 532 533
class VMC(base.Component):

    def extra_components(self):
        if self.cluster.synnefo:
534 535 536 537
            return [
                Image, GTools, GanetiCollectd,
                PithosBackend, Archip, ArchipGaneti
                ]
538
        else:
539
            return [ExtStorage, Archip, ArchipGaneti]
540

541 542 543 544 545 546 547 548 549 550 551
    def required_components(self):
        return [
            HW, SSH, DNS, DDNS, APT, Mount, Ganeti, Network,
            ] + self.extra_components()

    @update_cluster_admin
    def admin_post(self):
        self.MASTER.add_node(self.node)


class Ganeti(base.Component):
552 553 554 555 556 557 558 559 560 561
    REQUIRED_PACKAGES = [
        "qemu-kvm",
        "python-bitarray",
        "ganeti-htools",
        "ganeti-haskell",
        "snf-ganeti",
        "ganeti2",
        "bridge-utils",
        "lvm2",
        "drbd8-utils",
562
        "ganeti-instance-debootstrap",
563 564
        ]

565 566 567 568 569
    @update_admin
    def admin_pre(self):
        self.NS.update_ns()

    @base.run_cmds
570 571
    def check(self):
        commands = [
572 573
            "getent hosts %s | grep -v ^127" % self.node.hostname,
            "hostname -f | grep %s" % self.node.fqdn,
574 575 576
            ]
        return commands

577
    def _configure(self):
578 579 580 581
        return [
            ("/etc/ganeti/file-storage-paths", {}, {}),
            ]

582
    def _prepare_lvm(self):
583
        ret = []
584
        disk = self.node.extra_disk
585 586 587 588
        if disk:
            ret = [
                "test -e %s" % disk,
                "pvcreate %s" % disk,
589
                "vgcreate %s %s" % (self.cluster.vg, disk)
590 591
                ]
        return ret
592

593 594
    def _prepare_net_infra(self):
        br = config.common_bridge
595 596 597 598
        return [
            "brctl addbr {0}; ip link set {0} up".format(br)
            ]

599
    @base.run_cmds
600 601 602 603
    def prepare(self):
        return [
            "mkdir -p /srv/ganeti/file-storage/",
            "sed -i 's/^127.*$/127.0.0.1 localhost/g' /etc/hosts"
604
            ] + self._prepare_net_infra() + self._prepare_lvm()
605

606
    @base.run_cmds
607
    def restart(self):
608
        return ["/etc/init.d/ganeti restart"]
609 610


611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633
class Master(base.Component):

    @property
    def fqdn(self):
        return self.cluster

    def required_components(self):
        return [
            HW, SSH, DNS, DDNS, APT, Mount, Ganeti
            ]

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()

    @base.run_cmds
    def check(self):
        commands = [
            "host %s" % self.cluster.fqdn,
            ]
        return commands

    @base.run_cmds
634 635 636 637
    def add_qa_rapi_user(self):
        cmd = """
echo ganeti-qa qa_example_passwd write >> /var/lib/ganeti/rapi/users
"""
638
        return [cmd]
639

640 641 642
    def _add_rapi_user(self):
        user = config.synnefo_user
        passwd = config.synnefo_rapi_passwd
643
        x = "%s:Ganeti Remote API:%s" % (user, passwd)
644

645 646 647 648
        cmd = """
cat >> /var/lib/ganeti/rapi/users <<EOF
%s {HA1}$(echo -n %s | openssl md5 | sed 's/^.* //') write
EOF
649
""" % (user, x)
650

651
        return [cmd]
652

653 654
    @base.run_cmds
    def add_node(self, info):
655
        commands = [
656
            "gnt-node list " + info.fqdn + " || " +
657
            "gnt-node add --no-ssh-key-check --master-capable=yes " +
658
            "--vm-capable=yes " + info.fqdn,
659 660 661
            ]
        return commands

662 663
    def _try_use_vg(self):
        vg = self.cluster.vg
664 665 666 667 668 669
        return [
            "gnt-cluster modify --vg-name=%s || true" % vg,
            "gnt-cluster modify --disk-parameters=drbd:metavg=%s" % vg,
            "gnt-group modify --disk-parameters=drbd:metavg=%s default" % vg,
            ]

670
    @base.run_cmds
671
    def initialize(self):
672 673
        std = "cpu-count=1,disk-count=1,disk-size=1024"
        std += ",memory-size=128,nic-count=1,spindle-use=1"
674

675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697
        bound_min = "cpu-count=1,disk-count=1,disk-size=512"
        bound_min += ",memory-size=128,nic-count=0,spindle-use=1"

        bound_max = "cpu-count=8,disk-count=16,disk-size=1048576"
        bound_max += ",memory-size=32768,nic-count=8,spindle-use=12"
        cmd = """
gnt-cluster init --enabled-hypervisors=kvm \
    --no-lvm-storage --no-drbd-storage \
    --nic-parameters link={0},mode=bridged \
    --master-netdev {1} \
    --default-iallocator hail \
    --hypervisor-parameters kvm:kernel_path=,vnc_bind_address=0.0.0.0 \
    --no-ssh-init --no-etc-hosts \
    --enabled-disk-templates file,plain,ext,drbd \
    --ipolicy-std-specs {2} \
    --ipolicy-bounds-specs min:{3}/max:{4} \
    {5}
        """.format(config.common_bridge, self.cluster.netdev,
                   std, bound_min, bound_max, self.cluster.fqdn)

        return [cmd] + self._try_use_vg() + self._add_rapi_user()

    @base.run_cmds
698 699 700
    def restart(self):
        return ["/etc/init.d/ganeti restart"]

701 702 703 704 705 706 707 708 709
    @update_admin
    @update_cluster_admin
    def admin_post(self):
        if self.cluster.synnefo:
            self.CYCLADES._debug("Adding backend: %s" % self.cluster.fqdn)
            self.CYCLADES.add_backend()
            self.CYCLADES.list_backends()
            self.CYCLADES.undrain_backend()

710

711
class Image(base.Component):
712 713 714 715
    REQUIRED_PACKAGES = [
        "snf-image",
        ]

716
    @base.run_cmds
717
    def check(self):
718
        return ["mkdir -p %s" % config.image_dir]
719

720
    @base.run_cmds
721
    def prepare(self):
722 723
        url = config.debian_base_url
        d = config.image_dir
724 725 726 727 728
        image = "debian_base.diskdump"
        return [
            "test -e %s/%s || wget %s -O %s/%s" % (d, image, url, d, image)
            ]

729
    def _configure(self):
730 731
        tmpl = "/etc/default/snf-image"
        replace = {
732 733 734
            "synnefo_user": config.synnefo_user,
            "synnefo_db_passwd": config.synnefo_db_passwd,
            "pithos_dir": config.pithos_dir,
735
            "db_node": self.ctx.db.cname,
736
            "image_dir": config.image_dir,
737 738 739
            }
        return [(tmpl, replace, {})]

740
    @base.run_cmds
741 742 743 744
    def initialize(self):
        return ["snf-image-update-helper -y"]


745
class GTools(base.Component):
746 747 748 749
    REQUIRED_PACKAGES = [
        "snf-cyclades-gtools",
        ]

750
    @base.run_cmds
751
    def check(self):
752
        return ["ping -c1 %s" % self.ctx.mq.cname]
753

754
    @base.run_cmds
755 756 757 758 759
    def prepare(self):
        return [
            "sed -i 's/false/true/' /etc/default/snf-ganeti-eventd",
            ]

760
    def _configure(self):
761 762
        tmpl = "/etc/synnefo/gtools.conf"
        replace = {
763 764
            "synnefo_user": config.synnefo_user,
            "synnefo_rabbitmq_passwd": config.synnefo_rabbitmq_passwd,
765
            "mq_node": self.ctx.mq.cname,
766 767 768
            }
        return [(tmpl, replace, {})]

769
    @base.run_cmds
770 771 772 773
    def restart(self):
        return ["/etc/init.d/snf-ganeti-eventd restart"]


774
class Network(base.Component):
775 776 777 778 779 780
    REQUIRED_PACKAGES = [
        "python-nfqueue",
        "snf-network",
        "nfdhcpd",
        ]

781
    def _configure(self):
782
        r1 = {
783
            "ns_node_ip": self.ctx.ns.ip
784 785
            }
        r2 = {
786 787 788 789 790 791
            "common_bridge": config.common_bridge,
            "public_iface": self.node.public_iface,
            "subnet": config.synnefo_public_network_subnet,
            "gateway": config.synnefo_public_network_gateway,
            "router_ip": self.ctx.router.ip,
            "node_ip": self.node.ip,
792 793
            }
        r3 = {
794 795 796
            "domain": self.node.domain,
            "server": self.ctx.ns.ip,
            "keyfile": config.ddns_private_key,
797 798 799 800 801 802 803 804
            }

        return [
            ("/etc/nfdhcpd/nfdhcpd.conf", r1, {}),
            ("/etc/rc.local", r2, {"mode": 0755}),
            ("/etc/default/snf-network", r3, {}),
            ]

805
    @base.run_cmds
806 807 808
    def initialize(self):
        return ["/etc/init.d/rc.local start"]

809
    @base.run_cmds
810 811 812 813
    def restart(self):
        return ["/etc/init.d/nfdhcpd restart"]


814
class Apache(base.Component):
815 816 817 818
    REQUIRED_PACKAGES = [
        "apache2",
        ]

819
    @base.run_cmds
820 821 822 823 824 825 826
    def prepare(self):
        return [
            "a2enmod ssl", "a2enmod rewrite", "a2dissite default",
            "a2enmod headers",
            "a2enmod proxy_http", "a2dismod autoindex",
            ]

827 828
    def _configure(self):
        r1 = {"HOST": self.node.fqdn}
829 830 831 832 833
        return [
            ("/etc/apache2/sites-available/synnefo", r1, {}),
            ("/etc/apache2/sites-available/synnefo-ssl", r1, {}),
            ]

834
    @base.run_cmds
835 836 837 838 839
    def initialize(self):
        return [
            "a2ensite synnefo", "a2ensite synnefo-ssl",
            ]

840
    @base.run_cmds
841 842
    def restart(self):
        return [
843 844
            "/etc/init.d/apache2 restart",
            ]
845 846


847
class Gunicorn(base.Component):
848 849 850 851
    REQUIRED_PACKAGES = [
        "gunicorn",
        ]

852
    @base.run_cmds
853 854 855 856 857
    def prepare(self):
        return [
            "chown root.www-data /var/log/gunicorn",
            ]

858 859
    def _configure(self):
        r1 = {"HOST": self.node.fqdn}
860 861 862 863
        return [
            ("/etc/gunicorn.d/synnefo", r1, {}),
            ]

864
    @base.run_cmds
865 866
    def restart(self):
        return [
867 868
            "/etc/init.d/gunicorn restart",
            ]
869 870


871
class Common(base.Component):
872 873 874 875 876 877 878 879 880
    REQUIRED_PACKAGES = [
        # snf-common
        "python-objpool",
        "snf-common",
        "python-astakosclient",
        "snf-django-lib",
        "snf-branding",
        ]

881
    def _configure(self):
882
        r1 = {
883 884 885 886
            "EMAIL_SUBJECT_PREFIX": self.node.hostname,
            "domain": self.node.domain,
            "HOST": self.node.fqdn,
            "MAIL_DIR": config.mail_dir,
887 888 889 890 891
            }
        return [
            ("/etc/synnefo/common.conf", r1, {}),
            ]

892
    @base.run_cmds
893
    def initialize(self):
894
        return ["mkdir -p {0}; chmod 777 {0}".format(config.mail_dir)]
895

896
    @base.run_cmds
897 898
    def restart(self):
        return [
899 900
            "/etc/init.d/gunicorn restart",
            ]
901 902


903
class WEB(base.Component):
904 905 906 907 908 909 910
    REQUIRED_PACKAGES = [
        "snf-webproject",
        "python-psycopg2",
        "python-gevent",
        "python-django",
        ]

911
    @base.run_cmds
912
    def check(self):
913
        return ["ping -c1 %s" % self.ctx.db.cname]
914

915
    def _configure(self):
916
        r1 = {
917 918
            "synnefo_user": config.synnefo_user,
            "synnefo_db_passwd": config.synnefo_db_passwd,
919
            "db_node": self.ctx.db.cname,
920
            "domain": self.node.domain,
921
            "webproject_secret": config.webproject_secret,
922 923 924 925 926
            }
        return [
            ("/etc/synnefo/webproject.conf", r1, {}),
            ]

927
    @base.run_cmds
928 929
    def restart(self):
        return [
930 931
            "/etc/init.d/gunicorn restart",
            ]
932 933


934
class Astakos(base.Component):
935 936 937 938
    REQUIRED_PACKAGES = [
        "python-django-south",
        "snf-astakos-app",
        "kamaki",
939
        "python-openssl",
940 941
        ]

942
    alias = constants.ASTAKOS
943
    service = constants.ASTAKOS
944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964

    def required_components(self):
        return [HW, SSH, DNS, APT, Apache, Gunicorn, Common, WEB]

    @base.run_cmds
    def setup_user(self):
        self._debug("Setting up user")
        return self._set_default_quota() + \
            self._add_user() + self._activate_user()

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()
        self.DB.allow_db_access()
        self.DB.restart()

    @property
    def conflicts(self):
        return [CMS]

    @base.run_cmds
965
    def export_service(self):
966
        f = config.jsonfile
967 968 969 970
        return [
            "snf-manage service-export-astakos > %s" % f
            ]

971
    @base.run_cmds
972
    def import_service(self):
973
        f = config.jsonfile
974 975 976 977
        return [
            "snf-manage service-import --json=%s" % f
            ]

978 979
    @base.run_cmds
    def set_astakos_default_quota(self):
980 981 982
        cmd = "snf-manage resource-modify"
        return [
            "%s --base-default 2 astakos.pending_app" % cmd,
983 984 985 986 987 988 989
            "%s --project-default 0 astakos.pending_app" % cmd,
            ]

    @base.run_cmds
    def set_cyclades_default_quota(self):
        cmd = "snf-manage resource-modify"
        return [
990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005
            "%s --base-default 4 cyclades.vm" % cmd,
            "%s --base-default 40G cyclades.disk" % cmd,
            "%s --base-default 16G cyclades.total_ram" % cmd,
            "%s --base-default 8G cyclades.ram" % cmd,
            "%s --base-default 32 cyclades.total_cpu" % cmd,
            "%s --base-default 16 cyclades.cpu" % cmd,
            "%s --base-default 4 cyclades.network.private" % cmd,
            "%s --base-default 4 cyclades.floating_ip" % cmd,
            "%s --project-default 0 cyclades.vm" % cmd,
            "%s --project-default 0 cyclades.disk" % cmd,
            "%s --project-default inf cyclades.total_ram" % cmd,
            "%s --project-default 0 cyclades.ram" % cmd,
            "%s --project-default inf cyclades.total_cpu" % cmd,
            "%s --project-default 0 cyclades.cpu" % cmd,
            "%s --project-default 0 cyclades.network.private" % cmd,
            "%s --project-default 0 cyclades.floating_ip" % cmd,
1006 1007
            ]

1008 1009 1010 1011 1012 1013 1014 1015 1016
    @base.run_cmds
    def set_pithos_default_quota(self):
        cmd = "snf-manage resource-modify"
        return [
            "%s --base-default 40G pithos.diskspace" % cmd,
            "%s --project-default 0 pithos.diskspace" % cmd,
            ]

    @base.run_cmds
1017
    def modify_all_quota(self):
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
        cmd = "snf-manage project-modify --all-base-projects --limit"
        return [
            "%s pithos.diskspace 40G 40G" % cmd,
            "%s astakos.pending_app 2 2" % cmd,
            "%s cyclades.vm 4 4" % cmd,
            "%s cyclades.disk 40G 40G" % cmd,
            "%s cyclades.total_ram 16G 16G" % cmd,
            "%s cyclades.ram 8G 8G" % cmd,
            "%s cyclades.total_cpu 32 32" % cmd,
            "%s cyclades.cpu 16 16" % cmd,
            "%s cyclades.network.private 4 4" % cmd,
            "%s cyclades.floating_ip 4 4" % cmd,
1030 1031
            ]

1032 1033
    @parse_service_info
    @base.run_cmds
1034 1035 1036 1037 1038
    def get_services(self):
        return [
            "snf-manage component-list -o id,name,token"
            ]

1039
    def _configure(self):
1040
        r1 = {
1041
            "ACCOUNTS": self.ctx.astakos.cname,
1042
            "domain": self.node.domain,
1043 1044
            "CYCLADES": self.ctx.cyclades.cname,
            "PITHOS": self.ctx.pithos.cname,
1045 1046
            }
        return [
1047 1048
            ("/etc/synnefo/astakos.conf", r1, {}),
            ("/root/firefox_cert_override.py", {}, {})
1049 1050
            ]

1051
    @base.run_cmds
1052 1053 1054 1055 1056 1057 1058
    def initialize(self):
        return [
            "snf-manage syncdb --noinput",
            "snf-manage migrate im --delete-ghost-migrations",
            "snf-manage migrate quotaholder_app",
            "snf-manage migrate oa2",
            "snf-manage loaddata groups",
1059
            ] + self._astakos_oa2() + self._astakos_register_components()
1060

1061 1062
    def _astakos_oa2(self):
        secret = config.oa2_secret
1063
        view = "https://%s/pithos/ui/view" % self.ctx.pithos.cname
1064 1065 1066 1067 1068
        cmd = "snf-manage oauth2-client-add pithos-view \
                  --secret=%s --is-trusted --url %s || true" % (secret, view)
        return [cmd]

    def _astakos_register_components(self):
1069
        # base urls
1070 1071 1072 1073
        cbu = "https://%s/cyclades" % self.ctx.cyclades.cname
        pbu = "https://%s/pithos" % self.ctx.pithos.cname
        abu = "https://%s/astakos" % self.ctx.astakos.cname
        cmsurl = "https://%s/home" % self.ctx.cms.cname
1074 1075 1076 1077 1078 1079 1080 1081 1082

        cmd = "snf-manage component-add"
        h = "%s home --base-url %s --ui-url %s" % (cmd, cmsurl, cmsurl)
        c = "%s cyclades --base-url %s --ui-url %s/ui" % (cmd, cbu, cbu)
        p = "%s pithos --base-url %s --ui-url %s/ui" % (cmd, pbu, pbu)
        a = "%s astakos --base-url %s --ui-url %s/ui" % (cmd, abu, abu)

        return [h, c, p, a]

1083
    @base.run_cmds
1084 1085
    def add_user(self):
        info = (
1086 1087 1088 1089
            config.user_passwd,
            config.user_email,
            config.user_name,
            config.user_lastname,
1090 1091 1092 1093
            )
        cmd = "snf-manage user-add --password %s %s %s %s" % info
        return [cmd]

1094 1095
    @update_admin
    @base.run_cmds
1096
    def activate_user(self):
1097 1098
        self.DB.get_user_info_from_db()
        user_id = context.user_id
1099 1100 1101 1102 1103
        return [
            "snf-manage user-modify --verify %s" % user_id,
            "snf-manage user-modify --accept %s" % user_id,
            ]

1104 1105
    @update_admin
    @export_and_import_service
1106
    @cert_override
1107 1108 1109
    def admin_post(self):
        self.set_astakos_default_quota()

1110

1111
class CMS(base.Component):
1112 1113
    REQUIRED_PACKAGES = [
        "snf-cloudcms"
1114
        "python-openssl",
1115 1116
        ]

1117
    alias = constants.CMS
1118
    service = constants.CMS
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133

    def required_components(self):
        return [HW, SSH, DNS, APT, Apache, Gunicorn, Common, WEB]

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()
        self.DB.allow_db_access()
        self.DB.restart()

    @property
    def conflicts(self):
        return [Astakos, Pithos, Cyclades]

    def _configure(self):
1134
        r1 = {
1135
            "ACCOUNTS": self.ctx.astakos.cname
1136 1137
            }
        r2 = {
1138
            "DOMAIN": self.node.domain
1139 1140 1141 1142 1143 1144 1145
            }
        return [
            ("/etc/synnefo/cms.conf", r1, {}),
            ("/tmp/sites.json", r2, {}),
            ("/tmp/page.json", {}, {}),
            ]

1146
    @base.run_cmds
1147 1148 1149 1150 1151 1152 1153
    def initialize(self):
        return [
            "snf-manage syncdb",
            "snf-manage migrate --delete-ghost-migrations",
            "snf-manage loaddata /tmp/sites.json",
            "snf-manage loaddata /tmp/page.json",
            "snf-manage createsuperuser --username=admin \
1154
                  --email=admin@%s --noinput" % self.node.domain,
1155 1156
            ]

1157
    @base.run_cmds
1158 1159 1160
    def restart(self):
        return ["/etc/init.d/gunicorn restart"]

1161 1162 1163 1164 1165
    @update_admin
    @cert_override
    def admin_post(self):
        pass

1166

1167
class Mount(base.Component):
1168 1169 1170