components.py 57.3 KB
Newer Older
Vangelis Koukis's avatar
Vangelis Koukis committed
1
# Copyright (C) 2010-2014 GRNET S.A.
2
#
Vangelis Koukis's avatar
Vangelis Koukis committed
3
4
5
6
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
7
#
Vangelis Koukis's avatar
Vangelis Koukis committed
8
9
10
11
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
12
#
Vangelis Koukis's avatar
Vangelis Koukis committed
13
14
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
15

16
import re
17
import datetime
18
import simplejson
19
import copy
20
import os
21
22
23
24
from snfdeploy import base
from snfdeploy import config
from snfdeploy import constants
from snfdeploy import context
25
from snfdeploy.lib import FQDN, evaluate
26
27


28
29
30
_USER_INFO_RE = lambda x: \
    re.compile(r"(\d+)[ |]*(\S+)[ |]*(\S+)[ |]*%s.*" % x, re.M)
_USER_INFO = ["user_id", "user_auth_token", "user_uuid"]
31

32
33
_SERVICE_INFO_RE = lambda x: re.compile(r"(\d+)[ ]*%s[ ]*(\S+)" % x, re.M)
_SERVICE_INFO = ["service_id", "service_token"]
34

35
36
_BACKEND_INFO_RE = lambda x: re.compile(r"(\d+)[ ]*%s.*" % x, re.M)
_BACKEND_INFO = ["backend_id"]
37

38
39
_VOLUME_INFO_RE = lambda x: re.compile(r"(\d+)[ ]*%s.*" % x, re.M)
_VOLUME_INFO = ["volume_type_id"]
40
41


42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
# Helper decorator that wraps get_* methods of certain Components
# Those methods take one argument; the identity (mail, service, backend)
# to look for. It parses the output of those methods and updates the
# context's keys with the matched groups.
def parse(regex, keys):
    def wrap(f):
        def wrapped_f(cl, what):
            result = f(cl, what)
            match = regex(what).search(result)
            if config.dry_run:
                evaluate(context, **dict(zip(keys, ["dummy"] * len(keys))))
            elif match:
                evaluate(context, **dict(zip(keys, match.groups())))
            else:
                raise BaseException("Cannot parse info for %s" % what)
        return wrapped_f
    return wrap
59
60


61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
def update_admin(fn):
    """ Initializes the admin roles for each component

    Initialize the admin roles (NS, Astakos, Cyclades, etc.) and make them
    available under self.NS, self.ASTAKOS, etc. These have the same execution
    context of the current components besides the target node which gets
    derived from the corresponding config.

    """
    def wrapper(*args, **kwargs):
        """If used as decorator of a class method first argument is self."""
        cl = args[0]
        ctx = copy.deepcopy(cl.ctx)
        ctx.admin_service = cl.service
        ctx.admin_cluster = cl.cluster
        ctx.admin_node = cl.node
        ctx.admin_fqdn = cl.fqdn
        cl.NS = NS(node=ctx.ns.node, ctx=ctx)
79
        cl.CA = CA(node=ctx.ca.node, ctx=ctx)
80
81
82
83
        cl.NFS = NFS(node=ctx.nfs.node, ctx=ctx)
        cl.DB = DB(node=ctx.db.node, ctx=ctx)
        cl.ASTAKOS = Astakos(node=ctx.astakos.node, ctx=ctx)
        cl.CYCLADES = Cyclades(node=ctx.cyclades.node, ctx=ctx)
84
        cl.ADMIN = Admin(node=ctx.admin.node, ctx=ctx)
85
        cl.CLIENT = Client(node=ctx.client.node, ctx=ctx)
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
        return fn(*args, **kwargs)
    return wrapper


def update_cluster_admin(fn):
    """ Initializes the cluster admin roles for each component

    Finds the master role for the corresponding cluster

    """
    def wrapper(*args, **kwargs):
        """If used as decorator of a class method first argument is self."""
        cl = args[0]
        ctx = copy.deepcopy(cl.ctx)
        ctx.admin_cluster = cl.cluster
        cl.MASTER = Master(node=ctx.master.node, ctx=ctx)
        return fn(*args, **kwargs)
    return wrapper


def export_and_import_service(fn):
    """ Export and import synnefo service

    Used in Astakos, Pithos, and Cyclades service admin_post method

    """
    def wrapper(*args, **kwargs):
        cl = args[0]
        f = config.jsonfile
        cl.export_service()
        cl.get(f, f + ".local")
        cl.ASTAKOS.put(f + ".local", f)
        cl.ASTAKOS.import_service()
        return fn(*args, **kwargs)
    return wrapper


123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
def cert_override(fn):
    """ Create all needed entries for cert_override.txt file

    Append them in a tmp file and upload them to client node

    """
    def wrapper(*args, **kwargs):
        cl = args[0]
        f = "/tmp/" + constants.CERT_OVERRIDE + "_" + cl.service
        for domain in [cl.node.domain, cl.node.cname, cl.node.ip]:
            cmd = """
python /root/firefox_cert_override.py {0} {1}:443 >> {2}
""".format(constants.CERT_PATH, domain, f)
            cl.run(cmd)
        cl.get(f, f + ".local")
        cl.CLIENT.put(f + ".local", f)
        return fn(*args, **kwargs)
    return wrapper


143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
# ########################## Components ############################

# A Component() gets initialized with an execution context that is a
# configuration snapshot of the target setup, cluster and node. A
# component implements the following helper methods: check, install,
# prepare, configure, restart, initialize, and test. All those methods
# will be executed on the target node with this order during setup.
#
# Additionally each Component class implements admin_pre, and
# admin_post methods which invoke actions on different components on
# the same execution context before and after installation. For
# example before a backend gets installed, its FQDN must resolve to
# the master floating IP, so we have to run some actions on the ns
# node and after installation we must add it to cyclades (snf-manage
# backend-add on the cyclades node).
#
# Component() inherits ComponentRunner() which practically exports the
# setup() method. This will first check if the required components are
# installed, will install them if not and update the status of target
# node.
#
# ComponentRunner() inherits FabricRunner() which practically wraps
# basic fabric commands (put, get, run) with the correct execution
# environment.
#
# Each component gets initialized with an execution context and uses
# the config module for accessing global wide options. The context
# provides node, cluster, and setup related info.
171

172
class HW(base.Component):
173
174
175
176
177
178
179
180
181
182
183
184
185
186

    @base.run_cmds
    def prepare(self):
        return [
            # NOTE: This is needed because the NFS dir is owned by
            # archipelago:synnefo and IDs must be common across nodes
            "addgroup --system --gid 200 synnefo",
            "adduser --system --uid 200 --gid 200 --no-create-home \
                --gecos Synnefo synnefo",
            "addgroup --system --gid 300 archipelago",
            "adduser --system --uid 300 --gid 300 --no-create-home \
                --gecos Archipelago archipelago",
            ]

187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
    @base.check_if_testing
    def _configure(self):
        r1 = {
            "date": str(datetime.datetime.today()),
            }
        return [
            ("/etc/sysctl.d/disable-ipv6.conf", r1, {})
            ]

    @base.run_cmds
    @base.check_if_testing
    def initialize(self):
        return [
            "sysctl -f /etc/sysctl.d/disable-ipv6.conf",
            ]

203
    @base.run_cmds
204
205
    def test(self):
        return [
206
            "ping -c 1 %s" % self.node.ip,
207
208
209
210
            "ping -c 1 www.google.com",
            "apt-get update",
            ]

211

212
213
class SSH(base.Component):
    @base.run_cmds
214
215
216
217
218
219
220
    def prepare(self):
        return [
            "mkdir -p /root/.ssh",
            "for f in $(ls /root/.ssh/*); do cp $f $f.bak ; done",
            "echo StrictHostKeyChecking no >> /etc/ssh/ssh_config",
            ]

221
    def _configure(self):
222
223
224
        files = [
            "authorized_keys", "id_dsa", "id_dsa.pub", "id_rsa", "id_rsa.pub"
            ]
225
        ssh = [("/root/.ssh/%s" % f, {}, {"mode": 0600}) for f in files]
226
227
        return ssh

228
    @base.run_cmds
229
230
231
    def initialize(self):
        f = "/root/.ssh/authorized_keys"
        return [
232
            "test -e {0}.bak && cat {0}.bak >> {0} || true".format(f)
233
234
            ]

235
    @base.run_cmds
236
    def test(self):
237
238
        return ["ssh %s date" % self.node.ip]

239

240
241
242
243
class DNS(base.Component):
    @update_admin
    def admin_pre(self):
        self.NS.update_ns()
244

245
    @base.run_cmds
246
247
248
249
    def prepare(self):
        return [
            "chattr -i /etc/resolv.conf",
            "sed -i 's/^127.*$/127.0.0.1 localhost/g' /etc/hosts",
250
251
            "echo %s > /etc/hostname" % self.node.hostname,
            "hostname %s" % self.node.hostname
252
253
            ]

254
    def _configure(self):
255
256
        r1 = {
            "date": str(datetime.datetime.today()),
257
258
            "domain": self.node.domain,
            "ns_node_ip": self.ctx.ns.ip,
259
260
261
262
263
264
            }
        resolv = [
            ("/etc/resolv.conf", r1, {})
            ]
        return resolv

265
    @base.run_cmds
266
267
268
269
    def initialize(self):
        return ["chattr +i /etc/resolv.conf"]


270
class DDNS(base.Component):
271
272
273
274
    REQUIRED_PACKAGES = [
        "dnsutils",
        ]

275
    @base.run_cmds
276
277
278
279
280
    def prepare(self):
        return [
            "mkdir -p /root/ddns/"
            ]

281
    def _configure(self):
282
        return [
283
            ("/root/ddns/" + k, {}, {}) for k in config.ddns_keys
284
285
286
            ]


287
class NS(base.Component):
288
289
290
291
    REQUIRED_PACKAGES = [
        "bind9",
        ]

292
293
294
295
296
297
    alias = constants.NS

    def required_components(self):
        return [HW, SSH, DDNS]

    def _nsupdate(self, cmd):
298
299
300
301
302
303
        ret = """
nsupdate -k {0} > /dev/null <<EOF || true
server {1}
{2}
send
EOF
304
""".format(config.ddns_private_key, self.ctx.ns.ip, cmd)
305
306
        return ret

307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
    @base.run_cmds
    def update_ns(self, info=None):
        if not info:
            info = self.ctx.admin_fqdn
        return [
            self._nsupdate("update add %s" % info.arecord),
            self._nsupdate("update add %s" % info.ptrrecord),
            self._nsupdate("update add %s" % info.cnamerecord),
            ]

    def add_qa_instances(self):
        instances = [
            ("xen-test-inst1", "1.2.3.4"),
            ("xen-test-inst2", "1.2.3.5"),
            ("xen-test-inst3", "1.2.3.6"),
            ("xen-test-inst4", "1.2.3.7"),
            ]
        for name, ip in instances:
            info = {
                "name": name,
                "ip": ip,
                "domain": self.node.domain
                }
            node_info = FQDN(**info)
            self.update_ns(node_info)

    @base.run_cmds
334
335
336
337
338
339
340
341
    def prepare(self):
        return [
            "mkdir -p /etc/bind/zones",
            "chmod g+w /etc/bind/zones",
            "mkdir -p /etc/bind/rev",
            "chmod g+w /etc/bind/rev",
            ]

342
343
344
    def _configure(self):
        d = self.node.domain
        ip = self.node.ip
345
346
347
348
349
350
351
352
353
354
355
        return [
            ("/etc/bind/named.conf.local", {"domain": d}, {}),
            ("/etc/bind/zones/example.com",
             {"domain": d, "ns_node_ip": ip},
             {"remote": "/etc/bind/zones/%s" % d}),
            ("/etc/bind/zones/vm.example.com",
             {"domain": d, "ns_node_ip": ip},
             {"remote": "/etc/bind/zones/vm.%s" % d}),
            ("/etc/bind/rev/synnefo.in-addr.arpa.zone", {"domain": d}, {}),
            ("/etc/bind/rev/synnefo.ip6.arpa.zone", {"domain": d}, {}),
            ("/etc/bind/named.conf.options",
356
             {"node_ips": ";".join(self.ctx.all_ips)}, {}),
357
358
359
            ("/root/ddns/ddns.key", {}, {"remote": "/etc/bind/ddns.key"}),
            ]

360
    @base.run_cmds
361
362
363
364
    def restart(self):
        return ["/etc/init.d/bind9 restart"]


365
class APT(base.Component):
366
367
368
    """ Setup apt repos and check fqdns """
    REQUIRED_PACKAGES = ["curl"]

369
    @base.run_cmds
370
371
372
    def prepare(self):
        return [
            "echo 'APT::Install-Suggests \"false\";' >> /etc/apt/apt.conf",
373
374
            "curl -k https://dev.grnet.gr/files/apt-grnetdev.pub | \
                apt-key add -",
375
376
            ]

377
    def _configure(self):
378
379
380
381
        return [
            ("/etc/apt/sources.list.d/synnefo.wheezy.list", {}, {})
            ]

382
    @base.run_cmds
383
384
385
386
387
388
    def initialize(self):
        return [
            "apt-get update",
            ]


389
class MQ(base.Component):
390
391
    REQUIRED_PACKAGES = ["rabbitmq-server"]

392
393
394
395
396
397
398
399
400
401
    alias = constants.MQ

    def required_components(self):
        return [HW, SSH, DNS, APT]

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()

    @base.run_cmds
402
    def check(self):
403
        return ["ping -c 1 %s" % self.node.cname]
404

405
    @base.run_cmds
406
    def initialize(self):
407
408
        u = config.synnefo_user
        p = config.synnefo_rabbitmq_passwd
409
410
411
412
413
414
415
416
        return [
            "rabbitmqctl add_user %s %s" % (u, p),
            "rabbitmqctl set_permissions %s \".*\" \".*\" \".*\"" % u,
            "rabbitmqctl delete_user guest",
            "rabbitmqctl set_user_tags %s administrator" % u,
            ]


417
class DB(base.Component):
418
419
    REQUIRED_PACKAGES = ["postgresql"]

420
421
422
423
424
425
426
427
428
429
    alias = constants.DB

    def required_components(self):
        return [HW, SSH, DNS, APT]

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()

    @base.run_cmds
430
    def check(self):
431
        return ["ping -c 1 %s" % self.node.cname]
432

433
    @parse(_USER_INFO_RE, _USER_INFO)
434
    @base.run_cmds
435
    def get_user_info_from_db(self, user_email):
436
437
438
439
440
441
442
        cmd = """
cat > /tmp/psqlcmd <<EOF
select id, auth_token, uuid, email from auth_user, im_astakosuser \
where auth_user.id = im_astakosuser.user_ptr_id and auth_user.email = '{0}';
EOF

su - postgres -c  "psql -w -d snf_apps -f /tmp/psqlcmd"
443
""".format(user_email)
444
445
446

        return [cmd]

447
448
449
450
451
    @base.run_cmds
    def allow_db_access(self):
        user = "all"
        method = "md5"
        ip = self.ctx.admin_node.ip
452
453
        f = "/etc/postgresql/*/main/pg_hba.conf"
        cmd1 = "echo host all %s %s/32 %s >> %s" % \
454
            (user, ip, method, f)
455
        cmd2 = "sed -i 's/\(host.*127.0.0.1.*\)md5/\\1trust/' %s" % f
456
        return [cmd1, cmd2]
457

458
459
460
    def _configure(self):
        u = config.synnefo_user
        p = config.synnefo_db_passwd
461
462
463
464
465
        replace = {"synnefo_user": u, "synnefo_db_passwd": p}
        return [
            ("/tmp/db-init.psql", replace, {}),
            ]

466
    @base.check_if_testing
467
468
469
470
471
    def make_db_fast(self):
        f = "/etc/postgresql/*/main/postgresql.conf"
        opts = "fsync=off\nsynchronous_commit=off\nfull_page_writes=off\n"
        return ["""echo -e "%s" >> %s""" % (opts, f)]

472
    @base.run_cmds
473
474
    def prepare(self):
        f = "/etc/postgresql/*/main/postgresql.conf"
475
476
        ret = ["""echo "listen_addresses = '*'" >> %s""" % f]
        return ret + self.make_db_fast()
477

478
    @base.run_cmds
479
480
481
482
483
    def initialize(self):
        script = "/tmp/db-init.psql"
        cmd = "su - postgres -c \"psql -w -f %s\" " % script
        return [cmd]

484
    @base.run_cmds
485
486
487
    def restart(self):
        return ["/etc/init.d/postgresql restart"]

488
    @base.run_cmds
489
490
491
492
493
494
495
    def destroy_db(self):
        return [
            """su - postgres -c ' psql -w -c "drop database snf_apps" '""",
            """su - postgres -c ' psql -w -c "drop database snf_pithos" '"""
            ]


496
497
498
499
class VMC(base.Component):

    def extra_components(self):
        if self.cluster.synnefo:
500
501
            return [
                Image, GTools, GanetiCollectd,
502
                PithosBackend, ExtStorage, Archip, ArchipGaneti
503
                ]
504
        else:
505
            return [ExtStorage, Archip, ArchipGaneti]
506

507
508
    def required_components(self):
        return [
509
            HW, SSH, DNS, DDNS, APT, Mount, LVM, DRBD, Ganeti, Network,
510
511
512
513
514
            ] + self.extra_components()

    @update_cluster_admin
    def admin_post(self):
        self.MASTER.add_node(self.node)
515
516
        self.MASTER.enable_lvm()
        self.MASTER.enable_drbd()
517
518


519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
class LVM(base.Component):
    REQUIRED_PACKAGES = [
        "lvm2",
        ]

    @base.run_cmds
    def initialize(self):
        extra_disk_dev = self.node.extra_disk
        extra_disk_file = "/disk"
        # If extra disk found use it
        # else create a raw file and losetup it
        cmd = """
if [ -b "{0}" ]; then
  pvcreate {0} && vgcreate {2} {0}
else
  truncate -s {3} {1}
  loop_dev=$(losetup -f --show {1})
  pvcreate $loop_dev
  vgcreate {2} $loop_dev
fi
""".format(extra_disk_dev, extra_disk_file,
           self.cluster.vg, self.cluster.vg_size)

        return [cmd]


class DRBD(base.Component):
    REQUIRED_PACKAGES = [
        "drbd8-utils",
        ]

    def _configure(self):
        return [
            ("/etc/modprobe.d/drbd.conf", {}, {}),
            ]

    def prepare(self):
        return [
            "echo drbd >> /etc/modules",
            ]

    @base.run_cmds
    def initialize(self):
        return [
            "modprobe -rv drbd || true",
            "modprobe -v drbd",
            ]


568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
class CA(base.Component):
    REQUIRED_PACKAGES = [
        "openssl"
        ]

    alias = constants.CA
    service = constants.CA

    def required_components(self):
        return [
            HW, SSH, DNS, APT,
            ]

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()

    @base.run_cmds
    def prepare(self):
        return [
            "mkdir -p /root/ca"
            ]

    def _configure(self):
        r1 = {
            "domain": self.node.domain,
            }
        return [
            ("/root/create_root_ca.sh", {}, {"mode": 0755}),
            ("/root/ca/ca-x509-extensions.cnf", r1, {}),
            ("/root/ca/x509-extensions.cnf", r1, {}),
            ]

    @base.run_cmds
    def initialize(self):
        return [
            "/root/create_root_ca.sh"
            ]


608
class Ganeti(base.Component):
609
610
611
    REQUIRED_PACKAGES = [
        "qemu-kvm",
        "python-bitarray",
612
        "bridge-utils",
613
614
        "snf-ganeti",
        "ganeti2",
615
        "ganeti-instance-debootstrap"
616
617
        ]

618
619
620
621
622
    @update_admin
    def admin_pre(self):
        self.NS.update_ns()

    @base.run_cmds
623
624
    def check(self):
        commands = [
625
626
            "getent hosts %s | grep -v ^127" % self.node.hostname,
            "hostname -f | grep %s" % self.node.fqdn,
627
628
629
            ]
        return commands

630
    def _configure(self):
631
632
633
        r = {
            "SHARED_GANETI_DIR": config.ganeti_dir,
            }
634
        return [
635
            ("/etc/ganeti/file-storage-paths", r, {}),
636
            ("/etc/default/ganeti-instance-debootstrap", {}, {}),
637
638
            ]

639
640
    def _prepare_net_infra(self):
        br = config.common_bridge
641
642
643
644
        return [
            "brctl addbr {0}; ip link set {0} up".format(br)
            ]

645
    @base.run_cmds
646
647
    def prepare(self):
        return [
648
649
            "mkdir -p %s/file-storage/" % config.ganeti_dir,
            "mkdir -p %s/shared-file-storage/" % config.ganeti_dir,
650
            "sed -i 's/^127.*$/127.0.0.1 localhost/g' /etc/hosts",
651
            ] + self._prepare_net_infra()
652

653
    @base.run_cmds
654
    def restart(self):
655
        return ["/etc/init.d/ganeti restart"]
656
657


658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
class Master(base.Component):

    @property
    def fqdn(self):
        return self.cluster

    def required_components(self):
        return [
            HW, SSH, DNS, DDNS, APT, Mount, Ganeti
            ]

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()

    @base.run_cmds
    def check(self):
        commands = [
            "host %s" % self.cluster.fqdn,
            ]
        return commands

    @base.run_cmds
681
682
683
684
    def add_qa_rapi_user(self):
        cmd = """
echo ganeti-qa qa_example_passwd write >> /var/lib/ganeti/rapi/users
"""
685
        return [cmd]
686

687
688
689
    def _add_rapi_user(self):
        user = config.synnefo_user
        passwd = config.synnefo_rapi_passwd
690
        x = "%s:Ganeti Remote API:%s" % (user, passwd)
691

692
693
694
695
        cmd = """
cat >> /var/lib/ganeti/rapi/users <<EOF
%s {HA1}$(echo -n %s | openssl md5 | sed 's/^.* //') write
EOF
696
""" % (user, x)
697

698
        return [cmd]
699

700
701
    @base.run_cmds
    def add_node(self, info):
702
703
704
705
706
707
708
709
710
711
712
713
714
        add = """
gnt-node list {0} || gnt-node add --no-ssh-key-check {0}
""".format(info.fqdn)

        mod_vm = """
gnt-node modify --vm-capable=yes {0}
""".format(info.fqdn)

        mod_master = """
gnt-node modify --master-capable=yes {0}
""".format(info.fqdn)

        return [add, mod_vm, mod_master]
715

716
717
    @base.run_cmds
    def enable_lvm(self):
718
        vg = self.cluster.vg
719
        return [
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
            # This is needed because MIN_VG_SIZE is constant and set to 20G
            # and cluster modify --vg-name may result to:
            # volume group 'ganeti' too small
            # But this check is made only ff a vm-capable node is found
            "gnt-cluster modify --enabled-disk-templates file,ext,plain \
                                --vg-name=%s" % vg,
            "gnt-cluster modify --ipolicy-disk-template file,ext,plain",
            ]

    @base.run_cmds
    def enable_drbd(self):
        vg = self.cluster.vg
        return [
            "gnt-cluster modify --enabled-disk-templates file,ext,plain,drbd \
                                --drbd-usermode-helper=/bin/true",
            "gnt-cluster modify --ipolicy-disk-template file,ext,plain,drbd",
736
737
738
739
            "gnt-cluster modify --disk-parameters=drbd:metavg=%s" % vg,
            "gnt-group modify --disk-parameters=drbd:metavg=%s default" % vg,
            ]

740
    @base.run_cmds
741
    def initialize(self):
742
743
        std = "cpu-count=1,disk-count=1,disk-size=1024"
        std += ",memory-size=128,nic-count=1,spindle-use=1"
744

745
746
747
748
749
        bound_min = "cpu-count=1,disk-count=1,disk-size=512"
        bound_min += ",memory-size=128,nic-count=0,spindle-use=1"

        bound_max = "cpu-count=8,disk-count=16,disk-size=1048576"
        bound_max += ",memory-size=32768,nic-count=8,spindle-use=12"
750
751

        init = """
752
753
754
755
756
757
758
759
gnt-cluster init --enabled-hypervisors=kvm \
    --nic-parameters link={0},mode=bridged \
    --master-netdev {1} \
    --default-iallocator hail \
    --hypervisor-parameters kvm:kernel_path=,vnc_bind_address=0.0.0.0 \
    --no-ssh-init --no-etc-hosts \
    --ipolicy-std-specs {2} \
    --ipolicy-bounds-specs min:{3}/max:{4} \
760
    --enabled-disk-templates file,ext \
761
762
763
764
    {5}
        """.format(config.common_bridge, self.cluster.netdev,
                   std, bound_min, bound_max, self.cluster.fqdn)

765
766
767
        modify = "gnt-node modify --vm-capable=no %s" % self.node.fqdn

        return [init, modify] + self._add_rapi_user()
768
769

    @base.run_cmds
770
771
772
    def restart(self):
        return ["/etc/init.d/ganeti restart"]

773
774
775
776
777
778
    @update_admin
    @update_cluster_admin
    def admin_post(self):
        if self.cluster.synnefo:
            self.CYCLADES._debug("Adding backend: %s" % self.cluster.fqdn)
            self.CYCLADES.add_backend()
779
            self.CYCLADES.list_backends(self.cluster.fqdn)
780
781
            self.CYCLADES.undrain_backend()

782

783
class Image(base.Component):
784
785
786
787
    REQUIRED_PACKAGES = [
        "snf-image",
        ]

788
    @base.run_cmds
789
    def check(self):
790
        return ["mkdir -p %s" % config.images_dir]
791

792
    @base.run_cmds
793
    def prepare(self):
794
        url = config.debian_base_url
795
        d = config.images_dir
796
797
        image = "debian_base.diskdump"
        return [
798
799
            "test -e /tmp/%s || wget -4 %s -O /tmp/%s" % (image, url, image),
            "cp /tmp/%s %s/%s" % (image, d, image),
800
            "mv /etc/default/snf-image /etc/default/snf-image.orig",
801
802
            ]

803
    def _configure(self):
804
805
        tmpl = "/etc/default/snf-image"
        replace = {
806
807
            "synnefo_user": config.synnefo_user,
            "synnefo_db_passwd": config.synnefo_db_passwd,
808
            "db_node": self.ctx.db.cname,
809
            "image_dir": config.images_dir,
810
811
812
            }
        return [(tmpl, replace, {})]

813
    @base.run_cmds
814
    def initialize(self):
815
816
817
        # This is done during postinstall phase
        # snf-image-update-helper -y
        return []
818
819


820
class GTools(base.Component):
821
822
823
824
    REQUIRED_PACKAGES = [
        "snf-cyclades-gtools",
        ]

825
    @base.run_cmds
826
    def check(self):
827
        return ["ping -c1 %s" % self.ctx.mq.cname]
828

829
    @base.run_cmds
830
831
832
833
834
    def prepare(self):
        return [
            "sed -i 's/false/true/' /etc/default/snf-ganeti-eventd",
            ]

835
    def _configure(self):
836
837
        tmpl = "/etc/synnefo/gtools.conf"
        replace = {
838
839
            "synnefo_user": config.synnefo_user,
            "synnefo_rabbitmq_passwd": config.synnefo_rabbitmq_passwd,
840
            "mq_node": self.ctx.mq.cname,
841
842
843
            }
        return [(tmpl, replace, {})]

844
    @base.run_cmds
845
846
847
848
    def restart(self):
        return ["/etc/init.d/snf-ganeti-eventd restart"]


849
class Network(base.Component):
850
851
852
853
854
855
    REQUIRED_PACKAGES = [
        "python-nfqueue",
        "snf-network",
        "nfdhcpd",
        ]

856
    def _configure(self):
857
        r1 = {
858
            "ns_node_ip": self.ctx.ns.ip
859
860
            }
        r2 = {
861
862
863
864
865
866
            "common_bridge": config.common_bridge,
            "public_iface": self.node.public_iface,
            "subnet": config.synnefo_public_network_subnet,
            "gateway": config.synnefo_public_network_gateway,
            "router_ip": self.ctx.router.ip,
            "node_ip": self.node.ip,
867
868
            }
        r3 = {
869
870
871
            "domain": self.node.domain,
            "server": self.ctx.ns.ip,
            "keyfile": config.ddns_private_key,
872
873
874
875
876
877
878
879
            }

        return [
            ("/etc/nfdhcpd/nfdhcpd.conf", r1, {}),
            ("/etc/rc.local", r2, {"mode": 0755}),
            ("/etc/default/snf-network", r3, {}),
            ]

880
    @base.run_cmds
881
882
883
    def initialize(self):
        return ["/etc/init.d/rc.local start"]

884
    @base.run_cmds
885
886
887
888
    def restart(self):
        return ["/etc/init.d/nfdhcpd restart"]


889
class Apache(base.Component):
890
891
    REQUIRED_PACKAGES = [
        "apache2",
892
        "python-openssl",
893
894
        ]

895
    @base.run_cmds
896
897
898
899
900
901
902
    def prepare(self):
        return [
            "a2enmod ssl", "a2enmod rewrite", "a2dissite default",
            "a2enmod headers",
            "a2enmod proxy_http", "a2dismod autoindex",
            ]

903
904
    def _configure(self):
        r1 = {"HOST": self.node.fqdn}
905
906
907
        return [
            ("/etc/apache2/sites-available/synnefo", r1, {}),
            ("/etc/apache2/sites-available/synnefo-ssl", r1, {}),
908
            ("/root/firefox_cert_override.py", {}, {})
909
910
            ]

911
    @base.run_cmds
912
913
914
915
916
    def initialize(self):
        return [
            "a2ensite synnefo", "a2ensite synnefo-ssl",
            ]

917
    @base.run_cmds
918
919
    def restart(self):
        return [
920
921
            "/etc/init.d/apache2 restart",
            ]
922
923


924
class Gunicorn(base.Component):
925
    REQUIRED_PACKAGES = [
926
        "python-gevent",
927
928
929
        "gunicorn",
        ]

930
931
    def _configure(self):
        r1 = {"HOST": self.node.fqdn}
932
933
934
935
        return [
            ("/etc/gunicorn.d/synnefo", r1, {}),
            ]

936
    @base.run_cmds
937
938
    def restart(self):
        return [
939
940
            "/etc/init.d/gunicorn restart",
            ]
941
942


943
class Common(base.Component):
944
    REQUIRED_PACKAGES = [
945
        "ntp",
946
947
948
949
        "snf-common",
        "snf-branding",
        ]

950
951
952
953
954
955
956
    @base.run_cmds
    def prepare(self):
        return [
            "mkdir -p %s" % config.mail_dir,
            "chmod 777 %s" % config.mail_dir,
            ]

957
    def _configure(self):
958
        r1 = {
959
960
961
962
            "EMAIL_SUBJECT_PREFIX": self.node.hostname,
            "domain": self.node.domain,
            "HOST": self.node.fqdn,
            "MAIL_DIR": config.mail_dir,
963
964
965
966
967
            }
        return [
            ("/etc/synnefo/common.conf", r1, {}),
            ]

968
    @base.run_cmds
969
970
    def restart(self):
        return [
971
972
            "/etc/init.d/gunicorn restart",
            ]
973
974


975
class Webproject(base.Component):
976
    REQUIRED_PACKAGES = [
977
        "python-psycopg2",
978
979
        "python-astakosclient",
        "snf-django-lib",
980
981
982
        "snf-webproject",
        ]

983
    @base.run_cmds
984
    def check(self):
985
        return ["ping -c1 %s" % self.ctx.db.cname]
986

987
    def _configure(self):
988
        r1 = {
989
990
            "synnefo_user": config.synnefo_user,
            "synnefo_db_passwd": config.synnefo_db_passwd,
991
            "db_node": self.ctx.db.cname,
992
            "domain": self.node.domain,
993
            "webproject_secret": config.webproject_secret,
994
995
996
997
998
            }
        return [
            ("/etc/synnefo/webproject.conf", r1, {}),
            ]

999
    @base.run_cmds
1000
1001
    def restart(self):
        return [
1002
1003
            "/etc/init.d/gunicorn restart",
            ]
1004
1005


1006
class Astakos(base.Component):
1007
1008
1009
1010
    REQUIRED_PACKAGES = [
        "snf-astakos-app",
        ]

1011
    alias = constants.ASTAKOS
1012
    service = constants.ASTAKOS
1013
1014

    def required_components(self):
1015
        return [HW, SSH, DNS, APT, Apache, Gunicorn, Common, Webproject]
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033

    @base.run_cmds
    def setup_user(self):
        self._debug("Setting up user")
        return self._set_default_quota() + \
            self._add_user() + self._activate_user()

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()
        self.DB.allow_db_access()
        self.DB.restart()

    @property
    def conflicts(self):
        return [CMS]

    @base.run_cmds
1034
    def export_service(self):
1035
        f = config.jsonfile
1036
1037
1038
1039
        return [
            "snf-manage service-export-astakos > %s" % f
            ]

1040
    @base.run_cmds
1041
    def import_service(self):
1042
        f = config.jsonfile
1043
1044
1045
1046
        return [
            "snf-manage service-import --json=%s" % f
            ]

1047
1048
    @base.run_cmds
    def set_astakos_default_quota(self):
1049
1050
        cmd = "snf-manage resource-modify"
        return [
1051
            "%s --system-default 2 astakos.pending_app" % cmd,
1052
1053
1054
1055
1056
1057
1058
            "%s --project-default 0 astakos.pending_app" % cmd,
            ]

    @base.run_cmds
    def set_cyclades_default_quota(self):
        cmd = "snf-manage resource-modify"
        return [
1059
1060
1061
1062
1063
1064
1065
1066
            "%s --system-default 4 cyclades.vm" % cmd,
            "%s --system-default 40G cyclades.disk" % cmd,
            "%s --system-default 16G cyclades.total_ram" % cmd,
            "%s --system-default 8G cyclades.ram" % cmd,
            "%s --system-default 32 cyclades.total_cpu" % cmd,
            "%s --system-default 16 cyclades.cpu" % cmd,
            "%s --system-default 4 cyclades.network.private" % cmd,
            "%s --system-default 4 cyclades.floating_ip" % cmd,
1067
1068
1069
1070
1071
1072
1073
1074
            "%s --project-default 0 cyclades.vm" % cmd,
            "%s --project-default 0 cyclades.disk" % cmd,
            "%s --project-default inf cyclades.total_ram" % cmd,
            "%s --project-default 0 cyclades.ram" % cmd,
            "%s --project-default inf cyclades.total_cpu" % cmd,
            "%s --project-default 0 cyclades.cpu" % cmd,
            "%s --project-default 0 cyclades.network.private" % cmd,
            "%s --project-default 0 cyclades.floating_ip" % cmd,
1075
1076
            ]

1077
1078
1079
1080
    @base.run_cmds
    def set_pithos_default_quota(self):
        cmd = "snf-manage resource-modify"
        return [
1081
            "%s --system-default 40G pithos.diskspace" % cmd,
1082
1083
1084
1085
            "%s --project-default 0 pithos.diskspace" % cmd,
            ]

    @base.run_cmds
1086
    def modify_all_quota(self):
1087
        cmd = "snf-manage project-modify --all-system-projects --limit"
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
        return [
            "%s pithos.diskspace 40G 40G" % cmd,
            "%s astakos.pending_app 2 2" % cmd,
            "%s cyclades.vm 4 4" % cmd,
            "%s cyclades.disk 40G 40G" % cmd,
            "%s cyclades.total_ram 16G 16G" % cmd,
            "%s cyclades.ram 8G 8G" % cmd,
            "%s cyclades.total_cpu 32 32" % cmd,
            "%s cyclades.cpu 16 16" % cmd,
            "%s cyclades.network.private 4 4" % cmd,
            "%s cyclades.floating_ip 4 4" % cmd,
1099
1100
            ]

1101
    @parse(_SERVICE_INFO_RE, _SERVICE_INFO)
1102
    @base.run_cmds
1103
    def get_services(self, service):
1104
1105
1106
1107
        return [
            "snf-manage component-list -o id,name,token"
            ]

1108
    def _configure(self):
1109
        r1 = {
1110
            "ACCOUNTS": self.ctx.astakos.cname,
1111
            "domain": self.node.domain,
1112
1113
            "CYCLADES": self.ctx.cyclades.cname,
            "PITHOS": self.ctx.pithos.cname,
1114
1115
            }
        return [
1116
            ("/etc/synnefo/astakos.conf", r1, {}),
1117
1118
            ]

1119
    @base.run_cmds
1120
1121
1122
1123
1124
1125
1126
    def initialize(self):
        return [
            "snf-manage syncdb --noinput",
            "snf-manage migrate im --delete-ghost-migrations",
            "snf-manage migrate quotaholder_app",
            "snf-manage migrate oa2",
            "snf-manage loaddata groups",
1127
            ] + self._astakos_oa2() + self._astakos_register_components()
1128

1129
1130
    def _astakos_oa2(self):
        secret = config.oa2_secret
1131
        view = "https://%s/pithos/ui/view" % self.ctx.pithos.cname
1132
1133
1134
1135
1136
        cmd = "snf-manage oauth2-client-add pithos-view \
                  --secret=%s --is-trusted --url %s || true" % (secret, view)
        return [cmd]

    def _astakos_register_components(self):
1137
        # base urls
1138
1139
1140
1141
        cbu = "https://%s/cyclades" % self.ctx.cyclades.cname
        pbu = "https://%s/pithos" % self.ctx.pithos.cname
        abu = "https://%s/astakos" % self.ctx.astakos.cname
        cmsurl = "https://%s/home" % self.ctx.cms.cname
1142
1143
1144
1145
1146
1147
1148
1149
1150

        cmd = "snf-manage component-add"
        h = "%s home --base-url %s --ui-url %s" % (cmd, cmsurl, cmsurl)
        c = "%s cyclades --base-url %s --ui-url %s/ui" % (cmd, cbu, cbu)
        p = "%s pithos --base-url %s --ui-url %s/ui" % (cmd, pbu, pbu)
        a = "%s astakos --base-url %s --ui-url %s/ui" % (cmd, abu, abu)

        return [h, c, p, a]

1151
    @base.run_cmds
1152
1153
    def add_user(self):
        info = (
1154
1155
1156
1157
            config.user_passwd,
            config.user_email,
            config.user_name,
            config.user_lastname,
1158
1159
1160
1161
            )
        cmd = "snf-manage user-add --password %s %s %s %s" % info
        return [cmd]

1162
1163
    @update_admin
    @base.run_cmds
1164
    def activate_user(self):
1165
        self.DB.get_user_info_from_db(config.user_email)
1166
        user_id = context.user_id
1167
1168
1169
1170
1171
        return [
            "snf-manage user-modify --verify %s" % user_id,
            "snf-manage user-modify --accept %s" % user_id,
            ]

1172
1173
    @update_admin
    @export_and_import_service
1174
    @cert_override
1175
1176
1177
    def admin_post(self):
        self.set_astakos_default_quota()

1178

1179
class CMS(base.Component):
1180
1181
1182
1183
    REQUIRED_PACKAGES = [
        "snf-cloudcms"
        ]

1184
    alias = constants.CMS
1185
    service = constants.CMS
1186
1187

    def required_components(self):
1188
        return [HW, SSH, DNS, APT, Apache, Gunicorn, Common, Webproject]
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()
        self.DB.allow_db_access()
        self.DB.restart()

    @property
    def conflicts(self):
        return [Astakos, Pithos, Cyclades]

    def _configure(self):
1201
        r1 = {
1202
            "ACCOUNTS": self.ctx.astakos.cname
1203
1204
            }
        r2 = {
1205
            "DOMAIN": self.node.domain
1206
1207
1208
1209
1210
1211
1212
            }
        return [
            ("/etc/synnefo/cms.conf", r1, {}),
            ("/tmp/sites.json", r2, {}),
            ("/tmp/page.json", {}, {}),
            ]

1213
    @base.run_cmds
1214
1215
1216
1217
1218
1219
1220
    def initialize(self):
        return [
            "snf-manage syncdb",
            "snf-manage migrate --delete-ghost-migrations",
            "snf-manage loaddata /tmp/sites.json",