components.py 48.4 KB
Newer Older
Vangelis Koukis's avatar
Vangelis Koukis committed
1
# Copyright (C) 2010-2014 GRNET S.A.
2
#
Vangelis Koukis's avatar
Vangelis Koukis committed
3
4
5
6
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
7
#
Vangelis Koukis's avatar
Vangelis Koukis committed
8
9
10
11
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
12
#
Vangelis Koukis's avatar
Vangelis Koukis committed
13
14
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
15

16
import re
17
import datetime
18
import simplejson
19
20
21
22
23
24
import copy
from snfdeploy import base
from snfdeploy import config
from snfdeploy import constants
from snfdeploy import context
from snfdeploy.lib import FQDN
25
26


27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
#
# Helper decorators that wrap methods of Component
# class and update its execution context (self, self.ctx, context):
#
def parse_user_info(fn):
    """ Parse the output of DB.get_user_info_from_db()

    For the given user email found in config,
    update user_id, user_auth_token and user_uuid attributes of context.

    """
    def wrapper(*args, **kwargs):
        user_email = config.user_email
        result = fn(*args, **kwargs)
        r = re.compile(r"(\d+)[ |]*(\S+)[ |]*(\S+)[ |]*" + user_email, re.M)
        match = r.search(result)
        if config.dry_run:
            context.user_id, context.user_auth_token, context.user_uuid = \
                ("dummy_uid", "dummy_user_auth_token", "dummy_user_uuid")
        elif match:
            context.user_id, context.user_auth_token, context.user_uuid = \
                match.groups()
        else:
            raise BaseException("Cannot parse info for user %s" % user_email)
        return result
    return wrapper


def parse_service_info(fn):
    """ Parse the output of Astakos.get_services()

    For the given service (found in ctx.admin_service)
    updates service_id and service_token attributes of context.

    """
    def wrapper(*args, **kwargs):
        cl = args[0]
        service = cl.ctx.admin_service
        result = fn(*args, **kwargs)
        r = re.compile(r"(\d+)[ ]*%s[ ]*(\S+)" % service, re.M)
        match = r.search(result)
        if config.dry_run:
            context.service_id, context.service_token = \
                ("dummy_service_id", "dummy_service_token")
        elif match:
            context.service_id, context.service_token = \
                match.groups()
        else:
            raise BaseException("Cannot parse info for service %s" % service)
        return result
    return wrapper


def parse_backend_info(fn):
    """ Parse the output of Cyclades.list_backends()

    For the given cluster (found in ctx.admin_cluster)
    updates the backend_id attributes of context.

    """
    def wrapper(*args, **kwargs):
        cl = args[0]
        fqdn = cl.ctx.admin_cluster.fqdn
        result = fn(*args, **kwargs)
        r = re.compile(r"(\d+)[ ]*%s.*" % fqdn, re.M)
        match = r.search(result)
        if config.dry_run:
            context.backend_id = "dummy_backend_id"
        elif match:
            context.backend_id, = match.groups()
        else:
            raise BaseException("Cannot parse info for backend %s" % fqdn)
        return result
    return wrapper


def update_admin(fn):
    """ Initializes the admin roles for each component

    Initialize the admin roles (NS, Astakos, Cyclades, etc.) and make them
    available under self.NS, self.ASTAKOS, etc. These have the same execution
    context of the current components besides the target node which gets
    derived from the corresponding config.

    """
    def wrapper(*args, **kwargs):
        """If used as decorator of a class method first argument is self."""
        cl = args[0]
        ctx = copy.deepcopy(cl.ctx)
        ctx.admin_service = cl.service
        ctx.admin_cluster = cl.cluster
        ctx.admin_node = cl.node
        ctx.admin_fqdn = cl.fqdn
        cl.NS = NS(node=ctx.ns.node, ctx=ctx)
        cl.NFS = NFS(node=ctx.nfs.node, ctx=ctx)
        cl.DB = DB(node=ctx.db.node, ctx=ctx)
        cl.ASTAKOS = Astakos(node=ctx.astakos.node, ctx=ctx)
        cl.CYCLADES = Cyclades(node=ctx.cyclades.node, ctx=ctx)
        return fn(*args, **kwargs)
    return wrapper


def update_cluster_admin(fn):
    """ Initializes the cluster admin roles for each component

    Finds the master role for the corresponding cluster

    """
    def wrapper(*args, **kwargs):
        """If used as decorator of a class method first argument is self."""
        cl = args[0]
        ctx = copy.deepcopy(cl.ctx)
        ctx.admin_cluster = cl.cluster
        cl.MASTER = Master(node=ctx.master.node, ctx=ctx)
        return fn(*args, **kwargs)
    return wrapper


def export_and_import_service(fn):
    """ Export and import synnefo service

    Used in Astakos, Pithos, and Cyclades service admin_post method

    """
    def wrapper(*args, **kwargs):
        cl = args[0]
        f = config.jsonfile
        cl.export_service()
        cl.get(f, f + ".local")
        cl.ASTAKOS.put(f + ".local", f)
        cl.ASTAKOS.import_service()
        return fn(*args, **kwargs)
    return wrapper


# ########################## Components ############################

# A Component() gets initialized with an execution context that is a
# configuration snapshot of the target setup, cluster and node. A
# component implements the following helper methods: check, install,
# prepare, configure, restart, initialize, and test. All those methods
# will be executed on the target node with this order during setup.
#
# Additionally each Component class implements admin_pre, and
# admin_post methods which invoke actions on different components on
# the same execution context before and after installation. For
# example before a backend gets installed, its FQDN must resolve to
# the master floating IP, so we have to run some actions on the ns
# node and after installation we must add it to cyclades (snf-manage
# backend-add on the cyclades node).
#
# Component() inherits ComponentRunner() which practically exports the
# setup() method. This will first check if the required components are
# installed, will install them if not and update the status of target
# node.
#
# ComponentRunner() inherits FabricRunner() which practically wraps
# basic fabric commands (put, get, run) with the correct execution
# environment.
#
# Each component gets initialized with an execution context and uses
# the config module for accessing global wide options. The context
# provides node, cluster, and setup related info.
190

191
192
class HW(base.Component):
    @base.run_cmds
193
194
    def test(self):
        return [
195
            "ping -c 1 %s" % self.node.ip,
196
197
198
199
            "ping -c 1 www.google.com",
            "apt-get update",
            ]

200

201
202
class SSH(base.Component):
    @base.run_cmds
203
204
205
206
207
208
209
    def prepare(self):
        return [
            "mkdir -p /root/.ssh",
            "for f in $(ls /root/.ssh/*); do cp $f $f.bak ; done",
            "echo StrictHostKeyChecking no >> /etc/ssh/ssh_config",
            ]

210
    def _configure(self):
211
212
213
        files = [
            "authorized_keys", "id_dsa", "id_dsa.pub", "id_rsa", "id_rsa.pub"
            ]
214
        ssh = [("/root/.ssh/%s" % f, {}, {"mode": 0600}) for f in files]
215
216
        return ssh

217
    @base.run_cmds
218
219
220
    def initialize(self):
        f = "/root/.ssh/authorized_keys"
        return [
221
            "test -e {0}.bak && cat {0}.bak >> {0} || true".format(f)
222
223
            ]

224
    @base.run_cmds
225
    def test(self):
226
227
        return ["ssh %s date" % self.node.ip]

228

229
230
231
232
class DNS(base.Component):
    @update_admin
    def admin_pre(self):
        self.NS.update_ns()
233

234
    @base.run_cmds
235
236
237
238
    def prepare(self):
        return [
            "chattr -i /etc/resolv.conf",
            "sed -i 's/^127.*$/127.0.0.1 localhost/g' /etc/hosts",
239
240
            "echo %s > /etc/hostname" % self.node.hostname,
            "hostname %s" % self.node.hostname
241
242
            ]

243
    def _configure(self):
244
245
        r1 = {
            "date": str(datetime.datetime.today()),
246
247
            "domain": self.node.domain,
            "ns_node_ip": self.ctx.ns.ip,
248
249
250
251
252
253
            }
        resolv = [
            ("/etc/resolv.conf", r1, {})
            ]
        return resolv

254
    @base.run_cmds
255
256
257
258
    def initialize(self):
        return ["chattr +i /etc/resolv.conf"]


259
class DDNS(base.Component):
260
261
262
263
    REQUIRED_PACKAGES = [
        "dnsutils",
        ]

264
    @base.run_cmds
265
266
267
268
269
    def prepare(self):
        return [
            "mkdir -p /root/ddns/"
            ]

270
    def _configure(self):
271
        return [
272
            ("/root/ddns/" + k, {}, {}) for k in config.ddns_keys
273
274
275
            ]


276
class NS(base.Component):
277
278
279
280
    REQUIRED_PACKAGES = [
        "bind9",
        ]

281
282
283
284
285
286
    alias = constants.NS

    def required_components(self):
        return [HW, SSH, DDNS]

    def _nsupdate(self, cmd):
287
288
289
290
291
292
        ret = """
nsupdate -k {0} > /dev/null <<EOF || true
server {1}
{2}
send
EOF
293
""".format(config.ddns_private_key, self.ctx.ns.ip, cmd)
294
295
        return ret

296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
    @base.run_cmds
    def update_ns(self, info=None):
        if not info:
            info = self.ctx.admin_fqdn
        return [
            self._nsupdate("update add %s" % info.arecord),
            self._nsupdate("update add %s" % info.ptrrecord),
            self._nsupdate("update add %s" % info.cnamerecord),
            ]

    def add_qa_instances(self):
        instances = [
            ("xen-test-inst1", "1.2.3.4"),
            ("xen-test-inst2", "1.2.3.5"),
            ("xen-test-inst3", "1.2.3.6"),
            ("xen-test-inst4", "1.2.3.7"),
            ]
        for name, ip in instances:
            info = {
                "name": name,
                "ip": ip,
                "domain": self.node.domain
                }
            node_info = FQDN(**info)
            self.update_ns(node_info)

    @base.run_cmds
323
324
325
326
327
328
329
330
    def prepare(self):
        return [
            "mkdir -p /etc/bind/zones",
            "chmod g+w /etc/bind/zones",
            "mkdir -p /etc/bind/rev",
            "chmod g+w /etc/bind/rev",
            ]

331
332
333
    def _configure(self):
        d = self.node.domain
        ip = self.node.ip
334
335
336
337
338
339
340
341
342
343
344
        return [
            ("/etc/bind/named.conf.local", {"domain": d}, {}),
            ("/etc/bind/zones/example.com",
             {"domain": d, "ns_node_ip": ip},
             {"remote": "/etc/bind/zones/%s" % d}),
            ("/etc/bind/zones/vm.example.com",
             {"domain": d, "ns_node_ip": ip},
             {"remote": "/etc/bind/zones/vm.%s" % d}),
            ("/etc/bind/rev/synnefo.in-addr.arpa.zone", {"domain": d}, {}),
            ("/etc/bind/rev/synnefo.ip6.arpa.zone", {"domain": d}, {}),
            ("/etc/bind/named.conf.options",
345
             {"node_ips": ";".join(config.all_ips)}, {}),
346
347
348
            ("/root/ddns/ddns.key", {}, {"remote": "/etc/bind/ddns.key"}),
            ]

349
    @base.run_cmds
350
351
352
353
    def restart(self):
        return ["/etc/init.d/bind9 restart"]


354
class APT(base.Component):
355
356
357
    """ Setup apt repos and check fqdns """
    REQUIRED_PACKAGES = ["curl"]

358
    @base.run_cmds
359
360
361
    def prepare(self):
        return [
            "echo 'APT::Install-Suggests \"false\";' >> /etc/apt/apt.conf",
362
363
            "curl -k https://dev.grnet.gr/files/apt-grnetdev.pub | \
                apt-key add -",
364
365
            ]

366
    def _configure(self):
367
368
369
370
        return [
            ("/etc/apt/sources.list.d/synnefo.wheezy.list", {}, {})
            ]

371
    @base.run_cmds
372
373
374
375
376
377
    def initialize(self):
        return [
            "apt-get update",
            ]


378
class MQ(base.Component):
379
380
    REQUIRED_PACKAGES = ["rabbitmq-server"]

381
382
383
384
385
386
387
388
389
390
    alias = constants.MQ

    def required_components(self):
        return [HW, SSH, DNS, APT]

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()

    @base.run_cmds
391
    def check(self):
392
        return ["ping -c 1 %s" % self.node.cname]
393

394
    @base.run_cmds
395
    def initialize(self):
396
397
        u = config.synnefo_user
        p = config.synnefo_rabbitmq_passwd
398
399
400
401
402
403
404
405
        return [
            "rabbitmqctl add_user %s %s" % (u, p),
            "rabbitmqctl set_permissions %s \".*\" \".*\" \".*\"" % u,
            "rabbitmqctl delete_user guest",
            "rabbitmqctl set_user_tags %s administrator" % u,
            ]


406
class DB(base.Component):
407
408
    REQUIRED_PACKAGES = ["postgresql"]

409
410
411
412
413
414
415
416
417
418
    alias = constants.DB

    def required_components(self):
        return [HW, SSH, DNS, APT]

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()

    @base.run_cmds
419
    def check(self):
420
        return ["ping -c 1 %s" % self.node.cname]
421

422
423
    @parse_user_info
    @base.run_cmds
424
425
426
427
428
429
430
431
    def get_user_info_from_db(self):
        cmd = """
cat > /tmp/psqlcmd <<EOF
select id, auth_token, uuid, email from auth_user, im_astakosuser \
where auth_user.id = im_astakosuser.user_ptr_id and auth_user.email = '{0}';
EOF

su - postgres -c  "psql -w -d snf_apps -f /tmp/psqlcmd"
432
""".format(config.user_email)
433
434
435

        return [cmd]

436
437
438
439
440
    @base.run_cmds
    def allow_db_access(self):
        user = "all"
        method = "md5"
        ip = self.ctx.admin_node.ip
441
442
        f = "/etc/postgresql/*/main/pg_hba.conf"
        cmd1 = "echo host all %s %s/32 %s >> %s" % \
443
            (user, ip, method, f)
444
        cmd2 = "sed -i 's/\(host.*127.0.0.1.*\)md5/\\1trust/' %s" % f
445
        return [cmd1, cmd2]
446

447
448
449
    def _configure(self):
        u = config.synnefo_user
        p = config.synnefo_db_passwd
450
451
452
453
454
        replace = {"synnefo_user": u, "synnefo_db_passwd": p}
        return [
            ("/tmp/db-init.psql", replace, {}),
            ]

455
    @base.check_if_testing
456
457
458
459
460
    def make_db_fast(self):
        f = "/etc/postgresql/*/main/postgresql.conf"
        opts = "fsync=off\nsynchronous_commit=off\nfull_page_writes=off\n"
        return ["""echo -e "%s" >> %s""" % (opts, f)]

461
    @base.run_cmds
462
463
    def prepare(self):
        f = "/etc/postgresql/*/main/postgresql.conf"
464
465
        ret = ["""echo "listen_addresses = '*'" >> %s""" % f]
        return ret + self.make_db_fast()
466

467
    @base.run_cmds
468
469
470
471
472
    def initialize(self):
        script = "/tmp/db-init.psql"
        cmd = "su - postgres -c \"psql -w -f %s\" " % script
        return [cmd]

473
    @base.run_cmds
474
475
476
    def restart(self):
        return ["/etc/init.d/postgresql restart"]

477
    @base.run_cmds
478
479
480
481
482
483
484
    def destroy_db(self):
        return [
            """su - postgres -c ' psql -w -c "drop database snf_apps" '""",
            """su - postgres -c ' psql -w -c "drop database snf_pithos" '"""
            ]


485
486
487
488
class VMC(base.Component):

    def extra_components(self):
        if self.cluster.synnefo:
489
490
491
492
            return [
                Image, GTools, GanetiCollectd,
                PithosBackend, Archip, ArchipGaneti
                ]
493
        else:
494
            return [ExtStorage, Archip, ArchipGaneti]
495

496
497
498
499
500
501
502
503
504
505
506
    def required_components(self):
        return [
            HW, SSH, DNS, DDNS, APT, Mount, Ganeti, Network,
            ] + self.extra_components()

    @update_cluster_admin
    def admin_post(self):
        self.MASTER.add_node(self.node)


class Ganeti(base.Component):
507
508
509
510
511
512
513
514
515
516
    REQUIRED_PACKAGES = [
        "qemu-kvm",
        "python-bitarray",
        "ganeti-htools",
        "ganeti-haskell",
        "snf-ganeti",
        "ganeti2",
        "bridge-utils",
        "lvm2",
        "drbd8-utils",
517
        "ganeti-instance-debootstrap",
518
519
        ]

520
521
522
523
524
    @update_admin
    def admin_pre(self):
        self.NS.update_ns()

    @base.run_cmds
525
526
    def check(self):
        commands = [
527
528
            "getent hosts %s | grep -v ^127" % self.node.hostname,
            "hostname -f | grep %s" % self.node.fqdn,
529
530
531
            ]
        return commands

532
    def _configure(self):
533
534
535
536
        return [
            ("/etc/ganeti/file-storage-paths", {}, {}),
            ]

537
    def _prepare_lvm(self):
538
        ret = []
539
        disk = self.node.extra_disk
540
541
542
543
        if disk:
            ret = [
                "test -e %s" % disk,
                "pvcreate %s" % disk,
544
                "vgcreate %s %s" % (self.cluster.vg, disk)
545
546
                ]
        return ret
547

548
549
    def _prepare_net_infra(self):
        br = config.common_bridge
550
551
552
553
        return [
            "brctl addbr {0}; ip link set {0} up".format(br)
            ]

554
    @base.run_cmds
555
556
557
558
    def prepare(self):
        return [
            "mkdir -p /srv/ganeti/file-storage/",
            "sed -i 's/^127.*$/127.0.0.1 localhost/g' /etc/hosts"
559
            ] + self._prepare_net_infra() + self._prepare_lvm()
560

561
    @base.run_cmds
562
    def restart(self):
563
        return ["/etc/init.d/ganeti restart"]
564
565


566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
class Master(base.Component):

    @property
    def fqdn(self):
        return self.cluster

    def required_components(self):
        return [
            HW, SSH, DNS, DDNS, APT, Mount, Ganeti
            ]

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()

    @base.run_cmds
    def check(self):
        commands = [
            "host %s" % self.cluster.fqdn,
            ]
        return commands

    @base.run_cmds
589
590
591
592
    def add_qa_rapi_user(self):
        cmd = """
echo ganeti-qa qa_example_passwd write >> /var/lib/ganeti/rapi/users
"""
593
        return [cmd]
594

595
596
597
    def _add_rapi_user(self):
        user = config.synnefo_user
        passwd = config.synnefo_rapi_passwd
598
        x = "%s:Ganeti Remote API:%s" % (user, passwd)
599

600
601
602
603
        cmd = """
cat >> /var/lib/ganeti/rapi/users <<EOF
%s {HA1}$(echo -n %s | openssl md5 | sed 's/^.* //') write
EOF
604
""" % (user, x)
605

606
        return [cmd]
607

608
609
    @base.run_cmds
    def add_node(self, info):
610
        commands = [
611
            "gnt-node list " + info.fqdn + " || " +
612
            "gnt-node add --no-ssh-key-check --master-capable=yes " +
613
            "--vm-capable=yes " + info.fqdn,
614
615
616
            ]
        return commands

617
618
    def _try_use_vg(self):
        vg = self.cluster.vg
619
620
621
622
623
624
        return [
            "gnt-cluster modify --vg-name=%s || true" % vg,
            "gnt-cluster modify --disk-parameters=drbd:metavg=%s" % vg,
            "gnt-group modify --disk-parameters=drbd:metavg=%s default" % vg,
            ]

625
    @base.run_cmds
626
    def initialize(self):
627
628
        std = "cpu-count=1,disk-count=1,disk-size=1024"
        std += ",memory-size=128,nic-count=1,spindle-use=1"
629

630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
        bound_min = "cpu-count=1,disk-count=1,disk-size=512"
        bound_min += ",memory-size=128,nic-count=0,spindle-use=1"

        bound_max = "cpu-count=8,disk-count=16,disk-size=1048576"
        bound_max += ",memory-size=32768,nic-count=8,spindle-use=12"
        cmd = """
gnt-cluster init --enabled-hypervisors=kvm \
    --no-lvm-storage --no-drbd-storage \
    --nic-parameters link={0},mode=bridged \
    --master-netdev {1} \
    --default-iallocator hail \
    --hypervisor-parameters kvm:kernel_path=,vnc_bind_address=0.0.0.0 \
    --no-ssh-init --no-etc-hosts \
    --enabled-disk-templates file,plain,ext,drbd \
    --ipolicy-std-specs {2} \
    --ipolicy-bounds-specs min:{3}/max:{4} \
    {5}
        """.format(config.common_bridge, self.cluster.netdev,
                   std, bound_min, bound_max, self.cluster.fqdn)

        return [cmd] + self._try_use_vg() + self._add_rapi_user()

    @base.run_cmds
653
654
655
    def restart(self):
        return ["/etc/init.d/ganeti restart"]

656
657
658
659
660
661
662
663
664
    @update_admin
    @update_cluster_admin
    def admin_post(self):
        if self.cluster.synnefo:
            self.CYCLADES._debug("Adding backend: %s" % self.cluster.fqdn)
            self.CYCLADES.add_backend()
            self.CYCLADES.list_backends()
            self.CYCLADES.undrain_backend()

665

666
class Image(base.Component):
667
668
669
670
    REQUIRED_PACKAGES = [
        "snf-image",
        ]

671
    @base.run_cmds
672
    def check(self):
673
        return ["mkdir -p %s" % config.image_dir]
674

675
    @base.run_cmds
676
    def prepare(self):
677
678
        url = config.debian_base_url
        d = config.image_dir
679
680
681
682
683
        image = "debian_base.diskdump"
        return [
            "test -e %s/%s || wget %s -O %s/%s" % (d, image, url, d, image)
            ]

684
    def _configure(self):
685
686
        tmpl = "/etc/default/snf-image"
        replace = {
687
688
689
            "synnefo_user": config.synnefo_user,
            "synnefo_db_passwd": config.synnefo_db_passwd,
            "pithos_dir": config.pithos_dir,
690
            "db_node": self.ctx.db.cname,
691
            "image_dir": config.image_dir,
692
693
694
            }
        return [(tmpl, replace, {})]

695
    @base.run_cmds
696
697
698
699
    def initialize(self):
        return ["snf-image-update-helper -y"]


700
class GTools(base.Component):
701
702
703
704
    REQUIRED_PACKAGES = [
        "snf-cyclades-gtools",
        ]

705
    @base.run_cmds
706
    def check(self):
707
        return ["ping -c1 %s" % self.ctx.mq.cname]
708

709
    @base.run_cmds
710
711
712
713
714
    def prepare(self):
        return [
            "sed -i 's/false/true/' /etc/default/snf-ganeti-eventd",
            ]

715
    def _configure(self):
716
717
        tmpl = "/etc/synnefo/gtools.conf"
        replace = {
718
719
            "synnefo_user": config.synnefo_user,
            "synnefo_rabbitmq_passwd": config.synnefo_rabbitmq_passwd,
720
            "mq_node": self.ctx.mq.cname,
721
722
723
            }
        return [(tmpl, replace, {})]

724
    @base.run_cmds
725
726
727
728
    def restart(self):
        return ["/etc/init.d/snf-ganeti-eventd restart"]


729
class Network(base.Component):
730
731
732
733
734
735
    REQUIRED_PACKAGES = [
        "python-nfqueue",
        "snf-network",
        "nfdhcpd",
        ]

736
    def _configure(self):
737
        r1 = {
738
            "ns_node_ip": self.ctx.ns.ip
739
740
            }
        r2 = {
741
742
743
744
745
746
            "common_bridge": config.common_bridge,
            "public_iface": self.node.public_iface,
            "subnet": config.synnefo_public_network_subnet,
            "gateway": config.synnefo_public_network_gateway,
            "router_ip": self.ctx.router.ip,
            "node_ip": self.node.ip,
747
748
            }
        r3 = {
749
750
751
            "domain": self.node.domain,
            "server": self.ctx.ns.ip,
            "keyfile": config.ddns_private_key,
752
753
754
755
756
757
758
759
            }

        return [
            ("/etc/nfdhcpd/nfdhcpd.conf", r1, {}),
            ("/etc/rc.local", r2, {"mode": 0755}),
            ("/etc/default/snf-network", r3, {}),
            ]

760
    @base.run_cmds
761
762
763
    def initialize(self):
        return ["/etc/init.d/rc.local start"]

764
    @base.run_cmds
765
766
767
768
    def restart(self):
        return ["/etc/init.d/nfdhcpd restart"]


769
class Apache(base.Component):
770
771
772
773
    REQUIRED_PACKAGES = [
        "apache2",
        ]

774
    @base.run_cmds
775
776
777
778
779
780
781
    def prepare(self):
        return [
            "a2enmod ssl", "a2enmod rewrite", "a2dissite default",
            "a2enmod headers",
            "a2enmod proxy_http", "a2dismod autoindex",
            ]

782
783
    def _configure(self):
        r1 = {"HOST": self.node.fqdn}
784
785
786
787
788
        return [
            ("/etc/apache2/sites-available/synnefo", r1, {}),
            ("/etc/apache2/sites-available/synnefo-ssl", r1, {}),
            ]

789
    @base.run_cmds
790
791
792
793
794
    def initialize(self):
        return [
            "a2ensite synnefo", "a2ensite synnefo-ssl",
            ]

795
    @base.run_cmds
796
797
    def restart(self):
        return [
798
799
            "/etc/init.d/apache2 restart",
            ]
800
801


802
class Gunicorn(base.Component):
803
804
805
806
    REQUIRED_PACKAGES = [
        "gunicorn",
        ]

807
    @base.run_cmds
808
809
810
811
812
    def prepare(self):
        return [
            "chown root.www-data /var/log/gunicorn",
            ]

813
814
    def _configure(self):
        r1 = {"HOST": self.node.fqdn}
815
816
817
818
        return [
            ("/etc/gunicorn.d/synnefo", r1, {}),
            ]

819
    @base.run_cmds
820
821
    def restart(self):
        return [
822
823
            "/etc/init.d/gunicorn restart",
            ]
824
825


826
class Common(base.Component):
827
828
829
830
831
832
833
834
835
    REQUIRED_PACKAGES = [
        # snf-common
        "python-objpool",
        "snf-common",
        "python-astakosclient",
        "snf-django-lib",
        "snf-branding",
        ]

836
    def _configure(self):
837
        r1 = {
838
839
840
841
            "EMAIL_SUBJECT_PREFIX": self.node.hostname,
            "domain": self.node.domain,
            "HOST": self.node.fqdn,
            "MAIL_DIR": config.mail_dir,
842
843
844
845
846
            }
        return [
            ("/etc/synnefo/common.conf", r1, {}),
            ]

847
    @base.run_cmds
848
    def initialize(self):
849
        return ["mkdir -p {0}; chmod 777 {0}".format(config.mail_dir)]
850

851
    @base.run_cmds
852
853
    def restart(self):
        return [
854
855
            "/etc/init.d/gunicorn restart",
            ]
856
857


858
class WEB(base.Component):
859
860
861
862
863
864
865
    REQUIRED_PACKAGES = [
        "snf-webproject",
        "python-psycopg2",
        "python-gevent",
        "python-django",
        ]

866
    @base.run_cmds
867
    def check(self):
868
        return ["ping -c1 %s" % self.ctx.db.cname]
869

870
    def _configure(self):
871
        r1 = {
872
873
            "synnefo_user": config.synnefo_user,
            "synnefo_db_passwd": config.synnefo_db_passwd,
874
            "db_node": self.ctx.db.cname,
875
            "domain": self.node.domain,
876
877
878
879
880
            }
        return [
            ("/etc/synnefo/webproject.conf", r1, {}),
            ]

881
    @base.run_cmds
882
883
    def restart(self):
        return [
884
885
            "/etc/init.d/gunicorn restart",
            ]
886
887


888
class Astakos(base.Component):
889
890
891
892
893
894
    REQUIRED_PACKAGES = [
        "python-django-south",
        "snf-astakos-app",
        "kamaki",
        ]

895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
    alias = constants.ASTAKOS

    def required_components(self):
        return [HW, SSH, DNS, APT, Apache, Gunicorn, Common, WEB]

    @base.run_cmds
    def setup_user(self):
        self._debug("Setting up user")
        return self._set_default_quota() + \
            self._add_user() + self._activate_user()

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()
        self.DB.allow_db_access()
        self.DB.restart()

    @property
    def conflicts(self):
        return [CMS]

    @base.run_cmds
917
    def export_service(self):
918
        f = config.jsonfile
919
920
921
922
        return [
            "snf-manage service-export-astakos > %s" % f
            ]

923
    @base.run_cmds
924
    def import_service(self):
925
        f = config.jsonfile
926
927
928
929
        return [
            "snf-manage service-import --json=%s" % f
            ]

930
931
    @base.run_cmds
    def set_astakos_default_quota(self):
932
933
934
        cmd = "snf-manage resource-modify"
        return [
            "%s --base-default 2 astakos.pending_app" % cmd,
935
936
937
938
939
940
941
            "%s --project-default 0 astakos.pending_app" % cmd,
            ]

    @base.run_cmds
    def set_cyclades_default_quota(self):
        cmd = "snf-manage resource-modify"
        return [
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
            "%s --base-default 4 cyclades.vm" % cmd,
            "%s --base-default 40G cyclades.disk" % cmd,
            "%s --base-default 16G cyclades.total_ram" % cmd,
            "%s --base-default 8G cyclades.ram" % cmd,
            "%s --base-default 32 cyclades.total_cpu" % cmd,
            "%s --base-default 16 cyclades.cpu" % cmd,
            "%s --base-default 4 cyclades.network.private" % cmd,
            "%s --base-default 4 cyclades.floating_ip" % cmd,
            "%s --project-default 0 cyclades.vm" % cmd,
            "%s --project-default 0 cyclades.disk" % cmd,
            "%s --project-default inf cyclades.total_ram" % cmd,
            "%s --project-default 0 cyclades.ram" % cmd,
            "%s --project-default inf cyclades.total_cpu" % cmd,
            "%s --project-default 0 cyclades.cpu" % cmd,
            "%s --project-default 0 cyclades.network.private" % cmd,
            "%s --project-default 0 cyclades.floating_ip" % cmd,
958
959
            ]

960
961
962
963
964
965
966
967
968
    @base.run_cmds
    def set_pithos_default_quota(self):
        cmd = "snf-manage resource-modify"
        return [
            "%s --base-default 40G pithos.diskspace" % cmd,
            "%s --project-default 0 pithos.diskspace" % cmd,
            ]

    @base.run_cmds
969
    def modify_all_quota(self):
970
971
972
973
974
975
976
977
978
979
980
981
        cmd = "snf-manage project-modify --all-base-projects --limit"
        return [
            "%s pithos.diskspace 40G 40G" % cmd,
            "%s astakos.pending_app 2 2" % cmd,
            "%s cyclades.vm 4 4" % cmd,
            "%s cyclades.disk 40G 40G" % cmd,
            "%s cyclades.total_ram 16G 16G" % cmd,
            "%s cyclades.ram 8G 8G" % cmd,
            "%s cyclades.total_cpu 32 32" % cmd,
            "%s cyclades.cpu 16 16" % cmd,
            "%s cyclades.network.private 4 4" % cmd,
            "%s cyclades.floating_ip 4 4" % cmd,
982
983
            ]

984
985
    @parse_service_info
    @base.run_cmds
986
987
988
989
990
    def get_services(self):
        return [
            "snf-manage component-list -o id,name,token"
            ]

991
    def _configure(self):
992
        r1 = {
993
            "ACCOUNTS": self.ctx.astakos.cname,
994
            "domain": self.node.domain,
995
996
            "CYCLADES": self.ctx.cyclades.cname,
            "PITHOS": self.ctx.pithos.cname,
997
998
999
1000
1001
            }
        return [
            ("/etc/synnefo/astakos.conf", r1, {})
            ]

1002
    @base.run_cmds
1003
1004
1005
1006
1007
1008
1009
    def initialize(self):
        return [
            "snf-manage syncdb --noinput",
            "snf-manage migrate im --delete-ghost-migrations",
            "snf-manage migrate quotaholder_app",
            "snf-manage migrate oa2",
            "snf-manage loaddata groups",
1010
            ] + self._astakos_oa2() + self._astakos_register_components()
1011

1012
1013
    def _astakos_oa2(self):
        secret = config.oa2_secret
1014
        view = "https://%s/pithos/ui/view" % self.ctx.pithos.cname
1015
1016
1017
1018
1019
        cmd = "snf-manage oauth2-client-add pithos-view \
                  --secret=%s --is-trusted --url %s || true" % (secret, view)
        return [cmd]

    def _astakos_register_components(self):
1020
        # base urls
1021
1022
1023
1024
        cbu = "https://%s/cyclades" % self.ctx.cyclades.cname
        pbu = "https://%s/pithos" % self.ctx.pithos.cname
        abu = "https://%s/astakos" % self.ctx.astakos.cname
        cmsurl = "https://%s/home" % self.ctx.cms.cname
1025
1026
1027
1028
1029
1030
1031
1032
1033

        cmd = "snf-manage component-add"
        h = "%s home --base-url %s --ui-url %s" % (cmd, cmsurl, cmsurl)
        c = "%s cyclades --base-url %s --ui-url %s/ui" % (cmd, cbu, cbu)
        p = "%s pithos --base-url %s --ui-url %s/ui" % (cmd, pbu, pbu)
        a = "%s astakos --base-url %s --ui-url %s/ui" % (cmd, abu, abu)

        return [h, c, p, a]

1034
    @base.run_cmds
1035
1036
    def add_user(self):
        info = (
1037
1038
1039
1040
            config.user_passwd,
            config.user_email,
            config.user_name,
            config.user_lastname,
1041
1042
1043
1044
            )
        cmd = "snf-manage user-add --password %s %s %s %s" % info
        return [cmd]

1045
1046
    @update_admin
    @base.run_cmds
1047
    def activate_user(self):
1048
1049
        self.DB.get_user_info_from_db()
        user_id = context.user_id
1050
1051
1052
1053
1054
        return [
            "snf-manage user-modify --verify %s" % user_id,
            "snf-manage user-modify --accept %s" % user_id,
            ]

1055
1056
1057
1058
1059
    @update_admin
    @export_and_import_service
    def admin_post(self):
        self.set_astakos_default_quota()

1060

1061
class CMS(base.Component):
1062
1063
1064
1065
    REQUIRED_PACKAGES = [
        "snf-cloudcms"
        ]

1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
    alias = constants.CMS

    def required_components(self):
        return [HW, SSH, DNS, APT, Apache, Gunicorn, Common, WEB]

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()
        self.DB.allow_db_access()
        self.DB.restart()

    @property
    def conflicts(self):
        return [Astakos, Pithos, Cyclades]

    def _configure(self):
1082
        r1 = {
1083
            "ACCOUNTS": self.ctx.astakos.cname
1084
1085
            }
        r2 = {
1086
            "DOMAIN": self.node.domain
1087
1088
1089
1090
1091
1092
1093
            }
        return [
            ("/etc/synnefo/cms.conf", r1, {}),
            ("/tmp/sites.json", r2, {}),
            ("/tmp/page.json", {}, {}),
            ]

1094
    @base.run_cmds
1095
1096
1097
1098
1099
1100
1101
    def initialize(self):
        return [
            "snf-manage syncdb",
            "snf-manage migrate --delete-ghost-migrations",
            "snf-manage loaddata /tmp/sites.json",
            "snf-manage loaddata /tmp/page.json",
            "snf-manage createsuperuser --username=admin \
1102
                  --email=admin@%s --noinput" % self.node.domain,
1103
1104
            ]

1105
    @base.run_cmds
1106
1107
1108
1109
    def restart(self):
        return ["/etc/init.d/gunicorn restart"]


1110
class Mount(base.Component):
1111
1112
1113
1114
    REQUIRED_PACKAGES = [
        "nfs-common"
        ]

1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
    @update_admin
    def admin_pre(self):
        self.NFS.update_exports()
        self.NFS.restart()

    @property
    def conflicts(self):
        return [NFS]

    @base.run_cmds
1125
1126
    def prepare(self):
        ret = []
1127
        for d in [config.pithos_dir, config.image_dir, config.archip_dir]:
1128
1129
1130
1131
1132
            ret.append("mkdir -p %s" % d)
            cmd = """
cat >> /etc/fstab <<EOF
{0}:{1} {1}  nfs defaults,rw,noatime,rsize=131072,wsize=131072 0 0
EOF
1133
""".format(self.ctx.nfs.cname, d)
1134
1135
1136
1137
            ret.append(cmd)

        return ret

1138
    @base.run_cmds
1139
1140
    def initialize(self):
        ret = []
1141
        for d in [config.pithos_dir, config.image_dir, config.archip_dir]:
1142
1143
1144
1145
            ret.append("mount %s" % d)
        return ret


1146
class NFS(base.Component):
1147
1148
1149
1150
    REQUIRED_PACKAGES = [
        "nfs-kernel-server"
        ]

1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
    alias = constants.NFS

    def required_components(self):
        return [HW, SSH, DNS, APT]

    @property
    def conflicts(self):
        return [Mount]

    @base.run_cmds
1161
    def prepare(self):
1162
        p = config.pithos_dir
1163
        return [
1164
            "mkdir -p %s" % config.image_dir,
1165
            "mkdir -p %s/data" % p,
1166
1167
            "mkdir -p %s/blocks" % config.archip_dir,
            "mkdir -p %s/maps" % config.archip_dir,
1168
1169
            "chown www-data.www-data %s/data" % p,
            "chmod g+ws %s/data" % p,
1170
            ]
1171

1172
1173
    @base.run_cmds
    def update_exports(self):
1174
        fqdn = self.ctx.admin_node.fqdn
1175
        cmd = """
1176
1177
1178
1179
grep {3} /etc/exports || cat >> /etc/exports <<EOF
{0} {3}(rw,async,no_subtree_check,no_root_squash)
{1} {3}(rw,async,no_subtree_check,no_root_squash)
{2} {3}(rw,async,no_subtree_check,no_root_squash)
1180
EOF
1181
""".format(config.pithos_dir, config.image_dir, config.archip_dir, fqdn)
1182
        return [cmd]
1183

1184
    @base.run_cmds
1185
1186
1187
1188
    def restart(self):
        return ["exportfs -a"]


1189
class Pithos(base.Component):
1190
1191
    REQUIRED_PACKAGES = [
        "kamaki",
1192
        "python-svipc",
1193
1194
1195
1196
        "snf-pithos-app",
        "snf-pithos-webclient",
        ]

1197
1198
1199
1200
    alias = constants.PITHOS
    service = constants.PITHOS

    def required_components(self):
1201
1202
1203
1204
        return [
            HW, SSH, DNS, APT, Apache, Gunicorn, Common, WEB,
            PithosBackend, Archip
            ]
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217

    @update_admin
    def admin_pre(self):
        self.NS.update_ns()
        self.ASTAKOS.get_services()
        self.DB.allow_db_access()
        self.DB.restart()

    @property
    def conflicts(self):
        return [CMS]

    @base.run_cmds
1218
    def export_service(self):
1219
        f = config.jsonfile
1220
1221
1222
1223
        return [
            "snf-manage service-export-pithos > %s" % f
            ]

1224
    def _configure(self):
1225
        r1 = {
1226
1227
1228
            "ACCOUNTS": self.ctx.astakos.cname,
            "PITHOS": self.ctx.pithos.cname,
            "db_node": self.ctx.db.cname,
1229
1230
1231
1232
1233
            "synnefo_user": config.synnefo_user,
            "synnefo_db_passwd": config.synnefo_db_passwd,
            "pithos_dir": config.pithos_dir,
            "PITHOS_SERVICE_TOKEN": context.service_token,
            "oa2_secret": config.oa2_secret,
1234
1235
            }
        r2 = {
1236
            "ACCOUNTS": self.ctx.astakos.cname,
1237
            "PITHOS_UI_CLOUDBAR_ACTIVE_SERVICE": context.service_id,
1238
1239
1240
1241
1242
1243
1244
            }

        return [
            ("/etc/synnefo/pithos.conf", r1, {}),
            ("/etc/synnefo/webclient.conf", r2, {}),
            ]

1245
    @base.run_cmds
1246
1247
1248
    def initialize(self):
        return ["pithos-migrate stamp head"]

1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
    @base.run_cmds
    def restart(self):
        return [
            "/etc/init.d/gunicorn restart",
            ]

    @update_admin
    @export_and_import_service
    def admin_post(self):
        self.ASTAKOS.set_pithos_default_quota()
1259

1260
1261

class PithosBackend(base.Component):
1262
1263
1264
1265
    REQUIRED_PACKAGES = [
        "snf-pithos-backend",
        ]

1266
    def _configure(self):
1267
        r1 = {
1268
            "db_node": self.ctx.db.cname,