Commit 69bc52cd authored by Ioannis Tsafaras's avatar Ioannis Tsafaras

Fix various ansible task variables, some templates

parent ee9f3ae5
--- ---
- hosts: master - hosts: master
user: root user: root
gather_facts: yes
roles: roles:
- ../roles/apache-flink - ../roles/apache-flink
--- ---
- hosts: master
user: root
roles:
- ../roles/proxy
- hosts: all - hosts: all
user: root user: root
roles: roles:
...@@ -6,4 +12,3 @@ ...@@ -6,4 +12,3 @@
- ../roles/apache-hadoop - ../roles/apache-hadoop
- ../roles/apache-kafka - ../roles/apache-kafka
- ../roles/apache-flink - ../roles/apache-flink
...@@ -2,7 +2,8 @@ ...@@ -2,7 +2,8 @@
- name: Wait for port 22 to be ready - name: Wait for port 22 to be ready
hosts: master hosts: master
gather_facts: no
sudo: false
tasks: tasks:
- local_action: wait_for port=22 host="{{ inventory_hostname }}" search_regex=OpenSSH delay=5 - local_action: wait_for port=22 host="{{ inventory_hostname }}" search_regex=OpenSSH delay=15
tags: master tags: master
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
<configuration> <configuration>
<property> <property>
<name>fs.default.name</name> <name>fs.default.name</name>
<value>hdfs://{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:9000</value> <value>hdfs://{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:9000</value>
</property> </property>
<property> <property>
<name>hadoop.tmp.dir</name> <name>hadoop.tmp.dir</name>
......
...@@ -15,14 +15,14 @@ ...@@ -15,14 +15,14 @@
<configuration> <configuration>
<property> <property>
<name>yarn.resourcemanager.resource-tracker.address</name> <name>yarn.resourcemanager.resource-tracker.address</name>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8025</value> <value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:8025</value>
</property> </property>
<property> <property>
<name>yarn.resourcemanager.scheduler.address</name> <name>yarn.resourcemanager.scheduler.address</name>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8030</value> <value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:8030</value>
</property> </property>
<property> <property>
<name>yarn.resourcemanager.address</name> <name>yarn.resourcemanager.address</name>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8050</value> <value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:8050</value>
</property> </property>
</configuration> </configuration>
--- ---
- name: create topics - name: create topics
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master-node']['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic input" shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars[groups['master'][0]]['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic input"
notify: notify:
- create batch output topic - create batch output topic
- name: create batch output topic - name: create batch output topic
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master-node']['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic batch-output" shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars[groups['master'][0]]['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic batch-output"
notify: notify:
- create stream output topic - create stream output topic
- name: create stream output topic - name: create stream output topic
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master-node']['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic stream-output" shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars[groups['master'][0]]['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic stream-output"
--- ---
- name: Set hostname
hostname: name={{ inventory_hostname | replace(".vm.okeanos.grnet.gr",".local") }}
when: "'slaves' in group_names"
- name: Copy hosts file. - name: Copy hosts file.
template: src=hosts.j2 dest=/etc/hosts backup=no owner=root group=lambda mode=0750 template: src=hosts.j2 dest=/etc/hosts backup=no owner=root group=lambda mode=0750
...@@ -9,12 +13,14 @@ ...@@ -9,12 +13,14 @@
- name: Install the latest Java 7. - name: Install the latest Java 7.
apt: name=openjdk-7-jdk state=latest install_recommends=no update_cache=yes apt: name=openjdk-7-jdk state=latest install_recommends=no update_cache=yes
environment: proxy_env
- name: Copy environment file. - name: Copy environment file.
template: src=environment.j2 dest=/etc/environment backup=no owner=root group=lambda mode=0750 template: src=environment.j2 dest=/etc/environment backup=no owner=root group=lambda mode=0750
- name: Install sudo. - name: Install sudo.
apt: name=sudo state=latest apt: name=sudo state=latest
environment: proxy_env
- name: Add hduser to sudo group. - name: Add hduser to sudo group.
user: name=hduser group=sudo user: name=hduser group=sudo
......
{% for master in groups["master"] %} {% for master in groups["master"] %}
Host {{ master }} Host {{ master | replace(".vm.okeanos.grnet.gr","*") }}
StrictHostKeyChecking no StrictHostKeyChecking no
{% endfor %} {% endfor %}
......
--- ---
- name: Copy hosts file.
template: src=hosts.j2 dest=/etc/hosts backup=no owner=root mode=0750
- name: Install squid http proxy - name: Install squid http proxy
apt: name=squid3 state=latest apt: name=squid3 state=latest
tags: install tags: install
- name: Configure squid http proxy 1 - name: Configure squid http proxy
lineinfile: destfile="/etc/squid3/squid.conf" regexp="^acl localnet" insertafter="^#acl localnet" lineinfile: destfile="/etc/squid3/squid.conf" regexp="^acl localnet" insertafter="^#acl localnet"
line="acl localnet src {{ hostvars[groups['all'][0]]['local_net'] }}" line="acl localnet src {{ hostvars[groups['all'][0]]['local_net'] }}"
state=present state=present
notify: notify:
- restart squid3 - restart squid3
tags: config tags: config
- name: Configure squid http proxy 2 - name: Configure squid http proxy
lineinfile: destfile="/etc/squid3/squid.conf" regexp="^#http_access allow localnet" lineinfile: destfile="/etc/squid3/squid.conf" regexp="^#http_access allow localnet"
line="http_access allow localnet" line="http_access allow localnet"
state=present state=present
notify: notify:
- restart squid3 - restart squid3
tags: config tags: config
127.0.0.1 localhost
{% for master in groups["master"] %}
{{ hostvars[master]["internal_ip"] }} {{ master | replace(".vm.okeanos.grnet.gr",".local") }}
{% endfor %}
{% for slave in groups["slaves"] %}
{{ hostvars[slave]["internal_ip"] }} {{ slave | replace(".vm.okeanos.grnet.gr",".local") }}
{% endfor %}
# The following lines are desirable for IPv6 capable hosts
::1 localhost ip6-localhost ip6-loopback
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
...@@ -28,7 +28,7 @@ class Manager: ...@@ -28,7 +28,7 @@ class Manager:
% (self.temp_file, self.inventory['master']['name']) % (self.temp_file, self.inventory['master']['name'])
# ansible.constants.DEFAULT_PRIVATE_KEY_FILE = self.temp_file # ansible.constants.DEFAULT_PRIVATE_KEY_FILE = self.temp_file
ansible.constants.HOST_KEY_CHECKING = False ansible.constants.HOST_KEY_CHECKING = False
ansible.constants.DEFAULT_GATHERING = 'explicit' # ansible.constants.DEFAULT_GATHERING = 'explicit'
def create_inventory(self): def create_inventory(self):
""" """
...@@ -109,11 +109,13 @@ if __name__ == "__main__": ...@@ -109,11 +109,13 @@ if __name__ == "__main__":
# manager.run_playbook(playbook_file="../../ansible/playbooks/testproxy.yml", tags=['install']) # manager.run_playbook(playbook_file="../../ansible/playbooks/testproxy.yml", tags=['install'])
manager.run_playbook(playbook_file="../../ansible/playbooks/wait_for_ssh.yml") manager.run_playbook(playbook_file="../../ansible/playbooks/wait_for_ssh.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['master']) # manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['master'])
manager.run_playbook(playbook_file="../../ansible/playbooks/proxy/proxy.yml") # manager.run_playbook(playbook_file="../../ansible/playbooks/proxy/proxy.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['slaves']) # manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['slaves'])
manager.run_playbook(playbook_file="../../ansible/playbooks/apache-hadoop/hadoop-install.yml") # manager.run_playbook(playbook_file="../../ansible/playbooks/apache-hadoop/hadoop-install.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/apache-flink/flink-install.yml") # manager.run_playbook(playbook_file="../../ansible/playbooks/apache-flink/flink-install.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/apache-kafka/kafka-install.yml") # manager.run_playbook(playbook_file="../../ansible/playbooks/apache-kafka/kafka-install.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/cluster/cluster-install.yml")
manager.cleanup() manager.cleanup()
...@@ -4,6 +4,8 @@ from provisioner import Provisioner ...@@ -4,6 +4,8 @@ from provisioner import Provisioner
from ansible_manager import Manager from ansible_manager import Manager
if __name__ == "__main__": if __name__ == "__main__":
start_time = time.time()
parser = argparse.ArgumentParser(description="Okeanos VM provisioning") parser = argparse.ArgumentParser(description="Okeanos VM provisioning")
parser.add_argument('--cloud', type=str, dest="cloud", default="lambda") parser.add_argument('--cloud', type=str, dest="cloud", default="lambda")
parser.add_argument('--project-name', type=str, dest="project_name", parser.add_argument('--project-name', type=str, dest="project_name",
...@@ -11,7 +13,7 @@ if __name__ == "__main__": ...@@ -11,7 +13,7 @@ if __name__ == "__main__":
parser.add_argument('--name', type=str, dest='name', default="to mikro debian sto livadi") parser.add_argument('--name', type=str, dest='name', default="to mikro debian sto livadi")
parser.add_argument('--slaves', type=int, dest='slaves', default=1) parser.add_argument('--slaves', type=int, dest='slaves', default=3)
parser.add_argument('--vcpus_master', type=int, dest='vcpus_master', default=4) parser.add_argument('--vcpus_master', type=int, dest='vcpus_master', default=4)
parser.add_argument('--vcpus_slave', type=int, dest='vcpus_slave', default=4) parser.add_argument('--vcpus_slave', type=int, dest='vcpus_slave', default=4)
parser.add_argument('--ram_master', type=int, dest='ram_master', default=4096) # in MB parser.add_argument('--ram_master', type=int, dest='ram_master', default=4096) # in MB
...@@ -47,7 +49,8 @@ if __name__ == "__main__": ...@@ -47,7 +49,8 @@ if __name__ == "__main__":
provisioner_response['nodes']['slaves'][i]['internal_ip'] = slave_ip provisioner_response['nodes']['slaves'][i]['internal_ip'] = slave_ip
provisioner_response['pk'] = provisioner.get_private_key() provisioner_response['pk'] = provisioner.get_private_key()
print 'response = ', provisioner_response print 'response =', provisioner_response
provisioner_time = time.time()
manager = Manager(provisioner_response) manager = Manager(provisioner_response)
manager.create_inventory() manager.create_inventory()
...@@ -55,13 +58,21 @@ if __name__ == "__main__": ...@@ -55,13 +58,21 @@ if __name__ == "__main__":
# manager.run_playbook(playbook_file="../../ansible/playbooks/testproxy.yml", tags=['install']) # manager.run_playbook(playbook_file="../../ansible/playbooks/testproxy.yml", tags=['install'])
manager.run_playbook(playbook_file="../../ansible/playbooks/wait_for_ssh.yml") manager.run_playbook(playbook_file="../../ansible/playbooks/wait_for_ssh.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['master']) # manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['master'])
manager.run_playbook(playbook_file="../../ansible/playbooks/proxy/proxy.yml") # manager.run_playbook(playbook_file="../../ansible/playbooks/proxy/proxy.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['slaves']) # manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['slaves'])
manager.run_playbook(playbook_file="../../ansible/playbooks/apache-hadoop/hadoop-install.yml") # manager.run_playbook(playbook_file="../../ansible/playbooks/apache-hadoop/hadoop-install.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/apache-flink/flink-install.yml") # manager.run_playbook(playbook_file="../../ansible/playbooks/apache-flink/flink-install.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/apache-kafka/kafka-install.yml") # manager.run_playbook(playbook_file="../../ansible/playbooks/apache-kafka/kafka-install.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/cluster/cluster-install.yml")
# INSERT PLAYBOOKS HERE # INSERT PLAYBOOKS HERE
manager.cleanup() manager.cleanup()
provisioner_duration = provisioner_time - start_time
ansible_duration = time.time() - provisioner_time
print 'VM provisioning took', provisioner_duration, 'seconds'
print 'Ansible playbooks took', ansible_duration, 'seconds'
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment