Commit 69bc52cd authored by Ioannis Tsafaras's avatar Ioannis Tsafaras

Fix various ansible task variables, some templates

parent ee9f3ae5
---
- hosts: master
user: root
gather_facts: yes
roles:
- ../roles/apache-flink
---
- hosts: master
user: root
roles:
- ../roles/proxy
- hosts: all
user: root
roles:
......@@ -6,4 +12,3 @@
- ../roles/apache-hadoop
- ../roles/apache-kafka
- ../roles/apache-flink
......@@ -2,7 +2,8 @@
- name: Wait for port 22 to be ready
hosts: master
gather_facts: no
sudo: false
tasks:
- local_action: wait_for port=22 host="{{ inventory_hostname }}" search_regex=OpenSSH delay=5
- local_action: wait_for port=22 host="{{ inventory_hostname }}" search_regex=OpenSSH delay=15
tags: master
......@@ -19,7 +19,7 @@
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:9000</value>
<value>hdfs://{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:9000</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
......
......@@ -15,14 +15,14 @@
<configuration>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8025</value>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:8025</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8030</value>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:8030</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8050</value>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:8050</value>
</property>
</configuration>
---
- name: create topics
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master-node']['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic input"
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars[groups['master'][0]]['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic input"
notify:
- create batch output topic
- name: create batch output topic
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master-node']['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic batch-output"
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars[groups['master'][0]]['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic batch-output"
notify:
- create stream output topic
- name: create stream output topic
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master-node']['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic stream-output"
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars[groups['master'][0]]['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic stream-output"
---
- name: Set hostname
hostname: name={{ inventory_hostname | replace(".vm.okeanos.grnet.gr",".local") }}
when: "'slaves' in group_names"
- name: Copy hosts file.
template: src=hosts.j2 dest=/etc/hosts backup=no owner=root group=lambda mode=0750
......@@ -9,12 +13,14 @@
- name: Install the latest Java 7.
apt: name=openjdk-7-jdk state=latest install_recommends=no update_cache=yes
environment: proxy_env
- name: Copy environment file.
template: src=environment.j2 dest=/etc/environment backup=no owner=root group=lambda mode=0750
- name: Install sudo.
apt: name=sudo state=latest
environment: proxy_env
- name: Add hduser to sudo group.
user: name=hduser group=sudo
......
{% for master in groups["master"] %}
Host {{ master }}
Host {{ master | replace(".vm.okeanos.grnet.gr","*") }}
StrictHostKeyChecking no
{% endfor %}
......
---
- name: Copy hosts file.
template: src=hosts.j2 dest=/etc/hosts backup=no owner=root mode=0750
- name: Install squid http proxy
apt: name=squid3 state=latest
tags: install
- name: Configure squid http proxy 1
- name: Configure squid http proxy
lineinfile: destfile="/etc/squid3/squid.conf" regexp="^acl localnet" insertafter="^#acl localnet"
line="acl localnet src {{ hostvars[groups['all'][0]]['local_net'] }}"
state=present
notify:
- restart squid3
tags: config
- name: Configure squid http proxy 2
- name: Configure squid http proxy
lineinfile: destfile="/etc/squid3/squid.conf" regexp="^#http_access allow localnet"
line="http_access allow localnet"
state=present
notify:
- restart squid3
tags: config
127.0.0.1 localhost
{% for master in groups["master"] %}
{{ hostvars[master]["internal_ip"] }} {{ master | replace(".vm.okeanos.grnet.gr",".local") }}
{% endfor %}
{% for slave in groups["slaves"] %}
{{ hostvars[slave]["internal_ip"] }} {{ slave | replace(".vm.okeanos.grnet.gr",".local") }}
{% endfor %}
# The following lines are desirable for IPv6 capable hosts
::1 localhost ip6-localhost ip6-loopback
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
......@@ -28,7 +28,7 @@ class Manager:
% (self.temp_file, self.inventory['master']['name'])
# ansible.constants.DEFAULT_PRIVATE_KEY_FILE = self.temp_file
ansible.constants.HOST_KEY_CHECKING = False
ansible.constants.DEFAULT_GATHERING = 'explicit'
# ansible.constants.DEFAULT_GATHERING = 'explicit'
def create_inventory(self):
"""
......@@ -109,11 +109,13 @@ if __name__ == "__main__":
# manager.run_playbook(playbook_file="../../ansible/playbooks/testproxy.yml", tags=['install'])
manager.run_playbook(playbook_file="../../ansible/playbooks/wait_for_ssh.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['master'])
manager.run_playbook(playbook_file="../../ansible/playbooks/proxy/proxy.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['slaves'])
manager.run_playbook(playbook_file="../../ansible/playbooks/apache-hadoop/hadoop-install.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/apache-flink/flink-install.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/apache-kafka/kafka-install.yml")
# manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['master'])
# manager.run_playbook(playbook_file="../../ansible/playbooks/proxy/proxy.yml")
# manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['slaves'])
# manager.run_playbook(playbook_file="../../ansible/playbooks/apache-hadoop/hadoop-install.yml")
# manager.run_playbook(playbook_file="../../ansible/playbooks/apache-flink/flink-install.yml")
# manager.run_playbook(playbook_file="../../ansible/playbooks/apache-kafka/kafka-install.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/cluster/cluster-install.yml")
manager.cleanup()
......@@ -4,6 +4,8 @@ from provisioner import Provisioner
from ansible_manager import Manager
if __name__ == "__main__":
start_time = time.time()
parser = argparse.ArgumentParser(description="Okeanos VM provisioning")
parser.add_argument('--cloud', type=str, dest="cloud", default="lambda")
parser.add_argument('--project-name', type=str, dest="project_name",
......@@ -11,7 +13,7 @@ if __name__ == "__main__":
parser.add_argument('--name', type=str, dest='name', default="to mikro debian sto livadi")
parser.add_argument('--slaves', type=int, dest='slaves', default=1)
parser.add_argument('--slaves', type=int, dest='slaves', default=3)
parser.add_argument('--vcpus_master', type=int, dest='vcpus_master', default=4)
parser.add_argument('--vcpus_slave', type=int, dest='vcpus_slave', default=4)
parser.add_argument('--ram_master', type=int, dest='ram_master', default=4096) # in MB
......@@ -47,7 +49,8 @@ if __name__ == "__main__":
provisioner_response['nodes']['slaves'][i]['internal_ip'] = slave_ip
provisioner_response['pk'] = provisioner.get_private_key()
print 'response = ', provisioner_response
print 'response =', provisioner_response
provisioner_time = time.time()
manager = Manager(provisioner_response)
manager.create_inventory()
......@@ -55,13 +58,21 @@ if __name__ == "__main__":
# manager.run_playbook(playbook_file="../../ansible/playbooks/testproxy.yml", tags=['install'])
manager.run_playbook(playbook_file="../../ansible/playbooks/wait_for_ssh.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['master'])
manager.run_playbook(playbook_file="../../ansible/playbooks/proxy/proxy.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['slaves'])
manager.run_playbook(playbook_file="../../ansible/playbooks/apache-hadoop/hadoop-install.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/apache-flink/flink-install.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/apache-kafka/kafka-install.yml")
# manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['master'])
# manager.run_playbook(playbook_file="../../ansible/playbooks/proxy/proxy.yml")
# manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['slaves'])
# manager.run_playbook(playbook_file="../../ansible/playbooks/apache-hadoop/hadoop-install.yml")
# manager.run_playbook(playbook_file="../../ansible/playbooks/apache-flink/flink-install.yml")
# manager.run_playbook(playbook_file="../../ansible/playbooks/apache-kafka/kafka-install.yml")
manager.run_playbook(playbook_file="../../ansible/playbooks/cluster/cluster-install.yml")
# INSERT PLAYBOOKS HERE
manager.cleanup()
provisioner_duration = provisioner_time - start_time
ansible_duration = time.time() - provisioner_time
print 'VM provisioning took', provisioner_duration, 'seconds'
print 'Ansible playbooks took', ansible_duration, 'seconds'
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment