Fix various ansible task variables, some templates

ansible/playbooks/apache-flink/flink-install.yml

 ---
 - hosts: master
   user: root
+  gather_facts: yes
   roles:
     - ../roles/apache-flink
 

ansible/playbooks/cluster/cluster-install.yml

 ---
+
+  - hosts: master
+    user: root
+    roles:
+      - ../roles/proxy
+
   - hosts: all
     user: root
     roles:
@@ -6,4 +12,3 @@
       - ../roles/apache-hadoop
       - ../roles/apache-kafka
       - ../roles/apache-flink
-

ansible/playbooks/wait_for_ssh.yml

@@ -2,7 +2,8 @@
 
 - name: Wait for port 22 to be ready
   hosts: master
+  gather_facts: no
+  sudo: false
   tasks:
-    - local_action: wait_for port=22 host="{{ inventory_hostname }}"  search_regex=OpenSSH delay=5
+    - local_action: wait_for port=22 host="{{ inventory_hostname }}"  search_regex=OpenSSH delay=15
   tags: master
-

ansible/roles/apache-hadoop/templates/core-site.xml.j2

@@ -19,7 +19,7 @@
 <configuration>
   <property>
     <name>fs.default.name</name>
-    <value>hdfs://{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:9000</value>
+    <value>hdfs://{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:9000</value>
   </property>
   <property>
     <name>hadoop.tmp.dir</name>

ansible/roles/apache-hadoop/templates/yarn-site.xml.j2

@@ -15,14 +15,14 @@
 <configuration>
   <property>
     <name>yarn.resourcemanager.resource-tracker.address</name>
-    <value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8025</value>
+    <value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:8025</value>
   </property>
   <property>
     <name>yarn.resourcemanager.scheduler.address</name>
-    <value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8030</value>
+    <value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:8030</value>
   </property>
   <property>
     <name>yarn.resourcemanager.address</name>
-    <value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8050</value>
+    <value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:8050</value>
   </property>
 </configuration>

ansible/roles/apache-kafka/handlers/main.yml

 ---
 - name: create topics
-  shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master-node']['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic input"
+  shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars[groups['master'][0]]['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic input"
   notify:
     - create batch output topic
 
 - name: create batch output topic
-  shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master-node']['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic batch-output"
+  shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars[groups['master'][0]]['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic batch-output"
   notify:
     - create stream output topic
 
 - name: create stream output topic
-  shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master-node']['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic stream-output"
+  shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars[groups['master'][0]]['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic stream-output"
 

ansible/roles/common/tasks/common-1.yml

 ---
 
+  - name: Set hostname
+    hostname: name={{ inventory_hostname | replace(".vm.okeanos.grnet.gr",".local") }}
+    when: "'slaves' in group_names"
+
   - name: Copy hosts file.
     template: src=hosts.j2 dest=/etc/hosts backup=no owner=root group=lambda mode=0750
 
@@ -9,12 +13,14 @@
 
   - name: Install the latest Java 7.
     apt: name=openjdk-7-jdk state=latest install_recommends=no update_cache=yes
+    environment: proxy_env
 
   - name: Copy environment file.
     template: src=environment.j2 dest=/etc/environment backup=no owner=root group=lambda mode=0750
 
   - name: Install sudo.
     apt: name=sudo state=latest
+    environment: proxy_env
 
   - name: Add hduser to sudo group.
     user: name=hduser group=sudo

ansible/roles/common/templates/ssh-config.j2

 {% for master in groups["master"] %}
-Host {{ master }}
+Host {{ master | replace(".vm.okeanos.grnet.gr","*") }}
   StrictHostKeyChecking no
 {% endfor %}
 

ansible/roles/proxy/tasks/main.yml

 ---
 
+  - name: Copy hosts file.
+    template: src=hosts.j2 dest=/etc/hosts backup=no owner=root mode=0750
+
   - name: Install squid http proxy
     apt: name=squid3 state=latest
     tags: install
 
-  - name: Configure squid http proxy 1
+  - name: Configure squid http proxy
     lineinfile: destfile="/etc/squid3/squid.conf" regexp="^acl localnet" insertafter="^#acl localnet"
                 line="acl localnet src {{ hostvars[groups['all'][0]]['local_net'] }}"
                 state=present
     notify:
     - restart squid3
     tags: config
-    
-  - name: Configure squid http proxy 2
+
+  - name: Configure squid http proxy
     lineinfile: destfile="/etc/squid3/squid.conf" regexp="^#http_access allow localnet"
                 line="http_access allow localnet"
                 state=present
     notify:
     - restart squid3
     tags: config
-

ansible/roles/proxy/templates/hosts.j2

+127.0.0.1       localhost
+{% for master in groups["master"] %}
+{{ hostvars[master]["internal_ip"] }}	{{ master | replace(".vm.okeanos.grnet.gr",".local") }}
+{% endfor %}
+{% for slave in groups["slaves"] %}
+{{ hostvars[slave]["internal_ip"] }}	{{ slave | replace(".vm.okeanos.grnet.gr",".local") }}
+{% endfor %}
+
+# The following lines are desirable for IPv6 capable hosts
+::1     localhost ip6-localhost ip6-loopback
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters

core/fokia/ansible_manager.py

@@ -28,7 +28,7 @@ class Manager:
                                              % (self.temp_file, self.inventory['master']['name'])
         # ansible.constants.DEFAULT_PRIVATE_KEY_FILE = self.temp_file
         ansible.constants.HOST_KEY_CHECKING = False
-        ansible.constants.DEFAULT_GATHERING = 'explicit'
+        # ansible.constants.DEFAULT_GATHERING = 'explicit'
 
     def create_inventory(self):
         """
@@ -109,11 +109,13 @@ if __name__ == "__main__":
     # manager.run_playbook(playbook_file="../../ansible/playbooks/testproxy.yml", tags=['install'])
 
     manager.run_playbook(playbook_file="../../ansible/playbooks/wait_for_ssh.yml")
-    manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['master'])
-    manager.run_playbook(playbook_file="../../ansible/playbooks/proxy/proxy.yml")
-    manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['slaves'])
-    manager.run_playbook(playbook_file="../../ansible/playbooks/apache-hadoop/hadoop-install.yml")
-    manager.run_playbook(playbook_file="../../ansible/playbooks/apache-flink/flink-install.yml")
-    manager.run_playbook(playbook_file="../../ansible/playbooks/apache-kafka/kafka-install.yml")
+    # manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['master'])
+    # manager.run_playbook(playbook_file="../../ansible/playbooks/proxy/proxy.yml")
+    # manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['slaves'])
+    # manager.run_playbook(playbook_file="../../ansible/playbooks/apache-hadoop/hadoop-install.yml")
+    # manager.run_playbook(playbook_file="../../ansible/playbooks/apache-flink/flink-install.yml")
+    # manager.run_playbook(playbook_file="../../ansible/playbooks/apache-kafka/kafka-install.yml")
+    
+    manager.run_playbook(playbook_file="../../ansible/playbooks/cluster/cluster-install.yml")
 
     manager.cleanup()

core/fokia/cluster_creator.py

@@ -4,6 +4,8 @@ from provisioner import Provisioner
 from ansible_manager import Manager
 
 if __name__ == "__main__":
+    start_time = time.time()
+
     parser = argparse.ArgumentParser(description="Okeanos VM provisioning")
     parser.add_argument('--cloud', type=str, dest="cloud", default="lambda")
     parser.add_argument('--project-name', type=str, dest="project_name",
@@ -11,7 +13,7 @@ if __name__ == "__main__":
     parser.add_argument('--name', type=str, dest='name', default="to mikro debian sto livadi")
 
 
-    parser.add_argument('--slaves', type=int, dest='slaves', default=1)
+    parser.add_argument('--slaves', type=int, dest='slaves', default=3)
     parser.add_argument('--vcpus_master', type=int, dest='vcpus_master', default=4)
     parser.add_argument('--vcpus_slave', type=int, dest='vcpus_slave', default=4)
     parser.add_argument('--ram_master', type=int, dest='ram_master', default=4096)  # in MB
@@ -47,7 +49,8 @@ if __name__ == "__main__":
         provisioner_response['nodes']['slaves'][i]['internal_ip'] = slave_ip
     provisioner_response['pk'] = provisioner.get_private_key()
 
-    print 'response = ', provisioner_response
+    print 'response =', provisioner_response
+    provisioner_time = time.time()
 
     manager = Manager(provisioner_response)
     manager.create_inventory()
@@ -55,13 +58,21 @@ if __name__ == "__main__":
     # manager.run_playbook(playbook_file="../../ansible/playbooks/testproxy.yml", tags=['install'])
 
     manager.run_playbook(playbook_file="../../ansible/playbooks/wait_for_ssh.yml")
-    manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['master'])
-    manager.run_playbook(playbook_file="../../ansible/playbooks/proxy/proxy.yml")
-    manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['slaves'])
-    manager.run_playbook(playbook_file="../../ansible/playbooks/apache-hadoop/hadoop-install.yml")
-    manager.run_playbook(playbook_file="../../ansible/playbooks/apache-flink/flink-install.yml")
-    manager.run_playbook(playbook_file="../../ansible/playbooks/apache-kafka/kafka-install.yml")
+    # manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['master'])
+    # manager.run_playbook(playbook_file="../../ansible/playbooks/proxy/proxy.yml")
+    # manager.run_playbook(playbook_file="../../ansible/playbooks/common/install.yml", tags=['slaves'])
+    # manager.run_playbook(playbook_file="../../ansible/playbooks/apache-hadoop/hadoop-install.yml")
+    # manager.run_playbook(playbook_file="../../ansible/playbooks/apache-flink/flink-install.yml")
+    # manager.run_playbook(playbook_file="../../ansible/playbooks/apache-kafka/kafka-install.yml")
+    
+    manager.run_playbook(playbook_file="../../ansible/playbooks/cluster/cluster-install.yml")
 
     # INSERT PLAYBOOKS HERE
 
     manager.cleanup()
+
+    provisioner_duration = provisioner_time - start_time
+    ansible_duration = time.time() - provisioner_time
+
+    print 'VM provisioning took', provisioner_duration, 'seconds'
+    print 'Ansible playbooks took', ansible_duration, 'seconds'