Updated LAM-49 branch with new commits in devel branch.

ansible/roles/apache-flink/tasks/main.yml

 ---
-  - name: Download Apache Flink, Yarn version.
-    get_url: url="{{ mirror_url }}/flink-{{ version }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz"
+  - name: Include tasks for master.
+    include: master.yml
+    when: "'master' in group_names"
     tags:
-      - download
-
-  - name: Uncompress Apache Flink.
-    unarchive: src="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ installation_path }}" copy=no owner=flink group=lambda
-    tags:
-      - uncompress
-
-  - name: Create softlink for Apache Flink.
-    file: src="{{ installation_path }}/flink-{{ version }}" dest="{{ installation_path }}/flink" state=link
-    tags:
-      - uncompress
-
-  - name: Configure Apache Flink.
-    template: src=flink-conf.j2 dest="{{ installation_path }}/flink/conf/flink-conf.yaml" owner=flink group=lambda mode=0644
-    tags:
-      - configure
-
-  - name: Start Apache Flink.
-    shell: "{{ installation_path }}/flink/bin/yarn-session.sh -n {{ number_of_taskmanagers }} -tm {{ ram_per_task_manager }}"
-    async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
-    poll: 0
-    tags:
-      - start
+      - master-install

ansible/roles/apache-flink/tasks/master.yml

+---
+  - name: Download Apache Flink, Yarn version.
+    get_url: url="{{ mirror_url }}/flink-{{ version }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz"
+    tags:
+      - download
+
+  - name: Uncompress Apache Flink.
+    unarchive: src="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ installation_path }}" copy=no
+    tags:
+      - uncompress
+
+  - name: Create softlink for Apache Flink.
+    file: src="{{ installation_path }}/flink-{{ version }}" dest="{{ installation_path }}/flink" state=link
+    tags:
+      - uncompress
+
+  - name: Configure Apache Flink.
+    template: src=flink-conf.j2 dest="{{ installation_path }}/flink/conf/flink-conf.yaml" owner=root group=root mode=0644
+    tags:
+      - configure
+
+  - name: Copy Apache Flink init script.
+    template: src=flink-init.j2 dest=/etc/init.d/flink-init owner=root group=root mode=0755
+
+  - name: Start Apache Flink.
+    shell: /etc/init.d/flink-init start # Should use Ansible module "service: name=flink-init state=started" when the bug in version 1.9.1 is fixed.
+    tags:
+      - start
+

ansible/roles/apache-flink/templates/flink-init.j2

+#!/bin/bash
+
+# Source environment file to make sure HADOOP_HOME variable is available.
+source /etc/environment
+
+# The path where Apache Flink is installed.
+INSTALLATION_PATH="{{ installation_path }}"
+
+# The full path of the lock file to use.
+LOCKFILE="$INSTALLATION_PATH/flink/flink-lock"
+
+# The command that will start Apache Flink.
+START_COMMAND="$INSTALLATION_PATH/flink/bin/yarn-session.sh -n {{ number_of_taskmanagers }} -tm {{ ram_per_task_manager }}"
+
+# The command that will stop Apache Flink. Note that, HADOOP_HOME home variable should be set outside from this script and
+# before Flink's deployment.
+STOP_COMMAND="$HADOOP_HOME/bin/yarn application --kill"
+
+start(){
+  nohup $START_COMMAND > /dev/null &
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && touch $LOCKFILE
+  return $RETVAL
+}
+
+stop(){
+  id=$($HADOOP_HOME/bin/yarn application --list | grep "Flink session" | cut -f1)
+  nohup $STOP_COMMAND $id > /dev/null &
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(rm -f $LOCKFILE)
+  return $RETVAL
+}
+
+restart(){
+  stop
+  start
+}
+
+RETVAL=0
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart|reload|force-reload)
+    restart
+    ;;
+  condrestart)
+    [ -f $LOCKFILE ] && restart || :
+    ;;
+  status)
+    [ -f $LOCKFILE ] && echo "Apache Flink is running." || echo "Apache Flink is not running."
+    RETVAL=$?
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
+    RETVAL=1
+esac
+
+exit $RETVAL
+

ansible/roles/apache-flink/vars/main.yml

@@ -6,8 +6,8 @@ download_path: "/root"
 installation_path: "/usr/local"
 jobmanager_heap_mb: 256 
 taskmanager_heap_mb: 512 
-taskmanager_numberOfTaskSlots: 2
-parallelization_degree_default: 2
-number_of_taskmanagers: 2
+taskmanager_numberOfTaskSlots: "{{ ansible_processor_count * ansible_processor_cores }}"
+parallelization_degree_default: "{{ (ansible_processor_count * ansible_processor_cores * groups['slaves']|count / 2)|int }}"
+number_of_taskmanagers: "{{ groups['slaves']|count }}"
 ram_per_task_manager: 768
 

ansible/roles/apache-hadoop/tasks/master.yml

 ---
+  - name: Copy Apache HDFS init script.
+    template: src=hdfs-init.j2 dest=/etc/init.d/hdfs-init owner=root group=root mode=0755
+
+  - name: Copy Apache Yarn init script.
+    template: src=yarn-init.j2 dest=/etc/init.d/yarn-init owner=root group=root mode=0755
+
   - name: Format Apache HDFS.
     shell: "yes | {{ installation_path }}/hadoop/bin/hadoop namenode -format"
     tags:
       - format-hdfs
 
   - name: Start Apache HDFS.
-    shell: "{{ installation_path }}/hadoop/sbin/start-dfs.sh"
+    shell: /etc/init.d/hdfs-init start # Should use Ansible module "service: name=hdfs-init state=started" when the bug in version 1.9.1 is fixed.
+    tags:
+      - start-hdfs
+
+  - name: Wait for Apache HDFS to become available.
+    wait_for: host="{{ hostvars[inventory_hostname]["internal_ip"] }}" port=9000
     tags:
       - start-hdfs
 
   - name: Start Apache Yarn.
-    shell: "{{ installation_path }}/hadoop/sbin/start-yarn.sh"
+    shell: /etc/init.d/yarn-init start # Should use Ansible module "service: name=yarn-init state=started" when the bug in version 1.9.1 is fixed.
+    tags:
+      - start-yarn
+
+  - name: Wait for Apache Yarn to become available.
+    wait_for: host="{{ hostvars[inventory_hostname]["internal_ip"] }}" port=8050 # 8050 is the port of the resource manager.
     tags:
       - start-yarn
 

ansible/roles/apache-hadoop/templates/hdfs-init.j2

+#!/bin/bash
+
+# Source environment file to make sure JAVA_HOME variable is available.
+source /etc/environment
+
+# The path where Apache Hadoop is installed.
+INSTALLATION_PATH="{{ installation_path }}"
+
+# The full path of the lock file to use.
+LOCKFILE="$INSTALLATION_PATH/hadoop/hdfs-lock"
+
+# The command that will start Apache HDFS.
+START_COMMAND="$INSTALLATION_PATH/hadoop/sbin/start-dfs.sh"
+
+# The command that will stop Apache HDFS.
+STOP_COMMAND="$INSTALLATION_PATH/hadoop/sbin/stop-dfs.sh"
+
+start(){
+  nohup $START_COMMAND > /dev/null &
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && touch $LOCKFILE
+  return $RETVAL
+}
+
+stop(){
+  nohup $STOP_COMMAND > /dev/null &
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(rm -f $LOCKFILE)
+  return $RETVAL
+}
+
+restart(){
+  stop
+  start
+}
+
+RETVAL=0
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart|reload|force-reload)
+    restart
+    ;;
+  condrestart)
+    [ -f $LOCKFILE ] && restart || :
+    ;;
+  status)
+    [ -f $LOCKFILE ] && echo "Apache HDFS is running." || echo "Apache HDFS is not running."
+    RETVAL=$?
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
+    RETVAL=1
+esac
+
+exit $RETVAL
+

ansible/roles/apache-hadoop/templates/yarn-init.j2

+#!/bin/bash
+
+# Source environment file to make sure JAVA_HOME variable is available.
+source /etc/environment
+
+# The path where Apache Hadoop is installed.
+INSTALLATION_PATH="{{ installation_path }}"
+
+# The full path of the lock file to use.
+LOCKFILE="$INSTALLATION_PATH/hadoop/yarn-lock"
+
+# The command that will start Apache Yarn.
+START_COMMAND="$INSTALLATION_PATH/hadoop/sbin/start-yarn.sh"
+
+# The command that will stop Apache Yarn.
+STOP_COMMAND="$INSTALLATION_PATH/hadoop/sbin/stop-yarn.sh"
+
+start(){
+  nohup $START_COMMAND > /dev/null &
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && touch $LOCKFILE
+  return $RETVAL
+}
+
+stop(){
+  nohup $STOP_COMMAND > /dev/null &
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(rm -f $LOCKFILE)
+  return $RETVAL
+}
+
+restart(){
+  stop
+  start
+}
+
+RETVAL=0
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart|reload|force-reload)
+    restart
+    ;;
+  condrestart)
+    [ -f $LOCKFILE ] && restart || :
+    ;;
+  status)
+    [ -f $LOCKFILE ] && echo "Apache Yarn is running." || echo "Apache Yarn is not running."
+    RETVAL=$?
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|restart|reload|force-reload|condrestart}"
+    RETVAL=1
+esac
+
+exit $RETVAL
+

ansible/roles/apache-hadoop/vars/main.yml

@@ -3,5 +3,5 @@ mirror_url: "http://mirrors.myaegean.gr/apache/hadoop/common"
 version: "2.7.0"
 download_path: "/root"
 installation_path: "/usr/local"
-dfs_replication: 2
+dfs_replication: "{{ groups['slaves']|count }}"
 

ansible/roles/apache-kafka/tasks/download.yml → ansible/roles/apache-kafka/tasks/install.yml

@@ -7,3 +7,6 @@
 
   - name: Create softlink for Apache Kafka.
     file: src="{{ installation_path }}/kafka_{{ scala_version }}-{{ version }}" dest="{{ installation_path }}/kafka" state=link
+
+  - name: Copy Apache Kafka init script.
+    template: src=kafka-init.j2 dest=/etc/init.d/kafka-init owner=kafka group=lambda mode=0754

ansible/roles/apache-kafka/tasks/master.yml

 ---
-  - name: Download and install Apache kafka.
-    include: download.yml
-    tags:
-      - download
+  - name: Install Apache kafka.
+    include: install.yml
+
+  - name: Copy Apache Zookeeper init script.
+    template: src=zookeeper-init.j2 dest=/etc/init.d/zookeeper-init owner=root group=root mode=0755
 
   - name: Configure Apache kafka.
     copy: src=server.properties dest="{{ installation_path }}/kafka/config/server.properties" owner=kafka group=lambda mode=0644
@@ -10,9 +11,7 @@
       - configure-kafka
 
   - name: Start Apache Zookeeper server.
-    shell: "{{ installation_path }}/kafka/bin/zookeeper-server-start.sh {{ installation_path }}/kafka/config/zookeeper.properties"
-    async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
-    poll: 0
+    shell: /etc/init.d/zookeeper-init start # Should use Ansible module "service: name=zookeeper-init state=started" when the bug in version 1.9.1 is fixed.
     tags:
       - start-zookeeper
 
@@ -22,9 +21,7 @@
       - start-zookeeper
 
   - name: Start Apache Kafka server.
-    shell: "{{ installation_path }}/kafka/bin/kafka-server-start.sh {{ installation_path }}/kafka/config/server.properties"
-    async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
-    poll: 0
+    shell: /etc/init.d/kafka-init start # Should use Ansible module "service: name=kafka-init state=started" when the bug in version 1.9.1 is fixed.
     tags:
       - start-kafka
 

ansible/roles/apache-kafka/tasks/slaves.yml

 ---
-  - name: Download and install Apache kafka.
-    include: download.yml
-    tags:
-      - download
+  - name: Install Apache kafka.
+    include: install.yml
 
   - name: Configure Apache kafka.
     template: src=server.properties.j2 dest="{{ installation_path }}/kafka/config/server.properties" owner=kafka group=lambda mode=0644
@@ -10,9 +8,7 @@
       - configure-kafka
 
   - name: Start Apache Kafka server.
-    shell: "{{ installation_path }}/kafka/bin/kafka-server-start.sh {{ installation_path }}/kafka/config/server.properties"
-    async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
-    poll: 0
+    shell: /etc/init.d/kafka-init start # Should use ansible module service: name=zookeeper-init state=started when the bug in version 1.9.1 is fixed.
     tags:
       - start-kafka
 

ansible/roles/apache-kafka/templates/kafka-init.j2

+#!/bin/bash
+
+# The path where Apache Kafka is installed.
+INSTALLATION_PATH="{{ installation_path }}"
+
+# The full path of the pid file to use.
+PIDFILE="$INSTALLATION_PATH/kafka/kafka.pid"
+
+# The full path of the lock file to use.
+LOCKFILE="$INSTALLATION_PATH/kafka/kafka-lock"
+
+# The command that will start Apache Kafka.
+START_COMMAND="$INSTALLATION_PATH/kafka/bin/kafka-server-start.sh $INSTALLATION_PATH/kafka/config/server.properties"
+
+start(){
+  touch $PIDFILE
+  nohup $START_COMMAND > /dev/null & echo $! > $PIDFILE
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && touch $LOCKFILE
+  return $RETVAL
+}
+
+stop(){
+  kill -s 9 $(cat $PIDFILE)
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(rm -f $LOCKFILE)
+  return $RETVAL
+}
+
+restart(){
+  stop
+  start
+}
+
+RETVAL=0
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart|reload|force-reload)
+    restart
+    ;;
+  condrestart)
+    [ -f $LOCKFILE ] && restart || :
+    ;;
+  status)
+    [ -f $LOCKFILE ] && echo "Apache Kafka is running." || echo "Apache kafka is not running."
+    RETVAL=$?
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
+    RETVAL=1
+esac
+
+exit $RETVAL
+

ansible/roles/apache-kafka/templates/server.properties.j2

@@ -117,7 +117,7 @@ log.cleaner.enable=false
 # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
 # You can also append an optional chroot string to the urls to specify the
 # root directory for all kafka znodes.
-zookeeper.connect={{ hostvars['master-node']['internal_ip'] }}:2181
+zookeeper.connect={{ hostvars[groups['master'][0]]['internal_ip'] }}:2181
 
 # Timeout in ms for connecting to zookeeper
 zookeeper.connection.timeout.ms=6000

ansible/roles/apache-kafka/templates/zookeeper-init.j2

+#!/bin/bash
+
+# The path where Apache Zookeeper is installed.
+INSTALLATION_PATH="{{ installation_path }}"
+
+# The full path of the pid file to use.
+PIDFILE="$INSTALLATION_PATH/kafka/zookeeper.pid"
+
+# The full path of the lock file to use.
+LOCKFILE="$INSTALLATION_PATH/kafka/zookeeper-lock"
+
+# The command that will start Apache Zookeeper.
+START_COMMAND="$INSTALLATION_PATH/kafka/bin/zookeeper-server-start.sh $INSTALLATION_PATH/kafka/config/zookeeper.properties"
+
+start(){
+  touch $PIDFILE
+  nohup $START_COMMAND > /dev/null & echo $! > $PIDFILE
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && touch $LOCKFILE
+  return $RETVAL
+}
+
+stop(){
+  kill -s 9 $(cat $PIDFILE)
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(rm -f $LOCKFILE)
+  return $RETVAL
+}
+
+restart(){
+  stop
+  start
+}
+
+RETVAL=0
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart|reload|force-reload)
+    restart
+    ;;
+  condrestart)
+    [ -f $LOCKFILE ] && restart || :
+    ;;
+  status)
+    [ -f $LOCKFILE ] && echo "Apache Zookeeper is running." || echo "Apache Zookeeper is not running."
+    RETVAL=$?
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
+    RETVAL=1
+esac
+
+exit $RETVAL
+

ansible/roles/common/files/environment

-export JAVA_HOME=/usr/
-export HADOOP_HOME=/usr/local/hadoop

ansible/roles/common/tasks/common-1.yml

@@ -9,7 +9,7 @@
     template: src=hosts.j2 dest=/etc/hosts backup=no owner=root group=lambda mode=0750
 
   - name: Copy environment file.
-    copy: src=environment dest=/etc/environment backup=no owner=root group=root mode=0750
+    template: src=environment.j2 dest=/etc/environment backup=no owner=root group=root mode=0750
 
   - name: Generate ssh key.
     shell: cat /dev/zero | ssh-keygen -q -N ""

ansible/roles/common/templates/environment.j2

+export JAVA_HOME={{ java_home }}
+export HADOOP_HOME={{ hadoop_home }}

ansible/roles/common/vars/main.yml

+---
+java_home: "/usr"
+hadoop_home: "/usr/local/hadoop"