Merge remote-tracking branch 'gouzouni/LAM-49-updated' into tsaftests

ansible/host_vars/slave-1

-internal_ip: "192.168.0.2"
-
-## Apache Kafka variables.
-id: 1
-

ansible/host_vars/slave-2

-internal_ip: "192.168.0.4"
-
-## Apache Kafka variables.
-id: 2
-

ansible/roles/apache-flink/tasks/main.yml

 ---
-  - name: Download Apache Flink, Yarn version.
-    get_url: url="{{ mirror_url }}/flink-{{ version }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz"
-    environment: proxy_env 
+  - name: Include tasks for master.
+    include: master.yml
+    when: "'master' in group_names"
     tags:
-      - download
-
-  - name: Uncompress Apache Flink.
-    unarchive: src="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ installation_path }}" copy=no
-    tags:
-      - uncompress
-
-  - name: Create softlink for Apache Flink.
-    file: src="{{ installation_path }}/flink-{{ version }}" dest="{{ installation_path }}/flink" state=link
-    tags:
-      - uncompress
-
-  - name: Configure Apache Flink.
-    template: src=flink-conf.j2 dest="{{ installation_path }}/flink/conf/flink-conf.yaml" owner=root group=root mode=0644
-    tags:
-      - configure
-
-  - name: Start Apache Flink.
-    shell: "{{ installation_path }}/flink/bin/yarn-session.sh -n {{ number_of_taskmanagers }} -tm {{ ram_per_task_manager }}"
-    async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
-    poll: 0
-    tags:
-      - start
-
+      - master-install

ansible/roles/apache-flink/tasks/master.yml

+---
+  - name: Download Apache Flink, Yarn version.
+    get_url: url="{{ mirror_url }}/flink-{{ version }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz"
+    environment: proxy_env 
+    tags:
+      - download
+
+  - name: Uncompress Apache Flink.
+    unarchive: src="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ installation_path }}" copy=no owner=flink group=lambda
+    tags:
+      - uncompress
+
+  - name: Create softlink for Apache Flink.
+    file: src="{{ installation_path }}/flink-{{ version }}" dest="{{ installation_path }}/flink" state=link
+    tags:
+      - uncompress
+
+  - name: Configure Apache Flink.
+    template: src=flink-conf.j2 dest="{{ installation_path }}/flink/conf/flink-conf.yaml" owner=flink group=lambda mode=0644
+    tags:
+      - configure
+
+  - name: Copy Apache Flink init script.
+    template: src=flink-init.j2 dest=/etc/init.d/flink-init owner=flink group=lambda mode=0740
+
+  - name: Start Apache Flink.
+    shell: /etc/init.d/flink-init start > /dev/null &
+    tags:
+      - start
+

ansible/roles/apache-flink/templates/flink-init.j2

+#!/bin/bash
+
+# Source environment file to make sure HADOOP_HOME variable is available.
+source /etc/environment
+
+# Define the user that should run this script.
+SCRIPT_USER=flink
+
+# The path where Apache Flink is installed.
+INSTALLATION_PATH="{{ installation_path }}"
+
+# The full path of the lock file to use.
+LOCKFILE="$INSTALLATION_PATH/flink/flink-lock"
+
+# The command that will start Apache Flink.
+START_COMMAND="$INSTALLATION_PATH/flink/bin/yarn-session.sh -n {{ number_of_taskmanagers }} -tm {{ ram_per_task_manager }}"
+
+# The command that will stop Apache Flink. Note that, HADOOP_HOME home variable should be set outside from this script and
+# before Flink's deployment.
+STOP_COMMAND="$HADOOP_HOME/bin/yarn application --kill"
+
+start(){
+  sudo -E -u $SCRIPT_USER nohup $START_COMMAND > /dev/null &
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER touch $LOCKFILE)
+  return $RETVAL
+}
+
+stop(){
+  id=$(sudo -u $SCRIPT_USER $HADOOP_HOME/bin/yarn application --list | grep "Flink session" | cut -f1)
+  sudo -E -u $SCRIPT_USER nohup $STOP_COMMAND $id > /dev/null &
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER rm -f $LOCKFILE)
+  return $RETVAL
+}
+
+restart(){
+  stop
+  start
+}
+
+RETVAL=0
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart|reload|force-reload)
+    restart
+    ;;
+  condrestart)
+    [ -f $LOCKFILE ] && restart || :
+    ;;
+  status)
+    [ -f $LOCKFILE ] && echo "Apache Flink is running." || echo "Apache Flink is not running."
+    RETVAL=$?
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
+    RETVAL=1
+esac
+
+exit $RETVAL
+

ansible/roles/apache-flink/vars/main.yml

@@ -6,8 +6,8 @@ download_path: "/root"
 installation_path: "/usr/local"
 jobmanager_heap_mb: 256 
 taskmanager_heap_mb: 512 
-taskmanager_numberOfTaskSlots: 2
-parallelization_degree_default: 2
-number_of_taskmanagers: 2
+taskmanager_numberOfTaskSlots: "{{ ansible_processor_count * ansible_processor_cores }}"
+parallelization_degree_default: "{{ (ansible_processor_count * ansible_processor_cores * groups['slaves']|count / 2)|int }}"
+number_of_taskmanagers: "{{ groups['slaves']|count }}"
 ram_per_task_manager: 768
 

ansible/roles/apache-hadoop/tasks/master.yml

 ---
+  - name: Copy Apache HDFS init script.
+    template: src=hdfs-init.j2 dest=/etc/init.d/hdfs-init owner=hduser group=lambda mode=0740
+
+  - name: Copy Apache Yarn init script.
+    template: src=yarn-init.j2 dest=/etc/init.d/yarn-init owner=hduser group=lambda mode=0740
+
   - name: Format Apache HDFS.
-    shell: "yes | {{ installation_path }}/hadoop/bin/hadoop namenode -format"
+    shell: su - hduser -c "yes | {{ installation_path }}/hadoop/bin/hadoop namenode -format"
     tags:
       - format-hdfs
 
   - name: Start Apache HDFS.
-    shell: "{{ installation_path }}/hadoop/sbin/start-dfs.sh"
+    shell: /etc/init.d/hdfs-init start > /dev/null &
+    tags:
+      - start-hdfs
+
+  - name: Wait for Apache HDFS to become available.
+    wait_for: host="{{ hostvars[inventory_hostname]["internal_ip"] }}" port=9000
     tags:
       - start-hdfs
 
   - name: Start Apache Yarn.
-    shell: "{{ installation_path }}/hadoop/sbin/start-yarn.sh"
+    shell: /etc/init.d/yarn-init start > /dev/null &
+    tags:
+      - start-yarn
+
+  - name: Wait for Apache Yarn to become available.
+    wait_for: host="{{ hostvars[inventory_hostname]["internal_ip"] }}" port=8050 # 8050 is the port of the resource manager.
     tags:
       - start-yarn
 

ansible/roles/apache-hadoop/tasks/setup.yml

 ---
   - name: Download Apache Hadoop.
     get_url: url="{{ mirror_url }}/hadoop-{{ version }}/hadoop-{{ version }}.tar.gz" dest="{{ download_path }}/hadoop-{{ version }}.tar.gz"
-             use_proxy=yes
     environment: proxy_env
 
   - name: Uncompress Apache Hadoop.
-    unarchive: src="{{ download_path }}/hadoop-{{ version }}.tar.gz" dest="{{ installation_path }}" copy=no
+    unarchive: src="{{ download_path }}/hadoop-{{ version }}.tar.gz" dest="{{ installation_path }}" copy=no owner=hduser group=lambda
 
   - name: Create softlink for Apache Hadoop.
     file: src="{{ installation_path }}/hadoop-{{ version }}" dest="{{ installation_path }}/hadoop" state=link
 
   - name: Set JAVA_HOME in Apache Hadoop environment.
-    lineinfile: dest="{{ installation_path }}/hadoop/etc/hadoop/hadoop-env.sh" regexp="^export JAVA_HOME=" line="export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64"
+    lineinfile: dest="{{ installation_path }}/hadoop/etc/hadoop/hadoop-env.sh" regexp="^export JAVA_HOME=" line="export JAVA_HOME=/usr"
 
   - name: Configure slaves.
-    template: src=slaves.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/slaves" owner=root group=root mode=0644
+    template: src=slaves.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/slaves" owner=hduser group=lambda mode=0644
 
   - name: Configure core.
-    template: src=core-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/core-site.xml" owner=root group=root mode=0644
-  
+    template: src=core-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/core-site.xml" owner=hduser group=lambda mode=0644
+
   - name: Configure Apache HDFS.
-    template: src=hdfs-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/hdfs-site.xml" backup=no owner=root group=root mode=0644
-  
-  - name: Configure Apache Yarn.
-    template: src=yarn-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/yarn-site.xml" owner=root group=root mode=0644
+    template: src=hdfs-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/hdfs-site.xml" backup=no owner=hduser group=lambda mode=0644
 
+  - name: Configure Apache Yarn.
+    template: src=yarn-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/yarn-site.xml" owner=hduser group=lambda mode=0644

ansible/roles/apache-hadoop/templates/hdfs-init.j2

+#!/bin/bash
+
+# Source environment file to make sure JAVA_HOME variable is available.
+source /etc/environment
+
+# Define the user that should run this script.
+SCRIPT_USER=hduser
+
+# The path where Apache Hadoop is installed.
+INSTALLATION_PATH="{{ installation_path }}"
+
+# The full path of the lock file to use.
+LOCKFILE="$INSTALLATION_PATH/hadoop/hdfs-lock"
+
+# The command that will start Apache HDFS.
+START_COMMAND="$INSTALLATION_PATH/hadoop/sbin/start-dfs.sh"
+
+# The command that will stop Apache HDFS.
+STOP_COMMAND="$INSTALLATION_PATH/hadoop/sbin/stop-dfs.sh"
+
+start(){
+  sudo -u $SCRIPT_USER nohup $START_COMMAND > /dev/null &
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER touch $LOCKFILE)
+  return $RETVAL
+}
+
+stop(){
+  sudo -u $SCRIPT_USER nohup $STOP_COMMAND > /dev/null &
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER rm -f $LOCKFILE)
+  return $RETVAL
+}
+
+restart(){
+  stop
+  start
+}
+
+RETVAL=0
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart|reload|force-reload)
+    restart
+    ;;
+  condrestart)
+    [ -f $LOCKFILE ] && restart || :
+    ;;
+  status)
+    [ -f $LOCKFILE ] && echo "Apache HDFS is running." || echo "Apache HDFS is not running."
+    RETVAL=$?
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
+    RETVAL=1
+esac
+
+exit $RETVAL
+

ansible/roles/apache-hadoop/templates/yarn-init.j2

+#!/bin/bash
+
+# Source environment file to make sure JAVA_HOME variable is available.
+source /etc/environment
+
+# Define the user that should run this script.
+SCRIPT_USER=hduser
+
+# The path where Apache Hadoop is installed.
+INSTALLATION_PATH="{{ installation_path }}"
+
+# The full path of the lock file to use.
+LOCKFILE="$INSTALLATION_PATH/hadoop/yarn-lock"
+
+# The command that will start Apache Yarn.
+START_COMMAND="$INSTALLATION_PATH/hadoop/sbin/start-yarn.sh"
+
+# The command that will stop Apache Yarn.
+STOP_COMMAND="$INSTALLATION_PATH/hadoop/sbin/stop-yarn.sh"
+
+start(){
+  sudo -u $SCRIPT_USER nohup $START_COMMAND > /dev/null &
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER touch $LOCKFILE)
+  return $RETVAL
+}
+
+stop(){
+  sudo -u $SCRIPT_USER nohup $STOP_COMMAND > /dev/null &
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER rm -f $LOCKFILE)
+  return $RETVAL
+}
+
+restart(){
+  stop
+  start
+}
+
+RETVAL=0
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart|reload|force-reload)
+    restart
+    ;;
+  condrestart)
+    [ -f $LOCKFILE ] && restart || :
+    ;;
+  status)
+    [ -f $LOCKFILE ] && echo "Apache Yarn is running." || echo "Apache Yarn is not running."
+    RETVAL=$?
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|restart|reload|force-reload|condrestart}"
+    RETVAL=1
+esac
+
+exit $RETVAL
+

ansible/roles/apache-hadoop/vars/main.yml

@@ -3,5 +3,5 @@ mirror_url: "http://mirrors.myaegean.gr/apache/hadoop/common"
 version: "2.7.0"
 download_path: "/root"
 installation_path: "/usr/local"
-dfs_replication: 2
+dfs_replication: "{{ groups['slaves']|count }}"
 

ansible/roles/apache-kafka/tasks/download.yml → ansible/roles/apache-kafka/tasks/install.yml

@@ -4,8 +4,10 @@
     environment: proxy_env
 
   - name: Uncompress Apache Kafka.
-    unarchive: src="{{ download_path }}/kafka_{{ scala_version }}-{{ version }}.tgz" dest="{{ installation_path }}" copy=no
+    unarchive: src="{{ download_path }}/kafka_{{ scala_version }}-{{ version }}.tgz" dest="{{ installation_path }}" copy=no owner=kafka group=lambda
 
   - name: Create softlink for Apache Kafka.
     file: src="{{ installation_path }}/kafka_{{ scala_version }}-{{ version }}" dest="{{ installation_path }}/kafka" state=link
 
+  - name: Copy Apache Kafka init script.
+    template: src=kafka-init.j2 dest=/etc/init.d/kafka-init owner=kafka group=lambda mode=0740

ansible/roles/apache-kafka/tasks/master.yml

 ---
-  - name: Download and install Apache kafka.
-    include: download.yml
-    tags:
-      - download
+  - name: Install Apache kafka.
+    include: install.yml
+
+  - name: Copy Apache Zookeeper init script.
+    template: src=zookeeper-init.j2 dest=/etc/init.d/zookeeper-init owner=kafka group=lambda mode=0740
 
   - name: Configure Apache kafka.
-    copy: src=server.properties dest="{{ installation_path }}/kafka/config/server.properties" owner=root group=root mode=0644
+    copy: src=server.properties dest="{{ installation_path }}/kafka/config/server.properties" owner=kafka group=lambda mode=0644
     tags:
       - configure-kafka
 
   - name: Start Apache Zookeeper server.
-    shell: "{{ installation_path }}/kafka/bin/zookeeper-server-start.sh {{ installation_path }}/kafka/config/zookeeper.properties"
-    async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
-    poll: 0
+    shell: /etc/init.d/zookeeper-init start > /dev/null &
     tags:
       - start-zookeeper
 
@@ -22,9 +21,7 @@
       - start-zookeeper
 
   - name: Start Apache Kafka server.
-    shell: "{{ installation_path }}/kafka/bin/kafka-server-start.sh {{ installation_path }}/kafka/config/server.properties"
-    async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
-    poll: 0
+    shell: /etc/init.d/kafka-init start > /dev/null &
     tags:
       - start-kafka
 
@@ -35,4 +32,3 @@
       - create topics
     tags:
       - start-kafka
-

ansible/roles/apache-kafka/tasks/slaves.yml

 ---
-  - name: Download and install Apache kafka.
-    include: download.yml
-    tags:
-      - download
+  - name: Install Apache kafka.
+    include: install.yml
 
   - name: Configure Apache kafka.
-    template: src=server.properties.j2 dest="{{ installation_path }}/kafka/config/server.properties" owner=root group=root mode=0644
+    template: src=server.properties.j2 dest="{{ installation_path }}/kafka/config/server.properties" owner=kafka group=lambda mode=0644
     tags:
       - configure-kafka
 
   - name: Start Apache Kafka server.
-    shell: "{{ installation_path }}/kafka/bin/kafka-server-start.sh {{ installation_path }}/kafka/config/server.properties"
-    async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
-    poll: 0
+    shell: /etc/init.d/kafka-init start > /dev/null &
     tags:
       - start-kafka
 
@@ -20,4 +16,3 @@
     wait_for: port=9092
     tags:
       - start-kafka
-

ansible/roles/apache-kafka/templates/kafka-init.j2

+#!/bin/bash
+
+# Define the user that should run this script.
+SCRIPT_USER=kafka
+
+# The path where Apache Kafka is installed.
+INSTALLATION_PATH="{{ installation_path }}"
+
+# The full path of the pid file to use.
+PIDFILE="$INSTALLATION_PATH/kafka/kafka.pid"
+
+# The full path of the lock file to use.
+LOCKFILE="$INSTALLATION_PATH/kafka/kafka-lock"
+
+# The command that will start Apache Kafka.
+START_COMMAND="$INSTALLATION_PATH/kafka/bin/kafka-server-start.sh $INSTALLATION_PATH/kafka/config/server.properties"
+
+start(){
+  sudo -u $SCRIPT_USER /sbin/start-stop-daemon --start --background --make-pidfile --pidfile $PIDFILE --exec $START_COMMAND
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER touch $LOCKFILE)
+  return $RETVAL
+}
+
+stop(){
+  sudo -u $SCRIPT_USER  /sbin/start-stop-daemon --stop --remove-pidfile --pidfile $PIDFILE
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER rm -f $LOCKFILE)
+  return $RETVAL
+}
+
+restart(){
+  stop
+  start
+}
+
+RETVAL=0
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart|reload|force-reload)
+    restart
+    ;;
+  condrestart)
+    [ -f $LOCKFILE ] && restart || :
+    ;;
+  status)
+    [ -f $LOCKFILE ] && echo "Apache Kafka is running." || echo "Apache kafka is not running."
+    RETVAL=$?
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
+    RETVAL=1
+esac
+
+exit $RETVAL
+

ansible/roles/apache-kafka/templates/server.properties.j2

@@ -117,7 +117,7 @@ log.cleaner.enable=false
 # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
 # You can also append an optional chroot string to the urls to specify the
 # root directory for all kafka znodes.
-zookeeper.connect={{ hostvars['master-node']['internal_ip'] }}:2181
+zookeeper.connect={{ hostvars[groups['master'][0]]['internal_ip'] }}:2181
 
 # Timeout in ms for connecting to zookeeper
 zookeeper.connection.timeout.ms=6000

ansible/roles/apache-kafka/templates/zookeeper-init.j2

+#!/bin/bash
+
+# Define the user that should run this script.
+SCRIPT_USER=kafka
+
+# The path where Apache Zookeeper is installed.
+INSTALLATION_PATH="{{ installation_path }}"
+
+# The full path of the pid file to use.
+PIDFILE="$INSTALLATION_PATH/kafka/zookeeper.pid"
+
+# The full path of the lock file to use.
+LOCKFILE="$INSTALLATION_PATH/kafka/zookeeper-lock"
+
+# The command that will start Apache Zookeeper.
+START_COMMAND="$INSTALLATION_PATH/kafka/bin/zookeeper-server-start.sh $INSTALLATION_PATH/kafka/config/zookeeper.properties"
+
+start(){
+  sudo -u $SCRIPT_USER /sbin/start-stop-daemon --start --background --make-pidfile --pidfile $PIDFILE --exec $START_COMMAND
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER touch $LOCKFILE)
+  return $RETVAL
+}
+
+stop(){
+  sudo -u $SCRIPT_USER /sbin/start-stop-daemon --stop --remove-pidfile --pidfile $PIDFILE
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER rm -f $LOCKFILE)
+  return $RETVAL
+}
+
+restart(){
+  stop
+  start
+}
+
+RETVAL=0
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart|reload|force-reload)
+    restart
+    ;;
+  condrestart)
+    [ -f $LOCKFILE ] && restart || :
+    ;;
+  status)
+    [ -f $LOCKFILE ] && echo "Apache Zookeeper is running." || echo "Apache Zookeeper is not running."
+    RETVAL=$?
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
+    RETVAL=1
+esac
+
+exit $RETVAL
+

ansible/roles/common/files/environment

-export JAVA_HOME=/usr/
-export HADOOP_HOME=/usr/local/hadoop

ansible/roles/common/tasks/common-1.yml

 ---
 
   - name: Copy hosts file.
-    template: src=hosts.j2 dest=/etc/hosts backup=no owner=root group=root mode=0750
+    template: src=hosts.j2 dest=/etc/hosts backup=no owner=root group=lambda mode=0750
 
   - name: Upgrade packages.
     apt: upgrade=dist update_cache=yes
@@ -9,13 +9,18 @@
 
   - name: Install the latest Java 7.
     apt: name=openjdk-7-jdk state=latest install_recommends=no update_cache=yes
-    environment: proxy_env
 
   - name: Copy environment file.
-    copy: src=environment dest=/etc/environment backup=no owner=root group=root mode=0750
+    template: src=environment.j2 dest=/etc/environment backup=no owner=root group=lambda mode=0750
+
+  - name: Install sudo.
+    apt: name=sudo state=latest
+
+  - name: Add hduser to sudo group.
+    user: name=hduser group=sudo
 
-#  - name: Generate ssh key.
-#    shell: cat /dev/zero | ssh-keygen -q -N ""
-#    args:
-#      creates: /root/.ssh/id_rsa.pub
+  - name: Add flink user to sudo group.
+    user: name=flink group=sudo
 
+  - name: Add kafka user to sudo group.
+    user: name=kafka group=sudo

ansible/roles/common/tasks/common-2.yml

 ---