Unverified Commit ee9f3ae5 authored by Avraam Tsantekidis's avatar Avraam Tsantekidis

Merge remote-tracking branch 'gouzouni/LAM-49-updated' into tsaftests

parents 749dd946 ada5acb9
internal_ip: "192.168.0.2"
## Apache Kafka variables.
id: 1
internal_ip: "192.168.0.4"
## Apache Kafka variables.
id: 2
--- ---
- name: Download Apache Flink, Yarn version. - name: Include tasks for master.
get_url: url="{{ mirror_url }}/flink-{{ version }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz" include: master.yml
environment: proxy_env when: "'master' in group_names"
tags: tags:
- download - master-install
- name: Uncompress Apache Flink.
unarchive: src="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ installation_path }}" copy=no
tags:
- uncompress
- name: Create softlink for Apache Flink.
file: src="{{ installation_path }}/flink-{{ version }}" dest="{{ installation_path }}/flink" state=link
tags:
- uncompress
- name: Configure Apache Flink.
template: src=flink-conf.j2 dest="{{ installation_path }}/flink/conf/flink-conf.yaml" owner=root group=root mode=0644
tags:
- configure
- name: Start Apache Flink.
shell: "{{ installation_path }}/flink/bin/yarn-session.sh -n {{ number_of_taskmanagers }} -tm {{ ram_per_task_manager }}"
async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
poll: 0
tags:
- start
---
- name: Download Apache Flink, Yarn version.
get_url: url="{{ mirror_url }}/flink-{{ version }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz"
environment: proxy_env
tags:
- download
- name: Uncompress Apache Flink.
unarchive: src="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ installation_path }}" copy=no owner=flink group=lambda
tags:
- uncompress
- name: Create softlink for Apache Flink.
file: src="{{ installation_path }}/flink-{{ version }}" dest="{{ installation_path }}/flink" state=link
tags:
- uncompress
- name: Configure Apache Flink.
template: src=flink-conf.j2 dest="{{ installation_path }}/flink/conf/flink-conf.yaml" owner=flink group=lambda mode=0644
tags:
- configure
- name: Copy Apache Flink init script.
template: src=flink-init.j2 dest=/etc/init.d/flink-init owner=flink group=lambda mode=0740
- name: Start Apache Flink.
shell: /etc/init.d/flink-init start > /dev/null &
tags:
- start
#!/bin/bash
# Source environment file to make sure HADOOP_HOME variable is available.
source /etc/environment
# Define the user that should run this script.
SCRIPT_USER=flink
# The path where Apache Flink is installed.
INSTALLATION_PATH="{{ installation_path }}"
# The full path of the lock file to use.
LOCKFILE="$INSTALLATION_PATH/flink/flink-lock"
# The command that will start Apache Flink.
START_COMMAND="$INSTALLATION_PATH/flink/bin/yarn-session.sh -n {{ number_of_taskmanagers }} -tm {{ ram_per_task_manager }}"
# The command that will stop Apache Flink. Note that, HADOOP_HOME home variable should be set outside from this script and
# before Flink's deployment.
STOP_COMMAND="$HADOOP_HOME/bin/yarn application --kill"
start(){
sudo -E -u $SCRIPT_USER nohup $START_COMMAND > /dev/null &
RETVAL=$?
[ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER touch $LOCKFILE)
return $RETVAL
}
stop(){
id=$(sudo -u $SCRIPT_USER $HADOOP_HOME/bin/yarn application --list | grep "Flink session" | cut -f1)
sudo -E -u $SCRIPT_USER nohup $STOP_COMMAND $id > /dev/null &
RETVAL=$?
[ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER rm -f $LOCKFILE)
return $RETVAL
}
restart(){
stop
start
}
RETVAL=0
case "$1" in
start)
start
;;
stop)
stop
;;
restart|reload|force-reload)
restart
;;
condrestart)
[ -f $LOCKFILE ] && restart || :
;;
status)
[ -f $LOCKFILE ] && echo "Apache Flink is running." || echo "Apache Flink is not running."
RETVAL=$?
;;
*)
echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
RETVAL=1
esac
exit $RETVAL
...@@ -6,8 +6,8 @@ download_path: "/root" ...@@ -6,8 +6,8 @@ download_path: "/root"
installation_path: "/usr/local" installation_path: "/usr/local"
jobmanager_heap_mb: 256 jobmanager_heap_mb: 256
taskmanager_heap_mb: 512 taskmanager_heap_mb: 512
taskmanager_numberOfTaskSlots: 2 taskmanager_numberOfTaskSlots: "{{ ansible_processor_count * ansible_processor_cores }}"
parallelization_degree_default: 2 parallelization_degree_default: "{{ (ansible_processor_count * ansible_processor_cores * groups['slaves']|count / 2)|int }}"
number_of_taskmanagers: 2 number_of_taskmanagers: "{{ groups['slaves']|count }}"
ram_per_task_manager: 768 ram_per_task_manager: 768
--- ---
- name: Copy Apache HDFS init script.
template: src=hdfs-init.j2 dest=/etc/init.d/hdfs-init owner=hduser group=lambda mode=0740
- name: Copy Apache Yarn init script.
template: src=yarn-init.j2 dest=/etc/init.d/yarn-init owner=hduser group=lambda mode=0740
- name: Format Apache HDFS. - name: Format Apache HDFS.
shell: "yes | {{ installation_path }}/hadoop/bin/hadoop namenode -format" shell: su - hduser -c "yes | {{ installation_path }}/hadoop/bin/hadoop namenode -format"
tags: tags:
- format-hdfs - format-hdfs
- name: Start Apache HDFS. - name: Start Apache HDFS.
shell: "{{ installation_path }}/hadoop/sbin/start-dfs.sh" shell: /etc/init.d/hdfs-init start > /dev/null &
tags:
- start-hdfs
- name: Wait for Apache HDFS to become available.
wait_for: host="{{ hostvars[inventory_hostname]["internal_ip"] }}" port=9000
tags: tags:
- start-hdfs - start-hdfs
- name: Start Apache Yarn. - name: Start Apache Yarn.
shell: "{{ installation_path }}/hadoop/sbin/start-yarn.sh" shell: /etc/init.d/yarn-init start > /dev/null &
tags:
- start-yarn
- name: Wait for Apache Yarn to become available.
wait_for: host="{{ hostvars[inventory_hostname]["internal_ip"] }}" port=8050 # 8050 is the port of the resource manager.
tags: tags:
- start-yarn - start-yarn
......
--- ---
- name: Download Apache Hadoop. - name: Download Apache Hadoop.
get_url: url="{{ mirror_url }}/hadoop-{{ version }}/hadoop-{{ version }}.tar.gz" dest="{{ download_path }}/hadoop-{{ version }}.tar.gz" get_url: url="{{ mirror_url }}/hadoop-{{ version }}/hadoop-{{ version }}.tar.gz" dest="{{ download_path }}/hadoop-{{ version }}.tar.gz"
use_proxy=yes
environment: proxy_env environment: proxy_env
- name: Uncompress Apache Hadoop. - name: Uncompress Apache Hadoop.
unarchive: src="{{ download_path }}/hadoop-{{ version }}.tar.gz" dest="{{ installation_path }}" copy=no unarchive: src="{{ download_path }}/hadoop-{{ version }}.tar.gz" dest="{{ installation_path }}" copy=no owner=hduser group=lambda
- name: Create softlink for Apache Hadoop. - name: Create softlink for Apache Hadoop.
file: src="{{ installation_path }}/hadoop-{{ version }}" dest="{{ installation_path }}/hadoop" state=link file: src="{{ installation_path }}/hadoop-{{ version }}" dest="{{ installation_path }}/hadoop" state=link
- name: Set JAVA_HOME in Apache Hadoop environment. - name: Set JAVA_HOME in Apache Hadoop environment.
lineinfile: dest="{{ installation_path }}/hadoop/etc/hadoop/hadoop-env.sh" regexp="^export JAVA_HOME=" line="export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64" lineinfile: dest="{{ installation_path }}/hadoop/etc/hadoop/hadoop-env.sh" regexp="^export JAVA_HOME=" line="export JAVA_HOME=/usr"
- name: Configure slaves. - name: Configure slaves.
template: src=slaves.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/slaves" owner=root group=root mode=0644 template: src=slaves.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/slaves" owner=hduser group=lambda mode=0644
- name: Configure core. - name: Configure core.
template: src=core-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/core-site.xml" owner=root group=root mode=0644 template: src=core-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/core-site.xml" owner=hduser group=lambda mode=0644
- name: Configure Apache HDFS. - name: Configure Apache HDFS.
template: src=hdfs-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/hdfs-site.xml" backup=no owner=root group=root mode=0644 template: src=hdfs-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/hdfs-site.xml" backup=no owner=hduser group=lambda mode=0644
- name: Configure Apache Yarn.
template: src=yarn-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/yarn-site.xml" owner=root group=root mode=0644
- name: Configure Apache Yarn.
template: src=yarn-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/yarn-site.xml" owner=hduser group=lambda mode=0644
#!/bin/bash
# Source environment file to make sure JAVA_HOME variable is available.
source /etc/environment
# Define the user that should run this script.
SCRIPT_USER=hduser
# The path where Apache Hadoop is installed.
INSTALLATION_PATH="{{ installation_path }}"
# The full path of the lock file to use.
LOCKFILE="$INSTALLATION_PATH/hadoop/hdfs-lock"
# The command that will start Apache HDFS.
START_COMMAND="$INSTALLATION_PATH/hadoop/sbin/start-dfs.sh"
# The command that will stop Apache HDFS.
STOP_COMMAND="$INSTALLATION_PATH/hadoop/sbin/stop-dfs.sh"
start(){
sudo -u $SCRIPT_USER nohup $START_COMMAND > /dev/null &
RETVAL=$?
[ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER touch $LOCKFILE)
return $RETVAL
}
stop(){
sudo -u $SCRIPT_USER nohup $STOP_COMMAND > /dev/null &
RETVAL=$?
[ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER rm -f $LOCKFILE)
return $RETVAL
}
restart(){
stop
start
}
RETVAL=0
case "$1" in
start)
start
;;
stop)
stop
;;
restart|reload|force-reload)
restart
;;
condrestart)
[ -f $LOCKFILE ] && restart || :
;;
status)
[ -f $LOCKFILE ] && echo "Apache HDFS is running." || echo "Apache HDFS is not running."
RETVAL=$?
;;
*)
echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
RETVAL=1
esac
exit $RETVAL
#!/bin/bash
# Source environment file to make sure JAVA_HOME variable is available.
source /etc/environment
# Define the user that should run this script.
SCRIPT_USER=hduser
# The path where Apache Hadoop is installed.
INSTALLATION_PATH="{{ installation_path }}"
# The full path of the lock file to use.
LOCKFILE="$INSTALLATION_PATH/hadoop/yarn-lock"
# The command that will start Apache Yarn.
START_COMMAND="$INSTALLATION_PATH/hadoop/sbin/start-yarn.sh"
# The command that will stop Apache Yarn.
STOP_COMMAND="$INSTALLATION_PATH/hadoop/sbin/stop-yarn.sh"
start(){
sudo -u $SCRIPT_USER nohup $START_COMMAND > /dev/null &
RETVAL=$?
[ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER touch $LOCKFILE)
return $RETVAL
}
stop(){
sudo -u $SCRIPT_USER nohup $STOP_COMMAND > /dev/null &
RETVAL=$?
[ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER rm -f $LOCKFILE)
return $RETVAL
}
restart(){
stop
start
}
RETVAL=0
case "$1" in
start)
start
;;
stop)
stop
;;
restart|reload|force-reload)
restart
;;
condrestart)
[ -f $LOCKFILE ] && restart || :
;;
status)
[ -f $LOCKFILE ] && echo "Apache Yarn is running." || echo "Apache Yarn is not running."
RETVAL=$?
;;
*)
echo "Usage: $0 {start|stop|restart|reload|force-reload|condrestart}"
RETVAL=1
esac
exit $RETVAL
...@@ -3,5 +3,5 @@ mirror_url: "http://mirrors.myaegean.gr/apache/hadoop/common" ...@@ -3,5 +3,5 @@ mirror_url: "http://mirrors.myaegean.gr/apache/hadoop/common"
version: "2.7.0" version: "2.7.0"
download_path: "/root" download_path: "/root"
installation_path: "/usr/local" installation_path: "/usr/local"
dfs_replication: 2 dfs_replication: "{{ groups['slaves']|count }}"
...@@ -4,8 +4,10 @@ ...@@ -4,8 +4,10 @@
environment: proxy_env environment: proxy_env
- name: Uncompress Apache Kafka. - name: Uncompress Apache Kafka.
unarchive: src="{{ download_path }}/kafka_{{ scala_version }}-{{ version }}.tgz" dest="{{ installation_path }}" copy=no unarchive: src="{{ download_path }}/kafka_{{ scala_version }}-{{ version }}.tgz" dest="{{ installation_path }}" copy=no owner=kafka group=lambda
- name: Create softlink for Apache Kafka. - name: Create softlink for Apache Kafka.
file: src="{{ installation_path }}/kafka_{{ scala_version }}-{{ version }}" dest="{{ installation_path }}/kafka" state=link file: src="{{ installation_path }}/kafka_{{ scala_version }}-{{ version }}" dest="{{ installation_path }}/kafka" state=link
- name: Copy Apache Kafka init script.
template: src=kafka-init.j2 dest=/etc/init.d/kafka-init owner=kafka group=lambda mode=0740
--- ---
- name: Download and install Apache kafka. - name: Install Apache kafka.
include: download.yml include: install.yml
tags:
- download - name: Copy Apache Zookeeper init script.
template: src=zookeeper-init.j2 dest=/etc/init.d/zookeeper-init owner=kafka group=lambda mode=0740
- name: Configure Apache kafka. - name: Configure Apache kafka.
copy: src=server.properties dest="{{ installation_path }}/kafka/config/server.properties" owner=root group=root mode=0644 copy: src=server.properties dest="{{ installation_path }}/kafka/config/server.properties" owner=kafka group=lambda mode=0644
tags: tags:
- configure-kafka - configure-kafka
- name: Start Apache Zookeeper server. - name: Start Apache Zookeeper server.
shell: "{{ installation_path }}/kafka/bin/zookeeper-server-start.sh {{ installation_path }}/kafka/config/zookeeper.properties" shell: /etc/init.d/zookeeper-init start > /dev/null &
async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
poll: 0
tags: tags:
- start-zookeeper - start-zookeeper
...@@ -22,9 +21,7 @@ ...@@ -22,9 +21,7 @@
- start-zookeeper - start-zookeeper
- name: Start Apache Kafka server. - name: Start Apache Kafka server.
shell: "{{ installation_path }}/kafka/bin/kafka-server-start.sh {{ installation_path }}/kafka/config/server.properties" shell: /etc/init.d/kafka-init start > /dev/null &
async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
poll: 0
tags: tags:
- start-kafka - start-kafka
...@@ -35,4 +32,3 @@ ...@@ -35,4 +32,3 @@
- create topics - create topics
tags: tags:
- start-kafka - start-kafka
--- ---
- name: Download and install Apache kafka. - name: Install Apache kafka.
include: download.yml include: install.yml
tags:
- download
- name: Configure Apache kafka. - name: Configure Apache kafka.
template: src=server.properties.j2 dest="{{ installation_path }}/kafka/config/server.properties" owner=root group=root mode=0644 template: src=server.properties.j2 dest="{{ installation_path }}/kafka/config/server.properties" owner=kafka group=lambda mode=0644
tags: tags:
- configure-kafka - configure-kafka
- name: Start Apache Kafka server. - name: Start Apache Kafka server.
shell: "{{ installation_path }}/kafka/bin/kafka-server-start.sh {{ installation_path }}/kafka/config/server.properties" shell: /etc/init.d/kafka-init start > /dev/null &
async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
poll: 0
tags: tags:
- start-kafka - start-kafka
...@@ -20,4 +16,3 @@ ...@@ -20,4 +16,3 @@
wait_for: port=9092 wait_for: port=9092
tags: tags:
- start-kafka - start-kafka
#!/bin/bash
# Define the user that should run this script.
SCRIPT_USER=kafka
# The path where Apache Kafka is installed.
INSTALLATION_PATH="{{ installation_path }}"
# The full path of the pid file to use.
PIDFILE="$INSTALLATION_PATH/kafka/kafka.pid"
# The full path of the lock file to use.
LOCKFILE="$INSTALLATION_PATH/kafka/kafka-lock"
# The command that will start Apache Kafka.
START_COMMAND="$INSTALLATION_PATH/kafka/bin/kafka-server-start.sh $INSTALLATION_PATH/kafka/config/server.properties"
start(){
sudo -u $SCRIPT_USER /sbin/start-stop-daemon --start --background --make-pidfile --pidfile $PIDFILE --exec $START_COMMAND
RETVAL=$?
[ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER touch $LOCKFILE)
return $RETVAL
}
stop(){
sudo -u $SCRIPT_USER /sbin/start-stop-daemon --stop --remove-pidfile --pidfile $PIDFILE
RETVAL=$?
[ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER rm -f $LOCKFILE)
return $RETVAL
}
restart(){
stop
start
}
RETVAL=0
case "$1" in
start)
start
;;
stop)
stop
;;
restart|reload|force-reload)
restart
;;
condrestart)
[ -f $LOCKFILE ] && restart || :
;;
status)
[ -f $LOCKFILE ] && echo "Apache Kafka is running." || echo "Apache kafka is not running."
RETVAL=$?
;;
*)
echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
RETVAL=1
esac
exit $RETVAL
...@@ -117,7 +117,7 @@ log.cleaner.enable=false ...@@ -117,7 +117,7 @@ log.cleaner.enable=false
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
# You can also append an optional chroot string to the urls to specify the # You can also append an optional chroot string to the urls to specify the
# root directory for all kafka znodes. # root directory for all kafka znodes.
zookeeper.connect={{ hostvars['master-node']['internal_ip'] }}:2181 zookeeper.connect={{ hostvars[groups['master'][0]]['internal_ip'] }}:2181
# Timeout in ms for connecting to zookeeper # Timeout in ms for connecting to zookeeper
zookeeper.connection.timeout.ms=6000 zookeeper.connection.timeout.ms=6000
#!/bin/bash
# Define the user that should run this script.
SCRIPT_USER=kafka
# The path where Apache Zookeeper is installed.
INSTALLATION_PATH="{{ installation_path }}"
# The full path of the pid file to use.
PIDFILE="$INSTALLATION_PATH/kafka/zookeeper.pid"
# The full path of the lock file to use.
LOCKFILE="$INSTALLATION_PATH/kafka/zookeeper-lock"
# The command that will start Apache Zookeeper.
START_COMMAND="$INSTALLATION_PATH/kafka/bin/zookeeper-server-start.sh $INSTALLATION_PATH/kafka/config/zookeeper.properties"
start(){
sudo -u $SCRIPT_USER /sbin/start-stop-daemon --start --background --make-pidfile --pidfile $PIDFILE --exec $START_COMMAND
RETVAL=$?
[ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER touch $LOCKFILE)
return $RETVAL
}
stop(){
sudo -u $SCRIPT_USER /sbin/start-stop-daemon --stop --remove-pidfile --pidfile $PIDFILE
RETVAL=$?
[ $RETVAL -eq 0 ] && $(sudo -u $SCRIPT_USER rm -f $LOCKFILE)
return $RETVAL
}
restart(){
stop
start
}
RETVAL=0
case "$1" in
start)
start
;;
stop)
stop
;;
restart|reload|force-reload)
restart
;;
condrestart)