Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
itminedu
okeanos-LoD
Commits
6d18c39e
Commit
6d18c39e
authored
Jul 21, 2015
by
Georgios Ouzounis
Browse files
Options
Browse Files
Download
Plain Diff
Updated LAM-49 branch with new commits in devel branch.
parents
bac90107
f8b32c7d
Changes
18
Hide whitespace changes
Inline
Side-by-side
Showing
18 changed files
with
383 additions
and
52 deletions
+383
-52
ansible/roles/apache-flink/tasks/main.yml
ansible/roles/apache-flink/tasks/main.yml
+4
-25
ansible/roles/apache-flink/tasks/master.yml
ansible/roles/apache-flink/tasks/master.yml
+29
-0
ansible/roles/apache-flink/templates/flink-init.j2
ansible/roles/apache-flink/templates/flink-init.j2
+64
-0
ansible/roles/apache-flink/vars/main.yml
ansible/roles/apache-flink/vars/main.yml
+3
-3
ansible/roles/apache-hadoop/tasks/master.yml
ansible/roles/apache-hadoop/tasks/master.yml
+18
-2
ansible/roles/apache-hadoop/templates/hdfs-init.j2
ansible/roles/apache-hadoop/templates/hdfs-init.j2
+62
-0
ansible/roles/apache-hadoop/templates/yarn-init.j2
ansible/roles/apache-hadoop/templates/yarn-init.j2
+62
-0
ansible/roles/apache-hadoop/vars/main.yml
ansible/roles/apache-hadoop/vars/main.yml
+1
-1
ansible/roles/apache-kafka/tasks/install.yml
ansible/roles/apache-kafka/tasks/install.yml
+3
-0
ansible/roles/apache-kafka/tasks/master.yml
ansible/roles/apache-kafka/tasks/master.yml
+7
-10
ansible/roles/apache-kafka/tasks/slaves.yml
ansible/roles/apache-kafka/tasks/slaves.yml
+3
-7
ansible/roles/apache-kafka/templates/kafka-init.j2
ansible/roles/apache-kafka/templates/kafka-init.j2
+60
-0
ansible/roles/apache-kafka/templates/server.properties.j2
ansible/roles/apache-kafka/templates/server.properties.j2
+1
-1
ansible/roles/apache-kafka/templates/zookeeper-init.j2
ansible/roles/apache-kafka/templates/zookeeper-init.j2
+60
-0
ansible/roles/common/files/environment
ansible/roles/common/files/environment
+0
-2
ansible/roles/common/tasks/common-1.yml
ansible/roles/common/tasks/common-1.yml
+1
-1
ansible/roles/common/templates/environment.j2
ansible/roles/common/templates/environment.j2
+2
-0
ansible/roles/common/vars/main.yml
ansible/roles/common/vars/main.yml
+3
-0
No files found.
ansible/roles/apache-flink/tasks/main.yml
View file @
6d18c39e
---
-
name
:
Download Apache Flink, Yarn version.
get_url
:
url="{{ mirror_url }}/flink-{{ version }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz"
-
name
:
Include tasks for master.
include
:
master.yml
when
:
"
'master'
in
group_names"
tags
:
-
download
-
name
:
Uncompress Apache Flink.
unarchive
:
src="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ installation_path }}" copy=no owner=flink group=lambda
tags
:
-
uncompress
-
name
:
Create softlink for Apache Flink.
file
:
src="{{ installation_path }}/flink-{{ version }}" dest="{{ installation_path }}/flink" state=link
tags
:
-
uncompress
-
name
:
Configure Apache Flink.
template
:
src=flink-conf.j2 dest="{{ installation_path }}/flink/conf/flink-conf.yaml" owner=flink group=lambda mode=0644
tags
:
-
configure
-
name
:
Start Apache Flink.
shell
:
"
{{
installation_path
}}/flink/bin/yarn-session.sh
-n
{{
number_of_taskmanagers
}}
-tm
{{
ram_per_task_manager
}}"
async
:
31536000
# Stay alive for a year(1 year = 31536000 seconds).
poll
:
0
tags
:
-
start
-
master-install
ansible/roles/apache-flink/tasks/master.yml
0 → 100644
View file @
6d18c39e
---
-
name
:
Download Apache Flink, Yarn version.
get_url
:
url="{{ mirror_url }}/flink-{{ version }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz"
tags
:
-
download
-
name
:
Uncompress Apache Flink.
unarchive
:
src="{{ download_path }}/flink-{{ version }}-{{ version_for }}.tgz" dest="{{ installation_path }}" copy=no
tags
:
-
uncompress
-
name
:
Create softlink for Apache Flink.
file
:
src="{{ installation_path }}/flink-{{ version }}" dest="{{ installation_path }}/flink" state=link
tags
:
-
uncompress
-
name
:
Configure Apache Flink.
template
:
src=flink-conf.j2 dest="{{ installation_path }}/flink/conf/flink-conf.yaml" owner=root group=root mode=0644
tags
:
-
configure
-
name
:
Copy Apache Flink init script.
template
:
src=flink-init.j2 dest=/etc/init.d/flink-init owner=root group=root mode=0755
-
name
:
Start Apache Flink.
shell
:
/etc/init.d/flink-init start
# Should use Ansible module "service: name=flink-init state=started" when the bug in version 1.9.1 is fixed.
tags
:
-
start
ansible/roles/apache-flink/templates/flink-init.j2
0 → 100644
View file @
6d18c39e
#!/bin/bash
# Source environment file to make sure HADOOP_HOME variable is available.
source
/etc/environment
# The path where Apache Flink is installed.
INSTALLATION_PATH
=
"{{ installation_path }}"
# The full path of the lock file to use.
LOCKFILE
=
"
$INSTALLATION_PATH
/flink/flink-lock"
# The command that will start Apache Flink.
START_COMMAND
=
"
$INSTALLATION_PATH
/flink/bin/yarn-session.sh -n {{ number_of_taskmanagers }} -tm {{ ram_per_task_manager }}"
# The command that will stop Apache Flink. Note that, HADOOP_HOME home variable should be set outside from this script and
# before Flink's deployment.
STOP_COMMAND
=
"
$HADOOP_HOME
/bin/yarn application --kill"
start
(){
nohup
$START_COMMAND
>
/dev/null &
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
touch
$LOCKFILE
return
$RETVAL
}
stop
(){
id
=
$(
$HADOOP_HOME
/bin/yarn application
--list
|
grep
"Flink session"
|
cut
-f1
)
nohup
$STOP_COMMAND
$id
>
/dev/null &
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
rm
-f
$LOCKFILE
)
return
$RETVAL
}
restart
(){
stop
start
}
RETVAL
=
0
case
"
$1
"
in
start
)
start
;;
stop
)
stop
;;
restart|reload|force-reload
)
restart
;;
condrestart
)
[
-f
$LOCKFILE
]
&&
restart
||
:
;;
status
)
[
-f
$LOCKFILE
]
&&
echo
"Apache Flink is running."
||
echo
"Apache Flink is not running."
RETVAL
=
$?
;;
*
)
echo
"Usage:
$0
{start|stop|status|restart|reload|force-reload|condrestart}"
RETVAL
=
1
esac
exit
$RETVAL
ansible/roles/apache-flink/vars/main.yml
View file @
6d18c39e
...
...
@@ -6,8 +6,8 @@ download_path: "/root"
installation_path
:
"
/usr/local"
jobmanager_heap_mb
:
256
taskmanager_heap_mb
:
512
taskmanager_numberOfTaskSlots
:
2
parallelization_degree_default
:
2
number_of_taskmanagers
:
2
taskmanager_numberOfTaskSlots
:
"
{{
ansible_processor_count
*
ansible_processor_cores
}}"
parallelization_degree_default
:
"
{{
(ansible_processor_count
*
ansible_processor_cores
*
groups['slaves']|count
/
2)|int
}}"
number_of_taskmanagers
:
"
{{
groups['slaves']|count
}}"
ram_per_task_manager
:
768
ansible/roles/apache-hadoop/tasks/master.yml
View file @
6d18c39e
---
-
name
:
Copy Apache HDFS init script.
template
:
src=hdfs-init.j2 dest=/etc/init.d/hdfs-init owner=root group=root mode=0755
-
name
:
Copy Apache Yarn init script.
template
:
src=yarn-init.j2 dest=/etc/init.d/yarn-init owner=root group=root mode=0755
-
name
:
Format Apache HDFS.
shell
:
"
yes
|
{{
installation_path
}}/hadoop/bin/hadoop
namenode
-format"
tags
:
-
format-hdfs
-
name
:
Start Apache HDFS.
shell
:
"
{{
installation_path
}}/hadoop/sbin/start-dfs.sh"
shell
:
/etc/init.d/hdfs-init start
# Should use Ansible module "service: name=hdfs-init state=started" when the bug in version 1.9.1 is fixed.
tags
:
-
start-hdfs
-
name
:
Wait for Apache HDFS to become available.
wait_for
:
host="{{ hostvars[inventory_hostname]["internal_ip"] }}" port=9000
tags
:
-
start-hdfs
-
name
:
Start Apache Yarn.
shell
:
"
{{
installation_path
}}/hadoop/sbin/start-yarn.sh"
shell
:
/etc/init.d/yarn-init start
# Should use Ansible module "service: name=yarn-init state=started" when the bug in version 1.9.1 is fixed.
tags
:
-
start-yarn
-
name
:
Wait for Apache Yarn to become available.
wait_for
:
host="{{ hostvars[inventory_hostname]["internal_ip"] }}" port=8050
# 8050 is the port of the resource manager.
tags
:
-
start-yarn
...
...
ansible/roles/apache-hadoop/templates/hdfs-init.j2
0 → 100644
View file @
6d18c39e
#!/bin/bash
# Source environment file to make sure JAVA_HOME variable is available.
source
/etc/environment
# The path where Apache Hadoop is installed.
INSTALLATION_PATH
=
"{{ installation_path }}"
# The full path of the lock file to use.
LOCKFILE
=
"
$INSTALLATION_PATH
/hadoop/hdfs-lock"
# The command that will start Apache HDFS.
START_COMMAND
=
"
$INSTALLATION_PATH
/hadoop/sbin/start-dfs.sh"
# The command that will stop Apache HDFS.
STOP_COMMAND
=
"
$INSTALLATION_PATH
/hadoop/sbin/stop-dfs.sh"
start
(){
nohup
$START_COMMAND
>
/dev/null &
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
touch
$LOCKFILE
return
$RETVAL
}
stop
(){
nohup
$STOP_COMMAND
>
/dev/null &
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
rm
-f
$LOCKFILE
)
return
$RETVAL
}
restart
(){
stop
start
}
RETVAL
=
0
case
"
$1
"
in
start
)
start
;;
stop
)
stop
;;
restart|reload|force-reload
)
restart
;;
condrestart
)
[
-f
$LOCKFILE
]
&&
restart
||
:
;;
status
)
[
-f
$LOCKFILE
]
&&
echo
"Apache HDFS is running."
||
echo
"Apache HDFS is not running."
RETVAL
=
$?
;;
*
)
echo
"Usage:
$0
{start|stop|status|restart|reload|force-reload|condrestart}"
RETVAL
=
1
esac
exit
$RETVAL
ansible/roles/apache-hadoop/templates/yarn-init.j2
0 → 100644
View file @
6d18c39e
#!/bin/bash
# Source environment file to make sure JAVA_HOME variable is available.
source
/etc/environment
# The path where Apache Hadoop is installed.
INSTALLATION_PATH
=
"{{ installation_path }}"
# The full path of the lock file to use.
LOCKFILE
=
"
$INSTALLATION_PATH
/hadoop/yarn-lock"
# The command that will start Apache Yarn.
START_COMMAND
=
"
$INSTALLATION_PATH
/hadoop/sbin/start-yarn.sh"
# The command that will stop Apache Yarn.
STOP_COMMAND
=
"
$INSTALLATION_PATH
/hadoop/sbin/stop-yarn.sh"
start
(){
nohup
$START_COMMAND
>
/dev/null &
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
touch
$LOCKFILE
return
$RETVAL
}
stop
(){
nohup
$STOP_COMMAND
>
/dev/null &
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
rm
-f
$LOCKFILE
)
return
$RETVAL
}
restart
(){
stop
start
}
RETVAL
=
0
case
"
$1
"
in
start
)
start
;;
stop
)
stop
;;
restart|reload|force-reload
)
restart
;;
condrestart
)
[
-f
$LOCKFILE
]
&&
restart
||
:
;;
status
)
[
-f
$LOCKFILE
]
&&
echo
"Apache Yarn is running."
||
echo
"Apache Yarn is not running."
RETVAL
=
$?
;;
*
)
echo
"Usage:
$0
{start|stop|restart|reload|force-reload|condrestart}"
RETVAL
=
1
esac
exit
$RETVAL
ansible/roles/apache-hadoop/vars/main.yml
View file @
6d18c39e
...
...
@@ -3,5 +3,5 @@ mirror_url: "http://mirrors.myaegean.gr/apache/hadoop/common"
version
:
"
2.7.0"
download_path
:
"
/root"
installation_path
:
"
/usr/local"
dfs_replication
:
2
dfs_replication
:
"
{{
groups['slaves']|count
}}"
ansible/roles/apache-kafka/tasks/
download
.yml
→
ansible/roles/apache-kafka/tasks/
install
.yml
View file @
6d18c39e
...
...
@@ -7,3 +7,6 @@
-
name
:
Create softlink for Apache Kafka.
file
:
src="{{ installation_path }}/kafka_{{ scala_version }}-{{ version }}" dest="{{ installation_path }}/kafka" state=link
-
name
:
Copy Apache Kafka init script.
template
:
src=kafka-init.j2 dest=/etc/init.d/kafka-init owner=kafka group=lambda mode=0754
ansible/roles/apache-kafka/tasks/master.yml
View file @
6d18c39e
---
-
name
:
Download and install Apache kafka.
include
:
download.yml
tags
:
-
download
-
name
:
Install Apache kafka.
include
:
install.yml
-
name
:
Copy Apache Zookeeper init script.
template
:
src=zookeeper-init.j2 dest=/etc/init.d/zookeeper-init owner=root group=root mode=0755
-
name
:
Configure Apache kafka.
copy
:
src=server.properties dest="{{ installation_path }}/kafka/config/server.properties" owner=kafka group=lambda mode=0644
...
...
@@ -10,9 +11,7 @@
-
configure-kafka
-
name
:
Start Apache Zookeeper server.
shell
:
"
{{
installation_path
}}/kafka/bin/zookeeper-server-start.sh
{{
installation_path
}}/kafka/config/zookeeper.properties"
async
:
31536000
# Stay alive for a year(1 year = 31536000 seconds).
poll
:
0
shell
:
/etc/init.d/zookeeper-init start
# Should use Ansible module "service: name=zookeeper-init state=started" when the bug in version 1.9.1 is fixed.
tags
:
-
start-zookeeper
...
...
@@ -22,9 +21,7 @@
-
start-zookeeper
-
name
:
Start Apache Kafka server.
shell
:
"
{{
installation_path
}}/kafka/bin/kafka-server-start.sh
{{
installation_path
}}/kafka/config/server.properties"
async
:
31536000
# Stay alive for a year(1 year = 31536000 seconds).
poll
:
0
shell
:
/etc/init.d/kafka-init start
# Should use Ansible module "service: name=kafka-init state=started" when the bug in version 1.9.1 is fixed.
tags
:
-
start-kafka
...
...
ansible/roles/apache-kafka/tasks/slaves.yml
View file @
6d18c39e
---
-
name
:
Download and install Apache kafka.
include
:
download.yml
tags
:
-
download
-
name
:
Install Apache kafka.
include
:
install.yml
-
name
:
Configure Apache kafka.
template
:
src=server.properties.j2 dest="{{ installation_path }}/kafka/config/server.properties" owner=kafka group=lambda mode=0644
...
...
@@ -10,9 +8,7 @@
-
configure-kafka
-
name
:
Start Apache Kafka server.
shell
:
"
{{
installation_path
}}/kafka/bin/kafka-server-start.sh
{{
installation_path
}}/kafka/config/server.properties"
async
:
31536000
# Stay alive for a year(1 year = 31536000 seconds).
poll
:
0
shell
:
/etc/init.d/kafka-init start
# Should use ansible module service: name=zookeeper-init state=started when the bug in version 1.9.1 is fixed.
tags
:
-
start-kafka
...
...
ansible/roles/apache-kafka/templates/kafka-init.j2
0 → 100644
View file @
6d18c39e
#!/bin/bash
# The path where Apache Kafka is installed.
INSTALLATION_PATH
=
"{{ installation_path }}"
# The full path of the pid file to use.
PIDFILE
=
"
$INSTALLATION_PATH
/kafka/kafka.pid"
# The full path of the lock file to use.
LOCKFILE
=
"
$INSTALLATION_PATH
/kafka/kafka-lock"
# The command that will start Apache Kafka.
START_COMMAND
=
"
$INSTALLATION_PATH
/kafka/bin/kafka-server-start.sh
$INSTALLATION_PATH
/kafka/config/server.properties"
start
(){
touch
$PIDFILE
nohup
$START_COMMAND
>
/dev/null &
echo
$!
>
$PIDFILE
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
touch
$LOCKFILE
return
$RETVAL
}
stop
(){
kill
-s
9
$(
cat
$PIDFILE
)
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
rm
-f
$LOCKFILE
)
return
$RETVAL
}
restart
(){
stop
start
}
RETVAL
=
0
case
"
$1
"
in
start
)
start
;;
stop
)
stop
;;
restart|reload|force-reload
)
restart
;;
condrestart
)
[
-f
$LOCKFILE
]
&&
restart
||
:
;;
status
)
[
-f
$LOCKFILE
]
&&
echo
"Apache Kafka is running."
||
echo
"Apache kafka is not running."
RETVAL
=
$?
;;
*
)
echo
"Usage:
$0
{start|stop|status|restart|reload|force-reload|condrestart}"
RETVAL
=
1
esac
exit
$RETVAL
ansible/roles/apache-kafka/templates/server.properties.j2
View file @
6d18c39e
...
...
@@ -117,7 +117,7 @@ log.cleaner.enable=false
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
# You can also append an optional chroot string to the urls to specify the
# root directory for all kafka znodes.
zookeeper.connect={{ hostvars['master
-node'
]['internal_ip'] }}:2181
zookeeper.connect={{ hostvars[
groups[
'master
'][0]
]['internal_ip'] }}:2181
# Timeout in ms for connecting to zookeeper
zookeeper.connection.timeout.ms=6000
ansible/roles/apache-kafka/templates/zookeeper-init.j2
0 → 100644
View file @
6d18c39e
#!/bin/bash
# The path where Apache Zookeeper is installed.
INSTALLATION_PATH
=
"{{ installation_path }}"
# The full path of the pid file to use.
PIDFILE
=
"
$INSTALLATION_PATH
/kafka/zookeeper.pid"
# The full path of the lock file to use.
LOCKFILE
=
"
$INSTALLATION_PATH
/kafka/zookeeper-lock"
# The command that will start Apache Zookeeper.
START_COMMAND
=
"
$INSTALLATION_PATH
/kafka/bin/zookeeper-server-start.sh
$INSTALLATION_PATH
/kafka/config/zookeeper.properties"
start
(){
touch
$PIDFILE
nohup
$START_COMMAND
>
/dev/null &
echo
$!
>
$PIDFILE
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
touch
$LOCKFILE
return
$RETVAL
}
stop
(){
kill
-s
9
$(
cat
$PIDFILE
)
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
rm
-f
$LOCKFILE
)
return
$RETVAL
}
restart
(){
stop
start
}
RETVAL
=
0
case
"
$1
"
in
start
)
start
;;
stop
)
stop
;;
restart|reload|force-reload
)
restart
;;
condrestart
)
[
-f
$LOCKFILE
]
&&
restart
||
:
;;
status
)
[
-f
$LOCKFILE
]
&&
echo
"Apache Zookeeper is running."
||
echo
"Apache Zookeeper is not running."
RETVAL
=
$?
;;
*
)
echo
"Usage:
$0
{start|stop|status|restart|reload|force-reload|condrestart}"
RETVAL
=
1
esac
exit
$RETVAL
ansible/roles/common/files/environment
deleted
100644 → 0
View file @
bac90107
export JAVA_HOME=/usr/
export HADOOP_HOME=/usr/local/hadoop
ansible/roles/common/tasks/common-1.yml
View file @
6d18c39e
...
...
@@ -9,7 +9,7 @@
template
:
src=hosts.j2 dest=/etc/hosts backup=no owner=root group=lambda mode=0750
-
name
:
Copy environment file.
copy
:
src=environment dest=/etc/environment backup=no owner=root group=root mode=0750
template
:
src=environment
.j2
dest=/etc/environment backup=no owner=root group=root mode=0750
-
name
:
Generate ssh key.
shell
:
cat /dev/zero | ssh-keygen -q -N ""
...
...
ansible/roles/common/templates/environment.j2
0 → 100644
View file @
6d18c39e
export JAVA_HOME={{ java_home }}
export HADOOP_HOME={{ hadoop_home }}
ansible/roles/common/vars/main.yml
0 → 100644
View file @
6d18c39e
---
java_home
:
"
/usr"
hadoop_home
:
"
/usr/local/hadoop"
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment