Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
O
okeanos-LoD
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Incidents
Environments
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
itminedu
okeanos-LoD
Commits
de4fb9ca
Commit
de4fb9ca
authored
Sep 09, 2015
by
Christos Kanellopoulos
Browse files
Options
Browse Files
Download
Plain Diff
Merge pull request #75 from grnet/devel
Merge devel to master
parents
bb81b7d8
bef724a2
Changes
84
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
84 changed files
with
3922 additions
and
427 deletions
+3922
-427
.travis.yml
.travis.yml
+10
-35
ansible/README.md
ansible/README.md
+0
-3
ansible/hosts
ansible/hosts
+6
-0
ansible/playbooks/initialize.yml
ansible/playbooks/initialize.yml
+12
-0
ansible/roles/.DS_Store
ansible/roles/.DS_Store
+0
-0
ansible/roles/apache-flink/tasks/master.yml
ansible/roles/apache-flink/tasks/master.yml
+1
-1
ansible/roles/apache-flink/templates/flink-init.j2
ansible/roles/apache-flink/templates/flink-init.j2
+42
-2
ansible/roles/apache-hadoop/tasks/master.yml
ansible/roles/apache-hadoop/tasks/master.yml
+2
-2
ansible/roles/apache-hadoop/tasks/setup.yml
ansible/roles/apache-hadoop/tasks/setup.yml
+7
-2
ansible/roles/apache-hadoop/templates/core-site.xml.j2
ansible/roles/apache-hadoop/templates/core-site.xml.j2
+1
-1
ansible/roles/apache-hadoop/templates/hdfs-init.j2
ansible/roles/apache-hadoop/templates/hdfs-init.j2
+15
-2
ansible/roles/apache-hadoop/templates/hdfs-site-master.xml.j2
...ble/roles/apache-hadoop/templates/hdfs-site-master.xml.j2
+4
-0
ansible/roles/apache-hadoop/templates/hdfs-site-slave.xml.j2
ansible/roles/apache-hadoop/templates/hdfs-site-slave.xml.j2
+32
-0
ansible/roles/apache-hadoop/templates/yarn-init.j2
ansible/roles/apache-hadoop/templates/yarn-init.j2
+15
-2
ansible/roles/apache-kafka/tasks/master.yml
ansible/roles/apache-kafka/tasks/master.yml
+2
-2
ansible/roles/apache-kafka/templates/kafka-init.j2
ansible/roles/apache-kafka/templates/kafka-init.j2
+15
-2
ansible/roles/apache-kafka/templates/zookeeper-init.j2
ansible/roles/apache-kafka/templates/zookeeper-init.j2
+15
-2
ansible/roles/common/files/sources.list
ansible/roles/common/files/sources.list
+11
-0
ansible/roles/common/tasks/all.yml
ansible/roles/common/tasks/all.yml
+0
-10
ansible/roles/common/tasks/common-1.yml
ansible/roles/common/tasks/common-1.yml
+17
-0
ansible/roles/common/tasks/main.yml
ansible/roles/common/tasks/main.yml
+9
-4
ansible/roles/common/tasks/master.yml
ansible/roles/common/tasks/master.yml
+12
-8
ansible/roles/common/templates/lambda-init.j2
ansible/roles/common/templates/lambda-init.j2
+246
-0
ansible/roles/common/templates/supervisord-master.conf.j2
ansible/roles/common/templates/supervisord-master.conf.j2
+160
-0
ansible/roles/common/templates/supervisord-slaves.conf.j2
ansible/roles/common/templates/supervisord-slaves.conf.j2
+147
-0
ansible/roles/common/vars/main.yml
ansible/roles/common/vars/main.yml
+4
-0
core/README.md
core/README.md
+5
-5
core/fokia/ansible_manager.py
core/fokia/ansible_manager.py
+2
-1
core/fokia/cluster_creator.py
core/fokia/cluster_creator.py
+0
-90
core/fokia/lambda_instance_manager.py
core/fokia/lambda_instance_manager.py
+142
-0
core/fokia/provisioner.py
core/fokia/provisioner.py
+109
-130
core/fokia/utils.py
core/fokia/utils.py
+176
-0
core/setup.py
core/setup.py
+4
-1
core/tests/test_provisioner.py
core/tests/test_provisioner.py
+58
-51
core/tox.ini
core/tox.ini
+6
-5
setup.cfg
setup.cfg
+4
-0
webapp/ansible/README.md
webapp/ansible/README.md
+23
-1
webapp/ansible/ansible.cfg
webapp/ansible/ansible.cfg
+7
-0
webapp/ansible/playbooks/celery-install.yml
webapp/ansible/playbooks/celery-install.yml
+9
-0
webapp/ansible/playbooks/setup.yml
webapp/ansible/playbooks/setup.yml
+4
-1
webapp/ansible/roles/celery/files/supervisord-init
webapp/ansible/roles/celery/files/supervisord-init
+45
-0
webapp/ansible/roles/celery/files/supervisord.conf
webapp/ansible/roles/celery/files/supervisord.conf
+158
-0
webapp/ansible/roles/celery/handlers/main.yml
webapp/ansible/roles/celery/handlers/main.yml
+3
-0
webapp/ansible/roles/celery/tasks/install.yml
webapp/ansible/roles/celery/tasks/install.yml
+5
-0
webapp/ansible/roles/celery/tasks/main.yml
webapp/ansible/roles/celery/tasks/main.yml
+8
-0
webapp/ansible/roles/celery/tasks/supervisord.yml
webapp/ansible/roles/celery/tasks/supervisord.yml
+13
-0
webapp/ansible/roles/common/files/sources.list
webapp/ansible/roles/common/files/sources.list
+11
-0
webapp/ansible/roles/common/tasks/main.yml
webapp/ansible/roles/common/tasks/main.yml
+15
-0
webapp/ansible/roles/rabbitmq/tasks/install.yml
webapp/ansible/roles/rabbitmq/tasks/install.yml
+5
-0
webapp/ansible/roles/rabbitmq/tasks/main.yml
webapp/ansible/roles/rabbitmq/tasks/main.yml
+5
-0
webapp/ansible/roles/service-vm/tasks/main.yml
webapp/ansible/roles/service-vm/tasks/main.yml
+12
-14
webapp/ansible/roles/service-vm/templates/lambda-service-backend.conf.j2
...roles/service-vm/templates/lambda-service-backend.conf.j2
+5
-1
webapp/ansible/roles/service-vm/templates/settings.py.j2
webapp/ansible/roles/service-vm/templates/settings.py.j2
+58
-3
webapp/ansible/roles/service-vm/vars/main.yml
webapp/ansible/roles/service-vm/vars/main.yml
+2
-0
webapp/api-doc/CreateLambdaInstance.md
webapp/api-doc/CreateLambdaInstance.md
+72
-0
webapp/api-doc/docs/Authentication.md
webapp/api-doc/docs/Authentication.md
+58
-0
webapp/api-doc/docs/LambdaInstanceDestroy.md
webapp/api-doc/docs/LambdaInstanceDestroy.md
+67
-0
webapp/api-doc/docs/LambdaInstanceDetails.md
webapp/api-doc/docs/LambdaInstanceDetails.md
+101
-0
webapp/api-doc/docs/LambdaInstanceList.md
webapp/api-doc/docs/LambdaInstanceList.md
+100
-0
webapp/api-doc/docs/LambdaInstanceStart.md
webapp/api-doc/docs/LambdaInstanceStart.md
+61
-0
webapp/api-doc/docs/LambdaInstanceStatus.md
webapp/api-doc/docs/LambdaInstanceStatus.md
+74
-0
webapp/api-doc/docs/LambdaInstanceStop.md
webapp/api-doc/docs/LambdaInstanceStop.md
+61
-0
webapp/api-doc/docs/Upload.md
webapp/api-doc/docs/Upload.md
+58
-0
webapp/api-doc/docs/index.md
webapp/api-doc/docs/index.md
+21
-0
webapp/api-doc/docs/template.md
webapp/api-doc/docs/template.md
+11
-9
webapp/api-doc/mkdocs.yml
webapp/api-doc/mkdocs.yml
+15
-0
webapp/backend/admin.py
webapp/backend/admin.py
+0
-1
webapp/backend/authenticate_user.py
webapp/backend/authenticate_user.py
+42
-0
webapp/backend/events.py
webapp/backend/events.py
+72
-0
webapp/backend/models.py
webapp/backend/models.py
+247
-29
webapp/backend/queries.py
webapp/backend/queries.py
+304
-0
webapp/backend/serializers.py
webapp/backend/serializers.py
+42
-0
webapp/backend/tasks.py
webapp/backend/tasks.py
+206
-0
webapp/backend/templates/display-servers.html
webapp/backend/templates/display-servers.html
+39
-0
webapp/backend/tests.py
webapp/backend/tests.py
+0
-3
webapp/backend/urls.py
webapp/backend/urls.py
+22
-0
webapp/backend/views.py
webapp/backend/views.py
+346
-2
webapp/requirements.txt
webapp/requirements.txt
+7
-0
webapp/tests/__init__.py
webapp/tests/__init__.py
+2
-0
webapp/tests/test_user_files.py
webapp/tests/test_user_files.py
+37
-0
webapp/webapp/__init__.py
webapp/webapp/__init__.py
+7
-0
webapp/webapp/celery.py
webapp/webapp/celery.py
+22
-0
webapp/webapp/settings.py
webapp/webapp/settings.py
+148
-0
webapp/webapp/urls.py
webapp/webapp/urls.py
+1
-2
No files found.
.travis.yml
View file @
de4fb9ca
---
language
:
python
python
:
"
2.7"
before_install
:
# Make sure everything's up to date.
-
sudo apt-get update -qq
install
:
# Install Ansible.
-
pip install ansible
# Add ansible.cfg to pick up roles path.
-
"
printf
'[defaults]
\n
roles_path
=
../../'
>
ansible.cfg"
python
:
-
"
2.7"
install
:
-
pip install tox
-
pip install flake8-diff
script
:
# We'll add some commands to test the role here.
#- cd ansible/roles/common/tests && ansible-playbook -i inventory test.yml --syntax-check
#- ansible-playbook -i inventory test.yml --connection=local --sudo
#- "ansible-playbook -i inventory test.yml --connection=local --sudo | tee /tmp/output.txt; grep -q 'changed=0.*failed=0' /tmp/output.txt && (echo 'Idempotence test: pass' && exit 0) || (echo 'Idempotence test: fail' && exit 1)"
-
cd ansible/roles/apache-flink/tests
# Check the role/playbook's syntax.
-
ansible-playbook -i inventory test.yml --syntax-check
# Run the role/playbook with ansible-playbook.
-
ansible-playbook -i inventory test.yml --connection=local --sudo
# Run the role/playbook again, checking to make sure it's idempotent.
-
ansible-playbook -i inventory test.yml --connection=local --sudo
| grep -q 'changed=0.*failed=0'
&& (echo 'Idempotence test
:
pass' && exit 0)
|
|
(echo
'Idempotence
test:
fail'
&&
exit
1)
# Some MySQL debugging (show all the logs).
-
"
sudo
ls
-lah
/var/log"
-
"
sudo
cat
/var/log/apache-flink/error.log"
-
"
sudo
wget
-O
hamlet.txt
http://www.gutenberg.org/cache/epub/1787/pg1787.txt"
-
"
sudo
bin/flink
run
./examples/flink-java-examples-0.8.1-WordCount.jar
file://`pwd`/hamlet.txt
file://`pwd`/wordcount-result.txt"
-
git fetch origin $TRAVIS_BRANCH:travis_pr_branch
-
flake8-diff travis_pr_branch
-
cd core && tox -e $TOXENV_CORE
env
:
-
TOXENV_CORE=py27
ansible/README.md
View file @
de4fb9ca
...
...
@@ -69,9 +69,6 @@ There are four (4) roles and five (5) playbooks. These are:
-
Downloads and installs Apache Flink on master node.
-
Starts and Apache Flink, Yarn session.
## How to deploy
You can deploy the whole cluster by running the cluster-install playbook:
...
...
ansible/hosts
0 → 100644
View file @
de4fb9ca
[master]
snf-669832.vm.okeanos.grnet.gr
[slaves]
snf-669833.vm.okeanos.grnet.gr
snf-669834.vm.okeanos.grnet.gr
ansible/playbooks/initialize.yml
0 → 100644
View file @
de4fb9ca
---
-
hosts
:
master
user
:
root
gather_facts
:
no
roles
:
-
wait_for_ssh
-
hosts
:
master
user
:
root
roles
:
-
proxy
ansible/roles/.DS_Store
deleted
100644 → 0
View file @
bb81b7d8
File deleted
ansible/roles/apache-flink/tasks/master.yml
View file @
de4fb9ca
...
...
@@ -24,7 +24,7 @@
template
:
src=flink-init.j2 dest=/etc/init.d/flink-init owner=flink group=lambda mode=0740
-
name
:
Start Apache Flink.
shell
:
/etc/init.d/flink-init start > /dev/null
&
shell
:
/etc/init.d/flink-init start > /dev/null
tags
:
-
start
ansible/roles/apache-flink/templates/flink-init.j2
View file @
de4fb9ca
...
...
@@ -9,6 +9,10 @@ SCRIPT_USER=flink
# The path where Apache Flink is installed.
INSTALLATION_PATH
=
"{{ installation_path }}"
# The full path of the pid file to use. Apache Flink is run as an Apache Yarn application. The id of this application
# is stored on this file.
APPLICATION_ID
=
"
$INSTALLATION_PATH
/flink/flink.pid"
# The full path of the lock file to use.
LOCKFILE
=
"
$INSTALLATION_PATH
/flink/flink-lock"
...
...
@@ -20,15 +24,50 @@ START_COMMAND="$INSTALLATION_PATH/flink/bin/yarn-session.sh -n {{ number_of_task
STOP_COMMAND
=
"
$HADOOP_HOME
/bin/yarn application --kill"
start
(){
# Assert that there is no other Apache Flink instance, created with this script, running.
[
-f
$LOCKFILE
]
&&
return
0
# Execute the command to start Apache Flink.
sudo
-E
-u
$SCRIPT_USER
nohup
$START_COMMAND
>
/dev/null &
# Get the returned value and create a lock file to prevent multiple instantiations.
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
sudo
-u
$SCRIPT_USER
touch
$LOCKFILE
)
# Wait for a minute Apache Flink to start. The condition check whether an application under user "flink" has been
# started on Apache Yarn and whether a minute has passed.
i
=
0
while
[
"
$(
sudo
-u
$SCRIPT_USER
$HADOOP_HOME
/bin/yarn application
--list
|
cut
-f4
|
grep
"flink"
)
"
==
""
]
&&
[
$i
-lt
6
]
do
sleep
10
i
=
$((
i+1
))
done
# Save the application id of this Apache Flink application.
$(
sudo
-u
$SCRIPT_USER
$HADOOP_HOME
/bin/yarn application
--list
|
cut
-f1
,4 |
grep
"flink"
|
cut
-f1
>
$APPLICATION_ID
)
chown
$SCRIPT_USER
$APPLICATION_ID
chmod
644
$APPLICATION_ID
return
$RETVAL
}
stop
(){
id
=
$(
sudo
-u
$SCRIPT_USER
$HADOOP_HOME
/bin/yarn application
--list
|
grep
"Flink session"
|
cut
-f1
)
sudo
-E
-u
$SCRIPT_USER
nohup
$STOP_COMMAND
$id
>
/dev/null &
# Assert that an Apache Flink instance, created with this script, is running.
[
!
-f
$LOCKFILE
]
&&
return
0
# Read the application id and execute the command to stop Apache Flink. The command will block
# until the service has been stopped.
id
=
$(
sudo
-u
$SCRIPT_USER
cat
$APPLICATION_ID
)
sudo
-E
-u
$SCRIPT_USER
$STOP_COMMAND
$id
>
/dev/null
# Delete the files on Apache HDFS created when Apache Flink was started.
# These files are created by Apache Yarn to distribute Flink accross all nodes.
sudo
-u
$SCRIPT_USER
$HADOOP_HOME
/bin/hadoop fs
-rm
-r
-skipTrash
/user/flink/.flink/
$id
# Delete application id file.
sudo
-u
$SCRIPT_USER
rm
$APPLICATION_ID
# Get the returned value of the executed command and remove the lock file.
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
sudo
-u
$SCRIPT_USER
rm
-f
$LOCKFILE
)
return
$RETVAL
...
...
@@ -55,6 +94,7 @@ case "$1" in
[
-f
$LOCKFILE
]
&&
restart
||
:
;;
status
)
# If the lock file exists, then Apache Flink is running.
[
-f
$LOCKFILE
]
&&
echo
"Apache Flink is running."
||
echo
"Apache Flink is not running."
RETVAL
=
$?
;;
...
...
ansible/roles/apache-hadoop/tasks/master.yml
View file @
de4fb9ca
...
...
@@ -11,7 +11,7 @@
-
format-hdfs
-
name
:
Start Apache HDFS.
shell
:
/etc/init.d/hdfs-init start > /dev/null
&
shell
:
/etc/init.d/hdfs-init start > /dev/null
tags
:
-
start-hdfs
...
...
@@ -21,7 +21,7 @@
-
start-hdfs
-
name
:
Start Apache Yarn.
shell
:
/etc/init.d/yarn-init start > /dev/null
&
shell
:
/etc/init.d/yarn-init start > /dev/null
tags
:
-
start-yarn
...
...
ansible/roles/apache-hadoop/tasks/setup.yml
View file @
de4fb9ca
...
...
@@ -18,8 +18,13 @@
-
name
:
Configure core.
template
:
src=core-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/core-site.xml" owner=hduser group=lambda mode=0644
-
name
:
Configure Apache HDFS.
template
:
src=hdfs-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/hdfs-site.xml" backup=no owner=hduser group=lambda mode=0644
-
name
:
Configure Apache HDFS for master node.
template
:
src=hdfs-site-master.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/hdfs-site.xml" backup=no owner=hduser group=lambda mode=0644
when
:
"
'master'
in
group_names"
-
name
:
Configure Apache HDFS for slave nodes.
template
:
src=hdfs-site-slave.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/hdfs-site.xml" backup=no owner=hduser group=lambda mode=0644
when
:
"
'slaves'
in
group_names"
-
name
:
Configure Apache Yarn.
template
:
src=yarn-site.xml.j2 dest="{{ installation_path }}/hadoop/etc/hadoop/yarn-site.xml" owner=hduser group=lambda mode=0644
ansible/roles/apache-hadoop/templates/core-site.xml.j2
View file @
de4fb9ca
...
...
@@ -18,7 +18,7 @@
<configuration>
<property>
<name>
fs.default
.name
</name>
<name>
fs.default
FS
</name>
<value>
hdfs://{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:9000
</value>
</property>
<property>
...
...
ansible/roles/apache-hadoop/templates/hdfs-init.j2
View file @
de4fb9ca
...
...
@@ -19,14 +19,26 @@ START_COMMAND="$INSTALLATION_PATH/hadoop/sbin/start-dfs.sh"
STOP_COMMAND
=
"
$INSTALLATION_PATH
/hadoop/sbin/stop-dfs.sh"
start
(){
sudo
-u
$SCRIPT_USER
nohup
$START_COMMAND
>
/dev/null &
# Assert that there is no other Apache HDFS instance, created with this script, running.
[
-f
$LOCKFILE
]
&&
return
0
# Execute the command to start Apache HDFS. The command waits until HDFS has been started.
sudo
-u
$SCRIPT_USER
$START_COMMAND
>
/dev/null
# Get the returned value of the executed command and create a lock file to prevent multiple instantiations.
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
sudo
-u
$SCRIPT_USER
touch
$LOCKFILE
)
return
$RETVAL
}
stop
(){
sudo
-u
$SCRIPT_USER
nohup
$STOP_COMMAND
>
/dev/null &
# Assert that an Apache HDFS instance, created with this script, is running.
[
!
-f
$LOCKFILE
]
&&
return
0
# Execute the command to stop Apache HDFS. The command waits until HDFS has been stopped.
sudo
-u
$SCRIPT_USER
$STOP_COMMAND
>
/dev/null
# Get the returned value of the executed command and delete the lock file.
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
sudo
-u
$SCRIPT_USER
rm
-f
$LOCKFILE
)
return
$RETVAL
...
...
@@ -53,6 +65,7 @@ case "$1" in
[
-f
$LOCKFILE
]
&&
restart
||
:
;;
status
)
# If the lock file exists, then Apache HDFS is running.
[
-f
$LOCKFILE
]
&&
echo
"Apache HDFS is running."
||
echo
"Apache HDFS is not running."
RETVAL
=
$?
;;
...
...
ansible/roles/apache-hadoop/templates/hdfs-site.xml.j2
→
ansible/roles/apache-hadoop/templates/hdfs-site
-master
.xml.j2
View file @
de4fb9ca
...
...
@@ -25,4 +25,8 @@
<name>
dfs.permissions
</name>
<value>
false
</value>
</property>
<property>
<name>
dfs.namenode.name.dir
</name>
<value>
file://{{ installation_path }}/hadoop/hdfs/name
</value>
</property>
</configuration>
ansible/roles/apache-hadoop/templates/hdfs-site-slave.xml.j2
0 → 100644
View file @
de4fb9ca
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>
dfs.replication
</name>
<value>
{{ dfs_replication }}
</value>
</property>
<property>
<name>
dfs.permissions.enabled
</name>
<value>
false
</value>
</property>
<property>
<name>
dfs.datanode.data.dir
</name>
<value>
file://{{ installation_path }}/hadoop/hdfs/data
</value>
</property>
</configuration>
ansible/roles/apache-hadoop/templates/yarn-init.j2
View file @
de4fb9ca
...
...
@@ -19,14 +19,26 @@ START_COMMAND="$INSTALLATION_PATH/hadoop/sbin/start-yarn.sh"
STOP_COMMAND
=
"
$INSTALLATION_PATH
/hadoop/sbin/stop-yarn.sh"
start
(){
sudo
-u
$SCRIPT_USER
nohup
$START_COMMAND
>
/dev/null &
# Assert that there is no other Apache Yarn instance, created with this script, running.
[
-f
$LOCKFILE
]
&&
return
0
# Execute the command to start Apache Yarn. The command waits until Yarn has been started.
sudo
-u
$SCRIPT_USER
$START_COMMAND
>
/dev/null
# Get the returned value of the executed command and create a lock file to prevent multiple instantiations.
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
sudo
-u
$SCRIPT_USER
touch
$LOCKFILE
)
return
$RETVAL
}
stop
(){
sudo
-u
$SCRIPT_USER
nohup
$STOP_COMMAND
>
/dev/null &
# Assert that an Apache Yarn instance, created with this script, is running.
[
!
-f
$LOCKFILE
]
&&
return
0
# Execute the command to stop Apache Yarn. The command waits until Yarn has been stopped.
sudo
-u
$SCRIPT_USER
$STOP_COMMAND
>
/dev/null
# Get the returned value of the executed command and delete the lock file.
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
sudo
-u
$SCRIPT_USER
rm
-f
$LOCKFILE
)
return
$RETVAL
...
...
@@ -53,6 +65,7 @@ case "$1" in
[
-f
$LOCKFILE
]
&&
restart
||
:
;;
status
)
# If the lock file exists, then Apache Yarn is running.
[
-f
$LOCKFILE
]
&&
echo
"Apache Yarn is running."
||
echo
"Apache Yarn is not running."
RETVAL
=
$?
;;
...
...
ansible/roles/apache-kafka/tasks/master.yml
View file @
de4fb9ca
...
...
@@ -11,7 +11,7 @@
-
configure-kafka
-
name
:
Start Apache Zookeeper server.
shell
:
/etc/init.d/zookeeper-init start > /dev/null
&
shell
:
/etc/init.d/zookeeper-init start > /dev/null
tags
:
-
start-zookeeper
...
...
@@ -21,7 +21,7 @@
-
start-zookeeper
-
name
:
Start Apache Kafka server.
shell
:
/etc/init.d/kafka-init start > /dev/null
&
shell
:
/etc/init.d/kafka-init start > /dev/null
tags
:
-
start-kafka
...
...
ansible/roles/apache-kafka/templates/kafka-init.j2
View file @
de4fb9ca
...
...
@@ -16,14 +16,26 @@ LOCKFILE="$INSTALLATION_PATH/kafka/kafka-lock"
START_COMMAND
=
"
$INSTALLATION_PATH
/kafka/bin/kafka-server-start.sh
$INSTALLATION_PATH
/kafka/config/server.properties"
start
(){
sudo
-u
$SCRIPT_USER
/sbin/start-stop-daemon
--start
--background
--make-pidfile
--pidfile
$PIDFILE
--exec
$START_COMMAND
# Assert that there is no other Apache Kafka instance, created with this script, running.
[
-f
$LOCKFILE
]
&&
return
0
# Execute the command to start Apache Kafka and wait until the service has been started.
sudo
-u
$SCRIPT_USER
/sbin/start-stop-daemon
--start
--background
--make-pidfile
--pidfile
$PIDFILE
--exec
$START_COMMAND
--retry
5
# Get the returned value of the executed command and create a lock file to prevent multiple instantiations.
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
sudo
-u
$SCRIPT_USER
touch
$LOCKFILE
)
return
$RETVAL
}
stop
(){
sudo
-u
$SCRIPT_USER
/sbin/start-stop-daemon
--stop
--remove-pidfile
--pidfile
$PIDFILE
# Assert that an Apache Kafka instance, created with this script, is running.
[
!
-f
$LOCKFILE
]
&&
return
0
# Execute the command to stop Apache Kafka and wait until the service has been stopped.
sudo
-u
$SCRIPT_USER
/sbin/start-stop-daemon
--stop
--remove-pidfile
--pidfile
$PIDFILE
--retry
5
# Get the returned value of the executed command and delete the lock file.
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
sudo
-u
$SCRIPT_USER
rm
-f
$LOCKFILE
)
return
$RETVAL
...
...
@@ -50,6 +62,7 @@ case "$1" in
[
-f
$LOCKFILE
]
&&
restart
||
:
;;
status
)
# If the lock file exists, then Apache Kafka is running.
[
-f
$LOCKFILE
]
&&
echo
"Apache Kafka is running."
||
echo
"Apache kafka is not running."
RETVAL
=
$?
;;
...
...
ansible/roles/apache-kafka/templates/zookeeper-init.j2
View file @
de4fb9ca
...
...
@@ -16,14 +16,26 @@ LOCKFILE="$INSTALLATION_PATH/kafka/zookeeper-lock"
START_COMMAND
=
"
$INSTALLATION_PATH
/kafka/bin/zookeeper-server-start.sh
$INSTALLATION_PATH
/kafka/config/zookeeper.properties"
start
(){
sudo
-u
$SCRIPT_USER
/sbin/start-stop-daemon
--start
--background
--make-pidfile
--pidfile
$PIDFILE
--exec
$START_COMMAND
# Assert that there is no other Apache Zookeeper instance, created with this script, running.
[
-f
$LOCKFILE
]
&&
return
0
# Execute the command to start Apache Zookeeper and wait until the service has been started.
sudo
-u
$SCRIPT_USER
/sbin/start-stop-daemon
--start
--background
--make-pidfile
--pidfile
$PIDFILE
--exec
$START_COMMAND
--retry
5
# Get the returned value of the executed command and create a lock file to prevent multiple instantiations.
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
sudo
-u
$SCRIPT_USER
touch
$LOCKFILE
)
return
$RETVAL
}
stop
(){
sudo
-u
$SCRIPT_USER
/sbin/start-stop-daemon
--stop
--remove-pidfile
--pidfile
$PIDFILE
# Assert that an Apache Zookeeper instance, created with this script, is running.
[
!
-f
$LOCKFILE
]
&&
return
# Execute the command to stop Apache Zookeeper and wait until the service has been stopped.
sudo
-u
$SCRIPT_USER
/sbin/start-stop-daemon
--stop
--remove-pidfile
--pidfile
$PIDFILE
--retry
5
# Get the returned value of the executed command and delete the lock file.
RETVAL
=
$?
[
$RETVAL
-eq
0
]
&&
$(
sudo
-u
$SCRIPT_USER
rm
-f
$LOCKFILE
)
return
$RETVAL
...
...
@@ -50,6 +62,7 @@ case "$1" in
[
-f
$LOCKFILE
]
&&
restart
||
:
;;
status
)
# If the lock file exists, then Apache Zookeeper is running.
[
-f
$LOCKFILE
]
&&
echo
"Apache Zookeeper is running."
||
echo
"Apache Zookeeper is not running."
RETVAL
=
$?
;;
...
...
ansible/roles/common/files/sources.list
0 → 100644
View file @
de4fb9ca
deb http://ftp.de.debian.org/debian/ jessie main
deb-src http://ftp.de.debian.org/debian/ jessie main
deb http://security.debian.org/ jessie/updates main
deb-src http://security.debian.org/ jessie/updates main
# jessie-updates, previously known as 'volatile'
deb http://ftp.de.debian.org/debian/ jessie-updates main
deb-src http://ftp.de.debian.org/debian/ jessie-updates main
deb http://apt.dev.grnet.gr jessie/
ansible/roles/common/tasks/all.yml
deleted
100644 → 0
View file @
bb81b7d8
---
-
name
:
Create users for each application.
include
:
users.yml
when
:
"
'slaves'
in
group_names"
-
name
:
Include common tasks.
include
:
common-1.yml
-
name
:
Include more common tasks.
include
:
common-2.yml
ansible/roles/common/tasks/common-1.yml
View file @
de4fb9ca
---
-
name
:
Fix locale problem.
command
:
update-locale LANGUAGE="en_US.UTF-8" LC_ALL="en_US.UTF-8"
-
name
:
Copy sources list.
copy
:
src=sources.list dest=/etc/apt/sources.list owner=root group=root mode=0640
-
name
:
Set hostname
hostname
:
name={{ inventory_hostname | replace(".vm.okeanos.grnet.gr",".local") }}
...
...
@@ -30,3 +35,15 @@
-
name
:
Add kafka user to sudo group.
user
:
name=kafka group=sudo
-
name
:
Install supervisord with apt.
apt
:
name=supervisor state=latest
environment
:
proxy_env
-
name
:
Configure supervisord for master.
template
:
src=supervisord-master.conf.j2 dest=/etc/supervisor/supervisord.conf owner=root group=root mode=0600
when
:
"
'master'
in
group_names"
-
name
:
Configure supervisord for slaves.
template
:
src=supervisord-slaves.conf.j2 dest=/etc/supervisor/supervisord.conf owner=root group=root mode=0600
when
:
"
'slaves'
in
group_names"
ansible/roles/common/tasks/main.yml
View file @
de4fb9ca
...
...
@@ -5,7 +5,12 @@
tags
:
-
master
-
name
:
Include tasks for all nodes.
include
:
all.yml
tags
:
-
all
-
name
:
Create users for each application.
include
:
users.yml
when
:
"
'slaves'
in
group_names"
-
name
:
Include common tasks.
include
:
common-1.yml
-
name
:
Include more common tasks.
include
:
common-2.yml
ansible/roles/common/tasks/master.yml
View file @
de4fb9ca
...
...
@@ -2,14 +2,6 @@
-
name
:
Create users for each application.
include
:
users.yml
# - name: Generate ssh key for root.
# shell: cat /dev/zero | ssh-keygen -q -N ""
# args:
# creates: /root/.ssh/id_rsa.pub
# - name: Fetch id_rsa.pub file from root.
# fetch: src=/root/.ssh/id_rsa.pub dest=/tmp/fetched/root_id_rsa.pub flat=yes
-
name
:
Generate ssh key for hduser.
shell
:
cat /dev/zero | ssh-keygen -q -N ""
args
:
...
...
@@ -34,3 +26,15 @@
-
name
:
Set up ssh config for flink user.
template
:
src=ssh-config.j2 dest=/home/flink/.ssh/config owner=flink group=lambda mode=600
-
name
:
Copy Lambda Instance init script.
template
:
src=lambda-init.j2 dest=/etc/init.d/lambda-init owner=root group=lambda mode=0740
# The Lambda services will be started individually during the execution of the playbooks.
# The lock file neends to be created so that lambda-init script is aware that there is
# a Lambda Instance running.
-
name
:
Create Lambda Instance lock file.
file
:
path=/root/lambda-lock state=touch owner=root group=lambda mode=0640
-
name
:
Put lambda init script on boot and shutdown sequence.
command
:
update-rc.d lambda-init defaults
ansible/roles/common/templates/lambda-init.j2
0 → 100644
View file @
de4fb9ca
#!/bin/bash
# The full path of the lock file to use.
LOCKFILE
=
"/root/lambda-lock"
start
(){
# Assert that there is no other Lambda instance, created with this script, running.
[
-f
$LOCKFILE
]
&&
return
0
# Start Apache HDFS.
echo
"Starting Apache HDFS..."
/etc/init.d/hdfs-init start
returnedValue
=
$?
if
[
$returnedValue
-eq
0
]
then
echo
"Apache HDFS has been started!"
# Force Apache HDFS to exit safe mode so that Apache Flink can be started later on this script.
{{
hadoop_home
}}
/bin/hdfs dfsadmin
-safemode
leave
else
echo
"Apache HDFS has failed to start with returned code
$returnedValue
."
fi
# Start Apache Yarn.
echo
"Starting Apache Yarn..."
/etc/init.d/yarn-init start
returnedValue
=
$?
if
[
$returnedValue
-eq
0
]
then
echo
"Apache Yarn has been started!"
else
echo
"Apache Yarn has failed to start with returned code
$returnedValue
."
fi
# Start supervisord on master node.
echo
"Starting Supervisord..."
supervisord
-c
/etc/supervisor/supervisord.conf
--logfile
=
/root/supervisord.log
returnedValue
=
$?
if
[
$returnedValue
-eq
0
]
then
echo
"Supervisord on master node has been started!"
else
echo
"Supervisord on master node has failed to start with returned code
$returnedValue
."
fi
# Start Apache Zookeeper.
echo
"Starting Apache Zookeeper..."
supervisorctl start apache_zookeeper
# Wait for Apache Zookeeper to start.
while
[
"
$(
supervisorctl status apache_zookeeper |
tr
-s
' '
|
cut
-f2
-d
' '
)
"
==
"STARTING"
]
do
sleep
10
done
apache_zookeeper_status
=
$(
supervisorctl status apache_zookeeper |
tr
-s
' '
|
cut
-f2
-d
' '
)
if
[
"
$apache_zookeeper_status
"
!=
"RUNNING"
]
then
echo
"Apache Zookeeper has failed to start with code
$apache_zookeeper_status
."
else
echo
"Apache Zookeeper has been started!"
fi
# Start Apache Kafka on master node.
echo
"Starting Apache kafka..."
supervisorctl start apache_kafka
# Wait for Apache Kafka to start.
while
[
"
$(
supervisorctl status apache_kafka |
tr
-s
' '
|
cut
-f2
-d
' '
)
"
==
"STARTING"
]
do
sleep
10
done
apache_kafka_status
=
$(
supervisorctl status apache_kafka |
tr
-s
' '
|
cut
-f2
-d
' '
)
if
[
"
$apache_kafka_status
"
!=
"RUNNING"
]
then
echo
"Apache kafka has failed to start with code
$apache_kafka_status
."
else
echo
"Apache kafka has been started!"
fi
# Start Apache Kafka on each slave node.
for
node
in
$(
cat
/etc/hosts |
grep
"snf"
|
cut
-f2
)
do
if
[
"
$node
"
==
"
$(
hostname
)
.local"
]
then
continue
fi
echo
"Starting Apache Kafka at
$node
..."
ssh
-l
root
$node
supervisord
-c
/etc/supervisor/supervisord.conf
--logfile
=
/root/supervisord.log
echo
"Supervisord on
$node
has been started!"
ssh
-l
root
$node
supervisorctl start apache_kafka
# Wait for Apache kafka to start.
while
[
"
$(
ssh
-l
root
$node
supervisorctl status apache_kafka |
tr
-s
' '
|
cut
-f2
-d
' '
)
"
==
"STARTING"
]