Commit f183883a authored by Paschalis Korosoglou's avatar Paschalis Korosoglou
Browse files

Merge pull request #27 from ioantsaf/ansible-proxy

LAM-42 Ansible playbooks for http proxy, ssh bastion proxy, LAM-40 ansible inventory
parents 51104772 0361037b
......@@ -61,3 +61,5 @@ target/
MANIFEST
old_kamaki
*.idea
coverage.xml
......@@ -26,6 +26,7 @@ both the master and the slaves.
## Playbooks and Roles
There are four (4) roles and five (5) playbooks. These are:
- proxy role, run from proxy playbook.
- common role, run from common playbook.
- apache-hadoop role, run from apache-hadoop playbook.
- apache-kafka role, run from apache-kafka playbook.
......@@ -36,6 +37,12 @@ There are four (4) roles and five (5) playbooks. These are:
## Role Explanation
### proxy
- Installs squid http proxy on master node, using apt package manager.
- Configures the acls of the proxy, and sets the localnet to allowed.
- Restarts the http proxy service
### common
- Installs all the packages that are needed in order for the cluster to run.
......@@ -63,6 +70,8 @@ There are four (4) roles and five (5) playbooks. These are:
- Starts and Apache Flink, Yarn session.
## How to deploy
You can deploy the whole cluster by running the cluster-install playbook:
......
#
# config file for ansible
# https://raw.githubusercontent.com/ansible/ansible/devel/examples/ansible.cfg
#
[defaults]
remote_user = root
hostfile = hosts
#[ssh_connection]
#ssh_args = -o StrictHostKeyChecking=no -F ssh.config -q
---
- hosts: master
user: root
gather_facts: no
roles:
- wait_for_ssh
- hosts: master
user: root
roles:
- proxy
- hosts: all
user: root
roles:
- common
- apache-hadoop
- apache-kafka
- apache-flink
---
- hosts: all
user: root
roles:
- ../roles/common
- ../roles/apache-hadoop
- ../roles/apache-kafka
- ../roles/apache-flink
---
- hosts: master
user: root
gather_facts: yes
roles:
- ../roles/apache-flink
- apache-flink
---
- hosts: all
user: root
roles:
- ../roles/apache-hadoop
- apache-hadoop
---
- hosts: all
user: root
roles:
- ../roles/apache-kafka
- apache-kafka
---
- hosts: master
user: root
roles:
- proxy
---
- hosts: master
user: root
tasks:
- name: Touch master file
file: path=/root/master state=touch mode="u=rw,g=r,o=r"
tags: touch
- name: Remove master file
file: path=/root/master state=absent
tags: rm
- name: Copy hosts file
template: src=../roles/common/templates/hosts.j2 dest=/etc/hosts backup=no owner=root group=root mode=0750
tags: hosts
- hosts: slaves
user: root
gather_facts: False
tasks:
- name: Touch slave file
file: path=/root/slave state=touch mode="u=rw,g=r,o=r"
tags: touch
- name: Remove slave file
file: path=/root/slave state=absent
tags: rm
- name: Copy hosts file
template: src=../roles/common/templates/hosts.j2 dest=/etc/hosts backup=no owner=root group=root mode=0750
tags: hosts
---
- hosts: master
user: root
tasks:
- name: install screen to master without proxy
apt: name=screen state=installed
tags: install
- hosts: slaves
user: root
tasks:
- name: install screen to slaves using proxy
apt: name=screen state=installed
environment: proxy_env
tags: install
---
- hosts: master
user: root
gather_facts: no
roles:
- wait_for_ssh
......@@ -35,8 +35,8 @@
tags:
- create-dirs
- name: Create Apache HDFS user/root directory.
command: "{{ installation_path }}/hadoop/bin/hadoop fs -mkdir /user/root"
- name: Create Apache HDFS user/hduser directory.
command: "{{ installation_path }}/hadoop/bin/hadoop fs -mkdir /user/hduser"
tags:
- create-dirs
......@@ -19,7 +19,7 @@
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:9000</value>
<value>hdfs://{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:9000</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
......
......@@ -15,14 +15,14 @@
<configuration>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8025</value>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:8025</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8030</value>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:8030</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8050</value>
<value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr",".local") }}:8050</value>
</property>
</configuration>
---
- name: create topics
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master-node']['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic input"
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars[groups['master'][0]]['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic input"
notify:
- create batch output topic
- name: create batch output topic
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master-node']['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic batch-output"
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars[groups['master'][0]]['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic batch-output"
notify:
- create stream output topic
- name: create stream output topic
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master-node']['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic stream-output"
shell: "{{ installation_path }}/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars[groups['master'][0]]['internal_ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic stream-output"
---
- name: Create users for each application.
include: users.yml
when: "'slaves' in group_names"
- name: Include common tasks.
include: common-1.yml
......
---
- name: Set hostname
hostname: name={{ inventory_hostname | replace(".vm.okeanos.grnet.gr",".local") }}
when: "'slaves' in group_names"
- name: Copy hosts file.
template: src=hosts.j2 dest=/etc/hosts backup=no owner=root group=lambda mode=0750
- name: Upgrade packages.
apt: upgrade=dist update_cache=yes
environment: proxy_env
- name: Install the latest Java 7.
apt: name=openjdk-7-jdk state=latest install_recommends=no update_cache=yes
environment: proxy_env
- name: Copy environment file.
template: src=environment.j2 dest=/etc/environment backup=no owner=root group=lambda mode=0750
- name: Install sudo.
apt: name=sudo state=latest
environment: proxy_env
- name: Add hduser to sudo group.
user: name=hduser group=sudo
......
......@@ -5,8 +5,7 @@
tags:
- master
- name: Include tasks for slaves.
include: slaves.yml
when: "'slaves' in group_names"
- name: Include tasks for all nodes.
include: all.yml
tags:
- slaves
- all
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment