Merge pull request #2 from ioantsaf/ansible_hadoop

LAM-15 Ansible playbooks for hadoop deployment

ansible/files/usr/local/hadoop/etc/hadoop/hdfs-site.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+   <property>
+     <name>dfs.replication</name>
+     <value>2</value>
+   </property>
+   <property>
+     <name>dfs.permissions</name>
+     <value>false</value>
+   </property>
+</configuration>

ansible/hosts

+[master]
+
+[slaves]

ansible/playbooks/hadoop.yml

+---
+
+- hosts: all
+  user: root
+
+  tasks:
+    - include: ../tasks/hadoop/install.yml       tags=install
+    - include: ../tasks/hadoop/config.yml        tags=config
+
+
+- hosts: master
+  user: root
+
+  tasks:
+    - include: ../tasks/hadoop/hdfs_format.yml   tags=format
+    - include: ../tasks/hadoop/start.yml         tags=start
+    - include: ../tasks/hadoop/hdfs_dirs.yml     tags=mkdir

ansible/tasks/hadoop/config.yml

+---
+
+- name: Set hadoop env
+  lineinfile: dest=/usr/local/hadoop/etc/hadoop/hadoop-env.sh regexp="^export JAVA_HOME=" line="export JAVA_HOME=/usr/"
+  tags: env
+
+- name: Copy slaves configuration
+  template: src=../templates/usr/local/hadoop/etc/hadoop/slaves.j2
+            dest=/usr/local/hadoop/etc/hadoop/slaves
+            owner=root group=root mode=0644
+  tags: slaves
+
+- name: Copy core configuration
+  template: src=../templates/usr/local/hadoop/etc/hadoop/core-site.xml.j2
+            dest=/usr/local/hadoop/etc/hadoop/core-site.xml
+            owner=root group=root mode=0644
+  tags: core
+
+- name: Copy hdfs configuration
+  copy: src=../files/usr/local/hadoop/etc/hadoop/hdfs-site.xml
+        dest=/usr/local/hadoop/etc/hadoop/hdfs-site.xml backup=no
+        owner=root group=root mode=0644
+  tags: hdfs
+
+- name: Copy yarn configuration
+  template: src=../templates/usr/local/hadoop/etc/hadoop/yarn-site.xml.j2
+            dest=/usr/local/hadoop/etc/hadoop/yarn-site.xml
+            owner=root group=root mode=0644
+  tags: yarn

ansible/tasks/hadoop/hdfs_dirs.yml

+---
+
+- name: Make hdfs user dir
+  command: /usr/local/hadoop/bin/hadoop fs -mkdir /user
+
+- name: Make hdfs user/root dir
+  command: /usr/local/hadoop/bin/hadoop fs -mkdir /user/root

ansible/tasks/hadoop/hdfs_format.yml

+---
+
+- name: Format hdfs
+  command: /usr/local/hadoop/bin/hadoop namenode -format -nonInteractive

ansible/tasks/hadoop/install.yml

+---
+
+- name: Download hadoop from apache servers
+  get_url: url=http://mirrors.myaegean.gr/apache/hadoop/common/hadoop-2.7.0/hadoop-2.7.0.tar.gz
+           dest=/root/hadoop-2.7.0.tar.gz
+
+- name: Untar hadoop
+  unarchive: src=/root/hadoop-2.7.0.tar.gz copy=no
+             dest=/usr/local/
+
+- name: Make hadoop softlink
+  file: src=/usr/local/hadoop-2.7.0 dest=/usr/local/hadoop state=link

ansible/tasks/hadoop/start.yml

+---
+
+- name: Start hdfs
+  command: /usr/local/hadoop/sbin/start-dfs.sh
+
+- name: Start yarn
+  command: /usr/local/hadoop/sbin/start-yarn.sh

ansible/templates/usr/local/hadoop/etc/hadoop/core-site.xml.j2

+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+  <property>
+    <name>fs.default.name</name>
+    <value>hdfs://{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:9000</value>
+  </property>
+  <property>
+    <name>hadoop.tmp.dir</name>
+    <value>/usr/local/hadoop/tmp</value>
+  </property>
+</configuration>

ansible/templates/usr/local/hadoop/etc/hadoop/slaves.j2

+{{ groups.slaves | replace("'","") | replace("[","") | replace(", ","\n") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}

ansible/templates/usr/local/hadoop/etc/hadoop/yarn-site.xml.j2

+<?xml version="1.0"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+  <property>
+    <name>yarn.resourcemanager.resource-tracker.address</name>
+    <value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8025</value>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.scheduler.address</name>
+    <value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8030</value>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.address</name>
+    <value>{{ groups.master | replace("[","") | replace("'","") | replace("]","") | replace(".vm.okeanos.grnet.gr","") }}:8050</value>
+  </property>
+</configuration>