LAM-30 Created apache-kafka role to install Apache Kafka on a cluster of machines.

ansible/playbooks/apache-kafka/kafka-install.yml

 ---
-
-- hosts: master
+- hosts: all
   user: root
-  tasks:
-    - name: Download Apache Kafka.
-      get_url: url=http://mirrors.myaegean.gr/apache/kafka/0.8.2.1/kafka_2.10-0.8.2.1.tgz dest=/root/kafka_2.10-0.8.2.1.tgz
-      tags:
-        - download
-
-    - name: Uncompress Apache Kafka.
-      unarchive: src=/root/kafka_2.10-0.8.2.1.tgz dest=/usr/local copy=no
-      tags:
-        - uncompress
-
-    - name: Create softlink for Apache Kafka.
-      file: src=/usr/local/kafka_2.10-0.8.2.1 dest=/usr/local/kafka state=link
-      tags:
-        - uncompress
-
-    - name: Configure Apache kafka.
-      copy: src=../files/usr/local/kafka/config/server.properties dest=/usr/local/kafka/config/server.properties owner=root group=root mode=0644
-      tags:
-        - configure-kafka
-
-    - name: Start Apache Zookeeper server.
-      shell: /usr/local/kafka/bin/zookeeper-server-start.sh /usr/local/kafka/config/zookeeper.properties
-      async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
-      poll: 0
-      tags:
-        - start-zookeeper
-
-    - name: Wait for Apache Zookeeper to become available.
-      wait_for: port=2181
-      tags:
-        - start-zookeeper
-
-    - name: Start Apache Kafka server.
-      shell: /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties
-      async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
-      poll: 0
-      tags:
-        - start-kafka
-
-    - name: Wait for Apache Kafka server to become available.
-      wait_for: port=9092
-      tags:
-        - start-kafka
+  roles:
+    - ../roles/apache-kafka
 

ansible/playbooks/roles

+../roles/
\ No newline at end of file

ansible/roles/apache-kafka/handlers/main.yml

+---
+- name: create topics
+  shell: /usr/local/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master']['kafka-ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic input
+  notify:
+    - create batch output topic
+
+- name: create batch output topic
+  shell: /usr/local/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master']['kafka-ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic batch-output
+  notify:
+    - create stream output topic
+
+- name: create stream output topic
+  shell: /usr/local/kafka/bin/kafka-topics.sh --create --zookeeper {{ hostvars['master']['kafka-ip'] }}:2181 --replication-factor {{ groups['slaves']|count + 1 }} --partitions 1 --topic stream-output
+

ansible/roles/apache-kafka/tasks/common.yml

+---
+  - name: Download Apache Kafka.
+    get_url: url=http://mirrors.myaegean.gr/apache/kafka/0.8.2.1/kafka_2.10-0.8.2.1.tgz dest=/root/kafka_2.10-0.8.2.1.tgz
+    tags:
+      - download
+
+  - name: Uncompress Apache Kafka.
+    unarchive: src=/root/kafka_2.10-0.8.2.1.tgz dest=/usr/local copy=no
+    tags:
+      - uncompress
+
+  - name: Create softlink for Apache Kafka.
+    file: src=/usr/local/kafka_2.10-0.8.2.1 dest=/usr/local/kafka state=link
+    tags:
+      - uncompress
+

ansible/roles/apache-kafka/tasks/main.yml

+---
+  - name: Include common tasks.
+    include: common.yml
+  
+  - name: Include tasks for master.
+    include: master.yml
+    when: "'master' in group_names"
+    tags:
+      - master-install
+  
+  - name: Include tasks for slaves.
+    include: slaves.yml
+    when: "'slaves' in group_names"
+    tags:
+      - slaves-install
+  

ansible/roles/apache-kafka/tasks/master.yml

+---
+  - name: Configure Apache kafka.
+    copy: src=usr/local/kafka/config/server.properties dest=/usr/local/kafka/config/server.properties owner=root group=root mode=0644
+    tags:
+      - configure-kafka
+
+  - name: Start Apache Zookeeper server.
+    shell: /usr/local/kafka/bin/zookeeper-server-start.sh /usr/local/kafka/config/zookeeper.properties
+    async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
+    poll: 0
+    tags:
+      - start-zookeeper
+
+  - name: Wait for Apache Zookeeper to become available.
+    wait_for: port=2181
+    tags:
+      - start-zookeeper
+
+  - name: Start Apache Kafka server.
+    shell: /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties
+    async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
+    poll: 0
+    tags:
+      - start-kafka
+
+  - name: Wait for Apache Kafka server to become available.
+    wait_for: port=9092 timeout=30
+    changed_when: true
+    notify:
+      - create topics
+    tags:
+      - start-kafka
+

ansible/roles/apache-kafka/tasks/slaves.yml

+---
+  - name: Configure Apache kafka.
+    copy: src=usr/local/kafka/config/server.properties dest=/usr/local/kafka/config/server.properties owner=root group=root mode=0644
+    tags:
+      - configure-kafka
+
+  - name: Assign a unique id to this Apache Kafka server.
+    lineinfile: dest=/usr/local/kafka/config/server.properties regexp="^broker.id=0" line="broker.id={{ id }}"
+    tags:
+      - configure-kafka
+
+  - name: Configure the Apache Zookeeper host.
+    lineinfile: dest=/usr/local/kafka/config/server.properties regexp="^zookeeper.connect=localhost:2181" line="zookeeper.connect={{ hostvars['master']['kafka-ip'] }}:2181"
+    tags:
+      - configure-kafka
+
+  - name: Start Apache Kafka server.
+    shell: /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties
+    async: 31536000 # Stay alive for a year(1 year = 31536000 seconds).
+    poll: 0
+    tags:
+      - start-kafka
+
+  - name: Wait for Apache Kafka server to become available.
+    wait_for: port=9092
+    tags:
+      - start-kafka
+