Commit aac6a0d2 authored by Georgios Ouzounis's avatar Georgios Ouzounis
Browse files

LAM-23 Implemented consumer to read data from Kafka and store them to HDFS.

parent 3b9b59df
#!/bin/bash
# Get the messages from Kafka and store them to a temporary file.
/usr/local/kafka/bin/kafka-console-consumer.sh --consumer.config /usr/local/kafka/config/consumer-batch.properties --zookeeper localhost:2181 --topic input > temporary_input;
timestamp=$(date +"%F_%H_%M_%S");
# The last 12 lines of the file are created from the timeout exception thrown by the
# kafka-console-consumer.sh script and they should be removed.
head -n -12 temporary_input > temporary_input2;
# Store the final file on HDFS.
$(/usr/local/hadoop/bin/hdfs dfs -put temporary_input2 /user/root/input/tweets_$timestamp);
# Remove all temporary files.
rm temporary_input;
rm temporary_input2;
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# see kafka.consumer.ConsumerConfig for more details
# Zookeeper connection string
# comma separated host:port pairs, each corresponding to a zk
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002"
zookeeper.connect=127.0.0.1:2181
# timeout in ms for connecting to zookeeper
zookeeper.connection.timeout.ms=6000
#consumer group id
group.id=consumer-batch
#consumer timeout
consumer.timeout.ms=5000
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment