Commit ed3d443d authored by Themis Zamani's avatar Themis Zamani
Browse files

Merge pull request #9 from gouzouni625/LAM-23

LAM-23 Consumer to read data from Kafka and store them to HDFS.
parents fea5e84d aac6a0d2
# Get the messages from Kafka and store them to a temporary file.
/usr/local/kafka/bin/ --consumer.config /usr/local/kafka/config/ --zookeeper localhost:2181 --topic input > temporary_input;
timestamp=$(date +"%F_%H_%M_%S");
# The last 12 lines of the file are created from the timeout exception thrown by the
# script and they should be removed.
head -n -12 temporary_input > temporary_input2;
# Store the final file on HDFS.
$(/usr/local/hadoop/bin/hdfs dfs -put temporary_input2 /user/root/input/tweets_$timestamp);
# Remove all temporary files.
rm temporary_input;
rm temporary_input2;
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.
# see kafka.consumer.ConsumerConfig for more details
# Zookeeper connection string
# comma separated host:port pairs, each corresponding to a zk
# server. e.g. ",,"
# timeout in ms for connecting to zookeeper
#consumer group id
#consumer timeout
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment