Last active
February 20, 2023 06:13
-
-
Save Radamanf/6b017460098452a73006 to your computer and use it in GitHub Desktop.
Kafka daemon with System.d. You also can use bin/kafka for init.d. Just unzip ZooKeeper and Kafka to you home/user/soft/ dir and create listed three files then enable them to be autostarted by systemctl enable kafka.service and systemctl enable zookeeper.service.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
[Unit] | |
Description=Apache Kafka server (broker) | |
Documentation=http://kafka.apache.org/documentation.html | |
Requires=zookeeper.service network.target remote-fs.target | |
After=zookeeper.service network.target remote-fs.target | |
[Service] | |
Type=forking | |
PIDFile=/var/run/kafka.pid | |
ExecStart=/home/user/bin/kafka start | |
ExecStop=/home/user/bin/kafka stop | |
ExecReload=/home/user/bin/kafka restart | |
Restart=on-failure | |
SyslogIdentifier=kafka | |
[Install] | |
WantedBy=multi-user.target |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
[Unit] | |
Description=Apache Zookeeper | |
After=network.target | |
[Service] | |
Type=forking | |
#User=zookeeper | |
#Group=zookeeper | |
#SyslogIdentifier=zookeeper | |
Restart=always | |
RestartSec=0s | |
PIDFile=/var/lib/zookeeper/zookeeper_server.pid | |
ExecStart=/home/user/soft/zookeeper-3.4.6/bin/zkServer.sh start | |
ExecStop=/home/user/soft/zookeeper-3.4.6/bin/zkServer.sh stop | |
ExecReload=/home/user/soft/zookeeper-3.4.6/bin/zkServer.sh restart | |
[Install] | |
WantedBy=multi-user.target |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# Licence: GPLv3, MIT, BSD, Apache or whatever you prefer; FREE to use, modify, copy, no obligations | |
# Description: Bash Script to Start Kafka as Daemon on unit system | |
# Author: Andrew Bikadorov | |
# Script v0.4 | |
# For debugging purposes uncomment next line | |
#set -x | |
APP_NAME="kafkaServer" | |
APP_PATH="/home/user/soft/kafka_2.11-0.8.2.2" | |
APP_PID="/var/run/kafka.pid" # kafka-run-class.sh need to be hacked :( | |
# Should Not Be altered | |
TMP_FILE="/tmp/status_$APP_NAME" | |
# Start | |
start() { | |
checkpid | |
STATUS=$? | |
if [ $STATUS -ne 0 ] ; | |
then | |
#echo "Starting $APP_NAME..." | |
#echo "rm -rf "$log.dirs"/ " | |
cd $APP_PATH | |
$APP_PATH/bin/kafka-run-class.sh -daemon -name $APP_NAME -loggc kafka.Kafka config/server.properties | |
#bin/kafka-server-start.sh -daemon config/server.properties | |
# Get PID from running processes | |
ps ax | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk '{print $1}' > $APP_PID | |
#echo PID file `cat $APP_PID` | |
#statusit | |
else | |
echo "$APP_NAME Already Running" | |
fi | |
} | |
# Stop | |
stop() { | |
checkpid | |
STATUS=$? | |
if [ $STATUS -eq 0 ] ; | |
then | |
#echo "Stopping $APP_NAME..." | |
kill `cat $APP_PID` | |
rm $APP_PID | |
#statusit | |
else | |
echo "$APP_NAME - Already killed" | |
fi | |
} | |
# Status | |
checkpid(){ | |
STATUS=9 | |
if [ -f $APP_PID -a -s $APP_PID ] ; # AND NOT -s $APP_PID OR `cat $APP_PID` == 0 | |
then | |
#echo "Is Running if you can see next line with $APP_NAME" | |
#ps -Fp `cat $APP_PID` | grep -i 'kafka\.Kafka' > $TMP_FILE | |
ps ax | grep -i 'kafka\.Kafka' > $TMP_FILE | |
if [ -f $TMP_FILE -a -s $TMP_FILE ] ; | |
then | |
STATUS=0 | |
#"Is Running (PID `cat $APP_PID`)" | |
else | |
STATUS=2 | |
#"Stopped incorrectly" | |
fi | |
## Clean after yourself | |
rm -f $TMP_FILE | |
else | |
STATUS=1 | |
#"Not Running" | |
fi | |
return $STATUS | |
} | |
statusit() { | |
checkpid | |
#GET return value from previous function | |
STATUS=$? | |
case $STATUS in | |
0) | |
EXITSTATUS="Is Running" | |
;; | |
1) | |
EXITSTATUS="Not Running" | |
;; | |
2) | |
EXITSTATUS="Stopped incorrectly" | |
;; | |
*) | |
$EXITSTATUS="Default Status (should not be seen)" | |
;; | |
esac | |
echo $APP_NAME - $EXITSTATUS | |
} | |
## What option to run | |
case "$1" in | |
'start') | |
start | |
;; | |
'stop') | |
stop | |
;; | |
'restart') | |
stop | |
start | |
;; | |
'status') | |
statusit | |
;; | |
*) | |
echo "Usage: $0 { start | stop | restart | status }" | |
exit 1 | |
;; | |
esac | |
exit 0 |
!/bin/bash
-Simplified Console listener
TOPIC=test
OPTIONAL=""
if [ -n "$1" ]; then
TOPIC=$1
fi
if [ -n "$2" ]; then
OPTIONAL="--from-beginning"
fi
if [ -z "$1" ]; then
echo "Default usage FileName [param] [any]"
echo "First param - Topic name"
echo "Second param if not empty (any string) then display all messages from the beginning"
fi
echo "Using topic=$TOPIC"
bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic $TOPIC $OPTIONAL
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Use
http://kafka.apache.org/documentation.html#quickstart
https://zookeeper.apache.org/doc/trunk/zookeeperStarted.html
to download latest versions of both, I'm using separate installation of ZooKeeper.
Then you can run Producer and Listener localhost should be chenged to your IP
Producer
~/soft/kafka_2.11-0.8.2.2/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
Listener
~/soft/kafka_2.11-0.8.2.2/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning
Performance testing
~/soft/kafka_2.11-0.8.2.2/bin/kafka-producer-perf-test.sh --broker-list localhost:9092 --messages 150000 --topics test --threads 5 --message-size 800 --batch-size 250