Created
March 29, 2024 14:49
-
-
Save ram-pi/74c8dabed95bcf660a21c9b73a129bb4 to your computer and use it in GitHub Desktop.
Script for moving offset from one cluster to another
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
# example usage: ./offsets_mover.sh localhost:9092 client.1.properties localhost:9093 client.2.properties | |
# get bootstrap servers from args | |
BOOTSTRAP_SERVER=$1 | |
# get properties file location from args | |
PROPERTIES_FILE=$2 | |
# get new bootstrap servers from args | |
NEW_BOOTSTRAP_SERVER=$3 | |
# get new properties file location from args | |
NEW_PROPERTIES_FILE=$4 | |
# get list of all consumer groups | |
groups=$(kafka-consumer-groups --bootstrap-server $BOOTSTRAP_SERVER --command-config $PROPERTIES_FILE --all-groups --list) | |
# loop through all consumer groups | |
for group in $groups | |
do | |
# get group topic partition and current offset in format new-group pageviews 0 7825 | |
kafka-consumer-groups --bootstrap-server $BOOTSTRAP_SERVER --command-config $PROPERTIES_FILE --group $group --describe | awk '{print $1,$2,$3,$4}' | tail -n +3 > tmp.txt | |
# read tmp.txt file line by line and split each line into array | |
while IFS= read -r line | |
do | |
# split line into array | |
arr=($line) | |
# print group topic partition and current offset | |
echo "kafka-consumer-groups --bootstrap-server $NEW_BOOTSTRAP_SERVER --command-config $NEW_PROPERTIES_FILE --group ${arr[0]} --reset-offsets --to-offset ${arr[3]} --topic ${arr[1]}:${arr[2]} --dry-run" | |
done < tmp.txt | |
done |
yes correct, you usually need to do a dry-run first.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
In line 32, do we have to modify the
--dry-run
to be able to reset the offsets in the new cluster?