redis HSETALL (hash import)

yourkey.sh

#!/bin/bash

# to get this file run
# redis-cli --tls -u <url> HGETALL "yourkey" > yourkey.txt
FILE_PATH="./yourkey.txt"
REDIS_ERROR_FILE="./redis_error.txt"

if [[ ! -f $FILE_PATH ]]; then
    echo "Arquivo não encontrado: $FILE_PATH"
    exit 1
fi

batch_size=1000
batch_count=0
chunk=""
chunks=()
line_count=0
total_lines=$(wc -l < "$FILE_PATH")
duration=0

process_batch() {
    start=$(date +'%s')
    if [[ ${#chunks[@]} -eq 0 ]]; then
        return
    fi

    batch_count=$((batch_count + 1))
    echo "Processando lote: $batch_count ($((${line_count} * 100 / total_lines))%) (${line_count}/${total_lines})"

    echo "MULTI" > redis_batch_yourkey.txt

    for chunk in "${chunks[@]}"; do
        head=$(echo "$chunk" | head -n 1)
        content=$(echo "$chunk" | tail -n +2 | sed ':a;N;$!ba;s/\n/\\n/g')
        content=$(echo "$content" | sed 's/"/\\\"/g')
        if [[ -n "$head" && -n "$content" ]]; then
            echo "HSET last_updates $head \"$content\"" >> redis_batch_yourkey.txt
        fi
    done

    echo "EXEC" >> redis_batch_yourkey.txt

    redis_output=$(cat redis_batch_yourkey.txt | redis-cli --tls -u <url>)

    if [[ "$redis_output" =~ "ERR" || "$redis_output" =~ "Invalid argument(s)" ]]; then
        echo "ERROR $batch_count: $redis_output" >> "$REDIS_ERROR_FILE"
    fi

    rm redis_batch_yourkey.txt

    echo "Lote $batch_count concluído. ($((${line_count} * 100 / total_lines))%) - $(($(date +'%s') - $start))s"
    duration+=$(($(date +'%s') - $start))

    chunks=()
}

while IFS= read -r line || [[ -n "$line" ]]; do
    ((line_count++))
    if [[ -z "$line" ]]; then
        chunks+=("$chunk")
        chunk=""
    else
        chunk+="$line"$'\n'
    fi

    if [[ ${#chunks[@]} -eq $batch_size ]]; then
        process_batch
    fi
done < "$FILE_PATH"

if [[ -n "$chunk" ]]; then
    chunks+=("$chunk")
fi

process_batch

echo "Processamento concluído. (100%) em ${duration}s"