Frozen indices benchmark

create_repositories.sh

#/bin/bash

echo $(date) "Create snapshot repositories" 

curl -X PUT "localhost:9200/_snapshot/elasticlogs-nofm" -H 'Content-Type: application/json' -d'
{
  "type": "fs",
  "settings": {
    "location": "/data/snapshots/elasticlogs-nofm"
  }
}
'

curl -X PUT "localhost:9200/_snapshot/elasticlogs-fm" -H 'Content-Type: application/json' -d'
{
  "type": "fs",
  "settings": {
    "location": "/data/snapshots/elasticlogs-fm"
  }
}
'

echo $(date) "Snapshot not forcemerged data to the elasticlogs-nofm repository" 

curl -X PUT "localhost:9200/_snapshot/elasticlogs-nofm/1?wait_for_completion=true" -H 'Content-Type: application/json' -d'
{
  "indices": "elasticlogs",
  "ignore_unavailable": true,
  "include_global_state": false
}
'

echo $(date) "Forcemerge the index down to a single segment - wait for completion" 

curl -X POST "localhost:9200/elasticlogs/_forcemerge?max_num_segments=1&flush=true"

COUNT=$(curl -s localhost:9200/_cat/segments | grep elasticlogs | wc -l)

while [ $COUNT -gt 2 ]
do
	sleep 30  
    COUNT=$(curl -s localhost:9200/_cat/segments | grep elasticlogs | wc -l)

    if [ $COUNT == 2 ]
    then
        echo $(date) "Forcemerge of elasticlogs index has completed."
    else
    	echo $(date) "Forcemerge of elasticlogs index still not completed."
    fi
done

echo $(date) "Snapshot forcemerged data to the elasticlogs-fm repository" 

curl -X PUT "localhost:9200/_snapshot/elasticlogs-nofm/1?wait_for_completion=true" -H 'Content-Type: application/json' -d'
{
  "indices": "elasticlogs",
  "ignore_unavailable": true,
  "include_global_state": false
}
'

echo $(date) "Snapshots completed" 

freeze_all.sh

#!/bin/bash

INDEX_COUNT=$(curl -s localhost:9200/_cat/indices | grep elasticlogs | wc -l)

for i in $(seq 1 $INDEX_COUNT); do
    echo "Freeze elasticlogs-$i"
    curl -XPOST localhost:9200/elasticlogs-$i/_freeze 
done

freeze_half.sh

#!/bin/bash

INDEX_COUNT=$(curl -s localhost:9200/_cat/indices | grep elasticlogs | wc -l)
INDEX_HALF_COUNT=$((INDEX_COUNT / 2))
HALF_START=$((INDEX_HALF_COUNT + 1))

for i in $(seq 1 $INDEX_HALF_COUNT); do
    echo "Freeze elasticlogs-$i"
    curl -XPOST localhost:9200/elasticlogs-$i/_freeze 
done

for i in $(seq $HALF_START $INDEX_COUNT); do
    echo "Unfreeze elasticlogs-$i"
    curl -XPOST localhost:9200/elasticlogs-$i/_unfreeze
done

load_fm_data.sh

#!/bin/bash

INDEX_COUNT=20

echo "Delete existing indices"
curl -XDELETE localhost:9200/elasticlogs*

echo "restore 20 indices"
for i in $(seq 1 $INDEX_COUNT); do 
  ./restore_snapshot.sh localhost:9200 elasticlogs-fm elasticlogs 1 elasticlogs-$i
done

load_nofm_data.sh

#!/bin/bash

INDEX_COUNT=20

echo "Delete existing indices"
curl -XDELETE localhost:9200/elasticlogs*

echo "restore 20 indices"
for i in $(seq 1 $INDEX_COUNT); do 
  ./restore_snapshot.sh localhost:9200 elasticlogs-nofm elasticlogs 1 elasticlogs-$i
done

restore_snapshot.sh

#/bin/bash

TIMESTAMP=$(date +%s)
ES_HOST=$1
REPOSITORY=$2
INDEX_NAME=$3
SNAPSHOT_ID=$4
NEW_INDEX_NAME=$5


RESULT=$(curl -XPOST -s http://$ES_HOST/_snapshot/$REPOSITORY\/$SNAPSHOT_ID\/_restore -H 'Content-Type: application/json' -d "{
    \"indices\": \"$INDEX_NAME\",
    \"ignore_unavailable\": \"true\",
    \"include_global_state\": false,
    \"rename_pattern\": \"$INDEX_NAME\",
    \"rename_replacement\": \"$NEW_INDEX_NAME\"
}")

if [ "$RESULT" == '{"accepted":true}' ]
then
    echo $(date) "Snapshot initiated. Restoring" $INDEX_NAME "into" $NEW_INDEX_NAME "from repository" $REPOSITORY ". Await completion."
    COUNT=0

    while [ $COUNT == 0 ]
    do
	    sleep 30  
        COUNT=$(curl -s http://$ES_HOST/_cat/indices | grep $NEW_INDEX_NAME | grep green | wc -l)

        if [ $COUNT == 1 ]
        then
            echo $(date) "Snapshot restoration of" $NEW_INDEX_NAME "has completed."
        else
        	echo $(date) "Snapshot restoration of" $NEW_INDEX_NAME "still not completed."
        fi
    done
else
    echo $(date) "Initiation of snapshot restoration for" $NEW_INDEX_NAME "FAILED :" $RESULT
fi

run_cold_fm_benchmark.sh

#!/bin/bash

./unfreeze_all.sh

esrally --track-path=/home/ubuntu/rally-eventdata-track/eventdata --pipeline=benchmark-only --challenge=frozen-querying --client-options="use_ssl:false,timeout:600,request_timeout:600,max_retries:0" --track-params="node_type:'cold',instance:'d2.xlarge',data_vol:'1TB',forcemerged:'yes',throttled_threads:1,frozen_ratio_pct:0" --telemetry=node-stats --telemetry-params="node-stats-include-indices:true,node-stats-sample-interval:10" --on-error=abort

./freeze_all.sh

esrally --track-path=/home/ubuntu/rally-eventdata-track/eventdata --pipeline=benchmark-only --challenge=frozen-querying --client-options="use_ssl:false,timeout:600,request_timeout:600,max_retries:0" --track-params="node_type:'cold',instance:'d2.xlarge',data_vol:'1TB',forcemerged:'yes',throttled_threads:1,frozen_ratio_pct:100" --telemetry=node-stats --telemetry-params="node-stats-include-indices:true,node-stats-sample-interval:10" --on-error=abort

run_data_generation.sh

#/bin/bash

esrally --track-path=~/rally-eventdata-track/eventdata --pipeline=benchmark-only --challenge=frozen-data-generation

unfreeze_all.sh

#!/bin/bash

INDEX_COUNT=$(curl -s localhost:9200/_cat/indices | grep elasticlogs | wc -l)

for i in $(seq 1 $INDEX_COUNT); do
    echo "Unfreeze elasticlogs-$i"
    curl -XPOST localhost:9200/elasticlogs-$i/_unfreeze 
done