A quick shell script to scp only latest files based on file timestamp

scp_delta_load.sh

#!/bin/bash

SUB_FOLDER=$1
FILE_PATRN=$2

function usage {
    echo "Usage: $0 SUB_FOLDER [FILE_PATTERN]"
    exit 1
}

BASE_DIR=/data
[email protected]
SCP_TARGET_DIR=/home/scp_files

SOURCE_FOLDER=${BASE_DIR}/${SUB_FOLDER}
LAST_CHECKPOINT=$SOURCE_FOLDER/checkpoint

# Get all files matching the file pattern and newer than the checkpoint 
# In case there are multiple files, just choose the latest one to use
# If file pattern is not present, use *.csv as a default
FILE=$(find $SOURCE_FOLDER -name "${FILE_PATRN:-*.csv}" -newer ${LAST_CHECKPOINT} -type f | sort | tail -n 1 )

# Compare the timestamp with checkpoint
if [[ ${FILE} ]]; then
    echo "Newer file found! Copying ${FILE} to informatica..."
    scp "${FILE}" "${SCP_TARGET_HOST}:${SCP_TARGET_DIR}/${FILE_PATRN}.csv"

    if [[ "$?" -ne 0 ]]; then
        echo "Unable to scp file"
        exit 1
    fi

    if [[ "$?" -ne 0 ]]; then
        echo "Unable to complete SCP"
        exit 1
    else
        echo "Updating the checkpoint file.."
        touch $LAST_CHECKPOINT
    fi
else
    echo "No update since last check, skipping..."
fi