check_feed_link_statuses.sh

#!/usr/bin/env bash

# Enable error handling
set -o pipefail

# Check for feed URL argument
if [ $# -eq 0 ]; then
    echo "Usage: $0 <feed_url>"
    echo "Example: $0 https://www.gobankingrates.com/feed/external-limited"
    exit 1
fi

feed_url="$1"

# Create a temporary file for the feed
feed_file=$(mktemp)
trap 'rm -f "$feed_file"' EXIT

# Download the feed once
curl -s "$feed_url" > "$feed_file"

# Check if feed was downloaded successfully
if [ ! -s "$feed_file" ]; then
    echo "Error: Failed to download feed from $feed_url or feed is empty"
    exit 1
fi

# Process each item
while IFS= read -r item_link; do
    # Skip empty lines and the feed URL itself
    [ -z "$item_link" ] && continue
    [[ "$item_link" == *"/feed/"* ]] && continue
    
    echo "Checking article: $item_link" >&2
    
    # Get content for this article
    content=$(xmllint --xpath "//*[local-name()='encoded' and ../link[contains(text(),'$item_link')]]/text()" "$feed_file" 2>/dev/null)
    
    # Extract and check URLs
    if [ -n "$content" ]; then
        canonical_printed=false
        
        echo "$content" | grep -oE 'https?://[^"]+' | sort -u | \
        while read -r url; do
            [ -z "$url" ] && continue
            
            # Clean the URL
            cleaned_url="${url%[.,;:]}"
            
            # Check the URL status
            code=$(curl -A "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" \
                       -L \
                       -o /dev/null \
                       -s \
                       -w "%{http_code}" \
                       "$cleaned_url")
            
            # Output non-200 status codes
            if [ "$code" -ne 200 ]; then
                if [ "$canonical_printed" = false ]; then
                    echo "Canonical: $item_link"
                    canonical_printed=true
                fi
                echo "$code $cleaned_url"
            fi
        done
        
        # Add spacing if we printed anything
        [ "$canonical_printed" = true ] && echo ""
    fi
done < <(xmllint --xpath "//link/text()" "$feed_file" 2>/dev/null)