Reads a sitemap XML file and scans all pages for specified keywords. Caches pages so that rescans can be done quickly and without overloading sites.

site-keyword-search.sh

#!/bin/bash

#
# This script checks a sitemap for pages that contain any of the specified keywords in
# their main content, excluding menus, headers, and footers.
#
# It accomplishes this by fetching and parsing the sitemap with curl, extracting URLs with
# xmllint, and then for each URL:
#   1. Fetching the page content with curl.
#   2. Using perl to remove <nav>, <header>, and <footer> sections
#   3. Using lynx to extract the text content
#   4. Using grep to search for the specified keywords in the cleaned text.
#
# Caching to the temp directory is enabled by default to avoid re-fetching pages.
# Use --no-cache to disable caching, or -f/--force to force re-fetching and re-caching.
#
# Example usage:
#   ./site-keyword-search.sh https://example.com/sitemap.xml keyword1 keyword2 keyword3
#

MATCH_COUNT=0
TOTAL_PAGES=0
CACHE_DIR="${TMPDIR:-/tmp/}$(basename "$0")"
USE_CACHE=true
FORCE=false

# Required tools check
for cmd in curl lynx realpath perl shasum xmllint wc tr grep; do
    if ! command -v "$cmd" &> /dev/null; then
        echo "Error: '$cmd' is required but not installed."
        exit 1
    fi
done

write_usage() {
    {
        [[ -n "$1" ]] && echo "Error: $1"
        echo
        echo "Usage: $0 <sitemap_url> <list of keywords> [options]"
        echo "Example: $0 https://example.com/sitemap.xml keyword1 keyword2"
        echo "Options:"
        echo "  --no-cache     Do not read from or write to page cache"
        echo "  -f, --force    Force re-fetching and re-caching of pages"
        echo "  -h, --help     Show this help message and exit"
        echo
    } >&2
    exit 1
}

SITEMAP_URL=$1
[[ -z $SITEMAP_URL ]] && write_usage 'No sitemap URL provided.' && exit 1
shift

# Keywords and Options
words=()
options=()
for i in "$@"; do
    if [[ $i == -* ]]; then options+=("$i"); else words+=("$i"); fi
done
words_pattern=$(IFS='|'; echo "${words[*]}")
[[ ${#words[@]} -eq 0 ]] && write_usage "No keywords provided."

# Examine options
for option in "${options[@]}"; do
    [[ $option == "-h" || $option == "--help" ]] && write_usage && exit 0
    [[ $option == "-f" || $option == "--force" ]] && FORCE=true
    [[ $option == "--no-cache" ]] && USE_CACHE=false
done

# If they didn't provide a URL, check if it's local, and convert it to a file:// URL.
if [[ -n $SITEMAP_URL && ! $SITEMAP_URL =~ ^https?:// ]]; then
    [[ ! -f $SITEMAP_URL ]] && write_usage "File '$SITEMAP_URL' does not exist."
    SITEMAP_URL="file://$(realpath "$SITEMAP_URL")"
fi

# Init cache directory
$USE_CACHE && mkdir -p "$CACHE_DIR"

hash_url() {
    local hash
    hash=$(printf '%s' "$1" | shasum -a 256)
    echo "${hash%% *}"
}

get_cache_filename() {
    echo "$CACHE_DIR/$(hash_url "$1").html"
}

# Function to extract human-readable text, excluding menus/headers/footers. We do it a Q&D
# way with perl and lynx, as opposed to full HTML parsing with perhaps Node.js.
extract_text() {
    local url=$1
    local cache_file raw_html
    # Load from cache if enabled and available
    if $USE_CACHE; then
        cache_file=$(get_cache_filename "$url") || return 1
        ! $FORCE && [[ -f $cache_file ]] && echo -n '(cached) ' >&2 && raw_html=$(cat "$cache_file")
    fi
    # If we still don't have the HTML, fetch the page
    if [[ -z $raw_html ]]; then
        raw_html=$(curl -sL "$url") || return 1
        echo -n '(fetched) ' >&2
        $USE_CACHE && printf '%s' "$raw_html" > "$cache_file" 2>/dev/null
    fi
    # 1. curl fetches the raw HTML
    # 2. perl uses a regex (-0777 slurps the whole file to handle multi-line tags)
    #    to strip out <nav>, <header>, and <footer> tags and EVERYTHING inside them.
    # 3. lynx reads the cleaned HTML from standard input (-stdin) and renders it (-force_html).
    printf '%s' "$raw_html" | \
    perl -0777 -pe 's/<nav\b[^>]*>.*?<\/nav>//igs; s/<header\b[^>]*>.*?<\/header>//igs; s/<footer\b[^>]*>.*?<\/footer>//igs;' | \
    lynx -dump -force_html -stdin -nolist
}

echo "Retrieving sitemap from $SITEMAP_URL..."

# Extract URLs from the sitemap XML
URLS=$(xmllint --xpath "//*[local-name()='loc']/text()" <(curl -sL "$SITEMAP_URL"))
url_count=$(echo "$URLS" | wc -l | tr -d ' ')

if [ -z "$URLS" ]; then
    echo "Failed to retrieve any URLs from the sitemap."
    exit 1
fi

echo "Scanning $url_count pages (excluding menus)..."
echo "Looking for keywords: ${words[*]}"
echo "--------------------------------------------------------------------"

i=0
for url in $URLS; do
    ((TOTAL_PAGES++))
    ((i++))
    # Extract text and use grep to quietly search for whole words (\b), case-insensitive (-i)
    echo -n "$i. $url "
    if extract_text "$url" | grep -iqE "\b($words_pattern)\b"; then
        echo "✅"
        ((MATCH_COUNT++))
    else
        echo "❌"
    fi
done

echo "--------------------------------------------------------------------"
echo "Total matches in main content: $MATCH_COUNT out of $TOTAL_PAGES"