A web scanner that checks for secrets in the FE of a website as well as backups old compressed versions and accepts default patterns to search for.

sh

#!/bin/sh
# deep_scan_final.sh - Fixed & Improved version (2026)
# Usage: ./deep_scan_final.sh [--fast|--full] <url|file.txt> [custom-pattern]

# -------------------------------------------------------------------
# 0. ARGUMENT PARSING (mode flag must come first)
# -------------------------------------------------------------------
SCAN_MODE="full"   # default

case "$1" in
    --fast) SCAN_MODE="fast"; shift ;;
    --full) SCAN_MODE="full"; shift ;;
esac

usage() { echo "Usage: $0 [--fast|--full] <url|file.txt> [custom-pattern]"; exit 1; }
[ $# -eq 0 ] && usage

# If first argument is a file, read URLs from it
if [ -f "$1" ]; then
    while read -r line; do
        [ -z "$line" ] && continue
        "$0" "--${SCAN_MODE}" "$line" "${2:-}"   # recursive call for each URL
    done < "$1"
    exit 0
fi

BASE_URL="$1"
CUSTOM_PATTERN="${2:-}"

echo "[*] Scan mode: ${SCAN_MODE}"

# -------------------------------------------------------------------
# 1. PATTERNS – high confidence only
# -------------------------------------------------------------------
DEFAULT_PATTERN="sk-[a-zA-Z0-9_-]{48,}|sk-ant-[a-zA-Z0-9_-]{80,}|gsk_[a-zA-Z0-9_-]{40,}|AIza[0-9A-Za-z_-]{35}|sk_live_[0-9a-zA-Z]{20,}|pk_live_[0-9a-zA-Z]{20,}|AKIA[0-9A-Z]{16}|[0-9a-fA-F]{32}|[0-9a-fA-F]{64}|eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9|FLW(PUBK|SECK|ENC)[_-]|sk_(live|test)_|pk_(live|test)_|trello|supabase|openai|anthropic|groq|mistral|stripe|aws|clerk_|ghp_"

MAIN_PATTERN="${CUSTOM_PATTERN:-$DEFAULT_PATTERN}"

AWS_ID_PATTERN="AKIA[0-9A-Z]{16}"
AWS_SECRET_PATTERN="[A-Za-z0-9/+=]{40}"

SENSITIVE_KV_PATTERN='(api_key|apikey|secret|password|passwd|token|private_key|client_secret|db_password|auth_token|access_token|refresh_token)[[:space:]]*[:=][[:space:]]*['\''"]?[A-Za-z0-9_\-+=/]+'

ALL_PATTERNS="${MAIN_PATTERN}|${AWS_ID_PATTERN}|${AWS_SECRET_PATTERN}|${SENSITIVE_KV_PATTERN}"

# -------------------------------------------------------------------
# 2. SCAN MODE: file extension allow-lists
# -------------------------------------------------------------------
# Fast mode: JS/JSON/map only (high signal, low noise)
FAST_EXTENSIONS="js|json|map"

# Full mode: everything
FULL_EXTENSIONS="js|json|xml|env|yaml|yml|conf|txt|php|sql|log|csv|ini|toml|map|pem|key|bak|backup|swp|zip|tar|gz|rar"

if [ "$SCAN_MODE" = "fast" ]; then
    SCAN_EXTENSIONS="$FAST_EXTENSIONS"
else
    SCAN_EXTENSIONS="$FULL_EXTENSIONS"
fi

# Returns 0 if a URL's extension is allowed in the current scan mode
extension_allowed() {
    url="$1"
    # Strip query/fragment, extract extension
    ext=$(echo "$url" | sed 's/[?#].*//' | sed -E 's/.*\.([a-zA-Z0-9]+)$/\1/' | tr '[:upper:]' '[:lower:]')
    echo "$ext" | grep -qE "^(${SCAN_EXTENSIONS})$"
}

# -------------------------------------------------------------------
# 3. NOISE FILES – never scan these
# -------------------------------------------------------------------
NOISE_FILES="robots.txt sitemap.xml crossdomain.xml security.txt humans.txt ads.txt"

# -------------------------------------------------------------------
# 4. PATH LISTS (backups, phpmyadmin, adminer, well-known)
# -------------------------------------------------------------------
GENERIC_PATHS="
backend.tar new.tar old.tar 2020.tar 2021.tar 2022.tar 2023.tar 2024.tar 2025.tar 2019.tar 2018.tar
backup.tar backups.tar frontend.tar htdocs.tar www.tar public_html.tar server.tar website.tar
include.tar includes.tar php.tar html.tar oldsite.tar newsite.tar site.tar oldwebsite.tar
newwebsite.tar new-site.tar old-site.tar config.tar configs.tar full.tar fullsite.tar web.tar
newweb.tar files.tar file.tar back.tar bckup.tar configure.tar aws.tar s3.tar save.tar saved.tar
db.tar database.tar dbs.tar sql.tar mysql.tar postgresql.tar
backend.zip new.zip old.zip 2020.zip 2021.zip 2022.zip 2023.zip 2024.zip 2025.zip 2019.zip 2018.zip
backup.zip backups.zip frontend.zip htdocs.zip www.zip public_html.zip server.zip website.zip
include.zip includes.zip php.zip html.zip oldsite.zip newsite.zip site.zip oldwebsite.zip
newwebsite.zip new-site.zip old-site.zip config.zip configs.zip full.zip fullsite.zip web.zip
newweb.zip files.zip file.zip back.zip bckup.zip configure.zip aws.zip s3.zip save.zip saved.zip
db.zip database.zip dbs.zip sql.zip mysql.zip postgresql.zip
backend.gz new.gz old.gz backup.gz backups.gz sql.gz mysql.gz db.gz database.gz
backend.rar backup.rar backups.rar www.rar sql.rar db.rar
backup.tar.gz website.tar.gz sql.tar.gz db.tar.gz backup.tar.bz2
backend.backup backup.backup sql.backup db.backup config.zip.bak web.zip.bak backup.zip~
"

PHPMYADMIN_PATHS="
phpmyadmin/ phpMyAdmin/ db/phpmyadmin/ admin/phpmyadmin/ _phpmyadmin/ administrator/phpmyadmin/
mysql/admin/ sql/phpmyadmin/ pma/ PMA/ admin/pma/ db/pma/
"

ADMINER_PATHS="
adminer.php adminer/ db.php sql.php adminer-4.8.1.php adminer-4.8.0.php _adminer.php
"

WELL_KNOWN_FILES="
.env .env.production config.json secrets.json .git/config .htaccess phpinfo.php
"

# -------------------------------------------------------------------
# 5. PREPARATION
# -------------------------------------------------------------------
RANDOM_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/$(shuf -i 120-130 -n 1).0.0.0 Safari/537.36"
mkdir -p scans
touch scans/secrets_found.txt
touch scans/processed_files.txt
# NEW: store SHA-256 hashes of already-scanned content
touch scans/scanned_hashes.txt

# Rate limiting (increase if you get blocked)
RATE_DELAY=0.25

get_service_name() {
    key="$1"
    case "$key" in
        AIza[0-9A-Za-z_-]*) echo "Google API Key" ;;
        AKIA[0-9A-Z]*) echo "AWS Access Key" ;;
        sk_live_*|pk_live_*) echo "Stripe Live Key" ;;
        ghp_*) echo "GitHub Token" ;;
        * ) echo "Secret/Key" ;;
    esac
}

# -------------------------------------------------------------------
# 6. FETCH FUNCTION (wget with curl fallback)
# -------------------------------------------------------------------
fetch_url() {
    url="$1"
    content=$(wget -qO- --timeout=10 --tries=2 --no-cache \
        --user-agent="$RANDOM_UA" \
        --header="Accept: text/html,application/xhtml+xml" \
        --header="Accept-Language: en-US,en" \
        "$url" 2>/dev/null)
    if [ -n "$content" ]; then
        echo "$content"
        return 0
    fi
    content=$(curl -sL --max-time 10 --user-agent "$RANDOM_UA" \
        -H "Accept: text/html,application/xhtml+xml" \
        "$url" 2>/dev/null)
    if [ -n "$content" ]; then
        echo "$content"
        return 0
    fi
    return 1
}

# -------------------------------------------------------------------
# 6b. RELIABLE HTTP STATUS
# -------------------------------------------------------------------
get_http_status() {
    url="$1"
    curl -s -o /dev/null -w "%{http_code}" --max-time 10 \
        --user-agent "$RANDOM_UA" \
        -H "Accept: text/html,application/xhtml+xml" \
        "$url" 2>/dev/null || echo "000"
}

# -------------------------------------------------------------------
# 7. SHA-256 HELPER (portable: sha256sum or shasum)
# -------------------------------------------------------------------
sha256_file() {
    filepath="$1"
    if command -v sha256sum >/dev/null 2>&1; then
        sha256sum "$filepath" | awk '{print $1}'
    elif command -v shasum >/dev/null 2>&1; then
        shasum -a 256 "$filepath" | awk '{print $1}'
    else
        # Fallback: no dedup possible, return unique sentinel per call
        echo "nohash-$(date +%s%N)"
    fi
}

# -------------------------------------------------------------------
# 8. SCAN ANY FILE
#    Skips: noise files, disallowed extensions (by mode), duplicate content
# -------------------------------------------------------------------
scan_any_file() {
    file_url="$1"
    origin_route="$2"
    normalized=$(echo "$file_url" | sed 's/[?#].*//')

    # --- 8a. Skip noise files ---
    for noise in $NOISE_FILES; do
        case "$normalized" in
            *"/$noise") echo "   → SKIP (noise file): $normalized"; return ;;
        esac
    done

    # --- 8b. Extension filter (mode-aware) ---
    if ! extension_allowed "$normalized"; then
        echo "   → SKIP (extension not in ${SCAN_MODE} mode): $normalized"
        return
    fi

    echo "   → FETCH: $normalized"
    tmpfile=$(mktemp)
    wget -qO "$tmpfile" --timeout=15 --tries=2 --no-cache \
        --user-agent="$RANDOM_UA" "$normalized" 2>/dev/null || \
    curl -sL --max-time 15 --user-agent "$RANDOM_UA" -o "$tmpfile" "$normalized" 2>/dev/null

    if [ ! -s "$tmpfile" ]; then
        echo "      (failed or empty)"
        rm -f "$tmpfile"
        sleep "$RATE_DELAY"
        return
    fi

    # --- 8c. Content-hash deduplication (THE core fix) ---
    content_hash=$(sha256_file "$tmpfile")
    if grep -qFx "$content_hash" scans/scanned_hashes.txt 2>/dev/null; then
        echo "   → SKIP (duplicate content, hash: ${content_hash:0:16}…): $normalized"
        rm -f "$tmpfile"
        sleep "$RATE_DELAY"
        return
    fi
    # Record hash and URL
    echo "$content_hash" >> scans/scanned_hashes.txt
    echo "$normalized"   >> scans/processed_files.txt

    echo "   → SCAN: $normalized (hash: ${content_hash:0:16}…)"

    # --- 8d. Pattern matching ---
    matches=$(grep -a -oE "$ALL_PATTERNS" "$tmpfile" 2>/dev/null | sort -u)
    if [ -n "$matches" ]; then
        echo "$matches" | while read -r key; do
            [ ${#key} -lt 8 ] && continue
            [ "$key" = "10000000100040008000100000000000" ] && continue
            echo "$key" | grep -qE '^[0-9]+$' && continue
            service=$(get_service_name "$key")
            echo "Service: $service | Route: $origin_route | File: $normalized | Hash: $content_hash | Key: $key" >> scans/secrets_found.txt
        done
        echo "      ✅ found $(echo "$matches" | wc -l) candidate(s)"
    else
        echo "      (no secrets)"
    fi

    rm -f "$tmpfile"
    sleep "$RATE_DELAY"
}

# -------------------------------------------------------------------
# 9. CHECK A SINGLE PATH
# -------------------------------------------------------------------
check_path() {
    path_url="$1"
    origin_route="$2"
    full_url="${BASE_URL%/}/${path_url#/}"
    case "$path_url" in
        */) test_url="${full_url}index.php" ;;
        *) test_url="$full_url" ;;
    esac

    status=$(get_http_status "$test_url")
    if [ "$status" = "200" ]; then
        echo "   → FOUND: $test_url"
        if [ "${path_url%/}" != "$path_url" ]; then
            config_url="${full_url}config.inc.php"
            scan_any_file "$config_url" "$origin_route"
        else
            scan_any_file "$test_url" "$origin_route"
        fi
    fi
}

# -------------------------------------------------------------------
# 10. PARSE SITEMAP -> EXTRACT ALL ROUTES
# -------------------------------------------------------------------
parse_sitemap() {
    sitemap_url="${BASE_URL%/}/sitemap.xml"
    echo "→ Checking sitemap: $sitemap_url"
    sitemap=$(fetch_url "$sitemap_url")
    if [ -z "$sitemap" ]; then
        echo "   No sitemap found or inaccessible."
        return 1
    fi
    echo "$sitemap" | grep -oE '<loc>[^<]+</loc>' | sed 's/<loc>//;s/<\/loc>//' | \
        grep -vE '\.(xml|txt|jpg|png|css|js)$' > /tmp/sitemap_urls_$$.txt
    count=$(wc -l < /tmp/sitemap_urls_$$.txt | tr -d ' ')
    echo "   Sitemap contains $count URLs (excluding static assets)."
    while read -r url; do
        [ -z "$url" ] && continue
        if echo "$url" | grep -q "^${BASE_URL%/}"; then
            rel=$(echo "$url" | sed "s|${BASE_URL%/}||")
            echo "   Route from sitemap: $rel"
            echo "$rel" >> /tmp/discovered_routes_$$.txt
        fi
    done < /tmp/sitemap_urls_$$.txt
    rm -f /tmp/sitemap_urls_$$.txt
    return 0
}

# -------------------------------------------------------------------
# 11. PROCESS A ROUTE
#     Heavy checks (archives, phpMyAdmin, adminer) are ROOT-ONLY.
# -------------------------------------------------------------------
process_route() {
    route_url="$1"
    is_base_url=0
    [ "$route_url" = "$BASE_URL" ] || [ "$route_url" = "${BASE_URL%/}/" ] && is_base_url=1

    echo "=========================================="
    echo "Processing route: $route_url"
    [ "$is_base_url" = "1" ] && echo "(base URL – full probe enabled)"
    echo "=========================================="

    html=$(fetch_url "$route_url")
    if [ -z "$html" ]; then
        echo "   Failed to fetch route (maybe SPA, skipping HTML extraction)"
    else
        # Extract linked asset URLs matching current scan mode extensions
        echo "$html" | grep -oE '(https?:)?//[^"'\''<>[:space:]]+\.('"$SCAN_EXTENSIONS"')' | \
        sed -E 's|^//|https://|' | sed -E "s|^([^h])|${BASE_URL%/}/\1|" | sed 's|[?#].*||' | sort -u > /tmp/linked_files_$$.txt

        while read -r file_url; do
            [ -z "$file_url" ] && continue
            scan_any_file "$file_url" "$route_url"
        done < /tmp/linked_files_$$.txt
        rm -f /tmp/linked_files_$$.txt
    fi

    # --- Well-known config files (all routes, mode-aware) ---
    # In fast mode, only scan config.json from the well-known list
    if [ "$SCAN_MODE" = "fast" ]; then
        check_path "config.json"  "$route_url"
        check_path ".env"         "$route_url"
    else
        for f in $WELL_KNOWN_FILES; do
            check_path "$f" "$route_url"
        done
    fi

    # --- ROOT-ONLY heavy checks ---
    if [ "$is_base_url" = "1" ]; then
        if [ "$SCAN_MODE" = "full" ]; then
            echo "   [root] Probing generic backup paths…"
            for path in $GENERIC_PATHS; do
                check_path "$path" "$route_url"
            done
        fi

        echo "   [root] Probing phpMyAdmin / Adminer paths…"
        for path in $PHPMYADMIN_PATHS; do
            check_path "$path" "$route_url"
        done
        for path in $ADMINER_PATHS; do
            check_path "$path" "$route_url"
        done
    else
        echo "   (deep route – skipping archive/backup/phpMyAdmin probes)"
    fi
}

# -------------------------------------------------------------------
# 12. MAIN: DISCOVER ROUTES
# -------------------------------------------------------------------
echo "[*] Discovering routes on $BASE_URL"
> /tmp/discovered_routes_$$.txt

parse_sitemap
if [ ! -s /tmp/discovered_routes_$$.txt ]; then
    echo "→ No sitemap or empty. Falling back to HTML link extraction."
    homepage=$(fetch_url "$BASE_URL")
    if [ -z "$homepage" ]; then
        echo "Error: Cannot fetch $BASE_URL even with fallback. Exiting."
        exit 1
    fi
    domain=$(echo "$BASE_URL" | sed -E 's|https?://([^/]+).*|\1|')
    all_links=$(echo "$homepage" | grep -oE 'href="[^"]+"' | sed 's/href="//;s/"//' | grep -E "^(/|https?://$domain)")
    interesting_paths=$(echo "$all_links" | grep -iE '/(payment|map|chat|api|dashboard|admin|profile|account|billing|checkout|cart|wallet|transfer|auth|login|signup)' | sort -u)

    if [ -z "$interesting_paths" ]; then
        for path in payment maps chat api dashboard admin profile account; do
            test_url="${BASE_URL%/}/$path"
            status=$(get_http_status "$test_url")
            [ "$status" = "200" ] && interesting_paths="$interesting_paths
/$path"
        done
    fi
    echo "$interesting_paths" | sed '/^$/d' | sort -u >> /tmp/discovered_routes_$$.txt
fi

discovered_routes=$(sort -u /tmp/discovered_routes_$$.txt | sed '/^$/d')
rm -f /tmp/discovered_routes_$$.txt

echo "[*] Routes to scan (total: $(echo "$discovered_routes" | wc -l)):"
echo "$discovered_routes" | while read -r r; do echo "   $r"; done

# -------------------------------------------------------------------
# 13. SCAN EACH ROUTE
# -------------------------------------------------------------------
echo ""
echo "[*] Starting scan (mode=${SCAN_MODE}, content-hash dedup, noise filtered)…"
for rel_path in $discovered_routes; do
    full_url="${BASE_URL%/}${rel_path}"
    process_route "$full_url"
done

# Always scan the base URL itself
process_route "$BASE_URL"

# Cleanup
rm -f /tmp/*_$$.txt 2>/dev/null || true

echo ""
echo "[*] Done! Results saved to scans/secrets_found.txt"
echo "Processed URLs:   scans/processed_files.txt"
echo "Scanned hashes:   scans/scanned_hashes.txt"