Last active
June 4, 2026 20:14
-
-
Save AldeRoberge/ffe92736c0c70476fe9cf69a1746d50a to your computer and use it in GitHub Desktop.
Find 404 links, crawls a website and checks if links are broken. Written by Claude Sonnet 4.6 (low/high) after 35+ prompts.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #Requires -Version 5.1 | |
| <# | |
| .SYNOPSIS | |
| Dead Link Checker - Crawls a website and reports broken links, with Google-powered suggested replacements. | |
| .PARAMETER StartUrl Root URL to crawl. Omit to reuse last saved URL. | |
| .PARAMETER MaxDepth Max crawl depth (default 3). | |
| .PARAMETER MaxParallel Max concurrent requests (default 10). | |
| .PARAMETER OutputCsv CSV output path (default .\dead-links-<timestamp>.csv). | |
| .PARAMETER SameDomainOnly Stay on same domain (default $true). | |
| .PARAMETER TimeoutSec Request timeout seconds (default 15). | |
| .PARAMETER RateLimitDelayMs Base delay ms on 429 (default 2000). | |
| .PARAMETER SuggestFixes Search Google for replacement URLs on broken links (default $true). | |
| .EXAMPLE | |
| .\Invoke-DeadLinkChecker.ps1 -StartUrl "https://example.com" | |
| .EXAMPLE | |
| .\Invoke-DeadLinkChecker.ps1 # resumes with last used URL | |
| .EXAMPLE | |
| .\Invoke-DeadLinkChecker.ps1 -StartUrl "https://example.com" -SuggestFixes $false | |
| #> | |
| [CmdletBinding()] | |
| param( | |
| [Parameter()][string]$StartUrl = '', | |
| [Parameter()][ValidateRange(1,20)][int]$MaxDepth = 100, | |
| [Parameter()][ValidateRange(1,50)][int]$MaxParallel = 50, | |
| [Parameter()][string]$OutputCsv = '', | |
| [Parameter()][bool]$SameDomainOnly = $true, | |
| [Parameter()][ValidateRange(3,120)][int]$TimeoutSec = 10, | |
| [Parameter()][ValidateRange(500,60000)][int]$RateLimitDelayMs = 2000, | |
| [Parameter()][string]$UserAgent = "DeadLinkChecker/1.0 (PowerShell)", | |
| [Parameter()][bool]$SuggestFixes = $true | |
| ) | |
| $ErrorActionPreference = "Stop" | |
| $ScriptBase = if ($PSScriptRoot -and $PSScriptRoot -ne '') { $PSScriptRoot } else { (Get-Location).Path } | |
| $ConfigFile = Join-Path $ScriptBase 'deadlinkchecker.config.json' | |
| function Read-Config { | |
| if (Test-Path $ConfigFile) { | |
| try { return (Get-Content $ConfigFile -Raw | ConvertFrom-Json) } catch {} | |
| } | |
| return [PSCustomObject]@{ LastUrl = '' } | |
| } | |
| function Save-Config([string]$url) { | |
| try { [PSCustomObject]@{ LastUrl = $url } | ConvertTo-Json | Set-Content $ConfigFile -Encoding UTF8 } catch {} | |
| } | |
| $cfg = Read-Config | |
| if ($StartUrl -eq '') { | |
| $sep80 = '-' * 80 | |
| Write-Host "" | |
| Write-Host " DEAD LINK CHECKER" -ForegroundColor Cyan | |
| Write-Host " $sep80" -ForegroundColor DarkGray | |
| if ($cfg.LastUrl -ne '') { | |
| Write-Host (" Last URL : {0}" -f $cfg.LastUrl) -ForegroundColor DarkGray | |
| Write-Host " $sep80" -ForegroundColor DarkGray | |
| Write-Host "" | |
| Write-Host " Press ENTER to reuse the last URL, or type a new one:" -ForegroundColor White | |
| Write-Host " > " -NoNewline -ForegroundColor Cyan | |
| $typed = (Read-Host).Trim() | |
| if ($typed -eq '') { | |
| $StartUrl = $cfg.LastUrl | |
| Write-Host " Using: $StartUrl" -ForegroundColor Yellow | |
| } else { | |
| $StartUrl = $typed | |
| } | |
| } else { | |
| Write-Host " $sep80" -ForegroundColor DarkGray | |
| Write-Host "" | |
| Write-Host " Enter the URL to crawl:" -ForegroundColor White | |
| Write-Host " > " -NoNewline -ForegroundColor Cyan | |
| $typed = (Read-Host).Trim() | |
| if ($typed -eq '') { | |
| Write-Host " ERROR: No URL provided." -ForegroundColor Red | |
| exit 1 | |
| } | |
| $StartUrl = $typed | |
| } | |
| Write-Host "" | |
| } | |
| if (-not [Uri]::IsWellFormedUriString($StartUrl, [System.UriKind]::Absolute)) { | |
| Write-Host " ERROR: Invalid URL: $StartUrl" -ForegroundColor Red; exit 1 | |
| } | |
| if ($StartUrl -match '^http://') { | |
| $StartUrl = $StartUrl -replace '^http://', 'https://' | |
| Write-Host " NOTE: Upgraded StartUrl scheme to https://" -ForegroundColor DarkGray | |
| } | |
| Save-Config $StartUrl | |
| if ($OutputCsv -eq '') { | |
| $OutputCsv = Join-Path $ScriptBase ("dead-links-" + (Get-Date -Format 'yyyyMMdd-HHmmss') + ".csv") | |
| } | |
| $Visited = [System.Collections.Concurrent.ConcurrentDictionary[string,bool]]::new( | |
| [System.StringComparer]::OrdinalIgnoreCase) | |
| $BrokenSeen = [System.Collections.Concurrent.ConcurrentDictionary[string,bool]]::new( | |
| [System.StringComparer]::OrdinalIgnoreCase) | |
| $Queue = [System.Collections.Concurrent.ConcurrentQueue[PSCustomObject]]::new() | |
| $Results = [System.Collections.Concurrent.ConcurrentBag[PSCustomObject]]::new() | |
| $CsvLock = New-Object System.Threading.SemaphoreSlim(1,1) | |
| $Stats = [System.Collections.Hashtable]::Synchronized(@{ | |
| Total=0; OK=0; Broken=0; Errors=0; RateLimited=0 | |
| }) | |
| $StartTime = Get-Date | |
| $BaseDomain = ([Uri]$StartUrl).Host | |
| function Get-ConsoleWidth { | |
| try { $w = $Host.UI.RawUI.WindowSize.Width; if ($w -gt 10) { return [Math]::Min($w,100) } } catch {} | |
| return 80 | |
| } | |
| function Write-Banner { | |
| Clear-Host | |
| $sep = '-' * (Get-ConsoleWidth) | |
| Write-Host "" | |
| Write-Host " DEAD LINK CHECKER" -ForegroundColor Cyan | |
| Write-Host " $sep" -ForegroundColor DarkGray | |
| Write-Host " Target : $StartUrl" -ForegroundColor White | |
| Write-Host (" Depth : {0} | Parallel: {1} | Domain-only: {2} | Suggest-fixes: {3}" -f $MaxDepth, $MaxParallel, $SameDomainOnly, $SuggestFixes) -ForegroundColor DarkGray | |
| Write-Host " Report : $OutputCsv" -ForegroundColor DarkGray | |
| Write-Host " $sep" -ForegroundColor DarkGray | |
| Write-Host "" | |
| } | |
| function Get-StatusColor([int]$c) { | |
| if ($c -eq 0) { 'DarkGray' } | |
| elseif($c -lt 300) { 'Green' } | |
| elseif($c -lt 400) { 'Yellow' } | |
| elseif($c -eq 429) { 'Magenta' } | |
| elseif($c -lt 500) { 'Red' } | |
| else { 'DarkRed' } | |
| } | |
| function Get-StatusLabel([int]$c) { | |
| if ($c -eq 0) { '[ERR]' } | |
| elseif($c -lt 300) { '[OK ]' } | |
| elseif($c -lt 400) { '[RDR]' } | |
| elseif($c -eq 429) { '[429]' } | |
| else { "[{0}]" -f $c } | |
| } | |
| function Initialize-Csv { | |
| Set-Content -Path $OutputCsv -Encoding UTF8 -Value ` | |
| '"StatusCode","BrokenUrl","LinkText","FoundOnPage","RootPage","Depth","ResponseMs","StatusText","CheckedAt","SuggestedUrl","SuggestedTitle","SuggestedEngine"' | |
| } | |
| function Append-CsvRow([PSCustomObject]$r) { | |
| $CsvLock.Wait() | Out-Null | |
| try { | |
| $row = '{0},"{1}","{2}","{3}","{4}",{5},{6},"{7}","{8}","{9}","{10}","{11}"' -f ` | |
| $r.StatusCode, | |
| ($r.Url -replace '"','""'), | |
| ($r.LinkText -replace '"','""'), | |
| ($r.FoundOnPage -replace '"','""'), | |
| ($r.RootPage -replace '"','""'), | |
| $r.Depth, | |
| $r.ResponseMs, | |
| ($r.StatusText -replace '"','""'), | |
| $r.CheckedAt, | |
| ($r.SuggestedUrl -replace '"','""'), | |
| ($r.SuggestedTitle -replace '"','""'), | |
| ($r.SuggestedEngine -replace '"','""') | |
| Add-Content -Path $OutputCsv -Encoding UTF8 -Value $row | |
| } finally { $CsvLock.Release() | Out-Null } | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Build-SearchTerms (shared by both engines) | |
| # --------------------------------------------------------------------------- | |
| function Build-SearchTerms { | |
| param([string]$BrokenUrl, [string]$LinkText) | |
| $uri = [Uri]$BrokenUrl | |
| $domain = $uri.Host | |
| $genericPhrases = @('click here','here','link','read more','more','lire la suite', | |
| 'en savoir plus','cliquez ici','voir plus','telecharger','download', | |
| 'pdf','document','fichier','page','site') | |
| $queryTerms = $LinkText.Trim() | |
| if ($queryTerms.Length -lt 3 -or $genericPhrases -contains $queryTerms.ToLower()) { | |
| $segments = $uri.AbsolutePath.Trim('/').Split('/') | | |
| Where-Object { $_ -ne '' -and $_ -notmatch '^\d{4}$' } | | |
| ForEach-Object { | |
| $s = [System.Net.WebUtility]::UrlDecode($_) | |
| $s = [System.IO.Path]::GetFileNameWithoutExtension($s) | |
| $s -replace '[-_]',' ' | |
| } | |
| $queryTerms = ($segments | Select-Object -Last 2) -join ' ' | |
| } | |
| return [PSCustomObject]@{ Domain = $domain; Terms = $queryTerms.Trim() } | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Search-Google | |
| # Tries the Google web search scrape. Returns $null if blocked (CAPTCHA / | |
| # 429 / no matching result) so the caller can fall through to DuckDuckGo. | |
| # --------------------------------------------------------------------------- | |
| function Search-Google { | |
| param([string]$Domain, [string]$Terms) | |
| try { | |
| $q = [System.Net.WebUtility]::UrlEncode("site:$Domain $Terms") | |
| $searchUrl = "https://www.google.com/search?q=$q&num=5&hl=en" | |
| $req = [System.Net.HttpWebRequest]::Create($searchUrl) | |
| $req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" | |
| $req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" | |
| $req.Headers.Add("Accept-Language", "en-US,en;q=0.9,fr;q=0.8") | |
| $req.Timeout = 12000 | |
| $req.AllowAutoRedirect = $true | |
| $req.KeepAlive = $false | |
| $resp = $req.GetResponse() | |
| $statusCode = [int]$resp.StatusCode | |
| $sr = [System.IO.StreamReader]::new($resp.GetResponseStream(), [System.Text.Encoding]::UTF8) | |
| $html = $sr.ReadToEnd() | |
| $sr.Close(); $resp.Close() | |
| # Google returns 200 even for CAPTCHA — detect by known page markers | |
| if ($html -match 'id="captcha"' -or | |
| $html -match 'name="captcha"' -or | |
| $html -match '/sorry/index' -or | |
| $html -match 'detected unusual traffic' -or | |
| $statusCode -eq 429) { | |
| return $null # blocked — caller will try DuckDuckGo | |
| } | |
| $suggestedUrl = $null | |
| $suggestedTitle = $null | |
| # Pattern 1: /url?q= redirect (most common organic result wrapper) | |
| $m1 = [regex]::Match($html, '/url\?q=(https?://[^&"]+)&') | |
| if ($m1.Success) { | |
| $raw = [System.Net.WebUtility]::UrlDecode($m1.Groups[1].Value) | |
| if ($raw -match [regex]::Escape($Domain)) { $suggestedUrl = $raw } | |
| } | |
| # Pattern 2: direct href to target domain (featured snippets / newer layout) | |
| if (-not $suggestedUrl) { | |
| $m2 = [regex]::Match($html, 'href="(https?://' + [regex]::Escape($Domain) + '[^"]*)"') | |
| if ($m2.Success) { $suggestedUrl = $m2.Groups[1].Value } | |
| } | |
| if (-not $suggestedUrl) { return $null } | |
| $suggestedUrl = ($suggestedUrl -split '#')[0].TrimEnd('&?') | |
| $mt = [regex]::Match($html, '<h3[^>]*>(.*?)</h3>', | |
| [System.Text.RegularExpressions.RegexOptions]::Singleline) | |
| if ($mt.Success) { | |
| $suggestedTitle = [regex]::Replace($mt.Groups[1].Value, '<[^>]+>', '').Trim() | |
| $suggestedTitle = [System.Net.WebUtility]::HtmlDecode($suggestedTitle) | |
| if ($suggestedTitle.Length -gt 120) { $suggestedTitle = $suggestedTitle.Substring(0,120) } | |
| } | |
| return [PSCustomObject]@{ Url = $suggestedUrl; Title = if ($suggestedTitle) { $suggestedTitle } else { '' }; Engine = 'Google' } | |
| } | |
| catch { return $null } | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Search-DuckDuckGo | |
| # Uses https://html.duckduckgo.com/html/ — the lightweight HTML-only | |
| # endpoint that never serves CAPTCHAs and needs no API key. | |
| # | |
| # DDG result HTML layout: | |
| # Organic links appear as: | |
| # <a class="result__a" href="/l/?uddg=<percent-encoded-url>&...">Title</a> | |
| # or occasionally as direct hrefs. We try the redirect pattern first, | |
| # then fall back to any direct href pointing at the target domain. | |
| # --------------------------------------------------------------------------- | |
| function Search-DuckDuckGo { | |
| param([string]$Domain, [string]$Terms) | |
| try { | |
| $q = [System.Net.WebUtility]::UrlEncode("site:$Domain $Terms") | |
| $searchUrl = "https://html.duckduckgo.com/html/?q=$q" | |
| $req = [System.Net.HttpWebRequest]::Create($searchUrl) | |
| $req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" | |
| $req.Accept = "text/html,application/xhtml+xml,*/*;q=0.8" | |
| $req.Headers.Add("Accept-Language", "en-US,en;q=0.9,fr;q=0.8") | |
| $req.Timeout = 12000 | |
| $req.AllowAutoRedirect = $true | |
| $req.KeepAlive = $false | |
| $resp = $req.GetResponse() | |
| $sr = [System.IO.StreamReader]::new($resp.GetResponseStream(), [System.Text.Encoding]::UTF8) | |
| $html = $sr.ReadToEnd() | |
| $sr.Close(); $resp.Close() | |
| $suggestedUrl = $null | |
| $suggestedTitle = $null | |
| $domainEsc = [regex]::Escape($Domain) | |
| # Pattern 1: DDG /l/?uddg= redirect — decode and check it matches domain | |
| foreach ($m in [regex]::Matches($html, '/l/\?uddg=([^"&]+)', | |
| [System.Text.RegularExpressions.RegexOptions]::IgnoreCase)) { | |
| $decoded = [System.Net.WebUtility]::UrlDecode($m.Groups[1].Value) | |
| if ($decoded -match ('^https?://' + $domainEsc)) { | |
| $suggestedUrl = $decoded | |
| break | |
| } | |
| } | |
| # Pattern 2: direct href pointing to the target domain | |
| if (-not $suggestedUrl) { | |
| $m2 = [regex]::Match($html, | |
| 'href="(https?://' + $domainEsc + '[^"]*)"', | |
| [System.Text.RegularExpressions.RegexOptions]::IgnoreCase) | |
| if ($m2.Success) { $suggestedUrl = $m2.Groups[1].Value } | |
| } | |
| if (-not $suggestedUrl) { return $null } | |
| $suggestedUrl = ($suggestedUrl -split '#')[0].TrimEnd('&?') | |
| # Title: first result__a anchor text | |
| $mt = [regex]::Match($html, 'class="result__a"[^>]*>(.*?)</a>', | |
| [System.Text.RegularExpressions.RegexOptions]::Singleline) | |
| if ($mt.Success) { | |
| $suggestedTitle = [regex]::Replace($mt.Groups[1].Value, '<[^>]+>', '').Trim() | |
| $suggestedTitle = [System.Net.WebUtility]::HtmlDecode($suggestedTitle) | |
| if ($suggestedTitle.Length -gt 120) { $suggestedTitle = $suggestedTitle.Substring(0,120) } | |
| } | |
| return [PSCustomObject]@{ Url = $suggestedUrl; Title = if ($suggestedTitle) { $suggestedTitle } else { '' }; Engine = 'DDG' } | |
| } | |
| catch { return $null } | |
| } | |
| # --------------------------------------------------------------------------- | |
| # Find-SuggestedUrl (orchestrator — Google first, DuckDuckGo fallback) | |
| # --------------------------------------------------------------------------- | |
| function Find-SuggestedUrl { | |
| param( | |
| [string]$BrokenUrl, | |
| [string]$LinkText, | |
| [string]$UserAgent | |
| ) | |
| try { | |
| $st = Build-SearchTerms -BrokenUrl $BrokenUrl -LinkText $LinkText | |
| if ($st.Terms.Length -lt 3) { return $null } | |
| # 1. Try Google | |
| $result = Search-Google -Domain $st.Domain -Terms $st.Terms | |
| if ($result -and $result.Url -and $result.Url -ne $BrokenUrl) { return $result } | |
| # 2. Google blocked or empty — fall back to DuckDuckGo (no CAPTCHA) | |
| Start-Sleep -Milliseconds 400 | |
| $result = Search-DuckDuckGo -Domain $st.Domain -Terms $st.Terms | |
| if ($result -and $result.Url -and $result.Url -ne $BrokenUrl) { return $result } | |
| return $null | |
| } | |
| catch { return $null } | |
| } | |
| $WorkerScript = { | |
| param( | |
| [string]$Url, | |
| [string]$FoundOnPage, | |
| [string]$RootPage, | |
| [string]$LinkText, | |
| [int]$Depth, | |
| [int]$TimeoutSec, | |
| [string]$UserAgent, | |
| [bool]$SameDomainOnly, | |
| [string]$BaseDomain, | |
| [int]$MaxDepth, | |
| [int]$RateLimitDelayMs, | |
| $Queue, | |
| $Visited | |
| ) | |
| # ------------------------------------------------------------------------- | |
| # Normalize-Href: converts any href to an absolute URL to check. | |
| # ------------------------------------------------------------------------- | |
| function Normalize-Href([string]$href, [string]$pageUrl) { | |
| $href = $href.Trim() | |
| if ($href -eq '') { return $null } | |
| # Non-HTTP schemes — skip | |
| if ($href -match '^(mailto:|tel:|javascript:|data:|ftp:|#)') { return $null } | |
| # Fix common typo: https//: or http//: (colon after slashes instead of before) | |
| # e.g. href="https//:www.creehealth.org/" → "https://www.creehealth.org/" | |
| if ($href -match '^(https?)//:(.*)') { $href = $Matches[1] + '://' + $Matches[2] } | |
| # Already absolute (including after typo repair above) | |
| if ($href -match '^https?://') { | |
| if ($href -match '^http://') { $href = $href -replace '^http://','https://' } | |
| return $href | |
| } | |
| # Protocol-relative: //host/path → always treat as https://host/path | |
| if ($href -match '^//') { | |
| $href = 'https:' + $href | |
| return $href | |
| } | |
| # Explicitly relative: starts with /, ., ?, or # — let Uri resolve normally | |
| if ($href -match '^[/.?#]') { | |
| try { | |
| $u = [Uri]::new([Uri]::new($pageUrl), $href) | |
| $r = $u.GetLeftPart([System.UriPartial]::Query) | |
| if ($r -match '^http://') { $r = $r -replace '^http://','https://' } | |
| return $r | |
| } catch { return $null } | |
| } | |
| # ---- Bare path (no leading slash, dot, or scheme) ------------------- | |
| # Handled in priority order: | |
| # | |
| # CASE B — Bare path that starts with the page's own domain: | |
| # href="ville.valdor.qc.ca/plan-economique" | |
| # CMS omitted the scheme. Strip the leading host so it becomes | |
| # /plan-economique then resolve normally. | |
| # | |
| # CASE A — Bare external hostname without scheme: | |
| # href="spcavaldor.org" or href="www.example.com/page" | |
| # First segment contains a dot → treat as an external https:// URL. | |
| # (Will be dropped later by the SameDomainOnly filter if needed.) | |
| # | |
| # DEFAULT — Plain root-relative path segment: | |
| # href="services" → https://origin/services | |
| $pageDomain = ([Uri]$pageUrl).Host # e.g. "ville.valdor.qc.ca" | |
| # Case B: bare path starts with the page's own domain | |
| if ($href -match ('^' + [regex]::Escape($pageDomain) + '(/|$)')) { | |
| $rest = $href.Substring($pageDomain.Length) | |
| if ($rest -eq '') { $rest = '/' } | |
| try { | |
| $origin = ([Uri]$pageUrl).GetLeftPart([System.UriPartial]::Authority) | |
| $r = $origin + $rest | |
| if ($r -match '^http://') { $r = $r -replace '^http://','https://' } | |
| return $r | |
| } catch { return $null } | |
| } | |
| # Case A: first path segment contains a dot → external bare domain | |
| $firstSegment = ($href -split '/')[0] | |
| if ($firstSegment -match '\.') { | |
| return "https://$href" | |
| } | |
| # Default: root-relative path on the same origin | |
| try { | |
| $origin = ([Uri]$pageUrl).GetLeftPart([System.UriPartial]::Authority) | |
| $r = "$origin/$href" | |
| if ($r -match '^http://') { $r = $r -replace '^http://','https://' } | |
| return $r | |
| } catch { return $null } | |
| } | |
| $maxRetries = 3 | |
| $statusCode = 0 | |
| $statusText = 'Unknown Error' | |
| $responseMs = 0 | |
| for ($attempt = 0; $attempt -le $maxRetries; $attempt++) { | |
| $sw = [System.Diagnostics.Stopwatch]::StartNew() | |
| try { | |
| $req = [System.Net.HttpWebRequest]::Create($Url) | |
| $req.Method = 'GET' | |
| $req.UserAgent = $UserAgent | |
| $req.Accept = 'text/html,application/xhtml+xml,*/*' | |
| $req.Timeout = $TimeoutSec * 1000 | |
| $req.AllowAutoRedirect = $true | |
| $req.MaximumAutomaticRedirections = 5 | |
| $req.KeepAlive = $false | |
| $resp = $req.GetResponse() | |
| $statusCode = [int]$resp.StatusCode | |
| $statusText = $resp.StatusDescription | |
| $contentType = $resp.ContentType | |
| $html = $null | |
| if ($Depth -lt $MaxDepth -and $contentType -match 'text/html') { | |
| $sr = [System.IO.StreamReader]::new($resp.GetResponseStream()) | |
| $html = $sr.ReadToEnd() | |
| $sr.Close() | |
| } | |
| $resp.Close() | |
| $sw.Stop() | |
| $responseMs = $sw.ElapsedMilliseconds | |
| if ($html) { | |
| $pat = '(?i)<a\s[^>]*href\s*=\s*(?:"([^"]*?)"|''([^'']*?)''|([^\s>]+))[^>]*>(.*?)</a>' | |
| foreach ($m in [regex]::Matches($html, $pat, | |
| [System.Text.RegularExpressions.RegexOptions]::Singleline)) { | |
| $href = if ($m.Groups[1].Success) { $m.Groups[1].Value } | |
| elseif ($m.Groups[2].Success) { $m.Groups[2].Value } | |
| else { $m.Groups[3].Value } | |
| $lt = ([regex]::Replace($m.Groups[4].Value,'<[^>]+>','').Trim()) -replace '\s+',' ' | |
| $lt = [System.Net.WebUtility]::HtmlDecode($lt) | |
| if ($lt.Length -gt 100) { $lt = $lt.Substring(0,100) } | |
| $nu = Normalize-Href $href $Url | |
| if (-not $nu) { continue } | |
| if ($SameDomainOnly) { | |
| try { if (([Uri]$nu).Host -ne $BaseDomain) { continue } } catch { continue } | |
| } | |
| $childRoot = if ($Depth -eq 0) { $Url } else { $RootPage } | |
| if ($Visited.TryAdd($nu, $true)) { | |
| $Queue.Enqueue([PSCustomObject]@{ | |
| Url = $nu | |
| FoundOnPage = $Url | |
| RootPage = $childRoot | |
| LinkText = $lt | |
| Depth = ($Depth + 1) | |
| }) | |
| } | |
| } | |
| } | |
| $attempt = $maxRetries + 1 | |
| } | |
| catch [System.Net.WebException] { | |
| $sw.Stop(); $responseMs = $sw.ElapsedMilliseconds | |
| $ex = $_.Exception | |
| if ($null -ne $ex.Response) { | |
| $statusCode = [int]$ex.Response.StatusCode | |
| $statusText = $ex.Response.StatusDescription | |
| if ($statusCode -eq 429) { | |
| $ra = $ex.Response.Headers['Retry-After'] | |
| $delay = $RateLimitDelayMs * [Math]::Pow(2, $attempt) | |
| if ($ra) { $p=0; if ([int]::TryParse($ra,[ref]$p)) { $delay = $p*1000 } } | |
| $ex.Response.Close() | |
| if ($attempt -lt $maxRetries) { Start-Sleep -Milliseconds $delay; continue } | |
| } else { $ex.Response.Close() } | |
| } else { | |
| $statusCode = 0 | |
| $statusText = $ex.Message | |
| if ($statusText.Length -gt 120) { $statusText = $statusText.Substring(0,120)+'...' } | |
| } | |
| } | |
| catch { | |
| $sw.Stop(); $responseMs = $sw.ElapsedMilliseconds | |
| $statusCode = 0 | |
| $statusText = $_.Exception.Message | |
| if ($statusText.Length -gt 120) { $statusText = $statusText.Substring(0,120)+'...' } | |
| } | |
| } | |
| return [PSCustomObject]@{ | |
| Url = $Url | |
| FoundOnPage = $FoundOnPage | |
| RootPage = $RootPage | |
| LinkText = $LinkText | |
| StatusCode = $statusCode | |
| StatusText = $statusText | |
| Depth = $Depth | |
| ResponseMs = $responseMs | |
| CheckedAt = (Get-Date -Format 'yyyy-MM-dd HH:mm:ss') | |
| IsBroken = ($statusCode -eq 0 -or $statusCode -ge 400) | |
| SuggestedUrl = '' | |
| SuggestedTitle = "" | |
| SuggestedEngine= "" | |
| } | |
| } | |
| Write-Banner | |
| Initialize-Csv | |
| Write-Host " Starting crawl..." -ForegroundColor White | |
| if ($SuggestFixes) { | |
| Write-Host " Fix suggestions : ON (Google first, DuckDuckGo fallback)" -ForegroundColor DarkGray | |
| } else { | |
| Write-Host " Fix suggestions : OFF (use -SuggestFixes `$true to enable)" -ForegroundColor DarkGray | |
| } | |
| Write-Host "" | |
| $Visited.TryAdd($StartUrl, $true) | Out-Null | |
| $Queue.Enqueue([PSCustomObject]@{ | |
| Url = $StartUrl | |
| FoundOnPage = '' | |
| RootPage = '' | |
| LinkText = 'START' | |
| Depth = 0 | |
| }) | |
| $Pool = [System.Management.Automation.Runspaces.RunspaceFactory]::CreateRunspacePool(1, $MaxParallel) | |
| $Pool.Open() | |
| $Jobs = [System.Collections.Generic.List[hashtable]]::new() | |
| do { | |
| $remaining = [System.Collections.Generic.List[hashtable]]::new() | |
| foreach ($job in $Jobs) { | |
| if ($job.PS.InvocationStateInfo.State -in 'Completed','Failed','Stopped') { | |
| try { | |
| $raw = @($job.PS.EndInvoke($job.Handle)) | |
| $r = @($raw | Where-Object { $_ -ne $null -and $_.GetType().Name -ne 'String' })[0] | |
| if ($null -ne $r) { | |
| $sc = [int]($r.StatusCode -as [int]) | |
| # Deserialized runspace objects have read-only properties — rebuild | |
| # as a fresh PSCustomObject so we can write SuggestedUrl/Title/Engine. | |
| $r = [PSCustomObject]@{ | |
| Url = [string]$r.Url | |
| FoundOnPage = [string]$r.FoundOnPage | |
| RootPage = [string]$r.RootPage | |
| LinkText = [string]$r.LinkText | |
| StatusCode = $sc | |
| StatusText = [string]$r.StatusText | |
| Depth = [int]$r.Depth | |
| ResponseMs = [long]$r.ResponseMs | |
| CheckedAt = [string]$r.CheckedAt | |
| IsBroken = [bool]$r.IsBroken | |
| SuggestedUrl = '' | |
| SuggestedTitle = '' | |
| SuggestedEngine= '' | |
| } | |
| # ---- Suggest a fix for broken links (Google -> DDG fallback) ---- | |
| if ($r.IsBroken -and $SuggestFixes) { | |
| Write-Host "" | |
| Write-Host (" [SRCH] {0}" -f ` | |
| $(if ($r.Url.Length -gt 62) { $r.Url.Substring(0,59)+'...' } else { $r.Url })) ` | |
| -ForegroundColor DarkCyan | |
| $suggestion = Find-SuggestedUrl -BrokenUrl $r.Url -LinkText $r.LinkText -UserAgent $UserAgent | |
| if ($suggestion -and $suggestion.Url -and $suggestion.Url -ne $r.Url) { | |
| $r.SuggestedUrl = $suggestion.Url | |
| $r.SuggestedTitle = $suggestion.Title | |
| $r.SuggestedEngine = if ($suggestion.Engine) { $suggestion.Engine } else { '' } | |
| $engineLabel = $r.SuggestedEngine; if (-not $engineLabel) { $engineLabel = '??' } | |
| $fixColor = if ($engineLabel -eq 'DDG') { 'Green' } else { 'Cyan' } | |
| $sshort = if ($r.SuggestedUrl.Length -gt 62) { $r.SuggestedUrl.Substring(0,59)+'...' } else { $r.SuggestedUrl } | |
| Write-Host (" [FIX?:{0}] {1}" -f $engineLabel, $sshort) -ForegroundColor $fixColor | |
| } else { | |
| Write-Host " [SRCH] no suggestion found" -ForegroundColor DarkGray | |
| } | |
| # Pace the lookups - Google is more sensitive than DDG | |
| Start-Sleep -Milliseconds 800 | |
| } | |
| # ------------------------------------------------------- | |
| $Results.Add($r) | |
| if ($r.IsBroken -and $BrokenSeen.TryAdd([string]$r.Url, $true)) { | |
| Append-CsvRow $r | |
| } | |
| $Stats.Total++ | |
| if ($sc -ge 200 -and $sc -lt 400) { $Stats.OK++ } | |
| elseif($sc -ge 400) { $Stats.Broken++ } | |
| else { $Stats.Errors++ } | |
| if ($sc -eq 429) { $Stats.RateLimited++ } | |
| $label = Get-StatusLabel $sc | |
| $color = Get-StatusColor $sc | |
| $url = [string]$r.Url | |
| $short = if ($url.Length -gt 68) { $url.Substring(0,65)+'...' } else { $url } | |
| $stext = [string]$r.StatusText | |
| $st = if ($stext.Length -gt 30) { $stext.Substring(0,30) } else { $stext } | |
| Write-Host "" | |
| $esc = [char]27 | |
| $osc8o = "${esc}]8;;$($r.Url)${esc}\" | |
| $osc8c = "${esc}]8;;${esc}\" | |
| $line = " {0} {1} {2,6}ms D{3} {4}" -f $label, $short, $r.ResponseMs, $r.Depth, $st | |
| Write-Host "${osc8o}${line}${osc8c}" -ForegroundColor $color | |
| } | |
| } catch { | |
| Write-Host "" | |
| Write-Host (" [WRKERR] " + $_.Exception.Message) -ForegroundColor DarkYellow | |
| } | |
| $job.PS.Dispose() | |
| } else { $remaining.Add($job) } | |
| } | |
| $Jobs = $remaining | |
| $item = $null | |
| while ($Jobs.Count -lt $MaxParallel -and $Queue.TryDequeue([ref]$item)) { | |
| if ($item.Depth -le $MaxDepth) { | |
| $ps = [System.Management.Automation.PowerShell]::Create() | |
| $ps.RunspacePool = $Pool | |
| [void]$ps.AddScript($WorkerScript) | |
| [void]$ps.AddParameters([ordered]@{ | |
| Url = $item.Url | |
| FoundOnPage = $item.FoundOnPage | |
| RootPage = $item.RootPage | |
| LinkText = $item.LinkText | |
| Depth = $item.Depth | |
| TimeoutSec = $TimeoutSec | |
| UserAgent = $UserAgent | |
| SameDomainOnly = $SameDomainOnly | |
| BaseDomain = $BaseDomain | |
| MaxDepth = $MaxDepth | |
| RateLimitDelayMs = $RateLimitDelayMs | |
| Queue = $Queue | |
| Visited = $Visited | |
| }) | |
| $Jobs.Add(@{ PS=$ps; Handle=$ps.BeginInvoke() }) | |
| } | |
| $item = $null | |
| } | |
| $elapsed = (Get-Date) - $StartTime | |
| $es = '{0:mm\:ss}' -f $elapsed | |
| $rate = if ($elapsed.TotalSeconds -gt 1) { [Math]::Round($Stats.Total / $elapsed.TotalSeconds, 1) } else { 0 } | |
| $ticker = (" {0} | OK:{1} BROKEN:{2} ERR:{3} RL:{4} | Queue:{5} Active:{6} | {7} req/s" -f ` | |
| $es, $Stats.OK, $Stats.Broken, $Stats.Errors, $Stats.RateLimited, | |
| $Queue.Count, $Jobs.Count, $rate) | |
| try { | |
| $cursor = $Host.UI.RawUI.CursorPosition | |
| $cursor.X = 0; [void]$cursor | |
| [void]($Host.UI.RawUI.CursorPosition = $cursor) | |
| $padded = $ticker.PadRight($Host.UI.RawUI.WindowSize.Width - 1) | |
| Write-Host $padded -ForegroundColor DarkGray -NoNewline | |
| } catch { | |
| Write-Host ("`r" + $ticker + " ") -ForegroundColor DarkGray -NoNewline | |
| } | |
| if ($Jobs.Count -eq 0 -and $Queue.Count -eq 0) { break } | |
| Start-Sleep -Milliseconds 200 | |
| } while ($true) | |
| $Pool.Close() | |
| $Pool.Dispose() | |
| $sep = '-' * (Get-ConsoleWidth) | |
| $elapsed = (Get-Date) - $StartTime | |
| $es = '{0:mm\:ss}' -f $elapsed | |
| $allResults = @($Results | Sort-Object IsBroken -Descending) | |
| $broken = @($allResults | Where-Object { $_.IsBroken }) | |
| Write-Host "`n`n $sep" -ForegroundColor DarkGray | |
| Write-Host " CRAWL COMPLETE" -ForegroundColor White | |
| Write-Host " $sep" -ForegroundColor DarkGray | |
| Write-Host (" Duration : {0}" -f $es) -ForegroundColor White | |
| Write-Host (" Total : {0}" -f $Stats.Total) -ForegroundColor White | |
| Write-Host (" OK / Redir : {0}" -f $Stats.OK) -ForegroundColor Green | |
| Write-Host (" Broken : {0}" -f $Stats.Broken) -ForegroundColor Red | |
| Write-Host (" Errors : {0}" -f $Stats.Errors) -ForegroundColor DarkYellow | |
| Write-Host (" Rate Ltd : {0}" -f $Stats.RateLimited) -ForegroundColor Magenta | |
| Write-Host "" | |
| function Write-Link { | |
| param( | |
| [string]$Label, | |
| [string]$Url, | |
| [string]$Prefix = "", | |
| [string]$Color = "Cyan" | |
| ) | |
| $esc = [char]27 | |
| $open = "${esc}]8;;${Url}${esc}\" | |
| $close = "${esc}]8;;${esc}\" | |
| Write-Host -NoNewline $Prefix | |
| Write-Host -NoNewline "${open}${Label}${close}" -ForegroundColor $Color | |
| Write-Host "" | |
| } | |
| if ($broken.Count -gt 0) { | |
| Write-Host " $sep" -ForegroundColor DarkGray | |
| Write-Host " BROKEN LINKS" -ForegroundColor Red | |
| Write-Host " $sep" -ForegroundColor DarkGray | |
| foreach ($r in $broken) { | |
| $short = if ($r.Url.Length -gt 68) { $r.Url.Substring(0,65)+'...' } else { $r.Url } | |
| Write-Link -Label (" [{0,3}] {1}" -f $r.StatusCode, $short) -Url $r.Url -Color Red | |
| Write-Host (" Link text : {0}" -f $r.LinkText) -ForegroundColor Yellow | |
| Write-Link -Label $r.FoundOnPage -Url $r.FoundOnPage ` | |
| -Prefix " Found on : " -Color Gray | |
| if ($r.RootPage -and $r.RootPage -ne '') { | |
| Write-Link -Label $r.RootPage -Url $r.RootPage ` | |
| -Prefix " >> Fix on : " -Color Cyan | |
| } | |
| # Show suggestion with engine tag | |
| if ($r.SuggestedUrl -and $r.SuggestedUrl -ne '') { | |
| Write-Host "" | |
| # SuggestedTitle may carry "via:DDG" or "via:Google" appended during storage — strip it | |
| $engTag = '' | |
| if ($r.SuggestedTitle -match '\[via:(\w+)\]$') { | |
| $engTag = $Matches[1] | |
| $dispTitle = $r.SuggestedTitle -replace '\s*\[via:\w+\]$','' | |
| } else { $dispTitle = $r.SuggestedTitle } | |
| $engSuffix = if ($engTag) { " [$engTag]" } else { '' } | |
| Write-Host (" Suggested{0} : " -f $engSuffix) -ForegroundColor DarkGray -NoNewline | |
| $esc = [char]27 | |
| $open = "${esc}]8;;$($r.SuggestedUrl)${esc}\" | |
| $close = "${esc}]8;;${esc}\" | |
| $sshort = if ($r.SuggestedUrl.Length -gt 62) { $r.SuggestedUrl.Substring(0,59)+'...' } else { $r.SuggestedUrl } | |
| Write-Host "${open}${sshort}${close}" -ForegroundColor Green | |
| if ($dispTitle -and $dispTitle -ne '') { | |
| Write-Host (" Title : {0}" -f $dispTitle) -ForegroundColor DarkGreen | |
| } | |
| } | |
| Write-Host "" | |
| } | |
| } | |
| Write-Host " $sep" -ForegroundColor DarkGray | |
| $csvUri = "file:///" + ($OutputCsv -replace "\\","/").TrimStart("/") | |
| $escF = [char]27 | |
| Write-Host -NoNewline " CSV saved : " | |
| Write-Host "${escF}]8;;${csvUri}${escF}\${OutputCsv}${escF}]8;;${escF}\" -ForegroundColor Cyan | |
| Write-Host " Config : $ConfigFile" -ForegroundColor DarkGray | |
| Write-Host " $sep`n" -ForegroundColor DarkGray | |
| return $allResults |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment