dale3h · June 29, 2018 15:47 · GwynethLlewelyn · Dec 15, 2023 · brendan-pike · Dec 20, 2023
diff --git a/jail.local b/jail.local
 # Add to file: /etc/fail2ban/jail.local
 [nginx-badbots]

 enabled  = true
 port     = http,https
 filter   = nginx-badbots
 logpath  = %(nginx_access_log)s
 maxretry = 1
 findtime = 604800
 bantime  = 604800
diff --git a/nginx-badbots.conf b/nginx-badbots.conf
 # Create as file: /etc/fail2ban/filter.d/nginx-badbots.conf 
 [Definition]

 badbots = 360Spider|404checker|404enemy|80legs|Abonti|Aboundex|Acunetix|ADmantX|AfD-Verbotsverfahren|AhrefsBot|AIBOT|AiHitBot|Aipbot|Alexibot|Alligator|AllSubmitter|AlphaBot|Anarchie|Apexoo|ASPSeek|Asterias|Attach|autoemailspider|BackDoorBot|Backlink-Ceck|backlink-check|BacklinkCrawler|BackStreet|BackWeb|Badass|Bandit|Barkrowler|BatchFTP|Battleztar Bazinga|BBBike|BDCbot|BDFetch|BetaBot|Bigfoot|Bitacle|Blackboard|Black Hole|BlackWidow|BLEXBot|Blow|BlowFish|Boardreader|Bolt|BotALot|Brandprotect|Brandwatch|Bubing|Buddy|BuiltBotTough|BuiltWith|Bullseye|BunnySlippers|BuzzSumo|Calculon|CATExplorador|CazoodleBot|CCBot|Cegbfeieh|CheeseBot|CherryPicker|ChinaClaw|Chlooe|Claritybot|Cliqzbot|Cloud mapping|coccocbot-web|Cogentbot|cognitiveseo|Collector|com\.plumanalytics|Copier|CopyRightCheck|Copyscape|Cosmos|Craftbot|crawler4j|crawler\.feedback|CrazyWebCrawler|Crescent|CSHttp|Curious|Custo|DatabaseDriverMysqli|DataCha0s|DBLBot|demandbase-bot|Demon|Deusu|Devil|Digincore|DigitalPebble|DIIbot|Dirbuster|Disco|Discobot|Discoverybot|DittoSpyder|DnyzBot|DomainAppender|DomainCrawler|DomainSigmaCrawler|DomainStatsBot|Dotbot|Download Wonder|Dragonfly|Drip|DTS Agent|EasyDL|Ebingbong|eCatch|ECCP/1\.0|Ecxi|EirGrabber|EMail Siphon|EMail Wolf|EroCrawler|evc-batch|Evil|Exabot|Express WebPictures|ExtLinksBot|Extractor|ExtractorPro|Extreme Picture Finder|EyeNetIE|Ezooms|FDM|FemtosearchBot|FHscan|Fimap|Firefox/7\.0|FlashGet|Flunky|Foobot|Freeuploader|FrontPage|Fyrebot|GalaxyBot|Genieo|GermCrawler|Getintent|GetRight|GetWeb|Gigablast|Gigabot|G-i-g-a-b-o-t|Go-Ahead-Got-It|Gotit|GoZilla|Go!Zilla|Grabber|GrabNet|Grafula|GrapeFX|GrapeshotCrawler|GridBot|GT\:\:WWW|Haansoft|HaosouSpider|Harvest|Havij|HEADMasterSEO|Heritrix|Hloader|HMView|HTMLparser|HTTP\:\:Lite|HTTrack|Humanlinks|HybridBot|Iblog|IDBot|Id-search|IlseBot|Image Fetch|Image Sucker|IndeedBot|Indy Library|InfoNaviRobot|InfoTekies|instabid|Intelliseek|InterGET|Internet Ninja|InternetSeer|internetVista monitor|ips-agent|Iria|IRLbot|Iskanie|IstellaBot|JamesBOT|Jbrofuzz|JennyBot|JetCar|JikeSpider|JOC Web Spider|Joomla|Jorgee|JustView|Jyxobot|Kenjin Spider|Keyword Density|Kozmosbot|Lanshanbot|Larbin|LeechFTP|LeechGet|LexiBot|Lftp|LibWeb|Libwhisker|Lightspeedsystems|Likse|Linkdexbot|LinkextractorPro|LinkpadBot|LinkScan|LinksManager|LinkWalker|LinqiaMetadataDownloaderBot|LinqiaRSSBot|LinqiaScrapeBot|Lipperhey|Litemage_walker|Lmspider|LNSpiderguy|Ltx71|lwp-request|LWP\:\:Simple|lwp-trivial|Magnet|Mag-Net|magpie-crawler|Mail\.RU_Bot|Majestic12|MarkMonitor|MarkWatch|Masscan|Mass Downloader|Mata Hari|MauiBot|Meanpathbot|mediawords|MegaIndex\.ru|Metauri|MFC_Tear_Sample|Microsoft Data Access|Microsoft URL Control|MIDown tool|MIIxpc|Mister PiX|MJ12bot|Mojeek|Morfeus Fucking Scanner|Mr\.4x3|MSFrontPage|MSIECrawler|Msrabot|MS Web Services Client Protocol|muhstik-scan|Musobot|Name Intelligence|Nameprotect|Navroad|NearSite|Needle|Nessus|NetAnts|Netcraft|netEstate NE Crawler|NetLyzer|NetMechanic|NetSpider|Nettrack|Net Vampire|Netvibes|NetZIP|NextGenSearchBot|Nibbler|NICErsPRO|Niki-bot|Nikto|NimbleCrawler|Ninja|Nmap|NPbot|Nutch|oBot|Octopus|Offline Explorer|Offline Navigator|Openfind|OpenLinkProfiler|Openvas|OrangeBot|OrangeSpider|OutclicksBot|OutfoxBot|PageAnalyzer|Page Analyzer|PageGrabber|page scorer|PageScorer|Panscient|Papa Foto|Pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|PHPCrawl|Picscout|Picsearch|PictureFinder|Pimonster|Pi-Monster|Pixray|PleaseCrawl|plumanalytics|Pockey|POE-Component-Client-HTTP|Probethenet|ProPowerBot|ProWebWalker|Psbot|Pump|PxBroker|PyCurl|QueryN Metasearch|Quick-Crawler|RankActive|RankActiveLinkBot|RankFlex|RankingBot|RankingBot2|Rankivabot|RankurBot|RealDownload|Reaper|RebelMouse|Recorder|RedesScrapy|ReGet|RepoMonkey|Ripper|RocketCrawler|Rogerbot|SalesIntelligent|SBIder|ScanAlert|Scanbot|scan\.lol|Scrapy|Screaming|ScreenerBot|Searchestate|SearchmetricsBot|Semrush|SemrushBot|SEOkicks|SEOlyticsCrawler|Seomoz|SEOprofiler|seoscanners|SEOstats|sexsearcher|Seznam|SeznamBot|Shodan|Siphon|SISTRIX|Sitebeam|SiteExplorer|Siteimprove|SiteLockSpider|SiteSnagger|SiteSucker|Site Sucker|Sitevigil|Slackbot-LinkExpanding|SlySearch|SmartDownload|SMTBot|Snake|Snapbot|Snoopy|SocialRankIOBot|Sogou web spider|Sosospider|Sottopop|SpaceBison|Spammen|SpankBot|Spanner|Spbot|Spinn3r|SputnikBot|Sqlmap|Sqlworm|Sqworm|Steeler|Stripper|Sucker|Sucuri|SuperBot|SuperHTTP|Surfbot|SurveyBot|Suzuran|Swiftbot|sysscan|Szukacz|T0PHackTeam|T8Abot|tAkeOut|Teleport|TeleportPro|Telesoft|Telesphoreo|Telesphorep|The Intraformant|TheNomad|TightTwatBot|Titan|Toata|Toweyabot|Trendiction|Trendictionbot|trendiction\.com|trendiction\.de|True_Robot|Turingos|Turnitin|TurnitinBot|TwengaBot|Twice|Typhoeus|UnisterBot|URLy\.Warning|URLy Warning|Vacuum|Vagabondo|VB Project|VCI|VeriCiteCrawler|VidibleScraper|Virusdie|VoidEYE|Voil|Voltron|Wallpapers/3\.0|WallpapersHD|WASALive-Bot|WBSearchBot|Webalta|WebAuto|Web Auto|WebBandit|WebCollage|Web Collage|WebCopier|WEBDAV|WebEnhancer|Web Enhancer|WebFetch|Web Fetch|WebFuck|Web Fuck|WebGo IS|WebImageCollector|WebLeacher|WebmasterWorldForumBot|webmeup-crawler|WebPix|Web Pix|WebReaper|WebSauger|Web Sauger|Webshag|WebsiteExtractor|WebsiteQuester|Website Quester|Webster|WebStripper|WebSucker|Web Sucker|WebWhacker|WebZIP|WeSEE|Whack|Whacker|Whatweb|Who\.is Bot|Widow|WinHTTrack|WiseGuys Robot|WISENutbot|Wonderbot|Woobot|Wotbox|Wprecon|WPScan|WWW-Collector-E|WWW-Mechanize|WWW\:\:Mechanize|WWWOFFLE|x09Mozilla|x22Mozilla|Xaldon_WebSpider|Xaldon WebSpider|Xenu|xpymep1\.exe|YoudaoBot|Zade|Zauba|zauba\.io|Zermelo|Zeus|zgrab|Zitebot|ZmEu|ZumBot|ZyBorg

 failregex = (?i)<HOST> -.*"(GET|POST|HEAD).*HTTP.*(?:%(badbots)s).*"$

 ignoreregex =
	# Add to file: /etc/fail2ban/jail.local
	[nginx-badbots]

	enabled = true
	port = http,https
	filter = nginx-badbots
	logpath = %(nginx_access_log)s
	maxretry = 1
	findtime = 604800
	bantime = 604800
	# Create as file: /etc/fail2ban/filter.d/nginx-badbots.conf
	[Definition]

	badbots = 360Spider\|404checker\|404enemy\|80legs\|Abonti\|Aboundex\|Acunetix\|ADmantX\|AfD-Verbotsverfahren\|AhrefsBot\|AIBOT\|AiHitBot\|Aipbot\|Alexibot\|Alligator\|AllSubmitter\|AlphaBot\|Anarchie\|Apexoo\|ASPSeek\|Asterias\|Attach\|autoemailspider\|BackDoorBot\|Backlink-Ceck\|backlink-check\|BacklinkCrawler\|BackStreet\|BackWeb\|Badass\|Bandit\|Barkrowler\|BatchFTP\|Battleztar Bazinga\|BBBike\|BDCbot\|BDFetch\|BetaBot\|Bigfoot\|Bitacle\|Blackboard\|Black Hole\|BlackWidow\|BLEXBot\|Blow\|BlowFish\|Boardreader\|Bolt\|BotALot\|Brandprotect\|Brandwatch\|Bubing\|Buddy\|BuiltBotTough\|BuiltWith\|Bullseye\|BunnySlippers\|BuzzSumo\|Calculon\|CATExplorador\|CazoodleBot\|CCBot\|Cegbfeieh\|CheeseBot\|CherryPicker\|ChinaClaw\|Chlooe\|Claritybot\|Cliqzbot\|Cloud mapping\|coccocbot-web\|Cogentbot\|cognitiveseo\|Collector\|com\.plumanalytics\|Copier\|CopyRightCheck\|Copyscape\|Cosmos\|Craftbot\|crawler4j\|crawler\.feedback\|CrazyWebCrawler\|Crescent\|CSHttp\|Curious\|Custo\|DatabaseDriverMysqli\|DataCha0s\|DBLBot\|demandbase-bot\|Demon\|Deusu\|Devil\|Digincore\|DigitalPebble\|DIIbot\|Dirbuster\|Disco\|Discobot\|Discoverybot\|DittoSpyder\|DnyzBot\|DomainAppender\|DomainCrawler\|DomainSigmaCrawler\|DomainStatsBot\|Dotbot\|Download Wonder\|Dragonfly\|Drip\|DTS Agent\|EasyDL\|Ebingbong\|eCatch\|ECCP/1\.0\|Ecxi\|EirGrabber\|EMail Siphon\|EMail Wolf\|EroCrawler\|evc-batch\|Evil\|Exabot\|Express WebPictures\|ExtLinksBot\|Extractor\|ExtractorPro\|Extreme Picture Finder\|EyeNetIE\|Ezooms\|FDM\|FemtosearchBot\|FHscan\|Fimap\|Firefox/7\.0\|FlashGet\|Flunky\|Foobot\|Freeuploader\|FrontPage\|Fyrebot\|GalaxyBot\|Genieo\|GermCrawler\|Getintent\|GetRight\|GetWeb\|Gigablast\|Gigabot\|G-i-g-a-b-o-t\|Go-Ahead-Got-It\|Gotit\|GoZilla\|Go!Zilla\|Grabber\|GrabNet\|Grafula\|GrapeFX\|GrapeshotCrawler\|GridBot\|GT\:\:WWW\|Haansoft\|HaosouSpider\|Harvest\|Havij\|HEADMasterSEO\|Heritrix\|Hloader\|HMView\|HTMLparser\|HTTP\:\:Lite\|HTTrack\|Humanlinks\|HybridBot\|Iblog\|IDBot\|Id-search\|IlseBot\|Image Fetch\|Image Sucker\|IndeedBot\|Indy Library\|InfoNaviRobot\|InfoTekies\|instabid\|Intelliseek\|InterGET\|Internet Ninja\|InternetSeer\|internetVista monitor\|ips-agent\|Iria\|IRLbot\|Iskanie\|IstellaBot\|JamesBOT\|Jbrofuzz\|JennyBot\|JetCar\|JikeSpider\|JOC Web Spider\|Joomla\|Jorgee\|JustView\|Jyxobot\|Kenjin Spider\|Keyword Density\|Kozmosbot\|Lanshanbot\|Larbin\|LeechFTP\|LeechGet\|LexiBot\|Lftp\|LibWeb\|Libwhisker\|Lightspeedsystems\|Likse\|Linkdexbot\|LinkextractorPro\|LinkpadBot\|LinkScan\|LinksManager\|LinkWalker\|LinqiaMetadataDownloaderBot\|LinqiaRSSBot\|LinqiaScrapeBot\|Lipperhey\|Litemage_walker\|Lmspider\|LNSpiderguy\|Ltx71\|lwp-request\|LWP\:\:Simple\|lwp-trivial\|Magnet\|Mag-Net\|magpie-crawler\|Mail\.RU_Bot\|Majestic12\|MarkMonitor\|MarkWatch\|Masscan\|Mass Downloader\|Mata Hari\|MauiBot\|Meanpathbot\|mediawords\|MegaIndex\.ru\|Metauri\|MFC_Tear_Sample\|Microsoft Data Access\|Microsoft URL Control\|MIDown tool\|MIIxpc\|Mister PiX\|MJ12bot\|Mojeek\|Morfeus Fucking Scanner\|Mr\.4x3\|MSFrontPage\|MSIECrawler\|Msrabot\|MS Web Services Client Protocol\|muhstik-scan\|Musobot\|Name Intelligence\|Nameprotect\|Navroad\|NearSite\|Needle\|Nessus\|NetAnts\|Netcraft\|netEstate NE Crawler\|NetLyzer\|NetMechanic\|NetSpider\|Nettrack\|Net Vampire\|Netvibes\|NetZIP\|NextGenSearchBot\|Nibbler\|NICErsPRO\|Niki-bot\|Nikto\|NimbleCrawler\|Ninja\|Nmap\|NPbot\|Nutch\|oBot\|Octopus\|Offline Explorer\|Offline Navigator\|Openfind\|OpenLinkProfiler\|Openvas\|OrangeBot\|OrangeSpider\|OutclicksBot\|OutfoxBot\|PageAnalyzer\|Page Analyzer\|PageGrabber\|page scorer\|PageScorer\|Panscient\|Papa Foto\|Pavuk\|pcBrowser\|PECL\:\:HTTP\|PeoplePal\|PHPCrawl\|Picscout\|Picsearch\|PictureFinder\|Pimonster\|Pi-Monster\|Pixray\|PleaseCrawl\|plumanalytics\|Pockey\|POE-Component-Client-HTTP\|Probethenet\|ProPowerBot\|ProWebWalker\|Psbot\|Pump\|PxBroker\|PyCurl\|QueryN Metasearch\|Quick-Crawler\|RankActive\|RankActiveLinkBot\|RankFlex\|RankingBot\|RankingBot2\|Rankivabot\|RankurBot\|RealDownload\|Reaper\|RebelMouse\|Recorder\|RedesScrapy\|ReGet\|RepoMonkey\|Ripper\|RocketCrawler\|Rogerbot\|SalesIntelligent\|SBIder\|ScanAlert\|Scanbot\|scan\.lol\|Scrapy\|Screaming\|ScreenerBot\|Searchestate\|SearchmetricsBot\|Semrush\|SemrushBot\|SEOkicks\|SEOlyticsCrawler\|Seomoz\|SEOprofiler\|seoscanners\|SEOstats\|sexsearcher\|Seznam\|SeznamBot\|Shodan\|Siphon\|SISTRIX\|Sitebeam\|SiteExplorer\|Siteimprove\|SiteLockSpider\|SiteSnagger\|SiteSucker\|Site Sucker\|Sitevigil\|Slackbot-LinkExpanding\|SlySearch\|SmartDownload\|SMTBot\|Snake\|Snapbot\|Snoopy\|SocialRankIOBot\|Sogou web spider\|Sosospider\|Sottopop\|SpaceBison\|Spammen\|SpankBot\|Spanner\|Spbot\|Spinn3r\|SputnikBot\|Sqlmap\|Sqlworm\|Sqworm\|Steeler\|Stripper\|Sucker\|Sucuri\|SuperBot\|SuperHTTP\|Surfbot\|SurveyBot\|Suzuran\|Swiftbot\|sysscan\|Szukacz\|T0PHackTeam\|T8Abot\|tAkeOut\|Teleport\|TeleportPro\|Telesoft\|Telesphoreo\|Telesphorep\|The Intraformant\|TheNomad\|TightTwatBot\|Titan\|Toata\|Toweyabot\|Trendiction\|Trendictionbot\|trendiction\.com\|trendiction\.de\|True_Robot\|Turingos\|Turnitin\|TurnitinBot\|TwengaBot\|Twice\|Typhoeus\|UnisterBot\|URLy\.Warning\|URLy Warning\|Vacuum\|Vagabondo\|VB Project\|VCI\|VeriCiteCrawler\|VidibleScraper\|Virusdie\|VoidEYE\|Voil\|Voltron\|Wallpapers/3\.0\|WallpapersHD\|WASALive-Bot\|WBSearchBot\|Webalta\|WebAuto\|Web Auto\|WebBandit\|WebCollage\|Web Collage\|WebCopier\|WEBDAV\|WebEnhancer\|Web Enhancer\|WebFetch\|Web Fetch\|WebFuck\|Web Fuck\|WebGo IS\|WebImageCollector\|WebLeacher\|WebmasterWorldForumBot\|webmeup-crawler\|WebPix\|Web Pix\|WebReaper\|WebSauger\|Web Sauger\|Webshag\|WebsiteExtractor\|WebsiteQuester\|Website Quester\|Webster\|WebStripper\|WebSucker\|Web Sucker\|WebWhacker\|WebZIP\|WeSEE\|Whack\|Whacker\|Whatweb\|Who\.is Bot\|Widow\|WinHTTrack\|WiseGuys Robot\|WISENutbot\|Wonderbot\|Woobot\|Wotbox\|Wprecon\|WPScan\|WWW-Collector-E\|WWW-Mechanize\|WWW\:\:Mechanize\|WWWOFFLE\|x09Mozilla\|x22Mozilla\|Xaldon_WebSpider\|Xaldon WebSpider\|Xenu\|xpymep1\.exe\|YoudaoBot\|Zade\|Zauba\|zauba\.io\|Zermelo\|Zeus\|zgrab\|Zitebot\|ZmEu\|ZumBot\|ZyBorg

	failregex = (?i)<HOST> -."(GET\|POST\|HEAD).HTTP.(?:%(badbots)s)."$

	ignoreregex =