Last active
June 11, 2024 08:38
-
-
Save hiranthi/9ba92cfc31e501a2a9c57665ec935ee3 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <IfModule mod_rewrite.c> | |
| RewriteEngine On | |
| # Het betreft geen request voor `robots.txt`/`bots-forbidden.php`: | |
| RewriteCond %{REQUEST_URI} !(robots.txt|bots-forbidden.php) | |
| # User-Agents van bots die geen toegang (behalve `robots.txt`/`bots-forbidden.php`) mogen hebben: | |
| RewriteCond %{HTTP_USER_AGENT} Amazonbot [NC,OR] | |
| RewriteCond %{HTTP_USER_AGENT} AhrefsBot [NC,OR] | |
| RewriteCond %{HTTP_USER_AGENT} Bytespider [NC,OR] | |
| RewriteCond %{HTTP_USER_AGENT} Bytedance [NC,OR] | |
| RewriteCond %{HTTP_USER_AGENT} CCBot [NC,OR] | |
| RewriteCond %{HTTP_USER_AGENT} Claudebot [NC,OR] | |
| RewriteCond %{HTTP_USER_AGENT} DotBot [NC,OR] | |
| RewriteCond %{HTTP_USER_AGENT} FacebookBot [NC,OR] | |
| RewriteCond %{HTTP_USER_AGENT} Google-Extended [NC,OR] | |
| RewriteCond %{HTTP_USER_AGENT} GPTBot [NC,OR] | |
| RewriteCond %{HTTP_USER_AGENT} MJ12Bot [NC,OR] | |
| RewriteCond %{HTTP_USER_AGENT} SemrushBot [NC,OR] | |
| RewriteCond %{HTTP_USER_AGENT} YandexBot [NC] | |
| # Liever alles op 1 regel? | |
| # Verwijder dan de `HTTP_USER_AGENT` RewriteCond regels van hierboven | |
| # en gebruik in plaats daarvan onderstaande regel (zonder de # aan het begin) | |
| # RewriteCond %{HTTP_USER_AGENT} (Amazonbot|AhrefsBot|Bytespider|Bytedance|CCBot|Claudebot|DotBot|FacebookBot|Google-Extended|GPTBot|MJ12Bot|SemrushBot|YandexBot) [NC] | |
| # Betreffende bots krijgen `bots-forbidden.php` gepresenteerd: | |
| RewriteRule .* /bots-forbidden.php [L] | |
| </IfModule> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #################################################################################################### | |
| # # | |
| # Een verkorte variant voor in `.htaccess`, bij deze is er ook geen `bots-forbidden.php` nodig # | |
| # # | |
| #################################################################################################### | |
| <IfModule mod_rewrite.c> | |
| RewriteEngine On | |
| # Het betreft geen request voor `robots.txt`: | |
| RewriteCond %{REQUEST_URI} !robots.txt | |
| # Liever alles op 1 regel? | |
| # Verwijder dan de `HTTP_USER_AGENT` RewriteCond regels van hierboven | |
| # en gebruik in plaats daarvan onderstaande regel (zonder de # aan het begin) | |
| # RewriteCond %{HTTP_USER_AGENT} (Amazonbot|AhrefsBot|Bytespider|Bytedance|CCBot|Claudebot|DotBot|FacebookBot|Google-Extended|GPTBot|MJ12Bot|SemrushBot|YandexBot) [NC] | |
| # Betreffende bots krijgen een `408 Request Timeout`: | |
| RewriteRule .* - [R=408,L] | |
| </IfModule> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <?php | |
| /** | |
| * bots-forbidden.php | |
| * | |
| * Bestand om bots naartoe te sturen die geen toegang tot de website moeten hebben en waarvoor een 403 Forbidden extra 'aantrekkelijk' lijkt te zijn. | |
| * | |
| * INFO: | |
| * 1. Dit bestand moet in de hoofdmap van je website komen te staan (dezelfde map als het `.htaccess`-bestand waarin je de toevoeging vanuit de `.htaccess` van deze Gist doet) | |
| * 2. De extra witregels zijn bewust. | |
| **/ | |
| $protocol = $_SERVER['SERVER_PROTOCOL']; | |
| if ( 'HTTP/1.1' != $protocol && 'HTTP/1.0' != $protocol ) | |
| $protocol = 'HTTP/1.0'; | |
| header("{$protocol} 408 Request Timeout", true, 408); | |
| header('Content-Type: text/html; charset=utf-8'); | |
| header('Connection: close'); | |
| ?><!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN"> | |
| <HTML><HEAD> | |
| <TITLE>408 Request Timeout</TITLE> | |
| </HEAD><BODY> | |
| <H1>Timeout Request</H1> | |
| The server was unable to complete your request | |
| <HR> | |
| <I><!--#echo var="HTTP_HOST" --></I> | |
| </BODY></HTML> | |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| User-agent: Amazonbot | |
| User-agent: AhrefsBot | |
| User-agent: Bytespider | |
| User-agent: Bytedance | |
| User-agent: CCBot | |
| User-agent: Claudebot | |
| User-agent: DotBot | |
| User-agent: FacebookBot | |
| User-agent: Google-Extended | |
| User-agent: GPTBot | |
| User-agent: MJ12Bot | |
| User-agent: SemrushBot | |
| User-agent: YandexBot | |
| Disallow: / | |
| ### Onderstaande alleen nodig voor WordPress + Yoast SEO gebruikt wordt: | |
| # START YOAST BLOCK | |
| # --------------------------- | |
| User-agent: * | |
| Disallow: | |
| Sitemap: https://DOMEINNAAM_WEBSITE/sitemap_index.xml | |
| # --------------------------- | |
| # END YOAST BLOCK |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment