219 lines
7.8 KiB
PHP
219 lines
7.8 KiB
PHP
<?php
|
|
|
|
/**
|
|
* BotGuard - Bot, AI Crawler, and Scraper detection & protection.
|
|
*
|
|
* Detecteert en blokkeert bots, AI-crawlers en scrapers op basis van de
|
|
* User-Agent string, en genereert dynamische robots.txt-inhoud op basis
|
|
* van de beveiligingsinstellingen.
|
|
*
|
|
* @since 2.6.4
|
|
*/
|
|
class BotGuard
|
|
{
|
|
/**
|
|
* Geeft de map met bot-handtekeningen per categorie en patroon.
|
|
*
|
|
* Retourneert een multi-dimensionale array met de categorieën 'ai',
|
|
* 'search' en 'scraper', elk met een mapping van patroon naar weergavelabel.
|
|
*
|
|
* @since 2.6.4
|
|
*
|
|
* @return array<string,array<string,string>> Bot-handtekeningen per categorie.
|
|
*/
|
|
public static function getBotSignatures(): array
|
|
{
|
|
return [
|
|
'ai' => [
|
|
'GPTBot' => 'AI (GPTBot)',
|
|
'ChatGPT-User' => 'AI (ChatGPT)',
|
|
'Claude-Web' => 'AI (Claude)',
|
|
'ClaudeBot' => 'AI (ClaudeBot)',
|
|
'anthropic-ai' => 'AI (Anthropic)',
|
|
'Google-Extended' => 'AI (Gemini/Google)',
|
|
'CCBot' => 'AI (CommonCrawl)',
|
|
'PerplexityBot' => 'AI (Perplexity)',
|
|
'Amazonbot' => 'AI (Amazon)',
|
|
'cohere-ai' => 'AI (Cohere)',
|
|
'OAI-SearchBot' => 'AI (OpenAI)',
|
|
'Bytespider' => 'AI (ByteDance)',
|
|
'FacebookBot' => 'AI (Meta/FB)',
|
|
'Applebot-Extended' => 'AI (Apple)',
|
|
'Meta-ExternalAgent' => 'AI (Meta)',
|
|
'Diffbot' => 'AI (Diffbot)',
|
|
'ImagesiftBot' => 'AI (Imagesift)',
|
|
'Omgilibot' => 'AI (Omgili)',
|
|
'Timpibot' => 'AI (Timpi)',
|
|
],
|
|
'search' => [
|
|
'Googlebot' => 'Zoekmachine (Google)',
|
|
'Bingbot' => 'Zoekmachine (Bing)',
|
|
'BingPreview' => 'Zoekmachine (Bing)',
|
|
'Slurp' => 'Zoekmachine (Yahoo)',
|
|
'DuckDuckBot' => 'Zoekmachine (DuckDuckGo)',
|
|
'Baiduspider' => 'Zoekmachine (Baidu)',
|
|
'YandexBot' => 'Zoekmachine (Yandex)',
|
|
'Sogou' => 'Zoekmachine (Sogou)',
|
|
'Exabot' => 'Zoekmachine (Exabot)',
|
|
'facebot' => 'Zoekmachine (Facebook)',
|
|
],
|
|
'scraper' => [
|
|
'HTTrack' => 'Scraper (HTTrack)',
|
|
'Scrapy' => 'Scraper (Scrapy)',
|
|
'PhantomJS' => 'Scraper (PhantomJS)',
|
|
'HeadlessChrome' => 'Scraper (Headless)',
|
|
'curl' => 'Scraper (cURL)',
|
|
'wget' => 'Scraper (Wget)',
|
|
'python-requests' => 'Scraper (Python)',
|
|
'python-urllib' => 'Scraper (Python)',
|
|
'Go-http-client' => 'Scraper (Go)',
|
|
'libwww-perl' => 'Scraper (Perl)',
|
|
'Java/' => 'Scraper (Java)',
|
|
'Postman' => 'Scraper (Postman)',
|
|
]
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Identificeer een User-Agent string.
|
|
*
|
|
* Vergelijkt de User-Agent met de bekende bot-handtekeningen (ai, search,
|
|
* scraper), daarna met een generieke bot-regex. Bij geen match wordt de
|
|
* bezoeker als mens beschouwd.
|
|
*
|
|
* @since 2.6.4
|
|
*
|
|
* @param string $ua User-Agent string.
|
|
* @return array{category:string,pattern:string,label:string} Array met category, pattern en display label.
|
|
*/
|
|
public static function identify(string $ua): array
|
|
{
|
|
if (trim($ua) === '') {
|
|
return [
|
|
'category' => 'empty',
|
|
'pattern' => 'empty',
|
|
'label' => 'Lege User-Agent'
|
|
];
|
|
}
|
|
|
|
$signatures = self::getBotSignatures();
|
|
|
|
foreach ($signatures['ai'] as $pattern => $label) {
|
|
if (stripos($ua, $pattern) !== false) {
|
|
return ['category' => 'ai', 'pattern' => $pattern, 'label' => $label];
|
|
}
|
|
}
|
|
|
|
foreach ($signatures['search'] as $pattern => $label) {
|
|
if (stripos($ua, $pattern) !== false) {
|
|
return ['category' => 'search', 'pattern' => $pattern, 'label' => $label];
|
|
}
|
|
}
|
|
|
|
foreach ($signatures['scraper'] as $pattern => $label) {
|
|
if (stripos($ua, $pattern) !== false) {
|
|
return ['category' => 'scraper', 'pattern' => $pattern, 'label' => $label];
|
|
}
|
|
}
|
|
|
|
if (preg_match('/(bot|crawler|spider|slurp)/i', $ua)) {
|
|
return ['category' => 'generic', 'pattern' => 'generic_bot', 'label' => 'Bot'];
|
|
}
|
|
|
|
return ['category' => 'human', 'pattern' => 'human', 'label' => 'Mens'];
|
|
}
|
|
|
|
/**
|
|
* Bepaal of een request geblokkeerd moet worden op basis van beveiligingsinstellingen.
|
|
*
|
|
* Controleert achtereenvolgens: lege User-Agent, custom blocklist, en
|
|
* categorie-specifieke blokkades (ai, search, scraper, generic bot).
|
|
*
|
|
* @since 2.6.4
|
|
*
|
|
* @param string $ua User-Agent string.
|
|
* @param array $securitySettings Beveiligingsconfiguratie-array.
|
|
* @return string|null Reden-string indien geblokkeerd (bijv. 'blocked:ai'), null indien toegestaan.
|
|
*/
|
|
public static function shouldBlock(string $ua, array $securitySettings): ?string
|
|
{
|
|
$trimmedUa = trim($ua);
|
|
|
|
// 1. Empty User-Agent check
|
|
if ($trimmedUa === '') {
|
|
if (!empty($securitySettings['block_empty_user_agent'])) {
|
|
return 'blocked:empty_ua';
|
|
}
|
|
return null;
|
|
}
|
|
|
|
// 2. Custom User-Agent blocklist
|
|
$customBlocked = $securitySettings['custom_blocked_agents'] ?? [];
|
|
if (is_array($customBlocked)) {
|
|
foreach ($customBlocked as $pattern) {
|
|
$pattern = trim($pattern);
|
|
if ($pattern !== '' && stripos($trimmedUa, $pattern) !== false) {
|
|
return 'blocked:custom_agent';
|
|
}
|
|
}
|
|
}
|
|
|
|
// 3. Category signature check
|
|
$identity = self::identify($trimmedUa);
|
|
$category = $identity['category'];
|
|
|
|
if ($category === 'ai' && !empty($securitySettings['block_ai_bots'])) {
|
|
return 'blocked:ai';
|
|
}
|
|
|
|
if ($category === 'search' && !empty($securitySettings['block_search_engines'])) {
|
|
return 'blocked:search';
|
|
}
|
|
|
|
if ($category === 'scraper' && !empty($securitySettings['block_scrapers'])) {
|
|
return 'blocked:scraper';
|
|
}
|
|
|
|
if ($category === 'generic' && (!empty($securitySettings['block_scrapers']) || !empty($securitySettings['block_ai_bots']))) {
|
|
return 'blocked:generic_bot';
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Genereer dynamische robots.txt-inhoud op basis van de beveiligingsinstellingen.
|
|
*
|
|
* Stelt globale regels op voor zoekmachines en voegt, indien AI-bots
|
|
* geblokkeerd worden, per AI-crawler een Disallow-blok toe.
|
|
*
|
|
* @since 2.6.4
|
|
*
|
|
* @param array $securitySettings Beveiligingsconfiguratie-array.
|
|
* @return string Robots.txt-inhoud.
|
|
*/
|
|
public static function generateRobotsTxt(array $securitySettings): string
|
|
{
|
|
$out = "# robots.txt generated dynamically by CodePress CMS\n\n";
|
|
|
|
// Global rule for search engines
|
|
if (!empty($securitySettings['block_search_engines'])) {
|
|
$out .= "User-agent: *\nDisallow: /\n\n";
|
|
} else {
|
|
$out .= "User-agent: *\nAllow: /\nDisallow: /admin\nDisallow: /cms\n\n";
|
|
}
|
|
|
|
// Block specific AI bots if enabled
|
|
if (!empty($securitySettings['block_ai_bots'])) {
|
|
$signatures = self::getBotSignatures();
|
|
$out .= "# Block AI Crawlers & Scrapers\n";
|
|
foreach (array_keys($signatures['ai']) as $aiBot) {
|
|
$out .= "User-agent: {$aiBot}\nDisallow: /\n";
|
|
}
|
|
$out .= "\n";
|
|
}
|
|
|
|
return $out;
|
|
}
|
|
}
|