[ 'GPTBot' => 'AI (GPTBot)', 'ChatGPT-User' => 'AI (ChatGPT)', 'Claude-Web' => 'AI (Claude)', 'ClaudeBot' => 'AI (ClaudeBot)', 'anthropic-ai' => 'AI (Anthropic)', 'Google-Extended' => 'AI (Gemini/Google)', 'CCBot' => 'AI (CommonCrawl)', 'PerplexityBot' => 'AI (Perplexity)', 'Amazonbot' => 'AI (Amazon)', 'cohere-ai' => 'AI (Cohere)', 'OAI-SearchBot' => 'AI (OpenAI)', 'Bytespider' => 'AI (ByteDance)', 'FacebookBot' => 'AI (Meta/FB)', 'Applebot-Extended' => 'AI (Apple)', 'Meta-ExternalAgent' => 'AI (Meta)', 'Diffbot' => 'AI (Diffbot)', 'ImagesiftBot' => 'AI (Imagesift)', 'Omgilibot' => 'AI (Omgili)', 'Timpibot' => 'AI (Timpi)', ], 'search' => [ 'Googlebot' => 'Zoekmachine (Google)', 'Bingbot' => 'Zoekmachine (Bing)', 'BingPreview' => 'Zoekmachine (Bing)', 'Slurp' => 'Zoekmachine (Yahoo)', 'DuckDuckBot' => 'Zoekmachine (DuckDuckGo)', 'Baiduspider' => 'Zoekmachine (Baidu)', 'YandexBot' => 'Zoekmachine (Yandex)', 'Sogou' => 'Zoekmachine (Sogou)', 'Exabot' => 'Zoekmachine (Exabot)', 'facebot' => 'Zoekmachine (Facebook)', ], 'scraper' => [ 'HTTrack' => 'Scraper (HTTrack)', 'Scrapy' => 'Scraper (Scrapy)', 'PhantomJS' => 'Scraper (PhantomJS)', 'HeadlessChrome' => 'Scraper (Headless)', 'curl' => 'Scraper (cURL)', 'wget' => 'Scraper (Wget)', 'python-requests' => 'Scraper (Python)', 'python-urllib' => 'Scraper (Python)', 'Go-http-client' => 'Scraper (Go)', 'libwww-perl' => 'Scraper (Perl)', 'Java/' => 'Scraper (Java)', 'Postman' => 'Scraper (Postman)', ] ]; } /** * Identify a User-Agent string * * @param string $ua User-Agent string * @return array Array with category, pattern, and display label */ public static function identify(string $ua): array { if (trim($ua) === '') { return [ 'category' => 'empty', 'pattern' => 'empty', 'label' => 'Lege User-Agent' ]; } $signatures = self::getBotSignatures(); foreach ($signatures['ai'] as $pattern => $label) { if (stripos($ua, $pattern) !== false) { return ['category' => 'ai', 'pattern' => $pattern, 'label' => $label]; } } foreach ($signatures['search'] as $pattern => $label) { if (stripos($ua, $pattern) !== false) { return ['category' => 'search', 'pattern' => $pattern, 'label' => $label]; } } foreach ($signatures['scraper'] as $pattern => $label) { if (stripos($ua, $pattern) !== false) { return ['category' => 'scraper', 'pattern' => $pattern, 'label' => $label]; } } if (preg_match('/(bot|crawler|spider|slurp)/i', $ua)) { return ['category' => 'generic', 'pattern' => 'generic_bot', 'label' => 'Bot']; } return ['category' => 'human', 'pattern' => 'human', 'label' => 'Mens']; } /** * Determine if a request should be blocked based on security settings * * @param string $ua User-Agent string * @param array $securitySettings Security configuration array * @return string|null Reason string if blocked, null if allowed */ public static function shouldBlock(string $ua, array $securitySettings): ?string { $trimmedUa = trim($ua); // 1. Empty User-Agent check if ($trimmedUa === '') { if (!empty($securitySettings['block_empty_user_agent'])) { return 'blocked:empty_ua'; } return null; } // 2. Custom User-Agent blocklist $customBlocked = $securitySettings['custom_blocked_agents'] ?? []; if (is_array($customBlocked)) { foreach ($customBlocked as $pattern) { $pattern = trim($pattern); if ($pattern !== '' && stripos($trimmedUa, $pattern) !== false) { return 'blocked:custom_agent'; } } } // 3. Category signature check $identity = self::identify($trimmedUa); $category = $identity['category']; if ($category === 'ai' && !empty($securitySettings['block_ai_bots'])) { return 'blocked:ai'; } if ($category === 'search' && !empty($securitySettings['block_search_engines'])) { return 'blocked:search'; } if ($category === 'scraper' && !empty($securitySettings['block_scrapers'])) { return 'blocked:scraper'; } if ($category === 'generic' && (!empty($securitySettings['block_scrapers']) || !empty($securitySettings['block_ai_bots']))) { return 'blocked:generic_bot'; } return null; } /** * Generate dynamic robots.txt content based on security settings * * @param array $securitySettings Security configuration array * @return string Robots.txt content */ public static function generateRobotsTxt(array $securitySettings): string { $out = "# robots.txt generated dynamically by CodePress CMS\n\n"; // Global rule for search engines if (!empty($securitySettings['block_search_engines'])) { $out .= "User-agent: *\nDisallow: /\n\n"; } else { $out .= "User-agent: *\nAllow: /\nDisallow: /admin\nDisallow: /cms\n\n"; } // Block specific AI bots if enabled if (!empty($securitySettings['block_ai_bots'])) { $signatures = self::getBotSignatures(); $out .= "# Block AI Crawlers & Scrapers\n"; foreach (array_keys($signatures['ai']) as $aiBot) { $out .= "User-agent: {$aiBot}\nDisallow: /\n"; } $out .= "\n"; } return $out; } }