CodePress/cms/core/class/BotGuard.php

<?php

/**
 * BotGuard - Bot, AI Crawler, and Scraper detection & protection
 */
class BotGuard
{
    /**
     * Map of bot signatures by category and pattern
     */
    public static function getBotSignatures(): array
    {
        return [
            'ai' => [
                'GPTBot' => 'AI (GPTBot)',
                'ChatGPT-User' => 'AI (ChatGPT)',
                'Claude-Web' => 'AI (Claude)',
                'ClaudeBot' => 'AI (ClaudeBot)',
                'anthropic-ai' => 'AI (Anthropic)',
                'Google-Extended' => 'AI (Gemini/Google)',
                'CCBot' => 'AI (CommonCrawl)',
                'PerplexityBot' => 'AI (Perplexity)',
                'Amazonbot' => 'AI (Amazon)',
                'cohere-ai' => 'AI (Cohere)',
                'OAI-SearchBot' => 'AI (OpenAI)',
                'Bytespider' => 'AI (ByteDance)',
                'FacebookBot' => 'AI (Meta/FB)',
                'Applebot-Extended' => 'AI (Apple)',
                'Meta-ExternalAgent' => 'AI (Meta)',
                'Diffbot' => 'AI (Diffbot)',
                'ImagesiftBot' => 'AI (Imagesift)',
                'Omgilibot' => 'AI (Omgili)',
                'Timpibot' => 'AI (Timpi)',
            ],
            'search' => [
                'Googlebot' => 'Zoekmachine (Google)',
                'Bingbot' => 'Zoekmachine (Bing)',
                'BingPreview' => 'Zoekmachine (Bing)',
                'Slurp' => 'Zoekmachine (Yahoo)',
                'DuckDuckBot' => 'Zoekmachine (DuckDuckGo)',
                'Baiduspider' => 'Zoekmachine (Baidu)',
                'YandexBot' => 'Zoekmachine (Yandex)',
                'Sogou' => 'Zoekmachine (Sogou)',
                'Exabot' => 'Zoekmachine (Exabot)',
                'facebot' => 'Zoekmachine (Facebook)',
            ],
            'scraper' => [
                'HTTrack' => 'Scraper (HTTrack)',
                'Scrapy' => 'Scraper (Scrapy)',
                'PhantomJS' => 'Scraper (PhantomJS)',
                'HeadlessChrome' => 'Scraper (Headless)',
                'curl' => 'Scraper (cURL)',
                'wget' => 'Scraper (Wget)',
                'python-requests' => 'Scraper (Python)',
                'python-urllib' => 'Scraper (Python)',
                'Go-http-client' => 'Scraper (Go)',
                'libwww-perl' => 'Scraper (Perl)',
                'Java/' => 'Scraper (Java)',
                'Postman' => 'Scraper (Postman)',
            ]
        ];
    }

    /**
     * Identify a User-Agent string
     *
     * @param string $ua User-Agent string
     * @return array Array with category, pattern, and display label
     */
    public static function identify(string $ua): array
    {
        if (trim($ua) === '') {
            return [
                'category' => 'empty',
                'pattern' => 'empty',
                'label' => 'Lege User-Agent'
            ];
        }

        $signatures = self::getBotSignatures();

        foreach ($signatures['ai'] as $pattern => $label) {
            if (stripos($ua, $pattern) !== false) {
                return ['category' => 'ai', 'pattern' => $pattern, 'label' => $label];
            }
        }

        foreach ($signatures['search'] as $pattern => $label) {
            if (stripos($ua, $pattern) !== false) {
                return ['category' => 'search', 'pattern' => $pattern, 'label' => $label];
            }
        }

        foreach ($signatures['scraper'] as $pattern => $label) {
            if (stripos($ua, $pattern) !== false) {
                return ['category' => 'scraper', 'pattern' => $pattern, 'label' => $label];
            }
        }

        if (preg_match('/(bot|crawler|spider|slurp)/i', $ua)) {
            return ['category' => 'generic', 'pattern' => 'generic_bot', 'label' => 'Bot'];
        }

        return ['category' => 'human', 'pattern' => 'human', 'label' => 'Mens'];
    }

    /**
     * Determine if a request should be blocked based on security settings
     *
     * @param string $ua User-Agent string
     * @param array $securitySettings Security configuration array
     * @return string|null Reason string if blocked, null if allowed
     */
    public static function shouldBlock(string $ua, array $securitySettings): ?string
    {
        $trimmedUa = trim($ua);

        // 1. Empty User-Agent check
        if ($trimmedUa === '') {
            if (!empty($securitySettings['block_empty_user_agent'])) {
                return 'blocked:empty_ua';
            }
            return null;
        }

        // 2. Custom User-Agent blocklist
        $customBlocked = $securitySettings['custom_blocked_agents'] ?? [];
        if (is_array($customBlocked)) {
            foreach ($customBlocked as $pattern) {
                $pattern = trim($pattern);
                if ($pattern !== '' && stripos($trimmedUa, $pattern) !== false) {
                    return 'blocked:custom_agent';
                }
            }
        }

        // 3. Category signature check
        $identity = self::identify($trimmedUa);
        $category = $identity['category'];

        if ($category === 'ai' && !empty($securitySettings['block_ai_bots'])) {
            return 'blocked:ai';
        }

        if ($category === 'search' && !empty($securitySettings['block_search_engines'])) {
            return 'blocked:search';
        }

        if ($category === 'scraper' && !empty($securitySettings['block_scrapers'])) {
            return 'blocked:scraper';
        }

        if ($category === 'generic' && (!empty($securitySettings['block_scrapers']) || !empty($securitySettings['block_ai_bots']))) {
            return 'blocked:generic_bot';
        }

        return null;
    }

    /**
     * Generate dynamic robots.txt content based on security settings
     *
     * @param array $securitySettings Security configuration array
     * @return string Robots.txt content
     */
    public static function generateRobotsTxt(array $securitySettings): string
    {
        $out = "# robots.txt generated dynamically by CodePress CMS\n\n";

        // Global rule for search engines
        if (!empty($securitySettings['block_search_engines'])) {
            $out .= "User-agent: *\nDisallow: /\n\n";
        } else {
            $out .= "User-agent: *\nAllow: /\nDisallow: /admin\nDisallow: /cms\n\n";
        }

        // Block specific AI bots if enabled
        if (!empty($securitySettings['block_ai_bots'])) {
            $signatures = self::getBotSignatures();
            $out .= "# Block AI Crawlers & Scrapers\n";
            foreach (array_keys($signatures['ai']) as $aiBot) {
                $out .= "User-agent: {$aiBot}\nDisallow: /\n";
            }
            $out .= "\n";
        }

        return $out;
    }
}