CodePress/cms/core/class/BotGuard.php

<?php

/**
 * BotGuard - Bot, AI Crawler, and Scraper detection & protection.
 *
 * Detecteert en blokkeert bots, AI-crawlers en scrapers op basis van de
 * User-Agent string, en genereert dynamische robots.txt-inhoud op basis
 * van de beveiligingsinstellingen.
 *
 * @since 2.6.5
 */
class BotGuard
{
    /**
     * Geeft de map met bot-handtekeningen per categorie en patroon.
     *
     * Retourneert een multi-dimensionale array met de categorieën 'ai',
     * 'search' en 'scraper', elk met een mapping van patroon naar weergavelabel.
     *
     * @since 2.6.5
     *
     * @return array<string,array<string,string>> Bot-handtekeningen per categorie.
     */
    public static function getBotSignatures(): array
    {
        return [
            'ai' => [
                'GPTBot' => 'AI (GPTBot)',
                'ChatGPT-User' => 'AI (ChatGPT)',
                'Claude-Web' => 'AI (Claude)',
                'ClaudeBot' => 'AI (ClaudeBot)',
                'anthropic-ai' => 'AI (Anthropic)',
                'Google-Extended' => 'AI (Gemini/Google)',
                'CCBot' => 'AI (CommonCrawl)',
                'PerplexityBot' => 'AI (Perplexity)',
                'Amazonbot' => 'AI (Amazon)',
                'cohere-ai' => 'AI (Cohere)',
                'OAI-SearchBot' => 'AI (OpenAI)',
                'Bytespider' => 'AI (ByteDance)',
                'FacebookBot' => 'AI (Meta/FB)',
                'Applebot-Extended' => 'AI (Apple)',
                'Meta-ExternalAgent' => 'AI (Meta)',
                'Diffbot' => 'AI (Diffbot)',
                'ImagesiftBot' => 'AI (Imagesift)',
                'Omgilibot' => 'AI (Omgili)',
                'Timpibot' => 'AI (Timpi)',
            ],
            'search' => [
                'Googlebot' => 'Zoekmachine (Google)',
                'Bingbot' => 'Zoekmachine (Bing)',
                'BingPreview' => 'Zoekmachine (Bing)',
                'Slurp' => 'Zoekmachine (Yahoo)',
                'DuckDuckBot' => 'Zoekmachine (DuckDuckGo)',
                'Baiduspider' => 'Zoekmachine (Baidu)',
                'YandexBot' => 'Zoekmachine (Yandex)',
                'Sogou' => 'Zoekmachine (Sogou)',
                'Exabot' => 'Zoekmachine (Exabot)',
                'facebot' => 'Zoekmachine (Facebook)',
            ],
            'scraper' => [
                'HTTrack' => 'Scraper (HTTrack)',
                'Scrapy' => 'Scraper (Scrapy)',
                'PhantomJS' => 'Scraper (PhantomJS)',
                'HeadlessChrome' => 'Scraper (Headless)',
                'curl' => 'Scraper (cURL)',
                'wget' => 'Scraper (Wget)',
                'python-requests' => 'Scraper (Python)',
                'python-urllib' => 'Scraper (Python)',
                'Go-http-client' => 'Scraper (Go)',
                'libwww-perl' => 'Scraper (Perl)',
                'Java/' => 'Scraper (Java)',
                'Postman' => 'Scraper (Postman)',
            ]
        ];
    }

    /**
     * Identificeer een User-Agent string.
     *
     * Vergelijkt de User-Agent met de bekende bot-handtekeningen (ai, search,
     * scraper), daarna met een generieke bot-regex. Bij geen match wordt de
     * bezoeker als mens beschouwd.
     *
     * @since 2.6.5
     *
     * @param string $ua User-Agent string.
     * @return array{category:string,pattern:string,label:string} Array met category, pattern en display label.
     */
    public static function identify(string $ua): array
    {
        if (trim($ua) === '') {
            return [
                'category' => 'empty',
                'pattern' => 'empty',
                'label' => 'Lege User-Agent'
            ];
        }

        $signatures = self::getBotSignatures();

        foreach ($signatures['ai'] as $pattern => $label) {
            if (stripos($ua, $pattern) !== false) {
                return ['category' => 'ai', 'pattern' => $pattern, 'label' => $label];
            }
        }

        foreach ($signatures['search'] as $pattern => $label) {
            if (stripos($ua, $pattern) !== false) {
                return ['category' => 'search', 'pattern' => $pattern, 'label' => $label];
            }
        }

        foreach ($signatures['scraper'] as $pattern => $label) {
            if (stripos($ua, $pattern) !== false) {
                return ['category' => 'scraper', 'pattern' => $pattern, 'label' => $label];
            }
        }

        if (preg_match('/(bot|crawler|spider|slurp)/i', $ua)) {
            return ['category' => 'generic', 'pattern' => 'generic_bot', 'label' => 'Bot'];
        }

        return ['category' => 'human', 'pattern' => 'human', 'label' => 'Mens'];
    }

    /**
     * Bepaal of een request geblokkeerd moet worden op basis van beveiligingsinstellingen.
     *
     * Controleert achtereenvolgens: lege User-Agent, custom blocklist, en
     * categorie-specifieke blokkades (ai, search, scraper, generic bot).
     *
     * @since 2.6.5
     *
     * @param string $ua               User-Agent string.
     * @param array  $securitySettings Beveiligingsconfiguratie-array.
     * @return string|null Reden-string indien geblokkeerd (bijv. 'blocked:ai'), null indien toegestaan.
     */
    public static function shouldBlock(string $ua, array $securitySettings): ?string
    {
        $trimmedUa = trim($ua);

        // 1. Empty User-Agent check
        if ($trimmedUa === '') {
            if (!empty($securitySettings['block_empty_user_agent'])) {
                return 'blocked:empty_ua';
            }
            return null;
        }

        // 2. Custom User-Agent blocklist
        $customBlocked = $securitySettings['custom_blocked_agents'] ?? [];
        if (is_array($customBlocked)) {
            foreach ($customBlocked as $pattern) {
                $pattern = trim($pattern);
                if ($pattern !== '' && stripos($trimmedUa, $pattern) !== false) {
                    return 'blocked:custom_agent';
                }
            }
        }

        // 3. Category signature check
        $identity = self::identify($trimmedUa);
        $category = $identity['category'];

        if ($category === 'ai' && !empty($securitySettings['block_ai_bots'])) {
            return 'blocked:ai';
        }

        if ($category === 'search' && !empty($securitySettings['block_search_engines'])) {
            return 'blocked:search';
        }

        if ($category === 'scraper' && !empty($securitySettings['block_scrapers'])) {
            return 'blocked:scraper';
        }

        if ($category === 'generic' && (!empty($securitySettings['block_scrapers']) || !empty($securitySettings['block_ai_bots']))) {
            return 'blocked:generic_bot';
        }

        return null;
    }

    /**
     * Genereer dynamische robots.txt-inhoud op basis van de beveiligingsinstellingen.
     *
     * Stelt globale regels op voor zoekmachines en voegt, indien AI-bots
     * geblokkeerd worden, per AI-crawler een Disallow-blok toe.
     *
     * @since 2.6.5
     *
     * @param array $securitySettings Beveiligingsconfiguratie-array.
     * @return string Robots.txt-inhoud.
     */
    public static function generateRobotsTxt(array $securitySettings): string
    {
        $out = "# robots.txt generated dynamically by CodePress CMS\n\n";

        // Global rule for search engines
        if (!empty($securitySettings['block_search_engines'])) {
            $out .= "User-agent: *\nDisallow: /\n\n";
        } else {
            $out .= "User-agent: *\nAllow: /\nDisallow: /admin\nDisallow: /cms\n\n";
        }

        // Block specific AI bots if enabled
        if (!empty($securitySettings['block_ai_bots'])) {
            $signatures = self::getBotSignatures();
            $out .= "# Block AI Crawlers & Scrapers\n";
            foreach (array_keys($signatures['ai']) as $aiBot) {
                $out .= "User-agent: {$aiBot}\nDisallow: /\n";
            }
            $out .= "\n";
        }

        return $out;
    }
}