CodePress CMS v1.8.0: BotGuard security engine, HAProxy docs & ARIA fix

- Implement BotGuard security engine (Bot, AI, Scraper & Empty UA blocking) - Add Admin Security page (/admin/security) with toggles, rate limiter & block/allowlists - Add per-IP RateLimiter handoff in index.php with HTTP 429 response - Add dynamic /robots.txt generation and noai/noimageai meta tags - Add RequestLogger status column and blocked badges in admin request logs - Fix ARIAComponents.php syntax errors on lines 67, 137, 262 - Add HAProxy / PFSense bot blocking & IP forwarding guide (docs/haproxy-bot-blocking.md) - Update version to 1.8.0 with release notes in version.php and guides
2026-07-29 14:26:48 +02:00
parent 5884877f18
commit 375a39c458
18 changed files with 618 additions and 117 deletions
@@ -64,7 +64,7 @@ class ARIAComponents {
        $label = $options['aria-label'] ?? 'Hoofdmenu';
        $orientation = $options['orientation'] ?? 'horizontal';
        
-        $html = '<nav id="' . $id . '" role="navigation" aria-label="' . htmlspecialchars($label, ENT_QUOTES, 'UTF-8) . '">';
+        $html = '<nav id="' . $id . '" role="navigation" aria-label="' . htmlspecialchars($label, ENT_QUOTES, 'UTF-8') . '">';
        $html .= '<ul role="menubar" aria-orientation="' . $orientation . '">';
        
        foreach ($menu as $index => $item) {
@@ -134,7 +134,7 @@ class ARIAComponents {
        $label = $options['aria-label'] ?? 'Formulier';
        
        $html = '<form id="' . $id . '" method="' . $method . '" action="' . htmlspecialchars($action, ENT_QUOTES, 'UTF-8') . '" ';
-        $html .= 'role="form" aria-label="' . htmlspecialchars($label, ENT_QUOTES, 'UTF-8) . '" ';
+        $html .= 'role="form" aria-label="' . htmlspecialchars($label, ENT_QUOTES, 'UTF-8') . '" ';
        $html .= 'novalidate>';
        
        foreach ($fields as $index => $field) {
@@ -259,7 +259,7 @@ class ARIAComponents {
    public static function createAccessibleBreadcrumb($breadcrumbs, $options = []) {
        $label = $options['aria-label'] ?? 'Broodkruimelnavigatie';
        
-        $html = '<nav aria-label="' . htmlspecialchars($label, ENT_QUOTES, 'UTF-8) . '">';
+        $html = '<nav aria-label="' . htmlspecialchars($label, ENT_QUOTES, 'UTF-8') . '">';
        $html .= '<ol class="breadcrumb">';
        
        foreach ($breadcrumbs as $index => $crumb) {
@@ -0,0 +1,189 @@
+<?php
+
+/**
+ * BotGuard - Bot, AI Crawler, and Scraper detection & protection
+ */
+class BotGuard
+{
+    /**
+     * Map of bot signatures by category and pattern
+     */
+    public static function getBotSignatures(): array
+    {
+        return [
+            'ai' => [
+                'GPTBot' => 'AI (GPTBot)',
+                'ChatGPT-User' => 'AI (ChatGPT)',
+                'Claude-Web' => 'AI (Claude)',
+                'ClaudeBot' => 'AI (ClaudeBot)',
+                'anthropic-ai' => 'AI (Anthropic)',
+                'Google-Extended' => 'AI (Gemini/Google)',
+                'CCBot' => 'AI (CommonCrawl)',
+                'PerplexityBot' => 'AI (Perplexity)',
+                'Amazonbot' => 'AI (Amazon)',
+                'cohere-ai' => 'AI (Cohere)',
+                'OAI-SearchBot' => 'AI (OpenAI)',
+                'Bytespider' => 'AI (ByteDance)',
+                'FacebookBot' => 'AI (Meta/FB)',
+                'Applebot-Extended' => 'AI (Apple)',
+                'Meta-ExternalAgent' => 'AI (Meta)',
+                'Diffbot' => 'AI (Diffbot)',
+                'ImagesiftBot' => 'AI (Imagesift)',
+                'Omgilibot' => 'AI (Omgili)',
+                'Timpibot' => 'AI (Timpi)',
+            ],
+            'search' => [
+                'Googlebot' => 'Zoekmachine (Google)',
+                'Bingbot' => 'Zoekmachine (Bing)',
+                'BingPreview' => 'Zoekmachine (Bing)',
+                'Slurp' => 'Zoekmachine (Yahoo)',
+                'DuckDuckBot' => 'Zoekmachine (DuckDuckGo)',
+                'Baiduspider' => 'Zoekmachine (Baidu)',
+                'YandexBot' => 'Zoekmachine (Yandex)',
+                'Sogou' => 'Zoekmachine (Sogou)',
+                'Exabot' => 'Zoekmachine (Exabot)',
+                'facebot' => 'Zoekmachine (Facebook)',
+            ],
+            'scraper' => [
+                'HTTrack' => 'Scraper (HTTrack)',
+                'Scrapy' => 'Scraper (Scrapy)',
+                'PhantomJS' => 'Scraper (PhantomJS)',
+                'HeadlessChrome' => 'Scraper (Headless)',
+                'curl' => 'Scraper (cURL)',
+                'wget' => 'Scraper (Wget)',
+                'python-requests' => 'Scraper (Python)',
+                'python-urllib' => 'Scraper (Python)',
+                'Go-http-client' => 'Scraper (Go)',
+                'libwww-perl' => 'Scraper (Perl)',
+                'Java/' => 'Scraper (Java)',
+                'Postman' => 'Scraper (Postman)',
+            ]
+        ];
+    }
+
+    /**
+     * Identify a User-Agent string
+     *
+     * @param string $ua User-Agent string
+     * @return array Array with category, pattern, and display label
+     */
+    public static function identify(string $ua): array
+    {
+        if (trim($ua) === '') {
+            return [
+                'category' => 'empty',
+                'pattern' => 'empty',
+                'label' => 'Lege User-Agent'
+            ];
+        }
+
+        $signatures = self::getBotSignatures();
+
+        foreach ($signatures['ai'] as $pattern => $label) {
+            if (stripos($ua, $pattern) !== false) {
+                return ['category' => 'ai', 'pattern' => $pattern, 'label' => $label];
+            }
+        }
+
+        foreach ($signatures['search'] as $pattern => $label) {
+            if (stripos($ua, $pattern) !== false) {
+                return ['category' => 'search', 'pattern' => $pattern, 'label' => $label];
+            }
+        }
+
+        foreach ($signatures['scraper'] as $pattern => $label) {
+            if (stripos($ua, $pattern) !== false) {
+                return ['category' => 'scraper', 'pattern' => $pattern, 'label' => $label];
+            }
+        }
+
+        if (preg_match('/(bot|crawler|spider|slurp)/i', $ua)) {
+            return ['category' => 'generic', 'pattern' => 'generic_bot', 'label' => 'Bot'];
+        }
+
+        return ['category' => 'human', 'pattern' => 'human', 'label' => 'Mens'];
+    }
+
+    /**
+     * Determine if a request should be blocked based on security settings
+     *
+     * @param string $ua User-Agent string
+     * @param array $securitySettings Security configuration array
+     * @return string|null Reason string if blocked, null if allowed
+     */
+    public static function shouldBlock(string $ua, array $securitySettings): ?string
+    {
+        $trimmedUa = trim($ua);
+
+        // 1. Empty User-Agent check
+        if ($trimmedUa === '') {
+            if (!empty($securitySettings['block_empty_user_agent'])) {
+                return 'blocked:empty_ua';
+            }
+            return null;
+        }
+
+        // 2. Custom User-Agent blocklist
+        $customBlocked = $securitySettings['custom_blocked_agents'] ?? [];
+        if (is_array($customBlocked)) {
+            foreach ($customBlocked as $pattern) {
+                $pattern = trim($pattern);
+                if ($pattern !== '' && stripos($trimmedUa, $pattern) !== false) {
+                    return 'blocked:custom_agent';
+                }
+            }
+        }
+
+        // 3. Category signature check
+        $identity = self::identify($trimmedUa);
+        $category = $identity['category'];
+
+        if ($category === 'ai' && !empty($securitySettings['block_ai_bots'])) {
+            return 'blocked:ai';
+        }
+
+        if ($category === 'search' && !empty($securitySettings['block_search_engines'])) {
+            return 'blocked:search';
+        }
+
+        if ($category === 'scraper' && !empty($securitySettings['block_scrapers'])) {
+            return 'blocked:scraper';
+        }
+
+        if ($category === 'generic' && (!empty($securitySettings['block_scrapers']) || !empty($securitySettings['block_ai_bots']))) {
+            return 'blocked:generic_bot';
+        }
+
+        return null;
+    }
+
+    /**
+     * Generate dynamic robots.txt content based on security settings
+     *
+     * @param array $securitySettings Security configuration array
+     * @return string Robots.txt content
+     */
+    public static function generateRobotsTxt(array $securitySettings): string
+    {
+        $out = "# robots.txt generated dynamically by CodePress CMS\n\n";
+
+        // Global rule for search engines
+        if (!empty($securitySettings['block_search_engines'])) {
+            $out .= "User-agent: *\nDisallow: /\n\n";
+        } else {
+            $out .= "User-agent: *\nAllow: /\nDisallow: /admin\nDisallow: /cms\n\n";
+        }
+
+        // Block specific AI bots if enabled
+        if (!empty($securitySettings['block_ai_bots'])) {
+            $signatures = self::getBotSignatures();
+            $out .= "# Block AI Crawlers & Scrapers\n";
+            foreach (array_keys($signatures['ai']) as $aiBot) {
+                $out .= "User-agent: {$aiBot}\nDisallow: /\n";
+            }
+            $out .= "\n";
+        }
+
+        return $out;
+    }
+}
@@ -1103,6 +1103,8 @@ class CodePressCMS {
            'author_git' => 'https://git.noorlander.info/E.Noorlander',
            'seo_description' => $this->config['seo']['description'] ?? 'CodePress CMS - Lightweight file-based content management system',
            'seo_keywords' => $this->config['seo']['keywords'] ?? 'cms, php, content management, file-based',
+            'block_ai_bots' => !empty($this->config['security']['block_ai_bots']),
+            'block_search_engines' => !empty($this->config['security']['block_search_engines']),
            'cms_version' => ($this->config['show_version'] ?? true) && isset($this->config['version_info']) ? $this->config['version_info']['version'] : '',
            // Theme colors
            'header_color' => $this->config['theme']['header_color'] ?? '#0d6efd',
@@ -9,7 +9,7 @@ class RequestLogger
        $this->logFile = $logFile;
    }

-    public function log(string $page, string $ip, string $userAgent, string $referrer, string $host, string $acceptLanguage): void
+    public function log(string $page, string $ip, string $userAgent, string $referrer, string $host, string $acceptLanguage, string $status = 'ok'): void
    {
        $dir = dirname($this->logFile);
        if (!is_dir($dir)) {
@@ -19,7 +19,7 @@ class RequestLogger
        $timestamp = date('Y-m-d H:i:s');
        $ua = substr(preg_replace('/[[:cntrl:]]/', '', $userAgent), 0, 500);
        $ref = substr(preg_replace('/[[:cntrl:]]/', '', $referrer), 0, 500);
-        $line = "[{$timestamp}] [{$ip}] [{$host}] [{$acceptLanguage}] [{$page}] [{$ua}] [{$ref}]\n";
+        $line = "[{$timestamp}] [{$ip}] [{$host}] [{$acceptLanguage}] [{$page}] [{$ua}] [{$ref}] [{$status}]\n";
        @file_put_contents($this->logFile, $line, FILE_APPEND | LOCK_EX);
    }

@@ -73,116 +73,50 @@ class RequestLogger

    public static function detectVisitorInfo(string $ua, string $user = ''): array
    {
-        if (empty($ua)) {
-            $userLabel = ($user && $user !== 'Gast') ? $user : 'Onbekend';
-            return ['type' => 'unknown', 'label' => $userLabel, 'badge' => 'secondary', 'icon' => 'bi-question-circle'];
-        }
+        if (class_exists('BotGuard')) {
+            $id = BotGuard::identify($ua);
+            $cat = $id['category'];
+            $label = $id['label'];

-        $bots = [
-            'AI' => [
-                'GPTBot' => 'AI (GPTBot)',
-                'ChatGPT-User' => 'AI (ChatGPT)',
-                'Claude-Web' => 'AI (Claude)',
-                'ClaudeBot' => 'AI (ClaudeBot)',
-                'anthropic-ai' => 'AI (Anthropic)',
-                'Google-Extended' => 'AI (Gemini/Google)',
-                'CCBot' => 'AI (CommonCrawl)',
-                'PerplexityBot' => 'AI (Perplexity)',
-                'Amazonbot' => 'AI (Amazon)',
-                'cohere-ai' => 'AI (Cohere)',
-                'OAI-SearchBot' => 'AI (OpenAI)',
-                'Bytespider' => 'AI (ByteDance)',
-                'FacebookBot' => 'AI (Meta/FB)',
-                'Applebot-Extended' => 'AI (Apple)',
-            ],
-            'Search' => [
-                'Googlebot' => 'Zoekmachine (Google)',
-                'Bingbot' => 'Zoekmachine (Bing)',
-                'BingPreview' => 'Zoekmachine (Bing)',
-                'Slurp' => 'Zoekmachine (Yahoo)',
-                'DuckDuckBot' => 'Zoekmachine (DuckDuckGo)',
-                'Baiduspider' => 'Zoekmachine (Baidu)',
-                'YandexBot' => 'Zoekmachine (Yandex)',
-                'Sogou' => 'Zoekmachine (Sogou)',
-                'Exabot' => 'Zoekmachine (Exabot)',
-                'facebot' => 'Zoekmachine (Facebook)',
-            ],
-            'Scraper' => [
-                'HTTrack' => 'Scraper (HTTrack)',
-                'Scrapy' => 'Scraper (Scrapy)',
-                'PhantomJS' => 'Scraper (PhantomJS)',
-                'HeadlessChrome' => 'Scraper (Headless)',
-                'curl' => 'Scraper (cURL)',
-                'wget' => 'Scraper (Wget)',
-                'python' => 'Scraper (Python)',
-                'Postman' => 'Scraper (Postman)',
-            ],
-        ];
-
-        foreach ($bots['AI'] as $pattern => $label) {
-            if (stripos($ua, $pattern) !== false) {
+            if ($cat === 'ai') {
                return ['type' => 'ai', 'label' => $label, 'badge' => 'danger', 'icon' => 'bi-robot'];
            }
-        }
-
-        foreach ($bots['Search'] as $pattern => $label) {
-            if (stripos($ua, $pattern) !== false) {
+            if ($cat === 'search') {
                return ['type' => 'search', 'label' => $label, 'badge' => 'primary', 'icon' => 'bi-search'];
            }
-        }
-
-        foreach ($bots['Scraper'] as $pattern => $label) {
-            if (stripos($ua, $pattern) !== false) {
+            if ($cat === 'scraper') {
                return ['type' => 'scraper', 'label' => $label, 'badge' => 'warning text-dark', 'icon' => 'bi-bug'];
            }
+            if ($cat === 'generic') {
+                return ['type' => 'bot', 'label' => 'Bot', 'badge' => 'secondary', 'icon' => 'bi-robot'];
+            }
+            if ($cat === 'empty') {
+                return ['type' => 'empty', 'label' => 'Lege UA', 'badge' => 'secondary', 'icon' => 'bi-slash-circle'];
+            }
+
+            $userPrefix = ($user && $user !== 'Gast') ? htmlspecialchars($user) . ' (' : '';
+            $userSuffix = ($user && $user !== 'Gast') ? ')' : '';
+
+            return [
+                'type' => 'human',
+                'label' => $userPrefix . 'Mens' . $userSuffix,
+                'badge' => 'success',
+                'icon' => 'bi-person-check',
+            ];
        }

-        if (preg_match('/(bot|crawler|spider|slurp)/i', $ua)) {
-            return ['type' => 'bot', 'label' => 'Bot', 'badge' => 'secondary', 'icon' => 'bi-robot'];
-        }
-
-        $userPrefix = ($user && $user !== 'Gast') ? htmlspecialchars($user) . ' (' : '';
-        $userSuffix = ($user && $user !== 'Gast') ? ')' : '';
-
-        return [
-            'type' => 'human',
-            'label' => $userPrefix . 'Mens' . $userSuffix,
-            'badge' => 'success',
-            'icon' => 'bi-person-check',
-        ];
+        return ['type' => 'unknown', 'label' => 'Bezoeker', 'badge' => 'secondary', 'icon' => 'bi-person'];
    }

    public static function detectBot(): ?string
    {
        $ua = $_SERVER['HTTP_USER_AGENT'] ?? '';
-        if (empty($ua)) return null;
-
-        $bots = [
-            'AI' => [
-                'GPTBot', 'ChatGPT-User', 'Claude-Web', 'ClaudeBot',
-                'anthropic-ai', 'Google-Extended', 'CCBot',
-                'PerplexityBot', 'Amazonbot', 'cohere-ai',
-                'OAI-SearchBot', 'Bytespider', 'FacebookBot',
-                'Applebot-Extended',
-            ],
-            'Search' => [
-                'Googlebot', 'Bingbot', 'BingPreview', 'Slurp',
-                'DuckDuckBot', 'Baiduspider', 'YandexBot',
-                'Sogou', 'Exabot', 'facebot',
-            ],
-            'Scraper' => [
-                'HTTrack', 'Scrapy', 'PhantomJS', 'HeadlessChrome',
-            ],
-        ];
-
-        foreach ($bots as $category => $patterns) {
-            foreach ($patterns as $pattern) {
-                if (stripos($ua, $pattern) !== false) {
-                    return $category;
-                }
+        if (class_exists('BotGuard')) {
+            $id = BotGuard::identify($ua);
+            if (in_array($id['category'], ['ai', 'search', 'scraper'], true)) {
+                return strtoupper($id['category']);
            }
        }
-
        return null;
    }

@@ -197,12 +131,15 @@ class RequestLogger
        $logs = [];

        foreach ($content as $line) {
-            if (preg_match('/^\[([^\]]+)\] \[([^\]]+)\] \[([^\]]+)\] \[([^\]]*)\] \[([^\]]+)\] \[([^\]]*)\] \[([^\]]*)\]$/', trim($line), $m)) {
+            $trimmed = trim($line);
+            if (preg_match('/^\[([^\]]+)\] \[([^\]]+)\] \[([^\]]+)\] \[([^\]]*)\] \[([^\]]+)\] \[([^\]]*)\] \[([^\]]*)\](?: \[([^\]]*)\])?$/', $trimmed, $m)) {
                $user = $m[3];
-                // Handle backwards compatibility where host (e.g. localhost:8080 or domain.com) was logged
                if (str_contains($user, '.') || str_contains($user, ':') || $user === 'cli') {
                    $user = 'Gast';
                }
+                $status = $m[8] ?? 'ok';
+                if ($status === '') $status = 'ok';
+
                $visitorInfo = self::detectVisitorInfo($m[6], $user);
                $logs[] = [
                    'time' => $m[1],
@@ -213,6 +150,7 @@ class RequestLogger
                    'page' => $m[5],
                    'ua' => $m[6],
                    'referrer' => $m[7],
+                    'status' => $status,
                ];
            }
        }