E.Noorlander/CodePress
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CodePress CMS v1.8.0: BotGuard security engine, HAProxy docs & ARIA fix

Browse Source 
- Implement BotGuard security engine (Bot, AI, Scraper & Empty UA blocking)
- Add Admin Security page (/admin/security) with toggles, rate limiter & block/allowlists
- Add per-IP RateLimiter handoff in index.php with HTTP 429 response
- Add dynamic /robots.txt generation and noai/noimageai meta tags
- Add RequestLogger status column and blocked badges in admin request logs
- Fix ARIAComponents.php syntax errors on lines 67, 137, 262
- Add HAProxy / PFSense bot blocking & IP forwarding guide (docs/haproxy-bot-blocking.md)
- Update version to 1.8.0 with release notes in version.php and guides
This commit is contained in:
E.Noorlander
2026-07-29 14:26:48 +02:00
parent 62dd7ddb9c
commit 239762fd3a
18 changed files with 618 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cms/core/class/ARIAComponents.php
+3 -3
View File
@@ -64,7 +64,7 @@ class ARIAComponents {
$label = $options['aria-label'] ?? 'Hoofdmenu';
$orientation = $options['orientation'] ?? 'horizontal';
$html = '<nav id="' . $id . '" role="navigation" aria-label="' . htmlspecialchars($label, ENT_QUOTES, 'UTF-8) . '">';
$html = '<nav id="' . $id . '" role="navigation" aria-label="' . htmlspecialchars($label, ENT_QUOTES, 'UTF-8') . '">';
$html .= '<ul role="menubar" aria-orientation="' . $orientation . '">';
foreach ($menu as $index => $item) {
@@ -134,7 +134,7 @@ class ARIAComponents {
$label = $options['aria-label'] ?? 'Formulier';
$html = '<form id="' . $id . '" method="' . $method . '" action="' . htmlspecialchars($action, ENT_QUOTES, 'UTF-8') . '" ';
$html .= 'role="form" aria-label="' . htmlspecialchars($label, ENT_QUOTES, 'UTF-8) . '" ';
$html .= 'role="form" aria-label="' . htmlspecialchars($label, ENT_QUOTES, 'UTF-8') . '" ';
$html .= 'novalidate>';
foreach ($fields as $index => $field) {
@@ -259,7 +259,7 @@ class ARIAComponents {
public static function createAccessibleBreadcrumb($breadcrumbs, $options = []) {
$label = $options['aria-label'] ?? 'Broodkruimelnavigatie';
$html = '<nav aria-label="' . htmlspecialchars($label, ENT_QUOTES, 'UTF-8) . '">';
$html = '<nav aria-label="' . htmlspecialchars($label, ENT_QUOTES, 'UTF-8') . '">';
$html .= '<ol class="breadcrumb">';
foreach ($breadcrumbs as $index => $crumb) {
cms/core/class/BotGuard.php
+189
View File
@@ -0,0 +1,189 @@
<?php
/**
* BotGuard - Bot, AI Crawler, and Scraper detection & protection
*/
class BotGuard
{
/**
* Map of bot signatures by category and pattern
*/
public static function getBotSignatures(): array
{
return [
'ai' => [
'GPTBot' => 'AI (GPTBot)',
'ChatGPT-User' => 'AI (ChatGPT)',
'Claude-Web' => 'AI (Claude)',
'ClaudeBot' => 'AI (ClaudeBot)',
'anthropic-ai' => 'AI (Anthropic)',
'Google-Extended' => 'AI (Gemini/Google)',
'CCBot' => 'AI (CommonCrawl)',
'PerplexityBot' => 'AI (Perplexity)',
'Amazonbot' => 'AI (Amazon)',
'cohere-ai' => 'AI (Cohere)',
'OAI-SearchBot' => 'AI (OpenAI)',
'Bytespider' => 'AI (ByteDance)',
'FacebookBot' => 'AI (Meta/FB)',
'Applebot-Extended' => 'AI (Apple)',
'Meta-ExternalAgent' => 'AI (Meta)',
'Diffbot' => 'AI (Diffbot)',
'ImagesiftBot' => 'AI (Imagesift)',
'Omgilibot' => 'AI (Omgili)',
'Timpibot' => 'AI (Timpi)',
],
'search' => [
'Googlebot' => 'Zoekmachine (Google)',
'Bingbot' => 'Zoekmachine (Bing)',
'BingPreview' => 'Zoekmachine (Bing)',
'Slurp' => 'Zoekmachine (Yahoo)',
'DuckDuckBot' => 'Zoekmachine (DuckDuckGo)',
'Baiduspider' => 'Zoekmachine (Baidu)',
'YandexBot' => 'Zoekmachine (Yandex)',
'Sogou' => 'Zoekmachine (Sogou)',
'Exabot' => 'Zoekmachine (Exabot)',
'facebot' => 'Zoekmachine (Facebook)',
],
'scraper' => [
'HTTrack' => 'Scraper (HTTrack)',
'Scrapy' => 'Scraper (Scrapy)',
'PhantomJS' => 'Scraper (PhantomJS)',
'HeadlessChrome' => 'Scraper (Headless)',
'curl' => 'Scraper (cURL)',
'wget' => 'Scraper (Wget)',
'python-requests' => 'Scraper (Python)',
'python-urllib' => 'Scraper (Python)',
'Go-http-client' => 'Scraper (Go)',
'libwww-perl' => 'Scraper (Perl)',
'Java/' => 'Scraper (Java)',
'Postman' => 'Scraper (Postman)',
]
];
}
/**
* Identify a User-Agent string
*
* @param string $ua User-Agent string
* @return array Array with category, pattern, and display label
*/
public static function identify(string $ua): array
{
if (trim($ua) === '') {
return [
'category' => 'empty',
'pattern' => 'empty',
'label' => 'Lege User-Agent'
];
}
$signatures = self::getBotSignatures();
foreach ($signatures['ai'] as $pattern => $label) {
if (stripos($ua, $pattern) !== false) {
return ['category' => 'ai', 'pattern' => $pattern, 'label' => $label];
}
}
foreach ($signatures['search'] as $pattern => $label) {
if (stripos($ua, $pattern) !== false) {
return ['category' => 'search', 'pattern' => $pattern, 'label' => $label];
}
}
foreach ($signatures['scraper'] as $pattern => $label) {
if (stripos($ua, $pattern) !== false) {
return ['category' => 'scraper', 'pattern' => $pattern, 'label' => $label];
}
}
if (preg_match('/(bot|crawler|spider|slurp)/i', $ua)) {
return ['category' => 'generic', 'pattern' => 'generic_bot', 'label' => 'Bot'];
}
return ['category' => 'human', 'pattern' => 'human', 'label' => 'Mens'];
}
/**
* Determine if a request should be blocked based on security settings
*
* @param string $ua User-Agent string
* @param array $securitySettings Security configuration array
* @return string|null Reason string if blocked, null if allowed
*/
public static function shouldBlock(string $ua, array $securitySettings): ?string
{
$trimmedUa = trim($ua);
// 1. Empty User-Agent check
if ($trimmedUa === '') {
if (!empty($securitySettings['block_empty_user_agent'])) {
return 'blocked:empty_ua';
}
return null;
}
// 2. Custom User-Agent blocklist
$customBlocked = $securitySettings['custom_blocked_agents'] ?? [];
if (is_array($customBlocked)) {
foreach ($customBlocked as $pattern) {
$pattern = trim($pattern);
if ($pattern !== '' && stripos($trimmedUa, $pattern) !== false) {
return 'blocked:custom_agent';
}
}
}
// 3. Category signature check
$identity = self::identify($trimmedUa);
$category = $identity['category'];
if ($category === 'ai' && !empty($securitySettings['block_ai_bots'])) {
return 'blocked:ai';
}
if ($category === 'search' && !empty($securitySettings['block_search_engines'])) {
return 'blocked:search';
}
if ($category === 'scraper' && !empty($securitySettings['block_scrapers'])) {
return 'blocked:scraper';
}
if ($category === 'generic' && (!empty($securitySettings['block_scrapers']) || !empty($securitySettings['block_ai_bots']))) {
return 'blocked:generic_bot';
}
return null;
}
/**
* Generate dynamic robots.txt content based on security settings
*
* @param array $securitySettings Security configuration array
* @return string Robots.txt content
*/
public static function generateRobotsTxt(array $securitySettings): string
{
$out = "# robots.txt generated dynamically by CodePress CMS\n\n";
// Global rule for search engines
if (!empty($securitySettings['block_search_engines'])) {
$out .= "User-agent: *\nDisallow: /\n\n";
} else {
$out .= "User-agent: *\nAllow: /\nDisallow: /admin\nDisallow: /cms\n\n";
}
// Block specific AI bots if enabled
if (!empty($securitySettings['block_ai_bots'])) {
$signatures = self::getBotSignatures();
$out .= "# Block AI Crawlers & Scrapers\n";
foreach (array_keys($signatures['ai']) as $aiBot) {
$out .= "User-agent: {$aiBot}\nDisallow: /\n";
}
$out .= "\n";
}
return $out;
}
}
cms/core/class/CodePressCMS.php
+2
View File
@@ -1103,6 +1103,8 @@ class CodePressCMS {
'author_git' => 'https://git.noorlander.info/E.Noorlander',
'seo_description' => $this->config['seo']['description'] ?? 'CodePress CMS - Lightweight file-based content management system',
'seo_keywords' => $this->config['seo']['keywords'] ?? 'cms, php, content management, file-based',
'block_ai_bots' => !empty($this->config['security']['block_ai_bots']),
'block_search_engines' => !empty($this->config['security']['block_search_engines']),
'cms_version' => ($this->config['show_version'] ?? true) && isset($this->config['version_info']) ? $this->config['version_info']['version'] : '',
// Theme colors
'header_color' => $this->config['theme']['header_color'] ?? '#0d6efd',
cms/core/class/RequestLogger.php
+36 -98
View File
@@ -9,7 +9,7 @@ class RequestLogger
$this->logFile = $logFile;
}
public function log(string $page, string $ip, string $userAgent, string $referrer, string $host, string $acceptLanguage): void
public function log(string $page, string $ip, string $userAgent, string $referrer, string $host, string $acceptLanguage, string $status = 'ok'): void
{
$dir = dirname($this->logFile);
if (!is_dir($dir)) {
@@ -19,7 +19,7 @@ class RequestLogger
$timestamp = date('Y-m-d H:i:s');
$ua = substr(preg_replace('/[[:cntrl:]]/', '', $userAgent), 0, 500);
$ref = substr(preg_replace('/[[:cntrl:]]/', '', $referrer), 0, 500);
$line = "[{$timestamp}] [{$ip}] [{$host}] [{$acceptLanguage}] [{$page}] [{$ua}] [{$ref}]\n";
$line = "[{$timestamp}] [{$ip}] [{$host}] [{$acceptLanguage}] [{$page}] [{$ua}] [{$ref}] [{$status}]\n";
@file_put_contents($this->logFile, $line, FILE_APPEND | LOCK_EX);
}
@@ -73,116 +73,50 @@ class RequestLogger
public static function detectVisitorInfo(string $ua, string $user = ''): array
{
if (empty($ua)) {
$userLabel = ($user && $user !== 'Gast') ? $user : 'Onbekend';
return ['type' => 'unknown', 'label' => $userLabel, 'badge' => 'secondary', 'icon' => 'bi-question-circle'];
}
if (class_exists('BotGuard')) {
$id = BotGuard::identify($ua);
$cat = $id['category'];
$label = $id['label'];
$bots = [
'AI' => [
'GPTBot' => 'AI (GPTBot)',
'ChatGPT-User' => 'AI (ChatGPT)',
'Claude-Web' => 'AI (Claude)',
'ClaudeBot' => 'AI (ClaudeBot)',
'anthropic-ai' => 'AI (Anthropic)',
'Google-Extended' => 'AI (Gemini/Google)',
'CCBot' => 'AI (CommonCrawl)',
'PerplexityBot' => 'AI (Perplexity)',
'Amazonbot' => 'AI (Amazon)',
'cohere-ai' => 'AI (Cohere)',
'OAI-SearchBot' => 'AI (OpenAI)',
'Bytespider' => 'AI (ByteDance)',
'FacebookBot' => 'AI (Meta/FB)',
'Applebot-Extended' => 'AI (Apple)',
],
'Search' => [
'Googlebot' => 'Zoekmachine (Google)',
'Bingbot' => 'Zoekmachine (Bing)',
'BingPreview' => 'Zoekmachine (Bing)',
'Slurp' => 'Zoekmachine (Yahoo)',
'DuckDuckBot' => 'Zoekmachine (DuckDuckGo)',
'Baiduspider' => 'Zoekmachine (Baidu)',
'YandexBot' => 'Zoekmachine (Yandex)',
'Sogou' => 'Zoekmachine (Sogou)',
'Exabot' => 'Zoekmachine (Exabot)',
'facebot' => 'Zoekmachine (Facebook)',
],
'Scraper' => [
'HTTrack' => 'Scraper (HTTrack)',
'Scrapy' => 'Scraper (Scrapy)',
'PhantomJS' => 'Scraper (PhantomJS)',
'HeadlessChrome' => 'Scraper (Headless)',
'curl' => 'Scraper (cURL)',
'wget' => 'Scraper (Wget)',
'python' => 'Scraper (Python)',
'Postman' => 'Scraper (Postman)',
],
];
foreach ($bots['AI'] as $pattern => $label) {
if (stripos($ua, $pattern) !== false) {
if ($cat === 'ai') {
return ['type' => 'ai', 'label' => $label, 'badge' => 'danger', 'icon' => 'bi-robot'];
}
}
foreach ($bots['Search'] as $pattern => $label) {
if (stripos($ua, $pattern) !== false) {
if ($cat === 'search') {
return ['type' => 'search', 'label' => $label, 'badge' => 'primary', 'icon' => 'bi-search'];
}
}
foreach ($bots['Scraper'] as $pattern => $label) {
if (stripos($ua, $pattern) !== false) {
if ($cat === 'scraper') {
return ['type' => 'scraper', 'label' => $label, 'badge' => 'warning text-dark', 'icon' => 'bi-bug'];
}
if ($cat === 'generic') {
return ['type' => 'bot', 'label' => 'Bot', 'badge' => 'secondary', 'icon' => 'bi-robot'];
}
if ($cat === 'empty') {
return ['type' => 'empty', 'label' => 'Lege UA', 'badge' => 'secondary', 'icon' => 'bi-slash-circle'];
}
$userPrefix = ($user && $user !== 'Gast') ? htmlspecialchars($user) . ' (' : '';
$userSuffix = ($user && $user !== 'Gast') ? ')' : '';
return [
'type' => 'human',
'label' => $userPrefix . 'Mens' . $userSuffix,
'badge' => 'success',
'icon' => 'bi-person-check',
];
}
if (preg_match('/(bot|crawler|spider|slurp)/i', $ua)) {
return ['type' => 'bot', 'label' => 'Bot', 'badge' => 'secondary', 'icon' => 'bi-robot'];
}
$userPrefix = ($user && $user !== 'Gast') ? htmlspecialchars($user) . ' (' : '';
$userSuffix = ($user && $user !== 'Gast') ? ')' : '';
return [
'type' => 'human',
'label' => $userPrefix . 'Mens' . $userSuffix,
'badge' => 'success',
'icon' => 'bi-person-check',
];
return ['type' => 'unknown', 'label' => 'Bezoeker', 'badge' => 'secondary', 'icon' => 'bi-person'];
}
public static function detectBot(): ?string
{
$ua = $_SERVER['HTTP_USER_AGENT'] ?? '';
if (empty($ua)) return null;
$bots = [
'AI' => [
'GPTBot', 'ChatGPT-User', 'Claude-Web', 'ClaudeBot',
'anthropic-ai', 'Google-Extended', 'CCBot',
'PerplexityBot', 'Amazonbot', 'cohere-ai',
'OAI-SearchBot', 'Bytespider', 'FacebookBot',
'Applebot-Extended',
],
'Search' => [
'Googlebot', 'Bingbot', 'BingPreview', 'Slurp',
'DuckDuckBot', 'Baiduspider', 'YandexBot',
'Sogou', 'Exabot', 'facebot',
],
'Scraper' => [
'HTTrack', 'Scrapy', 'PhantomJS', 'HeadlessChrome',
],
];
foreach ($bots as $category => $patterns) {
foreach ($patterns as $pattern) {
if (stripos($ua, $pattern) !== false) {
return $category;
}
if (class_exists('BotGuard')) {
$id = BotGuard::identify($ua);
if (in_array($id['category'], ['ai', 'search', 'scraper'], true)) {
return strtoupper($id['category']);
}
}
return null;
}
@@ -197,12 +131,15 @@ class RequestLogger
$logs = [];
foreach ($content as $line) {
if (preg_match('/^\[([^\]]+)\] \[([^\]]+)\] \[([^\]]+)\] \[([^\]]*)\] \[([^\]]+)\] \[([^\]]*)\] \[([^\]]*)\]$/', trim($line), $m)) {
$trimmed = trim($line);
if (preg_match('/^\[([^\]]+)\] \[([^\]]+)\] \[([^\]]+)\] \[([^\]]*)\] \[([^\]]+)\] \[([^\]]*)\] \[([^\]]*)\](?: \[([^\]]*)\])?$/', $trimmed, $m)) {
$user = $m[3];
// Handle backwards compatibility where host (e.g. localhost:8080 or domain.com) was logged
if (str_contains($user, '.') || str_contains($user, ':') || $user === 'cli') {
$user = 'Gast';
}
$status = $m[8] ?? 'ok';
if ($status === '') $status = 'ok';
$visitorInfo = self::detectVisitorInfo($m[6], $user);
$logs[] = [
'time' => $m[1],
@@ -213,6 +150,7 @@ class RequestLogger
'page' => $m[5],
'ua' => $m[6],
'referrer' => $m[7],
'status' => $status,
];
}
}
cms/core/config.php
+12
View File
@@ -33,6 +33,18 @@ if (!file_exists($configJsonPath)) {
'auto_link_pages' => true,
'search_enabled' => true,
'breadcrumbs_enabled' => true
],
'security' => [
'block_ai_bots' => true,
'block_scrapers' => true,
'block_search_engines' => false,
'block_empty_user_agent' => true,
'rate_limit_enabled' => true,
'rate_limit_max' => 60,
'rate_limit_window' => 60,
'custom_blocked_agents' => [],
'blocked_ips' => [],
'allowed_ips' => []
]
];
@file_put_contents($configJsonPath, json_encode($defaultConfig, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE));
cms/core/index.php
+3
View File
@@ -33,6 +33,9 @@ if (file_exists($autoloader)) {
}
// Load template engine - renders HTML with {{variable}} placeholders and conditionals
require_once 'class/Cache.php';
require_once 'class/RateLimiter.php';
require_once 'class/BotGuard.php';
require_once 'class/SimpleTemplate.php';
// Load Logger class - structured logging with log levels