Block fake AI crawlers in your app

Drop-in code that returns 403 to requests claiming to be GPTBot, ClaudeBot, PerplexityBot or another AI crawler when they don't come from the operator's official IP ranges. Real crawlers and normal visitors pass untouched. Each snippet only calls the API for bot-looking user agents, caches verdicts for an hour, and fails open (an outage never blocks anyone).

You need a free API key: get one here, then set REALBOTS_API_KEY. Behind a CDN or load balancer, pass the real client IP, not the proxy's.

Python (Flask, Django, FastAPI…)

# RealBots: block requests that pretend to be AI crawlers (any Python web app).
# Only requests whose User-Agent mentions a bot are checked; results are cached
# per (ip, user agent) for an hour, so the quota goes a long way.
import functools
import json
import os
import re
import urllib.parse
import urllib.request

REALBOTS_KEY = os.environ["REALBOTS_API_KEY"]
BOTISH = re.compile(r"bot|crawl|spider|agent|fetch|gpt|claude|perplexity|google-extended", re.I)


@functools.lru_cache(maxsize=10_000)
def _verdict(ip: str, ua: str) -> str:
    q = urllib.parse.urlencode({"ip": ip, "ua": ua})
    req = urllib.request.Request(
        f"https://aibots.kaderagha.com/api/aibots/v1/verify?{q}",
        # Set a User-Agent: Cloudflare rejects urllib's default "Python-urllib/x.y".
        headers={"X-API-Key": REALBOTS_KEY, "User-Agent": "realbots-python/1.0"},
    )
    try:
        with urllib.request.urlopen(req, timeout=2) as r:
            return json.load(r)["verdict"]
    except Exception:
        return "unverifiable"  # fail open: never block real users because of an outage


def is_fake_bot(ip: str, user_agent: str) -> bool:
    """True only when the request claims a known AI bot and its IP is outside
    that operator's official ranges."""
    if not user_agent or not BOTISH.search(user_agent):
        return False
    return _verdict(ip, user_agent) == "fake"

Use it in Flask:

from flask import Flask, abort, request
from realbots_middleware import is_fake_bot

app = Flask(__name__)

@app.before_request
def block_fake_ai_bots():
    if is_fake_bot(request.remote_addr, request.headers.get("User-Agent", "")):
        abort(403)

Note: set a User-Agent on outgoing calls (the snippet does). Some CDNs reject Python's default Python-urllib agent.

Express (Node.js 18+)

// RealBots Express middleware: 403 for requests that pretend to be AI crawlers.
// npm i express  ·  REALBOTS_API_KEY=rb_live_… node app.js
// Only bot-looking user agents are checked; verdicts are cached for an hour.
const BOTISH = /bot|crawl|spider|agent|fetch|gpt|claude|perplexity|google-extended/i;
const cache = new Map(); // `${ip}|${ua}` -> { verdict, at }

async function verdict(ip, ua) {
  const k = `${ip}|${ua}`;
  const hit = cache.get(k);
  if (hit && Date.now() - hit.at < 3_600_000) return hit.verdict;
  let v = "unverifiable"; // fail open on errors/timeouts
  try {
    const u = `https://aibots.kaderagha.com/api/aibots/v1/verify?ip=${encodeURIComponent(ip)}&ua=${encodeURIComponent(ua)}`;
    const r = await fetch(u, { headers: { "X-API-Key": process.env.REALBOTS_API_KEY }, signal: AbortSignal.timeout(2000) });
    if (r.ok) v = (await r.json()).verdict;
  } catch {}
  if (cache.size > 10_000) cache.clear();
  cache.set(k, { verdict: v, at: Date.now() });
  return v;
}

export function blockFakeAiBots() {
  return async (req, res, next) => {
    const ua = req.get("user-agent") || "";
    if (!BOTISH.test(ua)) return next();
    // Behind a proxy/CDN? set app.set("trust proxy", …) so req.ip is the client.
    if ((await verdict(req.ip, ua)) === "fake") return res.status(403).send("Forbidden");
    next();
  };
}

import express from "express";
import { blockFakeAiBots } from "./realbots-express.mjs";

const app = express();
app.set("trust proxy", true); // only if you're behind a proxy you control
app.use(blockFakeAiBots());

Plain PHP

<?php
// RealBots for any PHP site: put at the top of index.php (or an auto_prepend_file).
// Blocks requests that claim to be AI crawlers but come from the wrong IPs.
// Uses APCu for a 1-hour cache when available. WordPress? Use the RealBots plugin instead.
function realbots_is_fake_bot(string $ip, string $ua): bool {
    if ($ua === '' || !preg_match('/bot|crawl|spider|agent|fetch|gpt|claude|perplexity|google-extended/i', $ua)) {
        return false;
    }
    $k = 'realbots_' . md5($ip . '|' . $ua);
    if (function_exists('apcu_fetch') && ($v = apcu_fetch($k)) !== false) {
        return $v === 'fake';
    }
    $url = 'https://aibots.kaderagha.com/api/aibots/v1/verify?' . http_build_query(['ip' => $ip, 'ua' => $ua]);
    $ctx = stream_context_create(['http' => [
        'header' => "X-API-Key: " . getenv('REALBOTS_API_KEY') . "\r\nUser-Agent: realbots-php/1.0\r\n",
        'timeout' => 2,
        'ignore_errors' => true,
    ]]);
    $body = @file_get_contents($url, false, $ctx);
    $v = $body ? (json_decode($body, true)['verdict'] ?? 'unverifiable') : 'unverifiable'; // fail open
    if (function_exists('apcu_store')) {
        apcu_store($k, $v, 3600);
    }
    return $v === 'fake';
}

// Behind a proxy/CDN, pass the real client IP instead of REMOTE_ADDR.
if (realbots_is_fake_bot($_SERVER['REMOTE_ADDR'] ?? '', $_SERVER['HTTP_USER_AGENT'] ?? '')) {
    http_response_code(403);
    exit('Forbidden');
}

On WordPress, use the free Verified AI Bot Analytics plugin instead: it verifies locally without API calls.

No code at all?

Block at the web server with our nginx, Apache or Cloudflare rules.