stealth-scraping

Stealth Scraping Patterns

curl_cffi — TLS Fingerprint Impersonation

curl_cffi impersonates Chrome's exact TLS ClientHello bytes. Most WAFs that reject httpx/requests accept curl_cffi transparently.

from curl_cffi import requests as curl_requests

# Correct: proxy=URL (singular string), impersonate=specific version
session = curl_requests.AsyncSession(
    impersonate="chrome131",  # Use: chrome120, chrome124, chrome131 (NOT chrome133 — unsupported)
    proxy=PROXY_URL,          # Singular string, NOT a dict
    timeout=20.0,
)

Common mistake: proxies={"http": url, "https": url} — WRONG. Use proxy=url (singular). Common mistake: impersonate="chrome133" — NOT supported by curl_cffi. Use chrome120, chrome124, or chrome131.

playwright-stealth — Correct API

from playwright.async_api import async_playwright
from playwright_stealth import Stealth

stealth = Stealth()
# ... create context ...
await stealth.apply_stealth_async(context)

NOT from playwright_stealth import stealth_async — that import doesn't exist.

Block Detection Heuristics

CRITICAL: Strip <script> and <style> tags BEFORE checking block patterns. Many legitimate news sites embed reCAPTCHA/Cloudflare JavaScript that contains strings like "captcha", "recaptcha", "challenge-platform" — these are false positives, not actual blocks.

import re

# Patterns that indicate REAL block pages (after JS/CSS stripping)
BLOCK_PATTERNS = [
    "access denied",
    "verify you are human",
    "enable javascript and cookies",
    "theme-beta",
    "rate limited",
    "too many requests",
    "just a moment",
    "cf-browser-verification",
    "enable javascript to run this app",
    "bot detection",
    "automated access",
    "your request originates from an undeclared automated tool",
    "please verify you are a human",
    "security check",
    "ddos protection",
]

# Context-dependent: only match on short pages (<2000 chars)
CONTEXT_BLOCK_PATTERNS = ["blocked", "robot"]

def _is_blocked(text: str) -> bool:
    if not text:
        return True
    # STRIP JS/CSS FIRST — this is the #1 fix for false positives
    cleaned = re.sub(r'<script[^>]*>.*?</script>', '', text, flags=re.DOTALL | re.IGNORECASE)
    cleaned = re.sub(r'<style[^>]*>.*?</style>', '', cleaned, flags=re.DOTALL | re.IGNORECASE)
    cleaned_lower = cleaned.lower()
    for pattern in BLOCK_PATTERNS:
        if pattern in cleaned_lower:
            return True
    if len(text) < 2000:
        visible_text = re.sub(r'<[^>]+>', '', cleaned).lower()
        visible_text = re.sub(r'\s+', ' ', visible_text).strip()
        for pattern in CONTEXT_BLOCK_PATTERNS:
            if pattern in visible_text:
                return True
    if len(text) < 500:
        stripped = re.sub(r'<[^>]+>', '', text)
        stripped = re.sub(r'[{};:.\\-_=+#/\\|@!~*()\[\]"\'<>,0-9\\s]', '', stripped)
        if len(stripped) < 50:
            return True
    return False