# Block no-value SEO/scraper crawlers and AI training bots to free server
# capacity. Search engines (Googlebot, bingbot, SeznamBot), social link-preview
# agents (facebookexternalhit, Twitterbot, …) and ChatGPT-User (user-triggered
# fetches) stay allowed via the final "*" group.
# NOTE: robots.txt is advisory — compliant bots obey it; scrapers that ignore it
# (e.g. Bytespider, fake-browser UAs) must be blocked at the server level instead.

# --- No-value SEO / scraper / no-audience crawlers ---
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
User-agent: Barkrowler
User-agent: BLEXBot
User-agent: PetalBot
User-agent: Amazonbot
User-agent: Baiduspider
Disallow: /

# --- AI training crawlers ---
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: Bytespider
User-agent: PerplexityBot
Disallow: /

# --- Everyone else (Googlebot, bingbot, SeznamBot, ChatGPT-User, social previews) ---
User-agent: *
Disallow: