User-agent: Baiduspider
User-agent: Baiduspider-image
User-agent: Yandex
User-agent: YandexBot
User-agent: YandexImages
User-agent: Sogou web spider
User-agent: Sogou inst spider
User-agent: 360Spider
User-agent: Mail.RU_Bot
User-agent: YisouSpider
User-agent: Bytespider
User-agent: Shenfeng
User-agent: Sosospider
Disallow: /

# ==========================================
# 2. NUISANCE BOTS, SEO SCRAPERS & GITHUB LISTS
# ==========================================
# Heavily blocked bots sourced from popular GitHub 
# blocklist repositories.
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: megaIndex.ru
User-agent: PetalBot
User-agent: Rogerbot
User-agent: Exabot
User-agent: Spinn3r
User-agent: Xenu
User-agent: Screaming Frog SEO Spider
User-agent: TurnitinBot
User-agent: PiplBot
User-agent: magpie-crawler
User-agent: paperli-bot
User-agent: Qwantify
Disallow: /

# ==========================================
# 3. AGGRESSIVE AI CRAWLERS
# ==========================================
# Comprehensive AI crawler list sourced from 
# Github's ai-robots.txt repository.
User-agent: CCBot
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: Google-Extended
User-agent: Anthropic-ai
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: Cohere-ai
User-agent: omgilibot
User-agent: omgili
User-agent: Diffbot
User-agent: Barkrowler
User-agent: Amazonbot
User-agent: FacebookBot
User-agent: PerplexityBot
User-agent: YouBot
Disallow: /

# ==========================================
# 4. BLOCK COMMON WEB CLIENT FRAMEWORKS
# ==========================================
# NOTE: Scripts ignore robots.txt. These must be 
# enforced by a Firewall or Server-level block.
User-agent: curl
User-agent: Wget
User-agent: python-requests
User-agent: Python-urllib
User-agent: Go-http-client
User-agent: Java
User-agent: libwww-perl
User-agent: GuzzleHttp
User-agent: okhttp
User-agent: Dispatch
User-agent: Apache-HttpClient
User-agent: HTTrack
Disallow: /

# Everyone else: index the public site, skip private areas.
User-agent: *
Disallow: /messages
Disallow: /settings
Disallow: /todo
Disallow: /notifications
Disallow: /feed
Disallow: /chat
Disallow: /logout
Disallow: /mod
Allow: /

Sitemap: http://localhost/sitemap.xml