# https://www.robotstxt.org/robotstxt.html

# === TRADITIONAL SEARCH ENGINES ===
User-agent: Googlebot
Allow: /

User-agent: bingbot
Allow: /

# === AI SEARCH / CITATION CRAWLERS (allow — these surface DabotCentral in AI answers) ===

# OpenAI search (ChatGPT answers, SearchGPT)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic real-time fetch (Claude answers)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

# Perplexity AI search
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# === AI TRAINING CRAWLERS (block — protect content from being used to train competitor models) ===

# OpenAI bulk training
User-agent: GPTBot
Disallow: /

# Google Generative AI training (does NOT affect Google Search rankings)
User-agent: Google-Extended
Disallow: /

# Common Crawl (used by many open-source LLMs for training)
User-agent: CCBot
Disallow: /

# Meta / Facebook AI training
User-agent: FacebookBot
Disallow: /

# ByteDance / TikTok
User-agent: Bytespider
Disallow: /

# Cohere AI training
User-agent: cohere-ai
Disallow: /

# Apple Intelligence training
User-agent: Applebot-Extended
Disallow: /

# === ALL OTHER BOTS: allow site, but protect private routes ===
# (Single User-agent: * group. Per the robots.txt spec a crawler obeys only
#  ONE matching group, so the fallback allow and the protected-route disallows
#  MUST live together here — splitting them risks a crawler seeing only the
#  allow and missing the disallows.)
User-agent: *
Disallow: /admin/
Disallow: /api/
Disallow: /manage/
Allow: /

# === SITEMAP ===
Sitemap: https://dabotcentral.com/sitemap.xml