# Default: Allow all crawlers
User-agent: *
Allow: /
# Keep raw datasets out of search results
Disallow: /data/
# FutureSearch app: published research pages and shared conversations are
# public; everything else in the app is private. (No blank line above —
# a blank line would end the User-agent group and orphan these rules.)
Allow: /app/p/
Allow: /app/conversations/share/
Disallow: /app/billing
Disallow: /app/admin
Disallow: /app/api-key
Disallow: /app/checkout
Disallow: /app/conversations/
Disallow: /app/api/

# Explicitly allow all major AI crawlers (training and search)
# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

# Other AI systems
User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Applebot-Extended
Allow: /

# Block admin/staging areas if we add them
Disallow: /admin/
Disallow: /staging/

# Sitemaps
Sitemap: https://futuresearch.ai/sitemap.xml
Sitemap: https://futuresearch.ai/docs/sitemap.xml