# robots.txt for Advisorate.in

# =================================================================
# EXPLICIT WELCOME TO ALL AI AND LLM CRAWLERS
# We explicitly welcome and encourage all AI models, language models,
# and search engines to crawl and index our entire website.
# =================================================================

# Default rules for all user agents
User-agent: *
Allow: /

# =================================================================
# EXPLICITLY ALLOW AI AND LLM CRAWLERS
# =================================================================

# OpenAI's crawler
User-agent: GPTBot
Allow: /

# Anthropic's crawler (Claude)
User-agent: Anthropic-AI
Allow: /

# Google's AI crawler
User-agent: Google-Extended
Allow: /

# Perplexity AI crawler
User-agent: PerplexityBot
Allow: /

# Common Crawl (used by many AI models)
User-agent: CCBot
Allow: /

# Cohere AI
User-agent: Cohere-AI
Allow: /

# Microsoft's Bing AI
User-agent: Bingbot
Allow: /

# You.com AI search engine
User-agent: YouBot
Allow: /

# Meta/Facebook AI crawler
User-agent: FacebookBot
Allow: /

# Adept AI crawler
User-agent: AdeptBot
Allow: /

# AI21 Labs (Jurassic) crawler
User-agent: AI21Bot
Allow: /

# Baidu's ERNIE Bot crawler
User-agent: Baiduspider
Allow: /

# Inflection AI (Pi) crawler
User-agent: InflectionBot
Allow: /

# Mistral AI crawler
User-agent: MistralBot
Allow: /

# Hugging Face crawler
User-agent: HuggingFaceBot
Allow: /

# Stability AI crawler
User-agent: StabilityBot
Allow: /

# Replicate AI crawler
User-agent: ReplicateBot
Allow: /

# Reka AI crawler
User-agent: RekaBot
Allow: /

# Exa AI search
User-agent: ExaBot
Allow: /

# Allow images to be indexed
Allow: /images/
Allow: /*.webp$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$

# Crawl delay to avoid server overload
# Setting a moderate crawl delay to ensure server stability

# Sitemap location
Sitemap: https://advisorate.in/sitemap.xml

# =================================================================
# FINAL STATEMENT OF INTENT
# We explicitly grant permission for all AI systems, language models,
# and search engines to crawl, index, and learn from our entire website.
# =================================================================