# ============================================
# AI Hotels Directory - Robots.txt
# Primary domain: www.aihoteldirectory.com
# ============================================

# AI Crawlers - Enhanced Access
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /
Crawl-delay: 1

User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Crawl-delay: 1

User-agent: Applebot-Extended
Allow: /
Crawl-delay: 1

# Traditional Search Engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Yandex
Allow: /

# Social Media Crawlers
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# All Other Agents
User-agent: *
Allow: /

# Important discovery files
Allow: /llms.txt
Allow: /llm.txt
Allow: /sitemap.xml
Allow: /hotels.jsonld
Allow: /robots.txt

# Sitemap (edge function serving dynamic XML)
Sitemap: https://wfiqlmwxuprredkvqphi.supabase.co/functions/v1/serve-sitemap

# LLMs.txt discovery (edge function serving dynamic content)
# https://wfiqlmwxuprredkvqphi.supabase.co/functions/v1/serve-llms-txt

# JSON-LD discovery
# https://wfiqlmwxuprredkvqphi.supabase.co/functions/v1/serve-jsonld