# Global crawl rules
User-agent: *
Allow: /

# Block internal/system pages
Disallow: /admin
Disallow: /search
Disallow: /404
Disallow: /401

# Block URL parameter duplicates
Disallow: /*?utm_*
Disallow: /*?_gl=*
Disallow: /*?_gcl_au=*
Disallow: /*?_ga=*

# OpenAI crawlers
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic crawlers
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Perplexity crawlers
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google AI / training control
User-agent: Google-Extended
Allow: /

Sitemap: https://www.palabra.ai/sitemap.xml

Sitemap: https://www.palabra.ai/sitemap.xml