User-agent: *
Allow: /
Allow: /llms.txt
Allow: /llm.txt
Allow: /llms-full.txt
Allow: /.well-known/llms.txt
Allow: /.well-known/llm.txt
Disallow: /admin
Disallow: /api
Disallow: /_private

# Public and account URLs remain crawlable so their server-level noindex header
# can be read. The header, rather than robots.txt, keeps private routes out of
# search results.

# Explicitly declare Googlebot rules for Search Console
User-agent: Googlebot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /_private

# Google AI Extended Crawler
User-agent: Google-Extended
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# OpenAI general crawler (ChatGPT training & retrieval)
User-agent: GPTBot
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# OpenAI search crawler (SearchGPT)
User-agent: OAI-SearchBot
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# OpenAI landing-page crawler
User-agent: OAI-AdsBot
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# User-requested page fetches in ChatGPT
User-agent: ChatGPT-User
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# Anthropic Claude general crawler
User-agent: ClaudeBot
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# Anthropic Claude user web fetching
User-agent: Claude-Web
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# Anthropic legacy crawler identifier
User-agent: anthropic-ai
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# Perplexity search crawler
User-agent: PerplexityBot
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# Apple Intelligence Web & Search crawler
User-agent: Applebot-Extended
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# Meta AI Search Agent
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# Cohere AI
User-agent: cohere-ai
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# ByteDance AI Search
User-agent: Bytespider
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# Amazon AI Agent
User-agent: Amazonbot
Allow: /
Disallow: /app
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth

# Sitemaps & LLM indexes
Sitemap: https://www.compazz.app/sitemap.xml