# ─────────────────────────────────────────────────────────────────────────────
# AI Crawlers — Explicit Allowlist (2026 GEO/LLM-SEO best practice)
# Allow these named bots to index our public marketing pages for AI Overviews,
# ChatGPT, Claude, Perplexity, and Gemini answers. Same scope as User-agent: *
# but explicitly named because some AI crawlers default to "deny" otherwise.
# ─────────────────────────────────────────────────────────────────────────────

# OpenAI — ChatGPT training crawler
User-agent: GPTBot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth/
Disallow: /checkout
Disallow: /invoices
Disallow: /seller/
Disallow: /creator
Disallow: /orders
Disallow: /profile
Disallow: /settings
Disallow: /connected-accounts
Disallow: /login
Disallow: /register
Disallow: /reset-password
Disallow: /update-password
Disallow: /logout
Disallow: /resend-verification
Disallow: /seller_onboarding

# OpenAI — ChatGPT browsing/search bot (for live answers)
User-agent: OAI-SearchBot
Allow: /

# OpenAI — Real-time user-initiated fetches from ChatGPT
User-agent: ChatGPT-User
Allow: /

# Anthropic — Claude training crawler
User-agent: ClaudeBot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth/
Disallow: /checkout
Disallow: /invoices
Disallow: /seller/
Disallow: /creator
Disallow: /orders
Disallow: /profile
Disallow: /settings
Disallow: /connected-accounts
Disallow: /login
Disallow: /register
Disallow: /reset-password
Disallow: /update-password
Disallow: /logout
Disallow: /resend-verification
Disallow: /seller_onboarding

# Anthropic — Real-time Claude fetches
User-agent: Claude-Web
Allow: /

# Perplexity AI — Live answer engine
User-agent: PerplexityBot
Allow: /

# Google — Gemini + AI Overviews training (separate from Googlebot)
User-agent: Google-Extended
Allow: /

# Common Crawl — Used as training data by many LLMs
User-agent: CCBot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth/
Disallow: /checkout
Disallow: /invoices
Disallow: /seller/
Disallow: /creator
Disallow: /orders
Disallow: /profile
Disallow: /settings
Disallow: /connected-accounts
Disallow: /login
Disallow: /register
Disallow: /reset-password
Disallow: /update-password
Disallow: /logout
Disallow: /resend-verification
Disallow: /seller_onboarding

# Perplexity AI — Real-time user fetches (separate from PerplexityBot crawler)
User-agent: Perplexity-User
Allow: /

# Apple — Apple Intelligence + Siri (future Apple search ecosystem)
User-agent: Applebot-Extended
Allow: /

# Amazon — Amazon AI/Rufus crawler (critical for Amazon seller audience)
User-agent: Amazonbot
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /auth/
Disallow: /checkout
Disallow: /invoices
Disallow: /seller/
Disallow: /creator
Disallow: /orders
Disallow: /profile
Disallow: /settings
Disallow: /connected-accounts
Disallow: /login
Disallow: /register
Disallow: /reset-password
Disallow: /update-password
Disallow: /logout
Disallow: /resend-verification
Disallow: /seller_onboarding

# Meta — Llama AI training crawler
User-agent: Meta-ExternalAgent
Allow: /

# ─────────────────────────────────────────────────────────────────────────────
# All other crawlers (Googlebot, Bingbot, etc) — existing rules
# ─────────────────────────────────────────────────────────────────────────────

User-agent: *
Allow: /

# Marketing pages - full access
Allow: /ai-for-ecommerce
Allow: /how-it-works
Allow: /for-ai-developers
Allow: /about
Allow: /contact
Allow: /become-seller
Allow: /privacy-policy
Allow: /terms-of-service
Allow: /pricing
Allow: /developer/pricing
Allow: /sellers

# Educational content - full access
Allow: /resources
Allow: /resources/ai-agent-basics
Allow: /resources/blog
Allow: /resources/video-tutorials

# Seller documentation - full access (public, indexable)
Allow: /seller-docs

# Private/sensitive areas - block from crawling
Disallow: /dashboard
Disallow: /admin
Disallow: /api
Disallow: /seller/
Disallow: /creator
Disallow: /orders
Disallow: /profile
Disallow: /settings
Disallow: /connected-accounts
Disallow: /auth/
Disallow: /login
Disallow: /register
Disallow: /reset-password
Disallow: /update-password
Disallow: /logout
Disallow: /checkout
Disallow: /invoices
Disallow: /resend-verification
Disallow: /seller_onboarding

# Note: /shorts/[id]/run pages use noindex meta tag instead of robots.txt blocking
# This allows public /shorts/[id] detail pages to be indexed while preventing run pages from being indexed

# Sitemap location
Sitemap: https://sellershorts.com/sitemap.xml

# 2025 Google Algorithm Optimizations:
# - Clean, simple robots.txt for better parsing
# - Explicit allow directives for important pages
# - Disallow private/sensitive areas to save crawl budget
# - Clear sitemap reference
# - No unnecessary restrictions that could hurt crawling