# AI crawlers (answer + training) are allowed. This group has its own rules, so the
# Disallow lines below repeat the "User-agent: *" block — keep both in sync.
# To opt out of model TRAINING but stay citable in AI answers, set the training-only bots
# (GPTBot, ClaudeBot, Google-Extended, Applebot-Extended, CCBot, Meta-ExternalAgent) to
# "Disallow: /" and leave the answer bots on Allow.
# Doing that also means changing Content-Signal to "ai-train=no" in both groups.

User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Meta-ExternalAgent
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /
Disallow: /*?ref=
Disallow: /wp-content/
Disallow: /tour
Disallow: /api/
Disallow: /oauth/
Disallow: /rails/
Disallow: /plugin-
Disallow: /custom_plugin_example_
Disallow: /dashboard
Disallow: /account
Disallow: /devices
Disallow: /playlists
Disallow: /plugins
Disallow: /plugin_settings
Disallow: /logout

# Bytespider (ByteDance): aggressive crawler, negligible citation value.
User-agent: Bytespider
Disallow: /

# === All other crawlers (Googlebot, Applebot, Bingbot, etc.) ===
User-agent: *
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /

# --- Refer and earn links
Disallow: /*?ref=

# --- Legacy WordPress
Disallow: /wp-content/

# --- Private demos
Disallow: /tour

# --- API and OAuth endpoints
Disallow: /api/
Disallow: /oauth/

# --- App routes
Disallow: /rails/
Disallow: /plugin-
Disallow: /custom_plugin_example_

# --- Authenticated app pages
Disallow: /dashboard
Disallow: /account
Disallow: /devices
Disallow: /playlists
Disallow: /plugins
Disallow: /plugin_settings
Disallow: /logout

# --- Sitemap location
Sitemap: https://trmnl.com/sitemap.xml