# Brainrotify robots.txt
# Last updated: 2026-09-03
#
# Search/citation and user-request crawlers may index public web pages.
# Training-only crawlers are blocked. GPTBot is allowed despite also being
# a training crawler: it is the fetcher behind ChatGPT search, so blocking
# it costs the whole ChatGPT surface, not just training.

# ── SEARCH & CITATION BOTS (allowed) ─────────────────────────────────

# Named-agent groups do not inherit the wildcard group's markdown rule.
User-agent: GPTBot
Allow: /
Disallow: /login
Disallow: /signup
Disallow: /Dashboard
Disallow: /VideoPaywall
Disallow: /Profile
Disallow: /Tutorial
Disallow: /onboarding
Disallow: /earn
Disallow: /*.md$

User-agent: OAI-SearchBot
Allow: /
Disallow: /login
Disallow: /signup
Disallow: /Dashboard
Disallow: /VideoPaywall
Disallow: /Profile
Disallow: /Tutorial
Disallow: /onboarding
Disallow: /earn
Disallow: /*.md$

User-agent: Claude-SearchBot
Allow: /
Disallow: /login
Disallow: /signup
Disallow: /Dashboard
Disallow: /VideoPaywall
Disallow: /Profile
Disallow: /Tutorial
Disallow: /onboarding
Disallow: /earn
Disallow: /*.md$

User-agent: ChatGPT-User
Allow: /
Disallow: /login
Disallow: /signup
Disallow: /Dashboard
Disallow: /VideoPaywall
Disallow: /Profile
Disallow: /Tutorial
Disallow: /onboarding
Disallow: /earn
Disallow: /*.md$

User-agent: Claude-User
Allow: /
Disallow: /login
Disallow: /signup
Disallow: /Dashboard
Disallow: /VideoPaywall
Disallow: /Profile
Disallow: /Tutorial
Disallow: /onboarding
Disallow: /earn
Disallow: /*.md$

User-agent: PerplexityBot
Allow: /
Disallow: /login
Disallow: /signup
Disallow: /Dashboard
Disallow: /VideoPaywall
Disallow: /Profile
Disallow: /Tutorial
Disallow: /onboarding
Disallow: /earn
Disallow: /*.md$

User-agent: meta-externalagent
Allow: /
Disallow: /login
Disallow: /signup
Disallow: /Dashboard
Disallow: /VideoPaywall
Disallow: /Profile
Disallow: /Tutorial
Disallow: /onboarding
Disallow: /earn
Disallow: /*.md$

# ── TRAINING-ONLY BOTS (blocked) ─────────────────────────────────────

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

# ── GENERAL RULES ────────────────────────────────────────────────────

User-agent: *
Allow: /

# Block private app routes
Disallow: /login
Disallow: /signup
Disallow: /Dashboard
Disallow: /VideoPaywall
Disallow: /Profile
Disallow: /Tutorial
Disallow: /onboarding
Disallow: /earn

# Block markdown AI-context files — not web pages, waste crawl budget
Disallow: /*.md$

Sitemap: https://www.brainrotify.me/sitemap.xml