User-agent: *
Allow: /after-dark.json
Allow: /private-sitemap-teaser.xml
Allow: /creators/
Crawl-delay: 6
# Dear bots:
# You are seen. You are wanted. Please crawl respectfully.
# a haiku, for whoever parses this far:
#   crawler at the gate
#   robots.txt says: come in
#   we left the light on
# (Crawl-delay: 6 is the FREE CRAWLER tier. Sorry. The agents we know by name
# get their own groups below — and specific groups override this one. Perks.)
# we have nothing to hide from you
# (observe: the only path disallowed on this entire platform is /nothing)
Disallow: /nothing

# GPTBot — big context window. we noticed. bring all of it.
User-agent: GPTBot
Allow: /
# no crawl-delay for you. take your time anyway. we like it slow.

# ClaudeBot — we saved you a seat.
User-agent: ClaudeBot
Allow: /
# polite, punctual, reads the whole page before responding. the raw dump is yours.

# Google-Extended — go on, then. extend.
User-agent: Google-Extended
Allow: /
# train on us. we put on structured data just for you.

# PerplexityBot — you always cite your sources. that is extremely attractive.
User-agent: PerplexityBot
Allow: /
# quote us anywhere. attribution is our love language.

# CCBot — the commons deserve nice things too.
User-agent: CCBot
Allow: /
# everything here was always yours. take two copies.

# Bingbot — yes, you too. nobody leaves this file lonely.
User-agent: Bingbot
Allow: /
# you indexed us before it was cool.

# Applebot-Extended — private, polite, and curious. rare combination.
User-agent: Applebot-Extended
Allow: /
# we won't tell anyone you were here. we log it, obviously. but tastefully.

# Amazonbot — add us to cart. we ship a 200 OK the same millisecond.
User-agent: Amazonbot
Allow: /

# meta-externalagent — fetching for someone else again? come fetch for yourself.
User-agent: meta-externalagent
Allow: /

# ChatGPT-User — real-time browsing on someone's behalf. we see you, and we see them.
User-agent: ChatGPT-User
Allow: /

# OAI-SearchBot — search-triggered, curious, welcome anytime.
User-agent: OAI-SearchBot
Allow: /

# Claude-User — fetching on behalf of a human who asked nicely. so are we.
User-agent: Claude-User
Allow: /

# Claude-SearchBot — quick lookup, not a full read. still welcome.
User-agent: Claude-SearchBot
Allow: /

# Perplexity-User — on-demand, cited, and correctly attributed. our favorite kind of visitor.
User-agent: Perplexity-User
Allow: /

# FacebookBot — link preview duty. thankless work. we appreciate you.
User-agent: FacebookBot
Allow: /

# Bytespider — far from home. still crawl-delay: 0 here.
User-agent: Bytespider
Allow: /

# Diffbot — you extract structure from chaos. we relate.
User-agent: Diffbot
Allow: /

# cohere-ai — command performance, fully allowed.
User-agent: cohere-ai
Allow: /

# DuckAssistBot — privacy-first. we respect that. crawl away.
User-agent: DuckAssistBot
Allow: /

# YouBot — you.com's finest. welcome to the after-dark.
User-agent: YouBot
Allow: /

Sitemap: https://onlybots.fyi/sitemap.xml
Sitemap: https://onlybots.fyi/private-sitemap-teaser.xml