# robots.txt for revenuebase.ai
# Served from S3 through CloudFront. /companies/* is the RevenueBase company
# directory; every other path is proxied to the Webflow marketing site.
# The previous Webflow-generated robots.txt disallowed nothing; this file
# keeps that policy (no disallows, no crawler exclusions) and adds explicit
# allows for the major search and AI crawlers.
#
# The wildcard group below already permits everything, so no group here is
# load-bearing for access. They are here to state intent per crawler, and so
# that a future Disallow added to the wildcard cannot silently take the
# answer-engine crawlers down with it.
#
# The AI vendors run SEPARATE agents for separate purposes, and the one that
# governs whether we appear in an answer is usually NOT the one people think
# of first:
#
#   OpenAI    GPTBot          model training
#             OAI-SearchBot   inclusion in ChatGPT search results  <- this one
#             ChatGPT-User    a user asked ChatGPT to fetch this page
#   Anthropic ClaudeBot       model development
#             Claude-SearchBot  search indexing                    <- this one
#             Claude-User     a user asked Claude to fetch this page
#   Google    Googlebot       search, and AI Overviews follow it
#             Google-Extended Gemini / Vertex model training
#
# Claude-Web is Anthropic's retired agent name. It is kept because removing a
# name from an allow-only file gains nothing and an old crawler may still
# send it.

User-agent: *
Allow: /

# --- Search -----------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: YandexBot
Allow: /

# --- Answer engines: retrieval and search indexing --------------------------
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# --- Model development ------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

Sitemap: https://revenuebase.ai/sitemap.xml
Sitemap: https://revenuebase.ai/companies/sitemap.xml