# Crawl policy (updated 2026-08-15): word pages are server-rendered per
# request, so crawler volume consumes Worker CPU/request quota. Keep major
# search crawlers allowed, and allow the low-risk AI search/user-fetch
# agents that bring referral/search visibility (OAI-SearchBot,
# ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot,
# Perplexity-User). Continue blocking AI training crawlers and high-volume
# low-value crawlers that can burn the Cloudflare daily request limit
# without clear ranking or referral value.

# ------------------------------------------------------------
# AI SEARCH / USER-INITIATED ACCESS — ALLOWED
# ------------------------------------------------------------

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /


# ------------------------------------------------------------
# AI TRAINING / NON-ESSENTIAL AI CRAWLERS — BLOCKED
# ------------------------------------------------------------

User-agent: GPTBot
Disallow: /

User-agent: OAI-AdsBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /


# ------------------------------------------------------------
# SEO / HIGH-VOLUME LOW-VALUE CRAWLERS — BLOCKED
# ------------------------------------------------------------

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Amzn-SearchBot
Disallow: /

User-agent: PetalBot
Disallow: /


# ------------------------------------------------------------
# OTHER CRAWLERS — ALLOWED
# ------------------------------------------------------------

User-agent: *
Allow: /

Sitemap: https://www.fluentstellar.com/sitemap.xml