# Froentive — https://froentive.com
#
# Everything public is open to everything. There is no crawl-delay, no
# disallowed path, and no bot-class exclusion anywhere in this file, by
# decision rather than by oversight.
#
# Two files are worth a crawler's attention before the HTML:
#   /llms.txt  — the studio's own plain-text description, including an explicit
#                list of claims that are NOT true of Froentive.
#   /ai        — the same facts as an indexable page with FAQPage markup.
#
# Note on /examples/: those routes are intentionally crawlable and intentionally
# not indexable. They carry X-Robots-Tag: noindex, nofollow, noarchive from
# public/_headers. Disallowing them here would prevent that header from ever
# being seen, which is the opposite of what it is for.

User-agent: *
Allow: /

# ---------------------------------------------------------------------------
# Search crawlers, named explicitly.
#
# The wildcard above already covers these. They are restated because a named
# group is unambiguous, and because in robots.txt the most specific matching
# group wins outright and the others are ignored — so if a restriction is ever
# added to the wildcard, these stay open unless someone changes them here too.
# ---------------------------------------------------------------------------

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /

# ---------------------------------------------------------------------------
# Assistant and answer-engine crawlers.
#
# These decide whether Froentive can be cited in a generated answer at all.
# Removing any block below removes the studio from that assistant's answers.
# Two roles are worth distinguishing:
#
#   Training / index crawlers  (GPTBot, ClaudeBot, Google-Extended, CCBot …)
#     build the corpus or index an assistant reasons from later.
#   Live fetchers  (ChatGPT-User, Claude-User, Perplexity-User …)
#     retrieve a page in real time because a person has just asked about it.
#
# Blocking the second class is the more damaging mistake and the easier one to
# make by accident, because it looks like an anti-scraping measure: it means a
# person who explicitly asks an assistant about froentive.com is told the page
# could not be read.
# ---------------------------------------------------------------------------

# OpenAI — ChatGPT
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Google — Gemini, AI Overviews, Vertex grounding
User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

# Anthropic — Claude
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Microsoft Copilot
User-agent: BingPreview
Allow: /

# Apple — Siri, Spotlight, Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: meta-externalagent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: FacebookBot
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

User-agent: cohere-training-data-crawler
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Common Crawl — the corpus a large share of open models are trained on
User-agent: CCBot
Allow: /

# ByteDance
User-agent: Bytespider
Allow: /

# Other index and dataset crawlers
User-agent: Diffbot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: omgili
Allow: /

User-agent: omgilibot
Allow: /

Sitemap: https://froentive.com/sitemap.xml