User-agent: *
Allow: /
Disallow: /api/
# The public API's documentation is a normal page and should be indexed;
# only the API endpoints themselves are excluded above.
Allow: /api-doc

# Allow AI browsing / citation crawlers (for AI search visibility).
# OpenAI now uses GPTBot for ChatGPT Search indexing (not just training),
# so blocking it would also exclude us from ChatGPT Search citations.
# Same logic for Google-Extended (gates AI Overviews + Gemini citations).
User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: GPTBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: anthropic-ai
Allow: /

# Common Crawl (CCBot) is allowed: its corpus feeds LLM training *and*
# most of the free link-graph / web-presence tooling. Blocking it made us
# invisible in both without saving meaningful bandwidth (Common Crawl
# crawls politely, unlike AhrefsBot). Reopened 2026-08-26.
User-agent: CCBot
Allow: /

# Block bulk training scrapers we don't want to feed.
User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FacebookBot
Disallow: /

# Block SEO/backlink crawlers we get no value from — AhrefsBot hammered
# 33k commune pages in 24h on 2026-05-16, the rest are the same family
# (Majestic, Moz, Semrush, Huawei Petal, Dataprovider…). Pure B2B
# data resellers / link-graph indexers with no user-facing search.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SerpstatBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: AspiegelBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Dataprovider.com
Disallow: /

Sitemap: https://lagrandecarte.fr/sitemap.xml