# robots.txt for seoforge.fr
#
# One shared rule group: every named AI crawler below PLUS the "*" wildcard.
# Per RFC 9309 a crawler obeys only the most specific group matching its user
# agent, so giving a bot its own group with a bare "Allow: /" would silently
# cancel the Disallow rules for that bot. Stacking the User-agent lines on a
# single group gives every crawler - AI or classic - identical rules.
#
# Each AI bot is listed deliberately (GEO: be visible and citable in AI answer
# engines). The owner can strike any single bot by moving its line to a
# dedicated group with "Disallow: /".

# GPTBot - OpenAI model-training crawler
User-agent: GPTBot
# OAI-SearchBot - ChatGPT search indexing
User-agent: OAI-SearchBot
# ChatGPT-User - fetches pages on behalf of ChatGPT users
User-agent: ChatGPT-User
# ClaudeBot - Anthropic training crawler
User-agent: ClaudeBot
# Claude-User - fetches pages on behalf of Claude users
User-agent: Claude-User
# anthropic-ai - legacy Anthropic crawler token
User-agent: anthropic-ai
# PerplexityBot - Perplexity search indexing
User-agent: PerplexityBot
# Perplexity-User - fetches pages on behalf of Perplexity users
User-agent: Perplexity-User
# Google-Extended - opt-in for Gemini training and grounding
User-agent: Google-Extended
# GoogleOther - Google research and product crawls
User-agent: GoogleOther
# CCBot - Common Crawl; feeds many AI training corpora (allowed by explicit owner decision)
User-agent: CCBot
# Amazonbot - Alexa and Amazon AI answers
User-agent: Amazonbot
# Applebot-Extended - opt-in for Apple Intelligence training
User-agent: Applebot-Extended
# meta-externalagent - Meta AI training crawler
User-agent: meta-externalagent
# Bytespider - ByteDance/TikTok AI crawler (known to ignore robots.txt; line is declarative)
User-agent: Bytespider
# All other crawlers
User-agent: *
Allow: /
# Non-public application areas
Disallow: /admin/
Disallow: /backoffice/
Disallow: /api/
Disallow: /inscription
Disallow: /switch-locale/
# HTML fragments of the speed-test tool (partials, not pages)
Disallow: /fr/test-vitesse-site-web/fragment
Disallow: /en/website-speed-test/fragment
# Post-submit confirmation pages (noindex, no crawl value)
Disallow: /fr/contact/confirmation
Disallow: /en/contact/confirmation
Disallow: /fr/nom-de-domaine-email-pro/confirmation
Disallow: /en/domain-name-business-email/confirmation
Disallow: /fr/test-vitesse-site-web/confirmation
Disallow: /en/website-speed-test/confirmation

Sitemap: https://seoforge.fr/sitemap.xml

# AI-readable site summary: https://seoforge.fr/llms.txt