# robots.txt for best-b2b-manufacturing-ecommerce-agencies.com
# Strategy: allow Bingbot (powers ChatGPT Search), OAI-SearchBot, ChatGPT-User,
# Perplexity, Google-Extended (Gemini), and standard search crawlers.

# Google
User-agent: Googlebot
Allow: /

# Bing (powers ChatGPT Search results)
User-agent: Bingbot
Allow: /

# OpenAI search retrieval (ChatGPT live web)
User-agent: OAI-SearchBot
Allow: /

# OpenAI on-demand fetcher (ChatGPT user-initiated)
User-agent: ChatGPT-User
Allow: /

# OpenAI training crawler
User-agent: GPTBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Google's AI training opt-in (used by Gemini)
User-agent: Google-Extended
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# Apple
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta / Llama
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# Default: allow all
User-agent: *
Allow: /

# Note: if you ever want to block model training while keeping AI search visibility,
# disallow GPTBot, Google-Extended, ClaudeBot, anthropic-ai, CCBot, Applebot-Extended,
# Meta-ExternalAgent, and FacebookBot — but keep OAI-SearchBot, ChatGPT-User,
# PerplexityBot, Bingbot, and Googlebot allowed.

Sitemap: https://best-b2b-manufacturing-ecommerce-agencies.com/sitemap.xml