# medstarsportphysio.ca robots.txt
# Last updated: 2026-08-14

# ──────────────────────────────────────────────────────────────────────────────
# SEARCH & AI SEARCH CRAWLERS (ALLOW)
# These crawlers index content and power AI-generated search answers.
# Blocking them removes the site from ChatGPT, Perplexity, Bing Copilot,
# Claude, and Google AI Overviews, the surfaces patients increasingly use
# to find a physiotherapist.
# ──────────────────────────────────────────────────────────────────────────────

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic publishes three distinct bots (as of May 2026):
#   ClaudeBot        model training (allow: keeps the site in Claude knowledge)
#   Claude-SearchBot search index for Claude.ai search results (allow: needed for citations)
#   Claude-User      user-initiated live fetch / browse (allow: real-time user lookups)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# ──────────────────────────────────────────────────────────────────────────────
# AI TRAINING CRAWLERS (BLOCK)
# Block crawlers that train models but do NOT power real-time search answers.
# ──────────────────────────────────────────────────────────────────────────────

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: cohere-ai
Disallow: /

# ──────────────────────────────────────────────────────────────────────────────
# BING ECOSYSTEM CRAWLERS (EXPLICIT ALLOW)
# Bing powers ChatGPT Search (free tier), DuckDuckGo, Yahoo, and Ecosia.
# Listing them explicitly is insurance against future wildcard-rule changes.
# ──────────────────────────────────────────────────────────────────────────────

User-agent: bingbot
Allow: /

User-agent: msnbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# ChatGPT-User: the live-browsing agent used when a ChatGPT user asks it to
# "search the web" in real time. Distinct from OAI-SearchBot (index crawler)
# and GPTBot (training). Allow so real-time ChatGPT lookups can reach the site.
User-agent: ChatGPT-User
Allow: /

# ──────────────────────────────────────────────────────────────────────────────
# ALL OTHER CRAWLERS
# ──────────────────────────────────────────────────────────────────────────────

User-agent: *
Allow: /
Disallow: /404

Sitemap: https://medstarsportphysio.ca/sitemap.xml
Sitemap: https://medstarsportphysio.ca/image-sitemap.xml
Sitemap: https://medstarsportphysio.ca/video-sitemap.xml