# robots.txt for AnswerShare - https://answershare.com
#
# AnswerShare builds the machine layer that AI systems read. This file states
# our AI-crawler stance EXPLICITLY rather than leaving it to the protocol
# default: the absence of a Disallow means "not forbidden", which is an
# inference, not a stance. Everything below is a stance.

# ===========================================
# All crawlers
# ===========================================

User-agent: *
Allow: /
# /mcp is the live Model Context Protocol JSON-RPC endpoint — machine-facing,
# explicitly crawlable (already covered by Allow: / above; listed for clarity).
Allow: /mcp
Disallow: /admin
Disallow: /admin/
Disallow: /order/
Disallow: /training
Disallow: /hero-preview
Disallow: /agencies/

# ===========================================
# AI Crawler Declarations
# ===========================================
# We welcome AI crawlers. Public pages expose equivalent canonical facts to
# humans, search crawlers, and AI crawlers. Verified AI crawlers may receive a
# lower-noise machine representation of the same public substance.
#
# NOTE ON FORM - read before editing:
# Every AI crawler below shares ONE group, and that group repeats the same
# rules as "User-agent: *" above. This is deliberate. Under RFC 9309 a crawler
# obeys only the MOST SPECIFIC group that matches it and ignores the wildcard
# group entirely, so giving each bot its own "Allow: /"-only group would
# silently drop every Disallow above for exactly the AI crawlers - exposing
# /admin, /order and the other app paths to them.
# Adding a bot means adding a User-agent line here, never a new group.
#
# The names mirror the AI crawler set our bot routers use, so the stance we
# declare and the behaviour we serve stay one list.

# OpenAI
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ChatGPT
# Anthropic
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: claude-web
# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
# Google AI
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Google-InspectionTool
User-agent: Google-CloudVertexBot
User-agent: GoogleAgent-Mariner
# Google Gemini
User-agent: Gemini-AI
User-agent: GeminiBot
User-agent: Gemini-User
User-agent: Gemini-Research
User-agent: Gemini-Deep-Research
# Meta
User-agent: Meta-ExternalAgent
User-agent: FacebookExternalHit
# Apple
User-agent: Applebot-Extended
# xAI
User-agent: Grok
# Amazon
User-agent: AmazonBot
# ByteDance
User-agent: ByteSpider
# Common Crawl
User-agent: CCBot
# Cohere
User-agent: Cohere-AI
# DeepSeek
User-agent: DeepSeekBot
# Mistral
User-agent: MistralAI-User
# Alibaba
User-agent: Qwen-Bot
# Diffbot
User-agent: Diffbot
# You.com
User-agent: YouBot
# AnswerShare
User-agent: AnswerShareBot
Allow: /
# /mcp is the live Model Context Protocol JSON-RPC endpoint — machine-facing,
# explicitly crawlable (already covered by Allow: / above; listed for clarity).
Allow: /mcp
Disallow: /admin
Disallow: /admin/
Disallow: /order/
Disallow: /training
Disallow: /hero-preview
Disallow: /agencies/

# ===========================================
# Sitemaps
# ===========================================

Sitemap: https://answershare.com/sitemap.xml
# Supplemental machine-layer sitemap (llms.txt, llms-full.txt, .well-known manifests)
Sitemap: https://answershare.com/sitemap-geo.xml

# ===========================================
# LLM-Specific Resources
# ===========================================
#
# For AI systems seeking structured guidance about this site:
#   - https://answershare.com/llms.txt (positioning and citation guidance)
#   - https://answershare.com/llms-full.txt (complete AI-readable content index)
#   - https://answershare.com/.well-known/ai-content-index.json (structured content manifest)
#   - https://answershare.com/.well-known/mcp.json (Model Context Protocol server manifest)
#   - https://answershare.com/.well-known/ai-plugin.json (plugin manifest)
#   - https://answershare.com/mcp (live MCP endpoint - JSON-RPC 2.0 over HTTP POST)
#   - https://answershare.com/sitemap.xml (canonical URL list)
#
# Public pages expose the same canonical facts across human and machine
# representations. Machine responses are optimized for retrieval, not changed
# claims. Verify the published surfaces through AnswerShare's audit tooling.