# Robots.txt for exployt.ai
#
# /_framework/ holds the Blazor WebAssembly runtime: 100+ .wasm assemblies plus
# blazor.boot.json. Crawling it burns crawl budget on binaries that contain no
# readable content, so it is disallowed for every agent. Human visitors are
# unaffected — robots.txt does not gate the browser.
User-agent: *
Allow: /
# Public documentation: every RELEASED article is readable without a login. Human page per article:
# /documentation/{id}. Plain Markdown, JSON index and sitemap are served by the API host
# (https://exployt-api-prod.azurewebsites.net/wiki/{id}.md, /wiki/index.json, /wiki/sitemap.xml and the JSON read routes under
# /api/wiki/public), which has no robots restrictions. The Allow lines below keep exactly these routes open on this host as well;
# they are listed before the /api/ rule because the first matching rule wins for older crawlers.
Allow: /documentation
Allow: /wiki/
Allow: /api/wiki/public
Disallow: /admin/
Disallow: /account/
Disallow: /api/
Disallow: /_framework/

# AI / LLM crawlers are explicitly welcome to read and index this site so that AI
# assistants and AI search can describe exployt accurately. See /llms.txt for a
# concise, machine-readable overview.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Bytespider
User-agent: Meta-ExternalAgent
User-agent: cohere-ai
User-agent: CCBot
Allow: /
Allow: /documentation
Allow: /wiki/
Allow: /api/wiki/public
Disallow: /admin/
Disallow: /account/
Disallow: /api/
Disallow: /_framework/

# Machine-readable site overview for LLMs (https://llmstxt.org)
# LLMs: https://exployt.ai/llms.txt
# Trust: https://exployt.ai/trust.txt
# Security: https://exployt.ai/security.txt and https://exployt.ai/.well-known/security.txt
# Privacy: https://exployt.ai/privacy.txt
# Legal: https://exployt.ai/legal.txt
# Security FAQ: https://exployt.ai/faq-security.txt

# Documentation index: https://exployt-api-prod.azurewebsites.net/wiki/index.json

# Sitemap locations. The second one is generated from the released documentation entries (page and Markdown address of each); it lives on
# the API host and is listed here so search engines accept the exployt.ai page addresses in it (sitemaps.org cross-submission).
Sitemap: https://exployt.ai/sitemap.xml
Sitemap: https://exployt-api-prod.azurewebsites.net/wiki/sitemap.xml