# robots.txt for opengrants.io
# All crawlers welcome unless listed otherwise below.
# AI/LLM guidance: https://opengrants.io/llms.txt
# Extended corpus: https://opengrants.io/llms-full.txt

User-agent: *
# Content usage preferences (https://contentsignals.org/). Repeated per group because a
# crawler matches its most specific group and ignores User-agent: * entirely.
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /admin/
Disallow: /learn/lesson-content/

# --- AI crawlers: explicitly allowed ---
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin/
Disallow: /learn/lesson-content/

User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin/
Disallow: /learn/lesson-content/

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin/
Disallow: /learn/lesson-content/

User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin/
Disallow: /learn/lesson-content/

User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin/
Disallow: /learn/lesson-content/

User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin/
Disallow: /learn/lesson-content/

User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin/
Disallow: /learn/lesson-content/

User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin/
Disallow: /learn/lesson-content/

User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin/
Disallow: /learn/lesson-content/

User-agent: Amazonbot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin/
Disallow: /learn/lesson-content/

User-agent: Meta-ExternalAgent
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin/
Disallow: /learn/lesson-content/

User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin/
Disallow: /learn/lesson-content/

# --- existing exclusions (carried over from previous rules) ---
User-agent: AdsBot
Disallow: /

Sitemap: https://opengrants.io/sitemap-index.xml