# Allow all search engines
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/admin/
Disallow: /(payload)/
Crawl-delay: 1

# Explicit AI crawler policy
User-agent: GPTBot
Allow: /
Crawl-delay: 5

User-agent: ClaudeBot
Allow: /
Crawl-delay: 5

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 2

User-agent: GoogleExtended
Allow: /
Crawl-delay: 5

User-agent: Google-Extended
Allow: /
Crawl-delay: 5

User-agent: GoogleOther
Allow: /
Crawl-delay: 2

# Sitemap
Sitemap: https://insightfulpipe.com/sitemap.xml

# Agent resources
# llms.txt: https://insightfulpipe.com/llms.txt
# llms-full.txt: https://insightfulpipe.com/llms-full.txt