# IA² Agency - robots.txt
# Updated: March 11, 2026

# Allow all agents by default
User-agent: *
Allow: /


# Block access to internal/private resources
Disallow: /.env
Disallow: /.git
Disallow: /node_modules
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /cache/

# Crawl delay for respectful crawling
Crawl-delay: 1

# Googlebot - accelerated crawling
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Bingbot configuration
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Respectful AI crawlers
User-agent: GPTBot
Allow: /
Crawl-delay: 2

User-agent: CCBot
Allow: /
Crawl-delay: 2

# Block aggressive/malicious bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Sitemap and usage guidelines
Sitemap: https://ia2.agency/sitemap.xml
Sitemap: https://ia2.agency/robots.txt

# LLM Guidelines
# For AI/LLM usage guidelines, see: https://ia2.agency/llms.txt