# Jacobian Engineering - robots.txt

# Allow all crawlers by default
User-agent: *
Allow: /

# Block paths that don't contribute to SEO
Disallow: /admin/
Disallow: /login/
Disallow: /dashboard/
Disallow: /private/
Disallow: /internal/

# Block query strings to prevent duplicate content
Disallow: /*?

# Block staging and test environments
Disallow: /staging/
Disallow: /dev/
Disallow: /test/

# Block form processing and confirmation pages
Disallow: /form-handler/
Disallow: /thank-you/
Disallow: /confirmation/

# Block file downloads from indexing
Disallow: /downloads/

# Bing - throttle crawl rate (Google ignores Crawl-delay; use Search Console)
User-agent: Bingbot
Crawl-delay: 1

# Block aggressive SEO crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: SemrushBot
Crawl-delay: 10

# Sitemaps
Sitemap: https://jacobianengineering.com/sitemap.xml
Sitemap: https://jacobianengineering.com/sitemap-resources.xml