# Allow only some web crawlers to access the entire website
# Disallow: /en/tech should remove the AJAX product detail pages

User-agent: Googlebot
Disallow: /en/tech/
Disallow: */product-datasheet/

User-agent: MSNBot
Disallow: /en/tech/
Disallow: */product-datasheet/

User-agent: *
Disallow: /
Disallow: /en/tech/
Disallow: */product-datasheet/

# Set a crawl delay
Crawl-delay: 60

# Exclude specific directories
Disallow: /cgi-bin/
Disallow: /en/tech/
Disallow: */product-datasheet/

User-agent: AhrefsSiteAudit
Disallow: /