# All robots will spider the domain User-agent: * Disallow: # Disallow directory /secure/ User-agent: * Disallow: /secure/ Crawl-delay: 60 # Disallow product pages User-agent: Googlebot Disallow: /*/product.aspx Disallow: /ProductSheet.aspx Disallow: /*/*.htm Disallow: /*/*/*.htm Allow: /*/*.html User-agent: Googlebot Disallow: /more-info.aspx User-agent: msnbot Crawl-delay: 60 User-agent: Slurp Crawl-delay: 60 User-agent: BecomeBot Crawl-Delay: 60 User-agent: ShopWiki Crawl-Delay: 60 User-agent: Googlebot Crawl-Delay: 60 User-agent: Googlebot-Mobile Crawl-Delay: 60 User-agent: Googlebot-Image Crawl-Delay: 60