Sitemap: http://www.thesitebox.com/sitemap.xml


User-Agent: *

Crawl-delay: 10         #number of seconds to wait between successive requests to the same server
Request-rate: 1/5       # maximum rate is one page every 5 seconds
Visit-time: 2200-0600   # only visit between 06:00 and 08:45 UTC (GMT)

Disallow: /images/
Disallow: /IMAGES/
Disallow: /Images/

Disallow: /files/
Disallow: /Files/
Disallow: /FILES/

Disallow: *ImageService.axd

User-agent: Slurp
Disallow: /Product
Disallow: /product
Disallow: /PRODUCT