# robots.txt for http://www.hhs.gov
# robots.txt for http://www.os.dhhs.gov
# robots.txt for http://www.dhhs.gov
# robots.txt for http://os.dhhs.gov:80

user-agent: *  # directed to all spiders, not just Scooter
Disallow: /CGI/
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /masters/
Disallow: /search/
Disallow: /test/
Disallow: /planguide/
Hit-rate: 30
# wait 30 seconds before starting a new URL request default=30
Visiting-hours: 01:00EST-05:00EST
# index this site between 1AM - 5AM EST
Concurrent-hits: 2
# limit concurrent active URLs to 2 for each index server