# This file tells web-crawling robots what not to look at
User-agent: *
Disallow: /private
Disallow: /cgi-bin
Disallow: /logs/
Disallow: /cgi-pub
Disallow: /~
Disallow: /htdocs
# disallow Slurp (inktomi) accesses to whole server
# it is accessing our site more than the advertised once/min,
# sometimes many times per second. This coincides with malloc
# errors on the web server - LMengel 9/24/2000
User-agent: Slurp
Disallow: /