User-agent: * psbot
Disallow: /cig-bin/
Disallow: /images/
Disallow: /scripts/
Disallow: /ws_ftp.log
Disallow: /xhis-bkm.htm
Disallow: /zhis-bkm.htm
Disallow: /zpages/
Disallow: /zseminar/

urllist.txt


# Must have 2 carrage returns to properly terminate file otherwise robot can
# ignore "robots.txt" file.
#
# Everything you wanted to know about robots.txt http://www.robotstxt.org/wc/norobots.html or
#  http://www.robotstxt.org/orig.html 
# "Any number of agent id(s) can be placed on the User-Agent line so long as
# they are separated by white space (WS), but the User-Agent line must have at
# least one agent id."
#
# Added ``psbot`` cause it was messing up Urchin v3.0 stats. 1/20/04
# Disallow: /stats/   removed 2/19/02 I think this caused password pbms. w/Bob
# Disallow: /scripts/  2/20/02 ; re-installed 3/4/02
# Disallow: /1webdir.htm  removed all these 3/1/01
# Disallow: /about/# Disallow: /affiliat/# Disallow: /coop-min/# Disallow: /howto/# Disallow: /liberty/# Disallow: /states/# Disallow: /steward/# Disallow: /usa/# Disallow: /youcando/
# Disallow: /zseminar/  removed this 6/21/01 having pbms w/slide-show. ; re-installed 3/02
# 3/17/02 Disallow: /postinfo.html Disallow: /_private/Disallow: /_vti_cnf/Disallow: /_vti_inf.html Disallow: /_vti_log/Disallow: /_vti_pvt/Disallow: /_vti_txt/
# 7/31/02 was User-agent: * ip3000.com petersnews.com cyveillance.com 216.130.69.194