User-agent: gsa-crawler-stsci
Allow: /

User-agent: Mozilla/4.0 (compatible; T-H-U-N-D-E-R-S-T-O-N-E) - search.stsci.edu
Allow: /institute/center/information/streaming/

User-agent: *
Disallow: /ftp/cdbs/
Disallow: /ftp/observing/phase2_finder_charts/
Disallow: /cgi-bin/spike-patch-request
Disallow: /dbsa/doc/address-change-request.html
Disallow: /dbsa/doc/new-acct-requests.html
Disallow: /dbsa/doc/sqr-requests.html
Disallow: /dbsa/doc/admin/travel-request.html
Disallow: /support/Status/bb
Disallow: /security
Disallow: /stsearch
Disallow: /support/Status/CISDevents
Disallow: /support/Status/ARSystem
Disallow: /cgi-bin/texis
Disallow: /internal
Disallow: /institute/WebSite/
Disallow: /instruments/observatory/dq/0527_97/
Disallow: /WebSearch?printable=1
Disallow: /svn
Disallow: /trac
Disallow: /phpbb
Disallow: /hst/observatory/cdbs/SIfileInfo
#
# In support of CNSHD198215
Disallow: /science/goods/co-i.html
Disallow: /astroweb/
Disallow: /ftp/observing/phase2_finder_charts/
Disallow: /institute/org/ot/Governance/ppt/ 
Disallow: /wiki/
Disallow: /ARSystem/
Disallow: /cgi-bin/get-proposal-info
Disallow: /support/Status/bb
Disallow: /hst/metrics/
#
#Block crawlers from indexing the handbooks.
Disallow: /instruments/acs/documents/handbooks/
Disallow: /hst/nicmos/documents/handbooks/v5/
Disallow: /hst/stis/documents/handbooks/cycle11/
#
#Block crawlers from the webcasts stats
Disallow: /ts/stats/
Disallow: /webstats/
Disallow: /ts/webcasting/
Disallow: /institute/center/information/streaming/

#Slow down web crawlers
Crawl-delay: 5

#Remove outdated pages from google
User-agent: Googlebot
Disallow: /stsci/service/wsf

#Prevent bots from running instrument reference queries
Disallow: /hst/observatory/cdbs/SIfileInfo