# robots.txt for http://www.nsf.gov/
# Change history:

#NSF K2
User-agent: k2spider
Disallow: /cgi-bin/ 
Disallow: /newawardsearch/

User-agent: LinkChecker/4.7
Disallow: /cgi-bin/
Disallow: /awardsearch/
Disallow: /newawardsearch/

User-agent: LinkChecker/4.8
Disallow: /cgi-bin/
Disallow: /awardsearch/
Disallow: /newawardsearch/

User-agent: vspider
Disallow: /cgi-bin/   
Disallow: /stats/    
Disallow: /home/nsforg/	
Disallow: /awards/
Disallow: /pubsys/data/     
Disallow: /search97cgi/  
Disallow: /seind98/topdemo.htm
Disallow: /nsf99338/topdemo.htm 
Disallow: /home/ebulletin/archive/ 
Disallow: /sbe/srs/start.htm
Disallow: /web/          
Disallow: /geo/
Disallow: /eng/
Disallow: /home/crssprgm/igert/survey/
Disallow: /staff/
Disallow: /ads-cgi/
Disallow: /awardsearch/
Disallow: /newawardsearch/

User-agent: *           
Disallow: /cgi-bin/    
Disallow: /nsf99338/topdemo.htm 
Disallow: /home/ebulletin/archive/ 
Disallow: /home/crssprgm/igert/survey/
Disallow: /staff/
Disallow: /ads-cgi/
Disallow: /awardsearch/
Crawl-delay: 5
Disallow: /sbe_2020/pdfs/
Disallow: /sbe/sbe_2020/2020_pdfs
Disallow: /newawardsearch/