# start of 'go away' User-agent: taptubot Disallow: / User-agent: discobot Disallow: / User-Agent: Slurp Disallow: / Crawl-delay: 25200 User-agent: Baiduspider Disallow: / Crawl-delay: 25200 User-agent: GurujiBot Disallow: / Crawl-delay: 25200 User-Agent: msnbot Disallow: / Disallow: /G* Disallow: /T* Disallow: /a* Disallow: /ca* Disallow: /cg* Disallow: /da* Disallow: /e* Disallow: /f* Disallow: /gel* Disallow: /im* Disallow: /me* Disallow: /n* Disallow: /z* # Only allows 1 -> 600 :-( Crawl-delay: 600 User-Agent: thunderstone Disallow: / Crawl-delay: 25200 User-Agent: T-H-U-N-D-E-R-S-T-O-N-E Disallow: / Crawl-delay: 25200 User-Agent: twiceler Disallow: / Crawl-delay: 25200 User-Agent: BioSearch Disallow: / Crawl-delay: 25200 # end of 'go away' User-agent: Googlebot-Image Disallow: / User-agent: googlebot Disallow: /_* Disallow: /2* Disallow: /asp* Disallow: /cg* Disallow: /f* Disallow: /ge* Disallow: /i* Disallow: /s* Disallow: /Ge* Crawl-delay: 25200 User-agent: Teoma Disallow: /_* Disallow: /2* Disallow: /asp* Disallow: /cgi* Disallow: /f* Disallow: /ge* Disallow: /i* Disallow: /s* Disallow: /Ge* Crawl-delay: 25200 # slow down the rest User-agent: * Disallow: /_* Disallow: /2* Disallow: /asp* Disallow: /cgi* Disallow: /f* Disallow: /ge* Disallow: /i* Disallow: /s* Disallow: /Ge* Crawl-delay: 25200