User-agent: Wget Disallow: /money/ Disallow: /finance/ Disallow: /cgi-bin/ Disallow: /cgi-perl/ Disallow: /~mschmid/ Disallow: /~joerg/ Disallow: /webalizer/ User-agent: * Disallow: /money/ Disallow: /finance/ Disallow: /cgi-bin/ Disallow: /cgi-perl/ Disallow: /~mschmid/ Disallow: /pub/ Disallow: /webalizer/ #User-agent: Mozilla/5.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html) #Disallow: / #User-agent: Slurp #Disallow: / #User-agent: TurnitinBot/1.4 http://www.turnitin.com/robot/crawlerinfo.html #Disallow: / #User-agent: TurnitinBot #Disallow: / #User-agent: bumblebee/1.0 (bumblebee@relevare.com; http://www.relevare.com/) #Disallow: / #User-agent: psbot #Disallow: / #User-agent: fast #Disallow: / #User-agent: ZyBorg #Disallow: / #User-agent: ia_archiver #Disallow: / #User-agent: mirago #Disallow: / #User-agent: Mirago #Disallow: / #User-Agent: Mercator #Disallow: /