## robots.txt 080408 #User-agent: * #Allow: / User-agent: * Disallow: /ct/nieuws/zoeken/artikel/ User-agent: Slurp Crawl-delay: 120 Disallow: /ct/nieuws/zoeken/artikel/ Disallow: /chip/nieuws/zoeken/artikel/ Disallow: /news/ User-agent: MSNBot Crawl-delay: 120 Disallow: /ct/nieuws/zoeken/artikel/ Disallow: /chip/nieuws/zoeken/artikel/ Disallow: /news/ User-agent: Teoma Crawl-delay: 120 Disallow: /ct/nieuws/zoeken/artikel/ Disallow: /chip/nieuws/zoeken/artikel/ Disallow: /news/ User-agent: Ask Jeeves Crawl-delay: 120 Disallow: /ct/nieuws/zoeken/artikel/ Disallow: /chip/nieuws/zoeken/artikel/ Disallow: /news/ User-agent: ShopWiki Disallow: / #User-agent: "Baiduspider+(+http://www.baidu.com/search/spider_jp.html)" User-agent: Baiduspider Disallow: / #User-agent: "Gigabot/3.0 (http://www.gigablast.com/spider.html)" User-agent: Gigabot Disallow: / #User-agent: "ia_archiver" User-agent: ia_archiver Disallow: / #User-agent: "Labrador/0.2; http://ir.dcs.gla.ac.uk/labrador; craigm@dcs.gla.ac.uk" User-agent: Labrador Disallow: / #User-agent: "larbin_2.6.3 (larbin2.6.3@unspecified.mail)" User-agent: larbin_2.6.3 Disallow: / #User-agent: "Mozilla/4.0 (compatible; Vagabondo/4.0Beta; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)" User-agent: Vagabondo Disallow: / #User-agent: "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)" User-agent: Exabot Disallow: / #User-agent: "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)" User-agent: ScoutJet Disallow: / #User-agent: "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)" User-agent: Yahoo! Slurp China Disallow: / #User-agent: "Mozilla/5.0 (Twiceler-0.9 http://www.cuill.com/twiceler/robot.html)" User-agent: Twiceler Disallow: / #User-agent: "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (http://www.voila.com/)" User-agent: VoilaBot BETA 1.2 Disallow: / #User-agent: "WebAlta Crawler/2.0 (http://www.webalta.net/ru/about_webmaster.html) (Windows; U; Windows NT 5.1; ru-RU)" User-agent: WebAlta Crawler Disallow: /