66.196.91.167 - - [01/Feb/2006:00:00:50 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
61.129.57.19 - - [01/Feb/2006:00:24:46 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)"
133.11.23.250 - - [01/Feb/2006:00:37:04 -0700] "GET /robots.txt HTTP/1.1" 200 33 "-" "Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)"
72.30.177.13 - - [01/Feb/2006:00:51:54 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
84.148.200.130 - - [01/Feb/2006:00:53:15 -0700] "GET /robots.txt HTTP/1.1" 304 - "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; MSIECrawler)"
68.142.251.170 - - [01/Feb/2006:01:03:26 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
65.55.246.117 - - [01/Feb/2006:01:37:42 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
207.68.146.104 - - [01/Feb/2006:03:59:38 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
64.5.245.27 - - [01/Feb/2006:04:22:14 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "genieBot ((http://64.5.245.11/faq/faq.html))"
68.142.251.170 - - [01/Feb/2006:05:42:53 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
66.196.91.130 - - [01/Feb/2006:06:19:35 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
66.196.91.136 - - [01/Feb/2006:06:27:08 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
66.196.91.137 - - [01/Feb/2006:07:06:29 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
65.55.246.117 - - [01/Feb/2006:07:14:16 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
66.196.91.167 - - [01/Feb/2006:07:49:58 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
59.160.131.33 - - [01/Feb/2006:09:20:31 -0700] "GET /robots.txt HTTP/1.0" 200 33 "http://www.watching-paint-dry.com/v7ndotcom-elursrebmem/" "Mozilla/5.0 (compatible; vermut +http://vermut.aol.com)"
61.129.57.29 - - [01/Feb/2006:09:40:54 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)"
66.249.66.43 - - [01/Feb/2006:10:00:21 -0700] "GET /robots.txt HTTP/1.1" 200 33 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
61.129.57.23 - - [01/Feb/2006:10:16:52 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)"
68.142.251.170 - - [01/Feb/2006:10:22:19 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
65.214.44.206 - - [01/Feb/2006:10:22:53 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)"
64.241.242.18 - - [01/Feb/2006:11:23:19 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "NutchCVS/0.05 (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)"
66.196.91.136 - - [01/Feb/2006:11:57:52 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
207.68.146.68 - - [01/Feb/2006:12:12:48 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
72.30.177.13 - - [01/Feb/2006:12:45:19 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
66.196.91.167 - - [01/Feb/2006:12:58:53 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
207.68.146.104 - - [01/Feb/2006:13:03:44 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
65.55.246.117 - - [01/Feb/2006:13:06:38 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
62.221.208.90 - - [01/Feb/2006:13:41:13 -0700] "GET /robots.txt HTTP/1.1" 200 33 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 1.1.4322; MSIECrawler)"
68.142.251.170 - - [01/Feb/2006:13:51:21 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
130.75.2.19 - - [01/Feb/2006:15:17:27 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Forschungsportal/0.8-dev (Testinstallation; http://www.forschungsportal.net/; nutch-agent@lucene.apache.org)"
68.142.251.170 - - [01/Feb/2006:16:27:01 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
143.81.47.250 - - [01/Feb/2006:16:34:42 -0700] "GET /robots.txt HTTP/1.0" 304 - "-" "CE-Preload"
72.30.177.13 - - [01/Feb/2006:17:17:53 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
64.13.136.2 - - [01/Feb/2006:17:45:35 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "WinkBot/0.06 (Wink.com search engine web crawler; http://www.wink.com/Wink:WinkBot; winkbot@wink.com)"
65.54.164.46 - - [01/Feb/2006:17:56:10 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
65.55.246.117 - - [01/Feb/2006:18:03:22 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
192.17.82.153 - - [01/Feb/2006:18:08:35 -0700] "GET /robots.txt HTTP/1.1" 200 33 "-" "Jakarta Commons-HttpClient/3.0-rc3"
209.237.238.224 - - [01/Feb/2006:18:31:36 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "ia_archiver"
207.46.98.125 - - [01/Feb/2006:18:57:03 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
66.196.91.136 - - [01/Feb/2006:19:52:30 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
66.196.91.167 - - [01/Feb/2006:19:59:57 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
207.68.146.62 - - [01/Feb/2006:20:23:08 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
66.249.64.36 - - [01/Feb/2006:20:41:04 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Googlebot/2.1 (+http://www.google.com/bot.html)"
131.84.1.2 - - [01/Feb/2006:21:04:58 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/4.0 (compatible; MSIE 4.01 compatible; Plumtree 4.5; Windows NT)"
66.196.91.124 - - [01/Feb/2006:21:20:26 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
66.249.71.40 - - [01/Feb/2006:22:32:40 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Googlebot/2.1 (+http://www.google.com/bot.html)"
64.13.136.2 - - [01/Feb/2006:22:35:26 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "WinkBot/0.06 (Wink.com search engine web crawler; http://www.wink.com/Wink:WinkBot; winkbot@wink.com)"
128.194.135.86 - - [01/Feb/2006:23:10:04 -0700] "GET /robots.txt HTTP/1.1" 200 33 "-" "IRLbot/2.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler)"
65.55.246.117 - - [01/Feb/2006:23:10:07 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"