65.55.246.117 - - [25/Jan/2006:00:54:00 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
217.212.224.162 - - [25/Jan/2006:01:14:46 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "psbot/0.1 (+http://www.picsearch.com/bot.html)"
66.196.91.173 - - [25/Jan/2006:01:14:55 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
217.212.224.147 - - [25/Jan/2006:01:15:50 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "psbot/0.1 (+http://www.picsearch.com/bot.html)"
219.142.118.62 - - [25/Jan/2006:01:19:52 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "-"
69.248.26.83 - - [25/Jan/2006:01:32:43 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Comrite/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)"
210.202.75.129 - - [25/Jan/2006:01:33:29 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "OmniFind"
217.212.224.159 - - [25/Jan/2006:01:48:01 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "psbot/0.1 (+http://www.picsearch.com/bot.html)"
72.30.177.14 - - [25/Jan/2006:02:59:16 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
65.54.164.47 - - [25/Jan/2006:03:13:24 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
133.11.23.250 - - [25/Jan/2006:03:48:42 -0700] "GET /robots.txt HTTP/1.1" 200 33 "-" "Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)"
133.11.23.250 - - [25/Jan/2006:04:28:09 -0700] "GET /robots.txt HTTP/1.1" 200 33 "-" "Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)"
207.68.146.104 - - [25/Jan/2006:05:03:45 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
65.214.44.206 - - [25/Jan/2006:05:25:36 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)"
66.196.91.136 - - [25/Jan/2006:05:42:26 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
207.46.98.126 - - [25/Jan/2006:05:53:14 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
66.196.90.15 - - [25/Jan/2006:05:53:54 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
65.55.246.117 - - [25/Jan/2006:07:10:51 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
66.196.91.136 - - [25/Jan/2006:07:19:10 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
65.54.164.47 - - [25/Jan/2006:07:26:31 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
68.142.251.156 - - [25/Jan/2006:07:57:05 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
207.46.98.126 - - [25/Jan/2006:09:56:51 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
66.196.91.173 - - [25/Jan/2006:10:46:10 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
66.196.90.15 - - [25/Jan/2006:11:04:07 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
64.94.163.137 - - [25/Jan/2006:11:56:40 -0700] "GET /robots.txt HTTP/1.1" 200 33 "-" "Jakarta Commons-HttpClient/3.0"
65.55.246.117 - - [25/Jan/2006:12:01:49 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
64.94.163.150 - - [25/Jan/2006:12:20:02 -0700] "GET /robots.txt HTTP/1.1" 200 33 "-" "Jakarta Commons-HttpClient/3.0"
65.54.164.47 - - [25/Jan/2006:12:21:35 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
64.94.163.152 - - [25/Jan/2006:12:29:32 -0700] "GET /robots.txt HTTP/1.1" 200 33 "-" "Jakarta Commons-HttpClient/3.0"
207.68.146.104 - - [25/Jan/2006:14:03:59 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
66.196.91.136 - - [25/Jan/2006:14:58:38 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
65.54.164.47 - - [25/Jan/2006:15:20:27 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
207.46.98.126 - - [25/Jan/2006:15:24:44 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
195.228.227.98 - - [25/Jan/2006:15:37:39 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)"
220.218.159.50 - - [25/Jan/2006:16:55:17 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)"
66.147.154.3 - - [25/Jan/2006:17:10:52 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "http://www.almaden.ibm.com/cs/crawler   [hc3]"
65.54.164.47 - - [25/Jan/2006:17:24:43 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
72.30.177.14 - - [25/Jan/2006:18:07:49 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
66.196.91.124 - - [25/Jan/2006:19:14:30 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
193.47.80.36 - - [25/Jan/2006:19:20:36 -0700] "GET /robots.txt HTTP/1.1" 200 33 "-" "Exabot/2.0"
24.97.214.158 - - [25/Jan/2006:19:54:22 -0700] "GET /robots.txt HTTP/1.1" 200 33 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)"
65.55.246.117 - - [25/Jan/2006:19:58:18 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
66.196.91.136 - - [25/Jan/2006:20:04:56 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
207.46.98.126 - - [25/Jan/2006:20:58:32 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
66.249.64.15 - - [25/Jan/2006:21:07:10 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Googlebot/2.1 (+http://www.google.com/bot.html)"
84.63.13.248 - - [25/Jan/2006:21:35:29 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (X11; U; SunOS sun4u; en-US; rv:1.7.8) Gecko/20050512 Firefox/1.0.4"
65.54.164.47 - - [25/Jan/2006:21:46:45 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
207.68.146.62 - - [25/Jan/2006:22:20:49 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
216.220.192.132 - - [25/Jan/2006:22:21:34 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "NutchCVS/0.7 (Experimental Nutch; host=e-0-21)"
66.196.91.173 - - [25/Jan/2006:23:47:14 -0700] "GET /robots.txt HTTP/1.0" 200 33 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"