65.55.233.146 - - [01/Apr/2006:00:08:48 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
207.46.98.123 - - [01/Apr/2006:00:33:20 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
66.147.154.3 - - [01/Apr/2006:00:33:50 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "http://www.almaden.ibm.com/cs/crawler   [fc9]"
68.142.251.165 - - [01/Apr/2006:01:19:42 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
207.68.146.68 - - [01/Apr/2006:01:32:56 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
66.196.90.252 - - [01/Apr/2006:01:43:49 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
68.142.251.165 - - [01/Apr/2006:01:57:17 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
207.68.146.68 - - [01/Apr/2006:02:31:54 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
68.142.251.165 - - [01/Apr/2006:02:46:08 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
66.196.90.252 - - [01/Apr/2006:03:03:56 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
131.107.0.85 - - [01/Apr/2006:03:05:52 -0700] "GET /robots.txt HTTP/1.1" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
68.142.251.165 - - [01/Apr/2006:03:22:49 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
72.30.177.15 - - [01/Apr/2006:03:41:19 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
24.197.210.3 - - [01/Apr/2006:03:43:25 -0700] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FunWebProducts; .NET CLR 1.1.4322; MSIECrawler)"
38.100.225.7 - - [01/Apr/2006:03:55:22 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "sproose/0.1-alpha (sproose crawler; http://www.sproose.com/bot.html; crawler@sproose.com)"
129.187.148.165 - - [01/Apr/2006:04:04:43 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "wwwster/1.4 (Beta, mailto:gue@cis.uni-muenchen.de)"
68.142.251.165 - - [01/Apr/2006:04:24:13 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
65.55.233.146 - - [01/Apr/2006:05:29:33 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
62.163.78.95 - - [01/Apr/2006:05:51:34 -0700] "GET /v7ndotcom-elursrebmem/results/robots/GET /robots.txt HTTP/1.0 HTTP/1.1" 404 238 "-" "Java/1.4.1_04"
62.163.78.95 - - [01/Apr/2006:05:51:35 -0700] "GET /v7ndotcom-elursrebmem/results/robots/GET /robots.txt HTTP/1.1 HTTP/1.1" 404 238 "-" "Java/1.4.1_04"
62.163.78.95 - - [01/Apr/2006:05:51:35 -0700] "GET /v7ndotcom-elursrebmem/results/robots/GET /v7ndotcom-elursrebmem/robots.txt HTTP/1.1 HTTP/1.1" 404 238 "-" "Java/1.4.1_04"
68.142.251.165 - - [01/Apr/2006:05:59:13 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
68.142.251.165 - - [01/Apr/2006:07:07:32 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
64.4.8.100 - - [01/Apr/2006:07:18:54 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
65.55.246.117 - - [01/Apr/2006:08:02:43 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
24.202.35.138 - - [01/Apr/2006:08:32:53 -0700] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; MSIECrawler)"
207.68.146.68 - - [01/Apr/2006:09:25:31 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
38.100.225.8 - - [01/Apr/2006:10:01:50 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "sproose/0.1-alpha (sproose crawler; http://www.sproose.com/bot.html; crawler@sproose.com)"
66.196.91.124 - - [01/Apr/2006:10:52:58 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
66.196.91.120 - - [01/Apr/2006:10:53:30 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
207.46.98.125 - - [01/Apr/2006:10:57:37 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
72.30.177.15 - - [01/Apr/2006:11:02:31 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
222.5.230.191 - - [01/Apr/2006:11:34:19 -0700] "GET /robots.txt HTTP/1.1" 200 69 "-" "RSS_READER (mctwist@mail.dr-k.info)"
24.202.35.138 - - [01/Apr/2006:12:36:33 -0700] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; MSIECrawler)"
64.241.242.18 - - [01/Apr/2006:12:37:42 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "NutchCVS/0.05 (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)"
131.107.0.96 - - [01/Apr/2006:12:41:37 -0700] "GET /robots.txt HTTP/1.1" 200 69 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
66.147.154.3 - - [01/Apr/2006:12:55:48 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "http://www.almaden.ibm.com/cs/crawler   [hc3]"
64.4.8.100 - - [01/Apr/2006:13:13:39 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
65.55.233.146 - - [01/Apr/2006:13:25:13 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
72.30.177.15 - - [01/Apr/2006:13:28:19 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
65.55.246.117 - - [01/Apr/2006:13:45:11 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
66.196.65.36 - - [01/Apr/2006:13:55:37 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
207.241.229.168 - - [01/Apr/2006:14:02:13 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "ia_archiver-web.archive.org"
65.55.233.146 - - [01/Apr/2006:14:20:01 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
24.202.35.138 - - [01/Apr/2006:15:01:28 -0700] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; MSIECrawler)"
68.142.251.165 - - [01/Apr/2006:15:20:39 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
38.100.225.3 - - [01/Apr/2006:15:22:23 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "sproose/0.1-alpha (sproose crawler; http://www.sproose.com/bot.html; crawler@sproose.com)"
64.4.8.100 - - [01/Apr/2006:15:24:23 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
207.68.146.68 - - [01/Apr/2006:15:24:42 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
24.202.35.138 - - [01/Apr/2006:16:19:10 -0700] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; MSIECrawler)"
66.196.91.124 - - [01/Apr/2006:16:27:35 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
68.142.251.165 - - [01/Apr/2006:16:47:19 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
65.55.233.146 - - [01/Apr/2006:18:02:33 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
69.115.241.158 - - [01/Apr/2006:18:09:43 -0700] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; MSIECrawler)"
84.147.203.149 - - [01/Apr/2006:18:10:19 -0700] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.10) Gecko/20050724 Firefox/1.0.6"
202.156.6.91 - - [01/Apr/2006:18:12:44 -0700] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)"
207.68.146.68 - - [01/Apr/2006:18:53:08 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
129.187.148.165 - - [01/Apr/2006:19:30:14 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "wwwster/1.4 (Beta, mailto:gue@cis.uni-muenchen.de)"
65.55.235.145 - - [01/Apr/2006:19:38:01 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
68.142.251.165 - - [01/Apr/2006:19:38:01 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
207.46.98.80 - - [01/Apr/2006:19:46:00 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
65.55.246.117 - - [01/Apr/2006:19:56:10 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
85.124.169.41 - - [01/Apr/2006:19:58:07 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)"
207.68.146.62 - - [01/Apr/2006:20:03:29 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
38.100.225.8 - - [01/Apr/2006:20:14:25 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "sproose/0.1-alpha (sproose crawler; http://www.sproose.com/bot.html; crawler@sproose.com)"
212.46.126.104 - - [01/Apr/2006:20:50:40 -0700] "GET /robots.txt HTTP/1.0" 200 69 "http://www.watching-paint-dry.com:80/" "yacy (www.yacy.net; v20040602; i386 Linux 2.6.15.4.k7.g4.acpi.apic; java 1.5.0_05; Europe/de)"
66.196.90.252 - - [01/Apr/2006:21:18:30 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
66.94.233.119 - - [01/Apr/2006:21:43:02 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Yahoo-MMCrawler/3.x (mms dash mmcrawler dash support at yahoo dash inc dot com)"
68.142.251.165 - - [01/Apr/2006:21:45:45 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
24.202.35.138 - - [01/Apr/2006:22:05:19 -0700] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; MSIECrawler)"
66.249.65.177 - - [01/Apr/2006:22:11:48 -0700] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
64.4.8.100 - - [01/Apr/2006:22:17:27 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
66.196.90.252 - - [01/Apr/2006:22:41:46 -0700] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
24.73.161.18 - - [01/Apr/2006:23:55:01 -0700] "GET /v7ndotcom-elursrebmem/results/robots/GET /robots.txt HTTP/1.0 HTTP/1.1" 404 238 "-" "Java/1.4.1_04"
24.73.161.18 - - [01/Apr/2006:23:55:01 -0700] "GET /v7ndotcom-elursrebmem/results/robots/GET /robots.txt HTTP/1.1 HTTP/1.1" 404 238 "-" "Java/1.4.1_04"
24.73.161.18 - - [01/Apr/2006:23:55:01 -0700] "GET /v7ndotcom-elursrebmem/results/robots/GET /v7ndotcom-elursrebmem/robots.txt HTTP/1.1 HTTP/1.1" 404 238 "-" "Java/1.4.1_04"