66.154.103.137 - - [05/May/2006:00:04:52 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Gigabot/2.0/gigablast.com/spider.html" 64.71.164.125 - - [05/May/2006:00:05:07 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Krugle/Krugle,Nutch/0.8+ (Krugle web crawler; http://www.krugle.com/crawler/info.html; webcrawler@krugle.com)" 72.30.177.12 - - [05/May/2006:00:18:09 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 148.235.159.234 - - [05/May/2006:00:18:10 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MSIECrawler)" 213.114.34.247 - - [05/May/2006:00:39:08 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "WebCrawler (internet@bredband.net)" 82.135.28.34 - - [05/May/2006:01:06:59 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "DataparkSearch/4.37 (+http://www.dataparksearch.org/)" 72.30.61.17 - - [05/May/2006:01:15:38 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 66.147.154.3 - - [05/May/2006:01:17:04 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "http://www.almaden.ibm.com/cs/crawler [fc9]" 194.209.141.156 - - [05/May/2006:01:21:23 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.0.1) Gecko/20060111 Firefox/1.5.0.1" 211.144.32.63 - - [05/May/2006:01:28:12 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin@sina.com)" 72.30.128.226 - - [05/May/2006:01:57:43 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 211.144.32.63 - - [05/May/2006:02:15:13 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin@sina.com)" 68.142.251.178 - - [05/May/2006:02:22:58 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 211.144.32.63 - - [05/May/2006:02:27:44 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin@sina.com)" 211.144.32.63 - - [05/May/2006:02:38:30 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin@sina.com)" 72.30.128.226 - - [05/May/2006:03:11:20 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 207.67.117.170 - - [05/May/2006:03:19:17 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)" 68.142.251.178 - - [05/May/2006:03:28:47 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 24.21.116.20 - - [05/May/2006:03:40:42 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MSIECrawler)" 147.69.95.243 - - [05/May/2006:03:45:06 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FunWebProducts; MSIECrawler)" 66.196.90.252 - - [05/May/2006:04:14:37 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 72.30.61.17 - - [05/May/2006:04:14:37 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 207.68.188.242 - - [05/May/2006:04:23:01 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 65.214.44.187 - - [05/May/2006:04:27:26 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)" 72.30.128.226 - - [05/May/2006:04:35:35 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 65.55.252.50 - - [05/May/2006:04:46:32 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 216.227.117.35 - - [05/May/2006:04:50:06 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; MSIECrawler)" 64.4.8.100 - - [05/May/2006:05:15:45 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 72.30.128.226 - - [05/May/2006:05:25:22 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 72.30.61.17 - - [05/May/2006:05:33:49 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 66.80.248.146 - - [05/May/2006:05:38:33 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)" 63.247.91.202 - - [05/May/2006:05:45:04 -0600] "GET /robots.txt HTTP/1.0" 200 69 "http://www.watching-paint-dry.com/v7ndotcom-elursrebmem/" "Mozilla/5.0 (compatible; heritrix/1.6.0 +http://www.creativeandlogics.com/)" 66.196.90.252 - - [05/May/2006:05:53:19 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 68.142.251.178 - - [05/May/2006:05:53:42 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 72.30.177.12 - - [05/May/2006:06:09:08 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 207.46.98.125 - - [05/May/2006:06:22:00 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 72.30.128.226 - - [05/May/2006:06:28:34 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 24.66.94.140 - - [05/May/2006:07:36:48 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FunWebProducts; .NET CLR 1.1.4322; .NET CLR 2.0.50727; MSIECrawler)" 84.103.175.182 - - [05/May/2006:07:49:34 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)" 207.68.146.62 - - [05/May/2006:08:04:20 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 66.196.90.252 - - [05/May/2006:08:17:40 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 131.247.225.39 - - [05/May/2006:08:25:37 -0600] "HEAD /robots.txt HTTP/1.0" 200 - "-" "Mozilla/4.0 (compatible; GPU p2p crawler http://gpu.sourceforge.net/search_engine.php)" 131.247.225.39 - - [05/May/2006:08:25:37 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/4.0 (compatible; GPU p2p crawler http://gpu.sourceforge.net/search_engine.php)" 69.209.34.37 - - [05/May/2006:08:52:31 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; YPC 3.2.0; FunWebProducts; .NET CLR 1.1.4322; .NET CLR 2.0.50727; MSIECrawler)" 72.30.128.226 - - [05/May/2006:08:58:11 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 207.68.146.56 - - [05/May/2006:09:16:38 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 84.103.175.182 - - [05/May/2006:09:22:46 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)" 202.153.132.101 - - [05/May/2006:09:45:25 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)" 72.30.61.17 - - [05/May/2006:09:57:10 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 72.30.128.226 - - [05/May/2006:10:04:03 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 65.55.252.50 - - [05/May/2006:10:05:16 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 66.196.91.124 - - [05/May/2006:10:27:14 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 66.27.197.88 - - [05/May/2006:10:31:01 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727; MSIECrawler)" 66.196.90.252 - - [05/May/2006:10:34:48 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 66.94.233.119 - - [05/May/2006:10:45:34 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Yahoo-MMCrawler/3.x (mms dash mmcrawler dash support at yahoo dash inc dot com)" 72.30.61.17 - - [05/May/2006:10:58:28 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 72.30.128.226 - - [05/May/2006:11:21:14 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 207.68.146.62 - - [05/May/2006:11:54:09 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 85.124.31.71 - - [05/May/2006:12:05:31 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)" 207.68.146.56 - - [05/May/2006:12:14:37 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 213.114.34.247 - - [05/May/2006:12:28:40 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "WebCrawler (internet@bredband.net)" 66.74.19.246 - - [05/May/2006:12:33:35 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; MSIECrawler)" 72.30.177.12 - - [05/May/2006:12:45:56 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 72.30.128.226 - - [05/May/2006:12:47:43 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 207.68.188.242 - - [05/May/2006:12:52:07 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 82.157.199.158 - - [05/May/2006:13:14:29 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; MSIECrawler)" 201.232.29.23 - - [05/May/2006:13:17:26 -0600] "GET /robots.txt HTTP/1.1" 304 - "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; IE6 B 04312002 MED 525pm; MSIECrawler)" 207.68.146.68 - - [05/May/2006:13:23:46 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 65.214.44.144 - - [05/May/2006:13:32:22 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)" 72.30.61.17 - - [05/May/2006:13:34:50 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 219.142.118.12 - - [05/May/2006:13:41:25 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "-" 72.30.128.226 - - [05/May/2006:14:37:35 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 68.227.69.123 - - [05/May/2006:14:38:46 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3" 72.64.65.249 - - [05/May/2006:14:38:47 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-US; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3" 65.205.209.10 - - [05/May/2006:14:42:42 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3" 213.84.24.41 - - [05/May/2006:14:48:29 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; nl; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3" 62.253.128.12 - - [05/May/2006:14:48:42 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3" 12.46.239.17 - - [05/May/2006:14:58:52 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3" 200.1.171.138 - - [05/May/2006:15:13:08 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MSIECrawler)" 64.230.0.67 - - [05/May/2006:15:17:52 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3" 202.160.183.218 - - [05/May/2006:15:19:29 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)" 201.232.29.23 - - [05/May/2006:15:20:41 -0600] "GET /robots.txt HTTP/1.1" 304 - "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; IE6 B 04312002 MED 525pm; MSIECrawler)" 207.68.146.68 - - [05/May/2006:15:21:03 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 72.30.61.84 - - [05/May/2006:15:28:05 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 138.11.0.153 - - [05/May/2006:15:36:23 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.2) Gecko/20060429 Firefox/1.5.0.2" 66.36.128.89 - - [05/May/2006:15:57:48 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3" 143.81.252.13 - - [05/May/2006:16:04:08 -0600] "GET /robots.txt HTTP/1.0" 304 - "-" "CE-Preload" 68.142.251.168 - - [05/May/2006:16:12:09 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 82.244.80.46 - - [05/May/2006:16:17:13 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:13 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:13 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:13 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:15 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:15 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:15 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:15 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:15 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:15 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:15 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:15 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:15 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:15 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:15 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 82.244.80.46 - - [05/May/2006:16:17:15 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.13) Gecko/20060418 Firefox/1.0.8 (Ubuntu package 1.0.8)" 72.30.128.226 - - [05/May/2006:16:21:11 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 85.124.31.71 - - [05/May/2006:16:23:46 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)" 68.142.251.159 - - [05/May/2006:16:30:24 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 24.238.204.58 - - [05/May/2006:16:33:07 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3" 72.30.177.12 - - [05/May/2006:16:36:33 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 65.55.252.50 - - [05/May/2006:16:41:18 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 64.247.118.39 - - [05/May/2006:17:04:09 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.2) Gecko/20060502 Ubuntu/dapper Firefox/1.5.0.2" 72.30.128.226 - - [05/May/2006:17:07:06 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 62.194.60.29 - - [05/May/2006:17:13:13 -0600] "GET /v7ndotcom-elursrebmem/results/robots/GET /robots.txt HTTP/1.0 HTTP/1.1" 404 238 "-" "Java/1.4.1_04" 62.194.60.29 - - [05/May/2006:17:13:13 -0600] "GET /v7ndotcom-elursrebmem/results/robots/GET /robots.txt HTTP/1.1 HTTP/1.1" 404 238 "-" "Java/1.4.1_04" 62.194.60.29 - - [05/May/2006:17:13:13 -0600] "GET /v7ndotcom-elursrebmem/results/robots/GET /v7ndotcom-elursrebmem/robots.txt HTTP/1.1 HTTP/1.1" 404 238 "-" "Java/1.4.1_04" 207.68.146.56 - - [05/May/2006:17:15:29 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 200.75.6.3 - - [05/May/2006:17:23:43 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; es-AR; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3" 64.4.8.119 - - [05/May/2006:17:26:36 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 24.91.251.99 - - [05/May/2006:17:31:18 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3" 207.68.146.48 - - [05/May/2006:17:34:18 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 84.63.23.233 - - [05/May/2006:17:55:18 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (X11; U; SunOS sun4u; en-US; rv:1.7.8) Gecko/20050512 Firefox/1.0.4" 72.30.128.226 - - [05/May/2006:18:08:08 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 72.30.61.17 - - [05/May/2006:18:09:24 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 66.196.90.252 - - [05/May/2006:18:10:13 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 72.30.61.84 - - [05/May/2006:18:38:23 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 65.88.178.10 - - [05/May/2006:18:48:18 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0" 72.30.128.226 - - [05/May/2006:18:48:23 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 66.249.65.238 - - [05/May/2006:18:48:46 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 202.153.132.101 - - [05/May/2006:18:51:19 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)" 68.142.251.159 - - [05/May/2006:18:53:46 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 67.166.194.195 - - [05/May/2006:18:59:52 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.1) Gecko/20060124 Firefox/1.5.0.1" 72.30.177.12 - - [05/May/2006:19:02:04 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 81.213.170.19 - - [05/May/2006:19:32:25 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; tr; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3" 68.142.251.159 - - [05/May/2006:19:38:37 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 72.30.61.17 - - [05/May/2006:19:40:27 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 64.4.8.100 - - [05/May/2006:19:55:40 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 66.154.103.137 - - [05/May/2006:20:14:48 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Gigabot/2.0/gigablast.com/spider.html" 207.68.146.62 - - [05/May/2006:20:17:44 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 211.144.32.63 - - [05/May/2006:20:20:08 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin@sina.com)" 68.40.211.93 - - [05/May/2006:20:35:25 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3" 72.30.128.226 - - [05/May/2006:20:53:45 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 207.68.146.48 - - [05/May/2006:20:54:01 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 202.96.51.151 - - [05/May/2006:21:00:57 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "bot/1.0" 68.82.81.154 - - [05/May/2006:21:20:55 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; MSIECrawler)" 65.55.252.50 - - [05/May/2006:21:52:40 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 157.82.254.59 - - [05/May/2006:22:17:47 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)" 69.165.134.150 - - [05/May/2006:22:21:50 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.1) Gecko/20060126 SUSE/1.5.0.1-5.1 Firefox/1.5.0.1" 65.55.246.118 - - [05/May/2006:22:22:35 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)" 198.102.112.18 - - [05/May/2006:22:26:50 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "-" 72.30.128.226 - - [05/May/2006:22:29:49 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 157.82.254.59 - - [05/May/2006:23:04:51 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)" 157.82.254.59 - - [05/May/2006:23:07:36 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)" 72.30.61.17 - - [05/May/2006:23:08:21 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 72.30.61.84 - - [05/May/2006:23:13:52 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 157.82.254.59 - - [05/May/2006:23:26:28 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)" 66.154.103.137 - - [05/May/2006:23:28:31 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Gigabot/2.0/gigablast.com/spider.html" 72.30.128.226 - - [05/May/2006:23:33:49 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 157.82.254.59 - - [05/May/2006:23:48:12 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)" 207.68.146.68 - - [05/May/2006:23:53:34 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)" 202.156.2.37 - - [05/May/2006:23:55:17 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Wget/1.10.2"