24.202.35.138 - - [15/Apr/2006:00:24:41 -0600] "GET /robots.txt HTTP/1.1" 304 - "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727; MSIECrawler)"
66.196.91.125 - - [15/Apr/2006:00:43:57 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
72.245.225.172 - - [15/Apr/2006:00:50:03 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "dev-spider3.ercko.com/1.3b"
64.4.8.100 - - [15/Apr/2006:01:25:12 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
24.202.35.138 - - [15/Apr/2006:02:16:57 -0600] "GET /robots.txt HTTP/1.1" 304 - "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727; MSIECrawler)"
207.68.146.62 - - [15/Apr/2006:02:20:02 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
207.46.98.121 - - [15/Apr/2006:02:26:29 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
207.68.146.62 - - [15/Apr/2006:03:12:04 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
207.68.188.242 - - [15/Apr/2006:03:17:07 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
207.68.146.56 - - [15/Apr/2006:03:28:25 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
207.67.117.173 - - [15/Apr/2006:03:40:16 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)"
212.143.56.110 - - [15/Apr/2006:03:46:45 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)"
207.68.146.48 - - [15/Apr/2006:03:57:37 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
70.106.169.113 - - [15/Apr/2006:04:01:17 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; MSIECrawler)"
24.202.35.138 - - [15/Apr/2006:04:05:58 -0600] "GET /robots.txt HTTP/1.1" 304 - "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727; MSIECrawler)"
207.68.146.68 - - [15/Apr/2006:04:10:51 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
66.147.154.3 - - [15/Apr/2006:04:43:25 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "http://www.almaden.ibm.com/cs/crawler   [hc3]"
64.124.85.75 - - [15/Apr/2006:04:44:25 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (compatible; BecomeBot/3.0; MSIE 6.0 compatible; +http://www.become.com/site_owners.html)"
64.124.85.75 - - [15/Apr/2006:05:26:38 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (compatible; BecomeBot/3.0; MSIE 6.0 compatible; +http://www.become.com/site_owners.html)"
65.214.44.186 - - [15/Apr/2006:05:36:34 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)"
24.202.35.138 - - [15/Apr/2006:05:47:10 -0600] "GET /robots.txt HTTP/1.1" 304 - "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727; MSIECrawler)"
68.142.251.159 - - [15/Apr/2006:05:57:07 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
72.30.177.12 - - [15/Apr/2006:06:20:13 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
66.249.66.2 - - [15/Apr/2006:06:25:57 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
65.55.233.146 - - [15/Apr/2006:06:31:05 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
24.202.35.138 - - [15/Apr/2006:07:22:13 -0600] "GET /robots.txt HTTP/1.1" 304 - "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727; MSIECrawler)"
72.30.177.12 - - [15/Apr/2006:07:32:51 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
212.143.56.110 - - [15/Apr/2006:07:40:23 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)"
65.214.44.144 - - [15/Apr/2006:08:44:56 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)"
202.96.51.151 - - [15/Apr/2006:09:04:07 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "bot/1.0"
72.30.177.12 - - [15/Apr/2006:09:37:26 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
24.202.35.138 - - [15/Apr/2006:09:46:35 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727; MSIECrawler)"
64.4.8.100 - - [15/Apr/2006:10:00:37 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
217.113.225.23 - - [15/Apr/2006:10:08:35 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; MSIECrawler)"
64.210.196.197 - - [15/Apr/2006:10:47:25 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Girafabot; girafabot at girafa dot com; http://www.girafa.com)"
207.46.98.126 - - [15/Apr/2006:10:49:08 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
70.160.27.206 - - [15/Apr/2006:10:53:35 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; MSIECrawler)"
24.202.35.138 - - [15/Apr/2006:11:00:00 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727; MSIECrawler)"
213.107.224.18 - - [15/Apr/2006:11:06:09 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; NN5.0.4.4; .NET CLR 1.1.4322; MSIECrawler)"
65.55.246.118 - - [15/Apr/2006:11:34:21 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
68.142.251.159 - - [15/Apr/2006:11:35:26 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
68.142.251.159 - - [15/Apr/2006:12:18:33 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
209.237.238.224 - - [15/Apr/2006:12:43:06 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" ""
219.142.118.12 - - [15/Apr/2006:12:43:41 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "-"
222.33.71.54 - - [15/Apr/2006:12:58:50 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; MSIECrawler)"
157.82.157.22 - - [15/Apr/2006:13:04:12 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "FeedChecker/0.01"
64.4.8.100 - - [15/Apr/2006:13:05:36 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
68.142.251.159 - - [15/Apr/2006:13:17:30 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
207.68.146.56 - - [15/Apr/2006:13:21:07 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
72.30.111.147 - - [15/Apr/2006:14:25:44 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
192.55.214.132 - - [15/Apr/2006:14:27:51 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)"
192.55.214.132 - - [15/Apr/2006:14:43:51 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "larbin_2.6.3 (larbin2.6.3@unspecified.mail)"
209.191.65.127 - - [15/Apr/2006:14:47:38 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Yahoo-MMCrawler/3.x (mms dash mmcrawler dash support at yahoo dash inc dot com)"
65.55.246.118 - - [15/Apr/2006:14:51:23 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
207.68.146.48 - - [15/Apr/2006:14:52:00 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
65.55.233.146 - - [15/Apr/2006:14:56:50 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
207.68.146.68 - - [15/Apr/2006:15:02:35 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
68.142.251.159 - - [15/Apr/2006:15:20:30 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
207.68.146.62 - - [15/Apr/2006:15:25:52 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
207.68.188.242 - - [15/Apr/2006:15:51:29 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
65.55.233.146 - - [15/Apr/2006:15:52:17 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
217.113.225.23 - - [15/Apr/2006:15:54:03 -0600] "GET /robots.txt HTTP/1.1" 304 - "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; MSIECrawler)"
207.68.146.56 - - [15/Apr/2006:17:46:09 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
128.8.126.112 - - [15/Apr/2006:18:02:08 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.10) Gecko/20050720 Fedora/1.0.6-1.1.fc3 Firefox/1.0.6"
24.202.35.138 - - [15/Apr/2006:18:07:14 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727; MSIECrawler)"
207.68.146.56 - - [15/Apr/2006:18:11:27 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
70.106.169.113 - - [15/Apr/2006:18:16:00 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; MSIECrawler)"
66.196.90.252 - - [15/Apr/2006:18:40:28 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
65.55.246.96 - - [15/Apr/2006:19:34:53 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/0.9 (+http://search.msn.com/msnbot.htm)"
62.2.169.218 - - [15/Apr/2006:19:49:54 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "augurfind V-2 beta1"
64.4.8.100 - - [15/Apr/2006:19:50:40 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
207.68.146.62 - - [15/Apr/2006:20:16:29 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
68.142.251.159 - - [15/Apr/2006:20:28:46 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
66.196.90.252 - - [15/Apr/2006:20:44:06 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
81.19.66.8 - - [15/Apr/2006:20:45:12 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "StackRambler/2.0 (MSIE incompatible)"
84.203.112.97 - - [15/Apr/2006:21:16:36 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Gizbot3"
24.117.114.193 - - [15/Apr/2006:21:17:42 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; MSIECrawler)"
24.202.35.138 - - [15/Apr/2006:21:47:28 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727; MSIECrawler)"
207.68.146.56 - - [15/Apr/2006:21:51:40 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"
64.124.85.74 - - [15/Apr/2006:21:57:20 -0600] "GET /robots.txt HTTP/1.1" 200 69 "-" "Mozilla/5.0 (compatible; BecomeBot/3.0; MSIE 6.0 compatible; +http://www.become.com/site_owners.html)"
65.214.44.144 - - [15/Apr/2006:22:02:08 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)"
65.55.233.146 - - [15/Apr/2006:23:06:29 -0600] "GET /robots.txt HTTP/1.0" 200 69 "-" "msnbot/1.0 (+http://search.msn.com/msnbot.htm)"