# robots.txt # Disallow all search bots to the infinite virtual URL space of # /cgi-bin, /reports, /reports-old, /reports-test, and /webadmin, and # /awstats and /stats User-agent: * Disallow: /awstats/ Disallow: /cgi-bin/ Disallow: /reports/ Disallow: /reports-old/ Disallow: /reports-test/ Disallow: /stats/ Disallow: /webadmin/ Disallow: /wordpress/wp-content/uploads/ # # # User-agent: * Robot-version: 2.0 Crawl-Delay: 10 Request-rate: 1/3 0000-0059 Request-rate: 1/3 1300-2359 Request-rate: 1/5 0100-1259 # Crawl-delay: 5 # Visit-time: 0100-1300 Disallow: /awstats/ Disallow: /cgi-bin/ Disallow: /reports/ Disallow: /reports-old/ Disallow: /reports-test/ Disallow: /stats/ Disallow: /webadmin/ Disallow: /wordpress/wp-content/uploads/