#http://www.robotstxt.org/wc/faq.html#robotstxt
#a robots.txt found on the drupal site
User-agent: *
Disallow: /admin/
Disallow: /error.php
Disallow: /cron.php
Disallow: /xmlrpc.php
Disallow: /database/
Disallow: /includes/
Disallow: /modules/
Disallow: /scripts/
Disallow: /themes/