# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
User-agent: *

Sitemap: http://www.abouttheartists.com/sitemap_index.xml

Disallow: /*edit
Disallow: /*new
Disallow: /*destroy
Disallow: /*update
Disallow: /*sort
Disallow: /movie
Disallow: /*set_default_img
Disallow: /*claim
Disallow: /session
Disallow: /user
Disallow: /galleries
Disallow: /articles
Disallow: /*search*


User-agent: MJ12bot
Disallow: /

User-agent: TwengaBot-Discover
Disallow: /

User-agent: CamontSpider
Disallow: /

User-agent: Sosospider
Disallow: /

User-agent: dotbot
Disallow: /

User-Agent: panscient.com
Disallow: /


User-agent: Exabot
Crawl-delay: 15