# TCRF is a wiki that doesn't change too often;
# repeatedly crawling the same pages is... not useful.
# Not only that, but we really don't need old revisions
# and other junk stored, so, please don't do that. thanks.
# (This is loosely based off of Wikipedia's robots.txt)

User-agent: *
Crawl-delay: 30
Allow: /api.php?action=mobileview&
Allow: /load.php?
Disallow: /api.php
Disallow: /index.php
Disallow: /Special:*

# Bing is the worst of the web-crawlers by a mile and has several
# documented instances of "I'm gonna download your entire server"
# with basically no delay across its various crawlers
User-agent: bingbot
Crawl-delay: 100

User-agent: msnbot
Crawl-delay: 100