# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent:webcrawler
Disallow:/old_site_backup
Disallow:/pages

User-agent:*
Disallow:/tmp
Disallow:/logs
Disallow:/old_site_backup
Disallow:/pages