# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

# User-agent: webcrawler
# Disallow:

# User-agent: lycra
# Disallow: /

User-agent: *
Disallow: /images
Disallow: /cgi-bin
Disallow: /Copy of CGI-BIN
Disallow: /Copy of polls
Disallow: /dim
Disallow: /Dload
Disallow: /dounas
Disallow: /flashkit poll
Disallow: /msl
Disallow: /sbie
Disallow: /search
Disallow: /amea