# robots.txt file
# This character signifies a comment tag

User-agent: *
Disallow: /cgi-bin/
User-agent: Teleport Pro 
Disallow: / 
User-agent: Offline Explorer 
Disallow: / 
User-agent: Googlebot
allow: /
User-agent: Slurp
allow: /
User-agent: Teleport
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: Offline
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: WWWOFFLE
Disallow: /
User-agent: HTTPClient
Disallow: /
User-agent: Pump
Disallow: /
User-agent: MegaSheep
Disallow: /