User-agent: *
Disallow: /cgi-bin
Disallow: /arcane

User-agent: aipbot
Disallow: /

User-agent: Slurp
Crawl-delay: 3

User-agent: baiduspider
Disallow: / 

User-agent: ia_archiver
Disallow: /

User-Agent: Charlotte
Disallow: / 

User-agent: 008
Disallow: /