User-agent: *
Disallow: /assets/
Disallow: /cgi-bin/
Disallow: /errors/

User-agent: psbot
Disallow: /

User-agent: Yahoo-MMCrawler
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: gigabot
Disallow: /