# User-agent:   * is All user agents
# Disallow:     do not allow if this matches part of requested path

User-agent: *
Disallow:   /cgi-bin/redirect.cgi
Disallow:   /cgi-bin/response.cgi
Disallow:   /cgi-bin/stats.cgi

User-agent: AhrefsBot
User-agent: almaden 
User-agent: ASPSeek 
User-agent: baiduspider 
User-agent: dumbBot 
User-agent: Generic 
User-agent: grub-client 
User-agent: MSIECrawler 
User-agent: NexaBot 
User-agent: NPBot 
User-agent: OWR_Crawler 
User-agent: psbot 
User-agent: rabaz 
User-agent: RPT-HTTPClient 
User-agent: ScoutAbout 
User-agent: semanticdiscovery 
User-agent: TurnitinBot 
User-agent: Wget 
User-agent: twiceler
User-agent: 008
User-agent: Yandex
User-agent: MJ12bot
User-agent: Ocelli
Disallow: /