User-Agent: * Disallow: /css/ User-Agent: gigabot Crawl-delay: 20 User-Agent: baidu Crawl-delay: 20 User-Agent: twiceler Crawl-delay: 20 User-Agent: ia_archiver Crawl-delay: 10 User-Agent: teoma Crawl-delay: 20 User-Agent:JRank-Bot Crawl-delay: 20 User-Agent:JRank Crawl-delay: 20 User-Agent: BecomeBot Crawl-delay: 30