# Robots.txt file User-agent: ia_archiver Disallow: / User-agent: ia_archiver/1.6 Disallow: / User-Agent: OmniExplorer_Bot/1.09 Disallow: / User-agent: TurnitinBot Disallow: / User-agent: SBIder Disallow: / User-agent: ConveraCrawler Disallow: / User-agent: Baiduspider Disallow: / User-agent: psbot Disallow: / User-agent: aipbot Disallow: / User-Agent: MJ12bot Disallow: / User-agent: IRLbot Disallow: / User-agent: BecomeBot Crawl-Delay: 30