User-agent: Googlebot # Google
User-agent: Bingbot # Bing
Disallow: /rpc/
Disallow: /job/
Disallow: /job-search/
Disallow: /view-job/
Disallow: /vanity/
Disallow: /empleo/
Disallow: /emploi/
Disallow: /vaga/
Disallow: /việc/
Disallow: /trabajo/
Disallow: /งาน/
Disallow: /job vacancy/
Disallow: /rss
Disallow: /style-guide/
Disallow: /iniciar-sesion
Disallow: /cdn-cgi/

User-agent: Amazonbot # Alexa home
User-agent: Baiduspider
User-agent: Blekkobot
User-agent: DuckDuckBot
User-agent: Ecosia
User-agent: ExaBot
User-agent: facebookexternalhit
User-agent: Yeti/Naverbot
User-agent: Slurp
User-agent: SeznamBot
User-agent: Sogou spider
User-agent: Soso Spider
User-agent: YandexBot
crawl-delay: 1
Disallow: /rpc/
Disallow: /job/
Disallow: /job-search/
Disallow: /view-job/
Disallow: /vanity/
Disallow: /empleo/
Disallow: /emploi/
Disallow: /vaga/
Disallow: /việc/
Disallow: /trabajo/
Disallow: /งาน/
Disallow: /job vacancy/
Disallow: /rss
Disallow: /style-guide/
Disallow: /iniciar-sesion
Disallow: /cdn-cgi/

User-agent: TwitterBot
crawl-delay: 1
Disallow: /rpc/
Disallow: /job-search/
Disallow: /view-job/
Disallow: /vanity/
Disallow: /vaga/
Disallow: /việc/
Disallow: /trabajo/
Disallow: /งาน/
Disallow: /job vacancy/
Disallow: /rss
Disallow: /style-guide/
Disallow: /iniciar-sesion
Disallow: /cdn-cgi/
Disallow: /rd/

User-agent: Screaming Frog SEO Spider
crawl-delay: 1
Allow: /

User-agent: Alexabot # Alexa Certification scan crawler
User-agent: APIs-Google # Google
User-agent: AdsBot-Google # Google
User-agent: AdsBot-Google-Mobile # Google
User-agent: AdsBot-Google-Mobile-Apps # Google
User-agent: DuplexWeb-Google # Google
User-agent: Googlebot-Image # Google
User-agent: Googlebot-News # Google
User-agent: Googlebot-Video # Google
User-agent: AdIdxBot # Bing
User-agent: BingPreview # Bing
User-agent: AhrefsBot # Ahrefs
User-agent: ArchitextSpider
User-agent: Crawler4j # Java Crawler
User-agent: RogerBot # Moz
User-agent: SEMrushBot # SEMRush
User-agent: ia_archiver # Alexa web-wide crawler
User-agent: Lycos_Spider_(T-Rex)
User-agent: speedy_spider
User-agent: Teoma
crawl-delay: 10
Disallow: /rpc/
Disallow: /job/
Disallow: /job-search/
Disallow: /view-job/
Disallow: /vanity/
Disallow: /empleo/
Disallow: /emploi/
Disallow: /vaga/
Disallow: /việc/
Disallow: /trabajo/
Disallow: /งาน/
Disallow: /job vacancy/
Disallow: /rss
Disallow: /style-guide/
Disallow: /iniciar-sesion
Disallow: /cdn-cgi/

# Disallowed UA
User-agent: TrovitBot
User-agent: LinkedInBot
User-agent: IndeedBot
User-agent: MJ12bot
User-agent: dotbot
User-agent: Sosospider
User-agent: sitebot
User-agent: Mediapartners-Google # Google
Disallow: /

User-agent: *
Disallow: /