User-agent: *
Disallow: /users/
Disallow: /user-settings/
Disallow: /jobs-search/
Disallow: /index/redirect/
Disallow: /furlough/
Disallow: /companies/inline-login/format/json
Disallow: /search/
Disallow: /companies/
User-agent: NetShelter ContentScan
Disallow: /

User-agent: JoobleBot
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: Mozilla/5.0 (compatible; DotBot/1.2; +https://opensiteexplorer.org/dotbot; help@moz.com)
Disallow: /

User-agent: python-requests/2.25.1
Disallow: /

User-Agent: omgilibot
Disallow: /

User-Agent: omgili
Disallow: /

User-agent: NetShelter
Disallow: /

User-agent: ^NetShelter
Disallow: /

User-agent: ^HubSpot Webcrawler
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /