User-agent: *
Disallow: /backup/
Disallow: /bin/
Disallow: /cache/
Disallow: /grav/
Disallow: /logs/
Disallow: /system/
Disallow: /vendor/
Disallow: /user/
Allow: /user/pages/
Allow: /user/themes/
Allow: /user/images/
Allow: /

User-agent: bingbot
Crawl-delay: 10

User-agent: DataForSeoBot
Disallow: /

User-agent: Yandex
Crawl-delay: 30

User-agent: BLEXBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: GrapeshotCrawler
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Clickagy Intelligence Bot v2
Disallow: /

User-agent: The Knowledge AI
Disallow: /

User-agent: BuiltWith
Disallow: /

User-agent: MaxPointCrawler
Disallow: /

User-agent: Buck
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: InfoTigerBot
Disallow: /

User-agent: barkrowler
Disallow: /

User-agent: CriteoBot
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: nbot
Disallow: /

User-agent: oBot
Disallow: /

User-agent: proximic
Disallow: /

User-agent: Amazonbot
Disallow: /

#https://www.comscore.com/Web-Crawler
#https://www.criteo.com/criteo-crawler/
#https://www.crunchbase.com/organization/admantx