User-agent: GPTBot
Crawl-Delay: 30

User-agent: AhrefsBot
Crawl-Delay: 30

User-agent: Scrapy
User-agent: BLEXBot
Disallow: /

User-agent: Amazonbot               # Amazon's user agent
Disallow: /

User-agent: Seekport Crawler
Disallow: /

User-agent: GeedoProductSearch
Disallow: /

User-Agent: MJ12bot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: barkrowler
Disallow: /

User-agent: *
Allow: /_system/content/*
Disallow: /bin/
Disallow: /admin/
Disallow: /_system/
Disallow: /_auth/
Disallow: /dev/
Disallow: /test/
Disallow: /_admin/
Disallow: /_dev/
Disallow: /_shop/
Disallow: /_www/
Disallow: /images/
Disallow: /images2/
Disallow: /PingTest.htm
Disallow: /_PingTest.htm
Allow: /