32 lines
390 B
Plaintext
32 lines
390 B
Plaintext
User-agent: dotbot
|
|
Disallow: /
|
|
|
|
User-agent: CCBot
|
|
Disallow: /
|
|
|
|
User-agent: barkrowler
|
|
Disallow: /
|
|
|
|
User-agent: AhrefsBot
|
|
Disallow: /
|
|
|
|
User-agent: SemrushBot
|
|
Disallow: /
|
|
|
|
User-agent: serpstatbot
|
|
Disallow: /
|
|
|
|
User-agent: InternetMeasurement
|
|
Disallow: /
|
|
|
|
User-agent: CensysInspect
|
|
Disallow: /
|
|
|
|
User-agent: MJ12bot
|
|
Disallow: /
|
|
|
|
User-agent: *
|
|
Crawl-delay: 3600
|
|
Disallow: /.env
|
|
Disallow: /api/v1
|