add robots txt file

2025-12-23 15:24:22 +01:00
parent a39862b5ab
commit 6280299770
1 changed files with 31 additions and 0 deletions
@@ -0,0 +1,31 @@
 User-agent: dotbot
 Disallow: /
 User-agent: CCBot
 Disallow: /
 User-agent: barkrowler
 Disallow: /
 User-agent: AhrefsBot
 Disallow: /
 User-agent: SemrushBot
 Disallow: /
 User-agent: serpstatbot
 Disallow: /
 User-agent: InternetMeasurement
 Disallow: /
 User-agent: CensysInspect
 Disallow: /
 User-agent: MJ12bot
 Disallow: /
 User-agent: *
 Crawl-delay: 3600
 Disallow: /.env
 Disallow: /api/v1