add robots txt file

2025-12-23 15:24:22 +01:00
parent a39862b5ab
commit 6280299770
1 changed files with 31 additions and 0 deletions
@@ -0,0 +1,31 @@
+User-agent: dotbot
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: barkrowler
+Disallow: /
+
+User-agent: AhrefsBot
+Disallow: /
+
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: serpstatbot
+Disallow: /
+
+User-agent: InternetMeasurement
+Disallow: /
+
+User-agent: CensysInspect
+Disallow: /
+
+User-agent: MJ12bot
+Disallow: /
+
+User-agent: *
+Crawl-delay: 3600
+Disallow: /.env
+Disallow: /api/v1