robots.txt: Block more SEO/marketing scrapers

5 years ago · 662d179548
1 changed files with 9 additions and 0 deletions
--- a/tildes/static/robots.txt
+++ b/tildes/static/robots.txt
@ -1,6 +1,12 @@
+User-agent: AhrefsBot
+Disallow: /
+
 User-agent: BLEXBot
 Disallow: /

+User-agent: Buck
+Disallow: /
+
 User-agent: dotbot
 Disallow: /

@ -9,3 +15,6 @@ Disallow: /

 User-agent: MegaIndex
 Disallow: /
+
+User-agent: SemrushBot
+Disallow: /