robots.txt: Block more SEO/marketing scrapers

5 years ago · 662d179548
1 changed files with 9 additions and 0 deletions
--- a/tildes/static/robots.txt
+++ b/tildes/static/robots.txt
@ -1,6 +1,12 @@
 User-agent: AhrefsBot
 Disallow: /
 User-agent: BLEXBot
 Disallow: /
 User-agent: Buck
 Disallow: /
 User-agent: dotbot
 Disallow: /
@ -9,3 +15,6 @@ Disallow: /
 User-agent: MegaIndex
 Disallow: /
 User-agent: SemrushBot
 Disallow: /