robots.txt: Block Majestic scraper (MJ12bot)

5 years ago · 983bae7039
1 changed files with 5 additions and 0 deletions
--- a/tildes/static/robots.txt
+++ b/tildes/static/robots.txt
@ -28,6 +28,11 @@ Disallow: /
 User-agent: MegaIndex
 Disallow: /

+# https://mj12bot.com/
+# Note: claims to be powering a search engine, actually sells SEO/marketing data
+User-agent: MJ12bot
+Disallow: /
+
 # https://www.semrush.com/bot/
 User-agent: SemrushBot
 Disallow: /