From 983bae70398c6e33c35a045c36760d829ac41060 Mon Sep 17 00:00:00 2001 From: Deimos Date: Tue, 27 Aug 2019 17:24:00 -0600 Subject: [PATCH] robots.txt: Block Majestic scraper (MJ12bot) --- tildes/static/robots.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tildes/static/robots.txt b/tildes/static/robots.txt index 8bf4f1a..70b95f3 100644 --- a/tildes/static/robots.txt +++ b/tildes/static/robots.txt @@ -28,6 +28,11 @@ Disallow: / User-agent: MegaIndex Disallow: / +# https://mj12bot.com/ +# Note: claims to be powering a search engine, actually sells SEO/marketing data +User-agent: MJ12bot +Disallow: / + # https://www.semrush.com/bot/ User-agent: SemrushBot Disallow: /