From f87d84759db3c7b13b30ed36662f7a51820b61fd Mon Sep 17 00:00:00 2001 From: Deimos Date: Sun, 24 Feb 2019 13:15:28 -0700 Subject: [PATCH] Block moz.com dotbot from crawling the site --- tildes/static/robots.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tildes/static/robots.txt b/tildes/static/robots.txt index 152a534..49504a0 100644 --- a/tildes/static/robots.txt +++ b/tildes/static/robots.txt @@ -1,2 +1,5 @@ User-agent: BLEXBot Disallow: / + +User-agent: dotbot +Disallow: /