From 7c331449955fb4ca587838b13f0eea31cdd6deaa Mon Sep 17 00:00:00 2001
From: Deimos <deimos@tildes.net>
Date: Sat, 31 Aug 2019 18:54:23 -0600
Subject: [PATCH] robots.txt: Block /login from all scrapers

Various scrapers hit this a lot because it's on all pages with a
differing query variable (for the "from url" to redirect back to after
logging in).
---
 tildes/static/robots.txt | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tildes/static/robots.txt b/tildes/static/robots.txt
index 70b95f3..1dfde65 100644
--- a/tildes/static/robots.txt
+++ b/tildes/static/robots.txt
@@ -4,6 +4,12 @@
 # Scrapers from SEO/marketing-type services will be blocked. Tildes data is not a
 # resource to be mined and sold.
 
+# urls blocked for all scrapers
+User-agent: *
+Disallow: /login
+
+# Specific, fully-blocked scrapers below
+
 # https://ahrefs.com/robot
 User-agent: AhrefsBot
 Disallow: /