From 673ffd26e17b951400ce1b5ace80c63b844011bc Mon Sep 17 00:00:00 2001
From: Deimos <deimos@tildes.net>
Date: Tue, 27 Aug 2019 16:47:49 -0600
Subject: [PATCH] robots.txt: Add comments

---
 tildes/static/robots.txt | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tildes/static/robots.txt b/tildes/static/robots.txt
index 47bb3cd..8bf4f1a 100644
--- a/tildes/static/robots.txt
+++ b/tildes/static/robots.txt
@@ -1,20 +1,33 @@
+# In general, scrapers are welcome if they are collecting data for informational uses
+# (such as search engines) and maintain a reasonable rate of scraping.
+#
+# Scrapers from SEO/marketing-type services will be blocked. Tildes data is not a
+# resource to be mined and sold.
+
+# https://ahrefs.com/robot
 User-agent: AhrefsBot
 Disallow: /
 
+# http://webmeup-crawler.com/
 User-agent: BLEXBot
 Disallow: /
 
+# https://app.hypefactors.com/media-monitoring/about.html
 User-agent: Buck
 Disallow: /
 
+# https://moz.com/help/moz-procedures/crawlers/dotbot
 User-agent: dotbot
 Disallow: /
 
+# Unknown/suspicious scraper - UA only contains a gmail address
 User-agent: MauiBot
 Disallow: /
 
+# https://megaindex.com/crawler
 User-agent: MegaIndex
 Disallow: /
 
+# https://www.semrush.com/bot/
 User-agent: SemrushBot
 Disallow: /