Add "heritrix" to bot user agent substrings

5 years ago · 82d0812a10
1 changed files with 1 additions and 0 deletions
--- a/tildes/tildes/request_methods.py
+++ b/tildes/tildes/request_methods.py
@ -24,6 +24,7 @@ def is_bot(request: Request) -> bool:
    bot_user_agent_substrings = (
        "bingbot",
        "Googlebot",
+        "heritrix",
        "http.rb",
        "qotnews scraper",
        "Prometheus",