diff --git a/tildes/tildes/request_methods.py b/tildes/tildes/request_methods.py index ec9fb76..4262fc6 100644 --- a/tildes/tildes/request_methods.py +++ b/tildes/tildes/request_methods.py @@ -19,6 +19,24 @@ def get_redis_connection(request: Request) -> Redis: return Redis(unix_socket_path=socket) +def is_bot(request: Request) -> bool: + """Return whether the request is by a known bot (e.g. search engine crawlers).""" + bot_user_agent_substrings = ( + "bingbot", + "Googlebot", + "qotnews scraper", + "Qwantify", + "YandexBot", + ) + + if request.user_agent: + return any( + [substring in request.user_agent for substring in bot_user_agent_substrings] + ) + + return False + + def is_safe_request_method(request: Request) -> bool: """Return whether the request method is "safe" (is GET or HEAD).""" return request.method in {"GET", "HEAD"} @@ -124,6 +142,7 @@ def current_listing_normal_url( def includeme(config: Configurator) -> None: """Attach the request methods to the Pyramid request object.""" + config.add_request_method(is_bot, "is_bot", reify=True) config.add_request_method(is_safe_request_method, "is_safe_method", reify=True) # Add the request.redis request method to access a redis connection. This is done in diff --git a/tildes/tildes/tweens.py b/tildes/tildes/tweens.py index 8c5a385..977093d 100644 --- a/tildes/tildes/tweens.py +++ b/tildes/tildes/tweens.py @@ -42,7 +42,7 @@ def metrics_tween_factory(handler: Callable, registry: Registry) -> Callable: request_histogram = Histogram( "tildes_pyramid_requests_seconds", "Request processing times", - labelnames=["route", "status_code", "method", "logged_in"], + labelnames=["route", "status_code", "method", "logged_in", "is_bot"], ) def metrics_tween(request: Request) -> Response: @@ -60,6 +60,7 @@ def metrics_tween_factory(handler: Callable, registry: Registry) -> Callable: status_code=response.status_code, method=request.method, logged_in=str(bool(request.user)).lower(), + is_bot=str(request.is_bot).lower(), ).observe(duration) return response