From 31afe0a8ba574988be4e879c9ab33792ceec9533 Mon Sep 17 00:00:00 2001 From: Deimos Date: Tue, 13 Jul 2021 22:11:14 -0600 Subject: [PATCH] Update mypy to 0.910 This also involved installing some new packages for the type stubs for a few of the major third-party libraries. I also had to change some of the imports in some model files in strange ways, I'm not sure why some of these were necessary. I suspect this might be a bug in mypy, but I'm not sure if I'll be able to build a reproduction of it to be able to report it. --- tildes/consumers/post_processing_script_runner.py | 4 ++++ tildes/consumers/topic_metadata_generator.py | 6 ++++++ tildes/mypy.ini | 3 +-- tildes/requirements-dev.in | 4 ++++ tildes/requirements-dev.txt | 8 ++++++-- tildes/tildes/lib/database.py | 8 ++++++-- tildes/tildes/lib/event_stream.py | 4 ++-- tildes/tildes/lib/markdown.py | 15 +++++++++++++-- tildes/tildes/models/comment/comment_bookmark.py | 3 ++- tildes/tildes/models/comment/comment_tree.py | 2 +- tildes/tildes/models/group/group_subscription.py | 2 +- tildes/tildes/models/group/group_wiki_page.py | 2 +- tildes/tildes/models/topic/topic_bookmark.py | 3 ++- tildes/tildes/models/topic/topic_ignore.py | 3 ++- tildes/tildes/models/topic/topic_schedule.py | 3 ++- tildes/tildes/models/user/user_group_settings.py | 3 ++- tildes/tildes/models/user/user_permissions.py | 3 ++- 17 files changed, 57 insertions(+), 19 deletions(-) diff --git a/tildes/consumers/post_processing_script_runner.py b/tildes/consumers/post_processing_script_runner.py index 52fd266..963ea5d 100644 --- a/tildes/consumers/post_processing_script_runner.py +++ b/tildes/consumers/post_processing_script_runner.py @@ -3,6 +3,8 @@ """Consumer that runs processing scripts on posts.""" +from typing import Type, Union + from sqlalchemy import desc from sqlalchemy.sql.expression import or_ @@ -21,6 +23,8 @@ class PostProcessingScriptRunner(EventStreamConsumer): def process_message(self, message: Message) -> None: """Process a message from the stream.""" + wrapper_class: Union[Type[TopicScriptingWrapper], Type[CommentScriptingWrapper]] + if "topic_id" in message.fields: post = ( self.db_session.query(Topic) diff --git a/tildes/consumers/topic_metadata_generator.py b/tildes/consumers/topic_metadata_generator.py index d09b05f..ff5f6a7 100644 --- a/tildes/consumers/topic_metadata_generator.py +++ b/tildes/consumers/topic_metadata_generator.py @@ -63,6 +63,9 @@ class TopicMetadataGenerator(EventStreamConsumer): @staticmethod def _generate_text_metadata(topic: Topic) -> Dict[str, Any]: """Generate metadata for a text topic (word count and excerpt).""" + if not topic.rendered_html: + return {} + extracted_text = extract_text_from_html(topic.rendered_html) # create a short excerpt by truncating the extracted string @@ -80,6 +83,9 @@ class TopicMetadataGenerator(EventStreamConsumer): def _generate_link_metadata(self, topic: Topic) -> Dict[str, Any]: """Generate metadata for a link topic (domain).""" + if not topic.link: + return {} + parsed_domain = get_domain_from_url(topic.link) if self._domain_is_ip_address(parsed_domain): diff --git a/tildes/mypy.ini b/tildes/mypy.ini index 55b6322..c7945a8 100644 --- a/tildes/mypy.ini +++ b/tildes/mypy.ini @@ -1,5 +1,6 @@ [mypy] mypy_path = /opt/tildes/stubs/ +exclude = ^(tests|alembic)/ disallow_untyped_defs = true ignore_missing_imports = true no_implicit_optional = true @@ -13,5 +14,3 @@ warn_unused_ignores = true [mypy-tasks] disallow_untyped_defs = false -[mypy-tests.*] -disallow_untyped_defs = false diff --git a/tildes/requirements-dev.in b/tildes/requirements-dev.in index a369b4c..09af633 100644 --- a/tildes/requirements-dev.in +++ b/tildes/requirements-dev.in @@ -8,4 +8,8 @@ pyramid-debugtoolbar pytest pytest-mock testing.redis +types-bleach +types-python-dateutil +types-redis +types-requests webtest diff --git a/tildes/requirements-dev.txt b/tildes/requirements-dev.txt index 81beb78..e2a04a4 100644 --- a/tildes/requirements-dev.txt +++ b/tildes/requirements-dev.txt @@ -38,7 +38,7 @@ markupsafe==1.1.1 marshmallow==3.9.0 mccabe==0.6.1 mypy-extensions==0.4.3 -mypy==0.790 +mypy==0.910 packaging==20.4 parso==0.7.1 pastedeploy==2.1.1 @@ -104,7 +104,11 @@ traitlets==5.0.5 transaction==3.0.0 translationstring==1.4 typed-ast==1.4.1 -typing-extensions==3.7.4.3 +types-bleach==3.3.3 +types-python-dateutil==0.1.4 +types-redis==3.5.4 +types-requests==2.25.0 +typing-extensions==3.10.0.0 urllib3==1.25.11 venusian==3.0.0 waitress==1.4.4 diff --git a/tildes/tildes/lib/database.py b/tildes/tildes/lib/database.py index f2b81d4..b0d7eab 100644 --- a/tildes/tildes/lib/database.py +++ b/tildes/tildes/lib/database.py @@ -157,14 +157,18 @@ class RecurrenceRule(TypeDecorator): impl = Text - def process_bind_param(self, value: rrule, dialect: Dialect) -> str: + def process_bind_param( + self, value: Optional[rrule], dialect: Dialect + ) -> Optional[str]: """Convert the rrule value to a string to store it.""" if value is None: return value return rrule_to_str(value) - def process_result_value(self, value: str, dialect: Dialect) -> rrule: + def process_result_value( + self, value: Optional[str], dialect: Dialect + ) -> Optional[rrule]: """Convert the stored string to an rrule.""" if value is None: return value diff --git a/tildes/tildes/lib/event_stream.py b/tildes/tildes/lib/event_stream.py index 092eba8..6dfa41b 100644 --- a/tildes/tildes/lib/event_stream.py +++ b/tildes/tildes/lib/event_stream.py @@ -6,7 +6,7 @@ import os from abc import abstractmethod from configparser import ConfigParser -from typing import Any, Dict, List, Sequence +from typing import Any, Dict, List, Optional, Sequence from prometheus_client import CollectorRegistry, Counter, start_http_server from redis import Redis, ResponseError @@ -47,7 +47,7 @@ class EventStreamConsumer: necessary. It relies on the environment variable INI_FILE being set. """ - METRICS_PORT = None + METRICS_PORT: Optional[int] = None def __init__( self, diff --git a/tildes/tildes/lib/markdown.py b/tildes/tildes/lib/markdown.py index 7e69e40..be3b232 100644 --- a/tildes/tildes/lib/markdown.py +++ b/tildes/tildes/lib/markdown.py @@ -5,7 +5,18 @@ import re from functools import partial -from typing import Any, Callable, Iterator, List, Match, Optional, Pattern, Tuple +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Match, + Optional, + Pattern, + Tuple, + Union, +) import bleach from bs4 import BeautifulSoup @@ -94,7 +105,7 @@ ALLOWED_HTML_TAGS = ( ) ALLOWED_LINK_PROTOCOLS = ("gemini", "http", "https", "mailto") -ALLOWED_HTML_ATTRIBUTES_DEFAULT = { +ALLOWED_HTML_ATTRIBUTES_DEFAULT: Dict[str, Union[List[str], Callable]] = { "a": ["href", "title"], "details": ["open"], "ol": ["start"], diff --git a/tildes/tildes/models/comment/comment_bookmark.py b/tildes/tildes/models/comment/comment_bookmark.py index ec0f156..e3c022c 100644 --- a/tildes/tildes/models/comment/comment_bookmark.py +++ b/tildes/tildes/models/comment/comment_bookmark.py @@ -7,9 +7,10 @@ from sqlalchemy.orm import relationship from sqlalchemy.sql.expression import text from tildes.models import DatabaseModel -from tildes.models.comment import Comment from tildes.models.user import User +from .comment import Comment + class CommentBookmark(DatabaseModel): """Model for a comment bookmark.""" diff --git a/tildes/tildes/models/comment/comment_tree.py b/tildes/tildes/models/comment/comment_tree.py index f9ec90d..fbf2895 100644 --- a/tildes/tildes/models/comment/comment_tree.py +++ b/tildes/tildes/models/comment/comment_tree.py @@ -272,7 +272,7 @@ class CommentInTree(ObjectProxy): self.replies: List[CommentInTree] = [] self.has_visible_descendant = False self.num_children = 0 - self.depth: Optional[int] = None + self.depth = 0 @property def has_uncollapsed_descendant(self) -> bool: diff --git a/tildes/tildes/models/group/group_subscription.py b/tildes/tildes/models/group/group_subscription.py index f899a42..2fec424 100644 --- a/tildes/tildes/models/group/group_subscription.py +++ b/tildes/tildes/models/group/group_subscription.py @@ -11,7 +11,7 @@ from sqlalchemy.sql.expression import text from tildes.metrics import incr_counter from tildes.models import DatabaseModel -from tildes.models.user import User +from tildes.models.user.user import User from .group import Group diff --git a/tildes/tildes/models/group/group_wiki_page.py b/tildes/tildes/models/group/group_wiki_page.py index 8ac3c03..914a2a1 100644 --- a/tildes/tildes/models/group/group_wiki_page.py +++ b/tildes/tildes/models/group/group_wiki_page.py @@ -20,7 +20,7 @@ from tildes.lib.html import add_anchors_to_headings from tildes.lib.markdown import convert_markdown_to_safe_html from tildes.lib.string import convert_to_url_slug from tildes.models import DatabaseModel -from tildes.models.user import User +from tildes.models.user.user import User from tildes.schemas.group_wiki_page import GroupWikiPageSchema, PAGE_NAME_MAX_LENGTH from tildes.typing import AclType diff --git a/tildes/tildes/models/topic/topic_bookmark.py b/tildes/tildes/models/topic/topic_bookmark.py index 3c810a5..a73ed1c 100644 --- a/tildes/tildes/models/topic/topic_bookmark.py +++ b/tildes/tildes/models/topic/topic_bookmark.py @@ -7,9 +7,10 @@ from sqlalchemy.orm import relationship from sqlalchemy.sql.expression import text from tildes.models import DatabaseModel -from tildes.models.topic import Topic from tildes.models.user import User +from .topic import Topic + class TopicBookmark(DatabaseModel): """Model for a topic bookmark.""" diff --git a/tildes/tildes/models/topic/topic_ignore.py b/tildes/tildes/models/topic/topic_ignore.py index 8c0759f..97fc6c2 100644 --- a/tildes/tildes/models/topic/topic_ignore.py +++ b/tildes/tildes/models/topic/topic_ignore.py @@ -7,9 +7,10 @@ from sqlalchemy.orm import relationship from sqlalchemy.sql.expression import text from tildes.models import DatabaseModel -from tildes.models.topic import Topic from tildes.models.user import User +from .topic import Topic + class TopicIgnore(DatabaseModel): """Model for an ignored topic.""" diff --git a/tildes/tildes/models/topic/topic_schedule.py b/tildes/tildes/models/topic/topic_schedule.py index dea585d..763a59a 100644 --- a/tildes/tildes/models/topic/topic_schedule.py +++ b/tildes/tildes/models/topic/topic_schedule.py @@ -25,10 +25,11 @@ from tildes.lib.database import RecurrenceRule, TagList from tildes.lib.datetime import utc_now from tildes.models import DatabaseModel from tildes.models.group import Group -from tildes.models.topic import Topic from tildes.models.user import User from tildes.schemas.topic import TITLE_MAX_LENGTH +from .topic import Topic + class TopicSchedule(DatabaseModel): """Model for scheduled topics (auto-posted, often repeatedly on a schedule). diff --git a/tildes/tildes/models/user/user_group_settings.py b/tildes/tildes/models/user/user_group_settings.py index 6b10d9a..d51b624 100644 --- a/tildes/tildes/models/user/user_group_settings.py +++ b/tildes/tildes/models/user/user_group_settings.py @@ -12,7 +12,8 @@ from sqlalchemy.orm import relationship from tildes.enums import TopicSortOption from tildes.models import DatabaseModel from tildes.models.group import Group -from tildes.models.user import User + +from .user import User class UserGroupSettings(DatabaseModel): diff --git a/tildes/tildes/models/user/user_permissions.py b/tildes/tildes/models/user/user_permissions.py index 92ed0bb..17804d1 100644 --- a/tildes/tildes/models/user/user_permissions.py +++ b/tildes/tildes/models/user/user_permissions.py @@ -10,7 +10,8 @@ from sqlalchemy.orm import relationship from tildes.enums import UserPermission, UserPermissionType from tildes.models import DatabaseModel from tildes.models.group import Group -from tildes.models.user import User + +from .user import User class UserPermissions(DatabaseModel):