diff --git a/ansible/roles/postgresql_tildes_dbs/tasks/main.yml b/ansible/roles/postgresql_tildes_dbs/tasks/main.yml index 6b36229..5786be6 100644 --- a/ansible/roles/postgresql_tildes_dbs/tasks/main.yml +++ b/ansible/roles/postgresql_tildes_dbs/tasks/main.yml @@ -4,7 +4,7 @@ name: - gcc - libpq-dev - - python3-dev + - python{{ python_version }}-dev - name: Install packages needed by Ansible community plugins pip: diff --git a/ansible/roles/python/tasks/main.yml b/ansible/roles/python/tasks/main.yml index bed87e2..05c8ae8 100644 --- a/ansible/roles/python/tasks/main.yml +++ b/ansible/roles/python/tasks/main.yml @@ -1,56 +1,10 @@ --- -- name: Check if the correct version of Python is already installed - stat: - path: /usr/local/bin/python{{ python_version }} - register: python_binary - -- name: Download and install Python - when: not python_binary.stat.exists - block: - - name: Download Python source code - get_url: - dest: /tmp/python.tar.gz - url: https://www.python.org/ftp/python/{{ python_full_version }}/Python-{{ python_full_version }}.tgz - checksum: sha256:1e71f006222666e0a39f5a47be8221415c22c4dd8f25334cc41aee260b3d379e - - - name: Create temp directory to extract Python to - file: - path: /tmp/python - state: directory - - - name: Extract Python - unarchive: - remote_src: true - src: /tmp/python.tar.gz - dest: /tmp/python - extra_opts: - - --strip-components=1 - - - name: Install build dependencies for Python - apt: - name: - - make - - build-essential - - libssl-dev - - zlib1g-dev - - libbz2-dev - - libreadline-dev - - libsqlite3-dev - - wget - - curl - - llvm - - libncurses5-dev - - libncursesw5-dev - - xz-utils - - tk-dev - - - name: Build and install Python (this can take a long time) - shell: - chdir: /tmp/python - cmd: | - ./configure --enable-optimizations --with-ensurepip=install - make - make altinstall +- name: Install Python and dependencies needed by packages + apt: + name: + - python{{ python_version }} + - python{{ python_version }}-venv + - libgit2-dev - name: Create dir for venvs file: diff --git a/ansible/vars.yml b/ansible/vars.yml index 61858db..d7df620 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -5,7 +5,6 @@ bin_dir: "{{ venv_dir }}/bin" static_sites_dir: /opt/tildes-static-sites -python_full_version: 3.9.20 -python_version: "{{ python_full_version.rpartition('.')[0] }}" +python_version: "3.11" is_docker: "{{ ansible_facts['virtualization_type'] == 'container' }}" \ No newline at end of file diff --git a/tildes/consumers/site_icon_downloader.py b/tildes/consumers/site_icon_downloader.py index 2b0b795..9f26921 100644 --- a/tildes/consumers/site_icon_downloader.py +++ b/tildes/consumers/site_icon_downloader.py @@ -10,7 +10,7 @@ from typing import Optional import publicsuffix import requests -from PIL import Image +from PIL import Image, IcoImagePlugin from tildes.enums import ScraperType from tildes.lib.event_stream import EventStreamConsumer, Message @@ -80,6 +80,10 @@ class SiteIconDownloader(EventStreamConsumer): return None if favicon.format == "ICO": + assert isinstance( + favicon, IcoImagePlugin.IcoImageFile + ) # tell mypy the type is more restricted now + # get the 32x32 size if it's present, otherwise resize the largest one if (32, 32) in favicon.ico.sizes(): return favicon.ico.getimage((32, 32)) diff --git a/tildes/consumers/topic_metadata_generator.py b/tildes/consumers/topic_metadata_generator.py index 7ea4842..b88a11d 100644 --- a/tildes/consumers/topic_metadata_generator.py +++ b/tildes/consumers/topic_metadata_generator.py @@ -45,6 +45,8 @@ class TopicMetadataGenerator(EventStreamConsumer): new_metadata = self._generate_text_metadata(topic) elif topic.is_link_type: new_metadata = self._generate_link_metadata(topic) + else: + new_metadata = {} # update the topic's content_metadata in a way that won't wipe out any existing # values, and can handle the column being null diff --git a/tildes/prospector.yaml b/tildes/prospector.yaml index d115a9a..df9408e 100644 --- a/tildes/prospector.yaml +++ b/tildes/prospector.yaml @@ -29,6 +29,8 @@ pylint: disable: - bad-continuation # let Black handle line-wrapping - comparison-with-callable # seems to have a lot of false positives + - consider-using-f-string # TBD if helpful [2025-01-16] + - consider-using-generator # TBD if helpful [2025-01-16] - cyclic-import # not sure what's triggering this, doesn't seem to work correctly - logging-fstring-interpolation # rather use f-strings than worry about this - no-else-return # elif after return - could refactor to enable this check @@ -39,8 +41,10 @@ pylint: - too-many-branches # almost never helpful - too-many-instance-attributes # models have many instance attributes - too-many-locals # almost never helpful + - too-many-positional-arguments # TBD if helpful [2025-01-16] - too-many-public-methods # almost never helpful - too-many-return-statements # almost never helpful - too-many-statements # almost never helpful - ungrouped-imports # let isort handle this - unnecessary-pass # I prefer using pass, even when it's not technically necessary + - use-yield-from # TBD if helpful [2025-01-16] diff --git a/tildes/pytest.ini b/tildes/pytest.ini index ededd1e..3422a1b 100644 --- a/tildes/pytest.ini +++ b/tildes/pytest.ini @@ -4,7 +4,6 @@ addopts = -p no:cacheprovider --strict-markers filterwarnings = ignore::DeprecationWarning ignore::PendingDeprecationWarning - ignore::yaml.YAMLLoadWarning markers = html_validation: mark a test as one that validates HTML using the Nu HTML Checker (very slow) webtest: mark a test as one that uses the WebTest library, which goes through the actual WSGI app and involves using HTTP/HTML (more of a "functional test" than "unit test") diff --git a/tildes/requirements-dev.in b/tildes/requirements-dev.in index 09af633..0299e91 100644 --- a/tildes/requirements-dev.in +++ b/tildes/requirements-dev.in @@ -3,7 +3,7 @@ black freezegun html5validator mypy -prospector @ git+https://github.com/Deimos/prospector.git#egg=prospector +prospector pyramid-debugtoolbar pytest pytest-mock diff --git a/tildes/requirements-dev.txt b/tildes/requirements-dev.txt index 9c897bb..650f9f6 100644 --- a/tildes/requirements-dev.txt +++ b/tildes/requirements-dev.txt @@ -2,20 +2,20 @@ ago==0.0.93 alembic==1.6.5 appdirs==1.4.4 argon2-cffi==20.1.0 -astroid==2.6.5 -attrs==21.2.0 +astroid==3.3.8 +attrs==24.3.0 backcall==0.2.0 beautifulsoup4==4.9.3 black==21.7b0 bleach==3.3.1 certifi==2021.5.30 -cffi==1.14.6 +cffi==1.17.1 charset-normalizer==2.0.3 click==8.0.1 cornice==5.2.0 decorator==5.0.9 dodgy==0.2.1 -flake8==3.9.2 +flake8==7.1.1 flake8-polyfill==1.0.2 freezegun==1.1.0 gunicorn==20.1.0 @@ -23,52 +23,52 @@ html5lib==1.1 html5validator==0.4.0 hupper==1.10.3 idna==3.2 -iniconfig==1.1.1 -invoke==1.6.0 +iniconfig==2.0.0 +invoke==2.2.0 ipython==7.25.0 ipython-genutils==0.2.0 isort==5.9.2 jedi==0.18.0 jinja2==3.0.1 lazy-object-proxy==1.6.0 -lupa==1.9 +lupa==2.4 mako==1.1.4 markupsafe==2.0.1 -marshmallow==3.13.0 +marshmallow==3.25.1 matplotlib-inline==0.1.2 -mccabe==0.6.1 -mypy==1.13.0 +mccabe==0.7.0 +mypy==1.14.1 mypy-extensions==1.0.0 -packaging==23.2 +packaging==24.2 parso==0.8.2 pastedeploy==2.1.1 pathspec==0.9.0 pep517==0.11.0 -pep8-naming==0.12.0 +pep8-naming==0.10.0 pexpect==4.8.0 pickleshare==0.7.5 -pillow==8.3.1 +pillow==11.1.0 pip-tools==6.2.0 plaster==1.0 plaster-pastedeploy==0.7 -pluggy==0.13.1 +pluggy==1.5.0 prometheus-client==0.11.0 prompt-toolkit==3.0.19 -git+https://github.com/Deimos/prospector.git#egg=prospector -psycopg2==2.9.1 +prospector==1.13.3 +psycopg2==2.9.10 ptyprocess==0.7.0 publicsuffix2==2.20160818 -py==1.10.0 -pycodestyle==2.7.0 +py==1.11.0 +pycodestyle==2.12.1 pycparser==2.20 pydocstyle==6.1.1 -pyflakes==2.3.1 -pygit2==1.6.1 +pyflakes==3.2.0 +pygit2==1.17.0 pygments==2.9.0 -pylint==2.9.5 -pylint-plugin-utils==0.6 +pylint==3.3.3 +pylint-plugin-utils==0.8.2 pyotp==2.6.0 -pyparsing==2.4.7 +pyparsing==3.2.1 pyramid==1.10.8 pyramid-debugtoolbar==4.9 pyramid-ipython==0.2 @@ -77,17 +77,17 @@ pyramid-mako==1.1.0 pyramid-session-redis==1.5.0 pyramid-tm==2.4 pyramid-webassets==0.10 -pytest==6.2.4 -pytest-mock==3.6.1 +pytest==8.3.4 +pytest-mock==3.14.0 python-dateutil==2.8.2 python-editor==1.0.4 -pyyaml==5.4.1 +pyyaml==6.0.2 qrcode==7.2 redis==3.5.3 regex==2021.7.6 repoze.lru==0.7 requests==2.26.0 -requirements-detector==0.7 +requirements-detector==1.3.2 sentry-sdk==1.3.0 setoptconf==0.3.0 six==1.16.0 @@ -119,7 +119,7 @@ webencodings==0.5.1 webob==1.8.7 webtest==2.0.35 wheel==0.36.2 -wrapt==1.12.1 +wrapt==1.17.2 zope.deprecation==4.4.0 zope.interface==5.4.0 zope.sqlalchemy==1.5 diff --git a/tildes/requirements.txt b/tildes/requirements.txt index f14d38d..8e7d847 100644 --- a/tildes/requirements.txt +++ b/tildes/requirements.txt @@ -5,7 +5,7 @@ backcall==0.2.0 beautifulsoup4==4.9.3 bleach==3.3.1 certifi==2021.5.30 -cffi==1.14.6 +cffi==1.17.1 charset-normalizer==2.0.3 click==8.0.1 cornice==5.2.0 @@ -14,36 +14,36 @@ gunicorn==20.1.0 html5lib==1.1 hupper==1.10.3 idna==3.2 -invoke==1.6.0 +invoke==2.2.0 ipython==7.25.0 ipython-genutils==0.2.0 jedi==0.18.0 jinja2==3.0.1 -lupa==1.9 +lupa==2.4 mako==1.1.4 markupsafe==2.0.1 -marshmallow==3.13.0 +marshmallow==3.25.1 matplotlib-inline==0.1.2 -packaging==23.2 +packaging==24.2 parso==0.8.2 pastedeploy==2.1.1 pep517==0.11.0 pexpect==4.8.0 pickleshare==0.7.5 -pillow==8.3.1 +pillow==11.1.0 pip-tools==6.2.0 plaster==1.0 plaster-pastedeploy==0.7 prometheus-client==0.11.0 prompt-toolkit==3.0.19 -psycopg2==2.9.1 +psycopg2==2.9.10 ptyprocess==0.7.0 publicsuffix2==2.20160818 pycparser==2.20 -pygit2==1.6.1 +pygit2==1.17.0 pygments==2.9.0 pyotp==2.6.0 -pyparsing==2.4.7 +pyparsing==3.2.1 pyramid==1.10.8 pyramid-ipython==0.2 pyramid-jinja2==2.8 @@ -52,7 +52,7 @@ pyramid-tm==2.4 pyramid-webassets==0.10 python-dateutil==2.8.2 python-editor==1.0.4 -pyyaml==5.4.1 +pyyaml==6.0.2 qrcode==7.2 redis==3.5.3 requests==2.26.0 @@ -75,7 +75,7 @@ webassets==2.0 webencodings==0.5.1 webob==1.8.7 wheel==0.36.2 -wrapt==1.12.1 +wrapt==1.17.2 zope.deprecation==4.4.0 zope.interface==5.4.0 zope.sqlalchemy==1.5 diff --git a/tildes/scripts/backup_database.py b/tildes/scripts/backup_database.py index 318ca0d..3b302b8 100644 --- a/tildes/scripts/backup_database.py +++ b/tildes/scripts/backup_database.py @@ -34,7 +34,7 @@ def create_encrypted_backup(gpg_recipient: str) -> str: filename = datetime.now().strftime(FILENAME_FORMAT) # dump the database to a file - with open(f"{filename}.sql", "w") as dump_file: + with open(f"{filename}.sql", "w", encoding="utf-8") as dump_file: subprocess.run( ["pg_dump", "-U", "tildes", "tildes"], stdout=dump_file, diff --git a/tildes/tildes/lib/database.py b/tildes/tildes/lib/database.py index e3d17bc..73e569f 100644 --- a/tildes/tildes/lib/database.py +++ b/tildes/tildes/lib/database.py @@ -68,7 +68,7 @@ class CIText(UserDefinedType): def get_col_spec(self, **kw: Any) -> str: """Return the type name (for creating columns and so on).""" - # pylint: disable=no-self-use,unused-argument + # pylint: disable=unused-argument return "CITEXT" def bind_processor(self, dialect: Dialect) -> Callable: diff --git a/tildes/tildes/lib/datetime.py b/tildes/tildes/lib/datetime.py index ae20688..54e19cb 100644 --- a/tildes/tildes/lib/datetime.py +++ b/tildes/tildes/lib/datetime.py @@ -43,6 +43,8 @@ class SimpleHoursPeriod: hours = count elif unit == "d": hours = count * 24 + else: + raise ValueError("Invalid time period") return cls(hours=hours) diff --git a/tildes/tildes/models/group/group_wiki_page.py b/tildes/tildes/models/group/group_wiki_page.py index 6e2404a..4b4a95b 100644 --- a/tildes/tildes/models/group/group_wiki_page.py +++ b/tildes/tildes/models/group/group_wiki_page.py @@ -130,7 +130,7 @@ class GroupWikiPage(DatabaseModel): def markdown(self) -> Optional[str]: """Return the wiki page's markdown.""" try: - return self.file_path.read_text().rstrip("\r\n") + return self.file_path.read_text(encoding="utf-8").rstrip("\r\n") except FileNotFoundError: return None @@ -141,7 +141,7 @@ class GroupWikiPage(DatabaseModel): if not new_markdown.endswith("\n"): new_markdown = new_markdown + "\n" - self.file_path.write_text(new_markdown) + self.file_path.write_text(new_markdown, encoding="utf-8") def edit(self, new_markdown: str, user: User, edit_message: str) -> None: """Set the page's markdown, render its HTML, and commit the repo.""" @@ -156,9 +156,10 @@ class GroupWikiPage(DatabaseModel): repo = Repository(self.BASE_PATH) author = Signature(user.username, user.username) - repo.index.read() - repo.index.add(str(self.file_path.relative_to(self.BASE_PATH))) - repo.index.write() + index = repo.index # type: ignore + index.read() + index.add(str(self.file_path.relative_to(self.BASE_PATH))) + index.write() # Prepend the group name and page path to the edit message - if you change the # format of this, make sure to also change the page-editing template to match @@ -169,6 +170,6 @@ class GroupWikiPage(DatabaseModel): author, author, edit_message, - repo.index.write_tree(), + index.write_tree(), [repo.head.target], ) diff --git a/tildes/tildes/models/model_query.py b/tildes/tildes/models/model_query.py index 74cde0d..3eaec0d 100644 --- a/tildes/tildes/models/model_query.py +++ b/tildes/tildes/models/model_query.py @@ -14,6 +14,7 @@ from sqlalchemy.orm import Load, undefer from sqlalchemy.orm.query import Query +# pylint: disable=invalid-name ModelType = TypeVar("ModelType") diff --git a/tildes/tildes/models/pagination.py b/tildes/tildes/models/pagination.py index 9727502..cf46fc6 100644 --- a/tildes/tildes/models/pagination.py +++ b/tildes/tildes/models/pagination.py @@ -16,6 +16,7 @@ from tildes.lib.id import id36_to_id, id_to_id36 from .model_query import ModelQuery +# pylint: disable=invalid-name ModelType = TypeVar("ModelType") @@ -136,9 +137,11 @@ class PaginatedQuery(ModelQuery): # an upper bound if the sort order is *ascending* is_anchor_upper_bound = not self.sort_desc + # pylint: disable=possibly-used-before-assignment subquery = self._anchor_subquery(anchor_id) # restrict the results to items on the right "side" of the anchor item + # pylint: disable=possibly-used-before-assignment if is_anchor_upper_bound: query = query.filter(func.row(*self.sorting_columns) < subquery) else: diff --git a/tildes/tildes/schemas/fields.py b/tildes/tildes/schemas/fields.py index ad8770b..48ff185 100644 --- a/tildes/tildes/schemas/fields.py +++ b/tildes/tildes/schemas/fields.py @@ -34,7 +34,7 @@ class Enum(Field): self._enum_class = enum_class def _serialize( - self, value: enum.Enum, attr: str, obj: object, **kwargs: Any + self, value: enum.Enum, attr: str | None, obj: object, **kwargs: Any ) -> str: """Serialize the enum value - lowercase version of its name.""" return value.name.lower() @@ -89,7 +89,7 @@ class ShortTimePeriod(Field): def _serialize( self, value: Optional[SimpleHoursPeriod], - attr: str, + attr: str | None, obj: object, **kwargs: Any, ) -> Optional[str]: @@ -131,7 +131,9 @@ class Markdown(Field): return value - def _serialize(self, value: str, attr: str, obj: object, **kwargs: Any) -> str: + def _serialize( + self, value: str, attr: str | None, obj: object, **kwargs: Any + ) -> str: """Serialize the value (no-op in this case).""" return value @@ -166,7 +168,9 @@ class SimpleString(Field): """Deserialize the string, removing/replacing as necessary.""" return simplify_string(value) - def _serialize(self, value: str, attr: str, obj: object, **kwargs: Any) -> str: + def _serialize( + self, value: str, attr: str | None, obj: object, **kwargs: Any + ) -> str: """Serialize the value (no-op in this case).""" return value @@ -179,7 +183,11 @@ class Ltree(Field): VALID_CHARS_REGEX = re.compile("^[A-Za-z0-9_.]+$") def _serialize( - self, value: sqlalchemy_utils.Ltree, attr: str, obj: object, **kwargs: Any + self, + value: sqlalchemy_utils.Ltree, + attr: str | None, + obj: object, + **kwargs: Any, ) -> str: """Serialize the Ltree value - use the (string) path.""" return value.path diff --git a/tildes/tildes/views/bookmarks.py b/tildes/tildes/views/bookmarks.py index 4c64fcc..4c68b01 100644 --- a/tildes/tildes/views/bookmarks.py +++ b/tildes/tildes/views/bookmarks.py @@ -32,7 +32,7 @@ def get_bookmarks( if post_type == "comment": post_cls = Comment bookmark_cls = CommentBookmark - elif post_type == "topic": + else: post_cls = Topic bookmark_cls = TopicBookmark diff --git a/tildes/tildes/views/decorators.py b/tildes/tildes/views/decorators.py index 6312178..677d3b8 100644 --- a/tildes/tildes/views/decorators.py +++ b/tildes/tildes/views/decorators.py @@ -16,9 +16,7 @@ from webargs import pyramidparser def use_kwargs( - argmap: Union[Schema, dict[str, Union[Field, type]]], - location: str = "query", - **kwargs: Any + argmap: Union[Schema, dict[str, Field]], location: str = "query", **kwargs: Any ) -> Callable: """Wrap the webargs @use_kwargs decorator with preferred default modifications. diff --git a/tildes/tildes/views/votes.py b/tildes/tildes/views/votes.py index 7f812ec..ba5f798 100644 --- a/tildes/tildes/views/votes.py +++ b/tildes/tildes/views/votes.py @@ -32,7 +32,7 @@ def get_voted_posts( if post_type == "comment": post_cls = Comment vote_cls = CommentVote - elif post_type == "topic": + else: post_cls = Topic vote_cls = TopicVote