Use prebuilt Python 3.11 instead of building Python 3.9

Closes tildes-community/tildes-cf#14

See merge request tildes-community/tildes-cf!8

ansible/roles/postgresql_tildes_dbs/tasks/main.yml

@@ -4,7 +4,7 @@
     name:
       - gcc
       - libpq-dev
-      - python3-dev
+      - python{{ python_version }}-dev
 
 - name: Install packages needed by Ansible community plugins
   pip:

ansible/roles/python/tasks/main.yml

@@ -1,56 +1,10 @@
 ---
-- name: Check if the correct version of Python is already installed
-  stat:
-    path: /usr/local/bin/python{{ python_version }}
-  register: python_binary
-
-- name: Download and install Python
-  when: not python_binary.stat.exists
-  block:
-    - name: Download Python source code
-      get_url:
-        dest: /tmp/python.tar.gz
-        url: https://www.python.org/ftp/python/{{ python_full_version }}/Python-{{ python_full_version }}.tgz
-        checksum: sha256:1e71f006222666e0a39f5a47be8221415c22c4dd8f25334cc41aee260b3d379e
-
-    - name: Create temp directory to extract Python to
-      file:
-        path: /tmp/python
-        state: directory
-
-    - name: Extract Python
-      unarchive:
-        remote_src: true
-        src: /tmp/python.tar.gz
-        dest: /tmp/python
-        extra_opts:
-          - --strip-components=1
-
-    - name: Install build dependencies for Python
+- name: Install Python and dependencies needed by packages
   apt:
     name:
-          - make
-          - build-essential
-          - libssl-dev
-          - zlib1g-dev
-          - libbz2-dev
-          - libreadline-dev
-          - libsqlite3-dev
-          - wget
-          - curl
-          - llvm
-          - libncurses5-dev
-          - libncursesw5-dev
-          - xz-utils
-          - tk-dev
-
-    - name: Build and install Python (this can take a long time)
-      shell:
-        chdir: /tmp/python
-        cmd: |
-          ./configure --enable-optimizations --with-ensurepip=install
-          make
-          make altinstall
+      - python{{ python_version }}
+      - python{{ python_version }}-venv
+      - libgit2-dev
 
 - name: Create dir for venvs
   file:

ansible/vars.yml

@@ -5,7 +5,6 @@ bin_dir: "{{ venv_dir }}/bin"
 
 static_sites_dir: /opt/tildes-static-sites
 
-python_full_version: 3.9.20
-python_version: "{{ python_full_version.rpartition('.')[0] }}"
+python_version: "3.11"
 
 is_docker: "{{ ansible_facts['virtualization_type'] == 'container' }}"

tildes/consumers/site_icon_downloader.py

@@ -10,7 +10,7 @@ from typing import Optional
 
 import publicsuffix
 import requests
-from PIL import Image
+from PIL import Image, IcoImagePlugin
 
 from tildes.enums import ScraperType
 from tildes.lib.event_stream import EventStreamConsumer, Message
@@ -80,6 +80,10 @@ class SiteIconDownloader(EventStreamConsumer):
             return None
 
         if favicon.format == "ICO":
+            assert isinstance(
+                favicon, IcoImagePlugin.IcoImageFile
+            )  # tell mypy the type is more restricted now
+
             # get the 32x32 size if it's present, otherwise resize the largest one
             if (32, 32) in favicon.ico.sizes():
                 return favicon.ico.getimage((32, 32))

tildes/consumers/topic_metadata_generator.py

@@ -45,6 +45,8 @@ class TopicMetadataGenerator(EventStreamConsumer):
             new_metadata = self._generate_text_metadata(topic)
         elif topic.is_link_type:
             new_metadata = self._generate_link_metadata(topic)
+        else:
+            new_metadata = {}
 
         # update the topic's content_metadata in a way that won't wipe out any existing
         # values, and can handle the column being null

tildes/prospector.yaml

@@ -29,6 +29,8 @@ pylint:
   disable:
     - bad-continuation  # let Black handle line-wrapping
     - comparison-with-callable  # seems to have a lot of false positives
+    - consider-using-f-string  # TBD if helpful [2025-01-16]
+    - consider-using-generator  # TBD if helpful [2025-01-16]
     - cyclic-import  # not sure what's triggering this, doesn't seem to work correctly
     - logging-fstring-interpolation  # rather use f-strings than worry about this
     - no-else-return  # elif after return - could refactor to enable this check
@@ -39,8 +41,10 @@ pylint:
     - too-many-branches  # almost never helpful
     - too-many-instance-attributes  # models have many instance attributes
     - too-many-locals  # almost never helpful
+    - too-many-positional-arguments  # TBD if helpful [2025-01-16]
     - too-many-public-methods  # almost never helpful
     - too-many-return-statements  # almost never helpful
     - too-many-statements  # almost never helpful
     - ungrouped-imports  # let isort handle this
     - unnecessary-pass  # I prefer using pass, even when it's not technically necessary
+    - use-yield-from  # TBD if helpful [2025-01-16]

tildes/pytest.ini

@@ -4,7 +4,6 @@ addopts = -p no:cacheprovider --strict-markers
 filterwarnings =
     ignore::DeprecationWarning
     ignore::PendingDeprecationWarning
-    ignore::yaml.YAMLLoadWarning
 markers =
     html_validation: mark a test as one that validates HTML using the Nu HTML Checker (very slow)
     webtest: mark a test as one that uses the WebTest library, which goes through the actual WSGI app and involves using HTTP/HTML (more of a "functional test" than "unit test")

tildes/requirements-dev.in

@@ -3,7 +3,7 @@ black
 freezegun
 html5validator
 mypy
-prospector @ git+https://github.com/Deimos/prospector.git#egg=prospector
+prospector
 pyramid-debugtoolbar
 pytest
 pytest-mock

tildes/requirements-dev.txt

@@ -2,20 +2,20 @@ ago==0.0.93
 alembic==1.6.5
 appdirs==1.4.4
 argon2-cffi==20.1.0
-astroid==2.6.5
-attrs==21.2.0
+astroid==3.3.8
+attrs==24.3.0
 backcall==0.2.0
 beautifulsoup4==4.9.3
 black==21.7b0
 bleach==3.3.1
 certifi==2021.5.30
-cffi==1.14.6
+cffi==1.17.1
 charset-normalizer==2.0.3
 click==8.0.1
 cornice==5.2.0
 decorator==5.0.9
 dodgy==0.2.1
-flake8==3.9.2
+flake8==7.1.1
 flake8-polyfill==1.0.2
 freezegun==1.1.0
 gunicorn==20.1.0
@@ -23,52 +23,52 @@ html5lib==1.1
 html5validator==0.4.0
 hupper==1.10.3
 idna==3.2
-iniconfig==1.1.1
-invoke==1.6.0
+iniconfig==2.0.0
+invoke==2.2.0
 ipython==7.25.0
 ipython-genutils==0.2.0
 isort==5.9.2
 jedi==0.18.0
 jinja2==3.0.1
 lazy-object-proxy==1.6.0
-lupa==1.9
+lupa==2.4
 mako==1.1.4
 markupsafe==2.0.1
-marshmallow==3.13.0
+marshmallow==3.25.1
 matplotlib-inline==0.1.2
-mccabe==0.6.1
-mypy==1.13.0
+mccabe==0.7.0
+mypy==1.14.1
 mypy-extensions==1.0.0
-packaging==23.2
+packaging==24.2
 parso==0.8.2
 pastedeploy==2.1.1
 pathspec==0.9.0
 pep517==0.11.0
-pep8-naming==0.12.0
+pep8-naming==0.10.0
 pexpect==4.8.0
 pickleshare==0.7.5
-pillow==8.3.1
+pillow==11.1.0
 pip-tools==6.2.0
 plaster==1.0
 plaster-pastedeploy==0.7
-pluggy==0.13.1
+pluggy==1.5.0
 prometheus-client==0.11.0
 prompt-toolkit==3.0.19
-git+https://github.com/Deimos/prospector.git#egg=prospector
-psycopg2==2.9.1
+prospector==1.13.3
+psycopg2==2.9.10
 ptyprocess==0.7.0
 publicsuffix2==2.20160818
-py==1.10.0
-pycodestyle==2.7.0
+py==1.11.0
+pycodestyle==2.12.1
 pycparser==2.20
 pydocstyle==6.1.1
-pyflakes==2.3.1
-pygit2==1.6.1
+pyflakes==3.2.0
+pygit2==1.17.0
 pygments==2.9.0
-pylint==2.9.5
-pylint-plugin-utils==0.6
+pylint==3.3.3
+pylint-plugin-utils==0.8.2
 pyotp==2.6.0
-pyparsing==2.4.7
+pyparsing==3.2.1
 pyramid==1.10.8
 pyramid-debugtoolbar==4.9
 pyramid-ipython==0.2
@@ -77,17 +77,17 @@ pyramid-mako==1.1.0
 pyramid-session-redis==1.5.0
 pyramid-tm==2.4
 pyramid-webassets==0.10
-pytest==6.2.4
-pytest-mock==3.6.1
+pytest==8.3.4
+pytest-mock==3.14.0
 python-dateutil==2.8.2
 python-editor==1.0.4
-pyyaml==5.4.1
+pyyaml==6.0.2
 qrcode==7.2
 redis==3.5.3
 regex==2021.7.6
 repoze.lru==0.7
 requests==2.26.0
-requirements-detector==0.7
+requirements-detector==1.3.2
 sentry-sdk==1.3.0
 setoptconf==0.3.0
 six==1.16.0
@@ -119,7 +119,7 @@ webencodings==0.5.1
 webob==1.8.7
 webtest==2.0.35
 wheel==0.36.2
-wrapt==1.12.1
+wrapt==1.17.2
 zope.deprecation==4.4.0
 zope.interface==5.4.0
 zope.sqlalchemy==1.5

tildes/requirements.txt

@@ -5,7 +5,7 @@ backcall==0.2.0
 beautifulsoup4==4.9.3
 bleach==3.3.1
 certifi==2021.5.30
-cffi==1.14.6
+cffi==1.17.1
 charset-normalizer==2.0.3
 click==8.0.1
 cornice==5.2.0
@@ -14,36 +14,36 @@ gunicorn==20.1.0
 html5lib==1.1
 hupper==1.10.3
 idna==3.2
-invoke==1.6.0
+invoke==2.2.0
 ipython==7.25.0
 ipython-genutils==0.2.0
 jedi==0.18.0
 jinja2==3.0.1
-lupa==1.9
+lupa==2.4
 mako==1.1.4
 markupsafe==2.0.1
-marshmallow==3.13.0
+marshmallow==3.25.1
 matplotlib-inline==0.1.2
-packaging==23.2
+packaging==24.2
 parso==0.8.2
 pastedeploy==2.1.1
 pep517==0.11.0
 pexpect==4.8.0
 pickleshare==0.7.5
-pillow==8.3.1
+pillow==11.1.0
 pip-tools==6.2.0
 plaster==1.0
 plaster-pastedeploy==0.7
 prometheus-client==0.11.0
 prompt-toolkit==3.0.19
-psycopg2==2.9.1
+psycopg2==2.9.10
 ptyprocess==0.7.0
 publicsuffix2==2.20160818
 pycparser==2.20
-pygit2==1.6.1
+pygit2==1.17.0
 pygments==2.9.0
 pyotp==2.6.0
-pyparsing==2.4.7
+pyparsing==3.2.1
 pyramid==1.10.8
 pyramid-ipython==0.2
 pyramid-jinja2==2.8
@@ -52,7 +52,7 @@ pyramid-tm==2.4
 pyramid-webassets==0.10
 python-dateutil==2.8.2
 python-editor==1.0.4
-pyyaml==5.4.1
+pyyaml==6.0.2
 qrcode==7.2
 redis==3.5.3
 requests==2.26.0
@@ -75,7 +75,7 @@ webassets==2.0
 webencodings==0.5.1
 webob==1.8.7
 wheel==0.36.2
-wrapt==1.12.1
+wrapt==1.17.2
 zope.deprecation==4.4.0
 zope.interface==5.4.0
 zope.sqlalchemy==1.5

tildes/scripts/backup_database.py

@@ -34,7 +34,7 @@ def create_encrypted_backup(gpg_recipient: str) -> str:
     filename = datetime.now().strftime(FILENAME_FORMAT)
 
     # dump the database to a file
-    with open(f"{filename}.sql", "w") as dump_file:
+    with open(f"{filename}.sql", "w", encoding="utf-8") as dump_file:
         subprocess.run(
             ["pg_dump", "-U", "tildes", "tildes"],
             stdout=dump_file,

tildes/tildes/lib/database.py

@@ -68,7 +68,7 @@ class CIText(UserDefinedType):
 
     def get_col_spec(self, **kw: Any) -> str:
         """Return the type name (for creating columns and so on)."""
-        # pylint: disable=no-self-use,unused-argument
+        # pylint: disable=unused-argument
         return "CITEXT"
 
     def bind_processor(self, dialect: Dialect) -> Callable:

tildes/tildes/lib/datetime.py

@@ -43,6 +43,8 @@ class SimpleHoursPeriod:
             hours = count
         elif unit == "d":
             hours = count * 24
+        else:
+            raise ValueError("Invalid time period")
 
         return cls(hours=hours)
 

tildes/tildes/models/group/group_wiki_page.py

@@ -130,7 +130,7 @@ class GroupWikiPage(DatabaseModel):
     def markdown(self) -> Optional[str]:
         """Return the wiki page's markdown."""
         try:
-            return self.file_path.read_text().rstrip("\r\n")
+            return self.file_path.read_text(encoding="utf-8").rstrip("\r\n")
         except FileNotFoundError:
             return None
 
@@ -141,7 +141,7 @@ class GroupWikiPage(DatabaseModel):
         if not new_markdown.endswith("\n"):
             new_markdown = new_markdown + "\n"
 
-        self.file_path.write_text(new_markdown)
+        self.file_path.write_text(new_markdown, encoding="utf-8")
 
     def edit(self, new_markdown: str, user: User, edit_message: str) -> None:
         """Set the page's markdown, render its HTML, and commit the repo."""
@@ -156,9 +156,10 @@ class GroupWikiPage(DatabaseModel):
         repo = Repository(self.BASE_PATH)
         author = Signature(user.username, user.username)
 
-        repo.index.read()
-        repo.index.add(str(self.file_path.relative_to(self.BASE_PATH)))
-        repo.index.write()
+        index = repo.index  # type: ignore
+        index.read()
+        index.add(str(self.file_path.relative_to(self.BASE_PATH)))
+        index.write()
 
         # Prepend the group name and page path to the edit message - if you change the
         # format of this, make sure to also change the page-editing template to match
@@ -169,6 +170,6 @@ class GroupWikiPage(DatabaseModel):
             author,
             author,
             edit_message,
-            repo.index.write_tree(),
+            index.write_tree(),
             [repo.head.target],
         )

tildes/tildes/models/model_query.py

@@ -14,6 +14,7 @@ from sqlalchemy.orm import Load, undefer
 from sqlalchemy.orm.query import Query
 
 
+# pylint: disable=invalid-name
 ModelType = TypeVar("ModelType")
 
 

tildes/tildes/models/pagination.py

@@ -16,6 +16,7 @@ from tildes.lib.id import id36_to_id, id_to_id36
 from .model_query import ModelQuery
 
 
+# pylint: disable=invalid-name
 ModelType = TypeVar("ModelType")
 
 
@@ -136,9 +137,11 @@ class PaginatedQuery(ModelQuery):
             # an upper bound if the sort order is *ascending*
             is_anchor_upper_bound = not self.sort_desc
 
+        # pylint: disable=possibly-used-before-assignment
         subquery = self._anchor_subquery(anchor_id)
 
         # restrict the results to items on the right "side" of the anchor item
+        # pylint: disable=possibly-used-before-assignment
         if is_anchor_upper_bound:
             query = query.filter(func.row(*self.sorting_columns) < subquery)
         else:

tildes/tildes/schemas/fields.py

@@ -34,7 +34,7 @@ class Enum(Field):
         self._enum_class = enum_class
 
     def _serialize(
-        self, value: enum.Enum, attr: str, obj: object, **kwargs: Any
+        self, value: enum.Enum, attr: str | None, obj: object, **kwargs: Any
     ) -> str:
         """Serialize the enum value - lowercase version of its name."""
         return value.name.lower()
@@ -89,7 +89,7 @@ class ShortTimePeriod(Field):
     def _serialize(
         self,
         value: Optional[SimpleHoursPeriod],
-        attr: str,
+        attr: str | None,
         obj: object,
         **kwargs: Any,
     ) -> Optional[str]:
@@ -131,7 +131,9 @@ class Markdown(Field):
 
         return value
 
-    def _serialize(self, value: str, attr: str, obj: object, **kwargs: Any) -> str:
+    def _serialize(
+        self, value: str, attr: str | None, obj: object, **kwargs: Any
+    ) -> str:
         """Serialize the value (no-op in this case)."""
         return value
 
@@ -166,7 +168,9 @@ class SimpleString(Field):
         """Deserialize the string, removing/replacing as necessary."""
         return simplify_string(value)
 
-    def _serialize(self, value: str, attr: str, obj: object, **kwargs: Any) -> str:
+    def _serialize(
+        self, value: str, attr: str | None, obj: object, **kwargs: Any
+    ) -> str:
         """Serialize the value (no-op in this case)."""
         return value
 
@@ -179,7 +183,11 @@ class Ltree(Field):
     VALID_CHARS_REGEX = re.compile("^[A-Za-z0-9_.]+$")
 
     def _serialize(
-        self, value: sqlalchemy_utils.Ltree, attr: str, obj: object, **kwargs: Any
+        self,
+        value: sqlalchemy_utils.Ltree,
+        attr: str | None,
+        obj: object,
+        **kwargs: Any,
     ) -> str:
         """Serialize the Ltree value - use the (string) path."""
         return value.path

tildes/tildes/views/bookmarks.py

@@ -32,7 +32,7 @@ def get_bookmarks(
     if post_type == "comment":
         post_cls = Comment
         bookmark_cls = CommentBookmark
-    elif post_type == "topic":
+    else:
         post_cls = Topic
         bookmark_cls = TopicBookmark
 

tildes/tildes/views/decorators.py

@@ -16,9 +16,7 @@ from webargs import pyramidparser
 
 
 def use_kwargs(
-    argmap: Union[Schema, dict[str, Union[Field, type]]],
-    location: str = "query",
-    **kwargs: Any
+    argmap: Union[Schema, dict[str, Field]], location: str = "query", **kwargs: Any
 ) -> Callable:
     """Wrap the webargs @use_kwargs decorator with preferred default modifications.
 

tildes/tildes/views/votes.py

@@ -32,7 +32,7 @@ def get_voted_posts(
     if post_type == "comment":
         post_cls = Comment
         vote_cls = CommentVote
-    elif post_type == "topic":
+    else:
         post_cls = Topic
         vote_cls = TopicVote