Add scheduled topics (no UI yet)

This adds the backend for scheduled topics, which can be set up to post
at a certain time and then (optionally) repeat on a schedule.

Currently, these topics must be text topics, and can have their title,
markdown, and tags set up. They can be configured to be posted by a
particular user, but if no user is chosen they will be posted by a
(newly added) generic user named "Tildes" that is intended to be used
for "owning" automatic actions like this.

salt/salt/cronjobs.sls

@@ -12,6 +12,11 @@ generate-site-icons-css-cronjob:
     - user: {{ app_username }}
     - minute: '*/5'
 
+post-scheduled-topics-cronjob:
+  cron.present:
+    - name: {{ bin_dir }}/python -c "from scripts.post_scheduled_topics import post_scheduled_topics; post_scheduled_topics('{{ app_dir }}/{{ pillar['ini_file'] }}')"
+    - user: {{ app_username }}
+
 update-common-topic-tags-cronjob:
   cron.present:
     - name: {{ bin_dir }}/python -c "from scripts.update_groups_common_topic_tags import update_common_topic_tags; update_common_topic_tags('{{ app_dir }}/{{ pillar['ini_file'] }}')"

tildes/alembic/versions/20b5f07e5f80_add_topic_schedule_table.py

@@ -0,0 +1,75 @@
+"""Add topic_schedule table
+
+Revision ID: 20b5f07e5f80
+Revises: d56e71257a86
+Create Date: 2019-10-02 22:08:13.324006
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+from tildes.lib.database import ArrayOfLtree
+
+
+# revision identifiers, used by Alembic.
+revision = "20b5f07e5f80"
+down_revision = "d56e71257a86"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "topic_schedule",
+        sa.Column("schedule_id", sa.Integer(), nullable=False),
+        sa.Column("group_id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.Integer(), nullable=True),
+        sa.Column(
+            "created_time",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("NOW()"),
+            nullable=False,
+        ),
+        sa.Column("title", sa.Text(), nullable=False),
+        sa.Column("markdown", sa.Text(), nullable=False),
+        sa.Column("tags", ArrayOfLtree(), server_default="{}", nullable=False),
+        sa.Column("next_post_time", sa.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("recurrence_rule", sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["group_id"],
+            ["groups.group_id"],
+            name=op.f("fk_topic_schedule_group_id_groups"),
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"], ["users.user_id"], name=op.f("fk_topic_schedule_user_id_users")
+        ),
+        sa.PrimaryKeyConstraint("schedule_id", name=op.f("pk_topic_schedule")),
+    )
+    op.create_index(
+        op.f("ix_topic_schedule_group_id"), "topic_schedule", ["group_id"], unique=False
+    )
+    op.create_index(
+        op.f("ix_topic_schedule_next_post_time"),
+        "topic_schedule",
+        ["next_post_time"],
+        unique=False,
+    )
+    op.create_check_constraint("title_length", "topic_schedule", "LENGTH(title) <= 200")
+
+    # add the generic Tildes user (used to post scheduled topics by default)
+    op.execute(
+        """
+        INSERT INTO users (user_id, username, password_hash)
+        VALUES(-1, 'Tildes', '')
+        ON CONFLICT DO NOTHING
+    """
+    )
+
+
+def downgrade():
+    op.drop_constraint("ck_topic_schedule_title_length", "topic_schedule")
+    op.drop_index(op.f("ix_topic_schedule_next_post_time"), table_name="topic_schedule")
+    op.drop_index(op.f("ix_topic_schedule_group_id"), table_name="topic_schedule")
+    op.drop_table("topic_schedule")
+
+    # don't delete the Tildes user, won't work if it posted any topics

tildes/requirements-to-freeze.txt

@@ -30,6 +30,7 @@ pyramid-tm
 pyramid-webassets
 pytest
 pytest-mock
+python-dateutil
 PyYAML  # needs to be installed separately for webassets
 qrcode
 requests

tildes/scripts/post_scheduled_topics.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2019 Tildes contributors <code@tildes.net>
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+"""Script to post any scheduled topics that are due."""
+
+from typing import Optional
+
+from sqlalchemy.orm.session import Session
+
+from tildes.lib.database import get_session_from_config
+from tildes.lib.datetime import utc_now
+from tildes.models.topic import TopicSchedule
+
+
+def _get_next_due_topic(db_session: Session) -> Optional[TopicSchedule]:
+    """Get the next due topic (if any).
+
+    Note that this also locks the topic's row with FOR UPDATE as well as using SKIP
+    LOCKED. This should (hypothetically) mean that multiple instances of this script
+    can run concurrently safely and will not attempt to post the same topics.
+    """
+    return (
+        db_session.query(TopicSchedule)
+        .filter(TopicSchedule.next_post_time <= utc_now())  # type: ignore
+        .order_by(TopicSchedule.next_post_time)
+        .with_for_update(skip_locked=True)
+        .first()
+    )
+
+
+def post_scheduled_topics(config_path: str) -> None:
+    """Post all scheduled topics that are due to be posted."""
+    db_session = get_session_from_config(config_path)
+
+    due_topic = _get_next_due_topic(db_session)
+
+    while due_topic:
+        db_session.add(due_topic.create_topic())
+        due_topic.advance_schedule()
+        db_session.add(due_topic)
+        db_session.commit()
+
+        due_topic = _get_next_due_topic(db_session)

tildes/sql/init/insert_base_data.sql

@@ -5,3 +5,7 @@
 -- outside the retention period, and similar uses
 INSERT INTO users (user_id, username, password_hash)
 VALUES (0, 'unknown user', '');
+
+-- add a generic "Tildes" user to attribute automatic actions to
+INSERT INTO users (user_id, username, password_hash)
+VALUES (-1, 'Tildes', '');

tildes/tests/test_datetime.py

@@ -3,7 +3,9 @@
 
 from datetime import datetime, timedelta, timezone
 
-from tildes.lib.datetime import descriptive_timedelta, utc_now
+from dateutil.rrule import rrulestr
+
+from tildes.lib.datetime import descriptive_timedelta, rrule_to_str, utc_now
 
 
 def test_utc_now_has_timezone():
@@ -60,3 +62,14 @@ def test_above_second_descriptive_timedelta():
     """Ensure it starts describing time in seconds above 1 second."""
     test_time = utc_now() - timedelta(seconds=1, microseconds=100)
     assert descriptive_timedelta(test_time) == "1 second ago"
+
+
+def test_rrule_to_str():
+    """Ensure the rrule_to_str method is giving an expected result.
+
+    This is mostly intended as protection in case dateutil ever changes the format
+    that they output when converting a rrule to string.
+    """
+    test_str = "FREQ=MONTHLY;INTERVAL=2"
+    rrule = rrulestr(test_str)
+    assert rrule_to_str(rrule) == test_str

tildes/tildes/database_models.py

@@ -16,5 +16,11 @@ from tildes.models.group import Group, GroupSubscription
 from tildes.models.log import Log
 from tildes.models.message import MessageConversation, MessageReply
 from tildes.models.scraper import ScraperResult
-from tildes.models.topic import Topic, TopicBookmark, TopicVisit, TopicVote
+from tildes.models.topic import (
+    Topic,
+    TopicBookmark,
+    TopicSchedule,
+    TopicVisit,
+    TopicVote,
+)
 from tildes.models.user import User, UserGroupSettings, UserInviteCode

tildes/tildes/lib/database.py

@@ -6,15 +6,18 @@
 import enum
 from typing import Any, Callable, List, Optional
 
+from dateutil.rrule import rrulestr
 from pyramid.paster import bootstrap
 from sqlalchemy import cast, func
 from sqlalchemy.dialects.postgresql import ARRAY
 from sqlalchemy.engine.interfaces import Dialect
 from sqlalchemy.orm.session import Session
-from sqlalchemy.types import UserDefinedType
+from sqlalchemy.types import Text, TypeDecorator, UserDefinedType
 from sqlalchemy_utils import LtreeType
 from sqlalchemy_utils.types.ltree import LQUERY
 
+from tildes.lib.datetime import rrule_to_str
+
 
 # https://www.postgresql.org/docs/current/static/errcodes-appendix.html
 NOT_NULL_ERROR_CODE = 23502
@@ -145,3 +148,25 @@ class ArrayOfLtree(ARRAY):  # pylint: disable=too-many-ancestors
                 return self.op("?")(cast(other, ARRAY(LQUERY)))
             else:
                 return self.op("~")(other)
+
+
+class RecurrenceRule(TypeDecorator):
+    """Stores a dateutil rrule in the database as text."""
+
+    # pylint: disable=abstract-method
+
+    impl = Text
+
+    def process_bind_param(self, value, dialect):  # type: ignore
+        """Convert the rrule value to a string to store it."""
+        if value is None:
+            return value
+
+        return rrule_to_str(value)
+
+    def process_result_value(self, value, dialect):  # type: ignore
+        """Convert the stored string to an rrule."""
+        if value is None:
+            return value
+
+        return rrulestr(value)

tildes/tildes/lib/datetime.py

@@ -7,6 +7,8 @@ import re
 from datetime import datetime, timedelta, timezone
 from typing import Any, Optional
 
+from dateutil.rrule import rrule
+
 from ago import human
 
 
@@ -156,3 +158,12 @@ def adaptive_date(
         format_str += ", %Y"
 
     return target.strftime(format_str)
+
+
+def rrule_to_str(rrule_obj: rrule) -> str:
+    """Convert a dateutil rrule to its string definition.
+
+    dateutil does this natively, but it always includes the start date, which we don't
+    always need or want to be storing. This gives only the rrule definition.
+    """
+    return str(rrule_obj).split("RRULE:")[1]

tildes/tildes/models/topic/__init__.py

@@ -3,5 +3,6 @@
 from .topic import EDIT_GRACE_PERIOD, Topic
 from .topic_bookmark import TopicBookmark
 from .topic_query import TopicQuery
+from .topic_schedule import TopicSchedule
 from .topic_visit import TopicVisit
 from .topic_vote import TopicVote

tildes/tildes/models/topic/topic_schedule.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2019 Tildes contributors <code@tildes.net>
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+"""Contains the TopicSchedule class."""
+
+from datetime import datetime
+from typing import List, Optional
+
+from dateutil.rrule import rrule
+from sqlalchemy import CheckConstraint, Column, ForeignKey, Integer, Text, TIMESTAMP
+from sqlalchemy.orm import relationship
+from sqlalchemy.orm.session import Session
+from sqlalchemy.sql.expression import text
+from sqlalchemy_utils import Ltree
+
+from tildes.lib.database import ArrayOfLtree, RecurrenceRule
+from tildes.models import DatabaseModel
+from tildes.models.group import Group
+from tildes.models.topic import Topic
+from tildes.models.user import User
+from tildes.schemas.topic import TITLE_MAX_LENGTH
+
+
+class TopicSchedule(DatabaseModel):
+    """Model for scheduled topics (auto-posted, often repeatedly on a schedule)."""
+
+    __tablename__ = "topic_schedule"
+
+    schedule_id: int = Column(Integer, primary_key=True)
+    group_id: int = Column(
+        Integer, ForeignKey("groups.group_id"), nullable=False, index=True
+    )
+    user_id: Optional[int] = Column(Integer, ForeignKey("users.user_id"), nullable=True)
+    created_time: datetime = Column(
+        TIMESTAMP(timezone=True), nullable=False, server_default=text("NOW()")
+    )
+    title: str = Column(
+        Text,
+        CheckConstraint(f"LENGTH(title) <= {TITLE_MAX_LENGTH}", name="title_length"),
+        nullable=False,
+    )
+    markdown: str = Column(Text, nullable=False)
+    tags: List[Ltree] = Column(ArrayOfLtree, nullable=False, server_default="{}")
+    next_post_time: Optional[datetime] = Column(
+        TIMESTAMP(timezone=True), nullable=True, index=True
+    )
+    recurrence_rule: Optional[rrule] = Column(RecurrenceRule, nullable=True)
+
+    group: Group = relationship("Group", innerjoin=True)
+    user: Optional[User] = relationship("User")
+
+    def __init__(
+        self,
+        group: Group,
+        title: str,
+        markdown: str,
+        tags: List[str],
+        next_post_time: datetime,
+        recurrence_rule: Optional[rrule] = None,
+        user: Optional[User] = None,
+    ) -> None:
+        """Create a new scheduled topic."""
+        # pylint: disable=too-many-arguments
+        self.group = group
+        self.title = title
+        self.markdown = markdown
+        self.tags = [Ltree(tag) for tag in tags]
+        self.next_post_time = next_post_time
+        self.recurrence_rule = recurrence_rule
+        self.user = user
+
+    def create_topic(self) -> Topic:
+        """Create and return an actual Topic for this scheduled topic."""
+        # if no user is specified, use the "generic"/automatic user (ID -1)
+        if self.user:
+            user = self.user
+        else:
+            user = (
+                Session.object_session(self)
+                .query(User)
+                .filter(User.user_id == -1)
+                .one()
+            )
+
+        topic = Topic.create_text_topic(self.group, user, self.title, self.markdown)
+        topic.tags = [str(tag) for tag in self.tags]
+
+        return topic
+
+    def advance_schedule(self) -> None:
+        """Advance the schedule, setting next_post_time appropriately."""
+        if self.recurrence_rule:
+            rule = self.recurrence_rule.replace(dtstart=self.next_post_time)
+            self.next_post_time = rule.after(self.next_post_time)
+        else:
+            self.next_post_time = None