From 1c34ca4f76a592ad88ee299488e690f598eda6c5 Mon Sep 17 00:00:00 2001 From: Deimos Date: Wed, 2 Oct 2019 21:14:26 -0600 Subject: [PATCH] Add scheduled topics (no UI yet) This adds the backend for scheduled topics, which can be set up to post at a certain time and then (optionally) repeat on a schedule. Currently, these topics must be text topics, and can have their title, markdown, and tags set up. They can be configured to be posted by a particular user, but if no user is chosen they will be posted by a (newly added) generic user named "Tildes" that is intended to be used for "owning" automatic actions like this. --- salt/salt/cronjobs.sls | 5 + .../20b5f07e5f80_add_topic_schedule_table.py | 75 +++++++++++++++ tildes/requirements-to-freeze.txt | 1 + tildes/scripts/post_scheduled_topics.py | 43 +++++++++ tildes/sql/init/insert_base_data.sql | 4 + tildes/tests/test_datetime.py | 15 ++- tildes/tildes/database_models.py | 8 +- tildes/tildes/lib/database.py | 27 +++++- tildes/tildes/lib/datetime.py | 11 +++ tildes/tildes/models/topic/__init__.py | 1 + tildes/tildes/models/topic/topic_schedule.py | 96 +++++++++++++++++++ 11 files changed, 283 insertions(+), 3 deletions(-) create mode 100644 tildes/alembic/versions/20b5f07e5f80_add_topic_schedule_table.py create mode 100644 tildes/scripts/post_scheduled_topics.py create mode 100644 tildes/tildes/models/topic/topic_schedule.py diff --git a/salt/salt/cronjobs.sls b/salt/salt/cronjobs.sls index 8b075a9..b6cafd6 100644 --- a/salt/salt/cronjobs.sls +++ b/salt/salt/cronjobs.sls @@ -12,6 +12,11 @@ generate-site-icons-css-cronjob: - user: {{ app_username }} - minute: '*/5' +post-scheduled-topics-cronjob: + cron.present: + - name: {{ bin_dir }}/python -c "from scripts.post_scheduled_topics import post_scheduled_topics; post_scheduled_topics('{{ app_dir }}/{{ pillar['ini_file'] }}')" + - user: {{ app_username }} + update-common-topic-tags-cronjob: cron.present: - name: {{ bin_dir }}/python -c "from scripts.update_groups_common_topic_tags import update_common_topic_tags; update_common_topic_tags('{{ app_dir }}/{{ pillar['ini_file'] }}')" diff --git a/tildes/alembic/versions/20b5f07e5f80_add_topic_schedule_table.py b/tildes/alembic/versions/20b5f07e5f80_add_topic_schedule_table.py new file mode 100644 index 0000000..fe01983 --- /dev/null +++ b/tildes/alembic/versions/20b5f07e5f80_add_topic_schedule_table.py @@ -0,0 +1,75 @@ +"""Add topic_schedule table + +Revision ID: 20b5f07e5f80 +Revises: d56e71257a86 +Create Date: 2019-10-02 22:08:13.324006 + +""" +from alembic import op +import sqlalchemy as sa + +from tildes.lib.database import ArrayOfLtree + + +# revision identifiers, used by Alembic. +revision = "20b5f07e5f80" +down_revision = "d56e71257a86" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "topic_schedule", + sa.Column("schedule_id", sa.Integer(), nullable=False), + sa.Column("group_id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=True), + sa.Column( + "created_time", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("NOW()"), + nullable=False, + ), + sa.Column("title", sa.Text(), nullable=False), + sa.Column("markdown", sa.Text(), nullable=False), + sa.Column("tags", ArrayOfLtree(), server_default="{}", nullable=False), + sa.Column("next_post_time", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("recurrence_rule", sa.Text(), nullable=True), + sa.ForeignKeyConstraint( + ["group_id"], + ["groups.group_id"], + name=op.f("fk_topic_schedule_group_id_groups"), + ), + sa.ForeignKeyConstraint( + ["user_id"], ["users.user_id"], name=op.f("fk_topic_schedule_user_id_users") + ), + sa.PrimaryKeyConstraint("schedule_id", name=op.f("pk_topic_schedule")), + ) + op.create_index( + op.f("ix_topic_schedule_group_id"), "topic_schedule", ["group_id"], unique=False + ) + op.create_index( + op.f("ix_topic_schedule_next_post_time"), + "topic_schedule", + ["next_post_time"], + unique=False, + ) + op.create_check_constraint("title_length", "topic_schedule", "LENGTH(title) <= 200") + + # add the generic Tildes user (used to post scheduled topics by default) + op.execute( + """ + INSERT INTO users (user_id, username, password_hash) + VALUES(-1, 'Tildes', '') + ON CONFLICT DO NOTHING + """ + ) + + +def downgrade(): + op.drop_constraint("ck_topic_schedule_title_length", "topic_schedule") + op.drop_index(op.f("ix_topic_schedule_next_post_time"), table_name="topic_schedule") + op.drop_index(op.f("ix_topic_schedule_group_id"), table_name="topic_schedule") + op.drop_table("topic_schedule") + + # don't delete the Tildes user, won't work if it posted any topics diff --git a/tildes/requirements-to-freeze.txt b/tildes/requirements-to-freeze.txt index 55e7f43..e811ed6 100644 --- a/tildes/requirements-to-freeze.txt +++ b/tildes/requirements-to-freeze.txt @@ -30,6 +30,7 @@ pyramid-tm pyramid-webassets pytest pytest-mock +python-dateutil PyYAML # needs to be installed separately for webassets qrcode requests diff --git a/tildes/scripts/post_scheduled_topics.py b/tildes/scripts/post_scheduled_topics.py new file mode 100644 index 0000000..ef0dce4 --- /dev/null +++ b/tildes/scripts/post_scheduled_topics.py @@ -0,0 +1,43 @@ +# Copyright (c) 2019 Tildes contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Script to post any scheduled topics that are due.""" + +from typing import Optional + +from sqlalchemy.orm.session import Session + +from tildes.lib.database import get_session_from_config +from tildes.lib.datetime import utc_now +from tildes.models.topic import TopicSchedule + + +def _get_next_due_topic(db_session: Session) -> Optional[TopicSchedule]: + """Get the next due topic (if any). + + Note that this also locks the topic's row with FOR UPDATE as well as using SKIP + LOCKED. This should (hypothetically) mean that multiple instances of this script + can run concurrently safely and will not attempt to post the same topics. + """ + return ( + db_session.query(TopicSchedule) + .filter(TopicSchedule.next_post_time <= utc_now()) # type: ignore + .order_by(TopicSchedule.next_post_time) + .with_for_update(skip_locked=True) + .first() + ) + + +def post_scheduled_topics(config_path: str) -> None: + """Post all scheduled topics that are due to be posted.""" + db_session = get_session_from_config(config_path) + + due_topic = _get_next_due_topic(db_session) + + while due_topic: + db_session.add(due_topic.create_topic()) + due_topic.advance_schedule() + db_session.add(due_topic) + db_session.commit() + + due_topic = _get_next_due_topic(db_session) diff --git a/tildes/sql/init/insert_base_data.sql b/tildes/sql/init/insert_base_data.sql index 9b1b3bc..fc45262 100644 --- a/tildes/sql/init/insert_base_data.sql +++ b/tildes/sql/init/insert_base_data.sql @@ -5,3 +5,7 @@ -- outside the retention period, and similar uses INSERT INTO users (user_id, username, password_hash) VALUES (0, 'unknown user', ''); + +-- add a generic "Tildes" user to attribute automatic actions to +INSERT INTO users (user_id, username, password_hash) +VALUES (-1, 'Tildes', ''); diff --git a/tildes/tests/test_datetime.py b/tildes/tests/test_datetime.py index 7826174..3eb02d7 100644 --- a/tildes/tests/test_datetime.py +++ b/tildes/tests/test_datetime.py @@ -3,7 +3,9 @@ from datetime import datetime, timedelta, timezone -from tildes.lib.datetime import descriptive_timedelta, utc_now +from dateutil.rrule import rrulestr + +from tildes.lib.datetime import descriptive_timedelta, rrule_to_str, utc_now def test_utc_now_has_timezone(): @@ -60,3 +62,14 @@ def test_above_second_descriptive_timedelta(): """Ensure it starts describing time in seconds above 1 second.""" test_time = utc_now() - timedelta(seconds=1, microseconds=100) assert descriptive_timedelta(test_time) == "1 second ago" + + +def test_rrule_to_str(): + """Ensure the rrule_to_str method is giving an expected result. + + This is mostly intended as protection in case dateutil ever changes the format + that they output when converting a rrule to string. + """ + test_str = "FREQ=MONTHLY;INTERVAL=2" + rrule = rrulestr(test_str) + assert rrule_to_str(rrule) == test_str diff --git a/tildes/tildes/database_models.py b/tildes/tildes/database_models.py index 4139832..553632e 100644 --- a/tildes/tildes/database_models.py +++ b/tildes/tildes/database_models.py @@ -16,5 +16,11 @@ from tildes.models.group import Group, GroupSubscription from tildes.models.log import Log from tildes.models.message import MessageConversation, MessageReply from tildes.models.scraper import ScraperResult -from tildes.models.topic import Topic, TopicBookmark, TopicVisit, TopicVote +from tildes.models.topic import ( + Topic, + TopicBookmark, + TopicSchedule, + TopicVisit, + TopicVote, +) from tildes.models.user import User, UserGroupSettings, UserInviteCode diff --git a/tildes/tildes/lib/database.py b/tildes/tildes/lib/database.py index 4f4937d..9e8fc26 100644 --- a/tildes/tildes/lib/database.py +++ b/tildes/tildes/lib/database.py @@ -6,15 +6,18 @@ import enum from typing import Any, Callable, List, Optional +from dateutil.rrule import rrulestr from pyramid.paster import bootstrap from sqlalchemy import cast, func from sqlalchemy.dialects.postgresql import ARRAY from sqlalchemy.engine.interfaces import Dialect from sqlalchemy.orm.session import Session -from sqlalchemy.types import UserDefinedType +from sqlalchemy.types import Text, TypeDecorator, UserDefinedType from sqlalchemy_utils import LtreeType from sqlalchemy_utils.types.ltree import LQUERY +from tildes.lib.datetime import rrule_to_str + # https://www.postgresql.org/docs/current/static/errcodes-appendix.html NOT_NULL_ERROR_CODE = 23502 @@ -145,3 +148,25 @@ class ArrayOfLtree(ARRAY): # pylint: disable=too-many-ancestors return self.op("?")(cast(other, ARRAY(LQUERY))) else: return self.op("~")(other) + + +class RecurrenceRule(TypeDecorator): + """Stores a dateutil rrule in the database as text.""" + + # pylint: disable=abstract-method + + impl = Text + + def process_bind_param(self, value, dialect): # type: ignore + """Convert the rrule value to a string to store it.""" + if value is None: + return value + + return rrule_to_str(value) + + def process_result_value(self, value, dialect): # type: ignore + """Convert the stored string to an rrule.""" + if value is None: + return value + + return rrulestr(value) diff --git a/tildes/tildes/lib/datetime.py b/tildes/tildes/lib/datetime.py index 64793ef..ec4863f 100644 --- a/tildes/tildes/lib/datetime.py +++ b/tildes/tildes/lib/datetime.py @@ -7,6 +7,8 @@ import re from datetime import datetime, timedelta, timezone from typing import Any, Optional +from dateutil.rrule import rrule + from ago import human @@ -156,3 +158,12 @@ def adaptive_date( format_str += ", %Y" return target.strftime(format_str) + + +def rrule_to_str(rrule_obj: rrule) -> str: + """Convert a dateutil rrule to its string definition. + + dateutil does this natively, but it always includes the start date, which we don't + always need or want to be storing. This gives only the rrule definition. + """ + return str(rrule_obj).split("RRULE:")[1] diff --git a/tildes/tildes/models/topic/__init__.py b/tildes/tildes/models/topic/__init__.py index 2255085..4ae5432 100644 --- a/tildes/tildes/models/topic/__init__.py +++ b/tildes/tildes/models/topic/__init__.py @@ -3,5 +3,6 @@ from .topic import EDIT_GRACE_PERIOD, Topic from .topic_bookmark import TopicBookmark from .topic_query import TopicQuery +from .topic_schedule import TopicSchedule from .topic_visit import TopicVisit from .topic_vote import TopicVote diff --git a/tildes/tildes/models/topic/topic_schedule.py b/tildes/tildes/models/topic/topic_schedule.py new file mode 100644 index 0000000..2e3c6c7 --- /dev/null +++ b/tildes/tildes/models/topic/topic_schedule.py @@ -0,0 +1,96 @@ +# Copyright (c) 2019 Tildes contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Contains the TopicSchedule class.""" + +from datetime import datetime +from typing import List, Optional + +from dateutil.rrule import rrule +from sqlalchemy import CheckConstraint, Column, ForeignKey, Integer, Text, TIMESTAMP +from sqlalchemy.orm import relationship +from sqlalchemy.orm.session import Session +from sqlalchemy.sql.expression import text +from sqlalchemy_utils import Ltree + +from tildes.lib.database import ArrayOfLtree, RecurrenceRule +from tildes.models import DatabaseModel +from tildes.models.group import Group +from tildes.models.topic import Topic +from tildes.models.user import User +from tildes.schemas.topic import TITLE_MAX_LENGTH + + +class TopicSchedule(DatabaseModel): + """Model for scheduled topics (auto-posted, often repeatedly on a schedule).""" + + __tablename__ = "topic_schedule" + + schedule_id: int = Column(Integer, primary_key=True) + group_id: int = Column( + Integer, ForeignKey("groups.group_id"), nullable=False, index=True + ) + user_id: Optional[int] = Column(Integer, ForeignKey("users.user_id"), nullable=True) + created_time: datetime = Column( + TIMESTAMP(timezone=True), nullable=False, server_default=text("NOW()") + ) + title: str = Column( + Text, + CheckConstraint(f"LENGTH(title) <= {TITLE_MAX_LENGTH}", name="title_length"), + nullable=False, + ) + markdown: str = Column(Text, nullable=False) + tags: List[Ltree] = Column(ArrayOfLtree, nullable=False, server_default="{}") + next_post_time: Optional[datetime] = Column( + TIMESTAMP(timezone=True), nullable=True, index=True + ) + recurrence_rule: Optional[rrule] = Column(RecurrenceRule, nullable=True) + + group: Group = relationship("Group", innerjoin=True) + user: Optional[User] = relationship("User") + + def __init__( + self, + group: Group, + title: str, + markdown: str, + tags: List[str], + next_post_time: datetime, + recurrence_rule: Optional[rrule] = None, + user: Optional[User] = None, + ) -> None: + """Create a new scheduled topic.""" + # pylint: disable=too-many-arguments + self.group = group + self.title = title + self.markdown = markdown + self.tags = [Ltree(tag) for tag in tags] + self.next_post_time = next_post_time + self.recurrence_rule = recurrence_rule + self.user = user + + def create_topic(self) -> Topic: + """Create and return an actual Topic for this scheduled topic.""" + # if no user is specified, use the "generic"/automatic user (ID -1) + if self.user: + user = self.user + else: + user = ( + Session.object_session(self) + .query(User) + .filter(User.user_id == -1) + .one() + ) + + topic = Topic.create_text_topic(self.group, user, self.title, self.markdown) + topic.tags = [str(tag) for tag in self.tags] + + return topic + + def advance_schedule(self) -> None: + """Advance the schedule, setting next_post_time appropriately.""" + if self.recurrence_rule: + rule = self.recurrence_rule.replace(dtstart=self.next_post_time) + self.next_post_time = rule.after(self.next_post_time) + else: + self.next_post_time = None