From bd350495a49d1133330854d8df08fa8eac474bea Mon Sep 17 00:00:00 2001 From: Deimos Date: Thu, 14 Mar 2019 19:05:01 -0600 Subject: [PATCH] Re-queue topic for some consumers on link edit After editing a topic's link, we want to re-process it through the scrapers, setting the domain in its metadata, etc. --- ...2b48_send_rabbitmq_message_on_link_edit.py | 32 +++++++++++++++++++ tildes/consumers/topic_embedly_extractor.py | 2 +- tildes/consumers/topic_metadata_generator.py | 2 +- tildes/consumers/topic_youtube_scraper.py | 2 +- tildes/sql/init/triggers/topics/rabbitmq.sql | 7 ++++ 5 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 tildes/alembic/versions/4ebc3ca32b48_send_rabbitmq_message_on_link_edit.py diff --git a/tildes/alembic/versions/4ebc3ca32b48_send_rabbitmq_message_on_link_edit.py b/tildes/alembic/versions/4ebc3ca32b48_send_rabbitmq_message_on_link_edit.py new file mode 100644 index 0000000..7209446 --- /dev/null +++ b/tildes/alembic/versions/4ebc3ca32b48_send_rabbitmq_message_on_link_edit.py @@ -0,0 +1,32 @@ +"""Send rabbitmq message on link edit + +Revision ID: 4ebc3ca32b48 +Revises: 24014adda7c3 +Create Date: 2019-03-15 00:59:57.713065 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "4ebc3ca32b48" +down_revision = "24014adda7c3" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + """ + CREATE TRIGGER send_rabbitmq_message_for_topic_link_edit + AFTER UPDATE ON topics + FOR EACH ROW + WHEN (OLD.link IS DISTINCT FROM NEW.link) + EXECUTE PROCEDURE send_rabbitmq_message_for_topic('link_edited'); + """ + ) + + +def downgrade(): + op.execute("DROP TRIGGER send_rabbitmq_message_for_topic_link_edit ON topics") diff --git a/tildes/consumers/topic_embedly_extractor.py b/tildes/consumers/topic_embedly_extractor.py index 79653e8..67c006a 100644 --- a/tildes/consumers/topic_embedly_extractor.py +++ b/tildes/consumers/topic_embedly_extractor.py @@ -93,5 +93,5 @@ if __name__ == "__main__": TopicEmbedlyExtractor( embedly_api_key, queue_name="topic_embedly_extractor.q", - routing_keys=["topic.created"], + routing_keys=["topic.created", "topic.link_edited"], ).consume_queue() diff --git a/tildes/consumers/topic_metadata_generator.py b/tildes/consumers/topic_metadata_generator.py index 9b31f19..44ceed6 100644 --- a/tildes/consumers/topic_metadata_generator.py +++ b/tildes/consumers/topic_metadata_generator.py @@ -77,5 +77,5 @@ class TopicMetadataGenerator(PgsqlQueueConsumer): if __name__ == "__main__": TopicMetadataGenerator( queue_name="topic_metadata_generator.q", - routing_keys=["topic.created", "topic.edited"], + routing_keys=["topic.created", "topic.edited", "topic.link_edited"], ).consume_queue() diff --git a/tildes/consumers/topic_youtube_scraper.py b/tildes/consumers/topic_youtube_scraper.py index 4089a28..e9ecdb2 100644 --- a/tildes/consumers/topic_youtube_scraper.py +++ b/tildes/consumers/topic_youtube_scraper.py @@ -96,5 +96,5 @@ if __name__ == "__main__": TopicYoutubeScraper( youtube_api_key, queue_name="topic_youtube_scraper.q", - routing_keys=["topic.created"], + routing_keys=["topic.created", "topic.link_edited"], ).consume_queue() diff --git a/tildes/sql/init/triggers/topics/rabbitmq.sql b/tildes/sql/init/triggers/topics/rabbitmq.sql index 670e6ee..91a5a28 100644 --- a/tildes/sql/init/triggers/topics/rabbitmq.sql +++ b/tildes/sql/init/triggers/topics/rabbitmq.sql @@ -32,3 +32,10 @@ CREATE TRIGGER send_rabbitmq_message_for_topic_edit FOR EACH ROW WHEN (OLD.markdown IS DISTINCT FROM NEW.markdown) EXECUTE PROCEDURE send_rabbitmq_message_for_topic('edited'); + + +CREATE TRIGGER send_rabbitmq_message_for_topic_link_edit + AFTER UPDATE ON topics + FOR EACH ROW + WHEN (OLD.link IS DISTINCT FROM NEW.link) + EXECUTE PROCEDURE send_rabbitmq_message_for_topic('link_edited');