Browse Source

Re-queue topic for some consumers on link edit

After editing a topic's link, we want to re-process it through the
scrapers, setting the domain in its metadata, etc.
merge-requests/64/head
Deimos 6 years ago
parent
commit
bd350495a4
  1. 32
      tildes/alembic/versions/4ebc3ca32b48_send_rabbitmq_message_on_link_edit.py
  2. 2
      tildes/consumers/topic_embedly_extractor.py
  3. 2
      tildes/consumers/topic_metadata_generator.py
  4. 2
      tildes/consumers/topic_youtube_scraper.py
  5. 7
      tildes/sql/init/triggers/topics/rabbitmq.sql

32
tildes/alembic/versions/4ebc3ca32b48_send_rabbitmq_message_on_link_edit.py

@ -0,0 +1,32 @@
"""Send rabbitmq message on link edit
Revision ID: 4ebc3ca32b48
Revises: 24014adda7c3
Create Date: 2019-03-15 00:59:57.713065
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = "4ebc3ca32b48"
down_revision = "24014adda7c3"
branch_labels = None
depends_on = None
def upgrade():
op.execute(
"""
CREATE TRIGGER send_rabbitmq_message_for_topic_link_edit
AFTER UPDATE ON topics
FOR EACH ROW
WHEN (OLD.link IS DISTINCT FROM NEW.link)
EXECUTE PROCEDURE send_rabbitmq_message_for_topic('link_edited');
"""
)
def downgrade():
op.execute("DROP TRIGGER send_rabbitmq_message_for_topic_link_edit ON topics")

2
tildes/consumers/topic_embedly_extractor.py

@ -93,5 +93,5 @@ if __name__ == "__main__":
TopicEmbedlyExtractor( TopicEmbedlyExtractor(
embedly_api_key, embedly_api_key,
queue_name="topic_embedly_extractor.q", queue_name="topic_embedly_extractor.q",
routing_keys=["topic.created"],
routing_keys=["topic.created", "topic.link_edited"],
).consume_queue() ).consume_queue()

2
tildes/consumers/topic_metadata_generator.py

@ -77,5 +77,5 @@ class TopicMetadataGenerator(PgsqlQueueConsumer):
if __name__ == "__main__": if __name__ == "__main__":
TopicMetadataGenerator( TopicMetadataGenerator(
queue_name="topic_metadata_generator.q", queue_name="topic_metadata_generator.q",
routing_keys=["topic.created", "topic.edited"],
routing_keys=["topic.created", "topic.edited", "topic.link_edited"],
).consume_queue() ).consume_queue()

2
tildes/consumers/topic_youtube_scraper.py

@ -96,5 +96,5 @@ if __name__ == "__main__":
TopicYoutubeScraper( TopicYoutubeScraper(
youtube_api_key, youtube_api_key,
queue_name="topic_youtube_scraper.q", queue_name="topic_youtube_scraper.q",
routing_keys=["topic.created"],
routing_keys=["topic.created", "topic.link_edited"],
).consume_queue() ).consume_queue()

7
tildes/sql/init/triggers/topics/rabbitmq.sql

@ -32,3 +32,10 @@ CREATE TRIGGER send_rabbitmq_message_for_topic_edit
FOR EACH ROW FOR EACH ROW
WHEN (OLD.markdown IS DISTINCT FROM NEW.markdown) WHEN (OLD.markdown IS DISTINCT FROM NEW.markdown)
EXECUTE PROCEDURE send_rabbitmq_message_for_topic('edited'); EXECUTE PROCEDURE send_rabbitmq_message_for_topic('edited');
CREATE TRIGGER send_rabbitmq_message_for_topic_link_edit
AFTER UPDATE ON topics
FOR EACH ROW
WHEN (OLD.link IS DISTINCT FROM NEW.link)
EXECUTE PROCEDURE send_rabbitmq_message_for_topic('link_edited');
Loading…
Cancel
Save