Browse Source

Set a maximum length for Redis event streams

Nothing should get anywhere near this limit for a very long time, but
it's better to have it set up in case anything ever gets out of control.
merge-requests/89/head
Deimos 5 years ago
parent
commit
bb1ccfe590
  1. 12
      tildes/scripts/postgresql_redis_bridge.py

12
tildes/scripts/postgresql_redis_bridge.py

@ -20,6 +20,11 @@ from tildes.lib.event_stream import REDIS_KEY_PREFIX
NOTIFY_CHANNEL = "postgresql_events"
# Stream entries seem to generally require about 20-50 bytes each, depending on which
# data fields they include. A max length of a million should mean that any individual
# stream shouldn't be able to take up more memory than 50 MB or so.
STREAM_MAX_LENGTH = 1_000_000
def postgresql_redis_bridge(config_path: str) -> None:
"""Listen for NOTIFY events and add them to Redis streams."""
@ -61,7 +66,12 @@ def postgresql_redis_bridge(config_path: str) -> None:
except json.decoder.JSONDecodeError:
continue
pipe.xadd(f"{REDIS_KEY_PREFIX}{stream_name}", fields)
pipe.xadd(
f"{REDIS_KEY_PREFIX}{stream_name}",
fields,
maxlen=STREAM_MAX_LENGTH,
approximate=True,
)
pipe.execute()

Loading…
Cancel
Save