Browse Source

Re-wrap comments with new max line-length

Black won't re-wrap comments because it has no way to determine when
that's a "safe" thing to do (that might break deliberate line-breaks).
So this is a quick pass through to re-wrap most multi-line comments and
docstrings to the new max line-length of 88.
merge-requests/26/head
Deimos 6 years ago
parent
commit
4d8998d8f9
  1. 5
      tildes/gunicorn_config.py
  2. 47
      tildes/scripts/breached_passwords.py
  3. 16
      tildes/scripts/clean_private_data.py
  4. 4
      tildes/scripts/initialize_db.py
  5. 36
      tildes/tests/conftest.py
  6. 4
      tildes/tests/test_comment_user_mentions.py
  7. 4
      tildes/tests/test_ratelimit.py
  8. 4
      tildes/tests/test_simplestring_field.py
  9. 10
      tildes/tests/test_string.py
  10. 6
      tildes/tests/webtests/test_user_page.py
  11. 8
      tildes/tildes/__init__.py
  12. 4
      tildes/tildes/api.py
  13. 24
      tildes/tildes/auth.py
  14. 20
      tildes/tildes/database.py
  15. 6
      tildes/tildes/enums.py
  16. 4
      tildes/tildes/json.py
  17. 9
      tildes/tildes/lib/__init__.py
  18. 17
      tildes/tildes/lib/amqp.py
  19. 10
      tildes/tildes/lib/database.py
  20. 22
      tildes/tildes/lib/datetime.py
  21. 8
      tildes/tildes/lib/hash.py
  22. 9
      tildes/tildes/lib/id.py
  23. 122
      tildes/tildes/lib/markdown.py
  24. 4
      tildes/tildes/lib/password.py
  25. 49
      tildes/tildes/lib/ratelimit.py
  26. 70
      tildes/tildes/lib/string.py
  27. 4
      tildes/tildes/metrics.py
  28. 28
      tildes/tildes/models/comment/comment.py
  29. 23
      tildes/tildes/models/comment/comment_notification.py
  30. 5
      tildes/tildes/models/comment/comment_query.py
  31. 23
      tildes/tildes/models/comment/comment_tree.py
  32. 4
      tildes/tildes/models/comment/comment_vote.py
  33. 37
      tildes/tildes/models/database_model.py
  34. 10
      tildes/tildes/models/group/group.py
  35. 5
      tildes/tildes/models/group/group_query.py
  36. 4
      tildes/tildes/models/group/group_subscription.py
  37. 5
      tildes/tildes/models/log/log.py
  38. 51
      tildes/tildes/models/message/message.py
  39. 38
      tildes/tildes/models/model_query.py
  40. 66
      tildes/tildes/models/pagination.py
  41. 22
      tildes/tildes/models/topic/topic.py
  42. 20
      tildes/tildes/models/topic/topic_query.py
  43. 14
      tildes/tildes/models/topic/topic_visit.py
  44. 4
      tildes/tildes/models/topic/topic_vote.py
  45. 14
      tildes/tildes/models/user/user.py
  46. 8
      tildes/tildes/models/user/user_invite_code.py
  47. 6
      tildes/tildes/resources/__init__.py
  48. 4
      tildes/tildes/resources/comment.py
  49. 8
      tildes/tildes/resources/group.py
  50. 4
      tildes/tildes/resources/topic.py
  51. 6
      tildes/tildes/routes.py
  52. 16
      tildes/tildes/schemas/__init__.py
  53. 4
      tildes/tildes/schemas/fields.py
  54. 10
      tildes/tildes/schemas/topic.py
  55. 12
      tildes/tildes/schemas/user.py
  56. 4
      tildes/tildes/views/__init__.py
  57. 29
      tildes/tildes/views/api/web/comment.py
  58. 12
      tildes/tildes/views/api/web/exceptions.py
  59. 4
      tildes/tildes/views/api/web/group.py
  60. 4
      tildes/tildes/views/api/web/topic.py
  61. 18
      tildes/tildes/views/api/web/user.py
  62. 12
      tildes/tildes/views/decorators.py
  63. 10
      tildes/tildes/views/message.py
  64. 8
      tildes/tildes/views/metrics.py
  65. 4
      tildes/tildes/views/notifications.py
  66. 12
      tildes/tildes/views/register.py
  67. 20
      tildes/tildes/views/topic.py
  68. 6
      tildes/tildes/views/user.py

5
tildes/gunicorn_config.py

@ -6,9 +6,8 @@ from prometheus_client import multiprocess
def child_exit(server, worker): # type: ignore def child_exit(server, worker): # type: ignore
"""Mark worker processes as dead for Prometheus when the worker exits. """Mark worker processes as dead for Prometheus when the worker exits.
Note that this uses the child_exit hook instead of worker_exit so that
it's handled by the master process (and will still be called if a worker
crashes).
Note that this uses the child_exit hook instead of worker_exit so that it's handled
by the master process (and will still be called if a worker crashes).
""" """
# pylint: disable=unused-argument # pylint: disable=unused-argument
multiprocess.mark_process_dead(worker.pid) multiprocess.mark_process_dead(worker.pid)

47
tildes/scripts/breached_passwords.py

@ -1,17 +1,15 @@
"""Command-line tools for managing a breached-passwords bloom filter. """Command-line tools for managing a breached-passwords bloom filter.
This tool will help with creating and updating a bloom filter in Redis (using
ReBloom: https://github.com/RedisLabsModules/rebloom) to hold hashes for
passwords that have been revealed through data breaches (to prevent users from
using these passwords here). The dumps are likely primarily sourced from Troy
Hunt's "Pwned Passwords" files:
This tool will help with creating and updating a bloom filter in Redis (using ReBloom:
https://github.com/RedisLabsModules/rebloom) to hold hashes for passwords that have been
revealed through data breaches (to prevent users from using these passwords here). The
dumps are likely primarily sourced from Troy Hunt's "Pwned Passwords" files:
https://haveibeenpwned.com/Passwords https://haveibeenpwned.com/Passwords
Specifically, the commands in this tool allow building the bloom filter
somewhere else, then the RDB file can be transferred to the production server.
Note that it is expected that a separate redis server instance is running
solely for holding this bloom filter. Replacing the RDB file will result in all
other keys being lost.
Specifically, the commands in this tool allow building the bloom filter somewhere else,
then the RDB file can be transferred to the production server. Note that it is expected
that a separate redis server instance is running solely for holding this bloom filter.
Replacing the RDB file will result in all other keys being lost.
Expected usage of this tool should look something like: Expected usage of this tool should look something like:
@ -20,8 +18,8 @@ On the machine building the bloom filter:
python breached_passwords.py addhashes pwned-passwords-1.0.txt python breached_passwords.py addhashes pwned-passwords-1.0.txt
python breached_passwords.py addhashes pwned-passwords-update-1.txt python breached_passwords.py addhashes pwned-passwords-update-1.txt
Then the RDB file can simply be transferred to the production server,
overwriting any previous RDB file.
Then the RDB file can simply be transferred to the production server, overwriting any
previous RDB file.
""" """
@ -90,14 +88,13 @@ def validate_init_error_rate(ctx: Any, param: Any, value: Any) -> float:
def init(estimate: int, error_rate: float) -> None: def init(estimate: int, error_rate: float) -> None:
"""Initialize a new bloom filter (destroying any existing one). """Initialize a new bloom filter (destroying any existing one).
It generally shouldn't be necessary to re-init a new bloom filter very
often with this command, only if the previous one was created with too low
of an estimate for number of passwords, or to change to a different false
positive rate. For choosing an estimate value, according to the ReBloom
documentation: "Performance will begin to degrade after adding more items
than this number. The actual degradation will depend on how far the limit
has been exceeded. Performance will degrade linearly as the number of
entries grow exponentially."
It generally shouldn't be necessary to re-init a new bloom filter very often with
this command, only if the previous one was created with too low of an estimate for
number of passwords, or to change to a different false positive rate. For choosing
an estimate value, according to the ReBloom documentation: "Performance will begin
to degrade after adding more items than this number. The actual degradation will
depend on how far the limit has been exceeded. Performance will degrade linearly as
the number of entries grow exponentially."
""" """
REDIS.delete(BREACHED_PASSWORDS_BF_KEY) REDIS.delete(BREACHED_PASSWORDS_BF_KEY)
@ -115,8 +112,8 @@ def init(estimate: int, error_rate: float) -> None:
def addhashes(filename: str) -> None: def addhashes(filename: str) -> None:
"""Add all hashes from a file to the bloom filter. """Add all hashes from a file to the bloom filter.
This uses the method of generating commands in Redis protocol and feeding
them into an instance of `redis-cli --pipe`, as recommended in
This uses the method of generating commands in Redis protocol and feeding them into
an instance of `redis-cli --pipe`, as recommended in
https://redis.io/topics/mass-insert https://redis.io/topics/mass-insert
""" """
# make sure the key exists and is a bloom filter # make sure the key exists and is a bloom filter
@ -146,9 +143,9 @@ def addhashes(filename: str) -> None:
for count, line in enumerate(open(filename), start=1): for count, line in enumerate(open(filename), start=1):
hashval = line.strip().lower() hashval = line.strip().lower()
# the Pwned Passwords hash lists now have a frequency count for each
# hash, which is separated from the hash with a colon, so we need to
# handle that if it's present
# the Pwned Passwords hash lists now have a frequency count for each hash, which
# is separated from the hash with a colon, so we need to handle that if it's
# present
hashval = hashval.split(":")[0] hashval = hashval.split(":")[0]
command = generate_redis_protocol("BF.ADD", BREACHED_PASSWORDS_BF_KEY, hashval) command = generate_redis_protocol("BF.ADD", BREACHED_PASSWORDS_BF_KEY, hashval)

16
tildes/scripts/clean_private_data.py

@ -24,8 +24,8 @@ RETENTION_PERIOD = timedelta(days=30)
def clean_all_data(config_path: str) -> None: def clean_all_data(config_path: str) -> None:
"""Clean all private/deleted data. """Clean all private/deleted data.
This should generally be the only function called in most cases, and will
initiate the full cleanup process.
This should generally be the only function called in most cases, and will initiate
the full cleanup process.
""" """
db_session = get_session_from_config(config_path) db_session = get_session_from_config(config_path)
@ -54,8 +54,8 @@ class DataCleaner:
def delete_old_log_entries(self) -> None: def delete_old_log_entries(self) -> None:
"""Delete all log entries older than the retention cutoff. """Delete all log entries older than the retention cutoff.
Note that this will also delete all entries from the child tables that
inherit from Log (LogTopics, etc.).
Note that this will also delete all entries from the child tables that inherit
from Log (LogTopics, etc.).
""" """
deleted = ( deleted = (
self.db_session.query(Log) self.db_session.query(Log)
@ -78,8 +78,8 @@ class DataCleaner:
def clean_old_deleted_comments(self) -> None: def clean_old_deleted_comments(self) -> None:
"""Clean the data of old deleted comments. """Clean the data of old deleted comments.
Change the comment's author to the "unknown user" (id 0), and delete
its contents.
Change the comment's author to the "unknown user" (id 0), and delete its
contents.
""" """
updated = ( updated = (
self.db_session.query(Comment) self.db_session.query(Comment)
@ -98,8 +98,8 @@ class DataCleaner:
def clean_old_deleted_topics(self) -> None: def clean_old_deleted_topics(self) -> None:
"""Clean the data of old deleted topics. """Clean the data of old deleted topics.
Change the topic's author to the "unknown user" (id 0), and delete its
title, contents, tags, and metadata.
Change the topic's author to the "unknown user" (id 0), and delete its title,
contents, tags, and metadata.
""" """
updated = ( updated = (
self.db_session.query(Topic) self.db_session.query(Topic)

4
tildes/scripts/initialize_db.py

@ -24,8 +24,8 @@ def initialize_db(config_path: str, alembic_config_path: Optional[str] = None) -
run_sql_scripts_in_dir("sql/init/", engine) run_sql_scripts_in_dir("sql/init/", engine)
# if an Alembic config file wasn't specified, assume it's alembic.ini in
# the same directory
# if an Alembic config file wasn't specified, assume it's alembic.ini in the same
# directory
if not alembic_config_path: if not alembic_config_path:
path = os.path.split(config_path)[0] path = os.path.split(config_path)[0]
alembic_config_path = os.path.join(path, "alembic.ini") alembic_config_path = os.path.join(path, "alembic.ini")

36
tildes/tests/conftest.py

@ -66,9 +66,9 @@ def overall_db_session(pyramid_config):
create_tables(session.connection()) create_tables(session.connection())
# SQL init scripts need to be executed "manually" instead of using psql
# like the normal database init process does, since the tables only exist
# inside this transaction
# SQL init scripts need to be executed "manually" instead of using psql like the
# normal database init process does, since the tables only exist inside this
# transaction
init_scripts_dir = "sql/init/" init_scripts_dir = "sql/init/"
for root, _, files in os.walk(init_scripts_dir): for root, _, files in os.walk(init_scripts_dir):
sql_files = [filename for filename in files if filename.endswith(".sql")] sql_files = [filename for filename in files if filename.endswith(".sql")]
@ -76,8 +76,8 @@ def overall_db_session(pyramid_config):
with open(os.path.join(root, sql_file)) as current_file: with open(os.path.join(root, sql_file)) as current_file:
session.execute(current_file.read()) session.execute(current_file.read())
# convert the Session to the wrapper class to enforce staying inside
# nested transactions in the test functions
# convert the Session to the wrapper class to enforce staying inside nested
# transactions in the test functions
session.__class__ = NestedSessionWrapper session.__class__ = NestedSessionWrapper
yield session yield session
@ -110,8 +110,8 @@ def db(overall_db_session):
@fixture(scope="session", autouse=True) @fixture(scope="session", autouse=True)
def overall_redis_session(): def overall_redis_session():
"""Create a session-level connection to a temporary redis server.""" """Create a session-level connection to a temporary redis server."""
# list of redis modules that need to be loaded (would be much nicer to do
# this automatically somehow, maybe reading from the real redis.conf?)
# list of redis modules that need to be loaded (would be much nicer to do this
# automatically somehow, maybe reading from the real redis.conf?)
redis_modules = ["/opt/redis-cell/libredis_cell.so"] redis_modules = ["/opt/redis-cell/libredis_cell.so"]
with RedisServer() as temp_redis_server: with RedisServer() as temp_redis_server:
@ -134,9 +134,9 @@ def redis(overall_redis_session):
@fixture(scope="session", autouse=True) @fixture(scope="session", autouse=True)
def session_user(sdb): def session_user(sdb):
"""Create a user named 'SessionUser' in the db for test session.""" """Create a user named 'SessionUser' in the db for test session."""
# note that some tests may depend on this username/password having these
# specific values, so make sure to search for and update those tests if you
# change the username or password for any reason
# note that some tests may depend on this username/password having these specific
# values, so make sure to search for and update those tests if you change the
# username or password for any reason
user = User("SessionUser", "session user password") user = User("SessionUser", "session user password")
sdb.add(user) sdb.add(user)
sdb.commit() sdb.commit()
@ -148,8 +148,8 @@ def session_user(sdb):
def session_user2(sdb): def session_user2(sdb):
"""Create a second user named 'OtherUser' in the db for test session. """Create a second user named 'OtherUser' in the db for test session.
This is useful for cases where two different users are needed, such as
when testing private messages.
This is useful for cases where two different users are needed, such as when testing
private messages.
""" """
user = User("OtherUser", "other user password") user = User("OtherUser", "other user password")
sdb.add(user) sdb.add(user)
@ -173,8 +173,8 @@ def base_app(overall_redis_session, sdb):
"""Configure a base WSGI app that webtest can create TestApps based on.""" """Configure a base WSGI app that webtest can create TestApps based on."""
testing_app = get_app("development.ini") testing_app = get_app("development.ini")
# replace the redis connection used by the redis-sessions library with a
# connection to the temporary server for this test session
# replace the redis connection used by the redis-sessions library with a connection
# to the temporary server for this test session
testing_app.app.registry._redis_sessions = overall_redis_session testing_app.app.registry._redis_sessions = overall_redis_session
def redis_factory(request): def redis_factory(request):
@ -183,8 +183,8 @@ def base_app(overall_redis_session, sdb):
testing_app.app.registry["redis_connection_factory"] = redis_factory testing_app.app.registry["redis_connection_factory"] = redis_factory
# replace the session factory function with one that will return the
# testing db session (inside a nested transaction)
# replace the session factory function with one that will return the testing db
# session (inside a nested transaction)
def session_factory(): def session_factory():
return sdb return sdb
@ -196,8 +196,8 @@ def base_app(overall_redis_session, sdb):
@fixture(scope="session") @fixture(scope="session")
def webtest(base_app): def webtest(base_app):
"""Create a webtest TestApp and log in as the SessionUser account in it.""" """Create a webtest TestApp and log in as the SessionUser account in it."""
# create the TestApp - note that specifying wsgi.url_scheme is necessary
# so that the secure cookies from the session library will work
# create the TestApp - note that specifying wsgi.url_scheme is necessary so that the
# secure cookies from the session library will work
app = TestApp( app = TestApp(
base_app, extra_environ={"wsgi.url_scheme": "https"}, cookiejar=CookieJar() base_app, extra_environ={"wsgi.url_scheme": "https"}, cookiejar=CookieJar()
) )

4
tildes/tests/test_comment_user_mentions.py

@ -68,8 +68,8 @@ def test_prevent_duplicate_notifications(db, user_list, topic):
"""Test that notifications are cleaned up for edits. """Test that notifications are cleaned up for edits.
Flow: Flow:
1. A comment is created by user A that mentions user B. Notifications
are generated, and yield A mentioning B.
1. A comment is created by user A that mentions user B. Notifications are
generated, and yield A mentioning B.
2. The comment is edited to mention C and not B. 2. The comment is edited to mention C and not B.
3. The comment is edited to mention B and C. 3. The comment is edited to mention B and C.
4. The comment is deleted. 4. The comment is deleted.

4
tildes/tests/test_ratelimit.py

@ -104,8 +104,8 @@ def test_time_until_retry(redis):
period = timedelta(seconds=60) period = timedelta(seconds=60)
limit = 2 limit = 2
# create an action with no burst allowed, which will force the actions to
# be spaced "evenly" across the limit
# create an action with no burst allowed, which will force the actions to be spaced
# "evenly" across the limit
action = RateLimitedAction( action = RateLimitedAction(
"test", period=period, limit=limit, max_burst=1, redis=redis "test", period=period, limit=limit, max_burst=1, redis=redis
) )

4
tildes/tests/test_simplestring_field.py

@ -13,8 +13,8 @@ class SimpleStringTestSchema(Schema):
def process_string(string): def process_string(string):
"""Deserialize a string with the field and return the "final" version. """Deserialize a string with the field and return the "final" version.
This also works for testing validation since .load() will raise a
ValidationError if an invalid string is attempted.
This also works for testing validation since .load() will raise a ValidationError if
an invalid string is attempted.
""" """
schema = SimpleStringTestSchema(strict=True) schema = SimpleStringTestSchema(strict=True)
result = schema.load({"subject": string}) result = schema.load({"subject": string})

10
tildes/tests/test_string.py

@ -96,17 +96,17 @@ def test_multibyte_url_slug():
def test_multibyte_conservative_truncation(): def test_multibyte_conservative_truncation():
"""Ensure truncating a multibyte url slug won't massively shorten it.""" """Ensure truncating a multibyte url slug won't massively shorten it."""
# this string has a comma as the 6th char which will be converted to an
# underscore, so if truncation amount isn't restricted, it would result in
# a 46-char slug instead of the full 100.
# this string has a comma as the 6th char which will be converted to an underscore,
# so if truncation amount isn't restricted, it would result in a 46-char slug
# instead of the full 100.
original = "パイソンは、汎用のプログラミング言語である" original = "パイソンは、汎用のプログラミング言語である"
assert len(convert_to_url_slug(original, 100)) == 100 assert len(convert_to_url_slug(original, 100)) == 100
def test_multibyte_whole_character_truncation(): def test_multibyte_whole_character_truncation():
"""Ensure truncation happens at the edge of a multibyte character.""" """Ensure truncation happens at the edge of a multibyte character."""
# each of these characters url-encodes to 3 bytes = 9 characters each, so
# only the first character should be included for all lengths from 9 - 17
# each of these characters url-encodes to 3 bytes = 9 characters each, so only the
# first character should be included for all lengths from 9 - 17
original = "コード" original = "コード"
for limit in range(9, 18): for limit in range(9, 18):
assert convert_to_url_slug(original, limit) == "%E3%82%B3" assert convert_to_url_slug(original, limit) == "%E3%82%B3"

6
tildes/tests/webtests/test_user_page.py

@ -1,9 +1,9 @@
def test_loggedout_username_leak(webtest_loggedout, session_user): def test_loggedout_username_leak(webtest_loggedout, session_user):
"""Ensure responses from existing and nonexistent users are the same. """Ensure responses from existing and nonexistent users are the same.
Since logged-out users are currently blocked from seeing user pages, this
makes sure that there isn't a data leak where it's possible to tell if a
particular username exists or not.
Since logged-out users are currently blocked from seeing user pages, this makes sure
that there isn't a data leak where it's possible to tell if a particular username
exists or not.
""" """
existing_user = webtest_loggedout.get( existing_user = webtest_loggedout.get(
"/user/" + session_user.username, expect_errors=True "/user/" + session_user.username, expect_errors=True

8
tildes/tildes/__init__.py

@ -38,8 +38,8 @@ def main(global_config: Dict[str, str], **settings: str) -> PrefixMiddleware:
config.add_request_method(is_safe_request_method, "is_safe_method", reify=True) config.add_request_method(is_safe_request_method, "is_safe_method", reify=True)
# Add the request.redis request method to access a redis connection. This
# is done in a bit of a strange way to support being overridden in tests.
# Add the request.redis request method to access a redis connection. This is done in
# a bit of a strange way to support being overridden in tests.
config.registry["redis_connection_factory"] = get_redis_connection config.registry["redis_connection_factory"] = get_redis_connection
# pylint: disable=unnecessary-lambda # pylint: disable=unnecessary-lambda
config.add_request_method( config.add_request_method(
@ -125,8 +125,8 @@ def current_listing_base_url(
) -> str: ) -> str:
"""Return the "base" url for the current listing route. """Return the "base" url for the current listing route.
The "base" url represents the current listing, including any filtering
options (or the fact that filters are disabled).
The "base" url represents the current listing, including any filtering options (or
the fact that filters are disabled).
The `query` argument allows adding query variables to the generated url. The `query` argument allows adding query variables to the generated url.
""" """

4
tildes/tildes/api.py

@ -19,8 +19,8 @@ class APIv0(Service):
super().__init__(name=name, path=path, **kwargs) super().__init__(name=name, path=path, **kwargs)
# Service.__init__ does this setup to support config.scan(), but it
# doesn't seem to inherit properly, so it needs to be done again here
# Service.__init__ does this setup to support config.scan(), but it doesn't seem
# to inherit properly, so it needs to be done again here
def callback(context: Any, name: Any, obj: Any) -> None: def callback(context: Any, name: Any, obj: Any) -> None:
# pylint: disable=unused-argument # pylint: disable=unused-argument
config = context.config.with_package(info.module) # noqa config = context.config.with_package(info.module) # noqa

24
tildes/tildes/auth.py

@ -15,10 +15,10 @@ from tildes.models.user import User
class DefaultRootFactory: class DefaultRootFactory:
"""Default root factory to grant everyone 'view' permission by default. """Default root factory to grant everyone 'view' permission by default.
Note that this will only be applied in cases where a view does not have a
factory specified at all (so request.context doesn't have a meaningful
value). Any classes that could be returned by a root factory must have
an __acl__ defined, they will not "fall back" to this one.
Note that this will only be applied in cases where a view does not have a factory
specified at all (so request.context doesn't have a meaningful value). Any classes
that could be returned by a root factory must have an __acl__ defined, they will not
"fall back" to this one.
""" """
__acl__ = ((Allow, Everyone, "view"),) __acl__ = ((Allow, Everyone, "view"),)
@ -42,8 +42,8 @@ def get_authenticated_user(request: Request) -> Optional[User]:
def auth_callback(user_id: int, request: Request) -> Optional[Sequence[str]]: def auth_callback(user_id: int, request: Request) -> Optional[Sequence[str]]:
"""Return authorization principals for a user_id from the session. """Return authorization principals for a user_id from the session.
This is a callback function needed by SessionAuthenticationPolicy. It
should return None if the user_id does not exist (such as a deleted user).
This is a callback function needed by SessionAuthenticationPolicy. It should return
None if the user_id does not exist (such as a deleted user).
""" """
if not request.user: if not request.user:
return None return None
@ -69,14 +69,14 @@ def includeme(config: Configurator) -> None:
# make all views require "view" permission unless specifically overridden # make all views require "view" permission unless specifically overridden
config.set_default_permission("view") config.set_default_permission("view")
# replace the default root factory with a custom one to more easily support
# the default permission
# replace the default root factory with a custom one to more easily support the
# default permission
config.set_root_factory(DefaultRootFactory) config.set_root_factory(DefaultRootFactory)
# Set the authorization policy to a custom one that always returns a
# "denied" result if the user isn't logged in. When overall site access is
# no longer being restricted, the AuthorizedOnlyPolicy class can just be
# replaced with the standard ACLAuthorizationPolicy
# Set the authorization policy to a custom one that always returns a "denied" result
# if the user isn't logged in. When overall site access is no longer being
# restricted, the AuthorizedOnlyPolicy class can just be replaced with the standard
# ACLAuthorizationPolicy
config.set_authorization_policy(AuthorizedOnlyPolicy()) config.set_authorization_policy(AuthorizedOnlyPolicy())
config.set_authentication_policy( config.set_authentication_policy(

20
tildes/tildes/database.py

@ -56,20 +56,18 @@ def includeme(config: Configurator) -> None:
Currently adds: Currently adds:
* request.db_session - the db session for the current request, managed by
pyramid_tm.
* request.query() - a factory method that will return a ModelQuery or
subclass for querying the model class supplied. This will generally be
used generatively, similar to standard SQLALchemy session.query(...).
* request.obtain_lock() - obtains a transaction-level advisory lock from
PostgreSQL.
* request.db_session - db session for the current request, managed by pyramid_tm.
* request.query() - a factory method that will return a ModelQuery or subclass for
querying the model class supplied. This will generally be used generatively,
similar to standard SQLALchemy session.query(...).
* request.obtain_lock() - obtains a transaction-level advisory lock from PostgreSQL.
""" """
settings = config.get_settings() settings = config.get_settings()
# Enable pyramid_tm's default_commit_veto behavior, which will abort the
# transaction if the response code starts with 4 or 5. The main benefit of
# this is to avoid aborting on exceptions that don't actually indicate a
# problem, such as a HTTPFound 302 redirect.
# Enable pyramid_tm's default_commit_veto behavior, which will abort the transaction
# if the response code starts with 4 or 5. The main benefit of this is to avoid
# aborting on exceptions that don't actually indicate a problem, such as a HTTPFound
# 302 redirect.
settings["tm.commit_veto"] = "pyramid_tm.default_commit_veto" settings["tm.commit_veto"] = "pyramid_tm.default_commit_veto"
config.include("pyramid_tm") config.include("pyramid_tm")

6
tildes/tildes/enums.py

@ -68,9 +68,9 @@ class TopicSortOption(enum.Enum):
def descending_description(self) -> str: def descending_description(self) -> str:
"""Describe this sort option when used in a "descending" order. """Describe this sort option when used in a "descending" order.
For example, the "votes" sort has a description of "most votes", since
using that sort in descending order means that topics with the most
votes will be listed first.
For example, the "votes" sort has a description of "most votes", since using
that sort in descending order means that topics with the most votes will be
listed first.
""" """
if self.name == "NEW": if self.name == "NEW":
return "newest" return "newest"

4
tildes/tildes/json.py

@ -13,8 +13,8 @@ from tildes.models.user import User
def serialize_model(model_item: DatabaseModel, request: Request) -> dict: def serialize_model(model_item: DatabaseModel, request: Request) -> dict:
"""Return serializable data for a DatabaseModel item. """Return serializable data for a DatabaseModel item.
Uses the .schema class attribute to serialize a model by using its
corresponding marshmallow schema.
Uses the .schema class attribute to serialize a model by using its corresponding
marshmallow schema.
""" """
# pylint: disable=unused-argument # pylint: disable=unused-argument
return model_item.schema.dump(model_item) return model_item.schema.dump(model_item)

9
tildes/tildes/lib/__init__.py

@ -1,9 +1,8 @@
"""Contains the overall "library" for the application. """Contains the overall "library" for the application.
Defining constants, behavior, etc. inside modules here (as opposed to other
locations such as in models) is encouraged, since it often makes it simpler to
import elsewhere for tests, when only a specific constant value is needed, etc.
Defining constants, behavior, etc. inside modules here (as opposed to other locations
such as in models) is encouraged, since it often makes it simpler to import elsewhere
for tests, when only a specific constant value is needed, etc.
Modules here should *never* import anything from models, to avoid circular
dependencies.
Modules here should *never* import anything from models, to avoid circular dependencies.
""" """

17
tildes/tildes/lib/amqp.py

@ -13,15 +13,14 @@ from tildes.lib.database import get_session_from_config
class PgsqlQueueConsumer(AbstractConsumer): class PgsqlQueueConsumer(AbstractConsumer):
"""Base class for consumers of events sent from PostgreSQL via rabbitmq. """Base class for consumers of events sent from PostgreSQL via rabbitmq.
This class is intended to be used in a completely "stand-alone" manner,
such as inside a script being run separately as a background job. As such,
it also includes connecting to rabbitmq, declaring the underlying queue and
bindings, and (optionally) connecting to the database to be able to fetch
and modify data as necessary. It relies on the environment variable
INI_FILE being set.
Note that all messages received by these consumers are expected to be in
JSON format.
This class is intended to be used in a completely "stand-alone" manner, such as
inside a script being run separately as a background job. As such, it also includes
connecting to rabbitmq, declaring the underlying queue and bindings, and
(optionally) connecting to the database to be able to fetch and modify data as
necessary. It relies on the environment variable INI_FILE being set.
Note that all messages received by these consumers are expected to be in JSON
format.
""" """
PGSQL_EXCHANGE_NAME = "pgsql_events" PGSQL_EXCHANGE_NAME = "pgsql_events"

10
tildes/tildes/lib/database.py

@ -35,9 +35,9 @@ def obtain_transaction_lock(
) -> None: ) -> None:
"""Obtain a transaction-level advisory lock from PostgreSQL. """Obtain a transaction-level advisory lock from PostgreSQL.
The lock_space arg must be either None or the name of one of the members of
the LockSpaces enum (case-insensitive). Contention for a lock will only
occur when both lock_space and lock_value have the same values.
The lock_space arg must be either None or the name of one of the members of the
LockSpaces enum (case-insensitive). Contention for a lock will only occur when both
lock_space and lock_value have the same values.
""" """
if lock_space: if lock_space:
try: try:
@ -125,8 +125,8 @@ class ArrayOfLtree(ARRAY): # pylint: disable=too-many-ancestors
class comparator_factory(ARRAY.comparator_factory): class comparator_factory(ARRAY.comparator_factory):
"""Add custom comparison functions. """Add custom comparison functions.
The ancestor_of and descendant_of functions are supported by LtreeType,
so this duplicates them here so they can be used on ArrayOfLtree too.
The ancestor_of and descendant_of functions are supported by LtreeType, so this
duplicates them here so they can be used on ArrayOfLtree too.
""" """
def ancestor_of(self, other): # type: ignore def ancestor_of(self, other): # type: ignore

22
tildes/tildes/lib/datetime.py

@ -43,9 +43,8 @@ class SimpleHoursPeriod:
def __str__(self) -> str: def __str__(self) -> str:
"""Return a representation of the period as a string. """Return a representation of the period as a string.
Will be of the form "4 hours", "2 days", "1 day, 6 hours", etc. except
for the special case of exactly "1 day", which is replaced with "24
hours".
Will be of the form "4 hours", "2 days", "1 day, 6 hours", etc. except for the
special case of exactly "1 day", which is replaced with "24 hours".
""" """
string = human(self.timedelta, past_tense="{}") string = human(self.timedelta, past_tense="{}")
if string == "1 day": if string == "1 day":
@ -63,8 +62,8 @@ class SimpleHoursPeriod:
def as_short_form(self) -> str: def as_short_form(self) -> str:
"""Return a representation of the period as a "short form" string. """Return a representation of the period as a "short form" string.
Uses "hours" representation unless the period is an exact multiple of
24 hours (except for 24 hours itself).
Uses "hours" representation unless the period is an exact multiple of 24 hours
(except for 24 hours itself).
""" """
if self.hours % 24 == 0 and self.hours != 24: if self.hours % 24 == 0 and self.hours != 24:
return "{}d".format(self.hours // 24) return "{}d".format(self.hours // 24)
@ -80,14 +79,13 @@ def utc_now() -> datetime:
def descriptive_timedelta(target: datetime, abbreviate: bool = False) -> str: def descriptive_timedelta(target: datetime, abbreviate: bool = False) -> str:
"""Return a descriptive string for how long ago a datetime was. """Return a descriptive string for how long ago a datetime was.
The returned string will be of a format like "4 hours ago" or
"3 hours, 21 minutes ago". The second "precision level" is only added if
it will be at least minutes, and only one "level" below the first unit.
That is, you'd never see anything like "4 hours, 5 seconds ago" or
"2 years, 3 hours ago".
The returned string will be of a format like "4 hours ago" or "3 hours, 21 minutes
ago". The second "precision level" is only added if it will be at least minutes, and
only one "level" below the first unit. That is, you'd never see anything like "4
hours, 5 seconds ago" or "2 years, 3 hours ago".
If `abbreviate` is true, the units will be shortened to return a string
like "12h 28m ago" instead of "12 hours, 28 minutes ago".
If `abbreviate` is true, the units will be shortened to return a string like
"12h 28m ago" instead of "12 hours, 28 minutes ago".
A time of less than a second returns "a moment ago". A time of less than a second returns "a moment ago".
""" """

8
tildes/tildes/lib/hash.py

@ -3,10 +3,10 @@
from argon2 import PasswordHasher from argon2 import PasswordHasher
from argon2.exceptions import VerifyMismatchError from argon2.exceptions import VerifyMismatchError
# These parameter values were chosen to achieve a hash-verification time of
# about 10ms on the current production server. They can be updated to different
# values if the server changes (consider upgrading old password hashes on login
# as well if that happens).
# These parameter values were chosen to achieve a hash-verification time of about 10ms
# on the current production server. They can be updated to different values if the
# server changes (consider upgrading old password hashes on login as well if that
# happens).
ARGON2_TIME_COST = 4 ARGON2_TIME_COST = 4
ARGON2_MEMORY_COST = 8092 ARGON2_MEMORY_COST = 8092

9
tildes/tildes/lib/id.py

@ -17,11 +17,10 @@ def id_to_id36(id_val: int) -> str:
# the "alphabet" of our ID36s - 0-9 followed by a-z # the "alphabet" of our ID36s - 0-9 followed by a-z
alphabet = string.digits + string.ascii_lowercase alphabet = string.digits + string.ascii_lowercase
# Repeatedly use divmod() on the value, which returns the quotient and
# remainder of each integer division - divmod(a, b) == (a // b, a % b).
# The remainder of each division works as an index into the alphabet, and
# doing this repeatedly will build up our ID36 string in reverse order
# (with the least-significant "digit" first).
# Repeatedly use divmod() on the value, which returns the quotient and remainder of
# each integer division - divmod(a, b) == (a // b, a % b). The remainder of each
# division works as an index into the alphabet, and doing this repeatedly will build
# up our ID36 string in reverse order (with the least-significant "digit" first).
quotient, index = divmod(id_val, 36) quotient, index = divmod(id_val, 36)
while quotient != 0: while quotient != 0:
reversed_chars.append(alphabet[index]) reversed_chars.append(alphabet[index])

122
tildes/tildes/lib/markdown.py

@ -79,18 +79,18 @@ HTML_ATTRIBUTE_WHITELIST = {
} }
PROTOCOL_WHITELIST = ("http", "https") PROTOCOL_WHITELIST = ("http", "https")
# Regex that finds ordered list markdown that was probably accidental - ones
# being initiated by anything except "1."
# Regex that finds ordered list markdown that was probably accidental - ones being
# initiated by anything except "1."
BAD_ORDERED_LIST_REGEX = re.compile( BAD_ORDERED_LIST_REGEX = re.compile(
r"((?:\A|\n\n)" # Either the start of the entire text, or a new paragraph r"((?:\A|\n\n)" # Either the start of the entire text, or a new paragraph
r"(?!1\.)\d+)" # A number that isn't "1" r"(?!1\.)\d+)" # A number that isn't "1"
r"\.\s" # Followed by a period and a space r"\.\s" # Followed by a period and a space
) )
# Type alias for the "namespaced attr dict" used inside bleach.linkify
# callbacks. This looks pretty ridiculous, but it's a dict where the keys are
# namespaced attr names, like `(None, 'href')`, and there's also a `_text`
# key for getting the innerText of the <a> tag.
# Type alias for the "namespaced attr dict" used inside bleach.linkify callbacks. This
# looks pretty ridiculous, but it's a dict where the keys are namespaced attr names,
# like `(None, 'href')`, and there's also a `_text` key for getting the innerText of the
# <a> tag.
NamespacedAttrDict = Dict[Union[Tuple[Optional[str], str], str], str] # noqa NamespacedAttrDict = Dict[Union[Tuple[Optional[str], str], str], str] # noqa
@ -155,15 +155,14 @@ def preprocess_markdown(markdown: str) -> str:
def escape_accidental_ordered_lists(markdown: str) -> str: def escape_accidental_ordered_lists(markdown: str) -> str:
"""Escape markdown that's probably an accidental ordered list. """Escape markdown that's probably an accidental ordered list.
It's a common markdown mistake to accidentally start a numbered list, by
beginning a post or paragraph with a number followed by a period. For
example, someone might try to write "1975. It was a long time ago.", and
the result will be a comment that says "1. It was a long time ago." since
that gets parsed into a numbered list.
It's a common markdown mistake to accidentally start a numbered list, by beginning a
post or paragraph with a number followed by a period. For example, someone might try
to write "1975. It was a long time ago.", and the result will be a comment that says
"1. It was a long time ago." since that gets parsed into a numbered list.
This fixes that quirk of markdown by escaping anything that would start a
numbered list except for "1. ". This will cause a few other edge cases, but
I believe they're less common/important than fixing this common error.
This fixes that quirk of markdown by escaping anything that would start a numbered
list except for "1. ". This will cause a few other edge cases, but I believe they're
less common/important than fixing this common error.
""" """
return BAD_ORDERED_LIST_REGEX.sub(r"\1\\. ", markdown) return BAD_ORDERED_LIST_REGEX.sub(r"\1\\. ", markdown)
@ -205,24 +204,24 @@ def apply_linkification(html: str, skip_tags: Optional[List[str]] = None) -> str
class LinkifyFilter(Filter): class LinkifyFilter(Filter):
"""html5lib Filter to convert custom text patterns to links. """html5lib Filter to convert custom text patterns to links.
This replaces references to group paths and usernames with links to the
relevant pages.
This replaces references to group paths and usernames with links to the relevant
pages.
This implementation is based heavily on the linkify implementation from
the Bleach library.
This implementation is based heavily on the linkify implementation from the Bleach
library.
""" """
# Regex that finds probable references to groups. This isn't "perfect",
# just a first pass to find likely candidates. The validity of the group
# path is checked more carefully later.
# Note: currently specifically excludes paths immediately followed by a
# tilde, but this may be possible to remove once strikethrough is
# implemented (since that's probably what they were trying to do)
# Regex that finds probable references to groups. This isn't "perfect", just a first
# pass to find likely candidates. The validity of the group path is checked more
# carefully later.
# Note: currently specifically excludes paths immediately followed by a tilde, but
# this may be possible to remove once strikethrough is implemented (since that's
# probably what they were trying to do)
GROUP_REFERENCE_REGEX = re.compile(r"(?<!\w)~([\w.]+)\b(?!~)") GROUP_REFERENCE_REGEX = re.compile(r"(?<!\w)~([\w.]+)\b(?!~)")
# Regex that finds probable references to users. As above, this isn't
# "perfect" either but works as an initial pass with the validity of
# the username checked more carefully later.
# Regex that finds probable references to users. As above, this isn't "perfect"
# either but works as an initial pass with the validity of the username checked more
# carefully later.
USERNAME_REFERENCE_REGEX = re.compile(r"(?<!\w)(?:/?u/|@)([\w-]+)\b") USERNAME_REFERENCE_REGEX = re.compile(r"(?<!\w)(?:/?u/|@)([\w-]+)\b")
def __init__( def __init__(
@ -230,8 +229,8 @@ class LinkifyFilter(Filter):
) -> None: ) -> None:
"""Initialize a linkification filter to apply to HTML. """Initialize a linkification filter to apply to HTML.
The skip_tags argument can be a list of tag names, and the contents of
any of those tags will be excluded from linkification.
The skip_tags argument can be a list of tag names, and the contents of any of
those tags will be excluded from linkification.
""" """
super().__init__(source) super().__init__(source)
self.skip_tags = skip_tags or [] self.skip_tags = skip_tags or []
@ -248,28 +247,27 @@ class LinkifyFilter(Filter):
token["type"] in ("StartTag", "EmptyTag") token["type"] in ("StartTag", "EmptyTag")
and token["name"] in self.skip_tags and token["name"] in self.skip_tags
): ):
# if this is the start of a tag we want to skip, add it to the
# list of skipped tags that we're currently inside
# if this is the start of a tag we want to skip, add it to the list of
# skipped tags that we're currently inside
inside_skipped_tags.append(token["name"]) inside_skipped_tags.append(token["name"])
elif inside_skipped_tags: elif inside_skipped_tags:
# if we're currently inside any skipped tags, the only thing we
# want to do is look for all the end tags we need to be able to
# finish skipping
# if we're currently inside any skipped tags, the only thing we want to
# do is look for all the end tags we need to be able to finish skipping
if token["type"] == "EndTag": if token["type"] == "EndTag":
try: try:
inside_skipped_tags.remove(token["name"]) inside_skipped_tags.remove(token["name"])
except ValueError: except ValueError:
pass pass
elif token["type"] == "Characters": elif token["type"] == "Characters":
# this is only reachable if inside_skipped_tags is empty, so
# this is a text token not inside a skipped tag - do the actual
# linkification replacements
# Note: doing the two replacements "iteratively" like this only
# works because they are "disjoint" and we know they're not
# competing to replace the same text. If more replacements are
# added in the future that might conflict with each other, this
# will need to be reworked somehow.
# this is only reachable if inside_skipped_tags is empty, so this is a
# text token not inside a skipped tag - do the actual linkification
# replacements
# Note: doing the two replacements "iteratively" like this only works
# because they are "disjoint" and we know they're not competing to
# replace the same text. If more replacements are added in the future
# that might conflict with each other, this will need to be reworked
# somehow.
replaced_tokens = self._linkify_tokens( replaced_tokens = self._linkify_tokens(
[token], [token],
filter_regex=self.GROUP_REFERENCE_REGEX, filter_regex=self.GROUP_REFERENCE_REGEX,
@ -281,13 +279,13 @@ class LinkifyFilter(Filter):
linkify_function=self._tokenize_username_match, linkify_function=self._tokenize_username_match,
) )
# yield all the tokens returned from the replacement process
# (will be just the original token if nothing was replaced)
# yield all the tokens returned from the replacement process (will be
# just the original token if nothing was replaced)
for new_token in replaced_tokens: for new_token in replaced_tokens:
yield new_token yield new_token
# we either yielded new tokens or the original one already, so
# we don't want to fall through and yield the original again
# we either yielded new tokens or the original one already, so we don't
# want to fall through and yield the original again
continue continue
yield token yield token
@ -298,11 +296,11 @@ class LinkifyFilter(Filter):
) -> List[dict]: ) -> List[dict]:
"""Check tokens for text that matches a regex and linkify it. """Check tokens for text that matches a regex and linkify it.
The `filter_regex` argument should be a compiled pattern that will be
applied to the text in all of the supplied tokens. If any matches are
found, they will each be used to call `linkify_function`, which will
validate the match and convert it back into tokens (representing an <a>
tag if it is valid for linkifying, or just text if not).
The `filter_regex` argument should be a compiled pattern that will be applied to
the text in all of the supplied tokens. If any matches are found, they will each
be used to call `linkify_function`, which will validate the match and convert it
back into tokens (representing an <a> tag if it is valid for linkifying, or just
text if not).
""" """
new_tokens = [] new_tokens = []
@ -316,8 +314,8 @@ class LinkifyFilter(Filter):
current_index = 0 current_index = 0
for match in filter_regex.finditer(original_text): for match in filter_regex.finditer(original_text):
# if there were some characters between the previous match and
# this one, add a token containing those first
# if there were some characters between the previous match and this one,
# add a token containing those first
if match.start() > current_index: if match.start() > current_index:
new_tokens.append( new_tokens.append(
{ {
@ -333,8 +331,8 @@ class LinkifyFilter(Filter):
# move the progress marker up to the end of this match # move the progress marker up to the end of this match
current_index = match.end() current_index = match.end()
# if there's still some text left over, add one more token for it
# (this will be the entire thing if there weren't any matches)
# if there's still some text left over, add one more token for it (this will
# be the entire thing if there weren't any matches)
if current_index < len(original_text): if current_index < len(original_text):
new_tokens.append( new_tokens.append(
{"type": "Characters", "data": original_text[current_index:]} {"type": "Characters", "data": original_text[current_index:]}
@ -345,14 +343,14 @@ class LinkifyFilter(Filter):
@staticmethod @staticmethod
def _tokenize_group_match(match: Match) -> List[dict]: def _tokenize_group_match(match: Match) -> List[dict]:
"""Convert a potential group reference into HTML tokens.""" """Convert a potential group reference into HTML tokens."""
# convert the potential group path to lowercase to allow people to use
# incorrect casing but still have it link properly
# convert the potential group path to lowercase to allow people to use incorrect
# casing but still have it link properly
group_path = match[1].lower() group_path = match[1].lower()
# Even though they're technically valid paths, we don't want to linkify
# things like "~10" or "~4.5" since that's just going to be someone
# using it in the "approximately" sense. So if the path consists of
# only numbers and/or periods, we won't linkify it
# Even though they're technically valid paths, we don't want to linkify things
# like "~10" or "~4.5" since that's just going to be someone using it in the
# "approximately" sense. So if the path consists of only numbers and/or periods,
# we won't linkify it
is_numeric = all(char in "0123456789." for char in group_path) is_numeric = all(char in "0123456789." for char in group_path)
# if it's a valid group path and not totally numeric, convert to <a> # if it's a valid group path and not totally numeric, convert to <a>

4
tildes/tildes/lib/password.py

@ -22,6 +22,6 @@ def is_breached_password(password: str) -> bool:
redis.execute_command("BF.EXISTS", BREACHED_PASSWORDS_BF_KEY, hashed) redis.execute_command("BF.EXISTS", BREACHED_PASSWORDS_BF_KEY, hashed)
) )
except (ConnectionError, ResponseError): except (ConnectionError, ResponseError):
# server isn't running, bloom filter doesn't exist or the key is a
# different data type
# server isn't running, bloom filter doesn't exist or the key is a different
# data type
return False return False

49
tildes/tildes/lib/ratelimit.py

@ -19,9 +19,8 @@ class RateLimitError(Exception):
class RateLimitResult: class RateLimitResult:
"""The result from a rate-limit check. """The result from a rate-limit check.
Includes data relating to whether the action should be allowed or blocked,
how much of the limit is remaining, how long until the action can be
retried, etc.
Includes data relating to whether the action should be allowed or blocked, how much
of the limit is remaining, how long until the action can be retried, etc.
""" """
def __init__( def __init__(
@ -100,23 +99,22 @@ class RateLimitResult:
def merged_result(cls, results: Sequence["RateLimitResult"]) -> "RateLimitResult": def merged_result(cls, results: Sequence["RateLimitResult"]) -> "RateLimitResult":
"""Merge any number of RateLimitResults into a single result. """Merge any number of RateLimitResults into a single result.
Basically, the merged result should be the "most restrictive"
combination of all the source results. That is, it should only allow
the action if *all* of the source results would allow it, the limit
counts should be the lowest of the set, and the waiting times should
be the highest of the set.
Basically, the merged result should be the "most restrictive" combination of all
the source results. That is, it should only allow the action if *all* of the
source results would allow it, the limit counts should be the lowest of the set,
and the waiting times should be the highest of the set.
Note: I think the behavior for time_until_max is not truly correct, but
it should be reasonable for now. Consider a situation like two
"overlapping" limits of 10/min and 100/hour and what the time_until_max
value of the combination should be. It might be a bit tricky.
Note: I think the behavior for time_until_max is not truly correct, but it
should be reasonable for now. Consider a situation like two "overlapping" limits
of 10/min and 100/hour and what the time_until_max value of the combination
should be. It might be a bit tricky.
""" """
# if there's only one result, just return that one # if there's only one result, just return that one
if len(results) == 1: if len(results) == 1:
return results[0] return results[0]
# time_until_retry is a bit trickier than the others because some/all
# of the source values might be None
# time_until_retry is a bit trickier than the others because some/all of the
# source values might be None
if all(r.time_until_retry is None for r in results): if all(r.time_until_retry is None for r in results):
time_until_retry = None time_until_retry = None
else: else:
@ -156,9 +154,9 @@ class RateLimitResult:
class RateLimitedAction: class RateLimitedAction:
"""Represents a particular action and the limits on its usage. """Represents a particular action and the limits on its usage.
This class uses the redis-cell Redis module to implement a Generic Cell
Rate Algorithm (GCRA) for rate-limiting, which includes several desirable
characteristics including a rolling time window and support for "bursts".
This class uses the redis-cell Redis module to implement a Generic Cell Rate
Algorithm (GCRA) for rate-limiting, which includes several desirable characteristics
including a rolling time window and support for "bursts".
""" """
def __init__( def __init__(
@ -173,13 +171,12 @@ class RateLimitedAction:
) -> None: ) -> None:
"""Initialize the limits on a particular action. """Initialize the limits on a particular action.
The action will be limited to a maximum of `limit` calls over the time
period specified in `period`. By default, up to half of the actions
inside a period may be used in a "burst", in which no specific time
restrictions are applied. This behavior is controlled by the
`max_burst` argument, which can range from 1 (no burst allowed,
requests must wait at least `period / limit` time between them), up to
the same value as `limit` (the full limit may be used at any rate, but
The action will be limited to a maximum of `limit` calls over the time period
specified in `period`. By default, up to half of the actions inside a period may
be used in a "burst", in which no specific time restrictions are applied. This
behavior is controlled by the `max_burst` argument, which can range from 1 (no
burst allowed, requests must wait at least `period / limit` time between them),
up to the same value as `limit` (the full limit may be used at any rate, but
never more than `limit` inside any given period). never more than `limit` inside any given period).
""" """
# pylint: disable=too-many-arguments # pylint: disable=too-many-arguments
@ -202,8 +199,8 @@ class RateLimitedAction:
self.by_user = by_user self.by_user = by_user
self.by_ip = by_ip self.by_ip = by_ip
# if a redis connection wasn't specified, it will need to be
# initialized before any checks or resets for this action can be done
# if a redis connection wasn't specified, it will need to be initialized before
# any checks or resets for this action can be done
self._redis = redis self._redis = redis
@property @property

70
tildes/tildes/lib/string.py

@ -34,14 +34,14 @@ def convert_to_url_slug(original: str, max_length: int = 100) -> str:
# url-encode the slug # url-encode the slug
encoded_slug = quote(slug) encoded_slug = quote(slug)
# if the slug's already short enough, just return without worrying about
# how it will need to be truncated
# if the slug's already short enough, just return without worrying about how it will
# need to be truncated
if len(encoded_slug) <= max_length: if len(encoded_slug) <= max_length:
return encoded_slug return encoded_slug
# Truncating a url-encoded slug can be tricky if there are any multi-byte
# unicode characters, since the %-encoded forms of them can be quite long.
# Check to see if the slug looks like it might contain any of those.
# Truncating a url-encoded slug can be tricky if there are any multi-byte unicode
# characters, since the %-encoded forms of them can be quite long. Check to see if
# the slug looks like it might contain any of those.
maybe_multi_bytes = bool(re.search("%..%", encoded_slug)) maybe_multi_bytes = bool(re.search("%..%", encoded_slug))
# if that matched, we need to take a more complicated approach # if that matched, we need to take a more complicated approach
@ -56,9 +56,8 @@ def convert_to_url_slug(original: str, max_length: int = 100) -> str:
def _truncate_multibyte_slug(original: str, max_length: int) -> str: def _truncate_multibyte_slug(original: str, max_length: int) -> str:
"""URL-encode and truncate a slug known to contain multi-byte chars.""" """URL-encode and truncate a slug known to contain multi-byte chars."""
# instead of the normal method of truncating "backwards" from the end of
# the string, build it up one encoded character at a time from the start
# until it's too long
# instead of the normal method of truncating "backwards" from the end of the string,
# build it up one encoded character at a time from the start until it's too long
encoded_slug = "" encoded_slug = ""
for character in original: for character in original:
encoded_char = quote(character) encoded_char = quote(character)
@ -69,14 +68,13 @@ def _truncate_multibyte_slug(original: str, max_length: int) -> str:
encoded_slug += encoded_char encoded_slug += encoded_char
# Now we know that the string is made up of "whole" characters and is close
# to the maximum length. We'd still like to truncate it at an underscore if
# possible, but some languages like Japanese and Chinese won't have many
# (or any) underscores in the slug, and we could end up losing a lot of the
# characters. So try breaking it at an underscore, but if it means more
# than 30% of the slug gets cut off, just leave it alone. This means that
# some url slugs in other languages will end in partial words, but
# determining the word edges is not simple.
# Now we know that the string is made up of "whole" characters and is close to the
# maximum length. We'd still like to truncate it at an underscore if possible, but
# some languages like Japanese and Chinese won't have many (or any) underscores in
# the slug, and we could end up losing a lot of the characters. So try breaking it
# at an underscore, but if it means more than 30% of the slug gets cut off, just
# leave it alone. This means that some url slugs in other languages will end in
# partial words, but determining the word edges is not simple.
acceptable_truncation = 0.7 acceptable_truncation = 0.7
truncated_slug = truncate_string_at_char(encoded_slug, "_") truncated_slug = truncate_string_at_char(encoded_slug, "_")
@ -95,15 +93,13 @@ def truncate_string(
) -> str: ) -> str:
"""Truncate a string to be no longer than a specified length. """Truncate a string to be no longer than a specified length.
If `truncate_at_chars` is specified (as a string, one or more characters),
the truncation will happen at the last occurrence of any of those chars
inside the remaining string after it has been initially cut down to the
desired length.
If `truncate_at_chars` is specified (as a string, one or more characters), the
truncation will happen at the last occurrence of any of those chars inside the
remaining string after it has been initially cut down to the desired length.
`overflow_str` will be appended to the result, and its length is included
in the final string length. So for example, if `overflow_str` has a length
of 3 and the target length is 10, at most 7 characters of the original
string will be kept.
`overflow_str` will be appended to the result, and its length is included in the
final string length. So for example, if `overflow_str` has a length of 3 and the
target length is 10, at most 7 characters of the original string will be kept.
""" """
if overflow_str is None: if overflow_str is None:
overflow_str = "" overflow_str = ""
@ -112,8 +108,8 @@ def truncate_string(
if len(original) <= length: if len(original) <= length:
return original return original
# cut the string down to the max desired length (leaving space for the
# overflow string if one is specified)
# cut the string down to the max desired length (leaving space for the overflow
# string if one is specified)
truncated = original[: length - len(overflow_str)] truncated = original[: length - len(overflow_str)]
# if we don't want to truncate at particular characters, we're done # if we don't want to truncate at particular characters, we're done
@ -129,17 +125,17 @@ def truncate_string(
def truncate_string_at_char(original: str, valid_chars: str) -> str: def truncate_string_at_char(original: str, valid_chars: str) -> str:
"""Truncate a string at the last occurrence of a particular character. """Truncate a string at the last occurrence of a particular character.
Supports passing multiple valid characters (as a string) for `valid_chars`,
for example valid_chars='.?!' would truncate at the "right-most" occurrence
of any of those 3 characters in the string.
Supports passing multiple valid characters (as a string) for `valid_chars`, for
example valid_chars='.?!' would truncate at the "right-most" occurrence of any of
those 3 characters in the string.
""" """
# work backwards through the string until we find one of the chars we want # work backwards through the string until we find one of the chars we want
for num_from_end, char in enumerate(reversed(original), start=1): for num_from_end, char in enumerate(reversed(original), start=1):
if char in valid_chars: if char in valid_chars:
break break
else: else:
# the loop didn't break, so we looked through the entire string and
# didn't find any of the desired characters - can't do anything
# the loop didn't break, so we looked through the entire string and didn't find
# any of the desired characters - can't do anything
return original return original
# a truncation char was found, so -num_from_end is the position to stop at # a truncation char was found, so -num_from_end is the position to stop at
@ -150,14 +146,14 @@ def truncate_string_at_char(original: str, valid_chars: str) -> str:
def simplify_string(original: str) -> str: def simplify_string(original: str) -> str:
"""Sanitize a string for usage in places where we need a "simple" one. """Sanitize a string for usage in places where we need a "simple" one.
This function is useful for sanitizing strings so that they're suitable to
be used in places like topic titles, message subjects, and so on.
This function is useful for sanitizing strings so that they're suitable to be used
in places like topic titles, message subjects, and so on.
Strings processed by this function: Strings processed by this function:
* have unicode chars from the "separator" category replaced with spaces * have unicode chars from the "separator" category replaced with spaces
* have unicode chars from the "other" category stripped out, except for
newlines, which are replaced with spaces
* have unicode chars from the "other" category stripped out, except for newlines,
which are replaced with spaces
* have leading and trailing whitespace removed * have leading and trailing whitespace removed
* have multiple consecutive spaces collapsed into a single space * have multiple consecutive spaces collapsed into a single space
""" """
@ -183,8 +179,8 @@ def _sanitize_characters(original: str) -> str:
# "separator" chars - replace with a normal space # "separator" chars - replace with a normal space
final_characters.append(" ") final_characters.append(" ")
elif category.startswith("C"): elif category.startswith("C"):
# "other" chars (control, formatting, etc.) - filter them out
# except for newlines, which are replaced with normal spaces
# "other" chars (control, formatting, etc.) - filter them out except for
# newlines, which are replaced with normal spaces
if char == "\n": if char == "\n":
final_characters.append(" ") final_characters.append(" ")
else: else:

4
tildes/tildes/metrics.py

@ -1,7 +1,7 @@
"""Contains Prometheus metric objects and functions for instrumentation.""" """Contains Prometheus metric objects and functions for instrumentation."""
# The prometheus_client classes work in a pretty crazy way, need to disable
# these pylint checks to avoid errors
# The prometheus_client classes work in a pretty crazy way, need to disable these pylint
# checks to avoid errors
# pylint: disable=no-value-for-parameter,redundant-keyword-arg # pylint: disable=no-value-for-parameter,redundant-keyword-arg
from typing import Callable from typing import Callable

28
tildes/tildes/models/comment/comment.py

@ -29,23 +29,23 @@ class Comment(DatabaseModel):
Trigger behavior: Trigger behavior:
Incoming: Incoming:
- num_votes will be incremented and decremented by insertions and
deletions in comment_votes.
- num_votes will be incremented and decremented by insertions and deletions in
comment_votes.
Outgoing: Outgoing:
- Inserting or deleting rows, or updating is_deleted/is_removed to
change visibility will increment or decrement num_comments
accordingly on the relevant topic.
- Inserting a row will increment num_comments on any topic_visit rows
for the comment's author and the relevant topic.
- Inserting a new comment or updating is_deleted or is_removed will
update last_activity_time on the relevant topic.
- Inserting or deleting rows, or updating is_deleted/is_removed to change
visibility will increment or decrement num_comments accordingly on the
relevant topic.
- Inserting a row will increment num_comments on any topic_visit rows for the
comment's author and the relevant topic.
- Inserting a new comment or updating is_deleted or is_removed will update
last_activity_time on the relevant topic.
- Setting is_deleted or is_removed to true will delete any rows in - Setting is_deleted or is_removed to true will delete any rows in
comment_notifications related to the comment. comment_notifications related to the comment.
- Changing is_deleted or is_removed will adjust num_comments on all
topic_visit rows for the relevant topic, where the visit_time was
after the time the comment was originally posted.
- Inserting a row or updating markdown will send a rabbitmq message
for "comment.created" or "comment.edited" respectively.
- Changing is_deleted or is_removed will adjust num_comments on all topic_visit
rows for the relevant topic, where the visit_time was after the time the
comment was originally posted.
- Inserting a row or updating markdown will send a rabbitmq message for
"comment.created" or "comment.edited" respectively.
Internal: Internal:
- deleted_time will be set or unset when is_deleted is changed - deleted_time will be set or unset when is_deleted is changed
- removed_time will be set or unset when is_removed is changed - removed_time will be set or unset when is_removed is changed

23
tildes/tildes/models/comment/comment_notification.py

@ -22,11 +22,10 @@ class CommentNotification(DatabaseModel):
Trigger behavior: Trigger behavior:
Incoming: Incoming:
- Rows will be deleted if the relevant comment has is_deleted set to
true.
- Rows will be deleted if the relevant comment has is_deleted set to true.
Outgoing: Outgoing:
- Inserting, deleting, or updating is_unread will increment or
decrement num_unread_notifications for the relevant user.
- Inserting, deleting, or updating is_unread will increment or decrement
num_unread_notifications for the relevant user.
""" """
__tablename__ = "comment_notifications" __tablename__ = "comment_notifications"
@ -122,16 +121,14 @@ class CommentNotification(DatabaseModel):
) -> Tuple[List["CommentNotification"], List["CommentNotification"]]: ) -> Tuple[List["CommentNotification"], List["CommentNotification"]]:
"""Filter new notifications for edited comments. """Filter new notifications for edited comments.
Protect against sending a notification for the same comment to
the same user twice. Edits can sent notifications to users
now mentioned in the content, but only if they weren't sent
a notification for that comment before.
Protect against sending a notification for the same comment to the same user
twice. Edits can sent notifications to users now mentioned in the content, but
only if they weren't sent a notification for that comment before.
This method returns a tuple of lists of this class. The first
item is the notifications that were previously sent for this
comment but need to be deleted (i.e. mentioned username was edited
out of the comment), and the second item is the notifications
that need to be added, as they're new.
This method returns a tuple of lists of this class. The first item is the
notifications that were previously sent for this comment but need to be deleted
(i.e. mentioned username was edited out of the comment), and the second item is
the notifications that need to be added, as they're new.
""" """
previous_notifications = ( previous_notifications = (
db_session.query(CommentNotification) db_session.query(CommentNotification)

5
tildes/tildes/models/comment/comment_query.py

@ -15,9 +15,8 @@ class CommentQuery(PaginatedQuery):
def __init__(self, request: Request) -> None: def __init__(self, request: Request) -> None:
"""Initialize a CommentQuery for the request. """Initialize a CommentQuery for the request.
If the user is logged in, additional user-specific data will be fetched
along with the comments. For the moment, this is whether the user has
voted on them.
If the user is logged in, additional user-specific data will be fetched along
with the comments. For the moment, this is whether the user has voted on them.
""" """
super().__init__(Comment, request) super().__init__(Comment, request)

23
tildes/tildes/models/comment/comment_tree.py

@ -14,8 +14,8 @@ class CommentTree:
The Comment objects held by this class have additional attributes added: The Comment objects held by this class have additional attributes added:
- `replies`: the list of all immediate children to that comment - `replies`: the list of all immediate children to that comment
- `has_visible_descendant`: whether the comment has any visible
descendants (if not, it can be pruned from the tree)
- `has_visible_descendant`: whether the comment has any visible descendants (if
not, it can be pruned from the tree)
""" """
def __init__(self, comments: Sequence[Comment], sort: CommentSortOption) -> None: def __init__(self, comments: Sequence[Comment], sort: CommentSortOption) -> None:
@ -23,8 +23,8 @@ class CommentTree:
self.tree: List[Comment] = [] self.tree: List[Comment] = []
self.sort = sort self.sort = sort
# sort the comments by date, since replies will always be posted later
# this will ensure that parent comments are always processed first
# sort the comments by date, since replies will always be posted later this will
# ensure that parent comments are always processed first
self.comments = sorted(comments, key=lambda c: c.created_time) self.comments = sorted(comments, key=lambda c: c.created_time)
# if there aren't any comments, we can just bail out here # if there aren't any comments, we can just bail out here
@ -33,11 +33,10 @@ class CommentTree:
self._build_tree() self._build_tree()
# The method of building the tree already sorts it by posting time, so
# there's no need to sort again if that's the desired sorting. Note
# also that because _sort_tree() uses sorted() which is a stable sort,
# this means that the "secondary sort" will always be by posting time
# as well.
# The method of building the tree already sorts it by posting time, so there's
# no need to sort again if that's the desired sorting. Note also that because
# _sort_tree() uses sorted() which is a stable sort, this means that the
# "secondary sort" will always be by posting time as well.
if sort != CommentSortOption.POSTED: if sort != CommentSortOption.POSTED:
with self._sorting_histogram().time(): with self._sorting_histogram().time():
self.tree = self._sort_tree(self.tree, self.sort) self.tree = self._sort_tree(self.tree, self.sort)
@ -75,9 +74,9 @@ class CommentTree:
def _sort_tree(tree: List[Comment], sort: CommentSortOption) -> List[Comment]: def _sort_tree(tree: List[Comment], sort: CommentSortOption) -> List[Comment]:
"""Sort the tree by the desired ordering (recursively). """Sort the tree by the desired ordering (recursively).
Because Python's sorted() function is stable, the ordering of any
comments that compare equal on the sorting method will be the same as
the order that they were originally in when passed to this function.
Because Python's sorted() function is stable, the ordering of any comments that
compare equal on the sorting method will be the same as the order that they were
originally in when passed to this function.
""" """
if sort == CommentSortOption.NEWEST: if sort == CommentSortOption.NEWEST:
tree = sorted(tree, key=lambda c: c.created_time, reverse=True) tree = sorted(tree, key=lambda c: c.created_time, reverse=True)

4
tildes/tildes/models/comment/comment_vote.py

@ -17,8 +17,8 @@ class CommentVote(DatabaseModel):
Trigger behavior: Trigger behavior:
Outgoing: Outgoing:
- Inserting or deleting a row will increment or decrement the num_votes
column for the relevant comment.
- Inserting or deleting a row will increment or decrement the num_votes column
for the relevant comment.
""" """
__tablename__ = "comment_votes" __tablename__ = "comment_votes"

37
tildes/tildes/models/database_model.py

@ -51,8 +51,8 @@ class DatabaseModelBase:
if not isinstance(other, self.__class__): if not isinstance(other, self.__class__):
return NotImplemented return NotImplemented
# loop over all the columns in the primary key - if any don't match,
# return False, otherwise return True if we get through all of them
# loop over all the columns in the primary key - if any don't match, return
# False, otherwise return True if we get through all of them
for column in self.__table__.primary_key: for column in self.__table__.primary_key:
if getattr(self, column.name) != getattr(other, column.name): if getattr(self, column.name) != getattr(other, column.name):
return False return False
@ -62,8 +62,8 @@ class DatabaseModelBase:
def __hash__(self) -> int: def __hash__(self) -> int:
"""Return the hash value of the model. """Return the hash value of the model.
This is implemented by mixing together the hash values of the primary
key columns used in __eq__, as recommended in the Python documentation.
This is implemented by mixing together the hash values of the primary key
columns used in __eq__, as recommended in the Python documentation.
""" """
primary_key_values = tuple( primary_key_values = tuple(
getattr(self, column.name) for column in self.__table__.primary_key getattr(self, column.name) for column in self.__table__.primary_key
@ -84,28 +84,25 @@ class DatabaseModelBase:
def _validate_new_value(self, attribute: str, value: Any) -> Any: def _validate_new_value(self, attribute: str, value: Any) -> Any:
"""Validate the new value for a column. """Validate the new value for a column.
This function will be attached to the SQLAlchemy ORM attribute event
for "set" and will be called whenever a new value is assigned to any of
a model's column attributes. It works by deserializing/loading the new
value through the marshmallow schema associated with the model class
(by its `schema` class attribute).
This function will be attached to the SQLAlchemy ORM attribute event for "set"
and will be called whenever a new value is assigned to any of a model's column
attributes. It works by deserializing/loading the new value through the
marshmallow schema associated with the model class (by its `schema` class
attribute).
The deserialization process can modify the value if desired (for
sanitization), or raise an exception which will prevent the assignment
from happening at all.
The deserialization process can modify the value if desired (for sanitization),
or raise an exception which will prevent the assignment from happening at all.
Note that if the schema does not have a Field defined for the column,
or the Field is declared dump_only, no validation/sanitization will be
applied.
Note that if the schema does not have a Field defined for the column, or the
Field is declared dump_only, no validation/sanitization will be applied.
""" """
if not self.schema_class: if not self.schema_class:
return value return value
# This is a bit "magic", but simplifies the interaction between this
# validation and SQLAlchemy hybrid properties. If the attribute being
# set starts with an underscore, assume that it's due to being set up
# as a hybrid property, and remove the underscore prefix when looking
# for a field to validate against.
# This is a bit "magic", but simplifies the interaction between this validation
# and SQLAlchemy hybrid properties. If the attribute being set starts with an
# underscore, assume that it's due to being set up as a hybrid property, and
# remove the underscore prefix when looking for a field to validate against.
if attribute.startswith("_"): if attribute.startswith("_"):
attribute = attribute[1:] attribute = attribute[1:]

10
tildes/tildes/models/group/group.py

@ -17,8 +17,8 @@ class Group(DatabaseModel):
Trigger behavior: Trigger behavior:
Incoming: Incoming:
- num_subscriptions will be incremented and decremented by insertions
and deletions in group_subscriptions.
- num_subscriptions will be incremented and decremented by insertions and
deletions in group_subscriptions.
""" """
schema_class = GroupSchema schema_class = GroupSchema
@ -45,9 +45,9 @@ class Group(DatabaseModel):
Boolean, nullable=False, server_default="false" Boolean, nullable=False, server_default="false"
) )
# Create a GiST index on path as well as the btree one that will be created
# by the index=True/unique=True keyword args to Column above. The GiST
# index supports additional operators for ltree queries: @>, <@, @, ~, ?
# Create a GiST index on path as well as the btree one that will be created by the
# index=True/unique=True keyword args to Column above. The GiST index supports
# additional operators for ltree queries: @>, <@, @, ~, ?
__table_args__ = (Index("ix_groups_path_gist", path, postgresql_using="gist"),) __table_args__ = (Index("ix_groups_path_gist", path, postgresql_using="gist"),)
def __repr__(self) -> str: def __repr__(self) -> str:

5
tildes/tildes/models/group/group_query.py

@ -15,9 +15,8 @@ class GroupQuery(ModelQuery):
def __init__(self, request: Request) -> None: def __init__(self, request: Request) -> None:
"""Initialize a GroupQuery for the request. """Initialize a GroupQuery for the request.
If the user is logged in, additional user-specific data will be fetched
along with the groups. For the moment, this is whether the user is
subscribed to them.
If the user is logged in, additional user-specific data will be fetched along
with the groups. For the moment, this is whether the user is subscribed to them.
""" """
super().__init__(Group, request) super().__init__(Group, request)

4
tildes/tildes/models/group/group_subscription.py

@ -17,8 +17,8 @@ class GroupSubscription(DatabaseModel):
Trigger behavior: Trigger behavior:
Outgoing: Outgoing:
- Inserting or deleting a row will increment or decrement the
num_subscriptions column for the relevant group.
- Inserting or deleting a row will increment or decrement the num_subscriptions
column for the relevant group.
""" """
__tablename__ = "group_subscriptions" __tablename__ = "group_subscriptions"

5
tildes/tildes/models/log/log.py

@ -74,9 +74,8 @@ class Log(DatabaseModel, BaseLog):
) -> None: ) -> None:
"""Create a new log entry. """Create a new log entry.
User and IP address info is extracted from the Request object.
`info` is an optional dict of arbitrary data that will be stored in
JSON form.
User and IP address info is extracted from the Request object. `info` is an
optional dict of arbitrary data that will be stored in JSON form.
""" """
self.user = request.user self.user = request.user
self.event_type = event_type self.event_type = event_type

51
tildes/tildes/models/message/message.py

@ -1,13 +1,12 @@
"""Contains the MessageConversation and MessageReply classes. """Contains the MessageConversation and MessageReply classes.
Note the difference between these two classes - MessageConversation represents
both the overall conversation and the initial message in a particular message
conversation/thread. Subsequent replies (if any) inside that same conversation
are represented by MessageReply.
This might feel a bit unusual since it splits "all messages" across two
tables/classes, but it simplifies a lot of things when organizing them into
threads.
Note the difference between these two classes - MessageConversation represents both the
overall conversation and the initial message in a particular message
conversation/thread. Subsequent replies (if any) inside that same conversation are
represented by MessageReply.
This might feel a bit unusual since it splits "all messages" across two tables/classes,
but it simplifies a lot of things when organizing them into threads.
""" """
from datetime import datetime from datetime import datetime
@ -44,13 +43,13 @@ class MessageConversation(DatabaseModel):
Trigger behavior: Trigger behavior:
Incoming: Incoming:
- num_replies, last_reply_time, and unread_user_ids are updated when a
new message_replies row is inserted for the conversation.
- num_replies and last_reply_time will be updated if a message_replies
row is deleted.
- num_replies, last_reply_time, and unread_user_ids are updated when a new
message_replies row is inserted for the conversation.
- num_replies and last_reply_time will be updated if a message_replies row is
deleted.
Outgoing: Outgoing:
- Inserting or updating unread_user_ids will update num_unread_messages
for all relevant users.
- Inserting or updating unread_user_ids will update num_unread_messages for all
relevant users.
""" """
schema_class = MessageConversationSchema schema_class = MessageConversationSchema
@ -95,9 +94,9 @@ class MessageConversation(DatabaseModel):
"MessageReply", order_by="MessageReply.created_time" "MessageReply", order_by="MessageReply.created_time"
) )
# Create a GIN index on the unread_user_ids column using the gin__int_ops
# operator class supplied by the intarray module. This should be the best
# index for "array contains" queries.
# Create a GIN index on the unread_user_ids column using the gin__int_ops operator
# class supplied by the intarray module. This should be the best index for "array
# contains" queries.
__table_args__ = ( __table_args__ = (
Index( Index(
"ix_message_conversations_unread_user_ids_gin", "ix_message_conversations_unread_user_ids_gin",
@ -151,8 +150,8 @@ class MessageConversation(DatabaseModel):
def other_user(self, viewer: User) -> User: def other_user(self, viewer: User) -> User:
"""Return the conversation's other user from viewer's perspective. """Return the conversation's other user from viewer's perspective.
That is, if the viewer is the sender, this will be the recipient, and
vice versa.
That is, if the viewer is the sender, this will be the recipient, and vice
versa.
""" """
if not self.is_participant(viewer): if not self.is_participant(viewer):
raise ValueError("User is not a participant in this conversation.") raise ValueError("User is not a participant in this conversation.")
@ -172,8 +171,8 @@ class MessageConversation(DatabaseModel):
def mark_unread_for_user(self, user: User) -> None: def mark_unread_for_user(self, user: User) -> None:
"""Mark the conversation unread for the specified user. """Mark the conversation unread for the specified user.
Uses the postgresql intarray union operator `|`, so there's no need to
worry about duplicate values, race conditions, etc.
Uses the postgresql intarray union operator `|`, so there's no need to worry
about duplicate values, race conditions, etc.
""" """
if not self.is_participant(user): if not self.is_participant(user):
raise ValueError("User is not a participant in this conversation.") raise ValueError("User is not a participant in this conversation.")
@ -184,9 +183,9 @@ class MessageConversation(DatabaseModel):
def mark_read_for_user(self, user: User) -> None: def mark_read_for_user(self, user: User) -> None:
"""Mark the conversation read for the specified user. """Mark the conversation read for the specified user.
Uses the postgresql intarray "remove element from array" operation, so
there's no need to worry about whether the value is present or not,
race conditions, etc.
Uses the postgresql intarray "remove element from array" operation, so there's
no need to worry about whether the value is present or not, race conditions,
etc.
""" """
if not self.is_participant(user): if not self.is_participant(user):
raise ValueError("User is not a participant in this conversation.") raise ValueError("User is not a participant in this conversation.")
@ -202,8 +201,8 @@ class MessageReply(DatabaseModel):
Trigger behavior: Trigger behavior:
Outgoing: Outgoing:
- Inserting will update num_replies, last_reply_time, and
unread_user_ids for the relevant conversation.
- Inserting will update num_replies, last_reply_time, and unread_user_ids for
the relevant conversation.
- Deleting will update num_replies and last_reply_time for the relevant - Deleting will update num_replies and last_reply_time for the relevant
conversation. conversation.
""" """

38
tildes/tildes/models/model_query.py

@ -30,8 +30,8 @@ class ModelQuery(Query):
def __iter__(self) -> Iterator[ModelType]: def __iter__(self) -> Iterator[ModelType]:
"""Iterate over the (processed) results of the query. """Iterate over the (processed) results of the query.
SQLAlchemy goes through __iter__ to execute the query and return the
results, so adding processing here should cover all the possibilities.
SQLAlchemy goes through __iter__ to execute the query and return the results, so
adding processing here should cover all the possibilities.
""" """
results = super().__iter__() results = super().__iter__()
return iter([self._process_result(result) for result in results]) return iter([self._process_result(result) for result in results])
@ -44,10 +44,10 @@ class ModelQuery(Query):
"""Finalize the query before it's executed.""" """Finalize the query before it's executed."""
# pylint: disable=protected-access # pylint: disable=protected-access
# Assertions are disabled to allow these functions to add more filters
# even though .limit() or .offset() may have already been called. This
# is potentially dangerous, but should be fine with the existing
# straightforward usage patterns.
# Assertions are disabled to allow these functions to add more filters even
# though .limit() or .offset() may have already been called. This is potentially
# dangerous, but should be fine with the existing straightforward usage
# patterns.
return ( return (
self.enable_assertions(False) self.enable_assertions(False)
._attach_extra_data() ._attach_extra_data()
@ -58,9 +58,9 @@ class ModelQuery(Query):
def _before_compile_listener(self) -> "ModelQuery": def _before_compile_listener(self) -> "ModelQuery":
"""Do any final adjustments to the query before it's compiled. """Do any final adjustments to the query before it's compiled.
Note that this method cannot be overridden by subclasses because of
the way it is subscribed to the event. Subclasses should override the
_finalize() method instead if necessary.
Note that this method cannot be overridden by subclasses because of the way it
is subscribed to the event. Subclasses should override the _finalize() method
instead if necessary.
""" """
return self._finalize() return self._finalize()
@ -81,13 +81,13 @@ class ModelQuery(Query):
def lock_based_on_request_method(self) -> "ModelQuery": def lock_based_on_request_method(self) -> "ModelQuery":
"""Lock the rows if request method implies it's needed (generative). """Lock the rows if request method implies it's needed (generative).
Applying this function to a query will cause the database to acquire
a row-level FOR UPDATE lock on any rows the query retrieves. This is
only done if the request method is DELETE, PATCH, or PUT, which all
imply that the item(s) being fetched are going to be modified.
Applying this function to a query will cause the database to acquire a row-level
FOR UPDATE lock on any rows the query retrieves. This is only done if the
request method is DELETE, PATCH, or PUT, which all imply that the item(s) being
fetched are going to be modified.
Note that POST is specifically not included, because the item being
POSTed to is not usually modified in a "dangerous" way as a result.
Note that POST is specifically not included, because the item being POSTed to is
not usually modified in a "dangerous" way as a result.
""" """
if self.request.method in {"DELETE", "PATCH", "PUT"}: if self.request.method in {"DELETE", "PATCH", "PUT"}:
return self.with_for_update(of=self.model_cls) return self.with_for_update(of=self.model_cls)
@ -109,8 +109,8 @@ class ModelQuery(Query):
def join_all_relationships(self) -> "ModelQuery": def join_all_relationships(self) -> "ModelQuery":
"""Eagerly join all lazy relationships (generative). """Eagerly join all lazy relationships (generative).
This is useful for being able to load an item "fully" in a single
query and avoid needing to make additional queries for related items.
This is useful for being able to load an item "fully" in a single query and
avoid needing to make additional queries for related items.
""" """
# pylint: disable=no-member # pylint: disable=no-member
self = self.options(Load(self.model_cls).joinedload("*")) self = self.options(Load(self.model_cls).joinedload("*"))
@ -129,8 +129,8 @@ class ModelQuery(Query):
return result return result
# add a listener so the _finalize() function will be called automatically just
# before the query executes
# add a listener so the _finalize() function will be called automatically just before
# the query executes
event.listen( event.listen(
ModelQuery, ModelQuery,
"before_compile", "before_compile",

66
tildes/tildes/models/pagination.py

@ -56,22 +56,20 @@ class PaginatedQuery(ModelQuery):
def is_reversed(self) -> bool: def is_reversed(self) -> bool:
"""Return whether the query is operating "in reverse". """Return whether the query is operating "in reverse".
This is a bit confusing. When moving "forward" through pages, items
will be queried in the same order that they are displayed. For example,
when displaying the newest topics, the query is simply for "newest N
topics" (where N is the number of items per page), with an optional
"after topic X" clause. Either way, the first result from the query
will have the highest created_time, and should be the first item
displayed.
However, things work differently when you are paging "backwards". Since
this is done by looking before a specific item, the query needs to
fetch items in the opposite order of how they will be displayed. For
the "newest" sort example, when paging backwards you need to query for
"*oldest* N items before topic X", so the query ordering is the exact
opposite of the desired display order. The first result from the query
will have the *lowest* created_time, so should be the last item
displayed. Because of this, the results need to be reversed.
This is a bit confusing. When moving "forward" through pages, items will be
queried in the same order that they are displayed. For example, when displaying
the newest topics, the query is simply for "newest N topics" (where N is the
number of items per page), with an optional "after topic X" clause. Either way,
the first result from the query will have the highest created_time, and should
be the first item displayed.
However, things work differently when you are paging "backwards". Since this is
done by looking before a specific item, the query needs to fetch items in the
opposite order of how they will be displayed. For the "newest" sort example,
when paging backwards you need to query for "*oldest* N items before topic X",
so the query ordering is the exact opposite of the desired display order. The
first result from the query will have the *lowest* created_time, so should be
the last item displayed. Because of this, the results need to be reversed.
""" """
return bool(self.before_id) return bool(self.before_id)
@ -100,20 +98,20 @@ class PaginatedQuery(ModelQuery):
query = self query = self
# determine the ID of the "anchor item" that we're using as an upper or
# lower bound, and which type of bound it is
# determine the ID of the "anchor item" that we're using as an upper or lower
# bound, and which type of bound it is
if self.after_id: if self.after_id:
anchor_id = self.after_id anchor_id = self.after_id
# since we're looking for other items "after" the anchor item, it
# will act as an upper bound when the sort order is descending,
# otherwise it's a lower bound
# since we're looking for other items "after" the anchor item, it will act
# as an upper bound when the sort order is descending, otherwise it's a
# lower bound
is_anchor_upper_bound = self.sort_desc is_anchor_upper_bound = self.sort_desc
elif self.before_id: elif self.before_id:
anchor_id = self.before_id anchor_id = self.before_id
# opposite of "after" behavior - when looking "before" the anchor
# item, it's an upper bound if the sort order is *ascending*
# opposite of "after" behavior - when looking "before" the anchor item, it's
# an upper bound if the sort order is *ascending*
is_anchor_upper_bound = not self.sort_desc is_anchor_upper_bound = not self.sort_desc
# create a subquery to get comparison values for the anchor item # create a subquery to get comparison values for the anchor item
@ -136,8 +134,8 @@ class PaginatedQuery(ModelQuery):
"""Finalize the query before execution.""" """Finalize the query before execution."""
query = super()._finalize() query = super()._finalize()
# if the query is reversed, we need to sort in the opposite dir
# (basically self.sort_desc XOR self.is_reversed)
# if the query is reversed, we need to sort in the opposite dir (basically
# self.sort_desc XOR self.is_reversed)
desc = self.sort_desc desc = self.sort_desc
if self.is_reversed: if self.is_reversed:
desc = not desc desc = not desc
@ -167,18 +165,18 @@ class PaginatedResults:
"""Fetch results from a PaginatedQuery.""" """Fetch results from a PaginatedQuery."""
self.per_page = per_page self.per_page = per_page
# if the query had `before` or `after` restrictions, there must be a
# page in that direction (it's where we came from)
# if the query had `before` or `after` restrictions, there must be a page in
# that direction (it's where we came from)
self.has_next_page = bool(query.before_id) self.has_next_page = bool(query.before_id)
self.has_prev_page = bool(query.after_id) self.has_prev_page = bool(query.after_id)
# fetch the results - try to get one more than we're actually going to
# display, so that we know if there's another page
# fetch the results - try to get one more than we're actually going to display,
# so that we know if there's another page
self.results = query.limit(per_page + 1).all() self.results = query.limit(per_page + 1).all()
# if we managed to get one more item than the page size, there's
# another page in the same direction that we're going - set the
# relevant attr and remove the extra item so it's not displayed
# if we managed to get one more item than the page size, there's another page in
# the same direction that we're going - set the relevant attr and remove the
# extra item so it's not displayed
if len(self.results) > per_page: if len(self.results) > per_page:
if query.is_reversed: if query.is_reversed:
self.results = self.results[1:] self.results = self.results[1:]
@ -187,8 +185,8 @@ class PaginatedResults:
self.has_next_page = True self.has_next_page = True
self.results = self.results[:-1] self.results = self.results[:-1]
# if the query came back empty for some reason, we won't be able to
# have next/prev pages since there are no items to base them on
# if the query came back empty for some reason, we won't be able to have
# next/prev pages since there are no items to base them on
if not self.results: if not self.results:
self.has_next_page = False self.has_next_page = False
self.has_prev_page = False self.has_prev_page = False

22
tildes/tildes/models/topic/topic.py

@ -46,15 +46,15 @@ class Topic(DatabaseModel):
Trigger behavior: Trigger behavior:
Incoming: Incoming:
- num_votes will be incremented and decremented by insertions and
deletions in topic_votes.
- num_comments will be incremented and decremented by insertions,
deletions, and updates to is_deleted in comments.
- last_activity_time will be updated by insertions, deletions, and
- num_votes will be incremented and decremented by insertions and deletions in
topic_votes.
- num_comments will be incremented and decremented by insertions, deletions, and
updates to is_deleted in comments. updates to is_deleted in comments.
- last_activity_time will be updated by insertions, deletions, and updates to
is_deleted in comments.
Outgoing: Outgoing:
- Inserting a row or updating markdown will send a rabbitmq message
for "topic.created" or "topic.edited" respectively.
- Inserting a row or updating markdown will send a rabbitmq message for
"topic.created" or "topic.edited" respectively.
Internal: Internal:
- deleted_time will be set when is_deleted is set to true - deleted_time will be set when is_deleted is set to true
""" """
@ -317,8 +317,8 @@ class Topic(DatabaseModel):
if not self.is_link_type or not self.link: if not self.is_link_type or not self.link:
raise ValueError("Non-link topics do not have a domain") raise ValueError("Non-link topics do not have a domain")
# get the domain from the content metadata if possible, but fall back
# to just parsing it from the link if it's not present
# get the domain from the content metadata if possible, but fall back to just
# parsing it from the link if it's not present
return self.get_content_metadata("domain") or get_domain_from_url(self.link) return self.get_content_metadata("domain") or get_domain_from_url(self.link)
@property @property
@ -329,8 +329,8 @@ class Topic(DatabaseModel):
def get_content_metadata(self, key: str) -> Any: def get_content_metadata(self, key: str) -> Any:
"""Get a piece of content metadata "safely". """Get a piece of content metadata "safely".
Will return None if the topic has no metadata defined, if this key
doesn't exist in the metadata, etc.
Will return None if the topic has no metadata defined, if this key doesn't exist
in the metadata, etc.
""" """
if not isinstance(self.content_metadata, dict): if not isinstance(self.content_metadata, dict):
return None return None

20
tildes/tildes/models/topic/topic_query.py

@ -21,10 +21,10 @@ class TopicQuery(PaginatedQuery):
def __init__(self, request: Request) -> None: def __init__(self, request: Request) -> None:
"""Initialize a TopicQuery for the request. """Initialize a TopicQuery for the request.
If the user is logged in, additional user-specific data will be fetched
along with the topics. For the moment, this is whether the user has
voted on the topics, and data related to their last visit - what time
they last visited, and how many new comments have been posted since.
If the user is logged in, additional user-specific data will be fetched along
with the topics. For the moment, this is whether the user has voted on the
topics, and data related to their last visit - what time they last visited, and
how many new comments have been posted since.
""" """
super().__init__(Topic, request) super().__init__(Topic, request)
@ -118,9 +118,9 @@ class TopicQuery(PaginatedQuery):
def inside_time_period(self, period: SimpleHoursPeriod) -> "TopicQuery": def inside_time_period(self, period: SimpleHoursPeriod) -> "TopicQuery":
"""Restrict the topics to inside a time period (generative).""" """Restrict the topics to inside a time period (generative)."""
# if the time period is too long, this will crash by creating a
# datetime outside the valid range - catch that and just don't filter
# by time period at all if the range is that large
# if the time period is too long, this will crash by creating a datetime outside
# the valid range - catch that and just don't filter by time period at all if
# the range is that large
try: try:
start_time = utc_now() - period.timedelta start_time = utc_now() - period.timedelta
except OverflowError: except OverflowError:
@ -130,9 +130,9 @@ class TopicQuery(PaginatedQuery):
def has_tag(self, tag: Ltree) -> "TopicQuery": def has_tag(self, tag: Ltree) -> "TopicQuery":
"""Restrict the topics to ones with a specific tag (generative).""" """Restrict the topics to ones with a specific tag (generative)."""
# casting tag to string really shouldn't be necessary, but some kind of
# strange interaction seems to be happening with the ArrayOfLtree
# class, this will need some investigation
# casting tag to string really shouldn't be necessary, but some kind of strange
# interaction seems to be happening with the ArrayOfLtree class, this will need
# some investigation
tag = str(tag) tag = str(tag)
# pylint: disable=protected-access # pylint: disable=protected-access

14
tildes/tildes/models/topic/topic_visit.py

@ -16,16 +16,16 @@ from .topic import Topic
class TopicVisit(DatabaseModel): class TopicVisit(DatabaseModel):
"""Model for a user's visit to a topic. """Model for a user's visit to a topic.
New visits should not be created through __init__(), but by executing the
statement returned by the `generate_insert_statement` method. This will
take advantage of postgresql's ability to update any existing visit.
New visits should not be created through __init__(), but by executing the statement
returned by the `generate_insert_statement` method. This will take advantage of
postgresql's ability to update any existing visit.
Trigger behavior: Trigger behavior:
Incoming: Incoming:
- num_comments will be incremented for the author's topic visit when
they post a comment in that topic.
- num_comments will be decremented when a comment is deleted, for all
visits to the topic that were after it was posted.
- num_comments will be incremented for the author's topic visit when they post a
comment in that topic.
- num_comments will be decremented when a comment is deleted, for all visits to
the topic that were after it was posted.
""" """
__tablename__ = "topic_visits" __tablename__ = "topic_visits"

4
tildes/tildes/models/topic/topic_vote.py

@ -17,8 +17,8 @@ class TopicVote(DatabaseModel):
Trigger behavior: Trigger behavior:
Outgoing: Outgoing:
- Inserting or deleting a row will increment or decrement the num_votes
column for the relevant topic.
- Inserting or deleting a row will increment or decrement the num_votes column
for the relevant topic.
""" """
__tablename__ = "topic_votes" __tablename__ = "topic_votes"

14
tildes/tildes/models/user/user.py

@ -39,12 +39,10 @@ class User(DatabaseModel):
Trigger behavior: Trigger behavior:
Incoming: Incoming:
- num_unread_notifications will be incremented and decremented by
insertions, deletions, and updates to is_unread in
comment_notifications.
- num_unread_messages will be incremented and decremented by
insertions, deletions, and updates to unread_user_ids in
message_conversations.
- num_unread_notifications will be incremented and decremented by insertions,
deletions, and updates to is_unread in comment_notifications.
- num_unread_messages will be incremented and decremented by insertions,
deletions, and updates to unread_user_ids in message_conversations.
""" """
schema_class = UserSchema schema_class = UserSchema
@ -142,8 +140,8 @@ class User(DatabaseModel):
@password.setter @password.setter
def password(self, value: str) -> None: def password(self, value: str) -> None:
# need to do manual validation since some password checks depend on
# checking the username at the same time (for similarity)
# need to do manual validation since some password checks depend on checking the
# username at the same time (for similarity)
self.schema.validate({"username": self.username, "password": value}) self.schema.validate({"username": self.username, "password": value})
self.password_hash = hash_string(value) self.password_hash = hash_string(value)

8
tildes/tildes/models/user/user_invite_code.py

@ -50,8 +50,8 @@ class UserInviteCode(DatabaseModel):
def __init__(self, user: User) -> None: def __init__(self, user: User) -> None:
"""Create a new (random) invite code owned by the user. """Create a new (random) invite code owned by the user.
Note that uniqueness is not confirmed here, so there is the potential
to create duplicate codes (which will fail to commit to the database).
Note that uniqueness is not confirmed here, so there is the potential to create
duplicate codes (which will fail to commit to the database).
""" """
self.user_id = user.user_id self.user_id = user.user_id
@ -64,8 +64,8 @@ class UserInviteCode(DatabaseModel):
# codes are stored in uppercase # codes are stored in uppercase
code = code.upper() code = code.upper()
# remove any characters that aren't in the code alphabet (allows
# dashes, spaces, etc. to be used to make the codes more readable)
# remove any characters that aren't in the code alphabet (allows dashes, spaces,
# etc. to be used to make the codes more readable)
code = "".join(letter for letter in code if letter in cls.ALPHABET) code = "".join(letter for letter in code if letter in cls.ALPHABET)
if len(code) > cls.LENGTH: if len(code) > cls.LENGTH:

6
tildes/tildes/resources/__init__.py

@ -8,9 +8,9 @@ from tildes.models import DatabaseModel, ModelQuery
def get_resource(request: Request, base_query: ModelQuery) -> DatabaseModel: def get_resource(request: Request, base_query: ModelQuery) -> DatabaseModel:
"""Prepare and execute base query from a root factory, returning result.""" """Prepare and execute base query from a root factory, returning result."""
# While the site is private, we don't want to leak information about which
# usernames or groups exist. So we should just always raise a 403 before
# doing a lookup and potentially raising a 404.
# While the site is private, we don't want to leak information about which usernames
# or groups exist. So we should just always raise a 403 before doing a lookup and
# potentially raising a 404.
if not request.user: if not request.user:
raise HTTPForbidden raise HTTPForbidden

4
tildes/tildes/resources/comment.py

@ -28,8 +28,8 @@ def notification_by_comment_id36(
) -> CommentNotification: ) -> CommentNotification:
"""Get a comment notification specified by {comment_id36} in the route. """Get a comment notification specified by {comment_id36} in the route.
Looks up a comment notification for the logged-in user with the
{comment_id36} specified in the route.
Looks up a comment notification for the logged-in user with the {comment_id36}
specified in the route.
""" """
if not request.user: if not request.user:
raise HTTPForbidden raise HTTPForbidden

8
tildes/tildes/resources/group.py

@ -16,10 +16,10 @@ from tildes.schemas.group import GroupSchema
) )
def group_by_path(request: Request, path: str) -> Group: def group_by_path(request: Request, path: str) -> Group:
"""Get a group specified by {group_path} in the route (or 404).""" """Get a group specified by {group_path} in the route (or 404)."""
# If loading the specified group path into the GroupSchema changed it, do a
# 301 redirect to the resulting group path. This will happen in cases like
# the original url including capital letters in the group path, where we
# want to redirect to the proper all-lowercase path instead.
# If loading the specified group path into the GroupSchema changed it, do a 301
# redirect to the resulting group path. This will happen in cases like the original
# url including capital letters in the group path, where we want to redirect to the
# proper all-lowercase path instead.
if path != request.matchdict["group_path"]: if path != request.matchdict["group_path"]:
request.matchdict["group_path"] = path request.matchdict["group_path"] = path
proper_url = request.route_url(request.matched_route.name, **request.matchdict) proper_url = request.route_url(request.matched_route.name, **request.matchdict)

4
tildes/tildes/resources/topic.py

@ -22,8 +22,8 @@ def topic_by_id36(request: Request, topic_id36: str) -> Topic:
topic = get_resource(request, query) topic = get_resource(request, query)
# if there's also a group specified in the route, check that it's the same
# group as the topic was posted in, otherwise redirect to correct group
# if there's also a group specified in the route, check that it's the same group as
# the topic was posted in, otherwise redirect to correct group
if "group_path" in request.matchdict: if "group_path" in request.matchdict:
path_from_route = request.matchdict["group_path"].lower() path_from_route = request.matchdict["group_path"].lower()
if path_from_route != topic.group.path: if path_from_route != topic.group.path:

6
tildes/tildes/routes.py

@ -149,9 +149,9 @@ def add_intercooler_routes(config: Configurator) -> None:
class LoggedInFactory: class LoggedInFactory:
"""Simple class to use as `factory` to restrict routes to logged-in users. """Simple class to use as `factory` to restrict routes to logged-in users.
This class can be used when a route should only be accessible to logged-in
users but doesn't already have another factory that would handle that by
checking access to a specific resource (such as a topic or message).
This class can be used when a route should only be accessible to logged-in users but
doesn't already have another factory that would handle that by checking access to a
specific resource (such as a topic or message).
""" """
__acl__ = ((Allow, Authenticated, "view"),) __acl__ = ((Allow, Authenticated, "view"),)

16
tildes/tildes/schemas/__init__.py

@ -2,14 +2,14 @@
These schemas are currently being used for several purposes: These schemas are currently being used for several purposes:
- Validation of data for models, such as checking the lengths of strings,
ensuring that they match a particular regex pattern, etc. Specific errors
can be generated for any data that is invalid.
- Validation of data for models, such as checking the lengths of strings, ensuring
that they match a particular regex pattern, etc. Specific errors can be generated
for any data that is invalid.
- Similarly, the webargs library uses the schemas to validate pieces of data
coming in via urls, POST data, etc. It can produce errors if the data is
not valid for the purpose it's intended for.
- Similarly, the webargs library uses the schemas to validate pieces of data coming in
via urls, POST data, etc. It can produce errors if the data is not valid for the
purpose it's intended for.
- Serialization of data, which the Pyramid JSON renderer uses to produce
data for the JSON API endpoints.
- Serialization of data, which the Pyramid JSON renderer uses to produce data for the
JSON API endpoints.
""" """

4
tildes/tildes/schemas/fields.py

@ -108,8 +108,8 @@ class SimpleString(Field):
These strings should generally not contain any special formatting (such as These strings should generally not contain any special formatting (such as
markdown), and have problematic whitespace/unicode/etc. removed. markdown), and have problematic whitespace/unicode/etc. removed.
See the simplify_string() function for full details of how these strings
are processed and sanitized.
See the simplify_string() function for full details of how these strings are
processed and sanitized.
""" """

10
tildes/tildes/schemas/topic.py

@ -68,11 +68,11 @@ class TopicSchema(Schema):
def validate_tags(self, value: typing.List[sqlalchemy_utils.Ltree]) -> None: def validate_tags(self, value: typing.List[sqlalchemy_utils.Ltree]) -> None:
"""Validate the tags field, raising an error if an issue exists. """Validate the tags field, raising an error if an issue exists.
Note that tags are validated by ensuring that each tag would be a valid
group path. This is definitely mixing concerns, but it's deliberate in
this case. It will allow for some interesting possibilities by ensuring
naming "compatibility" between groups and tags. For example, a popular
tag in a group could be converted into a sub-group easily.
Note that tags are validated by ensuring that each tag would be a valid group
path. This is definitely mixing concerns, but it's deliberate in this case. It
will allow for some interesting possibilities by ensuring naming "compatibility"
between groups and tags. For example, a popular tag in a group could be
converted into a sub-group easily.
""" """
group_schema = GroupSchema(partial=True) group_schema = GroupSchema(partial=True)
for tag in value: for tag in value:

12
tildes/tildes/schemas/user.py

@ -16,9 +16,9 @@ USERNAME_MAX_LENGTH = 20
# Valid username regex, encodes the following: # Valid username regex, encodes the following:
# - must start with a number or letter # - must start with a number or letter
# - must end with a number or letter # - must end with a number or letter
# - the middle can contain numbers, letters, underscores and dashes, but no
# more than one underscore/dash consecutively (this includes both "_-" and
# "-_" sequences being invalid)
# - the middle can contain numbers, letters, underscores and dashes, but no more than
# one underscore/dash consecutively (this includes both "_-" and "-_" sequences
# being invalid)
# Note: this regex does not contain any length checks, must be done separately # Note: this regex does not contain any length checks, must be done separately
# fmt: off # fmt: off
USERNAME_VALID_REGEX = re.compile( USERNAME_VALID_REGEX = re.compile(
@ -110,9 +110,9 @@ class UserSchema(Schema):
def is_valid_username(username: str) -> bool: def is_valid_username(username: str) -> bool:
"""Return whether the username is valid or not. """Return whether the username is valid or not.
Simple convenience wrapper that uses the schema to validate a username,
useful in cases where a simple valid/invalid result is needed without
worrying about the specific reason for invalidity.
Simple convenience wrapper that uses the schema to validate a username, useful in
cases where a simple valid/invalid result is needed without worrying about the
specific reason for invalidity.
""" """
schema = UserSchema(partial=True) schema = UserSchema(partial=True)
try: try:

4
tildes/tildes/views/__init__.py

@ -8,6 +8,6 @@ from pyramid.response import Response
IC_NOOP = Response(status_int=200) IC_NOOP = Response(status_int=200)
IC_NOOP_404 = Response(status_int=404) IC_NOOP_404 = Response(status_int=404)
# Because of the above, in order to deliberately cause Intercooler to replace
# an element with whitespace, the response needs to contain at least two spaces
# Because of the above, in order to deliberately cause Intercooler to replace an element
# with whitespace, the response needs to contain at least two spaces
IC_EMPTY = Response(" ") IC_EMPTY = Response(" ")

29
tildes/tildes/views/api/web/comment.py

@ -22,14 +22,13 @@ from tildes.views.decorators import ic_view_config
def _increment_topic_comments_seen(request: Request, comment: Comment) -> None: def _increment_topic_comments_seen(request: Request, comment: Comment) -> None:
"""Increment the number of comments in a topic the user has viewed. """Increment the number of comments in a topic the user has viewed.
If the user has the "track comment visits" feature enabled, we want to
increment the number of comments they've seen in the thread that the
comment came from, so that they don't *both* get a notification as well as
have the thread highlight with "(1 new)". This should only happen if their
last visit was before the comment was posted, however. Below, this is
implemented as a INSERT ... ON CONFLICT DO UPDATE so that it will insert
a new topic visit with 1 comment if they didn't previously have one at
all.
If the user has the "track comment visits" feature enabled, we want to increment the
number of comments they've seen in the thread that the comment came from, so that
they don't *both* get a notification as well as have the thread highlight with "(1
new)". This should only happen if their last visit was before the comment was
posted, however. Below, this is implemented as a INSERT ... ON CONFLICT DO UPDATE
so that it will insert a new topic visit with 1 comment if they didn't previously
have one at all.
""" """
if request.user.track_comment_visits: if request.user.track_comment_visits:
statement = ( statement = (
@ -190,8 +189,8 @@ def put_vote_comment(request: Request) -> dict:
request.db_session.add(new_vote) request.db_session.add(new_vote)
try: try:
# manually flush before attempting to commit, to avoid having all
# objects detached from the session in case of an error
# manually flush before attempting to commit, to avoid having all objects
# detached from the session in case of an error
request.db_session.flush() request.db_session.flush()
request.tm.commit() request.tm.commit()
except IntegrityError: except IntegrityError:
@ -254,8 +253,8 @@ def put_tag_comment(request: Request, name: CommentTagOption) -> Response:
request.db_session.add(tag) request.db_session.add(tag)
try: try:
# manually flush before attempting to commit, to avoid having all
# objects detached from the session in case of an error
# manually flush before attempting to commit, to avoid having all objects
# detached from the session in case of an error
request.db_session.flush() request.db_session.flush()
request.tm.commit() request.tm.commit()
except FlushError: except FlushError:
@ -309,9 +308,9 @@ def delete_tag_comment(request: Request, name: CommentTagOption) -> Response:
def put_mark_comments_read(request: Request, mark_all_previous: bool) -> Response: def put_mark_comments_read(request: Request, mark_all_previous: bool) -> Response:
"""Mark comment(s) read, clearing notifications. """Mark comment(s) read, clearing notifications.
The "main" notification (request.context) will always be marked read, and
if the query param mark_all_previous is Truthy, all notifications prior to
that one will be marked read as well.
The "main" notification (request.context) will always be marked read, and if the
query param mark_all_previous is Truthy, all notifications prior to that one will be
marked read as well.
""" """
notification = request.context notification = request.context

12
tildes/tildes/views/api/web/exceptions.py

@ -91,11 +91,11 @@ def httptoomanyrequests(request: Request) -> Response:
def httpfound(request: Request) -> Response: def httpfound(request: Request) -> Response:
"""Convert an HTTPFound to a 200 with the header for a redirect. """Convert an HTTPFound to a 200 with the header for a redirect.
Intercooler won't handle a 302 response as a "full" redirect, and will just
load the content of the destination page into the target element, the same
as any other response. However, it has support for a special X-IC-Redirect
header, which allows the response to trigger a client-side redirect. This
exception view will convert a 302 into a 200 with that header so it works
as a redirect for both standard requests as well as Intercooler ones.
Intercooler won't handle a 302 response as a "full" redirect, and will just load the
content of the destination page into the target element, the same as any other
response. However, it has support for a special X-IC-Redirect header, which allows
the response to trigger a client-side redirect. This exception view will convert a
302 into a 200 with that header so it works as a redirect for both standard requests
as well as Intercooler ones.
""" """
return Response(headers={"X-IC-Redirect": request.exception.location}) return Response(headers={"X-IC-Redirect": request.exception.location})

4
tildes/tildes/views/api/web/group.py

@ -32,8 +32,8 @@ def put_subscribe_group(request: Request) -> dict:
request.db_session.add(new_subscription) request.db_session.add(new_subscription)
try: try:
# manually flush before attempting to commit, to avoid having all
# objects detached from the session in case of an error
# manually flush before attempting to commit, to avoid having all objects
# detached from the session in case of an error
request.db_session.flush() request.db_session.flush()
request.tm.commit() request.tm.commit()
except IntegrityError: except IntegrityError:

4
tildes/tildes/views/api/web/topic.py

@ -87,8 +87,8 @@ def put_topic_vote(request: Request) -> Response:
request.db_session.add(new_vote) request.db_session.add(new_vote)
try: try:
# manually flush before attempting to commit, to avoid having all
# objects detached from the session in case of an error
# manually flush before attempting to commit, to avoid having all objects
# detached from the session in case of an error
request.db_session.flush() request.db_session.flush()
request.tm.commit() request.tm.commit()
except IntegrityError: except IntegrityError:

18
tildes/tildes/views/api/web/user.py

@ -66,10 +66,10 @@ def patch_change_email_address(
"""Change the user's email address (and descriptive note).""" """Change the user's email address (and descriptive note)."""
user = request.context user = request.context
# If the user already has an email address set, we need to retain the
# previous hash and description in the log. Otherwise, if an account is
# compromised and the attacker changes the email address, we'd have no way
# to support recovery for the owner.
# If the user already has an email address set, we need to retain the previous hash
# and description in the log. Otherwise, if an account is compromised and the
# attacker changes the email address, we'd have no way to support recovery for the
# owner.
log_info = None log_info = None
if user.email_address_hash: if user.email_address_hash:
log_info = { log_info = {
@ -155,8 +155,8 @@ def get_invite_code(request: Request) -> dict:
# obtain a lock to prevent concurrent requests generating multiple codes # obtain a lock to prevent concurrent requests generating multiple codes
request.obtain_lock("generate_invite_code", user.user_id) request.obtain_lock("generate_invite_code", user.user_id)
# it's possible to randomly generate an existing code, so we'll retry
# until we create a new one (will practically always be the first try)
# it's possible to randomly generate an existing code, so we'll retry until we
# create a new one (will practically always be the first try)
while True: while True:
savepoint = request.tm.savepoint() savepoint = request.tm.savepoint()
@ -169,9 +169,9 @@ def get_invite_code(request: Request) -> dict:
except IntegrityError: except IntegrityError:
savepoint.rollback() savepoint.rollback()
# doing an atomic decrement on request.user.invite_codes_remaining is going
# to make it unusable as an integer in the template, so store the expected
# value after the decrement first, to be able to use that instead
# doing an atomic decrement on request.user.invite_codes_remaining is going to make
# it unusable as an integer in the template, so store the expected value after the
# decrement first, to be able to use that instead
num_remaining = request.user.invite_codes_remaining - 1 num_remaining = request.user.invite_codes_remaining - 1
request.user.invite_codes_remaining = User.invite_codes_remaining - 1 request.user.invite_codes_remaining = User.invite_codes_remaining - 1

12
tildes/tildes/views/decorators.py

@ -28,9 +28,8 @@ def rate_limit_view(action_name: str) -> Callable:
Needs to be used with the name of the rate-limited action, such as: Needs to be used with the name of the rate-limited action, such as:
@rate_limit_view('register') @rate_limit_view('register')
If the ratelimit check comes back with the action being blocked, a 429
response with appropriate headers will be raised instead of calling the
decorated view.
If the ratelimit check comes back with the action being blocked, a 429 response with
appropriate headers will be raised instead of calling the decorated view.
""" """
def decorator(func: Callable) -> Callable: def decorator(func: Callable) -> Callable:
@ -51,10 +50,9 @@ def rate_limit_view(action_name: str) -> Callable:
def not_logged_in(func: Callable) -> Callable: def not_logged_in(func: Callable) -> Callable:
"""Decorate a view function to prevent access by logged-in users. """Decorate a view function to prevent access by logged-in users.
If a logged-in user attempts to access a view decorated by this function,
they will be redirected to the home page instead. This is useful for views
such as the login page, registration page, etc. which only logged-out users
should be accessing.
If a logged-in user attempts to access a view decorated by this function, they will
be redirected to the home page instead. This is useful for views such as the login
page, registration page, etc. which only logged-out users should be accessing.
""" """
def wrapper(request: Request, **kwargs: Any) -> Any: def wrapper(request: Request, **kwargs: Any) -> Any:

10
tildes/tildes/views/message.py

@ -22,9 +22,9 @@ def get_new_message_form(request: Request) -> dict:
@view_config(route_name="messages", renderer="messages.jinja2") @view_config(route_name="messages", renderer="messages.jinja2")
def get_user_messages(request: Request) -> dict: def get_user_messages(request: Request) -> dict:
"""Show the logged-in user's message conversations.""" """Show the logged-in user's message conversations."""
# select conversations where either the user is the recipient, or they
# were the sender and there is at least one reply (don't need to show
# conversations the user started but haven't been replied to)
# select conversations where either the user is the recipient, or they were the
# sender and there is at least one reply (don't need to show conversations the user
# started but haven't been replied to)
conversations = ( conversations = (
request.query(MessageConversation) request.query(MessageConversation)
.filter( .filter(
@ -63,8 +63,8 @@ def get_user_unread_messages(request: Request) -> dict:
@view_config(route_name="messages_sent", renderer="messages_sent.jinja2") @view_config(route_name="messages_sent", renderer="messages_sent.jinja2")
def get_user_sent_messages(request: Request) -> dict: def get_user_sent_messages(request: Request) -> dict:
"""Show the logged-in user's sent message conversations.""" """Show the logged-in user's sent message conversations."""
# select conversations where either the user was the sender, or they
# were the recipient and there is at least one reply
# select conversations where either the user was the sender, or they were the
# recipient and there is at least one reply
conversations = ( conversations = (
request.query(MessageConversation) request.query(MessageConversation)
.filter( .filter(

8
tildes/tildes/views/metrics.py

@ -13,10 +13,10 @@ def get_metrics(request: Request) -> str:
multiprocess.MultiProcessCollector(registry) multiprocess.MultiProcessCollector(registry)
data = generate_latest(registry) data = generate_latest(registry)
# When Prometheus accesses this page it will always create a new session.
# This session is useless and will never be used again, so we can just
# invalidate it to cause it to be deleted from storage. It would be even
# better to find a way to not create it in the first place.
# When Prometheus accesses this page it will always create a new session. This
# session is useless and will never be used again, so we can just invalidate it to
# cause it to be deleted from storage. It would be even better to find a way to not
# create it in the first place.
request.session.invalidate() request.session.invalidate()
return data.decode("utf-8") return data.decode("utf-8")

4
tildes/tildes/views/notifications.py

@ -22,8 +22,8 @@ def get_user_unread_notifications(request: Request) -> dict:
.all() .all()
) )
# if the user has the "automatically mark notifications as read" setting
# enabled, mark all their notifications as read
# if the user has the "automatically mark notifications as read" setting enabled,
# mark all their notifications as read
if request.user.auto_mark_notifications_read: if request.user.auto_mark_notifications_read:
for notification in notifications: for notification in notifications:
notification.is_unread = False notification.is_unread = False

12
tildes/tildes/views/register.py

@ -32,8 +32,8 @@ def get_register(request: Request) -> dict:
def user_schema_check_breaches(request: Request) -> UserSchema: def user_schema_check_breaches(request: Request) -> UserSchema:
"""Return a UserSchema that will check the password against breaches. """Return a UserSchema that will check the password against breaches.
It would probably be good to generalize this function at some point,
probably similar to:
It would probably be good to generalize this function at some point, probably
similar to:
http://webargs.readthedocs.io/en/latest/advanced.html#reducing-boilerplate http://webargs.readthedocs.io/en/latest/advanced.html#reducing-boilerplate
""" """
# pylint: disable=unused-argument # pylint: disable=unused-argument
@ -67,8 +67,8 @@ def post_register(
if password != password_confirm: if password != password_confirm:
raise HTTPUnprocessableEntity("Password and confirmation do not match.") raise HTTPUnprocessableEntity("Password and confirmation do not match.")
# attempt to fetch and lock the row for the specified invite code (lock
# prevents concurrent requests from using the same invite code)
# attempt to fetch and lock the row for the specified invite code (lock prevents
# concurrent requests from using the same invite code)
lookup_code = UserInviteCode.prepare_code_for_lookup(invite_code) lookup_code = UserInviteCode.prepare_code_for_lookup(invite_code)
code_row = ( code_row = (
request.query(UserInviteCode) request.query(UserInviteCode)
@ -95,8 +95,8 @@ def post_register(
except IntegrityError: except IntegrityError:
raise HTTPUnprocessableEntity("That username has already been registered.") raise HTTPUnprocessableEntity("That username has already been registered.")
# the flush above will generate the new user's ID, so use that to update
# the invite code with info about the user that registered with it
# the flush above will generate the new user's ID, so use that to update the invite
# code with info about the user that registered with it
code_row.invitee_id = user.user_id code_row.invitee_id = user.user_id
# subscribe the new user to all groups except ~test # subscribe the new user to all groups except ~test

20
tildes/tildes/views/topic.py

@ -47,8 +47,8 @@ def post_group_topics(
group=request.context, author=request.user, title=title, link=link group=request.context, author=request.user, title=title, link=link
) )
# if they specified both a link and markdown, use the markdown to post
# an initial comment on the topic
# if they specified both a link and markdown, use the markdown to post an
# initial comment on the topic
if markdown: if markdown:
new_comment = Comment( new_comment = Comment(
topic=new_topic, author=request.user, markdown=markdown topic=new_topic, author=request.user, markdown=markdown
@ -139,8 +139,8 @@ def get_group_topics(
period_options = [SimpleHoursPeriod(hours) for hours in (1, 12, 24, 72)] period_options = [SimpleHoursPeriod(hours) for hours in (1, 12, 24, 72)]
# add the current period to the bottom of the dropdown if it's not one of
# the "standard" ones
# add the current period to the bottom of the dropdown if it's not one of the
# "standard" ones
if period and period not in period_options: if period and period not in period_options:
period_options.append(period) period_options.append(period)
@ -177,8 +177,8 @@ def get_topic(request: Request, comment_order: CommentSortOption) -> dict:
"""View a single topic.""" """View a single topic."""
topic = request.context topic = request.context
# deleted and removed comments need to be included since they're necessary
# for building the tree if they have replies
# deleted and removed comments need to be included since they're necessary for
# building the tree if they have replies
comments = ( comments = (
request.query(Comment) request.query(Comment)
.include_deleted() .include_deleted()
@ -263,8 +263,8 @@ def _get_default_settings(request: Request, order: Any) -> DefaultSettings:
else: else:
default_order = TopicSortOption.ACTIVITY default_order = TopicSortOption.ACTIVITY
# the default period depends on what the order is, so we need to see if
# we're going to end up using the default order here as well
# the default period depends on what the order is, so we need to see if we're going
# to end up using the default order here as well
if order is missing: if order is missing:
order = default_order order = default_order
@ -275,8 +275,8 @@ def _get_default_settings(request: Request, order: Any) -> DefaultSettings:
user_default = request.user.home_default_period user_default = request.user.home_default_period
default_period = ShortTimePeriod().deserialize(user_default) default_period = ShortTimePeriod().deserialize(user_default)
else: else:
# Overall default periods, if the user doesn't have either a
# group-specific or a home default set up:
# Overall default periods, if the user doesn't have either a group-specific or a
# home default set up:
# * "all time" if sorting by new # * "all time" if sorting by new
# * "all time" if sorting by activity and inside a group # * "all time" if sorting by activity and inside a group
# * "3 days" if sorting by activity and on home page # * "3 days" if sorting by activity and on home page

6
tildes/tildes/views/user.py

@ -20,9 +20,9 @@ def _get_user_recent_activity(
) -> List[Union[Comment, Topic]]: ) -> List[Union[Comment, Topic]]:
page_size = 20 page_size = 20
# Since we don't know how many comments or topics will be needed to make
# up a page, we'll fetch the full page size of both types, merge them,
# and then trim down to the size afterwards
# Since we don't know how many comments or topics will be needed to make up a page,
# we'll fetch the full page size of both types, merge them, and then trim down to
# the size afterwards
query = ( query = (
request.query(Comment) request.query(Comment)
.filter(Comment.user == user) .filter(Comment.user == user)

Loading…
Cancel
Save