accumulated fixes and improvements from neodb

activities/migrations/0030_timelineevent_undismissed_idx.py

@@ -0,0 +1,21 @@
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("activities", "0029_post_url_db_index"),
+    ]
+
+    operations = [
+        AddIndexConcurrently(
+            model_name="timelineevent",
+            index=models.Index(
+                fields=["identity", "-id"],
+                condition=models.Q(dismissed=False),
+                name="te_identity_idneg_undismissed",
+            ),
+        ),
+    ]

activities/migrations/0031_quoteauthorization.py

@@ -0,0 +1,46 @@
+import core.snowflake
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("activities", "0030_timelineevent_undismissed_idx"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="QuoteAuthorization",
+            fields=[
+                (
+                    "id",
+                    models.BigIntegerField(
+                        default=core.snowflake.Snowflake.generate_post,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                ("interacting_object_uri", models.CharField(max_length=2048)),
+                (
+                    "request_uri",
+                    models.CharField(blank=True, max_length=2048, null=True),
+                ),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                (
+                    "target_post",
+                    models.ForeignKey(
+                        on_delete=models.deletion.CASCADE,
+                        related_name="quote_authorizations",
+                        to="activities.post",
+                    ),
+                ),
+            ],
+            options={
+                "indexes": [
+                    models.Index(
+                        fields=["target_post", "interacting_object_uri"],
+                        name="activities__target__448802_idx",
+                    ),
+                ],
+            },
+        ),
+    ]

activities/models/__init__.py

@@ -6,4 +6,5 @@
 from .post_attachment import PostAttachment, PostAttachmentStates  # noqa
 from .post_interaction import PostInteraction, PostInteractionStates  # noqa
 from .preview_card import PreviewCard, PreviewCardStates  # noqa
+from .quote_authorization import QuoteAuthorization  # noqa
 from .timeline_event import TimelineEvent  # noqa

activities/models/emoji.py

@@ -287,6 +287,9 @@ def by_ap_tag(cls, domain: Domain, data: dict, create: bool = False):
             if not create:
                 raise KeyError(f"No emoji with ID {data['id']}", data)
 
+        if domain is None:
+            raise ValueError("No domain for emoji", data)
+
         # Name could be a direct property, or in a language'd value
         if "name" in data:
             name = data["name"]

activities/models/hashtag.py

@@ -1,13 +1,13 @@
 import re
 import time
-from datetime import date, timedelta
+from datetime import date, datetime, timedelta
 
 import urlman
+from core.models import Config
 from django.conf import settings
 from django.db import connection, models, transaction
+from django.db.models.functions import TruncDate
 from django.utils import timezone
-
-from core.models import Config
 from stator.models import State, StateField, StateGraph, StatorModel
 
 
@@ -25,33 +25,81 @@ def handle_outdated(cls, instance: "Hashtag"):
         """
         from .post import Post
 
-        posts_query = Post.objects.local_public().tagged_with(instance)
-        total = posts_query.count()
-
         today = timezone.now().date()
-        total_today = posts_query.filter(created__date=today).count()
-        total_month = posts_query.filter(
-            created__year=today.year,
-            created__month=today.month,
-        ).count()
-        total_year = posts_query.filter(
-            created__year=today.year,
-        ).count()
-
-        history = []
-        tagged_posts = Post.objects.not_hidden().tagged_with(instance)
-        for i in range(8):
-            # Mastodon doesn't include today in history (let's do it anyway)
-            day = today - timedelta(days=i)
-            data = tagged_posts.filter(published__date=day).aggregate(
+        # Use timezone-aware datetime ranges instead of __date / __year /
+        # __month lookups so the FILTER predicates are plain timestamp
+        # comparisons instead of per-row AT TIME ZONE / EXTRACT calls
+        # (single aggregate was hitting ~800ms on popular hashtags because
+        # each candidate row had to be cast).
+        today_start = timezone.make_aware(datetime(today.year, today.month, today.day))
+        tomorrow_start = today_start + timedelta(days=1)
+        month_start = today_start.replace(day=1)
+        if today.month == 12:
+            next_month_start = timezone.make_aware(datetime(today.year + 1, 1, 1))
+        else:
+            next_month_start = timezone.make_aware(
+                datetime(today.year, today.month + 1, 1)
+            )
+        year_start = timezone.make_aware(datetime(today.year, 1, 1))
+        next_year_start = timezone.make_aware(datetime(today.year + 1, 1, 1))
+        # Collapse total / today / month / year into a single conditional
+        # aggregate so we hit the DB once instead of four times per hashtag
+        # transition (fired thousands of times daily by the stator loop).
+        totals = (
+            Post.objects.local_public()
+            .tagged_with(instance)
+            .aggregate(
+                total=models.Count("id"),
+                total_today=models.Count(
+                    "id",
+                    filter=models.Q(
+                        created__gte=today_start, created__lt=tomorrow_start
+                    ),
+                ),
+                total_month=models.Count(
+                    "id",
+                    filter=models.Q(
+                        created__gte=month_start, created__lt=next_month_start
+                    ),
+                ),
+                total_year=models.Count(
+                    "id",
+                    filter=models.Q(
+                        created__gte=year_start, created__lt=next_year_start
+                    ),
+                ),
+            )
+        )
+        total = totals["total"]
+        total_today = totals["total_today"]
+        total_month = totals["total_month"]
+        total_year = totals["total_year"]
+
+        # Mastodon doesn't include today in history (let's do it anyway).
+        # One GROUP BY per-day aggregate over the 8-day window replaces the
+        # previous 8 separate filter().aggregate() calls.
+        history_start = today - timedelta(days=7)
+        history_rows = (
+            Post.objects.not_hidden()
+            .tagged_with(instance)
+            .annotate(_day=TruncDate("published"))
+            .filter(_day__gte=history_start, _day__lte=today)
+            .values("_day")
+            .annotate(
                 total=models.Count("id"),
                 num_authors=models.Count("author", distinct=True),
             )
+        )
+        by_day = {row["_day"]: row for row in history_rows}
+        history = []
+        for i in range(8):
+            day = today - timedelta(days=i)
+            data = by_day.get(day)
             history.append(
                 {
                     "day": str(int(time.mktime(day.timetuple()))),
-                    "uses": str(data["total"]),
-                    "accounts": str(data["num_authors"]),
+                    "uses": str(data["total"]) if data else "0",
+                    "accounts": str(data["num_authors"]) if data else "0",
                 }
             )