✨(provisioning) add new endpoint to list DNS records for all domains

src/backend/core/api/viewsets/provisioning.py

@@ -4,6 +4,7 @@
 
 from django.core.exceptions import ValidationError
 from django.db import IntegrityError
+from django.db.models import Prefetch
 
 from drf_spectacular.utils import extend_schema
 from rest_framework import status
@@ -17,6 +18,7 @@
     MailboxLightSerializer,
     ProvisioningMailDomainSerializer,
 )
+from core.api.viewsets import Pagination
 from core.enums import ChannelApiKeyScope, MailboxRoleChoices
 
 logger = logging.getLogger(__name__)
@@ -92,6 +94,58 @@ def post(self, request):
         )
 
 
+class MailDomainDNSPagination(Pagination):
+    """Pagination allowing bigger pages, so a full DNS sync needs few requests."""
+
+    max_page_size = 1000
+
+
+class ProvisioningMailDomainDNSView(IsGlobalChannelMixin, APIView):
+    """List all mail domains with the DNS records we expect for them.
+
+    GET /api/v1.0/provisioning/maildomains/dns/?page=2&page_size=1000
+
+    Paginated with the standard ``page``/``page_size`` params (up to 1000
+    domains per page). Ordered by
+    creation date ascending (``id`` as tiebreaker) so domains created while
+    a caller is walking the pages are appended after the pages already
+    fetched, instead of shifting rows across page boundaries.
+
+    Global-only, like the other provisioning endpoints.
+    """
+
+    authentication_classes = [ChannelApiKeyAuthentication]
+    permission_classes = [channel_scope(ChannelApiKeyScope.MAILDOMAINS_READ)]
+
+    @extend_schema(exclude=True)
+    def get(self, request):
+        """Return a page of mail domains with their expected DNS records."""
+        # Only the fields needed to render the DKIM record, so the encrypted
+        # private keys are never loaded or decrypted. ``domain`` is required
+        # for the prefetch to map keys back to their domain.
+        active_dkim_keys = models.DKIMKey.objects.filter(is_active=True).only(
+            "selector", "public_key", "algorithm", "is_active", "domain"
+        )
+
+        queryset = models.MailDomain.objects.prefetch_related(
+            Prefetch("dkim_keys", queryset=active_dkim_keys)
+        ).order_by("created_at", "id")
+
+        paginator = MailDomainDNSPagination()
+        page = paginator.paginate_queryset(queryset, request, view=self)
+
+        results = [
+            {
+                "id": str(domain.id),
+                "name": domain.name,
+                "expected_dns_records": domain.get_expected_dns_records(),
+            }
+            for domain in page
+        ]
+
+        return paginator.get_paginated_response(results)
+
+
 def _serialize_mailbox_with_users(
     mailbox, role=None, maildomain_custom_attributes=None
 ):

src/backend/core/enums.py

@@ -399,6 +399,7 @@ class ChannelApiKeyScope(models.TextChoices):
 
     METRICS_READ = "metrics:read", "Read usage metrics"
     MAILBOXES_READ = "mailboxes:read", "Read mailboxes (and their users/roles)"
+    MAILDOMAINS_READ = "maildomains:read", "Read maildomains"
     MESSAGES_SEND = "messages:send", "Send outbound messages"
     MAILDOMAINS_CREATE = "maildomains:create", "Create new maildomains"
 
@@ -407,7 +408,6 @@ class ChannelApiKeyScope(models.TextChoices):
     # glance and so the WRITE/CREATE convention is documented.
     #
     # Reads:
-    #   MAILDOMAINS_READ   = "maildomains:read",       "Read maildomains"
     #   USERS_READ         = "users:read",             "Read users"
     #   LABELS_READ        = "labels:read",            "Read labels"
     #   CONTACTS_READ      = "contacts:read",          "Read contacts"

src/backend/core/models.py

@@ -436,6 +436,13 @@ def generate_dkim_key(
 
     def get_active_dkim_key(self):
         """Get the most recent active DKIM key for this domain."""
+        # Use prefetched keys if available to avoid a query per domain
+        if (
+            hasattr(self, "_prefetched_objects_cache")
+            and "dkim_keys" in self._prefetched_objects_cache
+        ):
+            return next((key for key in self.dkim_keys.all() if key.is_active), None)
+
         return (
             DKIMKey.objects.filter(
                 domain=self, is_active=True

src/backend/core/tests/api/test_provisioning_maildomains_dns.py

@@ -0,0 +1,250 @@
+"""Tests for the maildomain DNS provisioning endpoint."""
+# pylint: disable=redefined-outer-name, unused-argument
+
+import uuid
+
+from django.db import connection
+from django.test.utils import CaptureQueriesContext
+from django.urls import reverse
+
+import pytest
+
+from core.enums import ChannelApiKeyScope, ChannelScopeLevel
+from core.factories import MailDomainFactory, make_api_key_channel
+
+pytestmark = pytest.mark.django_db
+
+
+@pytest.fixture
+def url():
+    """Returns the URL for the maildomain DNS provisioning endpoint."""
+    return reverse("provisioning-maildomains-dns")
+
+
+@pytest.fixture
+def correctly_configured_header(db):
+    """
+    Returns the authentication headers for the endpoint via a
+    scope_level=global api_key Channel with maildomains:read.
+    """
+    channel, plaintext = make_api_key_channel(
+        scopes=(ChannelApiKeyScope.MAILDOMAINS_READ.value,),
+        name="provisioning-dns-test",
+    )
+    return {
+        "HTTP_X_CHANNEL_ID": str(channel.id),
+        "HTTP_X_API_KEY": plaintext,
+    }
+
+
+class TestProvisioningMailDomainsDNS:
+    """Tests for the maildomain DNS provisioning endpoint."""
+
+    def test_requires_auth(self, api_client, url, correctly_configured_header):
+        """Requires a valid api_key Channel with maildomains:read to access."""
+
+        assert api_client.get(url).status_code == 401
+
+        response = api_client.get(
+            url,
+            HTTP_X_CHANNEL_ID=str(uuid.uuid4()),
+            HTTP_X_API_KEY="invalid_token",
+        )
+        assert response.status_code == 401
+
+        response = api_client.get(url, **correctly_configured_header)
+        assert response.status_code == 200
+
+    def test_requires_maildomains_read_scope(self, api_client, url):
+        """An api_key Channel without maildomains:read is rejected."""
+
+        channel, plaintext = make_api_key_channel(
+            scopes=(ChannelApiKeyScope.MAILBOXES_READ.value,),
+            name="no-maildomains-read-scope",
+        )
+        response = api_client.get(
+            url,
+            HTTP_X_CHANNEL_ID=str(channel.id),
+            HTTP_X_API_KEY=plaintext,
+        )
+        assert response.status_code == 403
+
+    def test_requires_global_channel(self, api_client, url):
+        """A non-global api_key Channel is rejected even with the right scope."""
+
+        domain = MailDomainFactory()
+        channel, plaintext = make_api_key_channel(
+            scope_level=ChannelScopeLevel.MAILDOMAIN,
+            scopes=(ChannelApiKeyScope.MAILDOMAINS_READ.value,),
+            maildomain=domain,
+            name="maildomain-scope",
+        )
+        response = api_client.get(
+            url,
+            HTTP_X_CHANNEL_ID=str(channel.id),
+            HTTP_X_API_KEY=plaintext,
+        )
+        assert response.status_code == 403
+
+    def test_no_maildomains(self, api_client, url, correctly_configured_header):
+        """Returns an empty list when no maildomain exists."""
+
+        response = api_client.get(url, **correctly_configured_header)
+        assert response.status_code == 200
+        assert response.json() == {
+            "count": 0,
+            "next": None,
+            "previous": None,
+            "results": [],
+        }
+
+    def test_lists_all_maildomains_in_creation_order(
+        self, api_client, url, correctly_configured_header
+    ):
+        """All maildomains are returned, oldest first."""
+
+        MailDomainFactory(name="zeta.example.com")
+        MailDomainFactory(name="alpha.example.com")
+
+        response = api_client.get(url, **correctly_configured_header)
+        assert response.status_code == 200
+
+        content = response.json()
+        assert content["count"] == 2
+        assert [result["name"] for result in content["results"]] == [
+            "zeta.example.com",
+            "alpha.example.com",
+        ]
+
+    def test_expected_dns_records(self, api_client, url, correctly_configured_header):
+        """Each maildomain is returned with its expected DNS records."""
+
+        domain = MailDomainFactory(name="example-dns.com")
+
+        response = api_client.get(url, **correctly_configured_header)
+        assert response.status_code == 200
+
+        content = response.json()
+        assert content["count"] == 1
+        result = content["results"][0]
+        assert result["id"] == str(domain.id)
+        assert result["name"] == "example-dns.com"
+        assert result["expected_dns_records"] == domain.get_expected_dns_records()
+
+        records = result["expected_dns_records"]
+        assert len([r for r in records if r["type"] == "mx"]) == 2
+        assert any("v=spf1" in r["value"] for r in records)
+        assert any(r["target"] == "_dmarc" for r in records)
+
+        # A DKIM key is generated on domain creation
+        dkim_records = [r for r in records if "DKIM1" in r["value"]]
+        assert len(dkim_records) == 1
+        assert dkim_records[0]["target"].endswith("._domainkey")
+
+    def test_pagination(self, api_client, url, correctly_configured_header):
+        """Results are paginated with page/page_size, oldest first."""
+
+        for index in range(5):
+            MailDomainFactory(name=f"domain{index}.example.com")
+
+        response = api_client.get(f"{url}?page_size=2", **correctly_configured_header)
+        assert response.status_code == 200
+        content = response.json()
+        assert content["count"] == 5
+        assert content["previous"] is None
+        assert content["next"] is not None
+        assert [result["name"] for result in content["results"]] == [
+            "domain0.example.com",
+            "domain1.example.com",
+        ]
+
+        response = api_client.get(
+            f"{url}?page_size=2&page=3", **correctly_configured_header
+        )
+        assert response.status_code == 200
+        content = response.json()
+        assert content["next"] is None
+        assert [result["name"] for result in content["results"]] == [
+            "domain4.example.com"
+        ]
+
+        # Out-of-range pages are rejected by the paginator
+        response = api_client.get(
+            f"{url}?page_size=2&page=99", **correctly_configured_header
+        )
+        assert response.status_code == 404
+
+    def test_page_size_is_capped_at_1000(
+        self, api_client, url, correctly_configured_header
+    ):
+        """page_size is honored up to 1000, above which it is clamped."""
+
+        MailDomainFactory.create_batch(3)
+
+        response = api_client.get(
+            f"{url}?page_size=1000", **correctly_configured_header
+        )
+        assert response.status_code == 200
+        assert response.json()["next"] is None
+
+        # Asking for more than the max falls back to the max, not to an error
+        response = api_client.get(
+            f"{url}?page_size=5000", **correctly_configured_header
+        )
+        assert response.status_code == 200
+        assert len(response.json()["results"]) == 3
+
+    def test_domain_created_mid_walk_does_not_shift_pages(
+        self, api_client, url, correctly_configured_header
+    ):
+        """A domain created between two page requests lands after the pages already read."""
+
+        for index in range(4):
+            MailDomainFactory(name=f"domain{index}.example.com")
+
+        response = api_client.get(f"{url}?page_size=2", **correctly_configured_header)
+        first_page = [result["name"] for result in response.json()["results"]]
+        assert first_page == ["domain0.example.com", "domain1.example.com"]
+
+        MailDomainFactory(name="created-mid-walk.example.com")
+
+        response = api_client.get(
+            f"{url}?page_size=2&page=2", **correctly_configured_header
+        )
+        second_page = [result["name"] for result in response.json()["results"]]
+
+        # No row from page 1 reappears, and none is skipped
+        assert second_page == ["domain2.example.com", "domain3.example.com"]
+
+        response = api_client.get(
+            f"{url}?page_size=2&page=3", **correctly_configured_header
+        )
+        assert [result["name"] for result in response.json()["results"]] == [
+            "created-mid-walk.example.com"
+        ]
+
+    def test_dkim_keys_are_prefetched(
+        self, api_client, url, correctly_configured_header
+    ):
+        """The query count does not grow with the number of domains."""
+
+        MailDomainFactory()
+        with CaptureQueriesContext(connection) as baseline:
+            assert api_client.get(url, **correctly_configured_header).status_code == 200
+
+        MailDomainFactory.create_batch(3)
+        with CaptureQueriesContext(connection) as queries:
+            response = api_client.get(url, **correctly_configured_header)
+        assert response.status_code == 200
+        assert response.json()["count"] == 4
+
+        assert len(queries) == len(baseline)
+
+        # The encrypted private keys are never selected, and no deferred-field
+        # query fetches them afterwards
+        sql = [query["sql"] for query in queries.captured_queries]
+        assert any("public_key" in statement for statement in sql), (
+            "the DKIM prefetch query was not captured, the assertion below would pass "
+            "for the wrong reason"
+        )
+        assert not any("private_key" in statement for statement in sql)

src/backend/core/urls.py

@@ -46,6 +46,7 @@
 from core.api.viewsets.placeholder import DraftPlaceholderView, PlaceholderView
 from core.api.viewsets.provisioning import (
     ProvisioningMailboxView,
+    ProvisioningMailDomainDNSView,
     ProvisioningMailDomainView,
 )
 from core.api.viewsets.send import SendMessageView
@@ -306,6 +307,11 @@
         ProvisioningMailDomainView.as_view(),
         name="provisioning-maildomains",
     ),
+    path(
+        f"api/{settings.API_VERSION}/provisioning/maildomains/dns/",
+        ProvisioningMailDomainDNSView.as_view(),
+        name="provisioning-maildomains-dns",
+    ),
     path(
         f"api/{settings.API_VERSION}/submit/",
         SubmitRawEmailView.as_view(),