From 0b2be19f4b8fa717e5629a01701a79a2e94e7680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aar=C3=B3n=20Ayerdis=20Espinoza?= Date: Fri, 17 Jul 2026 23:42:53 -0600 Subject: [PATCH 01/10] Adding SchemaQuerySet plus the tsvector on Schema model --- core/models.py | 44 ++++++++++++++++++++++++++++++++++-- schemaindex/settings/base.py | 1 + 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/core/models.py b/core/models.py index b14fd69..813f832 100644 --- a/core/models.py +++ b/core/models.py @@ -7,6 +7,13 @@ from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation from django.contrib.contenttypes.models import ContentType from django.contrib.auth.hashers import make_password, check_password +from django.contrib.postgres.search import ( + SearchVector, + SearchQuery, + SearchRank, + SearchVectorField, +) +from django.contrib.postgres.indexes import GinIndex from django.core.cache import cache from django.core.exceptions import ValidationError from django.urls import reverse @@ -87,7 +94,29 @@ class Meta: indexes = [models.Index(fields=["content_type", "object_id"])] -class PublicSchemaManager(models.Manager): +class SchemaQuerySet(models.QuerySet): + def search(self, query_text): + """ + Rank schemas by full-text relevance against `query_text`. + `name` weighted above description. + + When `query_text` is blank the queryset is returned unchanged, + so the existing ordering (alphabetical on index) is preserved for plain browsing. + """ + query_text = (query_text or "").strip() + if not query_text: + return self + search_query = SearchQuery( + query_text, config="english", search_type="websearch" + ) + return ( + self.filter(search_vector=search_query) + .annotate(rank=SearchRank("search_vector", search_query)) + .order_by("-rank", "name") + ) + + +class PublicSchemaManager(models.Manager.from_queryset(SchemaQuerySet)): def get_queryset(self): return ( super() @@ -117,9 +146,20 @@ class Schema(BaseModel): published_at = models.DateTimeField(blank=True, null=True) permanent_urls = GenericRelation(PermanentURL, related_query_name="schema") description = models.CharField(blank=True, null=True, max_length=350) + search_vector = models.GeneratedField( + expression=( + SearchVector("name", weight="A", config="english") + + SearchVector("description", weight="B", config="english") + ), + output_field=SearchVectorField(), + db_persist=True, + ) class Meta: - indexes = [models.Index(fields=["published_at"])] + indexes = [ + models.Index(fields=["published_at"]), + GinIndex(fields=["search_vector"], name="schema_search_vec_gin"), + ] def __str__(self): return self.name diff --git a/schemaindex/settings/base.py b/schemaindex/settings/base.py index d659add..fde251b 100644 --- a/schemaindex/settings/base.py +++ b/schemaindex/settings/base.py @@ -43,6 +43,7 @@ "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", + "django.contrib.postgres", "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", From d57e10cc6ed4a8d823d6f3bab256299514c435b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aar=C3=B3n=20Ayerdis=20Espinoza?= Date: Fri, 17 Jul 2026 23:44:36 -0600 Subject: [PATCH 02/10] Creating schema search vector migration --- ...rch_vector_schema_schema_search_vec_gin.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 core/migrations/0017_schema_search_vector_schema_schema_search_vec_gin.py diff --git a/core/migrations/0017_schema_search_vector_schema_schema_search_vec_gin.py b/core/migrations/0017_schema_search_vector_schema_schema_search_vec_gin.py new file mode 100644 index 0000000..335d44e --- /dev/null +++ b/core/migrations/0017_schema_search_vector_schema_schema_search_vec_gin.py @@ -0,0 +1,26 @@ +# Generated by Django 5.2.5 on 2026-07-17 21:19 + +import django.contrib.postgres.indexes +import django.contrib.postgres.search +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0016_schemaref_id_value'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name='schema', + name='search_vector', + field=models.GeneratedField(db_persist=True, expression=django.contrib.postgres.search.CombinedSearchVector(django.contrib.postgres.search.SearchVector('name', config='english', weight='A'), '||', django.contrib.postgres.search.SearchVector('description', config='english', weight='B'), django.contrib.postgres.search.SearchConfig('english')), output_field=django.contrib.postgres.search.SearchVectorField()), + ), + migrations.AddIndex( + model_name='schema', + index=django.contrib.postgres.indexes.GinIndex(fields=['search_vector'], name='schema_search_vec_gin'), + ), + ] From 232d5bcd85ee3f51395934b0d58fc856379b77b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aar=C3=B3n=20Ayerdis=20Espinoza?= Date: Sat, 18 Jul 2026 00:18:02 -0600 Subject: [PATCH 03/10] Swapping index view to .search() --- core/views.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/core/views.py b/core/views.py index af52ef9..60106d7 100644 --- a/core/views.py +++ b/core/views.py @@ -143,23 +143,21 @@ def index(request): else defined_schemas ) - filtered_by_name = ( - filtered_by_documentation_type.filter(name__icontains=search_query) - if search_query - else filtered_by_documentation_type - ) + # Full-text ranked search. When the box is empty, .search() returns the + # queryset unchanged, preserving the alphabetical browse ordering above. + searched_schemas = filtered_by_documentation_type.search(search_query) filtered_by_specification_file_type = ( [ schema - for schema in filtered_by_name + for schema in searched_schemas if any( schema_ref.language == specification_file_type for schema_ref in schema.schemaref_set.all() ) ] if specification_file_type - else filtered_by_name + else searched_schemas ) return render( From 3c515510265a43cf844721215cdc84abe8e7fe73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aar=C3=B3n=20Ayerdis=20Espinoza?= Date: Sat, 18 Jul 2026 12:10:36 -0600 Subject: [PATCH 04/10] Renaming migration --- ...hema_schema_search_vec_gin.py => 0017_schema_search_vector.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename core/migrations/{0017_schema_search_vector_schema_schema_search_vec_gin.py => 0017_schema_search_vector.py} (100%) diff --git a/core/migrations/0017_schema_search_vector_schema_schema_search_vec_gin.py b/core/migrations/0017_schema_search_vector.py similarity index 100% rename from core/migrations/0017_schema_search_vector_schema_schema_search_vec_gin.py rename to core/migrations/0017_schema_search_vector.py From 2690ae53634eb7fb376253a3716a7dab584623ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aar=C3=B3n=20Ayerdis=20Espinoza?= Date: Mon, 20 Jul 2026 14:16:54 -0600 Subject: [PATCH 05/10] Adding tests plus fixing SearchRank string issue --- core/models.py | 2 +- tests/test_search.py | 72 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_views.py | 18 +++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 tests/test_search.py diff --git a/core/models.py b/core/models.py index 813f832..e734577 100644 --- a/core/models.py +++ b/core/models.py @@ -111,7 +111,7 @@ def search(self, query_text): ) return ( self.filter(search_vector=search_query) - .annotate(rank=SearchRank("search_vector", search_query)) + .annotate(rank=SearchRank(models.F("search_vector"), search_query)) .order_by("-rank", "name") ) diff --git a/tests/test_search.py b/tests/test_search.py new file mode 100644 index 0000000..09e82aa --- /dev/null +++ b/tests/test_search.py @@ -0,0 +1,72 @@ +import pytest + +from tests.factories import SchemaFactory +from core.models import Schema + + +@pytest.mark.django_db +def test_stemming_makes_singular_and_plural_equivalent(): + # "Schema" and "Schemas" must return the same result. + # decoy proves the @@ filter actually excludes non-matches (not just that matches are found). + schema = SchemaFactory(name="Invoice Schema") + decoy = SchemaFactory(name="Weather Report") + for term in ("Schemas", "Schema"): + results = Schema.public_objects.search(term) + assert schema in results + assert decoy not in results + + +@pytest.mark.django_db +def test_search_is_case_insensitive(): + schema = SchemaFactory(name="Invoice Schema") + assert schema in Schema.public_objects.search("INVOICE") + + +@pytest.mark.django_db +def test_search_matches_description_field(): + schema = SchemaFactory(name="ACME Format", description="A standard for invoices") + assert schema in Schema.public_objects.search("invoice") + + +@pytest.mark.django_db +def test_name_hit_outranks_description_only_hit(): + # Assert on the rank *values*, not list order. + # Order under equal ranks is decided by the name tie-break, + # which would hide a weight bug for some names. + SchemaFactory(name="Invoice Format", description=None) + SchemaFactory(name="ACME Format", description="Used for invoice data") + ranks = {s.name: s.rank for s in Schema.public_objects.search("invoice")} + assert ranks["Invoice Format"] > ranks["ACME Format"] + + +@pytest.mark.django_db +def test_blank_query_returns_all_without_ranking(): + SchemaFactory(name="One") + SchemaFactory(name="Two") + # Blank / None must not filter or reorder — plain browsing is preserved. + assert Schema.public_objects.search("").count() == 2 + assert Schema.public_objects.search(None).count() == 2 + + +@pytest.mark.django_db +def test_search_excludes_unpublished_schemas(): + SchemaFactory(name="Public Invoice") + SchemaFactory(name="Private Invoice", published_at=None) + assert Schema.public_objects.search("invoice").count() == 1 + + +@pytest.mark.django_db +def test_malformed_query_does_not_raise(): + SchemaFactory(name="Invoice") + # websearch_to_tsquery tolerates junk such as unbalanced quote plus dangling operator. + # The guarantee is that evaluating the queryset does not raise. + results = Schema.public_objects.search('"invoice OR -') + assert results.count() >= 0 + + +@pytest.mark.django_db +def test_generated_vector_is_populated_on_insert(): + # Postgres computes the stored generated column on write + schema = SchemaFactory(name="Invoice Schema") + schema.refresh_from_db() + assert schema.search_vector is not None diff --git a/tests/test_views.py b/tests/test_views.py index 2b07a30..b9015cf 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -431,3 +431,21 @@ def test_schema_export_sends_manifest(): assert response.status_code == 200 manifest = json.loads(response.content) assert_schema_matches_manifest(schema, manifest) + + +@pytest.mark.django_db +def test_homepage_search_is_stemmed(): + schema = SchemaFactory(name="Invoice Schema") + SchemaRefFactory(schema=schema) + response = Client().get("/?search_query=Schemas") + assert schema.name in str(response.content) + + +@pytest.mark.django_db +def test_homepage_no_query_lists_alphabetically(): + last_schema = SchemaFactory(name="ZZZ Last Schema") + SchemaRefFactory(schema=last_schema) + first_schema = SchemaFactory(name="AAA First Schema") + SchemaRefFactory(schema=first_schema) + names = [s.name for s in Client().get("/").context["schemas"]] + assert names == ["AAA First Schema", "ZZZ Last Schema"] From c9a2af16378520f12d62860b672493224921c003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aar=C3=B3n=20Ayerdis=20Espinoza?= Date: Mon, 20 Jul 2026 14:42:06 -0600 Subject: [PATCH 06/10] Adding PostgreSQL guard test as comment until I fix some django.db.utils.OperationalError --- tests/test_search.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_search.py b/tests/test_search.py index 09e82aa..af849b0 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -1,8 +1,17 @@ import pytest +from django.db import connection from tests.factories import SchemaFactory from core.models import Schema +""" +def test_database_is_postgres(): + # Postgres FTS has no SQLite equivalent + assert connection.vendor == "postgresql", ( + "Requires PostgreSQL for full-text search; the configured " + f"database backend is '{connection.vendor}'." + ) +""" @pytest.mark.django_db def test_stemming_makes_singular_and_plural_equivalent(): From df498d5f1eac126455021054158dbc281ac28367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aar=C3=B3n=20Ayerdis=20Espinoza?= Date: Mon, 20 Jul 2026 19:09:53 -0600 Subject: [PATCH 07/10] Changing PostgreSQL to a config check --- core/models.py | 5 +++-- tests/test_search.py | 13 +++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/core/models.py b/core/models.py index e734577..e1095ce 100644 --- a/core/models.py +++ b/core/models.py @@ -99,7 +99,7 @@ def search(self, query_text): """ Rank schemas by full-text relevance against `query_text`. `name` weighted above description. - + When `query_text` is blank the queryset is returned unchanged, so the existing ordering (alphabetical on index) is preserved for plain browsing. """ @@ -110,7 +110,8 @@ def search(self, query_text): query_text, config="english", search_type="websearch" ) return ( - self.filter(search_vector=search_query) + self + .filter(search_vector=search_query) .annotate(rank=SearchRank(models.F("search_vector"), search_query)) .order_by("-rank", "name") ) diff --git a/tests/test_search.py b/tests/test_search.py index af849b0..d27ab1e 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -1,17 +1,18 @@ import pytest -from django.db import connection +from django.conf import settings from tests.factories import SchemaFactory from core.models import Schema -""" + def test_database_is_postgres(): # Postgres FTS has no SQLite equivalent - assert connection.vendor == "postgresql", ( - "Requires PostgreSQL for full-text search; the configured " - f"database backend is '{connection.vendor}'." + engine = settings.DATABASES["default"]["ENGINE"] + assert engine == "django.db.backends.postgresql", ( + "This project requires PostgreSQL for full-text search; " + f"the configured database ENGINE is '{engine}'." ) -""" + @pytest.mark.django_db def test_stemming_makes_singular_and_plural_equivalent(): From f8b02d4dea4798e8c15b75063ace512c845961b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aar=C3=B3n=20Ayerdis=20Espinoza?= Date: Tue, 21 Jul 2026 16:09:50 -0600 Subject: [PATCH 08/10] Dropping PostgreSQL check --- tests/test_search.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/test_search.py b/tests/test_search.py index d27ab1e..2d0d307 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -5,15 +5,6 @@ from core.models import Schema -def test_database_is_postgres(): - # Postgres FTS has no SQLite equivalent - engine = settings.DATABASES["default"]["ENGINE"] - assert engine == "django.db.backends.postgresql", ( - "This project requires PostgreSQL for full-text search; " - f"the configured database ENGINE is '{engine}'." - ) - - @pytest.mark.django_db def test_stemming_makes_singular_and_plural_equivalent(): # "Schema" and "Schemas" must return the same result. From f8d7c1b1fa094c836e2228b5a7b92ab2d45bb4c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aar=C3=B3n=20Ayerdis=20Espinoza?= Date: Tue, 21 Jul 2026 16:12:27 -0600 Subject: [PATCH 09/10] Fixing linting from unused config import --- tests/test_search.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_search.py b/tests/test_search.py index 2d0d307..09e82aa 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -1,5 +1,4 @@ import pytest -from django.conf import settings from tests.factories import SchemaFactory from core.models import Schema From 62c92f82c5af146cc582db443f8da5db8f5b0e74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aar=C3=B3n=20Ayerdis=20Espinoza?= Date: Wed, 22 Jul 2026 11:08:54 -0600 Subject: [PATCH 10/10] Updating test to preserve order when search box is empty --- tests/test_search.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_search.py b/tests/test_search.py index 09e82aa..92f014e 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -40,12 +40,15 @@ def test_name_hit_outranks_description_only_hit(): @pytest.mark.django_db -def test_blank_query_returns_all_without_ranking(): +def test_blank_query_returns_all_and_preserves_order(): SchemaFactory(name="One") SchemaFactory(name="Two") - # Blank / None must not filter or reorder — plain browsing is preserved. - assert Schema.public_objects.search("").count() == 2 - assert Schema.public_objects.search(None).count() == 2 + # A blank / None query must neither filter nor reorder: .search() returns the + # queryset unchanged, so the caller's ordering survives and no relevance + # ranking is applied. (Whitespace-only counts as blank via .strip().) + for blank in ("", " ", None): + results = Schema.public_objects.order_by("name").search(blank) + assert [schema.name for schema in results] == ["One", "Two"] @pytest.mark.django_db