Skip to content

Commit b61ab0a

Browse files
committed
⚡️(backend) search users through the trigram indexes
The user search filtered on a similarity score computed on the raw email and full_name columns. The trigram indexes are built on f_unaccent(email) and f_unaccent(full_name) and only serve the % operator, so each search scanned the whole user table, and all the matches were then filtered and sorted in Python. The search now uses the % operator on the indexed expressions, with a similarity threshold scoped to the transaction. It filters the shared and same-domain users in SQL, and fetches no more users than the response limit.
1 parent 9c233b8 commit b61ab0a

3 files changed

Lines changed: 196 additions & 71 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ and this project adheres to
66

77
## [Unreleased]
88

9+
### Changed
10+
11+
- ⚡️(backend) search users through the trigram indexes instead of a full scan
12+
913
## [v6.0.3] - 2026-10-23
1014

1115
### Added

‎src/backend/core/api/viewsets.py‎

Lines changed: 81 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
from django.core.exceptions import ValidationError
1919
from django.core.files.storage import default_storage
2020
from django.core.validators import URLValidator
21-
from django.db import DatabaseError, transaction
21+
from django.db import DatabaseError, connection, transaction
2222
from django.db import models as db
2323
from django.db.models.expressions import RawSQL
2424
from django.db.models.functions import Greatest, Left, Length
@@ -104,6 +104,21 @@
104104

105105
# pylint: disable=too-many-ancestors
106106

107+
# Minimum trigram similarity between a user search and an email or a full name
108+
USERS_SEARCH_SIMILARITY_THRESHOLD = 0.2
109+
110+
111+
def set_trigram_similarity_threshold(threshold):
112+
"""
113+
Set the threshold of the trigram `%` operator for the current transaction only,
114+
so that it never leaks to the next users of a pooled connection.
115+
"""
116+
with connection.cursor() as cursor:
117+
cursor.execute(
118+
"SELECT set_config('pg_trgm.similarity_threshold', %s, true)",
119+
[str(threshold)],
120+
)
121+
107122

108123
class NestedGenericViewSet(viewsets.GenericViewSet):
109124
"""
@@ -233,94 +248,89 @@ def get_queryset(self):
233248

234249
filter_data = filterset.form.cleaned_data
235250
query = remove_accents(filter_data["q"])
251+
if not query:
252+
return []
236253

237-
# For emails, match emails by Levenstein distance to prevent typing errors
254+
limit = settings.API_USERS_LIST_LIMIT
255+
256+
# The trigram indexes of migration 0027 are built on these two expressions:
257+
# only a `%` filter on them uses the indexes, any other condition on the
258+
# user table (a similarity score, the raw columns) scans all the users.
259+
queryset = queryset.alias(
260+
unaccented_email=db.Func(
261+
"email", function="f_unaccent", output_field=db.TextField()
262+
),
263+
unaccented_full_name=db.Func(
264+
"full_name", function="f_unaccent", output_field=db.TextField()
265+
),
266+
)
267+
268+
# For emails, match emails by Levenstein distance to prevent typing errors,
269+
# among the users that the trigram index finds close enough to the query
238270
if "@" in query:
239-
return (
240-
queryset.annotate(
241-
distance=RawSQL(
242-
"levenshtein(unaccent(email::text), %s::text)", (query,)
271+
with transaction.atomic():
272+
set_trigram_similarity_threshold(USERS_SEARCH_SIMILARITY_THRESHOLD)
273+
return list(
274+
queryset.filter(unaccented_email__trigram_similar=query)
275+
.annotate(
276+
distance=RawSQL(
277+
"levenshtein(unaccent(email::text), %s::text)", (query,)
278+
)
243279
)
280+
.filter(distance__lte=3)
281+
.order_by("distance", "email")[:limit]
244282
)
245-
.filter(distance__lte=3)
246-
.order_by("distance", "email")[: settings.API_USERS_LIST_LIMIT]
247-
)
248283

249-
# Use trigram similarity for non-email-like queries
250-
# For performance reasons we filter first by similarity, which relies on an
251-
# index, then only calculate precise similarity scores for sorting purposes.
284+
# Use trigram similarity for non-email-like queries.
252285
#
253-
# Additionally results are reordered to prefer users "closer" to the current
254-
# user: users they recently shared documents with, then same email domain.
255-
# To achieve that without complex SQL, we build a proximity score in Python
256-
# and return the top N results.
257-
# For security results, users that match neither of these proximity criteria
286+
# Results are ordered to prefer users "closer" to the current user: users
287+
# they recently shared documents with, then users of the same email domain.
288+
# For security reasons, users that match neither of these proximity criteria
258289
# are not returned at all, to prevent email enumeration.
259290
current_user = self.request.user
260291
shared_map = users_sharing_documents_with(current_user.id)
292+
shared_ids = list(shared_map)
293+
user_email_domain = get_domain_from_email(current_user.email)
261294

262-
user_email_domain = get_domain_from_email(current_user.email) or ""
263-
264-
candidates = list(
265-
queryset.annotate(
266-
sim_email=TrigramSimilarity("email", query),
267-
sim_name=TrigramSimilarity("full_name", query),
295+
similar_users = queryset.filter(
296+
db.Q(unaccented_email__trigram_similar=query)
297+
| db.Q(unaccented_full_name__trigram_similar=query)
298+
).annotate(
299+
similarity=Greatest(
300+
TrigramSimilarity("unaccented_email", query),
301+
TrigramSimilarity("unaccented_full_name", query),
268302
)
269-
.annotate(similarity=Greatest("sim_email", "sim_name"))
270-
.filter(similarity__gt=0.2)
271-
.order_by("-similarity")
272303
)
273304

274-
# Keep only users that either share documents with the current user
275-
# or have an email with the same domain as the current user.
276-
filtered_candidates = []
277-
for u in candidates:
278-
candidate_domain = get_domain_from_email(u.email) or ""
279-
if shared_map.get(u.id) or (
280-
user_email_domain and candidate_domain == user_email_domain
281-
):
282-
filtered_candidates.append(u)
283-
284-
candidates = filtered_candidates
285-
286-
# Build ordering key for each candidate
287-
def _sort_key(u):
288-
# shared priority: most recent first
289-
# Use shared_last_at timestamp numeric for secondary ordering when shared.
290-
shared_last_at = shared_map.get(u.id)
291-
if shared_last_at:
292-
is_shared = 1
293-
shared_score = int(shared_last_at.timestamp())
294-
else:
295-
is_shared = 0
296-
shared_score = 0
297-
298-
# domain proximity
299-
candidate_email_domain = get_domain_from_email(u.email) or ""
305+
with transaction.atomic():
306+
set_trigram_similarity_threshold(USERS_SEARCH_SIMILARITY_THRESHOLD)
300307

301-
same_full_domain = (
302-
1
303-
if candidate_email_domain
304-
and candidate_email_domain == user_email_domain
305-
else 0
308+
# Bounded by the number of users the current user shares documents with
309+
shared_users = (
310+
list(similar_users.filter(id__in=shared_ids)) if shared_ids else []
306311
)
307312

308-
# similarity fallback
309-
sim = getattr(u, "similarity", 0) or 0
310-
311-
return (
312-
is_shared,
313-
shared_score,
314-
same_full_domain,
315-
sim,
316-
)
313+
# A domain can count many users: only fetch the ones that can still fit
314+
domain_users = []
315+
if user_email_domain and len(shared_users) < limit:
316+
domain_users = list(
317+
similar_users.filter(email__endswith=f"@{user_email_domain}")
318+
.exclude(id__in=shared_ids)
319+
.order_by("-similarity")[: limit - len(shared_users)]
320+
)
317321

318-
# Sort candidates by the key descending and return top N as a queryset-like
319-
# list. Keep return type consistent with previous behavior (QuerySet slice
320-
# was returned) by returning a list of model instances.
321-
candidates.sort(key=_sort_key, reverse=True)
322+
# Most recently shared first, then same domain, then the closest match
323+
shared_users.sort(
324+
key=lambda user: (
325+
shared_map[user.id],
326+
bool(user_email_domain)
327+
and get_domain_from_email(user.email) == user_email_domain,
328+
user.similarity,
329+
),
330+
reverse=True,
331+
)
322332

323-
return candidates[: settings.API_USERS_LIST_LIMIT]
333+
return (shared_users + domain_users)[:limit]
324334

325335
@drf.decorators.action(
326336
detail=False,

‎src/backend/core/tests/test_api_users.py‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
Test users API endpoints in the impress core app.
33
"""
44

5+
from django.db import connection
6+
from django.test.utils import CaptureQueriesContext
57
from django.utils import timezone
68

79
import pytest
@@ -430,6 +432,115 @@ def test_api_users_list_query_inactive():
430432
assert user_ids == [str(lennon.id)]
431433

432434

435+
def test_api_users_list_shared_users_before_more_similar_domain_users():
436+
"""
437+
A user sharing documents with the current user should come first even if
438+
enough users of the same domain match the query better to fill the response,
439+
and the remaining places should go to the users of the same domain.
440+
"""
441+
user = factories.UserFactory(
442+
email="martin.bernard@anct.gouv.fr", full_name="Martin Bernard"
443+
)
444+
445+
client = APIClient()
446+
client.force_login(user)
447+
448+
shared = factories.UserFactory(
449+
email="jp.dupont@beta.gouv.fr", full_name="Jean-Pierre Dupont-Martinez"
450+
)
451+
document = factories.DocumentFactory()
452+
factories.UserDocumentAccessFactory(user=user, document=document)
453+
factories.UserDocumentAccessFactory(user=shared, document=document)
454+
455+
same_domain = [
456+
factories.UserFactory(email=f"pierre.{i}@anct.gouv.fr", full_name="Pierre")
457+
for i in range(8)
458+
]
459+
460+
response = client.get("/api/v1.0/users/?q=Pierre")
461+
462+
assert response.status_code == 200
463+
user_ids = [user["id"] for user in response.json()]
464+
assert len(user_ids) == 5
465+
assert user_ids[0] == str(shared.id)
466+
assert set(user_ids[1:]) <= {str(other.id) for other in same_domain}
467+
468+
469+
@pytest.mark.parametrize(
470+
"query,indexes",
471+
[
472+
(
473+
"Bowman",
474+
["user_email_unaccent_trgm_idx", "user_full_name_unaccent_trgm_idx"],
475+
),
476+
("david.bowman@work.com", ["user_email_unaccent_trgm_idx"]),
477+
],
478+
)
479+
def test_api_users_list_query_uses_trigram_indexes(query, indexes):
480+
"""
481+
Searching users should go through the trigram indexes on the unaccented email
482+
and full name instead of scanning the whole user table.
483+
"""
484+
user = factories.UserFactory(email="user@work.com")
485+
dave = factories.UserFactory(
486+
email="david.bowman@work.com", full_name="David Bowman"
487+
)
488+
489+
client = APIClient()
490+
client.force_login(user)
491+
492+
with CaptureQueriesContext(connection) as context:
493+
response = client.get(f"/api/v1.0/users/?q={query}")
494+
495+
assert response.status_code == 200
496+
assert [user["id"] for user in response.json()] == [str(dave.id)]
497+
498+
searches = [
499+
captured["sql"]
500+
for captured in context.captured_queries
501+
if 'FROM "impress_user"' in captured["sql"] and query in captured["sql"]
502+
]
503+
assert searches
504+
505+
with connection.cursor() as cursor:
506+
# With so few users, a sequential scan would win even if an index can serve
507+
cursor.execute("SET LOCAL enable_seqscan = off")
508+
for search in searches:
509+
cursor.execute(f"EXPLAIN {search}")
510+
plan = "\n".join(row[0] for row in cursor.fetchall())
511+
for index in indexes:
512+
assert index in plan
513+
514+
515+
@pytest.mark.django_db(transaction=True)
516+
def test_api_users_list_similarity_threshold_does_not_leak():
517+
"""
518+
The trigram similarity threshold set for a search should not stay on the
519+
connection after the request, other requests share it through the pool.
520+
"""
521+
user = factories.UserFactory(email="user@work.com")
522+
dave = factories.UserFactory(
523+
email="david.bowman@work.com", full_name="David Bowman"
524+
)
525+
526+
client = APIClient()
527+
client.force_login(user)
528+
529+
with connection.cursor() as cursor:
530+
cursor.execute("SELECT show_limit()")
531+
default_threshold = cursor.fetchone()[0]
532+
533+
for query in ["Bowman", "david.bowman@work.com"]:
534+
response = client.get(f"/api/v1.0/users/?q={query}")
535+
536+
assert response.status_code == 200
537+
assert [user["id"] for user in response.json()] == [str(dave.id)]
538+
539+
with connection.cursor() as cursor:
540+
cursor.execute("SELECT show_limit()")
541+
assert cursor.fetchone()[0] == default_threshold
542+
543+
433544
def test_api_users_retrieve_me_anonymous():
434545
"""Anonymous users should not be allowed to list users."""
435546
factories.UserFactory.create_batch(2)

0 commit comments

Comments
 (0)