Skip to content

Commit bbef3d9

Browse files
authored
Merge pull request #1642 from MiquelRForgeFlow/fix/deterministic-query-ordering
fix: make API list ordering deterministic
2 parents f0deda3 + 42d9fe1 commit bbef3d9

4 files changed

Lines changed: 101 additions & 19 deletions

File tree

‎pokemon_v2/api.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1021,7 +1021,7 @@ def get(self, request: Request, pokemon_id: int) -> Response:
10211021
"encounterconditionvaluemap_set__encounter_condition_value",
10221022
"encounterpokemondetail_set",
10231023
)
1024-
.order_by("location_area_id", "version_id", "encounter_slot_id")
1024+
.order_by("location_area_id", "version_id", "encounter_slot_id", "pk")
10251025
)
10261026

10271027
grouped_data: list[dict[str, Any]] = []

‎pokemon_v2/models.py‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
# pyright: reportIncompatibleVariableOverride=false
22
from __future__ import annotations
33

4-
from typing import Any
4+
from typing import Any, ClassVar, TypeVar
55

66
from django.db import models
7-
from typing_extensions import override
7+
from typing_extensions import Self, override
8+
9+
_ModelT = TypeVar("_ModelT", bound=models.Model)
810

911
__all__: tuple[str, ...] = (
1012
"Ability",
@@ -165,6 +167,7 @@
165167
"PalPark",
166168
"PalParkArea",
167169
"PalParkAreaName",
170+
"PokeApiManager",
168171
"PokeApiModel",
169172
"PokeathlonStat",
170173
"PokeathlonStatName",
@@ -230,7 +233,24 @@
230233
############################
231234

232235

236+
class PokeApiManager(models.Manager[_ModelT]):
237+
"""Default manager ordering every queryset by primary key, so that the API
238+
serializes lists deterministically. Call sites needing another order
239+
override it with ``order_by()``.
240+
241+
A ``distinct(*fields)`` call has to spell out a matching ``order_by()``,
242+
since PostgreSQL requires the leading ``ORDER BY`` expressions to match the
243+
``DISTINCT ON`` ones. A plain ``distinct()`` is unaffected.
244+
"""
245+
246+
@override
247+
def get_queryset(self) -> models.QuerySet[_ModelT]:
248+
return super().get_queryset().order_by("pk")
249+
250+
233251
class PokeApiModel(models.Model):
252+
objects: ClassVar[PokeApiManager[Self]] = PokeApiManager()
253+
234254
class Meta:
235255
abstract = True
236256

‎pokemon_v2/serializers.py‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1053,7 +1053,7 @@ def get_method_rates(self, obj: LocationAreaEncounterRate) -> ReturnList[ReturnD
10531053
rates = (
10541054
LocationAreaEncounterRate.objects.filter(location_area=obj, encounter_method__isnull=False)
10551055
.select_related("encounter_method", "version")
1056-
.order_by("encounter_method_id")
1056+
.order_by("encounter_method_id", "version_id", "pk")
10571057
)
10581058
grouped_rates: list[dict[str, Any]] = [
10591059
{
@@ -1081,7 +1081,7 @@ def get_encounters(self, obj: LocationArea) -> ReturnList[ReturnDict[str, Any]]:
10811081
"encounterconditionvaluemap_set",
10821082
"encounterpokemondetail_set",
10831083
)
1084-
.order_by("pokemon_id", "version_id")
1084+
.order_by("pokemon_id", "version_id", "encounter_slot_id", "pk")
10851085
)
10861086

10871087
grouped_data: list[dict[str, Any]] = []
@@ -1597,7 +1597,7 @@ def get_held_by_pokemon(self, obj: Item) -> ReturnList[ReturnDict[str, Any]]:
15971597
pokemon_items = (
15981598
PokemonItem.objects.filter(item=obj)
15991599
.select_related("pokemon", "version")
1600-
.order_by("pokemon_id", "version_id")
1600+
.order_by("pokemon_id", "version_id", "pk")
16011601
)
16021602
grouped_data: list[dict[str, Any]] = [
16031603
{
@@ -1739,7 +1739,9 @@ class Meta:
17391739
@extend_schema_field(BerryFlavorBerryMapSerializer(many=True))
17401740
def get_berries_with_flavor(self, obj: BerryFlavor) -> ReturnList[ReturnDict[str, Any]]:
17411741
flavor_map_objects = (
1742-
BerryFlavorMap.objects.filter(berry_flavor=obj, potency__gt=0).select_related("berry").order_by("potency")
1742+
BerryFlavorMap.objects.filter(berry_flavor=obj, potency__gt=0)
1743+
.select_related("berry")
1744+
.order_by("potency", "berry_id", "pk")
17431745
)
17441746
return cast(
17451747
"ReturnList[ReturnDict[str, Any]]",
@@ -2629,7 +2631,7 @@ def get_pokemon_form_sprites(self, obj: PokemonForm) -> dict[str, Any]:
26292631

26302632
@extend_schema_field(PokemonFormTypeSerializer(many=True))
26312633
def get_pokemon_form_types(self, obj: PokemonForm) -> ReturnList[ReturnDict[str, Any]]:
2632-
form_types = PokemonFormType.objects.filter(pokemon_form=obj).select_related("type").order_by("slot")
2634+
form_types = PokemonFormType.objects.filter(pokemon_form=obj).select_related("type").order_by("slot", "pk")
26332635

26342636
if form_types:
26352637
return cast(
@@ -2638,7 +2640,7 @@ def get_pokemon_form_types(self, obj: PokemonForm) -> ReturnList[ReturnDict[str,
26382640
)
26392641

26402642
# Fall back to parent Pokemon's types if no form-specific types exist
2641-
pokemon_types = PokemonType.objects.filter(pokemon=obj.pokemon).select_related("type").order_by("slot")
2643+
pokemon_types = PokemonType.objects.filter(pokemon=obj.pokemon).select_related("type").order_by("slot", "pk")
26422644
return cast(
26432645
"ReturnList[ReturnDict[str, Any]]",
26442646
PokemonTypeSerializer(pokemon_types, many=True, context=self.context).data,
@@ -2946,7 +2948,7 @@ def get_pokemon_moves(self, obj: Pokemon) -> list[dict[str, Any]]:
29462948
pokemon_moves = (
29472949
PokemonMove.objects.filter(pokemon=obj, move__isnull=False)
29482950
.select_related("move", "version_group", "move_learn_method")
2949-
.order_by("move__id", "version_group_id")
2951+
.order_by("move__id", "version_group_id", "move_learn_method_id", "level", "pk")
29502952
)
29512953

29522954
vg_cache: dict[int, Any] = {}
@@ -2990,7 +2992,7 @@ def get_pokemon_held_items(self, obj: Pokemon) -> list[dict[str, Any]]:
29902992
pokemon_items = (
29912993
PokemonItem.objects.filter(pokemon=obj, item__isnull=False)
29922994
.select_related("item", "version")
2993-
.order_by("item__id", "version_id")
2995+
.order_by("item__id", "version_id", "pk")
29942996
)
29952997

29962998
version_cache: dict[int, Any] = {}
@@ -3033,7 +3035,7 @@ def get_past_pokemon_abilities(self, obj: Pokemon) -> list[dict[str, Any]]:
30333035
past_abilities = (
30343036
PokemonAbilityPast.objects.filter(pokemon=obj, generation__isnull=False)
30353037
.select_related("generation", "ability")
3036-
.order_by("generation_id")
3038+
.order_by("generation_id", "slot", "pk")
30373039
)
30383040

30393041
final_data: list[dict[str, Any]] = []
@@ -3061,7 +3063,7 @@ def get_past_pokemon_stats(self, obj: Pokemon) -> list[dict[str, Any]]:
30613063
past_stats = (
30623064
PokemonStatPast.objects.filter(pokemon=obj, generation__isnull=False)
30633065
.select_related("generation", "stat")
3064-
.order_by("generation_id")
3066+
.order_by("generation_id", "stat_id", "pk")
30653067
)
30663068

30673069
final_data: list[dict[str, Any]] = []
@@ -3086,7 +3088,7 @@ def get_past_pokemon_stats(self, obj: Pokemon) -> list[dict[str, Any]]:
30863088

30873089
@extend_schema_field(PokemonTypeSerializer(many=True))
30883090
def get_pokemon_types(self, obj: Pokemon) -> ReturnList[ReturnDict[str, Any]]:
3089-
types = PokemonType.objects.filter(pokemon=obj).select_related("type").order_by("slot")
3091+
types = PokemonType.objects.filter(pokemon=obj).select_related("type").order_by("slot", "pk")
30903092
return cast(
30913093
"ReturnList[ReturnDict[str, Any]]",
30923094
PokemonTypeSerializer(types, many=True, context=self.context).data,
@@ -3097,7 +3099,7 @@ def get_past_pokemon_types(self, obj: Pokemon) -> list[dict[str, Any]]:
30973099
past_types = (
30983100
PokemonTypePast.objects.filter(pokemon=obj, generation__isnull=False)
30993101
.select_related("generation", "type")
3100-
.order_by("generation_id", "slot")
3102+
.order_by("generation_id", "slot", "pk")
31013103
)
31023104

31033105
final_data: list[dict[str, Any]] = []
@@ -3361,7 +3363,7 @@ class Meta:
33613363

33623364
@extend_schema_field(EvolutionChainLinkSerializer)
33633365
def build_chain(self, obj: EvolutionChain) -> dict[str, Any]:
3364-
pokemon_objects = PokemonSpecies.objects.filter(evolution_chain=obj).order_by("order")
3366+
pokemon_objects = PokemonSpecies.objects.filter(evolution_chain=obj).order_by("order", "pk")
33653367
summary_data = cast(
33663368
"ReturnList[ReturnDict[str, Any]]",
33673369
PokemonSpeciesSummarySerializer(pokemon_objects, many=True, context=self.context).data,
@@ -3535,7 +3537,9 @@ class Meta:
35353537
@extend_schema_field(PokemonDexNumberSerializer(many=True))
35363538
def get_pokedex_entries(self, obj: Pokedex) -> ReturnList[ReturnDict[str, Any]]:
35373539
entries = (
3538-
PokemonDexNumber.objects.filter(pokedex=obj).select_related("pokemon_species").order_by("pokedex_number")
3540+
PokemonDexNumber.objects.filter(pokedex=obj)
3541+
.select_related("pokemon_species")
3542+
.order_by("pokedex_number", "pokemon_species_id", "pk")
35393543
)
35403544
return cast(
35413545
"ReturnList[ReturnDict[str, Any]]",

‎pokemon_v2/tests.py‎

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1309,6 +1309,7 @@ def setup_pokemon_species_data(
13091309
is_legendary=False,
13101310
is_mythical=False,
13111311
order=1,
1312+
pk=None,
13121313
):
13131314
generation = generation or cls.setup_generation_data(name="gen for " + name)
13141315

@@ -1321,6 +1322,7 @@ def setup_pokemon_species_data(
13211322
pokemon_habitat = pokemon_habitat or cls.setup_pokemon_habitat_data(name="pkm hbtt for " + name)
13221323

13231324
pokemon_species = PokemonSpecies.objects.create(
1325+
pk=pk,
13241326
name=name,
13251327
generation=generation,
13261328
evolves_from_species=evolves_from_species,
@@ -1573,10 +1575,11 @@ def setup_pokemon_item_data(cls, pokemon=None, item=None, version=None, rarity=5
15731575
return pokemon_item
15741576

15751577
@classmethod
1576-
def setup_pokemon_move_data(cls, pokemon, move, version_group, level=0, order=1):
1577-
move_learn_method = cls.setup_move_learn_method_data(name="mv lrn mthd for pkmn")
1578+
def setup_pokemon_move_data(cls, pokemon, move, version_group, level=0, order=1, move_learn_method=None, pk=None):
1579+
move_learn_method = move_learn_method or cls.setup_move_learn_method_data(name="mv lrn mthd for pkmn")
15781580

15791581
pokemon_move = PokemonMove.objects.create(
1582+
pk=pk,
15801583
pokemon=pokemon,
15811584
version_group=version_group,
15821585
move=move,
@@ -2686,6 +2689,29 @@ def test_growth_rate_api(self):
26862689
"{}{}/pokemon-species/{}/".format(TEST_HOST, API_V2, pokemon_species.pk),
26872690
)
26882691

2692+
def test_reverse_relation_lists_are_ordered_by_pk(self):
2693+
# Lists coming straight from a reverse relation have no order_by() of
2694+
# their own, so they fall back to the manager ordering by pk. The pks
2695+
# are inserted out of order, since an unordered query returns them in
2696+
# insertion order and would pass either way.
2697+
growth_rate = self.setup_growth_rate_data(name="grth rt for ordering")
2698+
species = [
2699+
self.setup_pokemon_species_data(
2700+
pk=pk,
2701+
growth_rate=growth_rate,
2702+
name="pkmn spcs for ordering {}".format(pk),
2703+
)
2704+
for pk in (30, 10, 20)
2705+
]
2706+
2707+
response = self.client.get("{}/growth-rate/{}/".format(API_V2, growth_rate.pk))
2708+
2709+
self.assertEqual(response.status_code, status.HTTP_200_OK)
2710+
self.assertEqual(
2711+
[entry["name"] for entry in response.data["pokemon_species"]],
2712+
[entry.name for entry in sorted(species, key=lambda entry: entry.pk)],
2713+
)
2714+
26892715
# Location Tests
26902716
def test_location_api(self):
26912717
location = self.setup_location_data(name="base lctn")
@@ -4591,6 +4617,38 @@ def test_pokemon_moves_with_non_contiguous_version_group_and_method_ids(self):
45914617
"{}{}/move-learn-method/{}/".format(TEST_HOST, API_V2, pokemon_move.move_learn_method.pk),
45924618
)
45934619

4620+
def test_pokemon_moves_version_group_details_are_deterministically_ordered(self):
4621+
# A pokemon can learn the same move in the same version group more than
4622+
# once, so these rows tie on every other order_by() field and fall back
4623+
# to the pk. The pks are inserted out of order on purpose.
4624+
pokemon_species = self.setup_pokemon_species_data(name="pkmn spcs for mv ordering")
4625+
pokemon = self.setup_pokemon_data(pokemon_species=pokemon_species, name="pkmn for mv ordering")
4626+
self.setup_pokemon_sprites_data(pokemon=pokemon)
4627+
self.setup_pokemon_cries_data(pokemon, latest=True, legacy=True)
4628+
4629+
move = self.setup_move_data(name="mv for mv ordering")
4630+
version_group = self.setup_version_group_data(name="ver grp for mv ordering")
4631+
move_learn_method = self.setup_move_learn_method_data(name="mv lrn mthd for mv ordering")
4632+
4633+
for pk, order in ((30, 3), (10, 1), (20, 2)):
4634+
self.setup_pokemon_move_data(
4635+
pk=pk,
4636+
pokemon=pokemon,
4637+
move=move,
4638+
version_group=version_group,
4639+
move_learn_method=move_learn_method,
4640+
level=5,
4641+
order=order,
4642+
)
4643+
4644+
response = self.client.get("{}/pokemon/{}/".format(API_V2, pokemon.pk), headers={"host": "testserver"})
4645+
4646+
self.assertEqual(response.status_code, status.HTTP_200_OK)
4647+
self.assertEqual(
4648+
[detail["order"] for detail in response.data["moves"][0]["version_group_details"]],
4649+
[1, 2, 3],
4650+
)
4651+
45944652
def test_pokemon_form_api(self):
45954653
pokemon_species = self.setup_pokemon_species_data()
45964654
pokemon = self.setup_pokemon_data(pokemon_species=pokemon_species)

0 commit comments

Comments
 (0)