Skip to content

Commit 42d9fe1

Browse files
fix: make API list ordering deterministic
Most ORM queries had no explicit ORDER BY, so the database was free to return rows in any order. The same dataset could then be serialized differently on every build, turning each api-data regeneration into a huge diff of pure reordering noise. Give PokeApiModel a default manager that orders by primary key. Managers are inherited from abstract bases and reverse relation managers derive from the model's default manager, so this covers plain queries, reverse relations and prefetches at once, including the declarative reverse fields (Generation, GrowthRate, PokemonColor, PokemonShape, PokemonHabitat) that no serializer call site could order. Add tie-breakers to the partial orderings that left equal keys undefined, ending every order_by() with the primary key: pokemon moves and held items, location area and pokemon encounters, past abilities, stats and types, form and pokemon types, berry flavors, pokedex entries and encounter method rates. The few calls left untouched already order by order_by("id"), which is the primary key. Document in the manager that a distinct(*fields) call has to spell out a matching order_by(), since PostgreSQL requires the leading ORDER BY expressions to match the DISTINCT ON ones. A plain distinct() is fine, because the primary key is already part of the selected row. Add two regression tests that enforce the ordering, one for a declarative reverse relation and one for rows tying on the whole ORDER BY. Both insert rows with primary keys out of order, so both fail without this fix.
1 parent f0deda3 commit 42d9fe1

4 files changed

Lines changed: 101 additions & 19 deletions

File tree

‎pokemon_v2/api.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1021,7 +1021,7 @@ def get(self, request: Request, pokemon_id: int) -> Response:
10211021
"encounterconditionvaluemap_set__encounter_condition_value",
10221022
"encounterpokemondetail_set",
10231023
)
1024-
.order_by("location_area_id", "version_id", "encounter_slot_id")
1024+
.order_by("location_area_id", "version_id", "encounter_slot_id", "pk")
10251025
)
10261026

10271027
grouped_data: list[dict[str, Any]] = []

‎pokemon_v2/models.py‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
# pyright: reportIncompatibleVariableOverride=false
22
from __future__ import annotations
33

4-
from typing import Any
4+
from typing import Any, ClassVar, TypeVar
55

66
from django.db import models
7-
from typing_extensions import override
7+
from typing_extensions import Self, override
8+
9+
_ModelT = TypeVar("_ModelT", bound=models.Model)
810

911
__all__: tuple[str, ...] = (
1012
"Ability",
@@ -165,6 +167,7 @@
165167
"PalPark",
166168
"PalParkArea",
167169
"PalParkAreaName",
170+
"PokeApiManager",
168171
"PokeApiModel",
169172
"PokeathlonStat",
170173
"PokeathlonStatName",
@@ -230,7 +233,24 @@
230233
############################
231234

232235

236+
class PokeApiManager(models.Manager[_ModelT]):
237+
"""Default manager ordering every queryset by primary key, so that the API
238+
serializes lists deterministically. Call sites needing another order
239+
override it with ``order_by()``.
240+
241+
A ``distinct(*fields)`` call has to spell out a matching ``order_by()``,
242+
since PostgreSQL requires the leading ``ORDER BY`` expressions to match the
243+
``DISTINCT ON`` ones. A plain ``distinct()`` is unaffected.
244+
"""
245+
246+
@override
247+
def get_queryset(self) -> models.QuerySet[_ModelT]:
248+
return super().get_queryset().order_by("pk")
249+
250+
233251
class PokeApiModel(models.Model):
252+
objects: ClassVar[PokeApiManager[Self]] = PokeApiManager()
253+
234254
class Meta:
235255
abstract = True
236256

‎pokemon_v2/serializers.py‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1053,7 +1053,7 @@ def get_method_rates(self, obj: LocationAreaEncounterRate) -> ReturnList[ReturnD
10531053
rates = (
10541054
LocationAreaEncounterRate.objects.filter(location_area=obj, encounter_method__isnull=False)
10551055
.select_related("encounter_method", "version")
1056-
.order_by("encounter_method_id")
1056+
.order_by("encounter_method_id", "version_id", "pk")
10571057
)
10581058
grouped_rates: list[dict[str, Any]] = [
10591059
{
@@ -1081,7 +1081,7 @@ def get_encounters(self, obj: LocationArea) -> ReturnList[ReturnDict[str, Any]]:
10811081
"encounterconditionvaluemap_set",
10821082
"encounterpokemondetail_set",
10831083
)
1084-
.order_by("pokemon_id", "version_id")
1084+
.order_by("pokemon_id", "version_id", "encounter_slot_id", "pk")
10851085
)
10861086

10871087
grouped_data: list[dict[str, Any]] = []
@@ -1597,7 +1597,7 @@ def get_held_by_pokemon(self, obj: Item) -> ReturnList[ReturnDict[str, Any]]:
15971597
pokemon_items = (
15981598
PokemonItem.objects.filter(item=obj)
15991599
.select_related("pokemon", "version")
1600-
.order_by("pokemon_id", "version_id")
1600+
.order_by("pokemon_id", "version_id", "pk")
16011601
)
16021602
grouped_data: list[dict[str, Any]] = [
16031603
{
@@ -1739,7 +1739,9 @@ class Meta:
17391739
@extend_schema_field(BerryFlavorBerryMapSerializer(many=True))
17401740
def get_berries_with_flavor(self, obj: BerryFlavor) -> ReturnList[ReturnDict[str, Any]]:
17411741
flavor_map_objects = (
1742-
BerryFlavorMap.objects.filter(berry_flavor=obj, potency__gt=0).select_related("berry").order_by("potency")
1742+
BerryFlavorMap.objects.filter(berry_flavor=obj, potency__gt=0)
1743+
.select_related("berry")
1744+
.order_by("potency", "berry_id", "pk")
17431745
)
17441746
return cast(
17451747
"ReturnList[ReturnDict[str, Any]]",
@@ -2629,7 +2631,7 @@ def get_pokemon_form_sprites(self, obj: PokemonForm) -> dict[str, Any]:
26292631

26302632
@extend_schema_field(PokemonFormTypeSerializer(many=True))
26312633
def get_pokemon_form_types(self, obj: PokemonForm) -> ReturnList[ReturnDict[str, Any]]:
2632-
form_types = PokemonFormType.objects.filter(pokemon_form=obj).select_related("type").order_by("slot")
2634+
form_types = PokemonFormType.objects.filter(pokemon_form=obj).select_related("type").order_by("slot", "pk")
26332635

26342636
if form_types:
26352637
return cast(
@@ -2638,7 +2640,7 @@ def get_pokemon_form_types(self, obj: PokemonForm) -> ReturnList[ReturnDict[str,
26382640
)
26392641

26402642
# Fall back to parent Pokemon's types if no form-specific types exist
2641-
pokemon_types = PokemonType.objects.filter(pokemon=obj.pokemon).select_related("type").order_by("slot")
2643+
pokemon_types = PokemonType.objects.filter(pokemon=obj.pokemon).select_related("type").order_by("slot", "pk")
26422644
return cast(
26432645
"ReturnList[ReturnDict[str, Any]]",
26442646
PokemonTypeSerializer(pokemon_types, many=True, context=self.context).data,
@@ -2946,7 +2948,7 @@ def get_pokemon_moves(self, obj: Pokemon) -> list[dict[str, Any]]:
29462948
pokemon_moves = (
29472949
PokemonMove.objects.filter(pokemon=obj, move__isnull=False)
29482950
.select_related("move", "version_group", "move_learn_method")
2949-
.order_by("move__id", "version_group_id")
2951+
.order_by("move__id", "version_group_id", "move_learn_method_id", "level", "pk")
29502952
)
29512953

29522954
vg_cache: dict[int, Any] = {}
@@ -2990,7 +2992,7 @@ def get_pokemon_held_items(self, obj: Pokemon) -> list[dict[str, Any]]:
29902992
pokemon_items = (
29912993
PokemonItem.objects.filter(pokemon=obj, item__isnull=False)
29922994
.select_related("item", "version")
2993-
.order_by("item__id", "version_id")
2995+
.order_by("item__id", "version_id", "pk")
29942996
)
29952997

29962998
version_cache: dict[int, Any] = {}
@@ -3033,7 +3035,7 @@ def get_past_pokemon_abilities(self, obj: Pokemon) -> list[dict[str, Any]]:
30333035
past_abilities = (
30343036
PokemonAbilityPast.objects.filter(pokemon=obj, generation__isnull=False)
30353037
.select_related("generation", "ability")
3036-
.order_by("generation_id")
3038+
.order_by("generation_id", "slot", "pk")
30373039
)
30383040

30393041
final_data: list[dict[str, Any]] = []
@@ -3061,7 +3063,7 @@ def get_past_pokemon_stats(self, obj: Pokemon) -> list[dict[str, Any]]:
30613063
past_stats = (
30623064
PokemonStatPast.objects.filter(pokemon=obj, generation__isnull=False)
30633065
.select_related("generation", "stat")
3064-
.order_by("generation_id")
3066+
.order_by("generation_id", "stat_id", "pk")
30653067
)
30663068

30673069
final_data: list[dict[str, Any]] = []
@@ -3086,7 +3088,7 @@ def get_past_pokemon_stats(self, obj: Pokemon) -> list[dict[str, Any]]:
30863088

30873089
@extend_schema_field(PokemonTypeSerializer(many=True))
30883090
def get_pokemon_types(self, obj: Pokemon) -> ReturnList[ReturnDict[str, Any]]:
3089-
types = PokemonType.objects.filter(pokemon=obj).select_related("type").order_by("slot")
3091+
types = PokemonType.objects.filter(pokemon=obj).select_related("type").order_by("slot", "pk")
30903092
return cast(
30913093
"ReturnList[ReturnDict[str, Any]]",
30923094
PokemonTypeSerializer(types, many=True, context=self.context).data,
@@ -3097,7 +3099,7 @@ def get_past_pokemon_types(self, obj: Pokemon) -> list[dict[str, Any]]:
30973099
past_types = (
30983100
PokemonTypePast.objects.filter(pokemon=obj, generation__isnull=False)
30993101
.select_related("generation", "type")
3100-
.order_by("generation_id", "slot")
3102+
.order_by("generation_id", "slot", "pk")
31013103
)
31023104

31033105
final_data: list[dict[str, Any]] = []
@@ -3361,7 +3363,7 @@ class Meta:
33613363

33623364
@extend_schema_field(EvolutionChainLinkSerializer)
33633365
def build_chain(self, obj: EvolutionChain) -> dict[str, Any]:
3364-
pokemon_objects = PokemonSpecies.objects.filter(evolution_chain=obj).order_by("order")
3366+
pokemon_objects = PokemonSpecies.objects.filter(evolution_chain=obj).order_by("order", "pk")
33653367
summary_data = cast(
33663368
"ReturnList[ReturnDict[str, Any]]",
33673369
PokemonSpeciesSummarySerializer(pokemon_objects, many=True, context=self.context).data,
@@ -3535,7 +3537,9 @@ class Meta:
35353537
@extend_schema_field(PokemonDexNumberSerializer(many=True))
35363538
def get_pokedex_entries(self, obj: Pokedex) -> ReturnList[ReturnDict[str, Any]]:
35373539
entries = (
3538-
PokemonDexNumber.objects.filter(pokedex=obj).select_related("pokemon_species").order_by("pokedex_number")
3540+
PokemonDexNumber.objects.filter(pokedex=obj)
3541+
.select_related("pokemon_species")
3542+
.order_by("pokedex_number", "pokemon_species_id", "pk")
35393543
)
35403544
return cast(
35413545
"ReturnList[ReturnDict[str, Any]]",

‎pokemon_v2/tests.py‎

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1309,6 +1309,7 @@ def setup_pokemon_species_data(
13091309
is_legendary=False,
13101310
is_mythical=False,
13111311
order=1,
1312+
pk=None,
13121313
):
13131314
generation = generation or cls.setup_generation_data(name="gen for " + name)
13141315

@@ -1321,6 +1322,7 @@ def setup_pokemon_species_data(
13211322
pokemon_habitat = pokemon_habitat or cls.setup_pokemon_habitat_data(name="pkm hbtt for " + name)
13221323

13231324
pokemon_species = PokemonSpecies.objects.create(
1325+
pk=pk,
13241326
name=name,
13251327
generation=generation,
13261328
evolves_from_species=evolves_from_species,
@@ -1573,10 +1575,11 @@ def setup_pokemon_item_data(cls, pokemon=None, item=None, version=None, rarity=5
15731575
return pokemon_item
15741576

15751577
@classmethod
1576-
def setup_pokemon_move_data(cls, pokemon, move, version_group, level=0, order=1):
1577-
move_learn_method = cls.setup_move_learn_method_data(name="mv lrn mthd for pkmn")
1578+
def setup_pokemon_move_data(cls, pokemon, move, version_group, level=0, order=1, move_learn_method=None, pk=None):
1579+
move_learn_method = move_learn_method or cls.setup_move_learn_method_data(name="mv lrn mthd for pkmn")
15781580

15791581
pokemon_move = PokemonMove.objects.create(
1582+
pk=pk,
15801583
pokemon=pokemon,
15811584
version_group=version_group,
15821585
move=move,
@@ -2686,6 +2689,29 @@ def test_growth_rate_api(self):
26862689
"{}{}/pokemon-species/{}/".format(TEST_HOST, API_V2, pokemon_species.pk),
26872690
)
26882691

2692+
def test_reverse_relation_lists_are_ordered_by_pk(self):
2693+
# Lists coming straight from a reverse relation have no order_by() of
2694+
# their own, so they fall back to the manager ordering by pk. The pks
2695+
# are inserted out of order, since an unordered query returns them in
2696+
# insertion order and would pass either way.
2697+
growth_rate = self.setup_growth_rate_data(name="grth rt for ordering")
2698+
species = [
2699+
self.setup_pokemon_species_data(
2700+
pk=pk,
2701+
growth_rate=growth_rate,
2702+
name="pkmn spcs for ordering {}".format(pk),
2703+
)
2704+
for pk in (30, 10, 20)
2705+
]
2706+
2707+
response = self.client.get("{}/growth-rate/{}/".format(API_V2, growth_rate.pk))
2708+
2709+
self.assertEqual(response.status_code, status.HTTP_200_OK)
2710+
self.assertEqual(
2711+
[entry["name"] for entry in response.data["pokemon_species"]],
2712+
[entry.name for entry in sorted(species, key=lambda entry: entry.pk)],
2713+
)
2714+
26892715
# Location Tests
26902716
def test_location_api(self):
26912717
location = self.setup_location_data(name="base lctn")
@@ -4591,6 +4617,38 @@ def test_pokemon_moves_with_non_contiguous_version_group_and_method_ids(self):
45914617
"{}{}/move-learn-method/{}/".format(TEST_HOST, API_V2, pokemon_move.move_learn_method.pk),
45924618
)
45934619

4620+
def test_pokemon_moves_version_group_details_are_deterministically_ordered(self):
4621+
# A pokemon can learn the same move in the same version group more than
4622+
# once, so these rows tie on every other order_by() field and fall back
4623+
# to the pk. The pks are inserted out of order on purpose.
4624+
pokemon_species = self.setup_pokemon_species_data(name="pkmn spcs for mv ordering")
4625+
pokemon = self.setup_pokemon_data(pokemon_species=pokemon_species, name="pkmn for mv ordering")
4626+
self.setup_pokemon_sprites_data(pokemon=pokemon)
4627+
self.setup_pokemon_cries_data(pokemon, latest=True, legacy=True)
4628+
4629+
move = self.setup_move_data(name="mv for mv ordering")
4630+
version_group = self.setup_version_group_data(name="ver grp for mv ordering")
4631+
move_learn_method = self.setup_move_learn_method_data(name="mv lrn mthd for mv ordering")
4632+
4633+
for pk, order in ((30, 3), (10, 1), (20, 2)):
4634+
self.setup_pokemon_move_data(
4635+
pk=pk,
4636+
pokemon=pokemon,
4637+
move=move,
4638+
version_group=version_group,
4639+
move_learn_method=move_learn_method,
4640+
level=5,
4641+
order=order,
4642+
)
4643+
4644+
response = self.client.get("{}/pokemon/{}/".format(API_V2, pokemon.pk), headers={"host": "testserver"})
4645+
4646+
self.assertEqual(response.status_code, status.HTTP_200_OK)
4647+
self.assertEqual(
4648+
[detail["order"] for detail in response.data["moves"][0]["version_group_details"]],
4649+
[1, 2, 3],
4650+
)
4651+
45944652
def test_pokemon_form_api(self):
45954653
pokemon_species = self.setup_pokemon_species_data()
45964654
pokemon = self.setup_pokemon_data(pokemon_species=pokemon_species)

0 commit comments

Comments
 (0)