From b3cdca37569e27cbebcdc1d2ea91f06e3f3e098e Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 19:16:42 +0000 Subject: [PATCH 01/17] fix: read paralysable and unknown creature values --- CHANGELOG.md | 12 ++ tests/parsers/test_creature.py | 167 +++++++++++++++++- .../content_creature_bonelord_totem.txt | 52 ++++++ tests/resources/content_creature_dragon.txt | 86 +++++++++ tests/tests_utils.py | 8 +- tibiawikisql/parsers/creature.py | 119 ++++++++++--- tibiawikisql/utils.py | 35 ++-- 7 files changed, 444 insertions(+), 35 deletions(-) create mode 100644 tests/resources/content_creature_bonelord_totem.txt create mode 100644 tests/resources/content_creature_dragon.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index addcb59..62d99da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ # Changelog +## Unreleased + +- Read creature `paralysable` from `paraimmune` the right way round, and store an unknown creature value as `NULL` + instead of a real-looking `0` or `false`. `paraimmune = yes` (immune) is `false` and `no` is `true`; before, an + immune creature like Dragon read `true`, a regression from upstream `1a6dabf`. `hitpoints`, `experience`, `armor`, + `mitigation`, `speed`, `runs_at` and every `modifier_*` keep a number, including 0, and are `NULL` for `?`, an empty + value, `--` or a colour word like `deep red` in `runsat`. `illusionable`, `pushable`, `push_objects`, + `sees_invisible` and `paralysable` are `NULL` for anything but `yes` or `no`, such as `?` or `no?`. `summon_cost` and + `convince_cost` keep 0 for `--` or `no`, which the wiki shows as "cannot", and are `NULL` for `?`, an empty value or + `yes`. Wiki comments are not read as values, so an `exp` found only inside a comment, as on Bonelord Totem, is `NULL` + instead of the comment's number. An absent value was already `NULL`, and `is_boss` is unchanged. + ## 9.0.0+tibiash.9 - Send the `tibiawikisql/ (+https://github.com/tibia-sh/tibiawiki-sql)` User-Agent with every TibiaWiki API diff --git a/tests/parsers/test_creature.py b/tests/parsers/test_creature.py index 67304a4..923f3c9 100644 --- a/tests/parsers/test_creature.py +++ b/tests/parsers/test_creature.py @@ -5,7 +5,7 @@ from tests import load_resource from tibiawikisql.api import Article from tibiawikisql.models import Creature -from tibiawikisql.parsers.creature import CreatureParser, parse_abilities, parse_maximum_damage +from tibiawikisql.parsers.creature import CreatureParser, parse_abilities, parse_mana_cost, parse_maximum_damage class TestCreatureParser(unittest.TestCase): @@ -170,3 +170,168 @@ def test_creature_race_id_ignores_a_trailing_comment(self): creature = self._parse(self._edited("| race_id = 35", "| race_id = 35 ")) self.assertEqual(35, creature.race_id) + + +class TestCreatureParserKnownValues(unittest.TestCase): + def _parse(self, content: str) -> Creature: + article = Article( + article_id=1, + title="Dragon", + timestamp=datetime.datetime.fromisoformat("2026-06-25T06:40:57+00:00"), + content=content, + ) + + return CreatureParser.from_article(article) + + def _edited(self, old: str, new: str) -> Creature: + content = load_resource("content_creature_dragon.txt") + self.assertIn(old, content) + return self._parse(content.replace(old, new)) + + def test_dragon(self): + creature = self._parse(load_resource("content_creature_dragon.txt")) + + self.assertIs(False, creature.paralysable) + self.assertEqual(0, creature.summon_cost) + self.assertEqual(0, creature.convince_cost) + self.assertEqual(1000, creature.hitpoints) + self.assertEqual(700, creature.experience) + self.assertEqual(25, creature.armor) + self.assertEqual(1.56, creature.mitigation) + self.assertEqual(86, creature.speed) + self.assertEqual(300, creature.runs_at) + self.assertIs(True, creature.illusionable) + self.assertIs(False, creature.pushable) + self.assertIs(True, creature.push_objects) + self.assertIs(True, creature.sees_invisible) + self.assertIs(False, creature.is_boss) + self.assertEqual(0, creature.modifier_fire) + self.assertEqual(20, creature.modifier_earth) + + def test_paralysable_when_not_immune(self): + creature = self._edited("| paraimmune = yes", "| paraimmune = no") + + self.assertIs(True, creature.paralysable) + + def test_paralysable_unknown(self): + creature = self._edited("| paraimmune = yes", "| paraimmune = ?") + + self.assertIsNone(creature.paralysable) + + def test_paralysable_absent(self): + creature = self._edited("| paraimmune = yes\n", "") + + self.assertIsNone(creature.paralysable) + + def test_armor_unknown(self): + creature = self._edited("| armor = 25", "| armor = ?") + + self.assertIsNone(creature.armor) + + def test_armor_zero(self): + creature = self._edited("| armor = 25", "| armor = 0") + + self.assertEqual(0, creature.armor) + + def test_stats_unknown(self): + creature = self._parse( + load_resource("content_creature_dragon.txt") + .replace("| hp = 1000", "| hp = ?") + .replace("| exp = 700", "| exp =") + .replace("| mitigation = 1.56", "| mitigation = ?") + .replace("| speed = 86", "| speed = --"), + ) + + self.assertIsNone(creature.hitpoints) + self.assertIsNone(creature.experience) + self.assertIsNone(creature.mitigation) + self.assertIsNone(creature.speed) + + def test_runs_at_colour(self): + creature = self._edited("| runsat = 300", "| runsat = deep red") + + self.assertIsNone(creature.runs_at) + + def test_runs_at_zero(self): + creature = self._edited("| runsat = 300", "| runsat = 0") + + self.assertEqual(0, creature.runs_at) + + def test_summon_unknown(self): + creature = self._edited("| summon = --", "| summon = ?") + + self.assertIsNone(creature.summon_cost) + + def test_convince_yes(self): + creature = self._edited("| convince = --", "| convince = yes") + + self.assertIsNone(creature.convince_cost) + + def test_summon_number(self): + creature = self._edited("| summon = --", "| summon = 1,000") + + self.assertEqual(1000, creature.summon_cost) + + def test_parse_mana_cost(self): + self.assertEqual(0, parse_mana_cost("--")) + self.assertEqual(0, parse_mana_cost(" No ")) + self.assertEqual(0, parse_mana_cost("0")) + self.assertEqual(390, parse_mana_cost("390")) + self.assertEqual(390, parse_mana_cost("390 ")) + self.assertEqual(0, parse_mana_cost("-- ")) + self.assertIsNone(parse_mana_cost("?")) + self.assertIsNone(parse_mana_cost("")) + self.assertIsNone(parse_mana_cost("yes")) + self.assertIsNone(parse_mana_cost("")) + + def test_push_objects_uncertain(self): + creature = self._edited("| pushobjects = Yes", "| pushobjects = no?") + + self.assertIsNone(creature.push_objects) + + def test_flags_unknown(self): + creature = self._parse( + load_resource("content_creature_dragon.txt") + .replace("| illusionable = yes", "| illusionable = ?") + .replace("| pushable = No", "| pushable =") + .replace("| senseinvis = yes\n", ""), + ) + + self.assertIsNone(creature.illusionable) + self.assertIsNone(creature.pushable) + self.assertIsNone(creature.sees_invisible) + + def test_experience_only_in_a_comment(self): + article = Article( + article_id=109773, + title="Bonelord Totem", + timestamp=datetime.datetime.fromisoformat("2026-07-13T17:46:07+00:00"), + content=load_resource("content_creature_bonelord_totem.txt"), + ) + + creature = CreatureParser.from_article(article) + + self.assertIsNone(creature.experience) + self.assertIsNone(creature.hitpoints) + self.assertEqual(0, creature.runs_at) + self.assertEqual(0, creature.modifier_fire) + + def test_stat_ignores_a_trailing_comment(self): + creature = self._edited("| hp = 1000", "| hp = 1000 ") + + self.assertEqual(1000, creature.hitpoints) + + def test_stat_hidden_by_an_unclosed_comment(self): + creature = self._edited("| hp = 1000", "| hp = +| armor = +| mitigation = +| summon = -- +| convince = -- +| illusionable = no +| creatureclass = +| primarytype = +| secondarytype = +| attacktype = +| usespells = +| spawntype = Triggered +| isboss = no +| abilities = +| usedelements = +| maxdmg = +| pushable = no +| pushobjects = no +| walksaround = -- +| walksthrough = -- +| paraimmune = yes +| senseinvis = yes +| physicalDmgMod = 0% +| earthDmgMod = 0% +| fireDmgMod = 0% +| deathDmgMod = 0% +| energyDmgMod = 0% +| holyDmgMod = 0% +| iceDmgMod = 0% +| healMod = 0% +| hpDrainDmgMod = 0% +| drownDmgMod = 0% +| sounds = {{Sound List|}} +| implemented = 15.20.7a5cc9 +| race_id = +| notes = Part of [[The Roost of the Graveborn Quest]], during the fight against [[The Gravedigger]]. +| behaviour = +| runsat = 0 +| speed = +| strategy = +| location = Boss room of [[The Gravedigger]]. +| loot = {{Loot Table|}} +| history = +| status = Active +}} \ No newline at end of file diff --git a/tests/resources/content_creature_dragon.txt b/tests/resources/content_creature_dragon.txt new file mode 100644 index 0000000..9a807c2 --- /dev/null +++ b/tests/resources/content_creature_dragon.txt @@ -0,0 +1,86 @@ +{{Infobox Creature|List={{{1|}}}|GetValue={{{GetValue|}}} +| name = Dragon +| article = a +| actualname = dragon +| plural = dragons +| hp = 1000 +| exp = 700 +| armor = 25 +| mitigation = 1.56 +| summon = -- +| convince = -- +| illusionable = yes +| creatureclass = Reptiles +| primarytype = Dragons +| bestiaryclass = Dragon +| bestiarylevel = Medium +| occurrence = Common +| isboss = no +| isarenaboss = no +| abilities = {{Ability List + |{{Melee|0-120}} + |{{Ability|Fire Wave|100-170|element=fire + |scene={{Scene|spell=8sqmwave|effect=Fireball Effect|look_direction=east}}}} + |{{Ability|Great Fireball|60-140|element=fire + |scene={{Scene|spell=2sqmballtarget|effect=Fireball Effect|missile=Fire Missile|missile_direction=south-east|missile_distance=2/2}}}} + |{{Healing|range=40-70 + |scene={{Scene|spell=buffspell|effect=Blue Sparkles Effect|look_direction=south|effect_on_caster=yes}}}} +}} +| maxdmg = {{Max Damage|physical=120|fire=310}} +| pushable = No +| pushobjects = Yes +| walksaround = -- +| walksthrough = Fire, Energy, Poison +| paraimmune = yes +| senseinvis = yes +| physicalDmgMod = 100% +| earthDmgMod = 20% +| fireDmgMod = 0% +| deathDmgMod = 100% +| energyDmgMod = 80% +| holyDmgMod = 100% +| iceDmgMod = 110% +| hpDrainDmgMod = 100% +| drownDmgMod = 100% +| healMod = 100% +| bestiaryname = dragon +| bestiarytext = Dragons were among the first creatures of Tibia and once ruled the whole continent. Nowadays, there are only a few of them left which live deep in the dungeons. Nevertheless, they are very powerful monsters and will strive for killing every intruder. Besides their immense strength, they shoot fireballs at their victims and spit fire. Moreover, they can heal themselves. +| sounds = {{Sound List|FCHHHHH|GROOAAARRR}} +| implemented = 4.0 +| race_id = 34 +| notes = These creatures can be skinned with an [[Obsidian Knife]] (see also: [[Green Dragon Leather/Skinning]]). +| behaviour = Dragons are known to [[Retargeting|retarget]]. +| runsat = 300 +| speed = 86 +| strategy = +| location = [[Thais]], [[Ancient Temple]], [[Darashia Dragon Lair]], [[Mount Sternum Dragon Cave]], [[Mintwallin]], deep in [[Fibula Dungeon]], [[Kazordoon Dragon Lair]] (near [[Dwarf Bridge]]), [[Plains of Havoc]], [[Elven Bane]] castle, [[Maze of Lost Souls]], southern cave and dragon tower in [[Shadowthorn]], [[Orc Fortress]], [[Venore Dragon Lair]], [[Pits of Inferno]], [[Behemoth Quest]] room in [[Edron]], [[Hero Cave]], deep [[Cyclopolis]], [[Edron Dragon Lair]], [[Goroma]], [[Kha'zeel_Dragon_Lairs/Kha'labal|Kha'labal Dragon Lairs]], [[Kha'zeel_Dragon_Lairs/Tiquanda|Tiquanda Dragon Lairs]], [[Draconia]], [[Dragonblaze Peaks]], some [[Ankrahmun Tombs]], underground of [[Fenrock]] (on the way to [[Beregar]]), [[Krailos Steppe]] and [[Crystal Lakes]]. +| loot = {{Loot Table + |{{Loot Item|1-105|Gold Coin|common}} + |{{Loot Item|1-3|Dragon Ham|common}} + |{{Loot Item|Steel Shield|uncommon}} + |{{Loot Item|Crossbow|uncommon}} + |{{Loot Item|Dragon's Tail|uncommon}} + |{{Loot Item|1-10|Burst Arrow|uncommon}} + |{{Loot Item|Longsword|semi-rare}} + |{{Loot Item|Steel Helmet|semi-rare}} + |{{Loot Item|Broadsword|semi-rare}} + |{{Loot Item|Plate Legs|semi-rare}} + |{{Loot Item|Green Dragon Leather|rare}} + |{{Loot Item|Wand of Inferno|rare}} + |{{Loot Item|Strong Health Potion|rare}} + |{{Loot Item|Green Dragon Scale|rare}} + |{{Loot Item|Double Axe|rare}} + |{{Loot Item|Dragon Hammer|rare}} + |{{Loot Item|Serpent Sword|rare}} + |{{Loot Item|Small Diamond|very rare}} + |{{Loot Item|Dragon Shield|very rare}} + |{{Loot Item|Life Crystal|very rare}} + |{{Loot Item|Dragonbone Staff|very rare}} + |{{Loot Item|Dragon Trophy|very rare}} +}} +| history = * Dragons are one of the oldest creatures in the game. +* In older times (prior to 2001), Dragons were [[Summon Creature|summonable]], and during these times it was possible to summon up to 8 creatures at once (requiring very high [[mana]]). Only the highest leveled [[mage]]s could summon them (back when the highest [[level]]s were only around level 60 or 70). It was a somewhat common occurrence to see mages walking the streets of [[Thais]] with several dragons summoned at one time. It was also possible to set summons free by logging out of the game, turning the summons into wild creatures. Often mages would leave the game after summoning as many as 8 dragons in the middle of major [[Hometowns|cities]], causing chaos. +| usespells = yes +| attacktype = Melee +| spawntype = Regular +}} \ No newline at end of file diff --git a/tests/tests_utils.py b/tests/tests_utils.py index b53ba6b..bd5f3a6 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -8,7 +8,7 @@ from tibiawikisql.utils import (clean_links, client_color_to_rgb, parse_boolean, parse_client_id, parse_currency, parse_first_integer, parse_float, parse_integer, parse_date, parse_loot_statistics, parse_min_max, parse_sounds, - parse_weapon_proficiency_name, parse_weapon_proficiency_tables) + parse_weapon_proficiency_name, parse_weapon_proficiency_tables, visible_text) class TestUtils(unittest.TestCase): @@ -78,6 +78,12 @@ def test_parse_first_integer(self): self.assertEqual(0, parse_first_integer("--")) self.assertIsNone(parse_first_integer("--", None)) + def test_visible_text(self): + self.assertEqual("35 7", visible_text("35 7")) + self.assertEqual("\n \n", visible_text("\n\n")) + self.assertEqual("35 ", visible_text("35 ")) diff --git a/tibiawikisql/parsers/creature.py b/tibiawikisql/parsers/creature.py index 457e424..01dbfef 100644 --- a/tibiawikisql/parsers/creature.py +++ b/tibiawikisql/parsers/creature.py @@ -25,6 +25,7 @@ parse_min_max, parse_sounds, strip_code, + visible_text, ) if TYPE_CHECKING: @@ -79,6 +80,78 @@ def parse_maximum_integer(value: str) -> int | None: return max((parse_integer(match) for match in matches), default=None) +def parse_known_integer(value: str) -> int | None: + """Parse a creature number, such as its hitpoints, from the visible text of its value. + + Args: + value: The raw value of the field. + + Returns: + The first number outside wiki comments, including 0, or ``None`` if there is none, as for ``?``, ``--`` or a + colour word. + + """ + return parse_integer(visible_text(value), None) + + +def parse_known_float(value: str) -> float | None: + """Parse a creature decimal number, such as its mitigation, from the visible text of its value. + + Args: + value: The raw value of the field. + + Returns: + The first number outside wiki comments, including 0, or ``None`` if there is none. + + """ + return parse_float(visible_text(value), None) + + +def parse_mana_cost(value: str) -> int | None: + """Parse the mana needed to summon or convince a creature. + + The template renders ``--`` as "cannot", so ``--`` and ``no`` are a cost of 0. + + Args: + value: The raw value of the ``summon`` or ``convince`` field. + + Returns: + The first number outside wiki comments, 0 for ``--`` or ``no``, or ``None`` for anything else, such as ``?`` + or ``yes``. + + """ + text = visible_text(value) + if text.strip().lower() in {"--", "no"}: + return 0 + return parse_integer(text, None) + + +def parse_known_boolean(value: str) -> bool | None: + """Parse a creature flag, such as whether it is pushable. + + Args: + value: The raw value of the field. + + Returns: + ``True`` for ``yes``, ``False`` for ``no``, or ``None`` for anything else, such as ``?`` or ``no?``. + + """ + return parse_boolean(value, None) + + +def parse_paralysable(value: str) -> bool | None: + """Parse whether a creature can be paralysed from its ``paraimmune`` field, which says whether it is immune. + + Args: + value: The raw value of the ``paraimmune`` field. + + Returns: + ``False`` for ``yes``, ``True`` for ``no``, or ``None`` for anything else, such as ``?``. + + """ + return parse_boolean(value, None, invert=True) + + def parse_loot(value: str) -> list[tuple[str, str]]: """Get every item drop entry of a creature's drops. @@ -220,32 +293,32 @@ class CreatureParser(BaseParser): "bestiary_level": AttributeParser.optional("bestiarylevel"), "bestiary_occurrence": AttributeParser.optional("occurrence"), "bosstiary_class": AttributeParser.optional("bosstiaryclass"), - "hitpoints": AttributeParser.optional("hp", parse_integer), - "experience": AttributeParser.optional("exp", parse_integer), - "armor": AttributeParser.optional("armor", parse_integer), - "mitigation": AttributeParser.optional("mitigation", parse_float), - "speed": AttributeParser.optional("speed", parse_integer), - "runs_at": AttributeParser.optional("runsat", parse_integer), - "summon_cost": AttributeParser.optional("summon", parse_integer), - "convince_cost": AttributeParser.optional("convince", parse_integer), - "illusionable": AttributeParser.optional("illusionable", parse_boolean), - "pushable": AttributeParser.optional("pushable", parse_boolean), - "push_objects": AttributeParser.optional("pushobjects", parse_boolean), - "sees_invisible": AttributeParser.optional("senseinvis", parse_boolean), - "paralysable": AttributeParser.optional("paraimmune", parse_boolean), + "hitpoints": AttributeParser.optional("hp", parse_known_integer), + "experience": AttributeParser.optional("exp", parse_known_integer), + "armor": AttributeParser.optional("armor", parse_known_integer), + "mitigation": AttributeParser.optional("mitigation", parse_known_float), + "speed": AttributeParser.optional("speed", parse_known_integer), + "runs_at": AttributeParser.optional("runsat", parse_known_integer), + "summon_cost": AttributeParser.optional("summon", parse_mana_cost), + "convince_cost": AttributeParser.optional("convince", parse_mana_cost), + "illusionable": AttributeParser.optional("illusionable", parse_known_boolean), + "pushable": AttributeParser.optional("pushable", parse_known_boolean), + "push_objects": AttributeParser.optional("pushobjects", parse_known_boolean), + "sees_invisible": AttributeParser.optional("senseinvis", parse_known_boolean), + "paralysable": AttributeParser.optional("paraimmune", parse_paralysable), "spawn_type": AttributeParser.optional("spawntype"), "is_boss": AttributeParser.optional("isboss", parse_boolean, False), "cooldown": AttributeParser.optional("cooldown", parse_float), - "modifier_physical": AttributeParser.optional("physicalDmgMod", parse_integer), - "modifier_earth": AttributeParser.optional("earthDmgMod", parse_integer), - "modifier_fire": AttributeParser.optional("fireDmgMod", parse_integer), - "modifier_energy": AttributeParser.optional("energyDmgMod", parse_integer), - "modifier_ice": AttributeParser.optional("iceDmgMod", parse_integer), - "modifier_death": AttributeParser.optional("deathDmgMod", parse_integer), - "modifier_holy": AttributeParser.optional("holyDmgMod", parse_integer), - "modifier_drown": AttributeParser.optional("drownDmgMod", parse_integer), - "modifier_lifedrain": AttributeParser.optional("hpDrainDmgMod", parse_integer), - "modifier_healing": AttributeParser.optional("healMod", parse_integer), + "modifier_physical": AttributeParser.optional("physicalDmgMod", parse_known_integer), + "modifier_earth": AttributeParser.optional("earthDmgMod", parse_known_integer), + "modifier_fire": AttributeParser.optional("fireDmgMod", parse_known_integer), + "modifier_energy": AttributeParser.optional("energyDmgMod", parse_known_integer), + "modifier_ice": AttributeParser.optional("iceDmgMod", parse_known_integer), + "modifier_death": AttributeParser.optional("deathDmgMod", parse_known_integer), + "modifier_holy": AttributeParser.optional("holyDmgMod", parse_known_integer), + "modifier_drown": AttributeParser.optional("drownDmgMod", parse_known_integer), + "modifier_lifedrain": AttributeParser.optional("hpDrainDmgMod", parse_known_integer), + "modifier_healing": AttributeParser.optional("healMod", parse_known_integer), "walks_through": AttributeParser.optional("walksthrough", parse_monster_walks), "walks_around": AttributeParser.optional("walksaround", parse_monster_walks), "location": AttributeParser.optional("location", clean_links), diff --git a/tibiawikisql/utils.py b/tibiawikisql/utils.py index 6e79649..d2db2cf 100644 --- a/tibiawikisql/utils.py +++ b/tibiawikisql/utils.py @@ -212,7 +212,7 @@ def find_templates(content: str, template_name: str, partial: bool = False, recu yield template -def parse_boolean(value: str, default: bool = False, invert: bool = False) -> bool: +def parse_boolean(value: str, default: bool | None = False, invert: bool = False) -> bool | None: """Parse a boolean value from a string. String must contain "yes" to be considered True. @@ -271,7 +271,7 @@ def parse_date(value: str) -> datetime.date: raise ValueError(msg) -def parse_float(value: str, default: float = 0.0) -> float: +def parse_float(value: str, default: float | None = 0.0) -> float | None: """From a string, parses a floating value. Args: @@ -343,20 +343,19 @@ def parse_whole_number(value: str) -> int | None: return None -def parse_client_id(value: str) -> int | None: - """Parse a client ID, ignoring wiki comments. +def visible_text(value: str) -> str: + """Remove the wiki comments from a value, leaving the text a reader of the page sees. Each comment is replaced by a space, so digits on either side of it stay apart. As in MediaWiki, an unclosed ```` or ```` or ``` reads as `yes`, and an `exp` found + only inside a comment, as on Bonelord Totem, is `NULL` instead of the comment's number. An absent value was already + `NULL`, and `is_boss` is unchanged. ## 9.0.0+tibiash.9 diff --git a/docs/schema.md b/docs/schema.md index e23fa0d..f70ce4d 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -137,7 +137,7 @@ The generated database has the following tables. | hitpoints | `INTEGER` | The number of hitpoints the creature has. May be `NULL` if unknown. | | experience | `INTEGER` | The number of experience the creature yields . May be `NULL` if unknown. | | armor | `INTEGER` | The armor value of the creature. May be `NULL` if unknown. | -| mitigation | `REAL` | The creature's mitigation value. | +| mitigation | `REAL` | The creature's mitigation value. May be `NULL` if unknown. | | speed | `INTEGER` | The speed value of the creature. May be `NULL` if unknown. | | creature_class | `TEXT` | The class this creature belongs to (e.g. `Demons`, `Humanoids`, `Mammals`). | | type_primary | `TEXT` | The type this creature belongs to (e.g. `Archdemons`, `Dwarves`, `Apes`). | @@ -146,14 +146,14 @@ The generated database has the following tables. | bestiary_level | `TEXT` | The bestiary level of this creature. `NULL` for creatures not in the bestiary. | | bestiary_occurrence | `TEXT` | The bestiary’s rarity value of this creature. `NULL` for creatures not in the bestiary. | | bosstiary_class | `TEXT` | The bosstiary category of this creature. `NULL` for creatures not in the bestiary. | -| runs_at | `INTEGER` | The amount of hitpoints when the creature starts to run away. 0 means it won't run away. | -| summon_cost | `INTEGER` | The mana cost to summon this creature. `0` means it is not summonable. | -| convince_cost | `INTEGER` | The mana cost to convince this creature. `0` means it is not convincible. | -| illusionable | `BOOLEAN` | Whether the player can turn into this creature with Creature Illusion. | -| pushable | `BOOLEAN` | Whether this creature can be pushed or not. | -| push_objects | `BOOLEAN` | Whether this creature can push objects or not. | -| paralysable | `BOOLEAN` | Whether this creature can be paralyzed or not. | -| sees_invisible | `INTEGER` | Whether this creature can see invisible players or not. | +| runs_at | `INTEGER` | The amount of hitpoints when the creature starts to run away. 0 means it won't run away. May be `NULL` if unknown. | +| summon_cost | `INTEGER` | The mana cost to summon this creature. `0` means it cannot be summoned. May be `NULL` if unknown. | +| convince_cost | `INTEGER` | The mana cost to convince this creature. `0` means it cannot be convinced. May be `NULL` if unknown. | +| illusionable | `BOOLEAN` | Whether the player can turn into this creature with Creature Illusion. May be `NULL` if unknown. | +| pushable | `BOOLEAN` | Whether this creature can be pushed or not. May be `NULL` if unknown. | +| push_objects | `BOOLEAN` | Whether this creature can push objects or not. May be `NULL` if unknown. | +| paralysable | `BOOLEAN` | Whether this creature can be paralyzed or not. May be `NULL` if unknown. | +| sees_invisible | `INTEGER` | Whether this creature can see invisible players or not. May be `NULL` if unknown. | | spawn_type | `TEXT` | The way this creature spawns. | | is_boss | `INTEGER` | Whether this creature is a boss or not. | | cooldown | `REAL` | The cooldown in hours to fight the boss again. | @@ -165,8 +165,8 @@ The generated database has the following tables. | modifier_death | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | | modifier_holy | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | | modifier_drown | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | -| modifier_lifedrain | `INTEGER` | The healing modifier. `NULL` if unknown. | -| modifier_healing | `INTEGER` | The healing modifier. `NULL` if unknown. | +| modifier_lifedrain | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | +| modifier_healing | `INTEGER` | Percentage of healing the creature receives. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | | walks_through | `TEXT` | The type of fields the creature will walk through. | | walks_around | `TEXT` | The type of fields the creature will walk around to avoid when possible. | | location | `TEXT` | The locations where the creature can be found. | diff --git a/tests/parsers/test_creature.py b/tests/parsers/test_creature.py index 923f3c9..b47b9d5 100644 --- a/tests/parsers/test_creature.py +++ b/tests/parsers/test_creature.py @@ -335,3 +335,38 @@ def test_modifier_absent(self): creature = self._edited("| fireDmgMod = 0%\n", "") self.assertIsNone(creature.modifier_fire) + + def test_flag_ignores_a_trailing_comment(self): + creature = self._edited("| pushable = No", "| pushable = yes ") + + self.assertIs(True, creature.pushable) + + def test_paralysable_ignores_a_trailing_comment(self): + creature = self._edited("| paraimmune = yes", "| paraimmune = yes ") + + self.assertIs(False, creature.paralysable) + + def test_flag_typo(self): + creature = self._edited("| pushable = No", "| pushable = yse") + + self.assertIsNone(creature.pushable) + + def test_mitigation_only_in_a_comment(self): + creature = self._edited("| mitigation = 1.56", "| mitigation = ") + + self.assertIsNone(creature.mitigation) + + def test_stat_absent(self): + creature = self._edited("| armor = 25\n", "") + + self.assertIsNone(creature.armor) + + def test_summon_absent(self): + creature = self._edited("| summon = --\n", "") + + self.assertIsNone(creature.summon_cost) + + def test_convince_no(self): + creature = self._edited("| convince = --", "| convince = no") + + self.assertEqual(0, creature.convince_cost) diff --git a/tibiawikisql/models/creature.py b/tibiawikisql/models/creature.py index 75313b1..44ead78 100644 --- a/tibiawikisql/models/creature.py +++ b/tibiawikisql/models/creature.py @@ -178,29 +178,29 @@ class Creature(WikiEntry, WithStatus, WithVersion, WithImage, RowModel, table=Cr hitpoints: int | None """The creature's hitpoints, may be `None` if unknown.""" experience: int | None - """Experience points yielded by the creature. Might be `None` if unknown.""" + """Experience points yielded by the creature. ``None`` if unknown.""" armor: int | None - """The creature's armor value.""" + """The creature's armor value. ``None`` if unknown.""" mitigation: float | None - """The creature's mitigation value.""" + """The creature's mitigation value. ``None`` if unknown.""" speed: int | None - """The creature's speed value.""" + """The creature's speed value. ``None`` if unknown.""" runs_at: int | None - """The amount of hitpoints when the creature starts to run away. 0 means it won't run away.""" + """The amount of hitpoints when the creature starts to run away. 0 means it won't run away. ``None`` if unknown.""" summon_cost: int | None - """The mana needed to summon this creature. 0 if not summonable.""" + """The mana needed to summon this creature. 0 if it cannot be summoned, ``None`` if unknown.""" convince_cost: int | None - """The mana needed to convince this creature. 0 if not convincible.""" + """The mana needed to convince this creature. 0 if it cannot be convinced, ``None`` if unknown.""" illusionable: bool | None - """Whether the creature can be illusioned into using `Creature Illusion`.""" + """Whether the creature can be illusioned into using `Creature Illusion`. ``None`` if unknown.""" pushable: bool | None - """Whether the creature can be pushed or not.""" + """Whether the creature can be pushed or not. ``None`` if unknown.""" push_objects: bool | None - """Whether the creature can push objects or not.""" + """Whether the creature can push objects or not. ``None`` if unknown.""" sees_invisible: bool | None - """Whether the creature can see invisible players or not.""" + """Whether the creature can see invisible players or not. ``None`` if unknown.""" paralysable: bool | None - """Whether the creature can be paralyzed or not.""" + """Whether the creature can be paralyzed or not. ``None`` if unknown.""" spawn_type: str | None """The way this creature spawns.""" is_boss: bool @@ -208,25 +208,25 @@ class Creature(WikiEntry, WithStatus, WithVersion, WithImage, RowModel, table=Cr cooldown: float | None """The cooldown in hours to fight the boss again.""" modifier_physical: int | None - """The percentage of damage received of physical damage. ``None`` if unknown.""" + """The percentage of damage received of physical damage. 0 if immune, ``None`` if unknown.""" modifier_earth: int | None - """The percentage of damage received of earth damage. ``None`` if unknown.""" + """The percentage of damage received of earth damage. 0 if immune, ``None`` if unknown.""" modifier_fire: int | None - """The percentage of damage received of fire damage. ``None`` if unknown.""" + """The percentage of damage received of fire damage. 0 if immune, ``None`` if unknown.""" modifier_energy: int | None - """The percentage of damage received of energy damage. ``None`` if unknown.""" + """The percentage of damage received of energy damage. 0 if immune, ``None`` if unknown.""" modifier_ice: int | None - """The percentage of damage received of ice damage. ``None`` if unknown.""" + """The percentage of damage received of ice damage. 0 if immune, ``None`` if unknown.""" modifier_death: int | None - """The percentage of damage received of death damage. ``None`` if unknown.""" + """The percentage of damage received of death damage. 0 if immune, ``None`` if unknown.""" modifier_holy: int | None - """The percentage of damage received of holy damage. ``None`` if unknown.""" + """The percentage of damage received of holy damage. 0 if immune, ``None`` if unknown.""" modifier_drown: int | None - """The percentage of damage received of drown damage. ``None`` if unknown.""" + """The percentage of damage received of drown damage. 0 if immune, ``None`` if unknown.""" modifier_lifedrain: int | None - """The percentage of damage received of life drain damage. ``None`` if unknown.""" + """The percentage of damage received of life drain damage. 0 if immune, ``None`` if unknown.""" modifier_healing: int | None - """The healing modifier. ``None`` if unknown.""" + """The percentage of healing received. 0 if immune, ``None`` if unknown.""" abilities: list[CreatureAbility] = Field([]) """A list of the creature abilities.""" max_damage: CreatureMaxDamage | None = None diff --git a/tibiawikisql/parsers/creature.py b/tibiawikisql/parsers/creature.py index 01dbfef..d222bed 100644 --- a/tibiawikisql/parsers/creature.py +++ b/tibiawikisql/parsers/creature.py @@ -127,16 +127,17 @@ def parse_mana_cost(value: str) -> int | None: def parse_known_boolean(value: str) -> bool | None: - """Parse a creature flag, such as whether it is pushable. + """Parse a creature flag, such as whether it is pushable, from the visible text of its value. Args: value: The raw value of the field. Returns: - ``True`` for ``yes``, ``False`` for ``no``, or ``None`` for anything else, such as ``?`` or ``no?``. + ``True`` for ``yes``, ``False`` for ``no``, or ``None`` for anything else, such as ``?`` or ``no?``. Wiki + comments are ignored, so ``yes `` is ``True``. """ - return parse_boolean(value, None) + return parse_boolean(visible_text(value), None) def parse_paralysable(value: str) -> bool | None: @@ -146,10 +147,11 @@ def parse_paralysable(value: str) -> bool | None: value: The raw value of the ``paraimmune`` field. Returns: - ``False`` for ``yes``, ``True`` for ``no``, or ``None`` for anything else, such as ``?``. + ``False`` for ``yes``, ``True`` for ``no``, or ``None`` for anything else, such as ``?``. Wiki comments are + ignored, as for :func:`parse_known_boolean`. """ - return parse_boolean(value, None, invert=True) + return parse_boolean(visible_text(value), None, invert=True) def parse_loot(value: str) -> list[tuple[str, str]]: From 34f6d0c7912800d713f0068203b1bf5f602b2cb9 Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 19:31:47 +0000 Subject: [PATCH 03/17] feat: read slot, spell id and attack type --- CHANGELOG.md | 6 +++ README.md | 5 ++ docs/schema.md | 73 +++++++++++++++-------------- tests/models/test_creature.py | 25 +++++++++- tests/models/test_item.py | 28 +++++++++++ tests/models/test_spell.py | 37 +++++++++++++++ tests/parsers/test_creature.py | 38 +++++++++++++++ tests/parsers/test_item.py | 79 ++++++++++++++++++++++++++++++++ tests/parsers/test_spell.py | 52 +++++++++++++++++++++ tests/tests_utils.py | 23 +++++++++- tibiawikisql/models/creature.py | 4 ++ tibiawikisql/models/item.py | 4 ++ tibiawikisql/models/spell.py | 2 + tibiawikisql/parsers/creature.py | 18 ++------ tibiawikisql/parsers/item.py | 4 ++ tibiawikisql/parsers/spell.py | 3 +- tibiawikisql/schema.py | 5 ++ tibiawikisql/utils.py | 29 ++++++++++++ 18 files changed, 384 insertions(+), 51 deletions(-) create mode 100644 tests/models/test_spell.py diff --git a/CHANGELOG.md b/CHANGELOG.md index beb2b76..b2cd161 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ `yes`. Wiki comments are ignored in all of these fields, so `yes ` reads as `yes`, and an `exp` found only inside a comment, as on Bonelord Totem, is `NULL` instead of the comment's number. An absent value was already `NULL`, and `is_boss` is unchanged. +- Add `item.slot`, `item.is_usable`, `spell.spell_id`, `creature.attack_type` and `creature.uses_spells`, read from + the `slot`, `usable`, `spellid`, `attacktype` and `usespells` fields. `slot` and `attack_type` are lower-cased, like + `neck` or `melee`, and `NULL` for an empty value or one with a `?`. `is_usable` and `uses_spells` are `true` for + `yes`, `false` for `no` and `NULL` for anything else, such as `?`. `spell_id` is the first number in the value and + `NULL` when there is none, as for `?`. Wiki comments are ignored in all of these fields, and an absent value is + `NULL`. ## 9.0.0+tibiash.9 diff --git a/README.md b/README.md index 981042e..01ab4fc 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,11 @@ The copy adds this data to the database: `NULL` when the mount has no price. * The `item.value_buy_currency` column. It holds the currency of an item's NPC price, like `Gold Coin` or `Theons`, `NULL` when NPCs do not sell the item. +* The `item.slot` and `item.is_usable` columns. They hold the slot an item goes in, in lower case, like `neck`, and + whether it can be used, `NULL` when the wiki does not say. +* The `spell.spell_id` column. It holds the ID of a spell, `NULL` when the wiki has none or only a comment. +* The `creature.attack_type` and `creature.uses_spells` columns. They hold how a creature attacks, in lower case, like + `melee`, and whether it uses spells, `NULL` when the wiki does not say. The [database schema](docs/schema.md) describes each of them. diff --git a/docs/schema.md b/docs/schema.md index f70ce4d..88dd53e 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -155,6 +155,8 @@ The generated database has the following tables. | paralysable | `BOOLEAN` | Whether this creature can be paralyzed or not. May be `NULL` if unknown. | | sees_invisible | `INTEGER` | Whether this creature can see invisible players or not. May be `NULL` if unknown. | | spawn_type | `TEXT` | The way this creature spawns. | +| attack_type | `TEXT` | How the creature attacks, in lower case (e.g. `melee`). May be `NULL` if unknown. | +| uses_spells | `BOOLEAN` | Whether this creature uses spells or not. May be `NULL` if unknown. | | is_boss | `INTEGER` | Whether this creature is a boss or not. | | cooldown | `REAL` | The cooldown in hours to fight the boss again. | | modifier_physical | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | @@ -330,6 +332,7 @@ The generated database has the following tables. | is_stackable | `BOOLEAN` | Whether this item is stackable or not. | | is_pickupable | `BOOLEAN` | Whether this item can be picked up or not. | | is_immobile | `BOOLEAN` | Whether the item can be moved around the map or not. | +| is_usable | `BOOLEAN` | Whether the item can be used or not. `NULL` if unknown. | | value_sell | `INTEGER` | The maximum value of this item when sold to NPCs | | value_buy | `INTEGER` | The maximum price of this item when bought from NPCs, in `value_buy_currency`. | | value_buy_currency | `TEXT` | The currency of `value_buy`: the article's `pricecurrency`, otherwise a currency written after the price in `npcprice`, otherwise Gold Coin. A linked currency gives the page it links to, like `Silver Token` for `[[Silver Token]]s`. `NULL` when `value_buy` is `NULL` or 0. | @@ -337,6 +340,7 @@ The generated database has the following tables. | item_class | `TEXT` | The class this item belongs to (e.g. `Body Equipment` , `Weapons`). | | item_type | `TEXT` | The category this item belongs to (e.g. `Helmets`, `Club Weapons`). | | type_secondary | `TEXT` | A secondary type this item belongs to, if any. | +| slot | `TEXT` | The slot this item goes in, in lower case (e.g. `neck`, `weapon hand`). `NULL` if unknown. | | flavor_text | `TEXT` | The extra text that is displayed when some items are looked at. | | light_color | `INTEGER` | The color of the light emitted by this item, if any. | | light_radius | `INTEGER` | The radius of the light emitted by this item, if any. | @@ -634,40 +638,41 @@ that doesn't form a range. ### spell -| Column | Type | Description | -| ------------------------ | --------------------- | ----------------------------------------------------------------------------- | -| article_id | `INTEGER` / `PRIMARY` | The id of the article containing this spell. | -| title | `TEXT` | The title of the article containing the spell. | -| name | `TEXT` | The name of the spell. | -| words | `TEXT` | The words used to cast the spell. | -| effect | `TEXT` | The effect of this spell. | -| spell_type | `TEXT` | Whether the spell is `Instant` or a `Rune` spell. | -| group_spell | `TEXT` | The cooldown group of this spell. | -| group_secondary | `TEXT` | The secondary cooldown group of this spell. | -| group_rune | `TEXT` | The cooldown group of the rune created by this spell. | -| element | `TEXT` | The type of damage this spell deals, if applicable. | -| base_power | `INTEGER` | The spell's base power, if applicable. | -| level | `INTEGER` | Level required to cast this spell | -| mana | `INTEGER` | Mana required to cast this spell. `0` means special conditions apply. | -| soul | `INTEGER` | Soul points required to cast this spell. | -| is_premium | `BOOLEAN` | Whether this spell requires premium account or not. `0` or `1`. | -| is_promotion | `BOOLEAN` | Whether this spell requires a promotion or not. `0` or `1`. | -| is_wheel_spell | `BOOLEAN` | Whether this spell is acquired through the Wheel of Destiny. | -| is_passive | `BOOLEAN` | Whether this spell is triggered automatically without casting. | -| cooldown | `INTEGER` | The individual cooldown in seconds of this spell | -| cooldown2 | `INTEGER` | The spell's individual cooldown for the level 2 perk of the Wheel of Destiny. | -| cooldown3 | `INTEGER` | The spell's individual cooldown for the level 3 perk of the Wheel of Destiny. | -| cooldown_group | `INTEGER` | The group cooldown of this spell. | -| cooldown_group_secondary | `INTEGER` | The secondary group cooldown of this spell. | -| knight | `BOOLEAN` | Whether this spell can be used by knights or not. | -| sorcerer | `BOOLEAN` | Whether this spell can be used by sorcerers or not. | -| druid | `BOOLEAN` | Whether this spell can be used by druids or not. | -| paladin | `BOOLEAN` | Whether this spell can be used by paladins or not. | -| monk | `BOOLEAN` | Whether the spell can be used by monks or not. | -| image | `BLOB` | The spell’s image bytes. | -| version | `TEXT` | Client version where this spell was implemented. | -| status | `TEXT` | The status of the spell in game. | -| timestamp | `TIMESTAMP` | ISO 8601 timestamp of the article's last edit. | +| Column | Type | Description | +| ------------------------ | --------------------- | --------------------------------------------------------------------------------------------------------------- | +| article_id | `INTEGER` / `PRIMARY` | The id of the article containing this spell. | +| title | `TEXT` | The title of the article containing the spell. | +| name | `TEXT` | The name of the spell. | +| spell_id | `INTEGER` | The ID of the spell, the first integer in the value after wiki comments are removed, `NULL` when there is none. | +| words | `TEXT` | The words used to cast the spell. | +| effect | `TEXT` | The effect of this spell. | +| spell_type | `TEXT` | Whether the spell is `Instant` or a `Rune` spell. | +| group_spell | `TEXT` | The cooldown group of this spell. | +| group_secondary | `TEXT` | The secondary cooldown group of this spell. | +| group_rune | `TEXT` | The cooldown group of the rune created by this spell. | +| element | `TEXT` | The type of damage this spell deals, if applicable. | +| base_power | `INTEGER` | The spell's base power, if applicable. | +| level | `INTEGER` | Level required to cast this spell | +| mana | `INTEGER` | Mana required to cast this spell. `0` means special conditions apply. | +| soul | `INTEGER` | Soul points required to cast this spell. | +| is_premium | `BOOLEAN` | Whether this spell requires premium account or not. `0` or `1`. | +| is_promotion | `BOOLEAN` | Whether this spell requires a promotion or not. `0` or `1`. | +| is_wheel_spell | `BOOLEAN` | Whether this spell is acquired through the Wheel of Destiny. | +| is_passive | `BOOLEAN` | Whether this spell is triggered automatically without casting. | +| cooldown | `INTEGER` | The individual cooldown in seconds of this spell | +| cooldown2 | `INTEGER` | The spell's individual cooldown for the level 2 perk of the Wheel of Destiny. | +| cooldown3 | `INTEGER` | The spell's individual cooldown for the level 3 perk of the Wheel of Destiny. | +| cooldown_group | `INTEGER` | The group cooldown of this spell. | +| cooldown_group_secondary | `INTEGER` | The secondary group cooldown of this spell. | +| knight | `BOOLEAN` | Whether this spell can be used by knights or not. | +| sorcerer | `BOOLEAN` | Whether this spell can be used by sorcerers or not. | +| druid | `BOOLEAN` | Whether this spell can be used by druids or not. | +| paladin | `BOOLEAN` | Whether this spell can be used by paladins or not. | +| monk | `BOOLEAN` | Whether the spell can be used by monks or not. | +| image | `BLOB` | The spell’s image bytes. | +| version | `TEXT` | Client version where this spell was implemented. | +| status | `TEXT` | The status of the spell in game. | +| timestamp | `TIMESTAMP` | ISO 8601 timestamp of the article's last edit. | diff --git a/tests/models/test_creature.py b/tests/models/test_creature.py index c1e5477..757f884 100644 --- a/tests/models/test_creature.py +++ b/tests/models/test_creature.py @@ -12,7 +12,12 @@ def setUp(self): self.conn.row_factory = sqlite3.Row schema.create_tables(self.conn) - def _creature(self, race_id: int | None) -> Creature: + def _creature( + self, + race_id: int | None, + attack_type: str | None = None, + uses_spells: bool | None = None, + ) -> Creature: return Creature( article_id=1, title="Demon", @@ -41,6 +46,8 @@ def _creature(self, race_id: int | None) -> Creature: sees_invisible=True, paralysable=False, spawn_type=None, + attack_type=attack_type, + uses_spells=uses_spells, is_boss=False, cooldown=None, modifier_physical=75, @@ -75,3 +82,19 @@ def test_creature_race_id_round_trip_none(self): loaded = Creature.get_one_by_field(self.conn, "article_id", 1) self.assertIsNone(loaded.race_id) + + def test_creature_attack_type_and_uses_spells_round_trip(self): + self._creature(35, attack_type="melee", uses_spells=False).insert(self.conn) + + loaded = Creature.get_one_by_field(self.conn, "article_id", 1) + + self.assertEqual("melee", loaded.attack_type) + self.assertIs(False, loaded.uses_spells) + + def test_creature_attack_type_and_uses_spells_round_trip_none(self): + self._creature(35).insert(self.conn) + + loaded = Creature.get_one_by_field(self.conn, "article_id", 1) + + self.assertIsNone(loaded.attack_type) + self.assertIsNone(loaded.uses_spells) diff --git a/tests/models/test_item.py b/tests/models/test_item.py index ece7ed9..28b6ba3 100644 --- a/tests/models/test_item.py +++ b/tests/models/test_item.py @@ -91,3 +91,31 @@ def test_item_value_buy_currency_round_trip(self): self.assertEqual(7197, loaded.value_buy) self.assertEqual("Theons", loaded.value_buy_currency) + + def test_item_slot_and_usable_round_trip(self): + article = Article( + article_id=1, + title="Amethyst Necklace", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=load_resource("content_item_amethyst_necklace.txt"), + ) + ItemParser.from_article(article).insert(self.conn) + + loaded = Item.get_one_by_field(self.conn, "article_id", 1) + + self.assertEqual("neck", loaded.slot) + self.assertIs(False, loaded.is_usable) + + def test_item_slot_and_usable_round_trip_none(self): + article = Article( + article_id=1, + title="Shallow Water", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=load_resource("content_item_shallow_water.txt"), + ) + ItemParser.from_article(article).insert(self.conn) + + loaded = Item.get_one_by_field(self.conn, "article_id", 1) + + self.assertIsNone(loaded.slot) + self.assertIsNone(loaded.is_usable) diff --git a/tests/models/test_spell.py b/tests/models/test_spell.py new file mode 100644 index 0000000..a5af6fd --- /dev/null +++ b/tests/models/test_spell.py @@ -0,0 +1,37 @@ +import datetime +import sqlite3 +import unittest + +from tests import load_resource +from tibiawikisql import schema +from tibiawikisql.api import Article +from tibiawikisql.models import Spell +from tibiawikisql.parsers import SpellParser + + +class TestSpell(unittest.TestCase): + def setUp(self): + self.conn = sqlite3.connect(":memory:") + self.conn.row_factory = sqlite3.Row + schema.create_tables(self.conn) + + def _round_trip(self, title: str, resource: str) -> Spell: + article = Article( + article_id=1, + title=title, + timestamp=datetime.datetime.fromisoformat("2026-08-06T21:35:46+00:00"), + content=load_resource(resource), + ) + SpellParser.from_article(article).insert(self.conn) + + return Spell.get_one_by_field(self.conn, "article_id", 1) + + def test_spell_id_round_trip(self): + loaded = self._round_trip("Flame Strike", "content_spell.txt") + + self.assertEqual(89, loaded.spell_id) + + def test_spell_id_round_trip_none(self): + loaded = self._round_trip("Second Wind", "content_spell_second_wind.txt") + + self.assertIsNone(loaded.spell_id) diff --git a/tests/parsers/test_creature.py b/tests/parsers/test_creature.py index b47b9d5..ebd7899 100644 --- a/tests/parsers/test_creature.py +++ b/tests/parsers/test_creature.py @@ -315,6 +315,8 @@ def test_experience_only_in_a_comment(self): self.assertIsNone(creature.hitpoints) self.assertEqual(0, creature.runs_at) self.assertEqual(0, creature.modifier_fire) + self.assertIsNone(creature.attack_type) + self.assertIsNone(creature.uses_spells) def test_stat_ignores_a_trailing_comment(self): creature = self._edited("| hp = 1000", "| hp = 1000 ") @@ -370,3 +372,39 @@ def test_convince_no(self): creature = self._edited("| convince = --", "| convince = no") self.assertEqual(0, creature.convince_cost) + + def test_attack_type_and_uses_spells(self): + creature = self._parse(load_resource("content_creature_dragon.txt")) + + self.assertEqual("melee", creature.attack_type) + self.assertIs(True, creature.uses_spells) + + def test_attack_type_unknown(self): + creature = self._edited("| attacktype = Melee", "| attacktype = ?") + + self.assertIsNone(creature.attack_type) + + def test_attack_type_absent(self): + creature = self._edited("| attacktype = Melee\n", "") + + self.assertIsNone(creature.attack_type) + + def test_attack_type_ignores_a_trailing_comment(self): + creature = self._edited("| attacktype = Melee", "| attacktype = Distance ") + + self.assertEqual("distance", creature.attack_type) + + def test_uses_spells_no(self): + creature = self._edited("| usespells = yes", "| usespells = no") + + self.assertIs(False, creature.uses_spells) + + def test_uses_spells_unknown(self): + creature = self._edited("| usespells = yes", "| usespells = ?") + + self.assertIsNone(creature.uses_spells) + + def test_uses_spells_absent(self): + creature = self._edited("| usespells = yes\n", "") + + self.assertIsNone(creature.uses_spells) diff --git a/tests/parsers/test_item.py b/tests/parsers/test_item.py index b41f655..4d2377a 100644 --- a/tests/parsers/test_item.py +++ b/tests/parsers/test_item.py @@ -260,3 +260,82 @@ def test_item_value_buy_currency_none_without_npcprice(self): self.assertIsNone(item.value_buy) self.assertIsNone(item.value_buy_currency) + + +class TestItemParserSlotAndUsable(unittest.TestCase): + def _parse_content(self, title: str, content: str): + article = Article( + article_id=1, + title=title, + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=content, + ) + + return ItemParser.from_article(article) + + def _edited(self, old: str, new: str): + content = load_resource("content_item_amethyst_necklace.txt") + self.assertIn(old, content) + return self._parse_content("Amethyst Necklace", content.replace(old, new)) + + def test_amethyst_necklace(self): + item = self._parse_content("Amethyst Necklace", load_resource("content_item_amethyst_necklace.txt")) + + self.assertEqual("neck", item.slot) + self.assertIs(False, item.is_usable) + + def test_blade_of_mayhem(self): + item = self._parse_content("Blade of Mayhem", load_resource("content_item_blade_of_mayhem.txt")) + + self.assertEqual("weapon hand", item.slot) + self.assertIs(True, item.is_usable) + + def test_slot_unknown(self): + item = self._edited("| slot = Neck", "| slot = ?") + + self.assertIsNone(item.slot) + + def test_slot_uncertain(self): + item = self._edited("| slot = Neck", "| slot = Neck?") + + self.assertIsNone(item.slot) + + def test_slot_empty(self): + item = self._edited("| slot = Neck", "| slot =") + + self.assertIsNone(item.slot) + + def test_slot_absent(self): + item = self._edited("| slot = Neck\n", "") + + self.assertIsNone(item.slot) + + def test_slot_ignores_a_trailing_comment(self): + item = self._edited("| slot = Neck", "| slot = Neck ") + + self.assertEqual("neck", item.slot) + + def test_slot_only_in_a_comment(self): + item = self._edited("| slot = Neck", "| slot = ") + + self.assertIsNone(item.slot) + + def test_usable_unknown(self): + item = self._edited("| usable = no", "| usable = ?") + + self.assertIsNone(item.is_usable) + + def test_usable_absent(self): + item = self._edited("| usable = no\n", "") + + self.assertIsNone(item.is_usable) + + def test_usable_typo(self): + item = self._edited("| usable = no", "| usable = noo") + + self.assertIsNone(item.is_usable) + + def test_usable_ignores_a_trailing_comment(self): + item = self._edited("| usable = no", "| usable = yes ") + + self.assertIs(True, item.is_usable) diff --git a/tests/parsers/test_spell.py b/tests/parsers/test_spell.py index e2600f4..9692a38 100644 --- a/tests/parsers/test_spell.py +++ b/tests/parsers/test_spell.py @@ -84,6 +84,58 @@ def test_sniper_without_effect(self): self.assertIsNone(spell.effect) +class TestSpellParserSpellId(unittest.TestCase): + def _parse(self, title: str, content: str) -> Spell: + article = Article( + article_id=1, + title=title, + timestamp=datetime.datetime.fromisoformat("2026-08-06T21:35:46+00:00"), + content=content, + ) + + return SpellParser.from_article(article) + + def _edited(self, old: str, new: str) -> Spell: + content = load_resource("content_spell.txt") + self.assertIn(old, content) + return self._parse("Flame Strike", content.replace(old, new)) + + def test_flame_strike(self): + spell = self._parse("Flame Strike", load_resource("content_spell.txt")) + + self.assertEqual(89, spell.spell_id) + + def test_ultimate_explosion(self): + spell = self._parse("Ultimate Explosion", load_resource("content_spell_ultimate_explosion.txt")) + + self.assertEqual(24, spell.spell_id) + + def test_second_wind_empty(self): + spell = self._parse("Second Wind", load_resource("content_spell_second_wind.txt")) + + self.assertIsNone(spell.spell_id) + + def test_absent(self): + spell = self._parse("Train Party", MANA_VARIES_CONTENT) + + self.assertIsNone(spell.spell_id) + + def test_unknown(self): + spell = self._edited("| spellid = 89", "| spellid = ?") + + self.assertIsNone(spell.spell_id) + + def test_ignores_a_trailing_comment(self): + spell = self._edited("| spellid = 89", "| spellid = 89 ") + + self.assertEqual(89, spell.spell_id) + + def test_only_in_a_comment(self): + spell = self._edited("| spellid = 89", "| spellid = ") + + self.assertIsNone(spell.spell_id) + + class TestParseWholeNumber(unittest.TestCase): def test_parse_whole_number(self): self.assertEqual(4, parse_whole_number("4")) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index bd5f3a6..6fb20fe 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -8,7 +8,8 @@ from tibiawikisql.utils import (clean_links, client_color_to_rgb, parse_boolean, parse_client_id, parse_currency, parse_first_integer, parse_float, parse_integer, parse_date, parse_loot_statistics, parse_min_max, parse_sounds, - parse_weapon_proficiency_name, parse_weapon_proficiency_tables, visible_text) + parse_known_boolean, parse_known_text, parse_weapon_proficiency_name, + parse_weapon_proficiency_tables, visible_text) class TestUtils(unittest.TestCase): @@ -84,6 +85,26 @@ def test_visible_text(self): self.assertEqual("35 ", visible_text("35 ")) + self.assertIsNone(parse_known_boolean("?")) + self.assertIsNone(parse_known_boolean("no?")) + self.assertIsNone(parse_known_boolean("")) + self.assertIsNone(parse_known_boolean("")) + + def test_parse_known_text(self): + self.assertEqual("weapon hand", parse_known_text("Weapon Hand")) + self.assertEqual("melee", parse_known_text(" Melee ")) + self.assertIsNone(parse_known_text("?")) + self.assertIsNone(parse_known_text("Melee?")) + self.assertIsNone(parse_known_text("")) + self.assertIsNone(parse_known_text(" ")) + self.assertIsNone(parse_known_text("")) + self.assertIsNone(parse_known_text("")) diff --git a/tibiawikisql/models/creature.py b/tibiawikisql/models/creature.py index 44ead78..4526a2a 100644 --- a/tibiawikisql/models/creature.py +++ b/tibiawikisql/models/creature.py @@ -203,6 +203,10 @@ class Creature(WikiEntry, WithStatus, WithVersion, WithImage, RowModel, table=Cr """Whether the creature can be paralyzed or not. ``None`` if unknown.""" spawn_type: str | None """The way this creature spawns.""" + attack_type: str | None = None + """How the creature attacks, in lower case, like ``melee`` or ``distance``. ``None`` if unknown.""" + uses_spells: bool | None = None + """Whether the creature uses spells or not. ``None`` if unknown.""" is_boss: bool """Whether the creature is a boss or not.""" cooldown: float | None diff --git a/tibiawikisql/models/item.py b/tibiawikisql/models/item.py index ef14521..f096bed 100644 --- a/tibiawikisql/models/item.py +++ b/tibiawikisql/models/item.py @@ -134,6 +134,8 @@ class Item(WikiEntry, WithVersion, WithStatus, WithImage, RowModel, table=ItemTa """Whether the item can be picked up or not.""" is_immobile: bool """Whether the item can be moved around the map or not.""" + is_usable: bool | None = None + """Whether the item can be used or not. `None` if unknown.""" value_sell: int | None """The highest price an NPC will buy this item for.""" value_buy: int | None @@ -148,6 +150,8 @@ class Item(WikiEntry, WithVersion, WithStatus, WithImage, RowModel, table=ItemTa """The item's type.""" type_secondary: str | None """The item's secondary type, if any.""" + slot: str | None = None + """The slot the item goes in, in lower case, like `neck` or `weapon hand`. `None` if unknown.""" flavor_text: str | None """The extra text that is displayed when some items are looked at.""" light_color: int | None = None diff --git a/tibiawikisql/models/spell.py b/tibiawikisql/models/spell.py index 2758bfd..9f6de8c 100644 --- a/tibiawikisql/models/spell.py +++ b/tibiawikisql/models/spell.py @@ -11,6 +11,8 @@ class Spell(WikiEntry, WithVersion, WithStatus, WithImage, RowModel, table=Spell name: str """The name of the spell.""" + spell_id: int | None = Field(None) + """The spell's ID. ``None`` if unknown.""" words: str | None = Field(None) """The spell's invocation words.""" effect: str | None = Field(None) diff --git a/tibiawikisql/parsers/creature.py b/tibiawikisql/parsers/creature.py index d222bed..9ce58de 100644 --- a/tibiawikisql/parsers/creature.py +++ b/tibiawikisql/parsers/creature.py @@ -22,6 +22,8 @@ parse_client_id, parse_float, parse_integer, + parse_known_boolean, + parse_known_text, parse_min_max, parse_sounds, strip_code, @@ -126,20 +128,6 @@ def parse_mana_cost(value: str) -> int | None: return parse_integer(text, None) -def parse_known_boolean(value: str) -> bool | None: - """Parse a creature flag, such as whether it is pushable, from the visible text of its value. - - Args: - value: The raw value of the field. - - Returns: - ``True`` for ``yes``, ``False`` for ``no``, or ``None`` for anything else, such as ``?`` or ``no?``. Wiki - comments are ignored, so ``yes `` is ``True``. - - """ - return parse_boolean(visible_text(value), None) - - def parse_paralysable(value: str) -> bool | None: """Parse whether a creature can be paralysed from its ``paraimmune`` field, which says whether it is immune. @@ -309,6 +297,8 @@ class CreatureParser(BaseParser): "sees_invisible": AttributeParser.optional("senseinvis", parse_known_boolean), "paralysable": AttributeParser.optional("paraimmune", parse_paralysable), "spawn_type": AttributeParser.optional("spawntype"), + "attack_type": AttributeParser.optional("attacktype", parse_known_text), + "uses_spells": AttributeParser.optional("usespells", parse_known_boolean), "is_boss": AttributeParser.optional("isboss", parse_boolean, False), "cooldown": AttributeParser.optional("cooldown", parse_float), "modifier_physical": AttributeParser.optional("physicalDmgMod", parse_known_integer), diff --git a/tibiawikisql/parsers/item.py b/tibiawikisql/parsers/item.py index f9cc39e..d162dcd 100644 --- a/tibiawikisql/parsers/item.py +++ b/tibiawikisql/parsers/item.py @@ -20,6 +20,8 @@ parse_float, parse_first_integer, parse_integer, + parse_known_boolean, + parse_known_text, parse_sounds, strip_code, ) @@ -72,6 +74,7 @@ class ItemParser(BaseParser): "is_stackable": AttributeParser.optional("stackable", parse_boolean, False), "is_pickupable": AttributeParser.optional("pickupable", parse_boolean, False), "is_immobile": AttributeParser.optional("immobile", parse_boolean, False), + "is_usable": AttributeParser.optional("usable", parse_known_boolean), "value_sell": AttributeParser.optional("npcvalue", parse_integer), "value_buy": AttributeParser.optional("npcprice", parse_integer), "weight": AttributeParser.optional("weight", parse_float), @@ -79,6 +82,7 @@ class ItemParser(BaseParser): "item_class": AttributeParser.optional("objectclass"), "item_type": AttributeParser.optional("primarytype"), "type_secondary": AttributeParser.optional("secondarytype"), + "slot": AttributeParser.optional("slot", parse_known_text), "light_color": AttributeParser.optional("lightcolor", lambda x: client_color_to_rgb(parse_integer(x))), "light_radius": AttributeParser.optional("lightradius", parse_integer), "version": AttributeParser.optional("implemented"), diff --git a/tibiawikisql/parsers/spell.py b/tibiawikisql/parsers/spell.py index afd52c6..78ee305 100644 --- a/tibiawikisql/parsers/spell.py +++ b/tibiawikisql/parsers/spell.py @@ -6,7 +6,7 @@ import tibiawikisql.schema from tibiawikisql.parsers.base import AttributeParser from tibiawikisql.parsers import BaseParser -from tibiawikisql.utils import clean_links, parse_boolean, parse_integer, parse_whole_number +from tibiawikisql.utils import clean_links, parse_boolean, parse_client_id, parse_integer, parse_whole_number class SpellParser(BaseParser): @@ -17,6 +17,7 @@ class SpellParser(BaseParser): template_name = "Infobox_Spell" attribute_map: ClassVar = { "name": AttributeParser.required("name"), + "spell_id": AttributeParser.optional("spellid", parse_client_id), "effect": AttributeParser.optional("effect", clean_links), "words": AttributeParser.optional("words"), "spell_type": AttributeParser.optional("type"), diff --git a/tibiawikisql/schema.py b/tibiawikisql/schema.py index 83c8a8a..45fdb8b 100644 --- a/tibiawikisql/schema.py +++ b/tibiawikisql/schema.py @@ -71,6 +71,8 @@ class CreatureTable(Table): paralysable = Column(Boolean) sees_invisible = Column(Integer) spawn_type = Column(Text) + attack_type = Column(Text) + uses_spells = Column(Boolean) is_boss = Column(Integer) cooldown = Column(Real) modifier_physical = Column(Integer) @@ -136,6 +138,7 @@ class ItemTable(Table): is_stackable = Column(Boolean, default=False) is_pickupable = Column(Boolean, default=True) is_immobile = Column(Boolean, default=True) + is_usable = Column(Boolean) value_sell = Column(Integer) value_buy = Column(Integer) value_buy_currency = Column(Text) @@ -143,6 +146,7 @@ class ItemTable(Table): item_class = Column(Text, index=True) item_type = Column(Text, index=True) type_secondary = Column(Text, index=True) + slot = Column(Text) flavor_text = Column(Text) light_color = Column(Integer) light_radius = Column(Integer) @@ -338,6 +342,7 @@ class SpellTable(Table): article_id = Column(Integer, primary_key=True) title = Column(Text, unique=True) name = Column(Text, no_case=True, index=True) + spell_id = Column(Integer) words = Column(Text, index=True) effect = Column(Text) spell_type = Column(Text, index=True) diff --git a/tibiawikisql/utils.py b/tibiawikisql/utils.py index d2db2cf..38c83ef 100644 --- a/tibiawikisql/utils.py +++ b/tibiawikisql/utils.py @@ -385,6 +385,35 @@ def parse_client_id(value: str) -> int | None: return parse_first_integer(visible_text(value), None) +def parse_known_boolean(value: str) -> bool | None: + """Parse a yes or no field, such as whether a creature is pushable, from the visible text of its value. + + Args: + value: The raw value of the field. + + Returns: + ``True`` for ``yes``, ``False`` for ``no``, or ``None`` for anything else, such as ``?`` or ``no?``. Wiki + comments are ignored, so ``yes `` is ``True``. + + """ + return parse_boolean(visible_text(value), None) + + +def parse_known_text(value: str) -> str | None: + """Parse a text field, such as an item's slot, from the visible text of its value, in lower case. + + Args: + value: The raw value of the field. + + Returns: + The text outside wiki comments, stripped and lower-cased, or ``None`` if it is empty or holds a ``?``, as for + ``?`` or ``Melee?``. + + """ + text = clean_question_mark(visible_text(value).strip()) + return text.lower() if text else None + + def parse_currency(value: str) -> str | None: """Parse the currency of a price, like the ``pricecurrency`` of an item or a mount. From 0e17c721bb2510a66477231e5abf1745edbd4082 Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 19:42:17 +0000 Subject: [PATCH 04/17] feat: keep every item client id --- CHANGELOG.md | 3 ++ README.md | 1 + docs/schema.md | 13 ++++++- tests/models/test_book.py | 11 +++++- tests/models/test_item.py | 36 +++++++++++++++++++ tests/parsers/test_item.py | 70 +++++++++++++++++++++++++++++++++++- tests/tests_utils.py | 16 ++++++++- tibiawikisql/models/item.py | 8 +++++ tibiawikisql/parsers/item.py | 9 +++++ tibiawikisql/schema.py | 7 ++++ tibiawikisql/utils.py | 16 +++++++++ 11 files changed, 186 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2cd161..f607593 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ `yes`, `false` for `no` and `NULL` for anything else, such as `?`. `spell_id` is the first number in the value and `NULL` when there is none, as for `?`. Wiki comments are ignored in all of these fields, and an absent value is `NULL`. +- Add the `item_client_id` table, with every client ID in an item's `itemid` in listed order, from position 0. Shallow + Water's `629,630,631,632,633,634` gives six rows. Wiki comments are ignored, and an `itemid` with no whole number, + like `?`, gives no rows. `item.client_id` is unchanged and still holds the first ID. ## 9.0.0+tibiash.9 diff --git a/README.md b/README.md index 01ab4fc..fe3080e 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ The copy adds this data to the database: * The `spell.spell_id` column. It holds the ID of a spell, `NULL` when the wiki has none or only a comment. * The `creature.attack_type` and `creature.uses_spells` columns. They hold how a creature attacks, in lower case, like `melee`, and whether it uses spells, `NULL` when the wiki does not say. +* The `item_client_id` table. It keeps every client ID of an item, in the order the wiki lists them, from position 0. The [database schema](docs/schema.md) describes each of them. diff --git a/docs/schema.md b/docs/schema.md index 88dd53e..10a4de3 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -27,6 +27,7 @@ The generated database has the following tables. | `imbuement_material` | Contains the item materials for imbuements. | | `item` | Contains information for all items. | | `item_attribute` | Contains extra attributes and properties of items that only apply to certain types. | +| `item_client_id` | Contains every client ID of items, in the order the wiki lists them. | | `item_key` | Contains the different key variations. | | `item_sound` | Contains all the sounds made by items. | | `item_proficiency_perk` | Contains weapon proficiency perks for items. | @@ -345,7 +346,7 @@ The generated database has the following tables. | light_color | `INTEGER` | The color of the light emitted by this item, if any. | | light_radius | `INTEGER` | The radius of the light emitted by this item, if any. | | version | `TEXT` | The client version this item was introduced to the game. | -| client_id | `INTEGER` | The client id of the item. | +| client_id | `INTEGER` | The client id of the item, the first integer in `itemid`. `item_client_id` has every client ID. | | image | `BLOB` | The item’s image bytes. | | status | `TEXT` | The status of the item in game. | | timestamp | `TIMESTAMP` | ISO 8601 timestamp of the article's last edit. | @@ -368,6 +369,16 @@ that doesn't form a range. +### item_client_id + +| Column | Type | Description | +| --------- | --------- | ------------------------------------------------------------------------------------ | +| item_id | `INTEGER` | The id of the item this client ID belongs to. | +| client_id | `INTEGER` | A client ID of the item, a whole number in `itemid` after wiki comments are removed. | +| position | `INTEGER` | The ID's place in the `itemid` list, from 0. | + + + ### item_key | Column | Type | Description | diff --git a/tests/models/test_book.py b/tests/models/test_book.py index ae90bf7..34f7e76 100644 --- a/tests/models/test_book.py +++ b/tests/models/test_book.py @@ -1,10 +1,18 @@ +# Changed by tibia.sh in 2026. See "About this copy" in README.md. import sqlite3 import unittest from polyfactory.factories.pydantic_factory import ModelFactory from tibiawikisql.models import Book, Item -from tibiawikisql.schema import BookTable, ItemAttributeTable, ItemSoundTable, ItemStoreOfferTable, ItemTable +from tibiawikisql.schema import ( + BookTable, + ItemAttributeTable, + ItemClientIdTable, + ItemSoundTable, + ItemStoreOfferTable, + ItemTable, +) class BookFactory(ModelFactory[Book]): @@ -31,6 +39,7 @@ def setUp(self): self.conn.executescript(ItemAttributeTable.get_create_table_statement()) self.conn.executescript(ItemStoreOfferTable.get_create_table_statement()) self.conn.executescript(ItemSoundTable.get_create_table_statement()) + self.conn.executescript(ItemClientIdTable.get_create_table_statement()) self.conn.executescript(BookTable.get_create_table_statement()) diff --git a/tests/models/test_item.py b/tests/models/test_item.py index 28b6ba3..e781793 100644 --- a/tests/models/test_item.py +++ b/tests/models/test_item.py @@ -119,3 +119,39 @@ def test_item_slot_and_usable_round_trip_none(self): self.assertIsNone(loaded.slot) self.assertIsNone(loaded.is_usable) + + def test_item_client_ids_round_trip(self): + article = Article( + article_id=1, + title="Shallow Water", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=load_resource("content_item_shallow_water.txt"), + ) + ItemParser.from_article(article).insert(self.conn) + + loaded = Item.get_one_by_field(self.conn, "article_id", 1) + rows = self.conn.execute( + "SELECT item_id, client_id, position FROM item_client_id ORDER BY position", + ).fetchall() + + self.assertEqual(629, loaded.client_id) + self.assertEqual([629, 630, 631, 632, 633, 634], loaded.client_ids) + self.assertEqual( + [(1, 629, 0), (1, 630, 1), (1, 631, 2), (1, 632, 3), (1, 633, 4), (1, 634, 5)], + [tuple(row) for row in rows], + ) + + def test_item_client_ids_round_trip_single(self): + article = Article( + article_id=1, + title="Amethyst Necklace", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=load_resource("content_item_amethyst_necklace.txt"), + ) + ItemParser.from_article(article).insert(self.conn) + + loaded = Item.get_one_by_field(self.conn, "article_id", 1) + rows = self.conn.execute("SELECT item_id, client_id, position FROM item_client_id").fetchall() + + self.assertEqual([35609], loaded.client_ids) + self.assertEqual([(1, 35609, 0)], [tuple(row) for row in rows]) diff --git a/tests/parsers/test_item.py b/tests/parsers/test_item.py index 4d2377a..bc545ab 100644 --- a/tests/parsers/test_item.py +++ b/tests/parsers/test_item.py @@ -1,7 +1,8 @@ import datetime +import os import unittest -from tests import load_resource +from tests import RESOURCES_PATH, load_resource from tibiawikisql.api import Article from tibiawikisql.parsers import ItemParser @@ -339,3 +340,70 @@ def test_usable_ignores_a_trailing_comment(self): item = self._edited("| usable = no", "| usable = yes ") self.assertIs(True, item.is_usable) + + +class TestItemParserClientIds(unittest.TestCase): + def _parse_content(self, title: str, content: str): + article = Article( + article_id=1, + title=title, + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=content, + ) + + return ItemParser.from_article(article) + + def _edited(self, old: str, new: str): + content = load_resource("content_item_amethyst_necklace.txt") + self.assertIn(old, content) + return self._parse_content("Amethyst Necklace", content.replace(old, new)) + + def test_shallow_water_keeps_every_id(self): + item = self._parse_content("Shallow Water", load_resource("content_item_shallow_water.txt")) + + self.assertEqual([629, 630, 631, 632, 633, 634], item.client_ids) + self.assertEqual(item.client_id, item.client_ids[0]) + + def test_single_id(self): + item = self._parse_content("Amethyst Necklace", load_resource("content_item_amethyst_necklace.txt")) + + self.assertEqual([35609], item.client_ids) + self.assertEqual(35609, item.client_id) + + def test_ids_separated_by_comma_and_space(self): + item = self._parse_content("Football", load_resource("content_item_no_attrib.txt")) + + self.assertEqual([2990, 9104], item.client_ids) + + def test_absent(self): + item = self._edited("| itemid = 35609\n", "") + + self.assertEqual([], item.client_ids) + + def test_unknown(self): + item = self._edited("| itemid = 35609", "| itemid = ?") + + self.assertEqual([], item.client_ids) + + def test_only_in_a_comment(self): + item = self._edited("| itemid = 35609", "| itemid = ") + + self.assertEqual([], item.client_ids) + + def test_ignores_a_trailing_comment(self): + item = self._edited("| itemid = 35609", "| itemid = 35609 ") + + self.assertEqual([35609], item.client_ids) + + def test_first_id_matches_client_id_in_every_item_fixture(self): + resources = sorted( + name for name in os.listdir(RESOURCES_PATH) + if name.startswith("content_item") and "{{Infobox Object" in load_resource(name) + ) + self.assertGreaterEqual(len(resources), 19) + for resource in resources: + with self.subTest(resource=resource): + item = self._parse_content(resource, load_resource(resource)) + + self.assertTrue(item.client_ids) + self.assertEqual(item.client_id, item.client_ids[0]) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 6fb20fe..cf6211a 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -5,7 +5,8 @@ import tibiawikisql from tests import load_resource -from tibiawikisql.utils import (clean_links, client_color_to_rgb, parse_boolean, parse_client_id, parse_currency, +from tibiawikisql.utils import (clean_links, client_color_to_rgb, parse_boolean, parse_client_id, parse_client_ids, + parse_currency, parse_first_integer, parse_float, parse_integer, parse_date, parse_loot_statistics, parse_min_max, parse_sounds, parse_known_boolean, parse_known_text, parse_weapon_proficiency_name, @@ -134,6 +135,19 @@ def test_parse_client_id(self): self.assertIsNone(parse_client_id("abc")) self.assertIsNone(parse_client_id("")) + def test_parse_client_ids(self): + self.assertEqual([629, 630, 631, 632, 633, 634], parse_client_ids("629,630,631,632,633,634")) + self.assertEqual([32759, 32760], parse_client_ids(" 32759, 32760")) + self.assertEqual([35], parse_client_ids("35")) + self.assertEqual([35, 36, 37], parse_client_ids("35
36
37")) + self.assertEqual([35, 36], parse_client_ids("35 36")) + self.assertEqual([35, 7], parse_client_ids("357")) + self.assertEqual([644, 645], parse_client_ids("644, 645")) + self.assertEqual([35], parse_client_ids("35 ")) + self.assertEqual([], parse_client_ids("?")) + self.assertEqual([], parse_client_ids("")) + def test_parse_date_ignores_comma_separated_time(self): self.assertEqual(datetime.date(2026, 1, 27), parse_date("January 27, 2026, 16:00")) diff --git a/tibiawikisql/models/item.py b/tibiawikisql/models/item.py index f096bed..29b5712 100644 --- a/tibiawikisql/models/item.py +++ b/tibiawikisql/models/item.py @@ -15,6 +15,7 @@ BookTable, CreatureDropTable, ItemAttributeTable, + ItemClientIdTable, ItemKeyTable, ItemProficiencyPerkTable, ItemSoundTable, @@ -160,6 +161,8 @@ class Item(WikiEntry, WithVersion, WithStatus, WithImage, RowModel, table=ItemTa """The radius of the light emitted by this item, if any.""" client_id: int | None """The internal id of the item in the client.""" + client_ids: list[int] = Field(default_factory=list) + """Every client id of the item, in the order the wiki lists them.""" attributes: list[ItemAttribute] = Field(default_factory=list) """The item's attributes.""" dropped_by: list[ItemDrop] = Field(default_factory=list) @@ -292,6 +295,8 @@ def insert(self, conn: Connection | Cursor) -> None: ItemAttributeTable.insert(conn, item_id=self.article_id, **attribute.model_dump()) for sound in self.sounds: ItemSoundTable.insert(conn, item_id=self.article_id, content=sound) + for position, client_id in enumerate(self.client_ids): + ItemClientIdTable.insert(conn, item_id=self.article_id, client_id=client_id, position=position) for offer in self.store_offers: ItemStoreOfferTable.insert(conn, item_id=self.article_id, **offer.model_dump()) @@ -320,6 +325,9 @@ def get_one_by_field(cls, conn: Connection | Cursor, field: str, value: Any, use sounds = ItemSoundTable.get_list_by_field(conn, "item_id", item.article_id) item.sounds = [r["content"] for r in sounds] + client_ids = ItemClientIdTable.get_list_by_field(conn, "item_id", item.article_id, sort_by="position") + item.client_ids = [r["client_id"] for r in client_ids] + item.bought_by = [ ItemOffer( npc_id=r["npc_id"], diff --git a/tibiawikisql/parsers/item.py b/tibiawikisql/parsers/item.py index d162dcd..13a4b35 100644 --- a/tibiawikisql/parsers/item.py +++ b/tibiawikisql/parsers/item.py @@ -16,6 +16,7 @@ find_templates, number_pattern, parse_boolean, + parse_client_ids, parse_currency, parse_float, parse_first_integer, @@ -163,6 +164,7 @@ def parse_attributes(cls, article: Article) -> dict[str, Any]: cls.parse_resistances(row) cls.parse_restores(row) cls.parse_sounds(row) + cls.parse_client_ids(row) cls.parse_store_value(row) return row @@ -313,6 +315,13 @@ def parse_sounds(cls, row): return row["sounds"] = parse_sounds(row["_raw_attributes"]["sounds"]) + @classmethod + def parse_client_ids(cls, row: dict[str, Any]) -> None: + """Read every client ID in the item's ``itemid`` into ``client_ids``, in listed order.""" + if "itemid" not in row["_raw_attributes"]: + return + row["client_ids"] = parse_client_ids(row["_raw_attributes"]["itemid"]) + @classmethod def parse_store_value(cls, row): if "storevalue" not in row["_raw_attributes"]: diff --git a/tibiawikisql/schema.py b/tibiawikisql/schema.py index 45fdb8b..2c910a8 100644 --- a/tibiawikisql/schema.py +++ b/tibiawikisql/schema.py @@ -163,6 +163,13 @@ class ItemSoundTable(Table, table_name="item_sound"): content = Column(Text, nullable=False) +class ItemClientIdTable(Table, table_name="item_client_id"): + """Contains every client ID of an item, in the order the wiki lists them.""" + item_id = Column(ForeignKey(Integer, table="item", column="article_id"), index=True) + client_id = Column(Integer, nullable=False) + position = Column(Integer, nullable=False) + + class ItemStoreOfferTable(Table, table_name="item_store_offer"): """Contains the Tibia store offers for an item.""" item_id = Column(ForeignKey(Integer, table="item", column="article_id"), index=True) diff --git a/tibiawikisql/utils.py b/tibiawikisql/utils.py index 38c83ef..e736e7b 100644 --- a/tibiawikisql/utils.py +++ b/tibiawikisql/utils.py @@ -24,6 +24,8 @@ min_max_pattern = re.compile(rf"({unsigned_number_rule})-({unsigned_number_rule})") """A range of two numbers without a sign, separated by a hyphen, like ``0-40`` or ``1,000-2,000``.""" int_pattern = re.compile(r"[+-]?\d+") +whole_number_pattern = re.compile(r"\d+") +"""A whole number: a run of digits, without a sign or separators.""" number_pattern = re.compile(rf"[+-]?(?:{unsigned_number_rule})") """A number whose thousands may be grouped with commas, like ``50,000``. Other commas end the number.""" loot_field_separator_pattern = re.compile(r",(?!\d{3}(?!\d))") @@ -385,6 +387,20 @@ def parse_client_id(value: str) -> int | None: return parse_first_integer(visible_text(value), None) +def parse_client_ids(value: str) -> list[int]: + """Parse every client ID in a list of IDs, ignoring wiki comments as :func:`visible_text` does. + + Args: + value: The raw value of a client ID field, like ``629,630,631``. + + Returns: + Every whole number in the value after comments are removed, in listed order. IDs may be separated by commas, + spaces or line breaks, and a sign is not read. The list is empty when there is no whole number, as for ``?``. + + """ + return [int(number) for number in whole_number_pattern.findall(visible_text(value))] + + def parse_known_boolean(value: str) -> bool | None: """Parse a yes or no field, such as whether a creature is pushable, from the visible text of its value. From f57b2667c9fdb603c167d13b884caee09e8daef4 Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 19:47:23 +0000 Subject: [PATCH 05/17] docs: say how item client ids read comments --- CHANGELOG.md | 2 ++ docs/schema.md | 4 ++++ tests/models/test_item.py | 33 +++++++++++++++++++++++++++++++++ tests/parsers/test_item.py | 28 +++++++++++++++++++++++++--- 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f607593..a55c9c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ - Add the `item_client_id` table, with every client ID in an item's `itemid` in listed order, from position 0. Shallow Water's `629,630,631,632,633,634` gives six rows. Wiki comments are ignored, and an `itemid` with no whole number, like `?`, gives no rows. `item.client_id` is unchanged and still holds the first ID. + It keeps its earlier reading, which counts comments, so when `itemid` starts with a comment holding a number, like + `644`, `client_id` is 7 while the rows start at 644. ## 9.0.0+tibiash.9 diff --git a/docs/schema.md b/docs/schema.md index 10a4de3..8012f63 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -377,6 +377,10 @@ that doesn't form a range. | client_id | `INTEGER` | A client ID of the item, a whole number in `itemid` after wiki comments are removed. | | position | `INTEGER` | The ID's place in the `itemid` list, from 0. | +These rows ignore wiki comments, but `item.client_id` keeps its earlier reading, the first integer in the raw value, +comments included. So when `itemid` starts with a comment holding a number, the two differ: `644,645` gives +`client_id` 7 and the rows 644 and 645, and `` gives `client_id` 7 and no rows. + ### item_key diff --git a/tests/models/test_item.py b/tests/models/test_item.py index e781793..df9e6cb 100644 --- a/tests/models/test_item.py +++ b/tests/models/test_item.py @@ -155,3 +155,36 @@ def test_item_client_ids_round_trip_single(self): self.assertEqual([35609], loaded.client_ids) self.assertEqual([(1, 35609, 0)], [tuple(row) for row in rows]) + + def _insert_amethyst_necklace_with_itemid(self, itemid: str) -> None: + content = load_resource("content_item_amethyst_necklace.txt") + self.assertIn("| itemid = 35609", content) + article = Article( + article_id=1, + title="Amethyst Necklace", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=content.replace("| itemid = 35609", f"| itemid = {itemid}"), + ) + ItemParser.from_article(article).insert(self.conn) + + def test_item_client_ids_round_trip_leading_comment(self): + self._insert_amethyst_necklace_with_itemid("644,645") + + loaded = Item.get_one_by_field(self.conn, "article_id", 1) + rows = self.conn.execute( + "SELECT item_id, client_id, position FROM item_client_id ORDER BY position", + ).fetchall() + + self.assertEqual(7, loaded.client_id) + self.assertEqual([644, 645], loaded.client_ids) + self.assertEqual([(1, 644, 0), (1, 645, 1)], [tuple(row) for row in rows]) + + def test_item_client_ids_round_trip_comment_only(self): + self._insert_amethyst_necklace_with_itemid("") + + loaded = Item.get_one_by_field(self.conn, "article_id", 1) + rows = self.conn.execute("SELECT item_id, client_id, position FROM item_client_id").fetchall() + + self.assertEqual(7, loaded.client_id) + self.assertEqual([], loaded.client_ids) + self.assertEqual([], rows) diff --git a/tests/parsers/test_item.py b/tests/parsers/test_item.py index bc545ab..b424e20 100644 --- a/tests/parsers/test_item.py +++ b/tests/parsers/test_item.py @@ -395,15 +395,37 @@ def test_ignores_a_trailing_comment(self): self.assertEqual([35609], item.client_ids) - def test_first_id_matches_client_id_in_every_item_fixture(self): + def test_client_id_reads_a_leading_comment_the_rows_ignore(self): + item = self._edited("| itemid = 35609", "| itemid = 644,645") + + self.assertEqual(7, item.client_id) + self.assertEqual([644, 645], item.client_ids) + + def test_client_id_reads_a_comment_only_value_with_no_rows(self): + item = self._edited("| itemid = 35609", "| itemid = ") + + self.assertEqual(7, item.client_id) + self.assertEqual([], item.client_ids) + + def _item_fixtures(self) -> list[str]: resources = sorted( name for name in os.listdir(RESOURCES_PATH) if name.startswith("content_item") and "{{Infobox Object" in load_resource(name) ) self.assertGreaterEqual(len(resources), 19) - for resource in resources: + return resources + + def test_every_item_fixture_has_a_client_id(self): + for resource in self._item_fixtures(): with self.subTest(resource=resource): item = self._parse_content(resource, load_resource(resource)) self.assertTrue(item.client_ids) - self.assertEqual(item.client_id, item.client_ids[0]) + + def test_first_id_matches_client_id_in_every_item_fixture(self): + for resource in self._item_fixtures(): + with self.subTest(resource=resource): + item = self._parse_content(resource, load_resource(resource)) + + if item.client_ids: + self.assertEqual(item.client_id, item.client_ids[0]) From 3c1d27cb22aabcccb1c4369ced02de662caa3b02 Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 19:55:51 +0000 Subject: [PATCH 06/17] feat: keep loot rarity and sample size --- CHANGELOG.md | 8 ++ README.md | 2 + docs/schema.md | 118 +++++++++++++------------- tests/models/test_creature.py | 44 +++++++++- tests/parsers/test_creature.py | 98 ++++++++++++++++++++- tests/test_generation.py | 105 +++++++++++++++++++++++ tibiawikisql/models/creature.py | 6 ++ tibiawikisql/models/item.py | 2 + tibiawikisql/parsers/creature.py | 35 ++++++-- tibiawikisql/schema.py | 4 + tibiawikisql/tasks/loot_statistics.py | 31 ++++++- 11 files changed, 384 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a55c9c1..ff1dde3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,14 @@ like `?`, gives no rows. `item.client_id` is unchanged and still holds the first ID. It keeps its earlier reading, which counts comments, so when `itemid` starts with a comment holding a number, like `644`, `client_id` is 7 while the rows start at 644. +- Add `creature_drop.rarity` and `creature.loot_statistics_kills`. `rarity` is the rarity word of a `{{Loot Item}}`, + with or without an amount, as in `{{Loot Item|1-3|Gold Coin|common}}` and `{{Loot Item|Gold Coin|common}}`, in lower + case: `always`, `common`, `uncommon`, `semi-rare`, `rare` or `very rare`. Any other word, `?`, an empty or absent + rarity is `NULL`, and wiki comments are ignored. When a Loot Statistics page replaces a drop's rows, the new row keeps + the rarity the old rows share, `NULL` included, and is `NULL` when they disagree, like Court Warlock's crystal coin + listed both `common` and `very rare`, or when the article does not list the item. `loot_statistics_kills` holds the + kills of the page the chances come from, and is `NULL` for a creature without a page or whose page has no positive + number of kills. Loot `min` and `max`, and the number of drop rows, are unchanged. ## 9.0.0+tibiash.9 diff --git a/README.md b/README.md index fe3080e..896a93b 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ The copy adds this data to the database: * The `creature.attack_type` and `creature.uses_spells` columns. They hold how a creature attacks, in lower case, like `melee`, and whether it uses spells, `NULL` when the wiki does not say. * The `item_client_id` table. It keeps every client ID of an item, in the order the wiki lists them, from position 0. +* The `creature_drop.rarity` and `creature.loot_statistics_kills` columns. They hold the rarity a creature's article + gives a drop, like `common`, and the kills on its Loot Statistics page that the drop chances come from. The [database schema](docs/schema.md) describes each of them. diff --git a/docs/schema.md b/docs/schema.md index 8012f63..8b059c0 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -127,57 +127,58 @@ The generated database has the following tables. ### creature -| Column | Type | Description | -| ------------------- | --------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | -| article_id | `INTEGER` / `PRIMARY` | The id of the article containing this creature. | -| title | `TEXT` | The title of the article containing this creature. | -| name | `TEXT` | The name of the creature in-game. | -| plural | `TEXT` | The plural of the name. | -| library_race | `TEXT` | The race name of the creature in Tibia.com's library. | -| article | `TEXT` | The grammatical article before the creature’s name. This is shown when looking at creatures. Bosses have no article. | -| hitpoints | `INTEGER` | The number of hitpoints the creature has. May be `NULL` if unknown. | -| experience | `INTEGER` | The number of experience the creature yields . May be `NULL` if unknown. | -| armor | `INTEGER` | The armor value of the creature. May be `NULL` if unknown. | -| mitigation | `REAL` | The creature's mitigation value. May be `NULL` if unknown. | -| speed | `INTEGER` | The speed value of the creature. May be `NULL` if unknown. | -| creature_class | `TEXT` | The class this creature belongs to (e.g. `Demons`, `Humanoids`, `Mammals`). | -| type_primary | `TEXT` | The type this creature belongs to (e.g. `Archdemons`, `Dwarves`, `Apes`). | -| type_secondary | `TEXT` | A secondary type this creature belongs to, if any. | -| bestiary_class | `TEXT` | The bestiary category of this creature. `NULL` for creatures not in the bestiary. | -| bestiary_level | `TEXT` | The bestiary level of this creature. `NULL` for creatures not in the bestiary. | -| bestiary_occurrence | `TEXT` | The bestiary’s rarity value of this creature. `NULL` for creatures not in the bestiary. | -| bosstiary_class | `TEXT` | The bosstiary category of this creature. `NULL` for creatures not in the bestiary. | -| runs_at | `INTEGER` | The amount of hitpoints when the creature starts to run away. 0 means it won't run away. May be `NULL` if unknown. | -| summon_cost | `INTEGER` | The mana cost to summon this creature. `0` means it cannot be summoned. May be `NULL` if unknown. | -| convince_cost | `INTEGER` | The mana cost to convince this creature. `0` means it cannot be convinced. May be `NULL` if unknown. | -| illusionable | `BOOLEAN` | Whether the player can turn into this creature with Creature Illusion. May be `NULL` if unknown. | -| pushable | `BOOLEAN` | Whether this creature can be pushed or not. May be `NULL` if unknown. | -| push_objects | `BOOLEAN` | Whether this creature can push objects or not. May be `NULL` if unknown. | -| paralysable | `BOOLEAN` | Whether this creature can be paralyzed or not. May be `NULL` if unknown. | -| sees_invisible | `INTEGER` | Whether this creature can see invisible players or not. May be `NULL` if unknown. | -| spawn_type | `TEXT` | The way this creature spawns. | -| attack_type | `TEXT` | How the creature attacks, in lower case (e.g. `melee`). May be `NULL` if unknown. | -| uses_spells | `BOOLEAN` | Whether this creature uses spells or not. May be `NULL` if unknown. | -| is_boss | `INTEGER` | Whether this creature is a boss or not. | -| cooldown | `REAL` | The cooldown in hours to fight the boss again. | -| modifier_physical | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | -| modifier_earth | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | -| modifier_fire | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | -| modifier_ice | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | -| modifier_energy | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | -| modifier_death | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | -| modifier_holy | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | -| modifier_drown | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | -| modifier_lifedrain | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | -| modifier_healing | `INTEGER` | Percentage of healing the creature receives. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | -| walks_through | `TEXT` | The type of fields the creature will walk through. | -| walks_around | `TEXT` | The type of fields the creature will walk around to avoid when possible. | -| location | `TEXT` | The locations where the creature can be found. | -| race_id | `INTEGER` | The client race ID of the creature, the first integer in the value after wiki comments are removed, `NULL` when there is none. | -| version | `TEXT` | The client version this creature was introduced to the game. | -| image | `BLOB` | The creature’s image bytes. | -| status | `TEXT` | The status of the creature in game. | -| timestamp | `TIMESTAMP` | ISO 8601 timestamp of the article's last edit. | +| Column | Type | Description | +| --------------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| article_id | `INTEGER` / `PRIMARY` | The id of the article containing this creature. | +| title | `TEXT` | The title of the article containing this creature. | +| name | `TEXT` | The name of the creature in-game. | +| plural | `TEXT` | The plural of the name. | +| library_race | `TEXT` | The race name of the creature in Tibia.com's library. | +| article | `TEXT` | The grammatical article before the creature’s name. This is shown when looking at creatures. Bosses have no article. | +| hitpoints | `INTEGER` | The number of hitpoints the creature has. May be `NULL` if unknown. | +| experience | `INTEGER` | The number of experience the creature yields . May be `NULL` if unknown. | +| armor | `INTEGER` | The armor value of the creature. May be `NULL` if unknown. | +| mitigation | `REAL` | The creature's mitigation value. May be `NULL` if unknown. | +| speed | `INTEGER` | The speed value of the creature. May be `NULL` if unknown. | +| creature_class | `TEXT` | The class this creature belongs to (e.g. `Demons`, `Humanoids`, `Mammals`). | +| type_primary | `TEXT` | The type this creature belongs to (e.g. `Archdemons`, `Dwarves`, `Apes`). | +| type_secondary | `TEXT` | A secondary type this creature belongs to, if any. | +| bestiary_class | `TEXT` | The bestiary category of this creature. `NULL` for creatures not in the bestiary. | +| bestiary_level | `TEXT` | The bestiary level of this creature. `NULL` for creatures not in the bestiary. | +| bestiary_occurrence | `TEXT` | The bestiary’s rarity value of this creature. `NULL` for creatures not in the bestiary. | +| bosstiary_class | `TEXT` | The bosstiary category of this creature. `NULL` for creatures not in the bestiary. | +| runs_at | `INTEGER` | The amount of hitpoints when the creature starts to run away. 0 means it won't run away. May be `NULL` if unknown. | +| summon_cost | `INTEGER` | The mana cost to summon this creature. `0` means it cannot be summoned. May be `NULL` if unknown. | +| convince_cost | `INTEGER` | The mana cost to convince this creature. `0` means it cannot be convinced. May be `NULL` if unknown. | +| illusionable | `BOOLEAN` | Whether the player can turn into this creature with Creature Illusion. May be `NULL` if unknown. | +| pushable | `BOOLEAN` | Whether this creature can be pushed or not. May be `NULL` if unknown. | +| push_objects | `BOOLEAN` | Whether this creature can push objects or not. May be `NULL` if unknown. | +| paralysable | `BOOLEAN` | Whether this creature can be paralyzed or not. May be `NULL` if unknown. | +| sees_invisible | `INTEGER` | Whether this creature can see invisible players or not. May be `NULL` if unknown. | +| spawn_type | `TEXT` | The way this creature spawns. | +| attack_type | `TEXT` | How the creature attacks, in lower case (e.g. `melee`). May be `NULL` if unknown. | +| uses_spells | `BOOLEAN` | Whether this creature uses spells or not. May be `NULL` if unknown. | +| is_boss | `INTEGER` | Whether this creature is a boss or not. | +| cooldown | `REAL` | The cooldown in hours to fight the boss again. | +| modifier_physical | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | +| modifier_earth | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | +| modifier_fire | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | +| modifier_ice | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | +| modifier_energy | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | +| modifier_death | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | +| modifier_holy | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | +| modifier_drown | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | +| modifier_lifedrain | `INTEGER` | Percentage of damage the creature receives from this damage type. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | +| modifier_healing | `INTEGER` | Percentage of healing the creature receives. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | +| walks_through | `TEXT` | The type of fields the creature will walk through. | +| walks_around | `TEXT` | The type of fields the creature will walk around to avoid when possible. | +| location | `TEXT` | The locations where the creature can be found. | +| race_id | `INTEGER` | The client race ID of the creature, the first integer in the value after wiki comments are removed, `NULL` when there is none. | +| loot_statistics_kills | `INTEGER` | The kills on the creature's Loot Statistics page that its drop chances come from. `NULL` when it has no page, or its page has no positive number of kills. | +| version | `TEXT` | The client version this creature was introduced to the game. | +| image | `BLOB` | The creature’s image bytes. | +| status | `TEXT` | The status of the creature in game. | +| timestamp | `TIMESTAMP` | ISO 8601 timestamp of the article's last edit. | @@ -195,13 +196,14 @@ The generated database has the following tables. ### creature_drop -| Column | Type | Description | -| ----------- | --------- | ------------------------------------------------------ | -| creature_id | `INTEGER` | The id of the creature that yields this drop. | -| item_id | `INTEGER` | The id of the dropped item. | -| chance | `REAL` | The chance percentage of this drop. `NULL` if unknown. | -| min | `INTEGER` | The minimum count of the dropped item. | -| max | `INTEGER` | The maximum count of the dropped item. | +| Column | Type | Description | +| ----------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| creature_id | `INTEGER` | The id of the creature that yields this drop. | +| item_id | `INTEGER` | The id of the dropped item. | +| chance | `REAL` | The chance percentage of this drop. `NULL` if unknown. | +| min | `INTEGER` | The minimum count of the dropped item. | +| max | `INTEGER` | The maximum count of the dropped item. | +| rarity | `TEXT` | The drop's rarity on the creature's article, in lower case: `always`, `common`, `uncommon`, `semi-rare`, `rare` or `very rare`. `NULL` if unknown, or if the article gives the item different rarities. | diff --git a/tests/models/test_creature.py b/tests/models/test_creature.py index 757f884..5887aa2 100644 --- a/tests/models/test_creature.py +++ b/tests/models/test_creature.py @@ -3,7 +3,8 @@ import unittest from tibiawikisql import schema -from tibiawikisql.models import Creature +from tibiawikisql.models import Creature, CreatureDrop, Item +from tibiawikisql.schema import ItemTable class TestCreature(unittest.TestCase): @@ -17,6 +18,8 @@ def _creature( race_id: int | None, attack_type: str | None = None, uses_spells: bool | None = None, + loot_statistics_kills: int | None = None, + loot: list[CreatureDrop] | None = None, ) -> Creature: return Creature( article_id=1, @@ -64,6 +67,8 @@ def _creature( walks_around=None, location="Demona, Edron Demon Forge.", race_id=race_id, + loot_statistics_kills=loot_statistics_kills, + loot=loot or [], version="4.0", status="active", timestamp=datetime.datetime.fromisoformat("2024-07-29T16:37:09+00:00"), @@ -98,3 +103,40 @@ def test_creature_attack_type_and_uses_spells_round_trip_none(self): self.assertIsNone(loaded.attack_type) self.assertIsNone(loaded.uses_spells) + + def test_creature_loot_statistics_kills_round_trip(self): + self._creature(35, loot_statistics_kills=36488).insert(self.conn) + + loaded = Creature.get_one_by_field(self.conn, "article_id", 1) + + self.assertEqual(36488, loaded.loot_statistics_kills) + + def test_creature_loot_statistics_kills_round_trip_none(self): + self._creature(35).insert(self.conn) + + loaded = Creature.get_one_by_field(self.conn, "article_id", 1) + + self.assertIsNone(loaded.loot_statistics_kills) + + def test_creature_drop_rarity_round_trip(self): + timestamp = datetime.datetime.fromisoformat("2024-07-29T16:37:09+00:00") + ItemTable.insert(self.conn, article_id=2, title="Gold Coin", name="Gold Coin", timestamp=timestamp) + ItemTable.insert(self.conn, article_id=3, title="Demon Horn", name="Demon Horn", timestamp=timestamp) + ItemTable.insert(self.conn, article_id=4, title="Talon", name="Talon", timestamp=timestamp) + self._creature(35, loot=[ + CreatureDrop(item_title="Gold Coin", min=0, max=100, rarity="common"), + CreatureDrop(item_id=3, item_title="Demon Horn", min=0, max=1, rarity="uncommon"), + CreatureDrop(item_title="Talon", min=0, max=1), + ]).insert(self.conn) + + loaded = Creature.get_one_by_field(self.conn, "article_id", 1) + gold_coin = Item.get_one_by_field(self.conn, "article_id", 2) + + self.assertEqual( + [("Gold Coin", 0, 100, "common"), ("Demon Horn", 0, 1, "uncommon"), ("Talon", 0, 1, None)], + [(drop.item_title, drop.min, drop.max, drop.rarity) for drop in loaded.loot], + ) + self.assertEqual( + [("Demon", 0, 100, "common")], + [(drop.creature_title, drop.min, drop.max, drop.rarity) for drop in gold_coin.dropped_by], + ) diff --git a/tests/parsers/test_creature.py b/tests/parsers/test_creature.py index ebd7899..723cafc 100644 --- a/tests/parsers/test_creature.py +++ b/tests/parsers/test_creature.py @@ -4,7 +4,7 @@ from tests import load_resource from tibiawikisql.api import Article -from tibiawikisql.models import Creature +from tibiawikisql.models import Creature, CreatureDrop from tibiawikisql.parsers.creature import CreatureParser, parse_abilities, parse_mana_cost, parse_maximum_damage @@ -408,3 +408,99 @@ def test_uses_spells_absent(self): creature = self._edited("| usespells = yes\n", "") self.assertIsNone(creature.uses_spells) + + +class TestCreatureParserLoot(unittest.TestCase): + GOLD_COIN = " |{{Loot Item|1-105|Gold Coin|common}}" + + def _parse(self, content: str) -> Creature: + article = Article( + article_id=1, + title="Dragon", + timestamp=datetime.datetime.fromisoformat("2026-06-25T06:40:57+00:00"), + content=content, + ) + + return CreatureParser.from_article(article) + + def _gold_coin(self, line: str) -> CreatureDrop: + content = load_resource("content_creature_dragon.txt") + self.assertIn(self.GOLD_COIN, content) + creature = self._parse(content.replace(self.GOLD_COIN, line)) + return next(drop for drop in creature.loot if drop.item_title == "Gold Coin") + + def test_rarity_with_amount(self): + drop = self._gold_coin(" |{{Loot Item|1-3|Gold Coin|common}}") + + self.assertEqual("common", drop.rarity) + self.assertEqual((1, 3), (drop.min, drop.max)) + + def test_rarity_without_amount(self): + drop = self._gold_coin(" |{{Loot Item|Gold Coin|common}}") + + self.assertEqual("common", drop.rarity) + self.assertEqual((0, 1), (drop.min, drop.max)) + + def test_rarity_absent(self): + drop = self._gold_coin(" |{{Loot Item|1-3|Gold Coin}}") + + self.assertIsNone(drop.rarity) + self.assertEqual((1, 3), (drop.min, drop.max)) + + def test_rarity_absent_without_amount(self): + drop = self._gold_coin(" |{{Loot Item|Gold Coin}}") + + self.assertIsNone(drop.rarity) + self.assertEqual((0, 1), (drop.min, drop.max)) + + def test_rarity_unknown(self): + drop = self._gold_coin(" |{{Loot Item|1-3|Gold Coin|?}}") + + self.assertIsNone(drop.rarity) + + def test_rarity_typo(self): + drop = self._gold_coin(" |{{Loot Item|1-3|Gold Coin|comon}}") + + self.assertIsNone(drop.rarity) + + def test_rarity_lower_cased_past_a_comment(self): + drop = self._gold_coin(" |{{Loot Item|1-3|Gold Coin|Very Rare }}") + + self.assertEqual("very rare", drop.rarity) + + def test_rarity_only_in_a_comment(self): + drop = self._gold_coin(" |{{Loot Item|1-3|Gold Coin|}}") + + self.assertIsNone(drop.rarity) + + def test_dragon_loot(self): + creature = self._parse(load_resource("content_creature_dragon.txt")) + + # Item, min and max are pinned to what the parser gave before rarity was read. + self.assertEqual( + [ + ("Gold Coin", 1, 105, "common"), + ("Dragon Ham", 1, 3, "common"), + ("Steel Shield", 0, 1, "uncommon"), + ("Crossbow", 0, 1, "uncommon"), + ("Dragon's Tail", 0, 1, "uncommon"), + ("Burst Arrow", 1, 10, "uncommon"), + ("Longsword", 0, 1, "semi-rare"), + ("Steel Helmet", 0, 1, "semi-rare"), + ("Broadsword", 0, 1, "semi-rare"), + ("Plate Legs", 0, 1, "semi-rare"), + ("Green Dragon Leather", 0, 1, "rare"), + ("Wand of Inferno", 0, 1, "rare"), + ("Strong Health Potion", 0, 1, "rare"), + ("Green Dragon Scale", 0, 1, "rare"), + ("Double Axe", 0, 1, "rare"), + ("Dragon Hammer", 0, 1, "rare"), + ("Serpent Sword", 0, 1, "rare"), + ("Small Diamond", 0, 1, "very rare"), + ("Dragon Shield", 0, 1, "very rare"), + ("Life Crystal", 0, 1, "very rare"), + ("Dragonbone Staff", 0, 1, "very rare"), + ("Dragon Trophy", 0, 1, "very rare"), + ], + [(drop.item_title, drop.min, drop.max, drop.rarity) for drop in creature.loot], + ) diff --git a/tests/test_generation.py b/tests/test_generation.py index 563b22a..9306984 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -456,6 +456,111 @@ def test_skips_page_without_kills(self): ) self.assertEqual([self.DRAGON_ROW], rows) + def seed_rows(self, *rows: tuple) -> None: + """Store drops the creatures' articles gave, as (creature_id, item_id, min, max, rarity) rows.""" + self.conn.executemany( + "INSERT INTO creature_drop(creature_id, item_id, min, max, rarity) VALUES(?,?,?,?,?)", + rows, + ) + + def rarities(self) -> list[tuple]: + return self.conn.execute( + "SELECT creature_id, item_id, rarity FROM creature_drop ORDER BY creature_id, item_id, rarity", + ).fetchall() + + def kills(self) -> list[tuple]: + return self.conn.execute( + "SELECT article_id, loot_statistics_kills FROM creature ORDER BY article_id", + ).fetchall() + + def test_keeps_rarity_and_stores_kills(self): + self.seed_rows((11, 20, 0, 105, "common")) + + rows = self.generate(("Dragon", self.DRAGON_PAGE)) + + self.assertEqual([self.DRAGON_ROW], rows) + self.assertEqual([(11, 20, "common")], self.rarities()) + self.assertEqual([(10, None), (11, 200)], self.kills()) + + def test_keeps_rarity_all_rows_agree_on(self): + self.seed_rows((11, 20, 0, 105, "common"), (11, 20, 0, 3, "common")) + + rows = self.generate(("Dragon", self.DRAGON_PAGE)) + + self.assertEqual([self.DRAGON_ROW], rows) + self.assertEqual([(11, 20, "common")], self.rarities()) + + def test_disagreeing_rarities_give_null(self): + # Court Warlock lists its crystal coin both common and very rare. + self.seed_rows((11, 20, 0, 1, "common"), (11, 20, 0, 1, "very rare")) + + rows = self.generate(("Dragon", self.DRAGON_PAGE)) + + self.assertEqual([self.DRAGON_ROW], rows) + self.assertEqual([(11, 20, None)], self.rarities()) + + def test_rarity_and_null_disagree(self): + self.seed_rows((11, 20, 0, 1, None), (11, 20, 0, 1, "common")) + + self.generate(("Dragon", self.DRAGON_PAGE)) + + self.assertEqual([(11, 20, None)], self.rarities()) + + def test_drop_missing_from_the_article_has_null_rarity(self): + rows = self.generate(("Dragon", self.DRAGON_PAGE)) + + self.assertEqual([self.DRAGON_ROW], rows) + self.assertEqual([(11, 20, None)], self.rarities()) + self.assertEqual([(10, None), (11, 200)], self.kills()) + + def test_item_listed_twice_keeps_its_rarity(self): + self.seed_rows((11, 20, 0, 105, "common")) + + rows = self.generate(("Dragon", ("{{Loot2\n|kills=200\n|Gold Coin, times:100, amount:1-50\n" + "|Gold Coin, times:20, amount:51-100\n}}"))) + + self.assertEqual([(11, 20, 10.0, 51, 100), self.DRAGON_ROW], sorted(rows, key=lambda row: row[2])) + self.assertEqual([(11, 20, "common"), (11, 20, "common")], self.rarities()) + + def test_creature_without_page_keeps_null_kills(self): + self.seed_rows((10, 20, 1, 3, "uncommon")) + + self.generate(("Dragon", self.DRAGON_PAGE)) + + self.assertEqual([(10, 20, "uncommon"), (11, 20, None)], self.rarities()) + self.assertEqual([(10, None), (11, 200)], self.kills()) + + def test_skipped_page_keeps_null_kills_and_rarity(self): + self.seed_rows((10, 20, 1, 3, "uncommon")) + + rows = self.generate( + ("Demon", "{{Loot2\n|kills=0\n|Gold Coin, times:5, amount:1-120\n}}"), + ("Dragon", self.DRAGON_PAGE), + ) + + self.assertEqual([(10, 20, None, 1, 3), self.DRAGON_ROW], rows) + self.assertEqual([(10, 20, "uncommon"), (11, 20, None)], self.rarities()) + self.assertEqual([(10, None), (11, 200)], self.kills()) + + def test_page_without_kills_keeps_null_kills(self): + self.generate(("Demon", "{{Loot2\n|Gold Coin, times:5, amount:1-120\n}}")) + + self.assertEqual([(10, None), (11, None)], self.kills()) + + def test_rewritten_pair_min_max_pinned(self): + # The task wrote these exact rows before rarity was kept: min and max come from the page, not the article. + self.seed_rows((10, 20, 0, 100, "common"), (10, 21, 3, 6, "common")) + + rows = self.generate( + ("Demon", ("{{Loot2\n|kills=36,488\n|Gold Coin, times:36,000, amount:1-199\n" + "|Platinum Coin, times:14,511, amount:1,000-2,000\n}}")), + ) + + self.assertEqual( + [(10, 20, 98.6625739969305, 1, 199), (10, 21, 39.7692392019294, 1000, 2000)], + rows, + ) + class TestGenerateCommand(unittest.TestCase): def setUp(self): diff --git a/tibiawikisql/models/creature.py b/tibiawikisql/models/creature.py index 4526a2a..d965738 100644 --- a/tibiawikisql/models/creature.py +++ b/tibiawikisql/models/creature.py @@ -72,6 +72,8 @@ class CreatureDrop(BaseModel): """The maximum possible amount of the dropped item.""" chance: float | None = None """The chance percentage of getting this item dropped by this creature.""" + rarity: str | None = None + """The rarity of the drop, in lower case, like ``common`` or ``very rare``. ``None`` if unknown.""" def insert(self, conn: sqlite3.Connection | sqlite3.Cursor, creature_id: int) -> None: if self.item_id is not None: @@ -87,6 +89,7 @@ def insert(self, conn: sqlite3.Connection | sqlite3.Cursor, creature_id: int) -> "min", "max", "chance", + "rarity", ) .insert( Parameter(":creature_id"), @@ -98,6 +101,7 @@ def insert(self, conn: sqlite3.Connection | sqlite3.Cursor, creature_id: int) -> Parameter(":min"), Parameter(":max"), Parameter(":chance"), + Parameter(":rarity"), ) ) query_str = q.get_sql() @@ -245,6 +249,8 @@ class Creature(WikiEntry, WithStatus, WithVersion, WithImage, RowModel, table=Cr """The locations where the creature can be found.""" race_id: int | None = None """The client race ID of the creature.""" + loot_statistics_kills: int | None = None + """The kills on the creature's Loot Statistics page that its drop chances come from. ``None`` without a page.""" loot: list[CreatureDrop] = Field([]) """The items dropped by this creature.""" diff --git a/tibiawikisql/models/item.py b/tibiawikisql/models/item.py index 29b5712..83fbb67 100644 --- a/tibiawikisql/models/item.py +++ b/tibiawikisql/models/item.py @@ -94,6 +94,8 @@ class ItemDrop(BaseModel): """The maximum possible amount of the dropped item.""" chance: float | None = None """The chance percentage of getting this item dropped by this creature.""" + rarity: str | None = None + """The rarity of the drop, in lower case, like ``common`` or ``very rare``. ``None`` if unknown.""" class ItemQuestReward(BaseModel): """A quest where the item is awarded.""" diff --git a/tibiawikisql/parsers/creature.py b/tibiawikisql/parsers/creature.py index 9ce58de..0a8f1ae 100644 --- a/tibiawikisql/parsers/creature.py +++ b/tibiawikisql/parsers/creature.py @@ -35,6 +35,9 @@ range_hyphen_pattern = re.compile(r"(?<=\d)-(?=\d)") +LOOT_RARITIES = ("always", "common", "uncommon", "semi-rare", "rare", "very rare") +"""The rarity words of the Loot Item template.""" + def parse_maximum_damage(value: str) -> dict[str, int]: """Parse the maximum damage template from TibiaWiki. @@ -142,14 +145,29 @@ def parse_paralysable(value: str) -> bool | None: return parse_boolean(visible_text(value), None, invert=True) -def parse_loot(value: str) -> list[tuple[str, str]]: +def parse_loot_rarity(value: str) -> str | None: + """Parse the rarity of a Loot Item template from the visible text of its value, in lower case. + + Args: + value: The raw value of the rarity parameter. + + Returns: + One of :data:`LOOT_RARITIES`, or ``None`` for any other value, such as ``?`` or a typo. + + """ + rarity = parse_known_text(value) + return rarity if rarity in LOOT_RARITIES else None + + +def parse_loot(value: str) -> list[tuple[str, str, str | None]]: """Get every item drop entry of a creature's drops. Args: value: A string containing item drops. Return: - A list of tuples containing the name of the item and the amount dropped (or empty for 1). + A list of tuples containing the name of the item, the amount dropped (or empty for 1) and the rarity of the + drop (or ``None`` if unknown). """ @@ -162,13 +180,15 @@ def match(k): for item_template in loot_items_templates: param_count = len(item_template.params) if param_count < 2: - loot.append((strip_code(item_template.get(1)), "")) + loot.append((strip_code(item_template.get(1)), "", None)) else: second_param = strip_code(item_template.get(2)) - if second_param in ["always", "common", "uncommon", "semi-rare", "rare", "very rare"]: - loot.append((strip_code(item_template.get(1)), "")) + if second_param in LOOT_RARITIES: + rarity = parse_loot_rarity(str(item_template.get(2).value)) + loot.append((strip_code(item_template.get(1)), "", rarity)) else: - loot.append((second_param, (strip_code(item_template.get(1))))) + rarity = parse_loot_rarity(str(item_template.get(3).value)) if item_template.has(3) else None + loot.append((second_param, (strip_code(item_template.get(1))), rarity)) return loot @@ -327,7 +347,7 @@ def parse_attributes(cls, article: Article) -> dict[str, Any]: if "loot" in raw_attributes: loot = parse_loot(raw_attributes["loot"]) loot_items = [] - for item, amounts in loot: + for item, amounts, rarity in loot: if not amounts: _min, _max = 0, 1 else: @@ -337,6 +357,7 @@ def parse_attributes(cls, article: Article) -> dict[str, Any]: item_title=item, min=_min, max=_max, + rarity=rarity, ), ) row["loot"] = loot_items diff --git a/tibiawikisql/schema.py b/tibiawikisql/schema.py index 2c910a8..4a03f82 100644 --- a/tibiawikisql/schema.py +++ b/tibiawikisql/schema.py @@ -89,6 +89,7 @@ class CreatureTable(Table): walks_around = Column(Text) location = Column(Text) race_id = Column(Integer) + loot_statistics_kills = Column(Integer) version = Column(Text, index=True) image = Column(Blob) status = Column(Text, default="active", nullable=False) @@ -194,6 +195,7 @@ class CreatureDropTable(Table, table_name="creature_drop"): chance = Column(Real) min = Column(Integer, nullable=False) max = Column(Integer, nullable=False) + rarity = Column(Text) @classmethod def get_by_creature_id(cls, conn: Connection | Cursor, creature_id: int): @@ -207,6 +209,7 @@ def get_by_creature_id(cls, conn: Connection | Cursor, creature_id: int): this.min, this.max, this.chance, + this.rarity, ) .join(item).on(this.item_id == item.article_id) ) @@ -224,6 +227,7 @@ def get_by_item_id(cls, conn: Connection | Cursor, item_id: int): this.min, this.max, this.chance, + this.rarity, ) .join(creature).on(this.creature_id == creature.article_id) ) diff --git a/tibiawikisql/tasks/loot_statistics.py b/tibiawikisql/tasks/loot_statistics.py index 484761d..dff81cc 100644 --- a/tibiawikisql/tasks/loot_statistics.py +++ b/tibiawikisql/tasks/loot_statistics.py @@ -12,6 +12,26 @@ import sqlite3 +def article_rarity(cursor: sqlite3.Cursor, creature_id: int, item_id: int) -> str | None: + """Get the rarity the creature's article gives a drop, before its rows are replaced. + + Args: + cursor: A cursor to the database. + creature_id: The article id of the creature. + item_id: The article id of the item. + + Returns: + The rarity of the drop's rows when they all have the same one, ``None`` included, and ``None`` when they + disagree, like a crystal coin listed both ``common`` and ``very rare``, or when there are no rows. + + """ + rows = cursor.execute( + "SELECT DISTINCT rarity FROM creature_drop WHERE creature_id = ? AND item_id = ?", + (creature_id, item_id), + ).fetchall() + return rows[0][0] if len(rows) == 1 else None + + def generate_loot_statistics( conn: sqlite3.Connection, data_store: dict[str, Any], @@ -47,7 +67,12 @@ def generate_loot_statistics( if kills <= 0: # Without a positive number of kills no chance can be computed. continue + cursor.execute( + "UPDATE creature SET loot_statistics_kills = ? WHERE article_id = ?", + (kills, creature_id), + ) rows = [] + rarities: dict[int, str | None] = {} for entry in loot_stats: if not entry: continue @@ -62,13 +87,15 @@ def generate_loot_statistics( amount = entry.get("amount", 1) percentage = min(times / kills * 100, 100) minimum, maximum = parse_min_max(amount) - rows.append((creature_id, item_id, percentage, minimum, maximum)) + if item_id not in rarities: + rarities[item_id] = article_rarity(cursor, creature_id, item_id) + rows.append((creature_id, item_id, percentage, minimum, maximum, rarities[item_id])) cursor.execute( "DELETE FROM creature_drop WHERE creature_id = ? AND item_id = ?", (creature_id, item_id), ) cursor.executemany( - "INSERT INTO creature_drop(creature_id, item_id, chance, min, max) VALUES(?,?,?,?,?)", + "INSERT INTO creature_drop(creature_id, item_id, chance, min, max, rarity) VALUES(?,?,?,?,?,?)", rows, ) if unknown_items: From 20a5b0e56ef3cd42d11bc91f9b35f108bdc27c16 Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 20:12:58 +0000 Subject: [PATCH 07/17] feat: keep creature ability conditions --- CHANGELOG.md | 12 + README.md | 3 + docs/schema.md | 14 +- tests/models/test_creature.py | 25 +- tests/parsers/test_creature.py | 242 +++++++++++++++++- .../resources/content_creature_abyssador.txt | 71 +++++ .../resources/content_creature_apocalypse.txt | 69 +++++ .../resources/content_creature_dreadbeast.txt | 64 +++++ .../content_creature_gnorre_chyllson.txt | 65 +++++ .../content_creature_green_djinn.txt | 83 ++++++ tests/resources/content_creature_mimic.txt | 48 ++++ .../content_creature_son_of_verminor.txt | 58 +++++ tibiawikisql/models/creature.py | 9 +- tibiawikisql/parsers/creature.py | 143 ++++++++++- tibiawikisql/schema.py | 2 + 15 files changed, 898 insertions(+), 10 deletions(-) create mode 100644 tests/resources/content_creature_abyssador.txt create mode 100644 tests/resources/content_creature_apocalypse.txt create mode 100644 tests/resources/content_creature_dreadbeast.txt create mode 100644 tests/resources/content_creature_gnorre_chyllson.txt create mode 100644 tests/resources/content_creature_green_djinn.txt create mode 100644 tests/resources/content_creature_mimic.txt create mode 100644 tests/resources/content_creature_son_of_verminor.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index ff1dde3..03f8197 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,18 @@ listed both `common` and `very rare`, or when the article does not list the item. `loot_statistics_kills` holds the kills of the page the chances come from, and is `NULL` for a creature without a page or whose page has no positive number of kills. Loot `min` and `max`, and the number of drop rows, are unchanged. +- Add `creature_ability.condition` and `creature_ability.condition_value`, and keep the `{{Haste}}`, `{{Debuff}}` and + `{{Outfit}}` abilities, which were dropped. `condition` is the `condition` of a `{{Melee}}` or `{{Ability}}` in lower + case, like `poisoned` or `slowed`, and `NULL` for `?`, an empty or absent value. A `{{Melee}}` with `poison=` is + `poisoned`, with the number in `condition_value`, which is `NULL` for `poison=?` and on every other row. Wiki + comments are ignored. A Haste row has the element `haste`, its spell as the name, like `Strong Haste`, or `Haste`, + and an empty effect. A Debuff row has the element `debuff`, its name or `Debuff`, and the effect + `value=; duration=; flags=`, like `value=50%; duration=10; flags=distance,melee`: the values + are copied as written, the flags set to `yes` are listed in the order `distance`, `melee`, `magic`, `shielding`, and + an empty part is left out. Flags given without a name, as in `{{Debuff|Debuff|yes}}`, are not read. An Outfit gives + one row per outfit, from parameter 2 or `thing`, then parameters 4 and up or `thing2` and up, with the element + `outfit` and the effect `yes` when it changes the victim and `no` when it changes the creature. Template names must + match exactly, as before, and the existing rows keep their values. ## 9.0.0+tibiash.9 diff --git a/README.md b/README.md index 896a93b..2c2871d 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,9 @@ The copy adds this data to the database: * The `item_client_id` table. It keeps every client ID of an item, in the order the wiki lists them, from position 0. * The `creature_drop.rarity` and `creature.loot_statistics_kills` columns. They hold the rarity a creature's article gives a drop, like `common`, and the kills on its Loot Statistics page that the drop chances come from. +* The `creature_ability.condition` and `creature_ability.condition_value` columns, and the `haste`, `debuff` and + `outfit` abilities. The columns hold the condition an ability causes, like `poisoned`, and the poison damage a melee + attack starts at. The [database schema](docs/schema.md) describes each of them. diff --git a/docs/schema.md b/docs/schema.md index 8b059c0..0fe9d20 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -185,12 +185,14 @@ The generated database has the following tables. ### creature_ability -| Column | Type | Description | -| ----------- | --------- | ----------------------------------------------------------------------------------------- | -| creature_id | `INTEGER` | The id of the creature that does this ability. | -| name | `TEXT` | The name of the ability. | -| effect | `TEXT` | The effect of the ability, or the damage range. | -| element | `TEXT` | The element of damage type of the ability. This could also be a status condition instead. | +| Column | Type | Description | +| --------------- | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| creature_id | `INTEGER` | The id of the creature that does this ability. | +| name | `TEXT` | The name of the ability. | +| effect | `TEXT` | The effect of the ability, or the damage range. For a `debuff`, the reduction as `value=; duration=; flags=`, without the empty parts, like `value=50%; duration=10; flags=distance,melee`. The flags are the reduced skills, in the order `distance`, `melee`, `magic`, `shielding`. For an `outfit`, `yes` if it changes the victim's outfit and `no` if it changes the creature's own. Empty for a `haste`. | +| element | `TEXT` | The element of damage type of the ability. This could also be a status condition instead. Haste, debuff and outfit change abilities are `haste`, `debuff` and `outfit`, and an outfit change has one row per outfit. | +| condition | `TEXT` | The condition the ability causes, in lower case, like `poisoned` or `slowed`. A melee attack that poisons is `poisoned`. `NULL` if unknown or none. | +| condition_value | `INTEGER` | The poison damage per turn a melee attack starts at, up to. `NULL` if unknown or not a poisoning melee attack. | diff --git a/tests/models/test_creature.py b/tests/models/test_creature.py index 5887aa2..fe4c239 100644 --- a/tests/models/test_creature.py +++ b/tests/models/test_creature.py @@ -3,7 +3,7 @@ import unittest from tibiawikisql import schema -from tibiawikisql.models import Creature, CreatureDrop, Item +from tibiawikisql.models import Creature, CreatureAbility, CreatureDrop, Item from tibiawikisql.schema import ItemTable @@ -140,3 +140,26 @@ def test_creature_drop_rarity_round_trip(self): [("Demon", 0, 100, "common")], [(drop.creature_title, drop.min, drop.max, drop.rarity) for drop in gold_coin.dropped_by], ) + + def test_creature_ability_condition_round_trip(self): + creature = self._creature(35) + creature.abilities = [ + CreatureAbility( + name="Melee", effect="0-3098", element="physical", condition="poisoned", condition_value=25, + ), + CreatureAbility(name="Debuff", effect="value=80-100%; duration=10", element="debuff"), + ] + creature.insert(self.conn) + + loaded = Creature.get_one_by_field(self.conn, "article_id", 1) + + self.assertEqual( + [ + ("Melee", "0-3098", "physical", "poisoned", 25), + ("Debuff", "value=80-100%; duration=10", "debuff", None, None), + ], + [ + (ability.name, ability.effect, ability.element, ability.condition, ability.condition_value) + for ability in loaded.abilities + ], + ) diff --git a/tests/parsers/test_creature.py b/tests/parsers/test_creature.py index 723cafc..0b382a5 100644 --- a/tests/parsers/test_creature.py +++ b/tests/parsers/test_creature.py @@ -5,7 +5,13 @@ from tests import load_resource from tibiawikisql.api import Article from tibiawikisql.models import Creature, CreatureDrop -from tibiawikisql.parsers.creature import CreatureParser, parse_abilities, parse_mana_cost, parse_maximum_damage +from tibiawikisql.parsers.creature import ( + CreatureParser, + format_debuff_effect, + parse_abilities, + parse_mana_cost, + parse_maximum_damage, +) class TestCreatureParser(unittest.TestCase): @@ -504,3 +510,237 @@ def test_dragon_loot(self): ], [(drop.item_title, drop.min, drop.max, drop.rarity) for drop in creature.loot], ) + + +class TestCreatureParserAbilities(unittest.TestCase): + def _abilities(self, resource: str) -> list[tuple]: + article = Article( + article_id=1, + title=resource, + timestamp=datetime.datetime.fromisoformat("2026-10-30T00:00:00+00:00"), + content=load_resource(resource), + ) + creature = CreatureParser.from_article(article) + return [ + (ability.name, ability.effect, ability.element, ability.condition, ability.condition_value) + for ability in creature.abilities + ] + + def _rows(self, content: str) -> list[tuple]: + return [ + (row["name"], row.get("effect"), row["element"], row.get("condition"), row.get("condition_value")) + for row in parse_abilities(content) + ] + + def test_debuff_effect(self): + self.assertEqual("value=50%; duration=10; flags=distance,melee", + format_debuff_effect("50%", "10", {"melee", "distance"})) + + def test_debuff_effect_orders_every_flag(self): + self.assertEqual("flags=distance,melee,magic,shielding", + format_debuff_effect("", "", ["shielding", "magic", "melee", "distance"])) + + def test_debuff_effect_without_flags(self): + self.assertEqual("value=50%; duration=10", format_debuff_effect("50%", "10", ())) + + def test_debuff_effect_without_value(self): + self.assertEqual("duration=10; flags=melee", format_debuff_effect("", "10", {"melee"})) + + def test_debuff_effect_only_duration(self): + self.assertEqual("duration=10", format_debuff_effect("", "10", ())) + + def test_debuff_effect_empty(self): + self.assertEqual("", format_debuff_effect("", "", ())) + + def test_unchanged_without_new_kinds(self): + # Pinned to what the parser gave before conditions were read: the new columns are the only change. + self.assertEqual( + [ + ("Melee", "0-500", "physical", None, None), + ("Great Fireball", "150-250", "fire", None, None), + ("Great Energy Beam", "300-480", "lifedrain", None, None), + ("Close-range Energy Strike", "210-300", "energy", None, None), + ("Mana Drain", "30-120", "manadrain", None, None), + ("Self-Healing", "80-250", "healing", None, None), + ("Shoots Fire Fields", "", "fire", None, None), + ("Distance Paralyze", "", "paralyze", None, None), + ("Fire Elemental", "1", "summon", None, None), + ], + self._abilities("content_creature.txt"), + ) + + def test_apocalypse(self): + self.assertEqual( + [ + ("Melee", "0-3098", "physical", "poisoned", 25), + ("Armageddon", "450-4000?", "death", None, None), + ("Great fireball", "100-900", "fire", None, None), + ("Teleport Wave", "600-1100", "mana drain", None, None), + ("Explosion Wave", "300-900", "physical", None, None), + ("Explosion Exori", "100?-500?", "energy", None, None), + ("Yellow Energy Beam", "500-1500", "energy", None, None), + ("Death Beam", "300?-1000?", "death", None, None), + ("Poisoning Beam", "up to 100 hp/turn", "physical", "poisoned", None), + ("Paralyze", "210-220 points for 60 seconds", "physical", "slowed", None), + # The unnamed yes after the name is not a flag. + ("Debuff", "value=80-100%; duration=10", "debuff", None, None), + ("Strong Haste", "", "haste", None, None), + ("Weak Self-Healing", "2000-3000", "healing", None, None), + ("Strong Self-Healing", "6000-10000", "healing", None, None), + ], + self._abilities("content_creature_apocalypse.txt"), + ) + + def test_gnorre_chyllson_debuff_flags_in_fixed_order(self): + abilities = self._abilities("content_creature_gnorre_chyllson.txt") + + self.assertIn(("Debuff", "value=34-54%; duration=4; flags=distance,melee,shielding", "debuff", None, None), + abilities) + self.assertIn(("Strong Haste", "", "haste", None, None), abilities) + self.assertIn(("Paralyze", "?", "", "slowed", None), abilities) + self.assertEqual(11, len(abilities)) + + def test_abyssador_debuffs_differ_by_flags(self): + abilities = self._abilities("content_creature_abyssador.txt") + + self.assertEqual( + [("Debuff", "flags=distance", "debuff", None, None), ("Debuff", "flags=melee", "debuff", None, None)], + [ability for ability in abilities if ability[2] == "debuff"], + ) + self.assertIn(("Poisoning Bomb", "?", "physical", "poisoned", None), abilities) + self.assertIn(("Melee", "0-1400+", "physical", None, None), abilities) + self.assertEqual(13, len(abilities)) + + def test_dreadbeast_unknown_poison(self): + abilities = self._abilities("content_creature_dreadbeast.txt") + + self.assertEqual( + [ + ("Melee", "0-50", "physical", "poisoned", None), + ("Drown Hit", "70-90", "drowning", None, None), + ("Life Drain Berserk", "0-80", "life drain", None, None), + ("Poisoning Berserk", "up to 6?hp/turn", "earth", "poisoned", None), + ("Debuff", "value=20-30%; duration=20secs; flags=melee", "debuff", None, None), + ("Self-Healing", "35-65", "healing", None, None), + ], + abilities, + ) + + def test_son_of_verminor_outfits(self): + abilities = self._abilities("content_creature_son_of_verminor.txt") + + self.assertEqual( + [ + ("Rat", "no", "outfit", None, None), + ("Larva", "no", "outfit", None, None), + ("Scorpion", "no", "outfit", None, None), + ("Slime", "no", "outfit", None, None), + ], + [ability for ability in abilities if ability[2] == "outfit"], + ) + # The template reads poison=, not poisoned=. + self.assertIn(("Melee", "0-460", "physical", None, None), abilities) + self.assertEqual(9, len(abilities)) + + def test_mimic_outfits(self): + self.assertEqual( + [ + ("Teddy Bear", "no", "outfit", None, None), + ("Demon Shield", "no", "outfit", None, None), + ("Warlord Sword", "no", "outfit", None, None), + ("Piggy Bank", "no", "outfit", None, None), + ("Green Frog", "no", "outfit", None, None), + ("Treasure Chest", "no", "outfit", None, None), + ("Rat", "no", "outfit", None, None), + ("Slime", "no", "outfit", None, None), + ("Pumpkin", "no", "outfit", None, None), + ("Jewel Case", "no", "outfit", None, None), + ("Giant Mushroom", "no", "outfit", None, None), + ("Present Box", "no", "outfit", None, None), + ("The Orb of Nature", "no", "outfit", None, None), + ("Vashresamun", "no", "outfit", None, None), + ("Turns Invisible", "very rare 1s", "invisible", None, None), + ], + self._abilities("content_creature_mimic.txt"), + ) + + def test_green_djinn_outfit_scene_is_not_a_name(self): + abilities = self._abilities("content_creature_green_djinn.txt") + + self.assertEqual([("rat", "yes", "outfit", None, None)], + [ability for ability in abilities if ability[2] == "outfit"]) + self.assertIn(("Drunkenness", "30 seconds", "drunk", None, None), abilities) + + def test_outfit_named_parameters(self): + rows = self._rows("{{Ability List|{{Outfit|victim=Yes|thing=Massive Fire Elemental|scene=|thing2=Football" + "|thing3=|thing4=Snowman}}}}") + + self.assertEqual( + [ + ("Massive Fire Elemental", "yes", "outfit", None, None), + ("Football", "yes", "outfit", None, None), + ("Snowman", "yes", "outfit", None, None), + ], + rows, + ) + + def test_outfit_positional_extra_wins_over_named(self): + rows = self._rows("{{Ability List|{{Outfit|no|Rat||Larva|thing2=Bug}}}}") + + self.assertEqual([("Rat", "no", "outfit", None, None), ("Larva", "no", "outfit", None, None)], rows) + + def test_haste_without_name(self): + self.assertEqual([("Haste", "", "haste", None, None)], self._rows("{{Ability List|{{Haste}}}}")) + + def test_haste_named_name(self): + self.assertEqual([("Strong Haste", "", "haste", None, None)], + self._rows("{{Ability List|{{Haste|name=[[Strong Haste]]}}}}")) + + def test_template_name_matches_exactly(self): + self.assertEqual([], self._rows("{{Ability List|{{Haste\n |scene=}}|{{haste}}}}")) + + def test_debuff_named_name(self): + self.assertEqual([("Icicle", "value=50%; flags=melee,shielding", "debuff", None, None)], + self._rows("{{Ability List|{{Debuff|Icicle|shielding=yes|melee=yes|value=50%}}}}")) + + def test_debuff_without_parameters(self): + self.assertEqual([("Debuff", "", "debuff", None, None)], self._rows("{{Ability List|{{Debuff}}}}")) + + def test_debuff_flag_not_yes(self): + self.assertEqual([("Debuff", "flags=magic", "debuff", None, None)], + self._rows("{{Ability List|{{Debuff|melee=no|distance=|magic=Yes }}}}")) + + def test_condition_unknown(self): + self.assertEqual([("Bomb", "?", "physical", None, None)], + self._rows("{{Ability List|{{Ability|Bomb|condition=?}}}}")) + + def test_condition_ignores_comments_and_case(self): + self.assertEqual([("Bomb", "?", "physical", "burning", None)], + self._rows("{{Ability List|{{Ability|Bomb|condition=Burning }}}}")) + + def test_melee_condition(self): + self.assertEqual([("Melee", "0-50", "physical", "bleeding", None)], + self._rows("{{Ability List|{{Melee|0-50|condition=bleeding}}}}")) + + def test_melee_empty_poison(self): + self.assertEqual([("Melee", "0-50", "physical", None, None)], + self._rows("{{Ability List|{{Melee|0-50|poison= }}}}")) + + def test_healing_and_summon_have_no_condition(self): + self.assertEqual( + [("Self-Healing", "10-20", "healing", None, None), ("Rat", "2", "summon", None, None)], + self._rows("{{Ability List|{{Healing|range=10-20|condition=x}}|{{Summon|Rat|2|condition=x}}}}"), + ) + + def test_no_template_and_plain_text_have_no_condition(self): + self.assertEqual([("Bites", None, "no_template", None, None)], self._rows("Bites")) + self.assertEqual([("Melee", "0-5", "physical", None, None), ("Bites", None, "plain_text", None, None)], + self._rows("{{Ability List|{{Melee|0-5}}|Bites}}")) + + def test_debuff_effect_rejects_an_unknown_flag(self): + with self.assertRaises(ValueError): + format_debuff_effect("50%", "10", {"melee", "speed"}) + + def test_outfit_named_victim_and_thing_win_over_positional(self): + self.assertEqual([("Bug", "yes", "outfit", None, None)], + self._rows("{{Ability List|{{Outfit|no|Rat|victim=yes|thing=Bug}}}}")) diff --git a/tests/resources/content_creature_abyssador.txt b/tests/resources/content_creature_abyssador.txt new file mode 100644 index 0000000..e728e71 --- /dev/null +++ b/tests/resources/content_creature_abyssador.txt @@ -0,0 +1,71 @@ +{{Infobox Creature|List={{{1|}}}|GetValue={{{GetValue|}}} +| name = Abyssador +| actualname = Abyssador +| hp = 340000 +| exp = 400000 +| armor = ? +| summon = -- +| convince = -- +| illusionable = no +| creatureclass = Elementals +| primarytype = Bio-Elementals +| secondarytype = Myriapods +| spawntype = Unique, Triggered, Unblockable +| isboss = yes +| isarenaboss = no +| abilities = {{Ability List + |{{Melee|0-1400+}} + |{{Ability|[[Invisible]]||invisible}} + |{{Ability|Poisoning Bomb|condition=poisoned}} + |{{Ability|Earth Wave|500-1100+|earth}} + |{{Ability|Earth Bomb around itself|300-700+|earth}} + |{{Ability|Distance Explosion Attack|300-600+|physical}} + |{{Ability|Stone Shower Beam|500-1200+}} + |{{Ability|Death Bomb|?|mana drain}} + |{{Healing|Ultimate [[Self-Healing]]|range=50000-300000, rare}} + |{{Healing|Medium [[Self-Healing]]|30000-40000, semi-rarely}} + |{{Healing|Light [[Self-Healing]]|3000-10000, often}} + |{{Debuff|distance=yes}} + |{{Debuff|melee=yes}} +}} +| maxdmg = {{Max Damage|physical=2000+|earth=3000+|manadrain=?}} +| pushable = no +| pushobjects = yes +| walksaround = ? +| walksthrough = ? +| paraimmune = yes +| senseinvis = yes +| physicalDmgMod = 100%'''[[Mage]]s''' should use [[Sudden Death Rune]]s and summon [[Fire Elemental]]s or [[Fire Devil]]s to deal more damage.
'''[[Paladin]]s''' should use [[Infernal Bolt]]s and [[Sharpshooter]] but have to be wary of the distance attacks. +| location = [[Warzone 3]]. +| loot = {{Loot Table + |{{Loot Item|Abyssador's Lash|always}} + |{{Loot Item|Crystal Crossbow}} + |{{Loot Item|Crystalline Axe}} + |{{Loot Item|Crystalline Sword}} + |{{Loot Item|Decorative Ribbon}} + |{{Loot Item|Mycological Bow}} + |{{Loot Item|Mycological Mace}} + |{{Loot Item|Shiny Blade}} +}} +| bosstiaryclass = Archfoe +| cooldown = 20 +| history = * As announced on {{OfficialNewsArchive|7858|April 23, 2024}}, the [[Updates/13.35.14931|patch]] increased its experience from 50,000 to 400,000 as part of the [[2024 Balancing Project]]. +* As announced on {{OfficialNewsArchive|7890|May 14, 2024}}, the [[Updates/13.36.d8d851|patch]] its loot is now cooperative as part of the [[2024 Balancing Project]]. +| bestiaryname = abyssador +}} \ No newline at end of file diff --git a/tests/resources/content_creature_apocalypse.txt b/tests/resources/content_creature_apocalypse.txt new file mode 100644 index 0000000..7e93d83 --- /dev/null +++ b/tests/resources/content_creature_apocalypse.txt @@ -0,0 +1,69 @@ +{{Infobox Creature|List={{{1|}}}|GetValue={{{GetValue|}}} +| name = Apocalypse +| actualname = Apocalypse +| plural = Apocalypse +| hp = 125000 +| exp = 30000 +| armor = 188 +| summon = -- +| convince = -- +| illusionable = no +| creatureclass = Demons +| primarytype = The Ruthless Seven +| secondarytype = Archdemons +| attacktype = Melee +| usespells = yes +| isboss = yes +| abilities = {{Ability List + |{{Melee|0-3098|poison=25}} + |{{Ability|[[Sudden_Death_Explosion|Armageddon]]|450-4000?|death + |scene={{Scene|spell=6sqmballself|effect=Death Effect|caster=Apocalypse|look_direction=|effect_on_caster=yes|effect_on_target=Cloud Effect}}}} + |{{Ability|Great fireball|100-900|fire|scene={{Scene|spell=5sqmballtarget|effect=Fireball Effect|caster=Apocalypse|missile=Fire Missile|missile_direction=south-east|missile_distance=5/5|effect_on_target=Fireball Effect}}}} + |{{Ability|Teleport Wave|600-1100|element=mana drain|scene={{Scene|spell=8sqmwave|caster=Apocalypse|look_direction=east|effect=Teleport Effect|effect_on_target=Red Sparkles Effect}}}} + |{{Ability|Explosion Wave|300-900|physical|scene={{Scene|spell=8sqmwave|caster=Apocalypse|look_direction=east|effect=Explosion Effect}}}} + |{{Ability|Explosion Exori|100?-500?|element=energy|scene={{Scene|spell=3x3spell|effect=Explosion Effect|caster=Apocalypse|effect_on_caster=yes|effect_on_target=Blue Electricity Effect}}}} + |{{Ability|Yellow Energy Beam|500-1500|energy|scene={{Scene|spell=8sqmbeam|effect=Yellow Electricity Effect|caster=Apocalypse|look_direction=east|effect_on_target=Blue Electricity Effect}}}} + |{{Ability|Death Beam|300?-1000?|element=death|scene={{Scene|spell=8sqmbeam|effect=Death Effect|caster=Apocalypse|look_direction=east}}}} + |{{Ability|Poisoning Beam|up to 100 hp/turn|condition=poisoned|scene={{Scene|spell=8sqmbeam|effect=Poison Effect 1|caster=Apocalypse|look_direction=east}}}} + |{{Ability|Paralyze|210-220 points for 60 seconds|condition=slowed|scene={{Scene|spell=4sqmball2self|effect=Poison Effect 3|caster=Apocalypse|effect_on_caster=yes}}}} + |{{Debuff|Debuff|yes|value=80-100%|duration=10 + |scene={{Scene|spell=3x3spell|effect=Trembling Effect|caster=Apocalypse|effect_on_caster=yes}}}} + |{{Haste|[[Strong Haste]]|scene={{Scene|spell=buffspell|effect=Red Sparkles Effect|caster=Apocalypse|effect_on_caster=yes}}}} + |{{Healing|[[Self-Healing|Weak Self-Healing]]|2000-3000|scene={{Scene|spell=buffspell|effect=Blue Sparkles Effect|caster=Apocalypse|effect_on_caster=yes}}}} + |{{Healing|[[Self-Healing|Strong Self-Healing]]|6000-10000|scene={{Scene|spell=buffspell|effect=Blue Sparkles Effect|caster=Apocalypse|effect_on_caster=yes}}}} +}} +| maxdmg = 12000? +| pushable = no +| pushobjects = yes +| walksaround = ? +| walksthrough = ? +| paraimmune = yes +| senseinvis = yes +| physicalDmgMod = 100%? +| earthDmgMod = 0% +| fireDmgMod = 0% +| deathDmgMod = 99%? +| energyDmgMod = 0% +| holyDmgMod = 100%? +| iceDmgMod = 90%? +| hpDrainDmgMod = 0% +| drownDmgMod = 100%? +| healMod = 100%? +| sounds = {{Sound List|BOW TO THE POWER OF THE RUTHLESS SEVEN!|DESTRUCTION!|CHAOS!|DEATH TO ALL!}} +| implemented = -- +| race_id = 234 +| notes = He was once the strongest monster in Tibia, but this is no longer the case. He is a Member of [[The Ruthless Seven]], The Total Destruction. Apocalypse's throne is in the [[Pits of Inferno]]. He was [[summon]]ed three times by CipSoft members on Test Servers (May 2006, November 2009 and July 2012). The last time it appeared it was slain, but unfortunately dropped no loot. + +Apocalypse2009.jpg|Apocalypse and its new look, seen in the [[Test Server]] of the [[Updates/8.53|Autumn Patch 2009]]. +Apocalypse Summer 2006.gif|Apocalypse from the private test server of the technical summer 2006 update. +Apocalypse1.JPG|[[Apocalypse]], seen in [[Test Server]] (May 2006). +Apocalypse SD Storm.jpg|Apocalypse hitting 2600 with [[Armageddon]] spell. + +| behaviour = +| runsat = 3000 +| speed = 350 +| strategy = A couple of stronger blockers ([[level]] 300+) using [[Stone Skin Amulet]] and [[Might Ring]] and a large group of [[mages]] shooting [[SD]]s... or just run for your life. +| location = Unknown. +| loot = {{Loot Table}} +| status = TS-only +}} \ No newline at end of file diff --git a/tests/resources/content_creature_dreadbeast.txt b/tests/resources/content_creature_dreadbeast.txt new file mode 100644 index 0000000..2eaf6c2 --- /dev/null +++ b/tests/resources/content_creature_dreadbeast.txt @@ -0,0 +1,64 @@ +{{Infobox Creature|List={{{1|}}}|GetValue={{{GetValue|}}} +| name = Dreadbeast +| article = a +| actualname = dreadbeast +| plural = dreadbeasts +| hp = 800 +| exp = 250 +| armor = ? +| summon = -- +| convince = 800 +| illusionable = no +| creatureclass = The Undead +| primarytype = Skeletons +| secondarytype = Ungulates +| isboss = no +| abilities = {{Ability List + |{{Melee|0-50|poison=?}} + |{{Ability|Drown Hit|70-90|element=drowning|scene={{Scene|spell=7sqmstrike|caster=Dreadbeast|look_direction=south|effect=Purple Electricity Effect|damage_effect=Ripple Effect|missile=Death Missile (Large)|missile_direction=south-east|missile_distance=7/7}}}} + |{{Ability|Life Drain Berserk|0-80|element=life drain + |scene={{Scene|spell=3x3spell|caster=Dreadbeast|look_direction=south|effect=Red Sparkles Effect|effect_on_caster=yes}}}} + |{{Ability|Poisoning Berserk|up to 6?hp/turn|element=earth|condition=poisoned|scene={{Scene|spell=3x3spell|caster=Dreadbeast|look_direction=south|effect=Poison Effect 3|effect_on_caster=yes}}}} + |{{Debuff|melee=yes|value=20-30%|duration=20secs + |scene={{Scene|spell=1sqmstrike|caster=Dreadbeast|look_direction=south|effect=Yellow Electricity Effect}}}} + |{{Healing|range=35-65 + |scene={{Scene|spell=buffspell|caster=Dreadbeast|look_direction=south|effect=Poison Effect 2|effect_on_caster=yes}}}} +}} +| maxdmg = 140 +| pushable = ? +| pushobjects = ? +| walksaround = Fire, ? +| walksthrough = ? +| paraimmune = ? +| senseinvis = ? +| physicalDmgMod = 70% +| earthDmgMod = 0% +| fireDmgMod = 45% +| deathDmgMod = 0% +| energyDmgMod = 85% +| holyDmgMod = 150% +| iceDmgMod = 60% +| hpDrainDmgMod = 0% +| drownDmgMod = 25% +| healMod = 100%? +| sounds = {{Sound List|}} +| implemented = 8.20 +| race_id = 404 +| notes = They are constantly summoned in [[Hellgorak|Hellgorak's]] Room. +| behaviour = Try to hit you with their [[Drown Damage|Drown Hits]]. +| speed = 68 +| strategy = Convince it and use it to kill the boss of the room at [[The Inquisition Quest]]. +| location = [[The Inquisition Quest]], [[Hellgorak|Hellgorak's]] Room. +| loot = {{Loot Table + |{{Loot Item|0-88|Gold Coin}} + |{{Loot Item|Bone}} + |{{Loot Item|Plate Armor|semi-rare}} + |{{Loot Item|Skull (Item)|semi-rare}} + |{{Loot Item|Big Bone|rare}} + |{{Loot Item|Bone Club|rare}} + |{{Loot Item|Bone Shield|rare}} + |{{Loot Item|Green Mushroom|rare}} + |{{Loot Item|Hardened Bone|rare}} + |{{Loot Item|Health Potion|rare}} +}} +}} \ No newline at end of file diff --git a/tests/resources/content_creature_gnorre_chyllson.txt b/tests/resources/content_creature_gnorre_chyllson.txt new file mode 100644 index 0000000..31a22cb --- /dev/null +++ b/tests/resources/content_creature_gnorre_chyllson.txt @@ -0,0 +1,65 @@ +{{Infobox Creature|List={{{1|}}}|GetValue={{{GetValue|}}} +| name = Gnorre Chyllson +| actualname = Gnorre Chyllson +| hp = 7150 +| exp = 4000 +| armor = ? +| summon = -- +| convince = -- +| illusionable = no +| creatureclass = Humans +| primarytype = Outlaws +| spawntype = Unique, Triggered +| isboss = yes +| isarenaboss = yes +| abilities = {{Ability List + |{{Melee|0-450+}} + |{{Ability|Musical Notes Beam|121-257|element=life drain + |scene={{Scene|spell=9sqmbeam|caster=Gnorre Chyllson|look_direction=east|effect=Blue Notes Effect|damage_effect=Red Sparkles Effect}}}} + |{{Ability|Berserk|65-191|element=life drain + |scene={{Scene|spell=3x3spell|caster=Gnorre Chyllson|effect=Trembling Effect|damage_effect=Red Sparkles Effect|effect_on_caster=yes}}}} + |{{Ability|Smoke Wave|? turns|element=drowning + |scene={{Scene|spell=7sqmwave|caster=Gnorre Chyllson|look_direction=east|effect=Poof Effect|damage_effect=None}}}} + |{{Ability|Hot Water Storm|? turns|element=drowning + |scene={{Scene|spell=2sqmballself|caster=Gnorre Chyllson|effect=Ripple Effect|damage_effect=Unknown|effect_on_caster=yes}}}} +|{{Debuff|melee=yes|distance=yes|shielding=yes|value=34-54%|duration=4 + |scene={{Scene|spell=3sqmball2self|caster=Gnorre Chyllson|effect=Poof Effect|damage_effect=None|effect_on_caster=yes}}}} + |{{Ability|Death Snowball|0-16|element=physical + |scene={{Scene|spell=5sqmstrike|caster=Gnorre Chyllson|look_direction=south|effect=Death Effect|missile=Snowball Missile|missile_direction=south-east|missile_distance=5/5}}}} + |{{Ability|Paralyze|element=|condition=slowed|scene={{Scene|spell=5sqmstrike|caster=Gnorre Chyllson|look_direction=south|effect=Death Effect|missile=Snowball Missile|missile_direction=south-east|missile_distance=5/5}}}} + |{{Healing|range=97-124 + |scene={{Scene|spell=buffspell|effect=Red Sparkles Effect|caster=Gnorre Chyllson|look_direction=south|effect_on_caster=yes}}}} + |{{Haste|[[Strong Haste]]| + |scene={{Scene|spell=buffspell|effect=Red Sparkles Effect|caster=Gnorre Chyllson|look_direction=south|effect_on_caster=yes}}}} + |{{Ability|Snowball|element=None + |scene={{Scene|spell=5sqmstrike|caster=Gnorre Chyllson|look_direction=south|effect=Ripple Effect|missile=Snowball Missile|missile_direction=south-east|missile_distance=5/5}}}} +}} +| usedelements = Physical, Life Drain, Drowning +| maxdmg = 1100+ +| pushable = no +| pushobjects = yes +| walksaround = Energy, Poison +| walksthrough = Fire +| paraimmune = yes +| senseinvis = yes +| physicalDmgMod = 101%>? +| earthDmgMod = 0% +| fireDmgMod = 0% +| deathDmgMod = 100%? +| energyDmgMod = 110% +| holyDmgMod = 99% [[Fallen Mooh'Tah Master Ghar|Next #8.]] +| runsat = 0 +| behaviour = +| speed = ? +| strategy = '''Although he shoots beams and waves, BE SURE THAT HIS FIELD ATTACKS are very strong.''' Keeping in diagonal helps to avoid waves. Also is '''recommended''' to use [[Protection Amulet]] to reduce physical damage.
[[Knight]]s should use [[Fierce Berserk]] since it is weak to [[Physical Damage]]. [[Paladin]]s may use [[Assassin Star]]s. +| location = In the [[Svargrond Arena]]. +| loot = {{Loot Table}} +}} \ No newline at end of file diff --git a/tests/resources/content_creature_green_djinn.txt b/tests/resources/content_creature_green_djinn.txt new file mode 100644 index 0000000..a21fe4a --- /dev/null +++ b/tests/resources/content_creature_green_djinn.txt @@ -0,0 +1,83 @@ + +:''This article is about the creature '''Green Djinn'''. For the NPCs who deal with players, see [[Alesar]] (equipment) or [[Yaman]] (magical).'' + +{{Infobox Creature|List={{{1|}}}|GetValue={{{GetValue|}}} +| name = Green Djinn +| article = a +| actualname = green djinn +| plural = green djinns +| hp = 330 +| exp = 215 +| armor = 20 +| mitigation = 1.22 +| summon = -- +| convince = -- +| illusionable = no +| creatureclass = Humanoids +| primarytype = Djinn +| bestiaryclass = Magical +| bestiarylevel = Medium +| occurrence = Common +| spawntype = Regular +| isboss = no +| abilities = {{Ability List + |{{Melee|0-110}} + |{{Ability|fire strike|45-75|fire + |scene={{Scene|spell=7sqmstrike|caster=Green Djinn|missile=Fire Missile|missile_direction=south-east|missile_distance=7/7|effect=None}}}} + |{{Ability|energy strike|50-90|electrified + |scene={{Scene|spell=7sqmstrike|caster=Green Djinn|missile=Energy Missile|missile_direction=south-east|missile_distance=7/7|effect=None}}}} + |{{Ability|life drain strike|55-105|life drain + |scene={{Scene|spell=7sqmstrike|caster=Green Djinn|missile=Death Missile|missile_direction=south-east|missile_distance=7/7|effect=None}}}} + |{{Ability|Drunkenness|30 seconds|drunk + |scene={{Scene|spell=7sqmstrike|effect=Teleport Effect|caster=Green Djinn|missile=Energy Missile|missile_direction=south-east|missile_distance=7/7}}}} + |{{Ability|[[Cancel Invisibility]]|element=invisible}} + |{{Outfit|yes|rat|{{Scene|spell=7sqmstrike|effect=Blue Sparkles Effect|caster=Green Djinn|missile_direction=south-east|missile_distance=7/7}}}} +}} +| usedelements = Fire > Death > Energy > Melee +| maxdmg = 320+ +| pushable = no +| pushobjects = yes +| walksaround = Fire, Energy, Poison +| walksthrough = -- +| paraimmune = yes +| senseinvis = yes +| physicalDmgMod = 80% +| earthDmgMod = 100% +| fireDmgMod = 20% +| deathDmgMod = 80% +| energyDmgMod = 50% +| holyDmgMod = 113% +| iceDmgMod = 110% +| hpDrainDmgMod = 100%? +| drownDmgMod = 100%? +| healMod = 100% +| bestiaryname = greendjinn +| bestiarytext = In ancient times the Green Djinns lost a great war against their blue brethren. Considered to be extinct, the last knowledge of these powerful beings turned into legends and legends became fairy-tales. Those who have encountered the recently resurfaced, self-proclaimed masters of evil can attest that they are indeed, a reality, and not a pleasant one. These magic-wielding enemies of all good embrace destruction and enjoy causing pain. Although only inferior members of the green-skinned race have been sighted until now, they are enemies to beware of. Their magical tricks make them more than a match for unexpecting adventurers. Despising all living beings and everything that is beautiful, their especially passionate hatred is reserved for their blue-skinned relatives, whom they regard as traitors to djinnkind. +| sounds = {{Sound List|Good wishes are for fairytales|I wish you a merry trip to hell!|Muhahahaha!|I grant you a deathwish!|Tell me your last wish!}} +| implemented = 7.3 +| race_id = 51 +| notes = These Magic [[Creature]]s are the counterpart of [[Blue Djinn]] and a lower rank than [[Efreet]]. They can turn you into a [[rat]] and cancel your invisibility. + +You can receive a task to kill 500 green djinns of the [[Killing in the Name of... Quest]]. When you complete this task you may fight [[Merikh the Slaughterer]]. +| behaviour = Green Djinns use strong melee when close to their opponents, and [[Energy|Energy Attacks]] when the opponent is at a [[distance]]. +| runsat = 0 +| speed = 110 +| strategy = It is highly recommended to wear a [[Dwarven Ring]]. +| location = [[Mal'ouquah]], [[Oasis Tomb]], under the [[Ankrahmun Library Tomb]], [[Serpentine Tower]] last floor behind the [[Magic Wall]]s, [[Deeper Banuta]], [[Goroma]] underground, [[Magician Quarter]]. +| loot = {{Loot Table + |{{Loot Item|0-115|Gold Coin|common}} + |{{Loot Item|Cheese|common}} + |{{Loot Item|Book (Green)|semi-rare}} + |{{Loot Item|0-4|Small Emerald|semi-rare}} + |{{Loot Item|Green Piece of Cloth|semi-rare}} + |{{Loot Item|0-2|Royal Spear|semi-rare}} + |{{Loot Item|Dirty Turban|semi-rare}} + |{{Loot Item|Small Oil Lamp|rare}} + |{{Loot Item|Grave Flower|rare}} + |{{Loot Item|Mana Potion|very rare}} + |{{Loot Item|Mystic Turban|very rare}} +}} +| history = Although the Green Djinn was technically implemented with the [[Updates/7.3|Summer Update 2004]], its first spawns were added to the game with the [[Updates/7.4|Christmas Update 2004]]. +| usespells = yes +| attacktype = Melee +}} \ No newline at end of file diff --git a/tests/resources/content_creature_mimic.txt b/tests/resources/content_creature_mimic.txt new file mode 100644 index 0000000..3866474 --- /dev/null +++ b/tests/resources/content_creature_mimic.txt @@ -0,0 +1,48 @@ +{{Infobox Creature|List={{{1|}}}|GetValue={{{GetValue|}}} +| name = Mimic +| article = a +| actualname = mimic +| plural = mimics +| hp = 1200 +| exp = 0 +| armor = ? +| summon = -- +| convince = -- +| illusionable = no +| creatureclass = Special +| primarytype = Shapeshifters +| isboss = no +| abilities = {{Ability List + |{{Outfit|no|Teddy Bear|thing2=Demon Shield|thing3=Warlord Sword|thing4=Piggy Bank}} + |{{Outfit|no|Green Frog|thing2=Treasure Chest|thing3=Rat|thing4=Slime}} + |{{Outfit|no|Pumpkin|thing2=Jewel Case|thing3=Giant Mushroom|thing4=Present Box}} + |{{Outfit|no|The Orb of Nature|thing2=Vashresamun}} + |{{Ability|Turns Invisible|very rare 1s|element=invisible}} +}} +| maxdmg = 0 +| pushable = ? +| pushobjects = ? +| walksaround = ? +| walksthrough = ? +| paraimmune = ? +| senseinvis = ? +| physicalDmgMod = 0% +| earthDmgMod = 0% +| fireDmgMod = 0% +| deathDmgMod = 0% +| energyDmgMod = 0% +| holyDmgMod = 0% +| iceDmgMod = 0% +| hpDrainDmgMod = 0% +| drownDmgMod = 0% +| healMod = 100%? +| sounds = {{Sound List|}} +| implemented = 7.3 +| race_id = 92 +| notes = They are shape shifters. Legends say they shapeshift into items that are of your inner-most desire. They were [[Summon Creature|summonable]] for a short period. +| behaviour = Keeps moving around shapeshifting. +| speed = ? +| strategy = They don't attack and you can't kill them either, so simply just watch them or ignore them. +| location = Third dream in the [[Dream Realm]] +| loot = {{Loot Table}} +}} \ No newline at end of file diff --git a/tests/resources/content_creature_son_of_verminor.txt b/tests/resources/content_creature_son_of_verminor.txt new file mode 100644 index 0000000..fae1677 --- /dev/null +++ b/tests/resources/content_creature_son_of_verminor.txt @@ -0,0 +1,58 @@ +{{Infobox Creature|List={{{1|}}}|GetValue={{{GetValue|}}} +| name = Son of Verminor +| article = a +| actualname = son of verminor +| hp = 8500 +| exp = 5900 +| armor = 48 +| mitigation = 2.09 +| summon = -- +| convince = -- +| illusionable = no +| creatureclass = Elementals +| primarytype = Blobs +| secondarytype = Shapeshifters +| bestiaryclass = Slime +| bestiarylevel = Hard +| occurrence = Common +| isboss = no +| spawntype = Regular +| abilities = {{Ability List + |{{Melee|0-460|poisoned=?}} + |{{Ability|Poison Hit|150-200|earth}} + |{{Ability|Poison Berserk|350-390|earth}} + |{{Ability|Curse Berserk|220-270|death}} + |{{Outfit|no|Rat||Larva|Scorpion|Slime}} + |{{Healing|range=250-350}} +}} +| maxdmg = {{Max Damage|physical=460|earth=590|death=270}} +| pushable = no +| pushobjects = yes +| walksaround = Fire +| walksthrough = Energy, Poison +| paraimmune = yes +| senseinvis = yes +| physicalDmgMod = 100% +| earthDmgMod = 0% +| fireDmgMod = 90% +| deathDmgMod = 100% +| energyDmgMod = 80% +| holyDmgMod = 100% +| iceDmgMod = 100% +| hpDrainDmgMod = 100%? +| drownDmgMod = 100%? +| healMod = 100% +| bestiaryname = sonofverminor +| bestiarytext = It is said that the dark lord of decay, Verminor, one of the Ruthless Seven, oozes the very essence of destruction. The power of that demon is so immense that some of his ooze gained a sort of sentience over the years. Instinctively devouring more and more of the demonic ooze, it grows in proportions and sentience unless a semi-intelligent creature is formed that is known as Son of Verminor.
The most powerful of them are recognised by their progenitor as his kin and serve as his minions. Usually not more than an almost mindless mass of decay and hunger, they can be used by their dark master almost like puppets to fulfil his unholy duties. It is assumed that they not only function as the long arm of their master, but also as his eyes, enabling him to sight what is happening in the vicinity of his 'son'. When encountered in the realm of man, Sons of Verminor are usually just a shambling ever-hungry mass that tries to consume everything that it can catch. Because of the mindless destruction they tend to cause, they are seldom summoned by the servants of the demons. The Sons of Verminor are usually used to make places as inhospitable as possible. Sometimes a greater number of them is summoned to make an area almost impassable for a while. +| sounds = {{Sound List|Blubb.}} +| implemented = 7.9 +| race_id = 265 +| behaviour = +| runsat = 0 +| speed = 120 +| strategy = Kill it fast using [[Ice Damage|ice]], [[Holy Damage|holy]], or [[Death Damage|death]] runes/spells and [[Exori Gran]]. +| location = Deep in the [[Pits of Inferno]], in Verminor's throne room. Also in the [[Demon Forge]] (The Vats and The Hive). +| loot = {{Loot Table}} +| usespells = yes +| attacktype = Melee +}} \ No newline at end of file diff --git a/tibiawikisql/models/creature.py b/tibiawikisql/models/creature.py index d965738..b9fa2ef 100644 --- a/tibiawikisql/models/creature.py +++ b/tibiawikisql/models/creature.py @@ -51,12 +51,19 @@ class CreatureAbility(BaseModel): name: str """The name of the ability.""" effect: str | None = None - """The effect of the ability, or the damage range.""" + """The effect of the ability, or the damage range. + For a ``debuff``, the reduction as ``value=; duration=; flags=``, without the empty parts. + For an ``outfit``, ``yes`` if it changes the victim's outfit and ``no`` if it changes the creature's own.""" element: str """The element of damage type of the ability. This could also be a status condition instead. + Haste, debuff and outfit change abilities are ``haste``, ``debuff`` and ``outfit``. For abilities that are just plain text, ``plain_text`` is set. For abilities that are not using the abilities templates in TibiaWiki, they are saved as a single entry with element: ``no_template``.""" + condition: str | None = None + """The condition the ability causes, in lower case, like ``poisoned`` or ``slowed``. ``None`` if unknown or none.""" + condition_value: int | None = None + """The poison damage per turn a melee attack starts at, up to. ``None`` if unknown or not a poisoning melee.""" class CreatureDrop(BaseModel): diff --git a/tibiawikisql/parsers/creature.py b/tibiawikisql/parsers/creature.py index 0a8f1ae..b167aa1 100644 --- a/tibiawikisql/parsers/creature.py +++ b/tibiawikisql/parsers/creature.py @@ -1,5 +1,6 @@ # Changed by tibia.sh in 2026. See "About this copy" in README.md. import re +from collections.abc import Collection from typing import Any, ClassVar, TYPE_CHECKING import mwparserfromhell @@ -38,6 +39,9 @@ LOOT_RARITIES = ("always", "common", "uncommon", "semi-rare", "rare", "very rare") """The rarity words of the Loot Item template.""" +DEBUFF_FLAGS = ("distance", "melee", "magic", "shielding") +"""The skills a Debuff template can reduce, in the order its effect lists them.""" + def parse_maximum_damage(value: str) -> dict[str, int]: """Parse the maximum damage template from TibiaWiki. @@ -192,9 +196,122 @@ def match(k): return loot -def parse_abilities(value: str) -> list[dict[str, str]]: +def template_value(template: "Template", *names: str) -> str: + """Get the plain text of the first given parameter of a template, as the template reads ``{{{a|{{{b|}}}}}}``. + + Args: + template: The template to read. + *names: The names of the parameters, in order of precedence, like ``"1"`` or ``"name"``. + + Returns: + The stripped plain text of the first given parameter, even if it is empty, or an empty string if none is given. + + """ + for name in names: + if template.has(name): + return strip_code(template.get(name)) + return "" + + +def format_debuff_effect(value: str, duration: str, flags: Collection[str]) -> str: + """Write the effect of a Debuff ability as ``value=; duration=; flags=``. + + An empty part is left out, with its separator, so a Debuff with only a duration is ``duration=10``. + + Args: + value: How much the skills are reduced, as the wiki writes it, like ``50%``. + duration: How long the reduction lasts, as the wiki writes it, like ``10``. + flags: The skills reduced, from :data:`DEBUFF_FLAGS`, in any order. They are written in the order of + :data:`DEBUFF_FLAGS`, separated by commas. + + Returns: + The effect, like ``value=50%; duration=10; flags=distance,melee``, or an empty string when every part is empty. + + Raises: + ValueError: If a flag is not one of :data:`DEBUFF_FLAGS`. + + """ + unknown = set(flags).difference(DEBUFF_FLAGS) + if unknown: + msg = f"Unknown debuff flags: {sorted(unknown)}" + raise ValueError(msg) + parts = [] + if value: + parts.append(f"value={value}") + if duration: + parts.append(f"duration={duration}") + ordered_flags = [flag for flag in DEBUFF_FLAGS if flag in flags] + if ordered_flags: + parts.append(f"flags={','.join(ordered_flags)}") + return "; ".join(parts) + + +def parse_debuff_effect(template: "Template") -> str: + """Parse the effect of a Debuff template with :func:`format_debuff_effect`. + + The value, duration and flags are read from the named parameters ``value``, ``duration``, ``distance``, + ``melee``, ``magic`` and ``shielding``. A flag is set when its value is ``yes``, in any case. Positional + parameters after the name are not read, so ``{{Debuff|Debuff|yes|value=80-100%}}`` has no flags. + + Args: + template: The Debuff template. + + Returns: + The effect of the Debuff. + + """ + flags = [flag for flag in DEBUFF_FLAGS if template_value(template, flag).lower() == "yes"] + return format_debuff_effect(template_value(template, "value"), template_value(template, "duration"), flags) + + +def parse_outfit_names(template: "Template") -> list[str]: + """Parse the outfits an Outfit template changes into. + + The first outfit is ``thing`` or parameter 2. Parameter 3 is never an outfit. Each extra outfit ``thingN``, from + ``thing2`` on, may also be given as parameter N + 2, which takes precedence. + + Args: + template: The Outfit template. + + Returns: + The names of the outfits, in order, without the empty ones. + + """ + last_extra = 1 + for param in template.params: + name = str(param.name).strip() + if name.isdigit(): + last_extra = max(last_extra, int(name) - 2) + elif name.startswith("thing") and name[len("thing"):].isdigit(): + last_extra = max(last_extra, int(name[len("thing"):])) + names = [template_value(template, "thing", "2")] + names.extend(template_value(template, str(number + 2), f"thing{number}") for number in range(2, last_extra + 1)) + return [name for name in names if name] + + +def parse_condition(template: "Template") -> str | None: + """Parse the condition an ability causes from its ``condition`` parameter, with :func:`parse_known_text`. + + Args: + template: The ability template. + + Returns: + The condition in lower case, like ``poisoned``, or ``None`` if the parameter is absent, empty or unknown. + + """ + if not template.has("condition"): + return None + return parse_known_text(str(template.get("condition").value)) + + +def parse_abilities(value: str) -> list[dict[str, str | int | None]]: """Parse the abilities of a creature. + Besides the Melee, Summon, Healing and Ability templates, Haste, Debuff and Outfit templates give abilities with + the element ``haste``, ``debuff`` and ``outfit``. An Outfit template gives one ability per outfit. A Melee or + Ability ``condition`` is kept, and a Melee ``poison`` makes the condition ``poisoned``, with the damage as its + value. Template names must match exactly, so ``{{haste}}`` gives nothing. + Args: value: A string containing the creature's abilities definition. @@ -230,7 +347,13 @@ def parse_abilities(value: str) -> list[dict[str, str]]: "name": ability_template.get("name", "Melee"), "effect": ability_template.get(1, ability_template.get("damage", "?")), "element": ability_template.get(2, ability_template.get("element", "physical")), + "condition": parse_condition(ability_template), } + if ability_template.has("poison"): + poison = visible_text(str(ability_template.get("poison").value)) + if poison.strip(): + ability["condition"] = "poisoned" + ability["condition_value"] = parse_known_integer(poison) if template_name == "Summon": ability = { "name": ability_template.get(1, ability_template.get("creature", None)), @@ -248,9 +371,27 @@ def parse_abilities(value: str) -> list[dict[str, str]]: "name": ability_template.get(1, ability_template.get("name", None)), "effect": ability_template.get(2, ability_template.get("damage", "?")), "element": ability_template.get(3, ability_template.get("element", "physical")), + "condition": parse_condition(ability_template), } if ability["name"] is None: ability["name"] = f'{ability["element"].title()} Damage' + if template_name == "Haste": + ability = { + "name": template_value(ability_template, "1", "name") or "Haste", + "effect": "", + "element": "haste", + } + if template_name == "Debuff": + ability = { + "name": template_value(ability_template, "1", "name") or "Debuff", + "effect": parse_debuff_effect(ability_template), + "element": "debuff", + } + if template_name == "Outfit": + victim = template_value(ability_template, "victim", "1").lower() + abilities.extend( + {"name": name, "effect": victim, "element": "outfit"} for name in parse_outfit_names(ability_template) + ) if ability: abilities.append(strip_code(ability)) return abilities diff --git a/tibiawikisql/schema.py b/tibiawikisql/schema.py index 4a03f82..9d2b0c1 100644 --- a/tibiawikisql/schema.py +++ b/tibiawikisql/schema.py @@ -102,6 +102,8 @@ class CreatureAbilityTable(Table, table_name="creature_ability"): name = Column(Text, nullable=False) effect = Column(Text) element = Column(Text) + condition = Column(Text) + condition_value = Column(Integer) class CreatureMaxDamageTable(Table, table_name="creature_max_damage"): From ccc566c82d77c5167c0e0614ddebeddf902cef1b Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 20:20:50 +0000 Subject: [PATCH 08/17] fix: read outfit extras without a range scan --- tests/parsers/test_creature.py | 20 ++++++++++++++++++++ tibiawikisql/parsers/creature.py | 22 +++++++++++++++------- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/tests/parsers/test_creature.py b/tests/parsers/test_creature.py index 0b382a5..207c8f8 100644 --- a/tests/parsers/test_creature.py +++ b/tests/parsers/test_creature.py @@ -744,3 +744,23 @@ def test_debuff_effect_rejects_an_unknown_flag(self): def test_outfit_named_victim_and_thing_win_over_positional(self): self.assertEqual([("Bug", "yes", "outfit", None, None)], self._rows("{{Ability List|{{Outfit|no|Rat|victim=yes|thing=Bug}}}}")) + + def test_outfit_far_extra_is_read_without_a_range_scan(self): + self.assertEqual([("Bug", "yes", "outfit", None, None), ("Rat", "yes", "outfit", None, None)], + self._rows("{{Ability List|{{Outfit|yes|Bug||thing1000000000=Rat}}}}")) + + def test_outfit_far_positional_extra(self): + self.assertEqual([("Bug", "yes", "outfit", None, None), ("Rat", "yes", "outfit", None, None)], + self._rows("{{Ability List|{{Outfit|yes|Bug||1000000000=Rat}}}}")) + + def test_outfit_ignores_non_ascii_digits(self): + self.assertEqual([("Bug", "yes", "outfit", None, None)], + self._rows("{{Ability List|{{Outfit|yes|Bug|thing²=Rat|²=Cat}}}}")) + + def test_outfit_ignores_a_number_too_long_to_read(self): + self.assertEqual([("Bug", "yes", "outfit", None, None)], + self._rows("{{Ability List|{{Outfit|yes|Bug|thing" + "9" * 5000 + "=Rat}}}}")) + + def test_outfit_extras_with_a_gap(self): + self.assertEqual([("A", "no", "outfit", None, None), ("B", "no", "outfit", None, None)], + self._rows("{{Ability List|{{Outfit|no|thing2=A|thing5=B}}}}")) diff --git a/tibiawikisql/parsers/creature.py b/tibiawikisql/parsers/creature.py index b167aa1..d75de6d 100644 --- a/tibiawikisql/parsers/creature.py +++ b/tibiawikisql/parsers/creature.py @@ -35,6 +35,8 @@ from mwparserfromhell.nodes import Template range_hyphen_pattern = re.compile(r"(?<=\d)-(?=\d)") +positional_parameter_pattern = re.compile(r"[0-9]+") +outfit_thing_pattern = re.compile(r"thing([0-9]+)") LOOT_RARITIES = ("always", "common", "uncommon", "semi-rare", "rare", "very rare") """The rarity words of the Loot Item template.""" @@ -268,7 +270,9 @@ def parse_outfit_names(template: "Template") -> list[str]: """Parse the outfits an Outfit template changes into. The first outfit is ``thing`` or parameter 2. Parameter 3 is never an outfit. Each extra outfit ``thingN``, from - ``thing2`` on, may also be given as parameter N + 2, which takes precedence. + ``thing2`` on, may also be given as parameter N + 2, which takes precedence. Only the parameters given are read, + in the order of N, so a gap between them costs nothing. A name with digits other than ASCII ones, or with a number + too long to read, is ignored. Args: template: The Outfit template. @@ -277,15 +281,19 @@ def parse_outfit_names(template: "Template") -> list[str]: The names of the outfits, in order, without the empty ones. """ - last_extra = 1 + extras: dict[int, str] = {} for param in template.params: name = str(param.name).strip() - if name.isdigit(): - last_extra = max(last_extra, int(name) - 2) - elif name.startswith("thing") and name[len("thing"):].isdigit(): - last_extra = max(last_extra, int(name[len("thing"):])) + thing = outfit_thing_pattern.fullmatch(name) + try: + if positional_parameter_pattern.fullmatch(name) and int(name) >= 4: + extras[int(name) - 2] = name + elif thing and int(thing[1]) >= 2: + extras.setdefault(int(thing[1]), name) + except ValueError: + continue names = [template_value(template, "thing", "2")] - names.extend(template_value(template, str(number + 2), f"thing{number}") for number in range(2, last_extra + 1)) + names.extend(template_value(template, extras[number]) for number in sorted(extras)) return [name for name in names if name] From 1754dd17dd60312e0a074b36d7d0a258e66045cf Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 20:36:22 +0000 Subject: [PATCH 09/17] fix: keep map coordinates out of locations --- CHANGELOG.md | 11 ++ README.md | 3 + docs/schema.md | 120 +++++++++----- tests/models/test_book.py | 2 + tests/models/test_creature.py | 34 +++- tests/models/test_item.py | 56 +++++++ tests/parsers/test_creature.py | 66 ++++++++ tests/parsers/test_item.py | 30 ++++ tests/parsers/test_npc.py | 43 +++++ tests/resources/content_creature_behemoth.txt | 84 ++++++++++ .../content_creature_black_sheep.txt | 53 +++++++ .../content_creature_warlord_ruzad.txt | 61 +++++++ .../resources/content_item_sneeze_blossom.txt | 16 ++ tests/tests_utils.py | 149 +++++++++++++++++- tibiawikisql/models/__init__.py | 1 + tibiawikisql/models/base.py | 14 ++ tibiawikisql/models/creature.py | 15 +- tibiawikisql/models/item.py | 16 +- tibiawikisql/parsers/creature.py | 11 +- tibiawikisql/parsers/item.py | 15 ++ tibiawikisql/parsers/npc.py | 39 ++--- tibiawikisql/schema.py | 21 +++ tibiawikisql/utils.py | 125 +++++++++++++++ 23 files changed, 909 insertions(+), 76 deletions(-) create mode 100644 tests/resources/content_creature_behemoth.txt create mode 100644 tests/resources/content_creature_black_sheep.txt create mode 100644 tests/resources/content_creature_warlord_ruzad.txt create mode 100644 tests/resources/content_item_sneeze_blossom.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 03f8197..0d997ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,17 @@ one row per outfit, from parameter 2 or `thing`, then parameters 4 and up or `thing2` and up, with the element `outfit` and the effect `yes` when it changes the victim and `no` when it changes the creature. Template names must match exactly, as before, and the existing rows keep their values. +- Keep map coordinates out of `creature.location`, and add `item.location` and the `creature_location_coordinate` and + `item_location_coordinate` tables. Each `{{Mapper Coords}}` template in a location is replaced by its `text` label, + so Black Sheep's `Femor Hills (127.88), cultist cave in Liberty Bay (126.192), Drefia` reads `Femor Hills (here), + cultist cave in Liberty Bay (here), Drefia`, and Behemoth's `double spawn text=here` reads `double spawn here`. A + template without a label is removed, with the spaces before it when a space, punctuation or the end follows, and + with parentheses it leaves empty. The rest of the text is cleaned as before, so a location without the template is + unchanged, and NPC, house and key locations are unchanged. Each template whose `x`, `y` and `z` read as numbers + gives a row with the position, its plain-text label or `NULL`, and its place in the list from 0. `x` and `y` are + read from the wiki's `major.minor` form, so `127.88` is 32600, and a malformed set like Warlord Ruzad's + `128.202,123.233,5-2` gives no row, while its label still replaces it in the text. `item.location` is the item's + `location`, cleaned the same way. ## 9.0.0+tibiash.9 diff --git a/README.md b/README.md index 2c2871d..a08476a 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,9 @@ The copy adds this data to the database: * The `creature_ability.condition` and `creature_ability.condition_value` columns, and the `haste`, `debuff` and `outfit` abilities. The columns hold the condition an ability causes, like `poisoned`, and the poison damage a melee attack starts at. +* The `item.location` column and the `creature_location_coordinate` and `item_location_coordinate` tables. The tables + keep the map positions written with `{{Mapper Coords}}` in a creature's or item's location, with their labels, and + the location text shows each label in place of its position. The [database schema](docs/schema.md) describes each of them. diff --git a/docs/schema.md b/docs/schema.md index 0fe9d20..4033ad9 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -10,46 +10,48 @@ The generated database has the following tables. ## Tables -| Table | Description | -| ----------------------- | ----------------------------------------------------------------------------------- | -| `achievement` | Contains information for all achievements. | -| `book` | Contains information about books. | -| `charm` | Contains information for all charms. | -| `creature` | Contains information for all creatures. | -| `creature_ability` | Contains all the abilities done by creatures. | -| `creature_drop` | Contains all the items dropped by creatures. | -| `creature_max_damage` | Contains the breakdown of max damage done by creatures. | -| `creature_sound` | Contains all the sounds made by creatures. | -| `database_info` | Contains information about the database itself. | -| `game_update` | Contains information about game updates. | -| `house` | Contains all houses and guildhalls. | -| `imbuement` | Contains information for all imbuements. | -| `imbuement_material` | Contains the item materials for imbuements. | -| `item` | Contains information for all items. | -| `item_attribute` | Contains extra attributes and properties of items that only apply to certain types. | -| `item_client_id` | Contains every client ID of items, in the order the wiki lists them. | -| `item_key` | Contains the different key variations. | -| `item_sound` | Contains all the sounds made by items. | -| `item_proficiency_perk` | Contains weapon proficiency perks for items. | -| `item_store_offer` | Contains all offers for items in the Tibia store. | -| `map` | Contains the world map’s images. | -| `mount` | Contains information for all mounts. | -| `npc` | Contains information for all NPCs. | -| `npc_destination` | Contains all the NPCs’ travel destinations. | -| `npc_job` | Contains all the NPCs’ jobs. | -| `npc_location` | Contains all the NPCs’ positions. | -| `npc_offer_buy` | Contains all the NPCs’ buy offers. | -| `npc_offer_sell` | Contains all the NPCs’ sell offers. | -| `npc_race` | Contains all the NPCs’ races. | -| `outfit` | Contains information for all outfits. | -| `outfit_image` | Contains images for all outfits. | -| `outfit_quest` | Contains outfit and addon rewards for quests. | -| `quest` | Contains information for all quests. | -| `quest_danger` | Contains creatures that can be found in a quest. | -| `quest_reward` | Contains item rewards for quests. | -| `rashid_position` | Contains the positions for the NPC Rashid every day of the week. | -| `spell` | Contains information for all spells. | -| `world` | Contains information for all worlds. | +| Table | Description | +| ------------------------------ | ----------------------------------------------------------------------------------- | +| `achievement` | Contains information for all achievements. | +| `book` | Contains information about books. | +| `charm` | Contains information for all charms. | +| `creature` | Contains information for all creatures. | +| `creature_ability` | Contains all the abilities done by creatures. | +| `creature_drop` | Contains all the items dropped by creatures. | +| `creature_location_coordinate` | Contains the map positions in the locations of creatures. | +| `creature_max_damage` | Contains the breakdown of max damage done by creatures. | +| `creature_sound` | Contains all the sounds made by creatures. | +| `database_info` | Contains information about the database itself. | +| `game_update` | Contains information about game updates. | +| `house` | Contains all houses and guildhalls. | +| `imbuement` | Contains information for all imbuements. | +| `imbuement_material` | Contains the item materials for imbuements. | +| `item` | Contains information for all items. | +| `item_attribute` | Contains extra attributes and properties of items that only apply to certain types. | +| `item_client_id` | Contains every client ID of items, in the order the wiki lists them. | +| `item_key` | Contains the different key variations. | +| `item_location_coordinate` | Contains the map positions in the locations of items. | +| `item_sound` | Contains all the sounds made by items. | +| `item_proficiency_perk` | Contains weapon proficiency perks for items. | +| `item_store_offer` | Contains all offers for items in the Tibia store. | +| `map` | Contains the world map’s images. | +| `mount` | Contains information for all mounts. | +| `npc` | Contains information for all NPCs. | +| `npc_destination` | Contains all the NPCs’ travel destinations. | +| `npc_job` | Contains all the NPCs’ jobs. | +| `npc_location` | Contains all the NPCs’ positions. | +| `npc_offer_buy` | Contains all the NPCs’ buy offers. | +| `npc_offer_sell` | Contains all the NPCs’ sell offers. | +| `npc_race` | Contains all the NPCs’ races. | +| `outfit` | Contains information for all outfits. | +| `outfit_image` | Contains images for all outfits. | +| `outfit_quest` | Contains outfit and addon rewards for quests. | +| `quest` | Contains information for all quests. | +| `quest_danger` | Contains creatures that can be found in a quest. | +| `quest_reward` | Contains item rewards for quests. | +| `rashid_position` | Contains the positions for the NPC Rashid every day of the week. | +| `spell` | Contains information for all spells. | +| `world` | Contains information for all worlds. | ## Table schemas @@ -172,7 +174,7 @@ The generated database has the following tables. | modifier_healing | `INTEGER` | Percentage of healing the creature receives. `0` being completely immune, `100` neutral. May be `NULL` if unknown. | | walks_through | `TEXT` | The type of fields the creature will walk through. | | walks_around | `TEXT` | The type of fields the creature will walk around to avoid when possible. | -| location | `TEXT` | The locations where the creature can be found. | +| location | `TEXT` | The locations where the creature can be found. Each Mapper Coords position shows as its label, or is left out without one. | | race_id | `INTEGER` | The client race ID of the creature, the first integer in the value after wiki comments are removed, `NULL` when there is none. | | loot_statistics_kills | `INTEGER` | The kills on the creature's Loot Statistics page that its drop chances come from. `NULL` when it has no page, or its page has no positive number of kills. | | version | `TEXT` | The client version this creature was introduced to the game. | @@ -210,6 +212,26 @@ The generated database has the following tables. +### creature_location_coordinate + +| Column | Type | Description | +| ----------- | --------- | ------------------------------------------------------------------------------------------- | +| creature_id | `INTEGER` | The id of the creature whose location has this map position. | +| x | `INTEGER` | The x coordinate of the position. | +| y | `INTEGER` | The y coordinate of the position. | +| z | `INTEGER` | The floor of the position. | +| label | `TEXT` | The text the location shows in place of the position, like `here`. `NULL` if it shows none. | +| position | `INTEGER` | The row's place in the order the location lists its map positions, from 0. | + +Each row is a `{{Mapper Coords}}` template in the creature's `location`, in the order they are written. `x` and `y` are +read from the wiki's `major.minor` form, so `127.88` is 32600, and `label` is the plain text of the template's `text`. +Named parameters do not count as positions, so `{{Mapper Coords|text=here|127.234|127.59|14}}` gives the same row as +`{{Mapper Coords|127.234|127.59|14|text=here}}`. A template whose `x`, `y` or `z` is missing or malformed, like Warlord +Ruzad's `{{Mapper Coords|128.202,123.233,5-2|text=here}}`, gives no row. In `creature.location`, each template is +replaced by its label, or left out when it has none, so no coordinate is left in the text. + + + ### creature_max_damage | Column | Type | Description | @@ -351,6 +373,7 @@ The generated database has the following tables. | light_radius | `INTEGER` | The radius of the light emitted by this item, if any. | | version | `TEXT` | The client version this item was introduced to the game. | | client_id | `INTEGER` | The client id of the item, the first integer in `itemid`. `item_client_id` has every client ID. | +| location | `TEXT` | Where the item can be found, cleaned as `creature.location` is. `item_location_coordinate` has its map positions. | | image | `BLOB` | The item’s image bytes. | | status | `TEXT` | The status of the item in game. | | timestamp | `TIMESTAMP` | ISO 8601 timestamp of the article's last edit. | @@ -406,6 +429,21 @@ comments included. So when `itemid` starts with a comment holding a number, the +### item_location_coordinate + +| Column | Type | Description | +| -------- | --------- | ------------------------------------------------------------------------------------------- | +| item_id | `INTEGER` | The id of the item whose location has this map position. | +| x | `INTEGER` | The x coordinate of the position. | +| y | `INTEGER` | The y coordinate of the position. | +| z | `INTEGER` | The floor of the position. | +| label | `TEXT` | The text the location shows in place of the position, like `here`. `NULL` if it shows none. | +| position | `INTEGER` | The row's place in the order the location lists its map positions, from 0. | + +The rows are read from the item's `location` as `creature_location_coordinate` rows are from a creature's. + + + ### item_sound | Column | Type | Description | diff --git a/tests/models/test_book.py b/tests/models/test_book.py index 34f7e76..7c53f3f 100644 --- a/tests/models/test_book.py +++ b/tests/models/test_book.py @@ -9,6 +9,7 @@ BookTable, ItemAttributeTable, ItemClientIdTable, + ItemLocationCoordinateTable, ItemSoundTable, ItemStoreOfferTable, ItemTable, @@ -40,6 +41,7 @@ def setUp(self): self.conn.executescript(ItemStoreOfferTable.get_create_table_statement()) self.conn.executescript(ItemSoundTable.get_create_table_statement()) self.conn.executescript(ItemClientIdTable.get_create_table_statement()) + self.conn.executescript(ItemLocationCoordinateTable.get_create_table_statement()) self.conn.executescript(BookTable.get_create_table_statement()) diff --git a/tests/models/test_creature.py b/tests/models/test_creature.py index fe4c239..75ab18f 100644 --- a/tests/models/test_creature.py +++ b/tests/models/test_creature.py @@ -3,7 +3,7 @@ import unittest from tibiawikisql import schema -from tibiawikisql.models import Creature, CreatureAbility, CreatureDrop, Item +from tibiawikisql.models import Creature, CreatureAbility, CreatureDrop, Item, LocationCoordinate from tibiawikisql.schema import ItemTable @@ -163,3 +163,35 @@ def test_creature_ability_condition_round_trip(self): for ability in loaded.abilities ], ) + + def test_creature_location_coordinates_round_trip(self): + creature = self._creature(35) + creature.location_coordinates = [ + LocationCoordinate(x=32732, y=32601, z=14, label="here"), + LocationCoordinate(x=32600, y=31802, z=4, label=None), + LocationCoordinate(x=32448, y=32767, z=7, label="exactly here"), + ] + creature.insert(self.conn) + + loaded = Creature.get_one_by_field(self.conn, "article_id", 1) + rows = self.conn.execute( + "SELECT creature_id, x, y, z, label, position FROM creature_location_coordinate ORDER BY position", + ).fetchall() + + self.assertEqual(creature.location_coordinates, loaded.location_coordinates) + self.assertEqual( + [ + (1, 32732, 32601, 14, "here", 0), + (1, 32600, 31802, 4, None, 1), + (1, 32448, 32767, 7, "exactly here", 2), + ], + [tuple(row) for row in rows], + ) + + def test_creature_location_coordinates_round_trip_none(self): + self._creature(35).insert(self.conn) + + loaded = Creature.get_one_by_field(self.conn, "article_id", 1) + + self.assertEqual([], loaded.location_coordinates) + self.assertEqual([], self.conn.execute("SELECT * FROM creature_location_coordinate").fetchall()) diff --git a/tests/models/test_item.py b/tests/models/test_item.py index df9e6cb..48ebbb7 100644 --- a/tests/models/test_item.py +++ b/tests/models/test_item.py @@ -156,6 +156,62 @@ def test_item_client_ids_round_trip_single(self): self.assertEqual([35609], loaded.client_ids) self.assertEqual([(1, 35609, 0)], [tuple(row) for row in rows]) + def test_item_location_round_trip(self): + article = Article( + article_id=1, + title="Sneeze Blossom", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=load_resource("content_item_sneeze_blossom.txt"), + ) + ItemParser.from_article(article).insert(self.conn) + + loaded = Item.get_one_by_field(self.conn, "article_id", 1) + rows = self.conn.execute( + "SELECT item_id, x, y, z, label, position FROM item_location_coordinate ORDER BY position", + ).fetchall() + + self.assertEqual("An example can be found at spot.", loaded.location) + self.assertEqual([(32709, 31842, 7, "spot")], [(c.x, c.y, c.z, c.label) for c in loaded.location_coordinates]) + self.assertEqual([(1, 32709, 31842, 7, "spot", 0)], [tuple(row) for row in rows]) + + def test_item_location_coordinates_round_trip_in_listed_order(self): + content = load_resource("content_item_sneeze_blossom.txt") + location = "{{Mapper Coords|127.197|124.98|7|3|text=spot}}" + self.assertIn(location, content) + article = Article( + article_id=1, + title="Sneeze Blossom", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=content.replace( + location, + location + ", {{Mapper Coords|126.4|128.108|7|2}} and {{Mapper Coords|125.1|128.109|3|4|text=x}}", + ), + ) + ItemParser.from_article(article).insert(self.conn) + + loaded = Item.get_one_by_field(self.conn, "article_id", 1) + + self.assertEqual("An example can be found at spot, and x.", loaded.location) + self.assertEqual( + [(32709, 31842, 7, "spot"), (32260, 32876, 7, None), (32001, 32877, 3, "x")], + [(c.x, c.y, c.z, c.label) for c in loaded.location_coordinates], + ) + + def test_item_location_round_trip_none(self): + article = Article( + article_id=1, + title="Amethyst Necklace", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=load_resource("content_item_amethyst_necklace.txt"), + ) + ItemParser.from_article(article).insert(self.conn) + + loaded = Item.get_one_by_field(self.conn, "article_id", 1) + + self.assertIsNone(loaded.location) + self.assertEqual([], loaded.location_coordinates) + self.assertEqual([], self.conn.execute("SELECT * FROM item_location_coordinate").fetchall()) + def _insert_amethyst_necklace_with_itemid(self, itemid: str) -> None: content = load_resource("content_item_amethyst_necklace.txt") self.assertIn("| itemid = 35609", content) diff --git a/tests/parsers/test_creature.py b/tests/parsers/test_creature.py index 207c8f8..fbf9233 100644 --- a/tests/parsers/test_creature.py +++ b/tests/parsers/test_creature.py @@ -764,3 +764,69 @@ def test_outfit_ignores_a_number_too_long_to_read(self): def test_outfit_extras_with_a_gap(self): self.assertEqual([("A", "no", "outfit", None, None), ("B", "no", "outfit", None, None)], self._rows("{{Ability List|{{Outfit|no|thing2=A|thing5=B}}}}")) + + +class TestCreatureParserLocation(unittest.TestCase): + def _parse(self, title: str, resource: str) -> Creature: + article = Article( + article_id=1, + title=title, + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=load_resource(resource), + ) + return CreatureParser.from_article(article) + + def _coordinates(self, creature: Creature) -> list[tuple[int, int, int, str | None]]: + return [(c.x, c.y, c.z, c.label) for c in creature.location_coordinates] + + def test_black_sheep_has_two_coordinates_and_clean_text(self): + creature = self._parse("Black Sheep", "content_creature_black_sheep.txt") + + self.assertEqual("Femor Hills (here), cultist cave in Liberty Bay (here), Drefia", creature.location) + self.assertEqual( + [(32600, 31802, 4, "here"), (32448, 32767, 7, "here")], + self._coordinates(creature), + ) + + def test_behemoth_reads_a_label_given_before_the_coordinates(self): + creature = self._parse("Behemoth", "content_creature_behemoth.txt") + + self.assertEqual( + "Cyclopolis, deepest part of Tarpit Tomb after the flame, Forbidden Lands, Vandura Mountain, Deeper Banuta " + "(single spawn here, double spawn here), Serpentine Tower (unreachable), deep into the Formorgar Mines, " + "Arena and Zoo Quarter, The Dark Path, Lower Spike, Chyllfroest, Medusa Tower and Underground Glooth " + "Factory (west side).", + creature.location, + ) + self.assertEqual( + [(32732, 32601, 14, "here"), (32746, 32571, 14, "here")], + self._coordinates(creature), + ) + + def test_warlord_ruzad_keeps_the_label_but_no_malformed_coordinates(self): + creature = self._parse("Warlord Ruzad", "content_creature_warlord_ruzad.txt") + + self.assertEqual("Orc Fortress, in the tower near Orc Powder Quest Room, here.", creature.location) + self.assertEqual([], creature.location_coordinates) + + def test_location_without_mapper_coords_is_unchanged(self): + creature = self._parse("Dragon", "content_creature_dragon.txt") + + self.assertTrue(creature.location.startswith("Thais, Ancient Temple, Darashia Dragon Lair, ")) + self.assertEqual([], creature.location_coordinates) + + def test_no_location(self): + content = load_resource("content_creature_mimic.txt") + location = "| location = Third dream in the [[Dream Realm]]\n" + self.assertIn(location, content) + article = Article( + article_id=1, + title="Mimic", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=content.replace(location, ""), + ) + + creature = CreatureParser.from_article(article) + + self.assertIsNone(creature.location) + self.assertEqual([], creature.location_coordinates) diff --git a/tests/parsers/test_item.py b/tests/parsers/test_item.py index b424e20..46ed935 100644 --- a/tests/parsers/test_item.py +++ b/tests/parsers/test_item.py @@ -4,6 +4,7 @@ from tests import RESOURCES_PATH, load_resource from tibiawikisql.api import Article +from tibiawikisql.models import Item from tibiawikisql.parsers import ItemParser @@ -429,3 +430,32 @@ def test_first_id_matches_client_id_in_every_item_fixture(self): if item.client_ids: self.assertEqual(item.client_id, item.client_ids[0]) + + +class TestItemParserLocation(unittest.TestCase): + def _parse(self, title: str, resource: str) -> Item: + article = Article( + article_id=1, + title=title, + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=load_resource(resource), + ) + return ItemParser.from_article(article) + + def test_sneeze_blossom_location_and_coordinates(self): + item = self._parse("Sneeze Blossom", "content_item_sneeze_blossom.txt") + + self.assertEqual("An example can be found at spot.", item.location) + self.assertEqual([(32709, 31842, 7, "spot")], [(c.x, c.y, c.z, c.label) for c in item.location_coordinates]) + + def test_location_without_mapper_coords(self): + item = self._parse("Shallow Water", "content_item_shallow_water.txt") + + self.assertEqual("All around the land of Tibia, rivers and water falls.", item.location) + self.assertEqual([], item.location_coordinates) + + def test_no_location(self): + item = self._parse("Amethyst Necklace", "content_item_amethyst_necklace.txt") + + self.assertIsNone(item.location) + self.assertEqual([], item.location_coordinates) diff --git a/tests/parsers/test_npc.py b/tests/parsers/test_npc.py index c4278ff..ff9893e 100644 --- a/tests/parsers/test_npc.py +++ b/tests/parsers/test_npc.py @@ -107,6 +107,49 @@ def test_npc_parser_locations_position_7_and_geolabel(self): self.assertEqual(31754, position_7.y) self.assertIsNone(position_7.z) + def test_npc_location_text_keeps_its_reading(self): + # NPC locations are cleaned as before: map coordinates are not replaced by their labels here. + article = Article( + article_id=1, + title="Buddel", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=load_resource("content_npc_locations.txt"), + ) + + npc = NpcParser.from_article(article) + + self.assertEqual( + "South of Svargrond, Tyrsung (126.76), Okolnir (125.224), Barbarian Settlements (125.20) and Helheim " + "(126.209).", + npc.location, + ) + + def test_npc_positions_in_every_fixture(self): + expected = { + "content_npc.txt": [(33045, 32620, 2)], + "content_npc_a_blue_stone.txt": [], + "content_npc_locations.txt": [ + (32254, 31195, 7), (32225, 31381, 7), (32464, 31172, 7), (32332, 31229, 7), (32019, 31293, 7), + ], + "content_npc_multi_origin.txt": [(32347, 32623, 7), (32027, 32814, 7)], + "content_npc_origin_note.txt": [(32649, 31293, 6), (32638, 31438, 7), (32563, 31313, 7)], + "content_npc_origin_position.txt": [(32856, 31548, 7), (32837, 31365, 7)], + "content_npc_shuttle.txt": [(32917, 31199, 7), (32940, 31182, 7)], + "content_npc_travel.txt": [(32310, 32210, 6)], + } + for resource, positions in expected.items(): + with self.subTest(resource=resource): + article = Article( + article_id=1, + title=resource, + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=load_resource(resource), + ) + + npc = NpcParser.from_article(article) + + self.assertEqual(positions, [(p.x, p.y, p.z) for p in npc.locations]) + def test_npc_parser_locations_malformed_and_gaps(self): content = load_resource("content_npc_locations.txt") content = re.sub(r"^\| posx2\s*=.*$", "| posx2 = 125", content, flags=re.MULTILINE) diff --git a/tests/resources/content_creature_behemoth.txt b/tests/resources/content_creature_behemoth.txt new file mode 100644 index 0000000..1412f25 --- /dev/null +++ b/tests/resources/content_creature_behemoth.txt @@ -0,0 +1,84 @@ +{{Infobox Creature|List={{{1|}}}|GetValue={{{GetValue|}}} +| name = Behemoth +| article = a +| actualname = behemoth +| plural = behemoths +| hp = 4000 +| exp = 2500 +| armor = 50 +| mitigation = 2.76 +| summon = -- +| convince = -- +| illusionable = no +| creatureclass = Humanoids +| primarytype = Giants +| bestiaryclass = Giant +| bestiarylevel = Medium +| occurrence = Common +| spawntype = Regular, Unblockable +| isboss = no +| abilities = {{Ability List + |{{Melee|0-455}} + |{{Ability|Boulder throw|0-200|element=physical + |scene={{Scene|spell=5sqmstrike|background=canvas|caster=Behemoth|look_direction=south|missile=Stone Missile|missile_direction=south-east|missile_distance=5/5}}}} + |{{Haste}} +}} +| usedelements = Physical +| maxdmg = {{Max Damage|physical=635}} +| pushable = No +| pushobjects = Yes +| walksaround = Energy +| walksthrough = Fire, Poison +| paraimmune = yes +| senseinvis = yes +| physicalDmgMod = 90% +| earthDmgMod = 20% +| fireDmgMod = 70% +| deathDmgMod = 105% +| energyDmgMod = 90% +| holyDmgMod = 70% +| iceDmgMod = 110% +| hpDrainDmgMod = 100%? +| drownDmgMod = 100%? +| healMod = 100% +| bestiaryname = behemoth +| bestiarytext = Behemoths are giant monsters that live in the ancient town deep below Edron. They look a bit like cross-breeds of humans and bulls and even talk in human language, but nobody knows yet where they come from. Although they are of minor intelligence, they are very strong fighters and throw huge stones at their opponents. +| sounds = {{Sound List|Crush the intruders!|You're so little!|Human flesh - delicious!}} +| implemented = 6.4 +| race_id = 55 +| notes = It can be [[Skinning|skinned]] in order to get [[Behemoth Fang]]s (see: [[Perfect Behemoth Fang/Skinning]]). + +You may hunt 700 of these as a task of the [[Killing in the Name of... Quest]]. Upon completion of the task you will face [[Stonecracker]]. +| behaviour = Behemoths attack their enemies and chase them with their great speed. +| runsat = 0 +| speed = 170 +| location = [[Cyclopolis]], deepest part of [[Tarpit Tomb]] after the flame, [[Forbidden Lands]], [[Vandura Mountain]], [[Deeper Banuta]] (single spawn {{Mapper Coords|127.220|127.89|14|2|text=here}}, double spawn {{Mapper Coords|text=here|127.234|127.59|14|2|1|0.50.5}}), [[Serpentine Tower]] (unreachable), deep into the [[Formorgar Mines]], [[Arena and Zoo Quarter]], [[Demon Forge|The Dark Path]], [[Lower Spike]], [[Chyllfroest]], [[Medusa Tower]] and [[Underground Glooth Factory]] (west side). +| loot = {{Loot Table + |{{Loot Item|1-200|Gold Coin|common}} + |{{Loot Item|1-5|Platinum Coin|common}} + |{{Loot Item|1-6|Meat|common}} + |{{Loot Item|Battle Stone|uncommon}} + |{{Loot Item|Double Axe|uncommon}} + |{{Loot Item|1-5|Assassin Star|uncommon}} + |{{Loot Item|1-5|Small Amethyst|uncommon}} + |{{Loot Item|Two Handed Sword|uncommon}} + |{{Loot Item|Great Health Potion|semi-rare}} + |{{Loot Item|Dark Armor|semi-rare}} + |{{Loot Item|Plate Armor|semi-rare}} + |{{Loot Item|Crystal Necklace|semi-rare}} + |{{Loot Item|Perfect Behemoth Fang|rare}} + |{{Loot Item|Giant Sword|rare}} + |{{Loot Item|Strange Symbol|rare}} + |{{Loot Item|Big Bone|rare}} + |{{Loot Item|Pick|rare}} + |{{Loot Item|Behemoth Claw|rare}} + |{{Loot Item|Steel Boots|very rare}} + |{{Loot Item|Behemoth Trophy|very rare}} + |{{Loot Item|Amphora|very rare}} + |{{Loot Item|Crowbar|very rare}} + |{{Loot Item|Titan Axe|very rare}} + |{{Loot Item|War Axe|very rare}} +}} +| usespells = yes +| attacktype = Melee +}} \ No newline at end of file diff --git a/tests/resources/content_creature_black_sheep.txt b/tests/resources/content_creature_black_sheep.txt new file mode 100644 index 0000000..b952ffd --- /dev/null +++ b/tests/resources/content_creature_black_sheep.txt @@ -0,0 +1,53 @@ +{{Infobox Creature|List={{{1|}}}|GetValue={{{GetValue|}}} +| name = Black Sheep +| article = a +| actualname = black sheep +| plural = black sheep +| hp = 20 +| exp = 0 +| armor = 1 +| mitigation = 0.08 +| summon = 250 +| convince = 250 +| illusionable = yes +| creatureclass = Mammals +| primarytype = Ungulates +| bestiaryclass = Mammal +| bestiarylevel = Trivial +| occurrence = Uncommon +| spawntype = Regular +| isboss = no +| abilities = {{Ability List|None}} +| maxdmg = 0 +| pushable = yes +| pushobjects = no +| walksaround = Fire, Energy, Poison +| walksthrough = -- +| paraimmune = no +| senseinvis = no +| physicalDmgMod = 100% +| earthDmgMod = 100% +| fireDmgMod = 105% +| deathDmgMod = 100% +| energyDmgMod = 100% +| holyDmgMod = 100% +| iceDmgMod = 100% +| hpDrainDmgMod = 100% +| drownDmgMod = 100% +| healMod = 100% +| sounds = {{Sound List|Maeh}} +| implemented = 6.1 +| race_id = 13 +| notes = They are the same as [[sheep]], with the exception of their color.
To tame this creature, use a [[Reins]] on it. For additional details, refer to the article on [[Black Sheep (Mount)]].
By successfully taming the [[Black Sheep]] you will earn the achievement [[Little Ball of Wool]]. +| behaviour = Black sheep will always run. +| speed = 58 +| strategy = Just attack them, they are slow. +| location = [[Femor Hills]] ({{Mapper Coords|127.88|124.58|4|4|text=here}}), cultist cave in [[Liberty Bay]] ({{Mapper Coords|126.192|127.255|7|2|text=here}}), [[Drefia]] +| loot = {{Loot Table + |{{Loot Item|1-5|Meat|common}} + |{{Loot Item|Black Wool|rare}} +}} +| history = In the early years of Tibia, many myths surrounded the black sheep. They were fairly rare, with only a couple known spawns, the most famous being in the [[Femor Hills]]. +| usespells = no +| attacktype = Melee +}} \ No newline at end of file diff --git a/tests/resources/content_creature_warlord_ruzad.txt b/tests/resources/content_creature_warlord_ruzad.txt new file mode 100644 index 0000000..37367d8 --- /dev/null +++ b/tests/resources/content_creature_warlord_ruzad.txt @@ -0,0 +1,61 @@ +{{Infobox Creature|List={{{1|}}}|GetValue={{{GetValue|}}} +| name = Warlord Ruzad +| hp = 1500 +| exp = 1700 +| armor = ? +| summon = -- +| convince = -- +| illusionable = no +| creatureclass = Humanoids +| primarytype = Orcs +| spawntype = Raid, Unique +| isboss = yes +| isarenaboss = no +| abilities = {{Ability List| {{Melee|0-300?}}|{{Summon|Orc Berserker|2}} |{{Healing|range=?-75}}}} +| maxdmg = ? +| pushable = ? +| pushobjects = ? +| walksaround = Poison +| walksthrough = ? +| paraimmune = yes +| senseinvis = yes +| physicalDmgMod = 100%? +| earthDmgMod = 110% +| fireDmgMod = 20% +| deathDmgMod = 110% +| energyDmgMod = 99%Orc.Boss.jpg +Warlord Ruzad Menera.jpg|Warlord Ruzad on Menera + +| behaviour = Treat like a strong [[Orc Warlord]], but it [[Summon]]s [[Orc Berserker]]s. +| speed = ? +| strategy = Be careful when facing this creature. Don't let him [[Summon]], or you can die easily. The strategy is almost the same than an [[Orc Warlord]]. Using [[Stealth Ring]]s or [[Invisibility]] would prevent you from being damaged by the summoned [[Orc Berserker]]s, but not the [[Orc leader]]s. +| location = [[Orc Fortress]], in the tower near [[The Explorer Society Quest|Orc Powder Quest Room]], {{Mapper Coords|128.202,123.233,5-2|text=here}}. +| loot = {{Loot Table + |{{Loot Item|0-59|Gold Coin}} + |{{Loot Item|0-14|Throwing Star}} + |{{Loot Item|Meat}} + |{{Loot Item|Fish}} + |{{Loot Item|Orc Leather}} + |{{Loot Item|Hunting Spear}} + |{{Loot Item|Scimitar}} + |{{Loot Item|Brass Legs}} + |{{Loot Item|Plate Legs}} + |{{Loot Item|Dark Helmet}} + |{{Loot Item|Orcish Axe}} + |{{Loot Item|Protection Amulet}} + |{{Loot Item|Plate Armor}} + |{{Loot Item|Two Handed Sword}} + |{{Loot Item|Magma Boots}} + |{{Loot Item|Butcher's Axe}} +}} +| bosstiaryclass = Nemesis +}} \ No newline at end of file diff --git a/tests/resources/content_item_sneeze_blossom.txt b/tests/resources/content_item_sneeze_blossom.txt new file mode 100644 index 0000000..4342ad9 --- /dev/null +++ b/tests/resources/content_item_sneeze_blossom.txt @@ -0,0 +1,16 @@ +{{Infobox Object|List={{{1|}}}|GetValue={{{GetValue|}}} +| name = Sneeze Blossom +| article = +| itemid = 3891 +| objectclass = Flora and Minerals +| primarytype = Flowers +| secondarytype = Steppe Plants +| implemented = 7.5 +| immobile = +| walkable = yes +| walkingspeed = +| pickupable = no +| attrib = +| location = An example can be found at {{Mapper Coords|127.197|124.98|7|3|text=spot}}. +| notes = +}} \ No newline at end of file diff --git a/tests/tests_utils.py b/tests/tests_utils.py index cf6211a..0a4dbaa 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -3,10 +3,14 @@ import datetime +import mwparserfromhell +from mwparserfromhell.nodes import Template + import tibiawikisql from tests import load_resource -from tibiawikisql.utils import (clean_links, client_color_to_rgb, parse_boolean, parse_client_id, parse_client_ids, - parse_currency, +from tibiawikisql.utils import (clean_links, clean_location, client_color_to_rgb, parse_boolean, parse_client_id, + parse_client_ids, parse_currency, parse_int, parse_location_coordinate, + parse_location_coordinates, parse_mapper_coords, parse_first_integer, parse_float, parse_integer, parse_date, parse_loot_statistics, parse_min_max, parse_sounds, parse_known_boolean, parse_known_text, parse_weapon_proficiency_name, @@ -336,6 +340,147 @@ def test_parse_weapon_proficiency_tables_level_two_headings(self): self.assertIn(1, tables["Sanguine 1H Axe"]) self.assertEqual("+1 Axe Fighting", tables["Sanguine 1H Axe"][1][0]["effect"]) + def test_parse_location_coordinate(self): + self.assertEqual(31754, parse_location_coordinate("124.10")) + self.assertEqual(32000, parse_location_coordinate("125")) + self.assertEqual(32000, parse_location_coordinate("125.")) + self.assertEqual(32000, parse_location_coordinate(" 125 . ")) + self.assertIsNone(parse_location_coordinate("125.1.2")) + self.assertIsNone(parse_location_coordinate("128.202,123.233,5-2")) + self.assertIsNone(parse_location_coordinate("abc")) + self.assertIsNone(parse_location_coordinate("125.abc")) + self.assertIsNone(parse_location_coordinate("")) + + def test_parse_int(self): + self.assertEqual(7, parse_int("7")) + self.assertEqual(7, parse_int(" 7 ")) + self.assertEqual(-1, parse_int("-1")) + self.assertIsNone(parse_int("")) + self.assertIsNone(parse_int("7a")) + self.assertIsNone(parse_int("1,000")) + + def _mapper_coords(self, wikitext: str) -> Template: + return mwparserfromhell.parse(wikitext).filter_templates()[0] + + def test_parse_mapper_coords(self): + self.assertEqual( + ((127 << 8) + 88, (124 << 8) + 58, 4), + parse_mapper_coords(self._mapper_coords("{{Mapper Coords|127.88|124.58|4|4|text=here}}")), + ) + + def test_parse_mapper_coords_named_text_first(self): + template = self._mapper_coords("{{Mapper Coords|text=here|127.234|127.59|14|2|1|0.50.5}}") + + self.assertEqual(((127 << 8) + 234, (127 << 8) + 59, 14), parse_mapper_coords(template)) + + def test_parse_mapper_coords_malformed(self): + # Warlord Ruzad writes the whole set in one parameter. + self.assertIsNone(parse_mapper_coords(self._mapper_coords("{{Mapper Coords|128.202,123.233,5-2|text=here}}"))) + self.assertIsNone(parse_mapper_coords(self._mapper_coords("{{Mapper Coords|127.88|124.58}}"))) + self.assertIsNone(parse_mapper_coords(self._mapper_coords("{{Mapper Coords|127.88|124.58|?}}"))) + self.assertIsNone(parse_mapper_coords(self._mapper_coords("{{Mapper Coords|127.88|abc|7}}"))) + self.assertIsNone(parse_mapper_coords(self._mapper_coords("{{Mapper Coords}}"))) + + def test_parse_mapper_coords_ignores_comments(self): + template = self._mapper_coords("{{Mapper Coords|127.88|124.58|7 }}") + + self.assertEqual(((127 << 8) + 88, (124 << 8) + 58, 7), parse_mapper_coords(template)) + + def test_clean_location_uses_labels(self): + self.assertEqual( + "Femor Hills (here), cultist cave in Liberty Bay (here), Drefia", + clean_location( + "[[Femor Hills]] ({{Mapper Coords|127.88|124.58|4|4|text=here}}), cultist cave in [[Liberty Bay]] " + "({{Mapper Coords|126.192|127.255|7|2|text=here}}), [[Drefia]]", + ), + ) + self.assertEqual( + "in front of the mirror, exactly here.", + clean_location("in front of the mirror, {{Mapper Coords|128.204|126.162|15|8|text= exactly here }}."), + ) + self.assertEqual( + "Orc Fortress, here.", + clean_location("[[Orc Fortress]], {{Mapper Coords|128.202,123.233,5-2|text=here}}."), + ) + self.assertEqual("the Fortress here", clean_location("the {{Mapper Coords|1.1|1.1|7|text=[[Fortress]]}} here")) + + def test_clean_location_matches_template_name_like_find_templates(self): + self.assertEqual("at here.", clean_location("at {{mapper_coords|127.88|124.58|4|text=here}}.")) + self.assertEqual("at here.", clean_location("at {{ Mapper Coords |127.88|124.58|4|text=here}}.")) + + def test_clean_location_removes_unlabelled(self): + self.assertEqual("Mintwallin.", clean_location("[[Mintwallin]] {{Mapper Coords|126.167|125.95|15}}.")) + self.assertEqual("Mintwallin", clean_location("[[Mintwallin]] {{Mapper Coords|126.167|125.95|15}}")) + self.assertEqual("Mintwallin.", clean_location("[[Mintwallin]] {{Mapper Coords|126.167|125.95|15|text=}}.")) + self.assertEqual( + "single spawn, double spawn", + clean_location("single spawn {{Mapper Coords|127.220|127.89|14}}, double spawn"), + ) + self.assertEqual( + "Tyrsung, Okolnir and Helheim.", + clean_location( + "[[Tyrsung]] ({{Mapper Coords|126.76|121.254|7}}), [[Okolnir]] ( {{Mapper Coords|125.224|122.150|7}} ) " + "and [[Helheim]].", + ), + ) + self.assertEqual( + "in a cave that is north", clean_location("in a cave {{Mapper Coords|1.1|1.1|7}} that is north"), + ) + self.assertEqual("north of the town", clean_location("{{Mapper Coords|1.1|1.1|7}} north of the town")) + self.assertEqual( + "Folda.\nnorth", + clean_location("[[Folda]] {{Mapper Coords|1.1|1.1|7}}.\n{{Mapper Coords|1.1|1.1|7}} north"), + ) + self.assertEqual("Folda.", clean_location("[[Folda]] {{Mapper Coords|1.1|1.1|7}} {{Mapper Coords|1.2|1.2|7}}.")) + self.assertEqual( + "Folda (and here)", + clean_location("[[Folda]] ({{Mapper Coords|1.1|1.1|7}} and {{Mapper Coords|1.2|1.2|7|text=here}})"), + ) + + def test_clean_location_replaces_a_template_inside_a_label(self): + self.assertEqual( + "Femor Hills (here)", + clean_location("[[Femor Hills]] ({{Mapper Coords|1.1|1.1|7|text={{Mapper Coords|1.2|1.2|7|text=here}}}})"), + ) + self.assertEqual( + "the Fortress", + clean_location( + "the {{Mapper Coords|1.1|1.1|7|text=[[Fortress|{{Mapper Coords|1.2|1.2|7|text=Fortress}}]]}}", + ), + ) + + def test_clean_location_without_mapper_coords_is_clean_links(self): + for value in ( + "[[Hero Cave]], [[Kharos|Ferumbras' Citadel]], deep in [[Pits of Inferno]] ( ) .", + "Isle ( {{Hover|x|y}} ) , here", + "[[Svargrond]] , {{Mapper Coord|1.1|1.1|7}}", + "", + ): + with self.subTest(value=value): + self.assertEqual(clean_links(value), clean_location(value)) + + def test_parse_location_coordinates(self): + value = ( + "[[Deeper Banuta]] (single spawn {{Mapper Coords|127.220|127.89|14|2|text=here}}, double spawn " + "{{Mapper Coords|text=here|127.234|127.59|14|2|1|0.50.5}}), " + "{{Mapper Coords|128.202,123.233,5-2|text=here}}, " + "{{Mapper Coords|128.1|123.94|11|4}}, {{Mapper Coords|128.204|126.162|15|8|text=[[Mirror|exactly here]]}}" + ) + + self.assertEqual( + [ + ((127 << 8) + 220, (127 << 8) + 89, 14, "here"), + ((127 << 8) + 234, (127 << 8) + 59, 14, "here"), + ((128 << 8) + 1, (123 << 8) + 94, 11, None), + ((128 << 8) + 204, (126 << 8) + 162, 15, "exactly here"), + ], + parse_location_coordinates(value), + ) + + def test_parse_location_coordinates_none(self): + self.assertEqual([], parse_location_coordinates("[[Thais]], [[Carlin]] (128.58)")) + self.assertEqual([], parse_location_coordinates("")) + def test_version_is_tibia_sh_build(self): # A release PR sets X.Y.Z+tibiash.N. An upstream sync takes upstream's bare X.Y.Z until the next release. self.assertRegex(tibiawikisql.__version__, r"\A[0-9]+\.[0-9]+\.[0-9]+(\+tibiash\.[0-9]+)?\Z") diff --git a/tibiawikisql/models/__init__.py b/tibiawikisql/models/__init__.py index 88af3e6..1f2d684 100644 --- a/tibiawikisql/models/__init__.py +++ b/tibiawikisql/models/__init__.py @@ -2,6 +2,7 @@ """Contains all the models representing TibiaWiki articles.""" from tibiawikisql.models.achievement import Achievement +from tibiawikisql.models.base import LocationCoordinate from tibiawikisql.models.charm import Charm from tibiawikisql.models.creature import Creature, CreatureAbility, CreatureDrop, CreatureMaxDamage from tibiawikisql.models.house import House diff --git a/tibiawikisql/models/base.py b/tibiawikisql/models/base.py index ced4126..5a60dc0 100644 --- a/tibiawikisql/models/base.py +++ b/tibiawikisql/models/base.py @@ -1,3 +1,4 @@ +# Changed by tibia.sh in 2026. See "About this copy" in README.md. """Module with base classes used by models.""" from __future__ import annotations @@ -33,6 +34,19 @@ class WithImage(BaseModel): """An image representing this article.""" +class LocationCoordinate(BaseModel): + """A map position written with ``{{Mapper Coords}}`` in a location.""" + + x: int + """The x coordinate of the position.""" + y: int + """The y coordinate of the position.""" + z: int + """The floor of the position.""" + label: str | None = None + """The text the location shows for the position, like ``here``. ``None`` if it shows none.""" + + class RowModel(BaseModel): """A mixin class to indicate that this model comes from a SQL table.""" diff --git a/tibiawikisql/models/creature.py b/tibiawikisql/models/creature.py index b9fa2ef..eb8d305 100644 --- a/tibiawikisql/models/creature.py +++ b/tibiawikisql/models/creature.py @@ -10,6 +10,7 @@ from tibiawikisql.api import WikiEntry from tibiawikisql.models.base import ( + LocationCoordinate, RowModel, WithImage, WithStatus, @@ -18,6 +19,7 @@ from tibiawikisql.schema import ( CreatureAbilityTable, CreatureDropTable, + CreatureLocationCoordinateTable, CreatureMaxDamageTable, CreatureSoundTable, CreatureTable, @@ -253,7 +255,9 @@ class Creature(WikiEntry, WithStatus, WithVersion, WithImage, RowModel, table=Cr sounds: list[str] = Field([]) """The "sounds" made by the creature.""" location: str | None - """The locations where the creature can be found.""" + """The locations where the creature can be found. Each map position is replaced by its label, if it has one.""" + location_coordinates: list[LocationCoordinate] = Field(default_factory=list) + """The map positions in the creature's location, in the order the wiki lists them.""" race_id: int | None = None """The client race ID of the creature.""" loot_statistics_kills: int | None = None @@ -310,6 +314,10 @@ def insert(self, conn: Connection | Cursor) -> None: CreatureSoundTable.insert(conn, creature_id=self.article_id, content=sound) for ability in self.abilities: CreatureAbilityTable.insert(conn, creature_id=self.article_id, **ability.model_dump()) + for position, coordinate in enumerate(self.location_coordinates): + CreatureLocationCoordinateTable.insert( + conn, creature_id=self.article_id, position=position, **coordinate.model_dump(), + ) if self.max_damage: CreatureMaxDamageTable.insert(conn, creature_id=self.article_id, **self.max_damage.model_dump()) @@ -328,6 +336,11 @@ def get_one_by_field(cls, conn: Connection | Cursor, field: str, value: Any, use abilities = CreatureAbilityTable.get_list_by_field(conn, "creature_id", creature.article_id) creature.abilities = [CreatureAbility(**dict(r)) for r in abilities] + coordinates = CreatureLocationCoordinateTable.get_list_by_field( + conn, "creature_id", creature.article_id, sort_by="position", + ) + creature.location_coordinates = [LocationCoordinate(**dict(r)) for r in coordinates] + drops = CreatureDropTable.get_by_creature_id(conn, creature.article_id) creature.loot = [CreatureDrop(**dict(r)) for r in drops] return creature diff --git a/tibiawikisql/models/item.py b/tibiawikisql/models/item.py index 83fbb67..195ada2 100644 --- a/tibiawikisql/models/item.py +++ b/tibiawikisql/models/item.py @@ -10,13 +10,14 @@ from typing_extensions import Self from tibiawikisql.api import WikiEntry -from tibiawikisql.models.base import RowModel, WithImage, WithStatus, WithVersion +from tibiawikisql.models.base import LocationCoordinate, RowModel, WithImage, WithStatus, WithVersion from tibiawikisql.schema import ( BookTable, CreatureDropTable, ItemAttributeTable, ItemClientIdTable, ItemKeyTable, + ItemLocationCoordinateTable, ItemProficiencyPerkTable, ItemSoundTable, ItemStoreOfferTable, @@ -165,6 +166,10 @@ class Item(WikiEntry, WithVersion, WithStatus, WithImage, RowModel, table=ItemTa """The internal id of the item in the client.""" client_ids: list[int] = Field(default_factory=list) """Every client id of the item, in the order the wiki lists them.""" + location: str | None = None + """Where the item can be found. Each map position is replaced by its label, if it has one.""" + location_coordinates: list[LocationCoordinate] = Field(default_factory=list) + """The map positions in the item's location, in the order the wiki lists them.""" attributes: list[ItemAttribute] = Field(default_factory=list) """The item's attributes.""" dropped_by: list[ItemDrop] = Field(default_factory=list) @@ -299,6 +304,10 @@ def insert(self, conn: Connection | Cursor) -> None: ItemSoundTable.insert(conn, item_id=self.article_id, content=sound) for position, client_id in enumerate(self.client_ids): ItemClientIdTable.insert(conn, item_id=self.article_id, client_id=client_id, position=position) + for position, coordinate in enumerate(self.location_coordinates): + ItemLocationCoordinateTable.insert( + conn, item_id=self.article_id, position=position, **coordinate.model_dump(), + ) for offer in self.store_offers: ItemStoreOfferTable.insert(conn, item_id=self.article_id, **offer.model_dump()) @@ -330,6 +339,11 @@ def get_one_by_field(cls, conn: Connection | Cursor, field: str, value: Any, use client_ids = ItemClientIdTable.get_list_by_field(conn, "item_id", item.article_id, sort_by="position") item.client_ids = [r["client_id"] for r in client_ids] + coordinates = ItemLocationCoordinateTable.get_list_by_field( + conn, "item_id", item.article_id, sort_by="position", + ) + item.location_coordinates = [LocationCoordinate(**dict(r)) for r in coordinates] + item.bought_by = [ ItemOffer( npc_id=r["npc_id"], diff --git a/tibiawikisql/parsers/creature.py b/tibiawikisql/parsers/creature.py index d75de6d..a2fbc6d 100644 --- a/tibiawikisql/parsers/creature.py +++ b/tibiawikisql/parsers/creature.py @@ -7,6 +7,7 @@ import tibiawikisql.schema from tibiawikisql.api import Article +from tibiawikisql.models.base import LocationCoordinate from tibiawikisql.models.creature import ( Creature, CreatureAbility, @@ -16,7 +17,7 @@ from tibiawikisql.parsers import BaseParser from tibiawikisql.parsers.base import AttributeParser from tibiawikisql.utils import ( - clean_links, + clean_location, find_template, number_pattern, parse_boolean, @@ -25,6 +26,7 @@ parse_integer, parse_known_boolean, parse_known_text, + parse_location_coordinates, parse_min_max, parse_sounds, strip_code, @@ -482,7 +484,7 @@ class CreatureParser(BaseParser): "modifier_healing": AttributeParser.optional("healMod", parse_known_integer), "walks_through": AttributeParser.optional("walksthrough", parse_monster_walks), "walks_around": AttributeParser.optional("walksaround", parse_monster_walks), - "location": AttributeParser.optional("location", clean_links), + "location": AttributeParser.optional("location", clean_location), "race_id": AttributeParser.optional("race_id", parse_client_id), "version": AttributeParser.optional("implemented"), "status": AttributeParser.status(), @@ -512,6 +514,11 @@ def parse_attributes(cls, article: Article) -> dict[str, Any]: row["loot"] = loot_items if "sounds" in raw_attributes: row["sounds"] = parse_sounds(raw_attributes["sounds"]) + if "location" in raw_attributes: + row["location_coordinates"] = [ + LocationCoordinate(x=x, y=y, z=z, label=label) + for x, y, z, label in parse_location_coordinates(raw_attributes["location"]) + ] if "abilities" in raw_attributes: abilities = parse_abilities(raw_attributes["abilities"]) if abilities: diff --git a/tibiawikisql/parsers/item.py b/tibiawikisql/parsers/item.py index 13a4b35..2664c8e 100644 --- a/tibiawikisql/parsers/item.py +++ b/tibiawikisql/parsers/item.py @@ -4,6 +4,7 @@ import mwparserfromhell from tibiawikisql.api import Article +from tibiawikisql.models.base import LocationCoordinate from tibiawikisql.models.item import Item, ItemAttribute, ItemStoreOffer from tibiawikisql.parsers import BaseParser from tibiawikisql.parsers.base import AttributeParser @@ -11,6 +12,7 @@ from tibiawikisql.utils import ( GOLD_COIN, clean_links, + clean_location, clean_question_mark, client_color_to_rgb, find_templates, @@ -23,6 +25,7 @@ parse_integer, parse_known_boolean, parse_known_text, + parse_location_coordinates, parse_sounds, strip_code, ) @@ -88,6 +91,7 @@ class ItemParser(BaseParser): "light_radius": AttributeParser.optional("lightradius", parse_integer), "version": AttributeParser.optional("implemented"), "client_id": AttributeParser.optional("itemid", parse_first_integer), + "location": AttributeParser.optional("location", clean_location), "status": AttributeParser.status(), } @@ -165,6 +169,7 @@ def parse_attributes(cls, article: Article) -> dict[str, Any]: cls.parse_restores(row) cls.parse_sounds(row) cls.parse_client_ids(row) + cls.parse_location_coordinates(row) cls.parse_store_value(row) return row @@ -322,6 +327,16 @@ def parse_client_ids(cls, row: dict[str, Any]) -> None: return row["client_ids"] = parse_client_ids(row["_raw_attributes"]["itemid"]) + @classmethod + def parse_location_coordinates(cls, row: dict[str, Any]) -> None: + """Read the map positions in the item's ``location`` into ``location_coordinates``, in listed order.""" + if "location" not in row["_raw_attributes"]: + return + row["location_coordinates"] = [ + LocationCoordinate(x=x, y=y, z=z, label=label) + for x, y, z, label in parse_location_coordinates(row["_raw_attributes"]["location"]) + ] + @classmethod def parse_store_value(cls, row): if "storevalue" not in row["_raw_attributes"]: diff --git a/tibiawikisql/parsers/npc.py b/tibiawikisql/parsers/npc.py index e24435f..8ab9957 100644 --- a/tibiawikisql/parsers/npc.py +++ b/tibiawikisql/parsers/npc.py @@ -8,7 +8,14 @@ from tibiawikisql.models.npc import Npc, NpcDestination, NpcLocation from tibiawikisql.parsers import BaseParser from tibiawikisql.parsers.base import AttributeParser -from tibiawikisql.utils import clean_links, convert_tibiawiki_position, find_template, strip_code +from tibiawikisql.utils import ( + clean_links, + convert_tibiawiki_position, + find_template, + parse_int, + parse_location_coordinate, + strip_code, +) ORIGIN_NOTE_PATTERN = re.compile(r"^\s*[Ff]rom\s+\[\[([^\]|]+)(?:\|[^\]]*)?\]\]\s*$") """A TransportCell note that consists only of a link to the place the leg starts from.""" @@ -104,35 +111,11 @@ def _parse_locations(cls, row: dict[str, Any]) -> None: city=clean_links(fields["city"]) or None, subarea=clean_links(fields["subarea"]) or None, geolabel=clean_links(fields["geolabel"]) or None, - x=cls._parse_location_coordinate(fields["posx"]), - y=cls._parse_location_coordinate(fields["posy"]), - z=cls._parse_int(fields["posz"]), + x=parse_location_coordinate(fields["posx"]), + y=parse_location_coordinate(fields["posy"]), + z=parse_int(fields["posz"]), )) - @classmethod - def _parse_location_coordinate(cls, value: str) -> int | None: - """Convert a TibiaWiki ``major.minor`` coordinate, returning ``None`` for anything malformed. - - Unlike :func:`convert_tibiawiki_position`, which falls back to 0, this rejects a non-integer part - or more than two parts. An absent or empty minor counts as 0. - """ - parts = value.split(".") - if len(parts) > 2: - return None - major = cls._parse_int(parts[0]) - minor = cls._parse_int(parts[1]) if len(parts) == 2 and parts[1].strip() else 0 - if major is None or minor is None: - return None - return (major << 8) + minor - - @classmethod - def _parse_int(cls, value: str) -> int | None: - """Convert a string to an integer, returning ``None`` if it is not one.""" - try: - return int(value) - except ValueError: - return None - @classmethod def _parse_shuttle_starts(cls, destinations: list[str], locations: list[NpcLocation]) -> list[str | None]: """Find where each leg of a shuttle NPC starts. diff --git a/tibiawikisql/schema.py b/tibiawikisql/schema.py index 9d2b0c1..f99d497 100644 --- a/tibiawikisql/schema.py +++ b/tibiawikisql/schema.py @@ -129,6 +129,16 @@ class CreatureSoundTable(Table, table_name="creature_sound"): content = Column(Text, nullable=False) +class CreatureLocationCoordinateTable(Table, table_name="creature_location_coordinate"): + """Contains the map positions in a creature's location, in the order the wiki lists them.""" + creature_id = Column(ForeignKey(Integer, table="creature", column="article_id"), index=True) + x = Column(Integer, nullable=False) + y = Column(Integer, nullable=False) + z = Column(Integer, nullable=False) + label = Column(Text) + position = Column(Integer, nullable=False) + + class ItemTable(Table): """Contains information about items and objects.""" article_id = Column(Integer, primary_key=True) @@ -155,6 +165,7 @@ class ItemTable(Table): light_radius = Column(Integer) version = Column(Text, index=True) client_id = Column(Integer) + location = Column(Text) image = Column(Blob) status = Column(Text, default="active", nullable=False) timestamp = Column(Timestamp, nullable=False) @@ -173,6 +184,16 @@ class ItemClientIdTable(Table, table_name="item_client_id"): position = Column(Integer, nullable=False) +class ItemLocationCoordinateTable(Table, table_name="item_location_coordinate"): + """Contains the map positions in an item's location, in the order the wiki lists them.""" + item_id = Column(ForeignKey(Integer, table="item", column="article_id"), index=True) + x = Column(Integer, nullable=False) + y = Column(Integer, nullable=False) + z = Column(Integer, nullable=False) + label = Column(Text) + position = Column(Integer, nullable=False) + + class ItemStoreOfferTable(Table, table_name="item_store_offer"): """Contains the Tibia store offers for an item.""" item_id = Column(ForeignKey(Integer, table="item", column="article_id"), index=True) diff --git a/tibiawikisql/utils.py b/tibiawikisql/utils.py index e736e7b..8ab526b 100644 --- a/tibiawikisql/utils.py +++ b/tibiawikisql/utils.py @@ -48,6 +48,23 @@ non_currency_link_pattern = re.compile(r"\s*:?\s*(?:File|Image|Category)\s*:", re.IGNORECASE) """The target of a link that cannot name a currency: a file, an image or a category.""" +MAPPER_COORDS = "mapper coords" +"""The name of the template that links a map position, in lower case with spaces.""" + +REMOVED_MAPPER_COORDS = "\x00" +"""Marks where a Mapper Coords template without a label was removed. The parser ends a text at this character, so a +parsed location never holds one of its own.""" + +removed_mapper_coords_rules = ( + (re.compile(r"[ \t]*\([ \t]*\x00+[ \t]*\)"), ""), + (re.compile(r"(^|\()[ \t]*\x00+[ \t]*", re.MULTILINE), r"\1"), + (re.compile(r"[ \t]*\x00+(?=[\s,.;:!?)]|\Z)"), ""), + (re.compile(r"\x00+"), ""), +) +"""How the text around a removed Mapper Coords template is tidied, in order: parentheses left empty go with the spaces +before them; at the start of a line or after an opening parenthesis, the spaces after it go; before a space, +punctuation or the end of the text, the spaces before it go; anywhere else only the template goes.""" + class Elapsed: """Holds the elapsed time measured by the [tibiawikisql.utils.timed][timed] context manager. @@ -171,6 +188,114 @@ def convert_tibiawiki_position(pos: str) -> int: return 0 +def parse_location_coordinate(value: str) -> int | None: + """Convert a TibiaWiki ``major.minor`` coordinate, returning ``None`` for anything malformed. + + Unlike :func:`convert_tibiawiki_position`, which falls back to 0, this rejects a non-integer part + or more than two parts. An absent or empty minor counts as 0. + """ + parts = value.split(".") + if len(parts) > 2: + return None + major = parse_int(parts[0]) + minor = parse_int(parts[1]) if len(parts) == 2 and parts[1].strip() else 0 + if major is None or minor is None: + return None + return (major << 8) + minor + + +def parse_int(value: str) -> int | None: + """Convert a string to an integer, returning ``None`` if it is not one.""" + try: + return int(value) + except ValueError: + return None + + +def is_mapper_coords(template: Template) -> bool: + """Check whether a template is a ``{{Mapper Coords}}``, matching its name as :func:`find_templates` does.""" + return strip_code(template.name).lower().replace("_", " ") == MAPPER_COORDS + + +def parse_mapper_coords(template: Template) -> tuple[int, int, int] | None: + """Read the position a ``{{Mapper Coords|x|y|z|...}}`` template points to. + + ``x`` and ``y`` are TibiaWiki ``major.minor`` coordinates, read by :func:`parse_location_coordinate`, and ``z`` is + the floor, read by :func:`parse_int`. Named parameters such as ``text`` do not count as positions, so + ``{{Mapper Coords|text=here|127.234|127.59|14}}`` is read like ``{{Mapper Coords|127.234|127.59|14|text=here}}``. + Wiki comments in the values are ignored. + + Args: + template: The Mapper Coords template. + + Returns: + The ``x``, ``y`` and ``z`` of the position, or ``None`` when any of them is missing or malformed, as in + Warlord Ruzad's ``{{Mapper Coords|128.202,123.233,5-2|text=here}}``. + + """ + values = [visible_text(str(template.get(name).value)) if template.has(name) else "" for name in ("1", "2", "3")] + x = parse_location_coordinate(values[0]) + y = parse_location_coordinate(values[1]) + z = parse_int(values[2]) + if x is None or y is None or z is None: + return None + return x, y, z + + +def _mapper_coords_label(template: Template) -> str: + """Get the raw ``text`` parameter of a Mapper Coords template, stripped, or an empty string without one.""" + return str(template.get("text").value).strip() if template.has("text") else "" + + +def clean_location(value: str) -> str: + """Clean a location as :func:`clean_links` does, putting each ``{{Mapper Coords}}`` template's label in its place. + + A Mapper Coords template is replaced by its ``text`` parameter, so ``Femor Hills ({{Mapper Coords|127.88|124.58|4| + text=here}})`` reads ``Femor Hills (here)``, and no coordinate is left in the text. A template without a label is + removed. Where one is removed, the spaces before it go too when a space, punctuation or the end of the text + follows, so ``Mintwallin {{Mapper Coords|...}}.`` reads ``Mintwallin.``; at the start of a line or after an opening + parenthesis, the spaces after it go; and parentheses left with nothing in them are removed with the spaces before + them. The rest of the text is cleaned exactly as :func:`clean_links` cleans it. + + Args: + value: The raw location. + + Returns: + The clean location. + + """ + code = mwparserfromhell.parse(value) + # Innermost first, so a template inside another's label is replaced before the label takes the outer one's place. + for template in reversed(code.filter_templates(matches=is_mapper_coords)): + code.replace(template, _mapper_coords_label(template) or Text(REMOVED_MAPPER_COORDS)) + text = str(code) + for pattern, replacement in removed_mapper_coords_rules: + text = pattern.sub(replacement, text) + return clean_links(text) + + +def parse_location_coordinates(value: str) -> list[tuple[int, int, int, str | None]]: + """Read the positions of the ``{{Mapper Coords}}`` templates in a location, in the order they are written. + + Args: + value: The raw location. + + Returns: + The ``x``, ``y``, ``z`` and label of each template whose position :func:`parse_mapper_coords` reads. The label + is the plain text of the ``text`` parameter, or ``None`` when it is absent or empty. A template whose position + is malformed gives nothing. + + """ + coordinates = [] + for template in mwparserfromhell.parse(value).filter_templates(matches=is_mapper_coords): + position = parse_mapper_coords(template) + if position is None: + continue + label = strip_code(mwparserfromhell.parse(_mapper_coords_label(template))) or None + coordinates.append((*position, label)) + return coordinates + + def find_template(content: str, template_name: str, partial: bool = False, recursive: bool = False) -> Template | None: """Find a template in a string containing wiki code. From 39c17a7747fb95152eaa1febf687224981cc42f5 Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 20:37:57 +0000 Subject: [PATCH 10/17] feat: record the item trade value note --- CHANGELOG.md | 3 + README.md | 2 + docs/schema.md | 1 + tests/models/test_item.py | 26 ++++++++ tests/parsers/test_item.py | 59 +++++++++++++++++++ tests/resources/content_item_amazon_armor.txt | 31 ++++++++++ tibiawikisql/models/item.py | 2 + tibiawikisql/parsers/item.py | 20 +++++++ tibiawikisql/schema.py | 1 + 9 files changed, 145 insertions(+) create mode 100644 tests/resources/content_item_amazon_armor.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d997ef..9307735 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,9 @@ read from the wiki's `major.minor` form, so `127.88` is 32600, and a malformed set like Warlord Ruzad's `128.202,123.233,5-2` gives no row, while its label still replaces it in the text. `item.location` is the item's `location`, cleaned the same way. +- Add `item.trade_value_note`, the item's `value` as the wiki writes it, with links and templates cleaned and wiki + comments dropped, like `Negotiable`, `Negotiable gp` or Amazon Armor's `55,000,000 - 70,000,000`. It is never read as + a number, so a range or `0` is kept as written, and it is `NULL` when the value is absent, empty, `?` or `--`. ## 9.0.0+tibiash.9 diff --git a/README.md b/README.md index a08476a..c6d6ca3 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ The copy adds this data to the database: * The `item.location` column and the `creature_location_coordinate` and `item_location_coordinate` tables. The tables keep the map positions written with `{{Mapper Coords}}` in a creature's or item's location, with their labels, and the location text shows each label in place of its position. +* The `item.trade_value_note` column. It holds what players trade an item for, as the wiki writes it, like `Negotiable` + or `55,000,000 - 70,000,000`. The [database schema](docs/schema.md) describes each of them. diff --git a/docs/schema.md b/docs/schema.md index 4033ad9..a6b9500 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -374,6 +374,7 @@ replaced by its label, or left out when it has none, so no coordinate is left in | version | `TEXT` | The client version this item was introduced to the game. | | client_id | `INTEGER` | The client id of the item, the first integer in `itemid`. `item_client_id` has every client ID. | | location | `TEXT` | Where the item can be found, cleaned as `creature.location` is. `item_location_coordinate` has its map positions. | +| trade_value_note | `TEXT` | What players trade this item for, the `value` as written with links and templates cleaned, like `Negotiable` or `55,000,000 - 70,000,000`. `NULL` when it is empty, `?` or `--`. | | image | `BLOB` | The item’s image bytes. | | status | `TEXT` | The status of the item in game. | | timestamp | `TIMESTAMP` | ISO 8601 timestamp of the article's last edit. | diff --git a/tests/models/test_item.py b/tests/models/test_item.py index 48ebbb7..5fef8a6 100644 --- a/tests/models/test_item.py +++ b/tests/models/test_item.py @@ -212,6 +212,32 @@ def test_item_location_round_trip_none(self): self.assertEqual([], loaded.location_coordinates) self.assertEqual([], self.conn.execute("SELECT * FROM item_location_coordinate").fetchall()) + def test_item_trade_value_note_round_trip(self): + article = Article( + article_id=1, + title="Amazon Armor", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=load_resource("content_item_amazon_armor.txt"), + ) + ItemParser.from_article(article).insert(self.conn) + + loaded = Item.get_one_by_field(self.conn, "article_id", 1) + + self.assertEqual("55,000,000 - 70,000,000", loaded.trade_value_note) + + def test_item_trade_value_note_round_trip_none(self): + article = Article( + article_id=1, + title="Shallow Water", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=load_resource("content_item_shallow_water.txt"), + ) + ItemParser.from_article(article).insert(self.conn) + + loaded = Item.get_one_by_field(self.conn, "article_id", 1) + + self.assertIsNone(loaded.trade_value_note) + def _insert_amethyst_necklace_with_itemid(self, itemid: str) -> None: content = load_resource("content_item_amethyst_necklace.txt") self.assertIn("| itemid = 35609", content) diff --git a/tests/parsers/test_item.py b/tests/parsers/test_item.py index 46ed935..12fb35d 100644 --- a/tests/parsers/test_item.py +++ b/tests/parsers/test_item.py @@ -6,6 +6,7 @@ from tibiawikisql.api import Article from tibiawikisql.models import Item from tibiawikisql.parsers import ItemParser +from tibiawikisql.parsers.item import parse_trade_value_note class TestItemParserRestores(unittest.TestCase): @@ -459,3 +460,61 @@ def test_no_location(self): self.assertIsNone(item.location) self.assertEqual([], item.location_coordinates) + + +class TestItemParserTradeValueNote(unittest.TestCase): + def _parse(self, title: str, content: str) -> Item: + article = Article( + article_id=1, + title=title, + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=content, + ) + return ItemParser.from_article(article) + + def _edited(self, value: str) -> Item: + content = load_resource("content_item_amethyst_necklace.txt") + old = "| value = Negotiable\n" + self.assertIn(old, content) + return self._parse("Amethyst Necklace", content.replace(old, value)) + + def test_fixtures_keep_the_note_as_written(self): + expected = { + "content_item_amazon_armor.txt": "55,000,000 - 70,000,000", + "content_item_amethyst_necklace.txt": "Negotiable", + "content_item_damage_reflection.txt": "Negotiable gp", + "content_item.txt": "4,000 - 8,000", + "content_item_store.txt": "< 50", + "content_item_adventurers_stone.txt": "0", + "content_item_potion_superior_mana_potion.txt": None, + "content_item_shallow_water.txt": None, + } + for resource, note in expected.items(): + with self.subTest(resource=resource): + self.assertEqual(note, self._parse(resource, load_resource(resource)).trade_value_note) + + def test_unknown(self): + self.assertIsNone(self._edited("| value = ?\n").trade_value_note) + self.assertIsNone(self._edited("| value = --\n").trade_value_note) + self.assertIsNone(self._edited("| value = ? \n").trade_value_note) + self.assertIsNone(self._edited("| value = \n").trade_value_note) + + def test_absent(self): + self.assertIsNone(self._edited("").trade_value_note) + + def test_cleans_links_and_templates(self): + self.assertEqual( + "About 5,000 or Negotiable", + self._edited("| value = About [[Gold Coin|5,000]] or {{Hover|Negotiable|ask}} \n") + .trade_value_note, + ) + + def test_parse_trade_value_note(self): + self.assertEqual("Negotiable", parse_trade_value_note(" Negotiable ")) + self.assertEqual("55,000,000 - 70,000,000", parse_trade_value_note("55,000,000 - 70,000,000")) + self.assertEqual("Negotiable?", parse_trade_value_note("Negotiable?")) + self.assertEqual("-", parse_trade_value_note("-")) + self.assertIsNone(parse_trade_value_note("?")) + self.assertIsNone(parse_trade_value_note("--")) + self.assertIsNone(parse_trade_value_note("")) + self.assertIsNone(parse_trade_value_note("")) diff --git a/tests/resources/content_item_amazon_armor.txt b/tests/resources/content_item_amazon_armor.txt new file mode 100644 index 0000000..19108c7 --- /dev/null +++ b/tests/resources/content_item_amazon_armor.txt @@ -0,0 +1,31 @@ +{{Infobox Object|List={{{1|}}}|GetValue={{{GetValue|}}} +| name = Amazon Armor +| article = an +| actualname = amazon armor +| plural = amazon armors +| itemid = 3394 +| objectclass = Body Equipment +| primarytype = Armors +| slot = Body +| implemented = 7.3 +| immobile = no +| walkable = yes +| pickupable = yes +| levelrequired = 60 +| vocrequired = paladins +| imbueslots = 2 +| upgradeclass = 4 +| attrib = distance fighting +3 +| armor = 13 +| weight = 45.00 +| stackable = no +| marketable = yes +| droppedby = {{Dropped By|Orc Warlord}} +| value = 55,000,000 - 70,000,000 +| npcvalue = 0 +| npcprice = 0 +| buyfrom = -- +| sellto = -- +| notes = It is only dropped during orc raids on [[Carlin Raids#Orc Raid on Femor Hills|Femor Hills]]. It is less rare than the [[Amazon Shield]], but still would take you 5 years on average to loot one if you killed every orc raid occurring. It is therefore extremely valuable. It is part of the [[Amazon Set]]. +| history = Although this item was implemented in [[Updates/7.3|Summer Update 2004]], it became obtainable only after [[Updates/7.4|Christmas Update 2004]] with the implementation of raids. +}} \ No newline at end of file diff --git a/tibiawikisql/models/item.py b/tibiawikisql/models/item.py index 195ada2..dc5aa97 100644 --- a/tibiawikisql/models/item.py +++ b/tibiawikisql/models/item.py @@ -170,6 +170,8 @@ class Item(WikiEntry, WithVersion, WithStatus, WithImage, RowModel, table=ItemTa """Where the item can be found. Each map position is replaced by its label, if it has one.""" location_coordinates: list[LocationCoordinate] = Field(default_factory=list) """The map positions in the item's location, in the order the wiki lists them.""" + trade_value_note: str | None = None + """What players trade the item for, as the wiki writes it, like `Negotiable`. `None` if unknown.""" attributes: list[ItemAttribute] = Field(default_factory=list) """The item's attributes.""" dropped_by: list[ItemDrop] = Field(default_factory=list) diff --git a/tibiawikisql/parsers/item.py b/tibiawikisql/parsers/item.py index 2664c8e..bc4dc0e 100644 --- a/tibiawikisql/parsers/item.py +++ b/tibiawikisql/parsers/item.py @@ -63,6 +63,25 @@ NOTE_PATTERN = re.compile(r"\([^()]*\)") """A note in parentheses, without parentheses inside.""" +UNKNOWN_TRADE_VALUES = frozenset(("", "?", "--")) +"""The cleaned ``value`` texts that say nothing about an item's trade value.""" + + +def parse_trade_value_note(value: str) -> str | None: + """Parse what players trade an item for, from its ``value`` field. + + Args: + value: The raw value of the field. + + Returns: + The value as written, with links and templates cleaned by :func:`clean_links`, which also drops wiki comments, + like ``Negotiable`` or ``55,000,000 - 70,000,000``. It is never read as a number. ``None`` when the value is + empty, ``?`` or ``--``. + + """ + note = clean_links(value) + return None if note in UNKNOWN_TRADE_VALUES else note + class ItemParser(BaseParser): """Parses items and objects.""" @@ -92,6 +111,7 @@ class ItemParser(BaseParser): "version": AttributeParser.optional("implemented"), "client_id": AttributeParser.optional("itemid", parse_first_integer), "location": AttributeParser.optional("location", clean_location), + "trade_value_note": AttributeParser.optional("value", parse_trade_value_note), "status": AttributeParser.status(), } diff --git a/tibiawikisql/schema.py b/tibiawikisql/schema.py index f99d497..8c0a816 100644 --- a/tibiawikisql/schema.py +++ b/tibiawikisql/schema.py @@ -166,6 +166,7 @@ class ItemTable(Table): version = Column(Text, index=True) client_id = Column(Integer) location = Column(Text) + trade_value_note = Column(Text) image = Column(Blob) status = Column(Text, default="active", nullable=False) timestamp = Column(Timestamp, nullable=False) From eba4649058e0fe139b90b106ca46ab927250bed2 Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 20:50:11 +0000 Subject: [PATCH 11/17] fix: bound map coordinates and keep labels --- CHANGELOG.md | 21 +++++++------ docs/schema.md | 12 +++++--- tests/models/test_creature.py | 48 +++++++++++++++++++++++++++++ tests/tests_utils.py | 29 ++++++++++++++++++ tibiawikisql/utils.py | 58 +++++++++++++++++++++++++---------- 5 files changed, 136 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9307735..fe7dc65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,16 +45,17 @@ `outfit` and the effect `yes` when it changes the victim and `no` when it changes the creature. Template names must match exactly, as before, and the existing rows keep their values. - Keep map coordinates out of `creature.location`, and add `item.location` and the `creature_location_coordinate` and - `item_location_coordinate` tables. Each `{{Mapper Coords}}` template in a location is replaced by its `text` label, - so Black Sheep's `Femor Hills (127.88), cultist cave in Liberty Bay (126.192), Drefia` reads `Femor Hills (here), - cultist cave in Liberty Bay (here), Drefia`, and Behemoth's `double spawn text=here` reads `double spawn here`. A - template without a label is removed, with the spaces before it when a space, punctuation or the end follows, and - with parentheses it leaves empty. The rest of the text is cleaned as before, so a location without the template is - unchanged, and NPC, house and key locations are unchanged. Each template whose `x`, `y` and `z` read as numbers - gives a row with the position, its plain-text label or `NULL`, and its place in the list from 0. `x` and `y` are - read from the wiki's `major.minor` form, so `127.88` is 32600, and a malformed set like Warlord Ruzad's - `128.202,123.233,5-2` gives no row, while its label still replaces it in the text. `item.location` is the item's - `location`, cleaned the same way. + `item_location_coordinate` tables. Each `{{Mapper Coords}}` template in a location is replaced by its label, its + `text` cleaned like the rest of the location, so Black Sheep's `Femor Hills (127.88), cultist cave in Liberty Bay + (126.192), Drefia` reads `Femor Hills (here), cultist cave in Liberty Bay (here), Drefia`, and Behemoth's + `double spawn text=here` reads `double spawn here`. A template without a label, or whose label is empty once + cleaned, is removed, with the spaces before it when a space, punctuation or the end follows, and with parentheses it + leaves empty. The rest of the text is cleaned as before, so a location without the template is unchanged, and NPC, + house and key locations are unchanged. Each template whose `x`, `y` and `z` read as numbers on the game's map, `x` + and `y` from 0 to 65535 and `z` from 0 to 15, gives a row with the position, the label the text shows or `NULL`, + and its place in the list from 0. `x` and `y` are read from the wiki's `major.minor` form, so `127.88` is 32600, and + a malformed set like Warlord Ruzad's `128.202,123.233,5-2`, or one off the map, gives no row, while its label still + replaces it in the text. `item.location` is the item's `location`, cleaned the same way. - Add `item.trade_value_note`, the item's `value` as the wiki writes it, with links and templates cleaned and wiki comments dropped, like `Negotiable`, `Negotiable gp` or Amazon Armor's `55,000,000 - 70,000,000`. It is never read as a number, so a range or `0` is kept as written, and it is `NULL` when the value is absent, empty, `?` or `--`. diff --git a/docs/schema.md b/docs/schema.md index a6b9500..33e7b07 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -224,11 +224,13 @@ The generated database has the following tables. | position | `INTEGER` | The row's place in the order the location lists its map positions, from 0. | Each row is a `{{Mapper Coords}}` template in the creature's `location`, in the order they are written. `x` and `y` are -read from the wiki's `major.minor` form, so `127.88` is 32600, and `label` is the plain text of the template's `text`. -Named parameters do not count as positions, so `{{Mapper Coords|text=here|127.234|127.59|14}}` gives the same row as -`{{Mapper Coords|127.234|127.59|14|text=here}}`. A template whose `x`, `y` or `z` is missing or malformed, like Warlord -Ruzad's `{{Mapper Coords|128.202,123.233,5-2|text=here}}`, gives no row. In `creature.location`, each template is -replaced by its label, or left out when it has none, so no coordinate is left in the text. +read from the wiki's `major.minor` form, so `127.88` is 32600. `label` is the template's `text`, cleaned as the +location is, so it is the text `creature.location` shows in the template's place: `text={{Hover|here|explanation}}` +gives `here`. Named parameters do not count as positions, so `{{Mapper Coords|text=here|127.234|127.59|14}}` gives the +same row as `{{Mapper Coords|127.234|127.59|14|text=here}}`. A template whose `x`, `y` or `z` is missing, malformed or +off the game's map, `x` and `y` from 0 to 65535 and `z` from 0 to 15, gives no row, like Warlord Ruzad's +`{{Mapper Coords|128.202,123.233,5-2|text=here}}`. In `creature.location`, each template is replaced by its label, or +left out when it has none, so no coordinate is left in the text. diff --git a/tests/models/test_creature.py b/tests/models/test_creature.py index 75ab18f..c33bb71 100644 --- a/tests/models/test_creature.py +++ b/tests/models/test_creature.py @@ -2,8 +2,11 @@ import sqlite3 import unittest +from tests import load_resource from tibiawikisql import schema +from tibiawikisql.api import Article from tibiawikisql.models import Creature, CreatureAbility, CreatureDrop, Item, LocationCoordinate +from tibiawikisql.parsers import CreatureParser, ItemParser from tibiawikisql.schema import ItemTable @@ -195,3 +198,48 @@ def test_creature_location_coordinates_round_trip_none(self): self.assertEqual([], loaded.location_coordinates) self.assertEqual([], self.conn.execute("SELECT * FROM creature_location_coordinate").fetchall()) + + def test_out_of_range_coordinates_insert_no_row(self): + content = load_resource("content_creature_black_sheep.txt") + coordinates = "{{Mapper Coords|127.88|124.58|4|4|text=here}}" + self.assertIn(coordinates, content) + for bad in ( + "{{Mapper Coords|" + "9" * 30 + ".88|124.58|4|text=here}}", + "{{Mapper Coords|127.88|124.58|" + "9" * 30 + "|text=here}}", + "{{Mapper Coords|-127.88|124.58|4|text=here}}", + "{{Mapper Coords|127.88|124.58|16|text=here}}", + ): + with self.subTest(bad=bad): + schema.create_tables(self.conn) + article = Article( + article_id=1, + title="Black Sheep", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=content.replace(coordinates, bad), + ) + + CreatureParser.from_article(article).insert(self.conn) + loaded = Creature.get_one_by_field(self.conn, "article_id", 1) + + self.assertEqual("Femor Hills (here), cultist cave in Liberty Bay (here), Drefia", loaded.location) + self.assertEqual( + [(32448, 32767, 7, "here")], + [(c.x, c.y, c.z, c.label) for c in loaded.location_coordinates], + ) + + def test_out_of_range_item_coordinates_insert_no_row(self): + content = load_resource("content_item_sneeze_blossom.txt") + coordinates = "{{Mapper Coords|127.197|124.98|7|3|text=spot}}" + self.assertIn(coordinates, content) + article = Article( + article_id=1, + title="Sneeze Blossom", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=content.replace(coordinates, "{{Mapper Coords|127.197|" + "9" * 30 + ".98|7|text=spot}}"), + ) + + ItemParser.from_article(article).insert(self.conn) + loaded = Item.get_one_by_field(self.conn, "article_id", 1) + + self.assertEqual("An example can be found at spot.", loaded.location) + self.assertEqual([], loaded.location_coordinates) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 0a4dbaa..208e2aa 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -381,6 +381,23 @@ def test_parse_mapper_coords_malformed(self): self.assertIsNone(parse_mapper_coords(self._mapper_coords("{{Mapper Coords|127.88|abc|7}}"))) self.assertIsNone(parse_mapper_coords(self._mapper_coords("{{Mapper Coords}}"))) + def test_parse_mapper_coords_bounds(self): + # The game's map: x and y from 0 to 65535, floors from 0 to 15. + self.assertEqual((0, 65535, 0), parse_mapper_coords(self._mapper_coords("{{Mapper Coords|0.0|255.255|0}}"))) + self.assertEqual((32000, 32000, 15), parse_mapper_coords(self._mapper_coords("{{Mapper Coords|125|125|15}}"))) + for template in ( + "{{Mapper Coords|" + "9" * 30 + ".1|124.58|7}}", + "{{Mapper Coords|127.88|" + "9" * 5000 + "|7}}", + "{{Mapper Coords|256.0|124.58|7}}", + "{{Mapper Coords|-1.0|124.58|7}}", + "{{Mapper Coords|127.88|-1.58|7}}", + "{{Mapper Coords|127.88|124.58|16}}", + "{{Mapper Coords|127.88|124.58|-1}}", + "{{Mapper Coords|127.88|124.58|99999999999999999999}}", + ): + with self.subTest(template=template): + self.assertIsNone(parse_mapper_coords(self._mapper_coords(template))) + def test_parse_mapper_coords_ignores_comments(self): template = self._mapper_coords("{{Mapper Coords|127.88|124.58|7 }}") @@ -449,6 +466,18 @@ def test_clean_location_replaces_a_template_inside_a_label(self): ), ) + def test_label_is_the_text_the_location_shows(self): + for value, text, label in ( + ("at {{Mapper Coords|127.88|124.58|4|text={{Hover|here|explanation}}}}.", "at here.", "here"), + ("at {{Mapper Coords|127.88|124.58|4|text=[[Femor Hills|the hills]]}}.", "at the hills.", "the hills"), + ("at {{Mapper Coords|127.88|124.58|4|text=here }}.", "at here.", "here"), + ("at {{Mapper Coords|127.88|124.58|4|text=}}.", "at.", None), + ("at {{Mapper Coords|127.88|124.58|4|text={{GP}}}}.", "at.", None), + ): + with self.subTest(value=value): + self.assertEqual(text, clean_location(value)) + self.assertEqual([(32600, 31802, 4, label)], parse_location_coordinates(value)) + def test_clean_location_without_mapper_coords_is_clean_links(self): for value in ( "[[Hero Cave]], [[Kharos|Ferumbras' Citadel]], deep in [[Pits of Inferno]] ( ) .", diff --git a/tibiawikisql/utils.py b/tibiawikisql/utils.py index 8ab526b..91b30bd 100644 --- a/tibiawikisql/utils.py +++ b/tibiawikisql/utils.py @@ -51,9 +51,15 @@ MAPPER_COORDS = "mapper coords" """The name of the template that links a map position, in lower case with spaces.""" +MAX_MAP_COORDINATE = 65535 +"""The highest x or y coordinate of the game's map.""" + +MAX_MAP_FLOOR = 15 +"""The highest floor of the game's map.""" + REMOVED_MAPPER_COORDS = "\x00" -"""Marks where a Mapper Coords template without a label was removed. The parser ends a text at this character, so a -parsed location never holds one of its own.""" +"""Marks where a Mapper Coords template without a label was removed. MediaWiki page text never holds this control +character, and the C tokenizer also ends a text at it.""" removed_mapper_coords_rules = ( (re.compile(r"[ \t]*\([ \t]*\x00+[ \t]*\)"), ""), @@ -230,7 +236,8 @@ def parse_mapper_coords(template: Template) -> tuple[int, int, int] | None: Returns: The ``x``, ``y`` and ``z`` of the position, or ``None`` when any of them is missing or malformed, as in - Warlord Ruzad's ``{{Mapper Coords|128.202,123.233,5-2|text=here}}``. + Warlord Ruzad's ``{{Mapper Coords|128.202,123.233,5-2|text=here}}``. A position off the game's map, with ``x`` + or ``y`` outside 0 to :data:`MAX_MAP_COORDINATE` or ``z`` outside 0 to :data:`MAX_MAP_FLOOR`, is malformed. """ values = [visible_text(str(template.get(name).value)) if template.has(name) else "" for name in ("1", "2", "3")] @@ -239,23 +246,41 @@ def parse_mapper_coords(template: Template) -> tuple[int, int, int] | None: z = parse_int(values[2]) if x is None or y is None or z is None: return None + if not (0 <= x <= MAX_MAP_COORDINATE and 0 <= y <= MAX_MAP_COORDINATE and 0 <= z <= MAX_MAP_FLOOR): + return None return x, y, z -def _mapper_coords_label(template: Template) -> str: - """Get the raw ``text`` parameter of a Mapper Coords template, stripped, or an empty string without one.""" - return str(template.get("text").value).strip() if template.has("text") else "" +def mapper_coords_label(template: Template) -> str | None: + """Get the label a Mapper Coords template shows: its ``text`` parameter, cleaned by :func:`clean_links`. + + Args: + template: The Mapper Coords template. + + Returns: + The clean label, like ``here`` for ``text={{Hover|here|explanation}}``, or ``None`` when the template has no + ``text``, it is empty once cleaned, as for a comment, or :func:`clean_links` cannot clean it because it holds a + template without parameters, like ``{{GP}}``. + + """ + if not template.has("text"): + return None + try: + label = clean_links(str(template.get("text").value)) + except IndexError: + return None + return label or None def clean_location(value: str) -> str: """Clean a location as :func:`clean_links` does, putting each ``{{Mapper Coords}}`` template's label in its place. - A Mapper Coords template is replaced by its ``text`` parameter, so ``Femor Hills ({{Mapper Coords|127.88|124.58|4| - text=here}})`` reads ``Femor Hills (here)``, and no coordinate is left in the text. A template without a label is - removed. Where one is removed, the spaces before it go too when a space, punctuation or the end of the text - follows, so ``Mintwallin {{Mapper Coords|...}}.`` reads ``Mintwallin.``; at the start of a line or after an opening - parenthesis, the spaces after it go; and parentheses left with nothing in them are removed with the spaces before - them. The rest of the text is cleaned exactly as :func:`clean_links` cleans it. + A Mapper Coords template is replaced by its label, from :func:`mapper_coords_label`, so ``Femor Hills ({{Mapper + Coords|127.88|124.58|4|text=here}})`` reads ``Femor Hills (here)``, and no coordinate is left in the text. A + template without a label is removed. Where one is removed, the spaces before it go too when a space, punctuation or + the end of the text follows, so ``Mintwallin {{Mapper Coords|...}}.`` reads ``Mintwallin.``; at the start of a line + or after an opening parenthesis, the spaces after it go; and parentheses left with nothing in them are removed with + the spaces before them. The rest of the text is cleaned exactly as :func:`clean_links` cleans it. Args: value: The raw location. @@ -267,7 +292,7 @@ def clean_location(value: str) -> str: code = mwparserfromhell.parse(value) # Innermost first, so a template inside another's label is replaced before the label takes the outer one's place. for template in reversed(code.filter_templates(matches=is_mapper_coords)): - code.replace(template, _mapper_coords_label(template) or Text(REMOVED_MAPPER_COORDS)) + code.replace(template, Text(mapper_coords_label(template) or REMOVED_MAPPER_COORDS)) text = str(code) for pattern, replacement in removed_mapper_coords_rules: text = pattern.sub(replacement, text) @@ -282,8 +307,8 @@ def parse_location_coordinates(value: str) -> list[tuple[int, int, int, str | No Returns: The ``x``, ``y``, ``z`` and label of each template whose position :func:`parse_mapper_coords` reads. The label - is the plain text of the ``text`` parameter, or ``None`` when it is absent or empty. A template whose position - is malformed gives nothing. + is the one :func:`clean_location` shows in its place, from :func:`mapper_coords_label`. A template whose + position is malformed gives nothing. """ coordinates = [] @@ -291,8 +316,7 @@ def parse_location_coordinates(value: str) -> list[tuple[int, int, int, str | No position = parse_mapper_coords(template) if position is None: continue - label = strip_code(mwparserfromhell.parse(_mapper_coords_label(template))) or None - coordinates.append((*position, label)) + coordinates.append((*position, mapper_coords_label(template))) return coordinates From 6bf92c3ff27a0bf6c88ee4cc2dcf6522e935edc0 Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 20:50:24 +0000 Subject: [PATCH 12/17] fix: keep trade notes past templates --- CHANGELOG.md | 7 ++++--- docs/schema.md | 2 +- tests/parsers/test_item.py | 13 ++++++++++--- tibiawikisql/parsers/item.py | 9 +++++---- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe7dc65..cb22da9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,9 +56,10 @@ and its place in the list from 0. `x` and `y` are read from the wiki's `major.minor` form, so `127.88` is 32600, and a malformed set like Warlord Ruzad's `128.202,123.233,5-2`, or one off the map, gives no row, while its label still replaces it in the text. `item.location` is the item's `location`, cleaned the same way. -- Add `item.trade_value_note`, the item's `value` as the wiki writes it, with links and templates cleaned and wiki - comments dropped, like `Negotiable`, `Negotiable gp` or Amazon Armor's `55,000,000 - 70,000,000`. It is never read as - a number, so a range or `0` is kept as written, and it is `NULL` when the value is absent, empty, `?` or `--`. +- Add `item.trade_value_note`, the item's `value` as the wiki writes it, in plain text: links become their text, and + templates and wiki comments are dropped, so `1,000 {{GP}}` is `1,000`. `Negotiable`, `Negotiable gp` and Amazon + Armor's `55,000,000 - 70,000,000` are kept. It is never read as a number, so a range or `0` is kept as written, and + it is `NULL` when the value is absent, `?`, `--` or empty once cleaned. ## 9.0.0+tibiash.9 diff --git a/docs/schema.md b/docs/schema.md index 33e7b07..7842728 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -376,7 +376,7 @@ left out when it has none, so no coordinate is left in the text. | version | `TEXT` | The client version this item was introduced to the game. | | client_id | `INTEGER` | The client id of the item, the first integer in `itemid`. `item_client_id` has every client ID. | | location | `TEXT` | Where the item can be found, cleaned as `creature.location` is. `item_location_coordinate` has its map positions. | -| trade_value_note | `TEXT` | What players trade this item for, the `value` as written with links and templates cleaned, like `Negotiable` or `55,000,000 - 70,000,000`. `NULL` when it is empty, `?` or `--`. | +| trade_value_note | `TEXT` | What players trade this item for, the `value` as written in plain text, links as their text and templates and comments dropped, like `Negotiable` or `55,000,000 - 70,000,000`. `NULL` when it is `?`, `--` or empty. | | image | `BLOB` | The item’s image bytes. | | status | `TEXT` | The status of the item in game. | | timestamp | `TIMESTAMP` | ISO 8601 timestamp of the article's last edit. | diff --git a/tests/parsers/test_item.py b/tests/parsers/test_item.py index 12fb35d..b0fc9e5 100644 --- a/tests/parsers/test_item.py +++ b/tests/parsers/test_item.py @@ -504,11 +504,18 @@ def test_absent(self): def test_cleans_links_and_templates(self): self.assertEqual( - "About 5,000 or Negotiable", - self._edited("| value = About [[Gold Coin|5,000]] or {{Hover|Negotiable|ask}} \n") - .trade_value_note, + "About 5,000", + self._edited("| value = About [[Gold Coin|5,000]] {{GP}} \n").trade_value_note, ) + def test_keeps_the_note_past_a_template_without_parameters(self): + self.assertEqual("1,000", parse_trade_value_note("1,000 {{GP}}")) + self.assertEqual("1,000", self._edited("| value = 1,000 {{GP}}\n").trade_value_note) + + def test_only_a_template(self): + self.assertIsNone(parse_trade_value_note("{{GP}}")) + self.assertIsNone(self._edited("| value = {{GP}}\n").trade_value_note) + def test_parse_trade_value_note(self): self.assertEqual("Negotiable", parse_trade_value_note(" Negotiable ")) self.assertEqual("55,000,000 - 70,000,000", parse_trade_value_note("55,000,000 - 70,000,000")) diff --git a/tibiawikisql/parsers/item.py b/tibiawikisql/parsers/item.py index bc4dc0e..f7f7363 100644 --- a/tibiawikisql/parsers/item.py +++ b/tibiawikisql/parsers/item.py @@ -74,12 +74,13 @@ def parse_trade_value_note(value: str) -> str | None: value: The raw value of the field. Returns: - The value as written, with links and templates cleaned by :func:`clean_links`, which also drops wiki comments, - like ``Negotiable`` or ``55,000,000 - 70,000,000``. It is never read as a number. ``None`` when the value is - empty, ``?`` or ``--``. + The value as written, like ``Negotiable`` or ``55,000,000 - 70,000,000``, in the plain text mwparserfromhell's + ``strip_code`` gives: links become their text, and templates and wiki comments are dropped, so ``1,000 {{GP}}`` + is ``1,000``. It never raises, and it is never read as a number. ``None`` when nothing is left, or the value is + ``?`` or ``--``. """ - note = clean_links(value) + note = mwparserfromhell.parse(value).strip_code().strip() return None if note in UNKNOWN_TRADE_VALUES else note From 3fa21110367f422c4272c726d0de9a2975aa58e6 Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 20:51:56 +0000 Subject: [PATCH 13/17] fix: keep trade notes that clean_links can read --- CHANGELOG.md | 10 ++++++---- docs/schema.md | 2 +- tests/parsers/test_item.py | 6 ++++++ tibiawikisql/parsers/item.py | 14 +++++++++----- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb22da9..362a459 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,10 +56,12 @@ and its place in the list from 0. `x` and `y` are read from the wiki's `major.minor` form, so `127.88` is 32600, and a malformed set like Warlord Ruzad's `128.202,123.233,5-2`, or one off the map, gives no row, while its label still replaces it in the text. `item.location` is the item's `location`, cleaned the same way. -- Add `item.trade_value_note`, the item's `value` as the wiki writes it, in plain text: links become their text, and - templates and wiki comments are dropped, so `1,000 {{GP}}` is `1,000`. `Negotiable`, `Negotiable gp` and Amazon - Armor's `55,000,000 - 70,000,000` are kept. It is never read as a number, so a range or `0` is kept as written, and - it is `NULL` when the value is absent, `?`, `--` or empty once cleaned. +- Add `item.trade_value_note`, the item's `value` as the wiki writes it, with links and templates cleaned and wiki + comments dropped, like `Negotiable`, `Negotiable gp` or Amazon Armor's `55,000,000 - 70,000,000`. A template gives + its first parameter, so `{{Hover|Negotiable|ask}}` is `Negotiable`, and a value holding a template without + parameters, which that cleaning cannot read, is read as plain text with its templates dropped, so `1,000 {{GP}}` is + `1,000`. It is never read as a number, so a range or `0` is kept as written, and it is `NULL` when the value is + absent, `?`, `--` or empty once cleaned. ## 9.0.0+tibiash.9 diff --git a/docs/schema.md b/docs/schema.md index 7842728..3f0379e 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -376,7 +376,7 @@ left out when it has none, so no coordinate is left in the text. | version | `TEXT` | The client version this item was introduced to the game. | | client_id | `INTEGER` | The client id of the item, the first integer in `itemid`. `item_client_id` has every client ID. | | location | `TEXT` | Where the item can be found, cleaned as `creature.location` is. `item_location_coordinate` has its map positions. | -| trade_value_note | `TEXT` | What players trade this item for, the `value` as written in plain text, links as their text and templates and comments dropped, like `Negotiable` or `55,000,000 - 70,000,000`. `NULL` when it is `?`, `--` or empty. | +| trade_value_note | `TEXT` | What players trade this item for, the `value` as written with links and templates cleaned, like `Negotiable` or `55,000,000 - 70,000,000`. A value with a template without parameters, like `1,000 {{GP}}`, drops its templates. `NULL` when it is `?`, `--` or empty. | | image | `BLOB` | The item’s image bytes. | | status | `TEXT` | The status of the item in game. | | timestamp | `TIMESTAMP` | ISO 8601 timestamp of the article's last edit. | diff --git a/tests/parsers/test_item.py b/tests/parsers/test_item.py index b0fc9e5..6ec9aae 100644 --- a/tests/parsers/test_item.py +++ b/tests/parsers/test_item.py @@ -508,6 +508,12 @@ def test_cleans_links_and_templates(self): self._edited("| value = About [[Gold Coin|5,000]] {{GP}} \n").trade_value_note, ) + def test_keeps_the_clean_links_reading(self): + # The reading of 39c17a7: a template with parameters gives its first one, and
a line break. + self.assertEqual("Negotiable", parse_trade_value_note("{{Hover|Negotiable|ask}}")) + self.assertEqual("Negotiable", self._edited("| value = {{Hover|Negotiable|ask}}\n").trade_value_note) + self.assertEqual("Negotiable\n(~5k)", parse_trade_value_note("Negotiable
(~5k)")) + def test_keeps_the_note_past_a_template_without_parameters(self): self.assertEqual("1,000", parse_trade_value_note("1,000 {{GP}}")) self.assertEqual("1,000", self._edited("| value = 1,000 {{GP}}\n").trade_value_note) diff --git a/tibiawikisql/parsers/item.py b/tibiawikisql/parsers/item.py index f7f7363..23fac65 100644 --- a/tibiawikisql/parsers/item.py +++ b/tibiawikisql/parsers/item.py @@ -74,13 +74,17 @@ def parse_trade_value_note(value: str) -> str | None: value: The raw value of the field. Returns: - The value as written, like ``Negotiable`` or ``55,000,000 - 70,000,000``, in the plain text mwparserfromhell's - ``strip_code`` gives: links become their text, and templates and wiki comments are dropped, so ``1,000 {{GP}}`` - is ``1,000``. It never raises, and it is never read as a number. ``None`` when nothing is left, or the value is - ``?`` or ``--``. + The value as written, like ``Negotiable`` or ``55,000,000 - 70,000,000``, cleaned by :func:`clean_links`, so + ``{{Hover|Negotiable|ask}}`` is ``Negotiable``. When :func:`clean_links` cannot clean it, because it holds a + template without parameters, it is the plain text mwparserfromhell's ``strip_code`` gives, with templates + dropped, so ``1,000 {{GP}}`` is ``1,000``. Wiki comments are dropped either way. It never raises, and it is + never read as a number. ``None`` when nothing is left, or the value is ``?`` or ``--``. """ - note = mwparserfromhell.parse(value).strip_code().strip() + try: + note = clean_links(value) + except IndexError: # clean_links reads params[0] of every template, which a template without parameters lacks. + note = mwparserfromhell.parse(value).strip_code().strip() return None if note in UNKNOWN_TRADE_VALUES else note From 38dba91fec057d19035d3135ebec13af6667dfc8 Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 20:58:30 +0000 Subject: [PATCH 14/17] fix: clean wiki text without raising --- CHANGELOG.md | 32 +++++----- docs/schema.md | 2 +- tests/models/test_creature.py | 31 +++++++++ tests/parsers/test_item.py | 6 ++ tests/tests_utils.py | 47 ++++++++++---- tibiawikisql/parsers/item.py | 15 ++--- tibiawikisql/utils.py | 117 ++++++++++++++++++++++------------ 7 files changed, 175 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 362a459..3b74f2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,22 +46,24 @@ match exactly, as before, and the existing rows keep their values. - Keep map coordinates out of `creature.location`, and add `item.location` and the `creature_location_coordinate` and `item_location_coordinate` tables. Each `{{Mapper Coords}}` template in a location is replaced by its label, its - `text` cleaned like the rest of the location, so Black Sheep's `Femor Hills (127.88), cultist cave in Liberty Bay - (126.192), Drefia` reads `Femor Hills (here), cultist cave in Liberty Bay (here), Drefia`, and Behemoth's - `double spawn text=here` reads `double spawn here`. A template without a label, or whose label is empty once - cleaned, is removed, with the spaces before it when a space, punctuation or the end follows, and with parentheses it - leaves empty. The rest of the text is cleaned as before, so a location without the template is unchanged, and NPC, - house and key locations are unchanged. Each template whose `x`, `y` and `z` read as numbers on the game's map, `x` - and `y` from 0 to 65535 and `z` from 0 to 15, gives a row with the position, the label the text shows or `NULL`, - and its place in the list from 0. `x` and `y` are read from the wiki's `major.minor` form, so `127.88` is 32600, and - a malformed set like Warlord Ruzad's `128.202,123.233,5-2`, or one off the map, gives no row, while its label still - replaces it in the text. `item.location` is the item's `location`, cleaned the same way. + `text` cleaned once, like the rest of the location, so Black Sheep's `Femor Hills (127.88), cultist cave in Liberty + Bay (126.192), Drefia` reads `Femor Hills (here), cultist cave in Liberty Bay (here), Drefia`, and Behemoth's `double + spawn text=here` reads `double spawn here`. A template without a label, or whose label is empty once cleaned, is + removed, with the spaces before it when a space, punctuation or the end follows, and with parentheses it leaves empty. + The rest of the text is cleaned as before, so a location without the template is unchanged, and NPC, house and key + locations are unchanged, except that a creature or item location the cleaning cannot read, such as one holding a + template inside another's later parameter, is read as plain text with its templates dropped instead of being `NULL`. + Each template whose `x`, `y` and `z` read as numbers on the game's map, `x` and `y` from 0 to 65535 and `z` from 0 to + 15, gives a row with the position, the label the text shows or `NULL`, and its place in the list from 0. `x` and `y` + are read from the wiki's `major.minor` form, so `127.88` is 32600, and a malformed set like Warlord Ruzad's + `128.202,123.233,5-2`, or one off the map, gives no row, while its label still replaces it in the text. + `item.location` is the item's `location`, cleaned the same way. - Add `item.trade_value_note`, the item's `value` as the wiki writes it, with links and templates cleaned and wiki - comments dropped, like `Negotiable`, `Negotiable gp` or Amazon Armor's `55,000,000 - 70,000,000`. A template gives - its first parameter, so `{{Hover|Negotiable|ask}}` is `Negotiable`, and a value holding a template without - parameters, which that cleaning cannot read, is read as plain text with its templates dropped, so `1,000 {{GP}}` is - `1,000`. It is never read as a number, so a range or `0` is kept as written, and it is `NULL` when the value is - absent, `?`, `--` or empty once cleaned. + comments dropped, like `Negotiable`, `Negotiable gp` or Amazon Armor's `55,000,000 - 70,000,000`. A template gives its + first parameter, so `{{Hover|Negotiable|ask}}` is `Negotiable`, and a value that cleaning cannot read, such as one + holding a template without parameters or a template inside another's later parameter, is read as plain text with its + templates dropped, so `1,000 {{GP}}` is `1,000`. It is never read as a number, so a range or `0` is kept as written, + and it is `NULL` when the value is absent, `?`, `--` or empty once cleaned. ## 9.0.0+tibiash.9 diff --git a/docs/schema.md b/docs/schema.md index 3f0379e..e2eb77d 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -376,7 +376,7 @@ left out when it has none, so no coordinate is left in the text. | version | `TEXT` | The client version this item was introduced to the game. | | client_id | `INTEGER` | The client id of the item, the first integer in `itemid`. `item_client_id` has every client ID. | | location | `TEXT` | Where the item can be found, cleaned as `creature.location` is. `item_location_coordinate` has its map positions. | -| trade_value_note | `TEXT` | What players trade this item for, the `value` as written with links and templates cleaned, like `Negotiable` or `55,000,000 - 70,000,000`. A value with a template without parameters, like `1,000 {{GP}}`, drops its templates. `NULL` when it is `?`, `--` or empty. | +| trade_value_note | `TEXT` | What players trade this item for, the `value` as written with links and templates cleaned, like `Negotiable` or `55,000,000 - 70,000,000`. A value that cannot be cleaned so, like `1,000 {{GP}}`, drops its templates. `NULL` when it is `?`, `--` or empty. | | image | `BLOB` | The item’s image bytes. | | status | `TEXT` | The status of the item in game. | | timestamp | `TIMESTAMP` | ISO 8601 timestamp of the article's last edit. | diff --git a/tests/models/test_creature.py b/tests/models/test_creature.py index c33bb71..3e66ea1 100644 --- a/tests/models/test_creature.py +++ b/tests/models/test_creature.py @@ -243,3 +243,34 @@ def test_out_of_range_item_coordinates_insert_no_row(self): self.assertEqual("An example can be found at spot.", loaded.location) self.assertEqual([], loaded.location_coordinates) + + def _black_sheep_with_first_label(self, label: str) -> Creature: + content = load_resource("content_creature_black_sheep.txt") + coordinates = "{{Mapper Coords|127.88|124.58|4|4|text=here}}" + self.assertIn(coordinates, content) + article = Article( + article_id=1, + title="Black Sheep", + timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), + content=content.replace(coordinates, "{{Mapper Coords|127.88|124.58|4|4|text=" + label + "}}"), + ) + CreatureParser.from_article(article).insert(self.conn) + return Creature.get_one_by_field(self.conn, "article_id", 1) + + def test_label_clean_links_cannot_read_keeps_the_article(self): + loaded = self._black_sheep_with_first_label("{{Hover|here|reached via {{Item|Rope}}}}") + + self.assertEqual("Femor Hills, cultist cave in Liberty Bay (here), Drefia", loaded.location) + self.assertEqual( + [(32600, 31802, 4, None), (32448, 32767, 7, "here")], + [(c.x, c.y, c.z, c.label) for c in loaded.location_coordinates], + ) + + def test_label_cleaned_once_keeps_the_location(self): + loaded = self._black_sheep_with_first_label("{{GP}}") + + self.assertEqual("Femor Hills ({{GP}}), cultist cave in Liberty Bay (here), Drefia", loaded.location) + self.assertEqual( + [(32600, 31802, 4, "{{GP}}"), (32448, 32767, 7, "here")], + [(c.x, c.y, c.z, c.label) for c in loaded.location_coordinates], + ) diff --git a/tests/parsers/test_item.py b/tests/parsers/test_item.py index 6ec9aae..15eb56e 100644 --- a/tests/parsers/test_item.py +++ b/tests/parsers/test_item.py @@ -518,6 +518,12 @@ def test_keeps_the_note_past_a_template_without_parameters(self): self.assertEqual("1,000", parse_trade_value_note("1,000 {{GP}}")) self.assertEqual("1,000", self._edited("| value = 1,000 {{GP}}\n").trade_value_note) + def test_nested_templates_do_not_raise(self): + self.assertIsNone(parse_trade_value_note("{{A|x|{{B|y}}}}")) + self.assertEqual("5,000", parse_trade_value_note("5,000 {{A|x|{{B|y}}}}")) + self.assertEqual("5,000", self._edited("| value = 5,000 {{A|x|{{B|y}}}}\n").trade_value_note) + self.assertIsNone(self._edited("| value = {{A|x|{{B|y}}}}\n").trade_value_note) + def test_only_a_template(self): self.assertIsNone(parse_trade_value_note("{{GP}}")) self.assertIsNone(self._edited("| value = {{GP}}\n").trade_value_note) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 208e2aa..eff0388 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -8,8 +8,8 @@ import tibiawikisql from tests import load_resource -from tibiawikisql.utils import (clean_links, clean_location, client_color_to_rgb, parse_boolean, parse_client_id, - parse_client_ids, parse_currency, parse_int, parse_location_coordinate, +from tibiawikisql.utils import (clean_links, clean_location, clean_text_safely, client_color_to_rgb, parse_boolean, + parse_client_id, parse_client_ids, parse_currency, parse_int, parse_location_coordinate, parse_location_coordinates, parse_mapper_coords, parse_first_integer, parse_float, parse_integer, parse_date, parse_loot_statistics, parse_min_max, parse_sounds, @@ -455,17 +455,42 @@ def test_clean_location_removes_unlabelled(self): ) def test_clean_location_replaces_a_template_inside_a_label(self): - self.assertEqual( - "Femor Hills (here)", - clean_location("[[Femor Hills]] ({{Mapper Coords|1.1|1.1|7|text={{Mapper Coords|1.2|1.2|7|text=here}}}})"), - ) - self.assertEqual( - "the Fortress", - clean_location( - "the {{Mapper Coords|1.1|1.1|7|text=[[Fortress|{{Mapper Coords|1.2|1.2|7|text=Fortress}}]]}}", - ), + value = "[[Femor Hills]] ({{Mapper Coords|1.1|1.1|7|text={{Mapper Coords|1.2|1.2|7|text=here}}}})" + self.assertEqual("Femor Hills (here)", clean_location(value)) + self.assertEqual([(257, 257, 7, "here"), (258, 258, 7, "here")], parse_location_coordinates(value)) + value = "the {{Mapper Coords|1.1|1.1|7|text=[[Fortress|{{Mapper Coords|1.2|1.2|7|text=Fortress}}]]}}" + self.assertEqual("the Fortress", clean_location(value)) + self.assertEqual([(257, 257, 7, "Fortress"), (258, 258, 7, "Fortress")], parse_location_coordinates(value)) + value = "at {{Mapper Coords|1.1|1.1|7|text=({{Mapper Coords|1.2|1.2|7}})}}." + self.assertEqual("at.", clean_location(value)) + self.assertEqual([(257, 257, 7, None), (258, 258, 7, None)], parse_location_coordinates(value)) + + def test_clean_text_safely(self): + for value in ("[[Holy Damage]]", "{{Hover|Negotiable|ask}}", "Negotiable
(~5k)", "Hello ", ""): + with self.subTest(value=value): + self.assertEqual(clean_links(value), clean_text_safely(value)) + self.assertEqual("1,000", clean_text_safely("1,000 {{GP}}")) + self.assertEqual("5,000", clean_text_safely("5,000 {{A|x|{{B|y}}}}")) + self.assertEqual("", clean_text_safely("{{Hover|here|reached via {{Item|Rope}}}}")) + + def test_location_labels_are_cleaned_once(self): + for value, text, label in ( + ("at {{Mapper Coords|127.88|124.58|4|text={{GP}}}}.", "at {{GP}}.", "{{GP}}"), + ("at {{Mapper Coords|127.88|124.58|4|text=&lt;b&gt;}}.", "at <b>.", "<b>"), + ): + with self.subTest(value=value): + self.assertEqual(text, clean_location(value)) + self.assertEqual([(32600, 31802, 4, label)], parse_location_coordinates(value)) + + def test_location_that_clean_links_cannot_read(self): + value = ( + "[[Femor Hills]] ({{Mapper Coords|127.88|124.58|4|4|text={{Hover|here|reached via {{Item|Rope}}}}}}), " + "[[Thais]] {{Hover|north|x {{Item|Rope}}}}, {{Mapper Coords|126.192|127.255|7|2|text=here}}" ) + self.assertEqual("Femor Hills, Thais , here", clean_location(value)) + self.assertEqual([(32600, 31802, 4, None), (32448, 32767, 7, "here")], parse_location_coordinates(value)) + def test_label_is_the_text_the_location_shows(self): for value, text, label in ( ("at {{Mapper Coords|127.88|124.58|4|text={{Hover|here|explanation}}}}.", "at here.", "here"), diff --git a/tibiawikisql/parsers/item.py b/tibiawikisql/parsers/item.py index 23fac65..6ac6b8f 100644 --- a/tibiawikisql/parsers/item.py +++ b/tibiawikisql/parsers/item.py @@ -14,6 +14,7 @@ clean_links, clean_location, clean_question_mark, + clean_text_safely, client_color_to_rgb, find_templates, number_pattern, @@ -74,17 +75,13 @@ def parse_trade_value_note(value: str) -> str | None: value: The raw value of the field. Returns: - The value as written, like ``Negotiable`` or ``55,000,000 - 70,000,000``, cleaned by :func:`clean_links`, so - ``{{Hover|Negotiable|ask}}`` is ``Negotiable``. When :func:`clean_links` cannot clean it, because it holds a - template without parameters, it is the plain text mwparserfromhell's ``strip_code`` gives, with templates - dropped, so ``1,000 {{GP}}`` is ``1,000``. Wiki comments are dropped either way. It never raises, and it is - never read as a number. ``None`` when nothing is left, or the value is ``?`` or ``--``. + The value as written, like ``Negotiable`` or ``55,000,000 - 70,000,000``, cleaned by :func:`clean_text_safely`: + as :func:`clean_links` cleans it, so ``{{Hover|Negotiable|ask}}`` is ``Negotiable``, or, where that fails, as + plain text with templates dropped, so ``1,000 {{GP}}`` is ``1,000``. Wiki comments are dropped either way. It + is never read as a number. ``None`` when nothing is left, or the value is ``?`` or ``--``. """ - try: - note = clean_links(value) - except IndexError: # clean_links reads params[0] of every template, which a template without parameters lacks. - note = mwparserfromhell.parse(value).strip_code().strip() + note = clean_text_safely(value) return None if note in UNKNOWN_TRADE_VALUES else note diff --git a/tibiawikisql/utils.py b/tibiawikisql/utils.py index 91b30bd..dbd574b 100644 --- a/tibiawikisql/utils.py +++ b/tibiawikisql/utils.py @@ -251,73 +251,112 @@ def parse_mapper_coords(template: Template) -> tuple[int, int, int] | None: return x, y, z -def mapper_coords_label(template: Template) -> str | None: - """Get the label a Mapper Coords template shows: its ``text`` parameter, cleaned by :func:`clean_links`. +def clean_text_safely(value: str) -> str: + """Clean wiki text as :func:`clean_links` does, falling back to plain text where :func:`clean_links` fails. Args: - template: The Mapper Coords template. + value: The wiki text. Returns: - The clean label, like ``here`` for ``text={{Hover|here|explanation}}``, or ``None`` when the template has no - ``text``, it is empty once cleaned, as for a comment, or :func:`clean_links` cannot clean it because it holds a - template without parameters, like ``{{GP}}``. + :func:`clean_links` of the value whenever it can clean it, so ``{{Hover|Negotiable|ask}}`` is ``Negotiable``. + When it raises :class:`IndexError` or :class:`ValueError`, the two errors it is known to raise on wiki text, + the stripped plain text mwparserfromhell's ``strip_code`` gives instead, where links become their text and + templates and comments are dropped, so ``1,000 {{GP}}`` is ``1,000`` and ``{{A|x|{{B|y}}}}`` is empty. So a + value :func:`clean_links` cannot clean still gives a text rather than an error. """ - if not template.has("text"): - return None try: - label = clean_links(str(template.get("text").value)) - except IndexError: - return None - return label or None + return clean_links(value) + # IndexError: a template without parameters has no first one. ValueError: a template inside another's later + # parameter is gone from the text once the outer one is replaced, so it cannot be replaced in turn. + except (IndexError, ValueError): + return mwparserfromhell.parse(value).strip_code().strip() -def clean_location(value: str) -> str: - """Clean a location as :func:`clean_links` does, putting each ``{{Mapper Coords}}`` template's label in its place. +def _unused_private_character(value: str) -> str: + """Get a character from Unicode's private use area that does not appear in a value. + + Raises: + StopIteration: If the value holds every one of the area's 6,400 characters. + + """ + return next(chr(code) for code in range(0xE000, 0xF900) if chr(code) not in value) + + +def parse_location(value: str) -> tuple[str, list[tuple[int, int, int, str | None]]]: + """Clean a location and read the positions of its ``{{Mapper Coords}}`` templates. - A Mapper Coords template is replaced by its label, from :func:`mapper_coords_label`, so ``Femor Hills ({{Mapper - Coords|127.88|124.58|4|text=here}})`` reads ``Femor Hills (here)``, and no coordinate is left in the text. A - template without a label is removed. Where one is removed, the spaces before it go too when a space, punctuation or - the end of the text follows, so ``Mintwallin {{Mapper Coords|...}}.`` reads ``Mintwallin.``; at the start of a line - or after an opening parenthesis, the spaces after it go; and parentheses left with nothing in them are removed with - the spaces before them. The rest of the text is cleaned exactly as :func:`clean_links` cleans it. + Each template's label is its ``text`` parameter, cleaned once, as the rest of the location is: removed templates + tidied as below, then :func:`clean_text_safely`. A template inside the label is cleaned the same way first, so in + ``text={{Mapper Coords|...|text=here}}`` the label is ``here``. A label that is empty once cleaned counts as none. + + In the text, each template is replaced by its label, so ``Femor Hills ({{Mapper Coords|127.88|124.58|4| + text=here}})`` reads ``Femor Hills (here)``, and no coordinate is left in it. A template without a label is + removed. Where one is removed, the spaces before it go too when a space, punctuation or the end of the text follows, + so ``Mintwallin {{Mapper Coords|...}}.`` reads ``Mintwallin.``; at the start of a line or after an opening + parenthesis, the spaces after it go; and parentheses left with nothing in them are removed with the spaces before + them. Labels are put in after the rest of the text is cleaned, through a private-use character absent from the + value, so each is cleaned once. A location without the template reads as :func:`clean_text_safely` reads it. + + Where :func:`clean_links` cannot clean the text or a label, :func:`clean_text_safely` gives its plain text instead, + so such a location gives a text and rows rather than an error. Args: value: The raw location. Returns: - The clean location. + The clean location, and the ``x``, ``y``, ``z`` and label of each template whose position + :func:`parse_mapper_coords` reads, in the order they are written. The label is the text the location shows in + the template's place, or ``None`` when it shows none. A template whose position is malformed gives no row. """ code = mwparserfromhell.parse(value) - # Innermost first, so a template inside another's label is replaced before the label takes the outer one's place. - for template in reversed(code.filter_templates(matches=is_mapper_coords)): - code.replace(template, Text(mapper_coords_label(template) or REMOVED_MAPPER_COORDS)) - text = str(code) - for pattern, replacement in removed_mapper_coords_rules: - text = pattern.sub(replacement, text) - return clean_links(text) + templates = code.filter_templates(matches=is_mapper_coords) + positions = [parse_mapper_coords(template) for template in templates] + mark = _unused_private_character(value) + placeholder = re.compile(f"{mark}([0-9]+){mark}") + labels: list[str | None] = [None] * len(templates) + + def clean(text: str) -> str: + for pattern, replacement in removed_mapper_coords_rules: + text = pattern.sub(replacement, text) + return placeholder.sub(lambda match: labels[int(match[1])], clean_text_safely(text)) + + # Innermost first, so a template inside another's label is replaced before that label is cleaned. + for index in reversed(range(len(templates))): + template = templates[index] + labels[index] = (clean(str(template.get("text").value)) if template.has("text") else "") or None + code.replace(template, Text(f"{mark}{index}{mark}" if labels[index] else REMOVED_MAPPER_COORDS)) + rows = [(*position, labels[index]) for index, position in enumerate(positions) if position is not None] + return clean(str(code)), rows + + +def clean_location(value: str) -> str: + """Clean a location, with each ``{{Mapper Coords}}`` template's label in its place, as :func:`parse_location` does. + + Args: + value: The raw location. + + Returns: + The clean location. + + """ + return parse_location(value)[0] def parse_location_coordinates(value: str) -> list[tuple[int, int, int, str | None]]: - """Read the positions of the ``{{Mapper Coords}}`` templates in a location, in the order they are written. + """Read the positions of the ``{{Mapper Coords}}`` templates in a location, as :func:`parse_location` does. Args: value: The raw location. Returns: - The ``x``, ``y``, ``z`` and label of each template whose position :func:`parse_mapper_coords` reads. The label - is the one :func:`clean_location` shows in its place, from :func:`mapper_coords_label`. A template whose - position is malformed gives nothing. + The ``x``, ``y``, ``z`` and label of each template whose position :func:`parse_mapper_coords` reads, in the + order they are written. The label is the text :func:`clean_location` shows in the template's place, or ``None`` + when it shows none. """ - coordinates = [] - for template in mwparserfromhell.parse(value).filter_templates(matches=is_mapper_coords): - position = parse_mapper_coords(template) - if position is None: - continue - coordinates.append((*position, mapper_coords_label(template))) - return coordinates + return parse_location(value)[1] def find_template(content: str, template_name: str, partial: bool = False, recursive: bool = False) -> Template | None: From 9ed27f3f6a622317e54258910ebd811b264bfcda Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 21:12:50 +0000 Subject: [PATCH 15/17] fix: keep location marks out of decoded text --- tests/models/test_creature.py | 41 ++++++++++++++++++-- tests/tests_utils.py | 65 +++++++++++++++++++++++++++++++- tibiawikisql/utils.py | 70 ++++++++++++++++++++++++++++------- 3 files changed, 157 insertions(+), 19 deletions(-) diff --git a/tests/models/test_creature.py b/tests/models/test_creature.py index 3e66ea1..554926a 100644 --- a/tests/models/test_creature.py +++ b/tests/models/test_creature.py @@ -244,19 +244,23 @@ def test_out_of_range_item_coordinates_insert_no_row(self): self.assertEqual("An example can be found at spot.", loaded.location) self.assertEqual([], loaded.location_coordinates) - def _black_sheep_with_first_label(self, label: str) -> Creature: + def _black_sheep_with(self, written: str, replacement: str) -> Creature: content = load_resource("content_creature_black_sheep.txt") - coordinates = "{{Mapper Coords|127.88|124.58|4|4|text=here}}" - self.assertIn(coordinates, content) + self.assertIn(written, content) article = Article( article_id=1, title="Black Sheep", timestamp=datetime.datetime.fromisoformat("2018-08-20T04:33:15+00:00"), - content=content.replace(coordinates, "{{Mapper Coords|127.88|124.58|4|4|text=" + label + "}}"), + content=content.replace(written, replacement), ) CreatureParser.from_article(article).insert(self.conn) return Creature.get_one_by_field(self.conn, "article_id", 1) + def _black_sheep_with_first_label(self, label: str) -> Creature: + return self._black_sheep_with( + "{{Mapper Coords|127.88|124.58|4|4|text=here}}", "{{Mapper Coords|127.88|124.58|4|4|text=" + label + "}}", + ) + def test_label_clean_links_cannot_read_keeps_the_article(self): loaded = self._black_sheep_with_first_label("{{Hover|here|reached via {{Item|Rope}}}}") @@ -274,3 +278,32 @@ def test_label_cleaned_once_keeps_the_location(self): [(32600, 31802, 4, "{{GP}}"), (32448, 32767, 7, "here")], [(c.x, c.y, c.z, c.label) for c in loaded.location_coordinates], ) + + def test_decoded_private_use_text_keeps_the_article(self): + mark = "\ue000" + for index, count in (("3", 2), ("0", 2)): + with self.subTest(index=index): + schema.create_tables(self.conn) + loaded = self._black_sheep_with("[[Drefia]]", f"[[Drefia]] {index}") + + self.assertEqual( + f"Femor Hills (here), cultist cave in Liberty Bay (here), Drefia {mark}{index}{mark}", + loaded.location, + ) + self.assertEqual(count, loaded.location.count("here")) + self.assertEqual( + [(32600, 31802, 4, "here"), (32448, 32767, 7, "here")], + [(c.x, c.y, c.z, c.label) for c in loaded.location_coordinates], + ) + + def test_location_holding_every_private_use_character_keeps_the_article(self): + characters = "".join(chr(code) for code in range(0xE000, 0xF900)) + loaded = self._black_sheep_with("[[Drefia]]", "[[Drefia]] " + characters) + + self.assertEqual( + "Femor Hills (127.88), cultist cave in Liberty Bay (126.192), Drefia " + characters, loaded.location, + ) + self.assertEqual( + [(32600, 31802, 4, "here"), (32448, 32767, 7, "here")], + [(c.x, c.y, c.z, c.label) for c in loaded.location_coordinates], + ) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index eff0388..0b59994 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -2,15 +2,17 @@ import unittest import datetime +from unittest import mock import mwparserfromhell from mwparserfromhell.nodes import Template import tibiawikisql from tests import load_resource +from tibiawikisql import utils from tibiawikisql.utils import (clean_links, clean_location, clean_text_safely, client_color_to_rgb, parse_boolean, - parse_client_id, parse_client_ids, parse_currency, parse_int, parse_location_coordinate, - parse_location_coordinates, parse_mapper_coords, + parse_client_id, parse_client_ids, parse_currency, parse_int, parse_location, + parse_location_coordinate, parse_location_coordinates, parse_mapper_coords, parse_first_integer, parse_float, parse_integer, parse_date, parse_loot_statistics, parse_min_max, parse_sounds, parse_known_boolean, parse_known_text, parse_weapon_proficiency_name, @@ -491,6 +493,65 @@ def test_location_that_clean_links_cannot_read(self): self.assertEqual("Femor Hills, Thais , here", clean_location(value)) self.assertEqual([(32600, 31802, 4, None), (32448, 32767, 7, "here")], parse_location_coordinates(value)) + def test_location_keeps_decoded_private_use_text(self): + mark = "\ue000" + removed = "à({{Mapper Coords|1.1|1.1|7}})00;" + for value, rows in ( + ("a 3 b", []), + ("a 3 b", []), + ("a 3 b", []), + (f"a {removed}3{removed} b", [(257, 257, 7, None), (257, 257, 7, None)]), + ): + with self.subTest(value=value): + self.assertEqual((f"a {mark}3{mark} b", rows), parse_location(value)) + + def test_location_label_goes_only_where_its_template_was(self): + mark = "\ue000" + here = "{{Mapper Coords|1.2|1.2|7|text=here}}" + removed = "à({{Mapper Coords|1.1|1.1|7}})00;" + for value, text, rows in ( + (f"a 0 {here}", f"a {mark}0{mark} here", [(258, 258, 7, "here")]), + (f"a 0 {here}", f"a {mark}0{mark} here", [(258, 258, 7, "here")]), + ( + f"a {removed}2{removed} {here}", + f"a {mark}2{mark} here", + [(257, 257, 7, None), (257, 257, 7, None), (258, 258, 7, "here")], + ), + ( + f"{{{{Mapper Coords|1.3|1.3|7|text=a {removed}3{removed}}}}} {here}", + f"a {mark}3{mark} here", + [(259, 259, 7, f"a {mark}3{mark}"), (257, 257, 7, None), (257, 257, 7, None), (258, 258, 7, "here")], + ), + ): + with self.subTest(value=value): + self.assertEqual((text, rows), parse_location(value)) + self.assertEqual(1, parse_location(value)[0].count("here")) + + def test_location_puts_labels_back_only_where_written(self): + # With the decoding check blinded, a decoded mark reaches the expansion, which must leave it as written. + mark = "\ue000" + here = "{{Mapper Coords|1.2|1.2|7|text=here}}" + with mock.patch.object(utils.html, "unescape", lambda text: text): + self.assertEqual((f"a {mark}3{mark} b", []), parse_location("a 3 b")) + self.assertEqual( + (f"a {mark}1{mark} here", [(259, 259, 7, f"a {mark}1{mark}"), (258, 258, 7, "here")]), + parse_location(f"{{{{Mapper Coords|1.3|1.3|7|text=a 1}}}} {here}"), + ) + + def test_location_holding_every_private_use_character(self): + characters = "".join(chr(code) for code in range(0xE000, 0xF900)) + here = " {{Mapper Coords|1.2|1.2|7|text=[[Fortress|here]]}}" + for value in ( + characters + here, + characters[:-1] + "" + here, + characters[:-1] + "ø({{Mapper Coords|1.1|1.1|7}})FF;" + here, + ): + with self.subTest(value=value[-60:]): + rows = [(258, 258, 7, "here")] + if "ø(" in value: + rows.insert(0, (257, 257, 7, None)) + self.assertEqual((clean_text_safely(value), rows), parse_location(value)) + def test_label_is_the_text_the_location_shows(self): for value, text, label in ( ("at {{Mapper Coords|127.88|124.58|4|text={{Hover|here|explanation}}}}.", "at here.", "here"), diff --git a/tibiawikisql/utils.py b/tibiawikisql/utils.py index dbd574b..af2b8d8 100644 --- a/tibiawikisql/utils.py +++ b/tibiawikisql/utils.py @@ -3,6 +3,7 @@ from __future__ import annotations import datetime +import html import re import time from collections import defaultdict @@ -273,14 +274,19 @@ def clean_text_safely(value: str) -> str: return mwparserfromhell.parse(value).strip_code().strip() -def _unused_private_character(value: str) -> str: - """Get a character from Unicode's private use area that does not appear in a value. +class _MarkCollisionError(Exception): + """A text about to be cleaned decodes to the placeholder mark, so the mark cannot tell its placeholders apart.""" - Raises: - StopIteration: If the value holds every one of the area's 6,400 characters. +def _unused_private_characters(value: str) -> Generator[str, None, None]: + """Get the characters of Unicode's private use area that appear neither in a value nor in its HTML-decoded form. + + Cleaning decodes the HTML entities in wiki text, so ```` reads as the area's first character. Wherever + mwparserfromhell decodes an entity to a private-use character, :func:`html.unescape` decodes it to the same one, and + it also decodes entities without the final semicolon. """ - return next(chr(code) for code in range(0xE000, 0xF900) if chr(code) not in value) + taken = set(value) | set(html.unescape(value)) + return (chr(code) for code in range(0xE000, 0xF900) if chr(code) not in taken) def parse_location(value: str) -> tuple[str, list[tuple[int, int, int, str | None]]]: @@ -295,11 +301,19 @@ def parse_location(value: str) -> tuple[str, list[tuple[int, int, int, str | Non removed. Where one is removed, the spaces before it go too when a space, punctuation or the end of the text follows, so ``Mintwallin {{Mapper Coords|...}}.`` reads ``Mintwallin.``; at the start of a line or after an opening parenthesis, the spaces after it go; and parentheses left with nothing in them are removed with the spaces before - them. Labels are put in after the rest of the text is cleaned, through a private-use character absent from the - value, so each is cleaned once. A location without the template reads as :func:`clean_text_safely` reads it. + them. Labels are put in after the rest of the text is cleaned, so each is cleaned once. A location without the + template reads as :func:`clean_text_safely` reads it. + + To put a label in, the template is first replaced by its index between two marks. The mark is a private-use + character that neither the value nor its HTML-decoded form holds, so not the one ```` decodes to, and that + no text about to be cleaned decodes to either, as when removing a template joins ``à`` and ``00;``. Only the + marked indices written into a text are put back in it; any other marked text stays as it is. A value that leaves no + such character, such as one holding all 6,400 of them, reads as :func:`clean_text_safely` reads it, coordinates + included, and each label is its ``text`` parameter cleaned on its own. Where :func:`clean_links` cannot clean the text or a label, :func:`clean_text_safely` gives its plain text instead, - so such a location gives a text and rows rather than an error. + so such a location gives a text and rows rather than an error. A label :func:`clean_links` drops from the text, as + in a ``{{Hover}}``'s later parameter, still gives its row a label the text does not show. Args: value: The raw location. @@ -309,24 +323,54 @@ def parse_location(value: str) -> tuple[str, list[tuple[int, int, int, str | Non :func:`parse_mapper_coords` reads, in the order they are written. The label is the text the location shows in the template's place, or ``None`` when it shows none. A template whose position is malformed gives no row. + """ + for mark in _unused_private_characters(value): + try: + return _parse_location_with_mark(value, mark) + except _MarkCollisionError: + continue + templates = mwparserfromhell.parse(value).filter_templates(matches=is_mapper_coords) + rows = [] + for template in templates: + position = parse_mapper_coords(template) + if position is not None: + label = clean_text_safely(str(template.get("text").value)) if template.has("text") else "" + rows.append((*position, label or None)) + return clean_text_safely(value), rows + + +def _parse_location_with_mark(value: str, mark: str) -> tuple[str, list[tuple[int, int, int, str | None]]]: + """Do :func:`parse_location`'s work with a placeholder mark absent from the value. + + Raises: + _MarkCollisionError: If a text about to be cleaned decodes to the mark. + """ code = mwparserfromhell.parse(value) templates = code.filter_templates(matches=is_mapper_coords) positions = [parse_mapper_coords(template) for template in templates] - mark = _unused_private_character(value) - placeholder = re.compile(f"{mark}([0-9]+){mark}") + placeholder = re.compile(f"{mark}[0-9]+{mark}") labels: list[str | None] = [None] * len(templates) + expansions: dict[str, str] = {} def clean(text: str) -> str: for pattern, replacement in removed_mapper_coords_rules: text = pattern.sub(replacement, text) - return placeholder.sub(lambda match: labels[int(match[1])], clean_text_safely(text)) + if html.unescape(text).count(mark) != text.count(mark): + raise _MarkCollisionError + written = {match[0] for match in placeholder.finditer(text)} + return placeholder.sub( + lambda match: expansions[match[0]] if match[0] in written else match[0], clean_text_safely(text), + ) # Innermost first, so a template inside another's label is replaced before that label is cleaned. for index in reversed(range(len(templates))): template = templates[index] - labels[index] = (clean(str(template.get("text").value)) if template.has("text") else "") or None - code.replace(template, Text(f"{mark}{index}{mark}" if labels[index] else REMOVED_MAPPER_COORDS)) + label = (clean(str(template.get("text").value)) if template.has("text") else "") or None + labels[index] = label + if label: + expansions[f"{mark}{index}{mark}"] = label + code.replace(template, Text(f"{mark}{index}{mark}" if label else REMOVED_MAPPER_COORDS)) rows = [(*position, labels[index]) for index, position in enumerate(positions) if position is not None] return clean(str(code)), rows From 19c24f0f66a3539ba887099002d9276d1ab7371e Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 21:24:32 +0000 Subject: [PATCH 16/17] fix: bound location mark retries --- tests/tests_utils.py | 24 ++++++++++++++++++++++++ tibiawikisql/utils.py | 40 +++++++++++++++++++++++++++++++--------- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 0b59994..0a17f1c 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -538,6 +538,30 @@ def test_location_puts_labels_back_only_where_written(self): parse_location(f"{{{{Mapper Coords|1.3|1.3|7|text=a 1}}}} {here}"), ) + def test_location_with_a_decimal_reference_past_the_digit_limit(self): + digits = "1" * 5000 + removed = "({{Mapper Coords|1.1|1.1|7}})" + self.assertEqual((f"a &#{digits}; b", []), parse_location(f"a &#{digits}; b")) + self.assertEqual( + (f"a &#{digits}; b", [(257, 257, 7, None)]), + parse_location(f"a &#{digits[:2500]}{removed}{digits[2500:]}; b"), + ) + mark = "\ue000" + self.assertEqual( + (f"a &#{digits}; {mark}0{mark} here", [(258, 258, 7, "here")]), + parse_location(f"a &#{digits}; 0 {{{{Mapper Coords|1.2|1.2|7|text=here}}}}"), + ) + + def test_location_whose_marks_keep_colliding(self): + removed = "({{Mapper Coords|1.1|1.1|7}})" + # Each entity, joined once its template is removed, decodes to the next candidate mark. + value = " ".join(f"&#x{code:X}"[:-2] + removed + f"{code:X}"[-2:] + ";" for code in range(0xE000, 0xE00A)) + value += " {{Mapper Coords|1.2|1.2|7|text=here}}" + self.assertEqual( + (clean_text_safely(value), [(257, 257, 7, None)] * 10 + [(258, 258, 7, "here")]), parse_location(value), + ) + self.assertIn("à(1.1)00;", parse_location(value)[0]) + def test_location_holding_every_private_use_character(self): characters = "".join(chr(code) for code in range(0xE000, 0xF900)) here = " {{Mapper Coords|1.2|1.2|7|text=[[Fortress|here]]}}" diff --git a/tibiawikisql/utils.py b/tibiawikisql/utils.py index af2b8d8..76afb87 100644 --- a/tibiawikisql/utils.py +++ b/tibiawikisql/utils.py @@ -8,6 +8,7 @@ import time from collections import defaultdict from contextlib import contextmanager +from itertools import islice from typing import Any, Literal, TYPE_CHECKING, overload import mwparserfromhell @@ -73,6 +74,13 @@ punctuation or the end of the text, the spaces before it go; anywhere else only the template goes.""" +LOCATION_MARK_ATTEMPTS = 4 +"""How many placeholder marks :func:`parse_location` tries before it reads the location as plain text.""" + +decimal_reference_pattern = re.compile(r"�*([0-9]{1,8})[0-9]*") +"""A decimal HTML character reference, with its leading zeros and its digits past the eighth significant one outside +the group. Past seven significant digits the value is above U+10FFFF, so eight are enough to decode it the same.""" + class Elapsed: """Holds the elapsed time measured by the [tibiawikisql.utils.timed][timed] context manager. @@ -282,13 +290,22 @@ def _unused_private_characters(value: str) -> Generator[str, None, None]: """Get the characters of Unicode's private use area that appear neither in a value nor in its HTML-decoded form. Cleaning decodes the HTML entities in wiki text, so ```` reads as the area's first character. Wherever - mwparserfromhell decodes an entity to a private-use character, :func:`html.unescape` decodes it to the same one, and - it also decodes entities without the final semicolon. + mwparserfromhell decodes an entity to a private-use character, :func:`_decode_entities` decodes it to the same one, + and it also decodes entities without the final semicolon. """ - taken = set(value) | set(html.unescape(value)) + taken = set(value) | set(_decode_entities(value)) return (chr(code) for code in range(0xE000, 0xF900) if chr(code) not in taken) +def _decode_entities(text: str) -> str: + """Decode the HTML entities in a text as :func:`html.unescape` does, without its error on long decimal references. + + :func:`html.unescape` reads a decimal reference with :class:`int`, which raises :class:`ValueError` past Python's + 4,300-digit limit, so each is shortened to its first eight significant digits first, which decodes the same. + """ + return html.unescape(decimal_reference_pattern.sub(r"&#\1", text)) + + def parse_location(value: str) -> tuple[str, list[tuple[int, int, int, str | None]]]: """Clean a location and read the positions of its ``{{Mapper Coords}}`` templates. @@ -306,10 +323,12 @@ def parse_location(value: str) -> tuple[str, list[tuple[int, int, int, str | Non To put a label in, the template is first replaced by its index between two marks. The mark is a private-use character that neither the value nor its HTML-decoded form holds, so not the one ```` decodes to, and that - no text about to be cleaned decodes to either, as when removing a template joins ``à`` and ``00;``. Only the - marked indices written into a text are put back in it; any other marked text stays as it is. A value that leaves no - such character, such as one holding all 6,400 of them, reads as :func:`clean_text_safely` reads it, coordinates - included, and each label is its ``text`` parameter cleaned on its own. + no text about to be cleaned decodes to either, as when removing a template joins ``à`` and ``00;``. A mark one + of those texts decodes to is found only once that text is reached, so at most :data:`LOCATION_MARK_ATTEMPTS` marks + are tried. Only the marked indices written into a text are put back in it; any other marked text stays as it is. A + value that leaves no such character, such as one holding all 6,400 of them, or whose texts decode to every mark + tried, reads as :func:`clean_text_safely` reads it, coordinates included, and each label is its ``text`` parameter + cleaned on its own. Where :func:`clean_links` cannot clean the text or a label, :func:`clean_text_safely` gives its plain text instead, so such a location gives a text and rows rather than an error. A label :func:`clean_links` drops from the text, as @@ -324,7 +343,10 @@ def parse_location(value: str) -> tuple[str, list[tuple[int, int, int, str | Non the template's place, or ``None`` when it shows none. A template whose position is malformed gives no row. """ - for mark in _unused_private_characters(value): + # A mark a text decodes to is found only after a whole attempt, so crafted text aiming entities at successive + # candidates would make each of the 6,400 cost one. Real wiki text holds no private-use entities, so a few + # attempts are plenty, and the cap bounds the work to that many. + for mark in islice(_unused_private_characters(value), LOCATION_MARK_ATTEMPTS): try: return _parse_location_with_mark(value, mark) except _MarkCollisionError: @@ -356,7 +378,7 @@ def _parse_location_with_mark(value: str, mark: str) -> tuple[str, list[tuple[in def clean(text: str) -> str: for pattern, replacement in removed_mapper_coords_rules: text = pattern.sub(replacement, text) - if html.unescape(text).count(mark) != text.count(mark): + if _decode_entities(text).count(mark) != text.count(mark): raise _MarkCollisionError written = {match[0] for match in placeholder.finditer(text)} return placeholder.sub( From 49dcade9101c4bc86c6fbf4a95a74d0ed62b5322 Mon Sep 17 00:00:00 2001 From: Jakub Mucha Date: Fri, 30 Oct 2026 21:31:46 +0000 Subject: [PATCH 17/17] docs: state the location cleaning limit --- tibiawikisql/utils.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tibiawikisql/utils.py b/tibiawikisql/utils.py index 76afb87..7c86a1e 100644 --- a/tibiawikisql/utils.py +++ b/tibiawikisql/utils.py @@ -270,8 +270,12 @@ def clean_text_safely(value: str) -> str: :func:`clean_links` of the value whenever it can clean it, so ``{{Hover|Negotiable|ask}}`` is ``Negotiable``. When it raises :class:`IndexError` or :class:`ValueError`, the two errors it is known to raise on wiki text, the stripped plain text mwparserfromhell's ``strip_code`` gives instead, where links become their text and - templates and comments are dropped, so ``1,000 {{GP}}`` is ``1,000`` and ``{{A|x|{{B|y}}}}`` is empty. So a - value :func:`clean_links` cannot clean still gives a text rather than an error. + templates and comments are dropped, so ``1,000 {{GP}}`` is ``1,000`` and ``{{A|x|{{B|y}}}}`` is empty. + + Raises: + ValueError: The known limit: the value holds a zero-padded decimal HTML entity of more than 4,300 digits. Both + :func:`clean_links` and the fallback decode it with mwparserfromhell's ``HTMLEntity.normalize``, which + reads it with :class:`int` past Python's digit limit. """ try: @@ -330,9 +334,9 @@ def parse_location(value: str) -> tuple[str, list[tuple[int, int, int, str | Non tried, reads as :func:`clean_text_safely` reads it, coordinates included, and each label is its ``text`` parameter cleaned on its own. - Where :func:`clean_links` cannot clean the text or a label, :func:`clean_text_safely` gives its plain text instead, - so such a location gives a text and rows rather than an error. A label :func:`clean_links` drops from the text, as - in a ``{{Hover}}``'s later parameter, still gives its row a label the text does not show. + Where :func:`clean_links` cannot clean the text or a label, :func:`clean_text_safely` gives its plain text instead. + A label :func:`clean_links` drops from the text, as in a ``{{Hover}}``'s later parameter, still gives its row a + label the text does not show. Args: value: The raw location. @@ -342,6 +346,10 @@ def parse_location(value: str) -> tuple[str, list[tuple[int, int, int, str | Non :func:`parse_mapper_coords` reads, in the order they are written. The label is the text the location shows in the template's place, or ``None`` when it shows none. A template whose position is malformed gives no row. + Raises: + ValueError: The known limit, from :func:`clean_text_safely`: the location holds a zero-padded decimal HTML + entity of more than 4,300 digits, written whole or split by a template whose removal joins it. + """ # A mark a text decodes to is found only after a whole attempt, so crafted text aiming entities at successive # candidates would make each of the 6,400 cost one. Real wiki text holds no private-use entities, so a few