Implement Unicode 16 (#208)

* Update to Unicode 16 test vectors
* Update README to reflect transitional processing is gone
* Pre-compute UTS46 tests as part of make-libdata
* Don't use ruff to format tests
* Remove extra UTS46 test vector
* Omit vectors known to be broken in test suite

Author: kjd

.github/workflows/python-package.yml

@@ -35,7 +35,7 @@ jobs:
         run: python -m pip install ruff mypy
       - name: Check code style with ruff
         run: |
-          ruff format --check idna tests
+          ruff format --check idna
           ruff check idna tests
       - name: Check types with mypy
         run: mypy --strict idna

README.rst

@@ -79,13 +79,10 @@ ways a user may input a domain name. This functionality, known as
 a “mapping”, is considered by the specification to be a local
 user-interface issue distinct from IDNA conversion functionality.
 
-This library provides one such mapping that was developed by the
-Unicode Consortium. Known as `Unicode IDNA Compatibility Processing
-<https://unicode.org/reports/tr46/>`_, it provides for both a regular
-mapping for typical applications, as well as a transitional mapping to
-help migrate from older IDNA 2003 applications. Strings are
-preprocessed according to Section 4.4 “Preprocessing for IDNA2008”
-prior to the IDNA operations.
+This library provides one such mapping — `Unicode IDNA Compatibility
+Processing <https://unicode.org/reports/tr46/>`_ developed by the Unicode
+Consortium. Strings are preprocessed according to Section 4.4
+“Preprocessing for IDNA2008” prior to the IDNA operations.
 
 For example, “Königsgäßchen” is not a permissible label as *LATIN
 CAPITAL LETTER K* is not allowed (nor are capital letters in general).
@@ -103,24 +100,6 @@ conversion.
     >>> print(idna.decode('xn--knigsgchen-b4a3dun'))
     königsgäßchen
 
-Transitional processing provides conversions to help transition from
-the older 2003 standard to the current standard. For example, in the
-original IDNA specification, the *LATIN SMALL LETTER SHARP S* (ß) was
-converted into two *LATIN SMALL LETTER S* (ss), whereas in the current
-IDNA specification this conversion is not performed.
-
-.. code-block:: pycon
-
-    >>> idna.encode('Königsgäßchen', uts46=True, transitional=True)
-    'xn--knigsgsschen-lcb0w'
-
-Implementers should use transitional processing with caution, only in
-rare cases where conversion from legacy labels to current labels must be
-performed (i.e. IDNA implementations that pre-date 2008). For typical
-applications that just need to convert labels, transitional processing
-is unlikely to be beneficial and could produce unexpected incompatible
-results.
-
 ``encodings.idna`` Compatibility
 ++++++++++++++++++++++++++++++++
 
@@ -206,3 +185,7 @@ Additional Notes
   may wish to consider trying the encode/decode operation in this library
   first, and then falling back to using `encodings.idna`. See `the Github
   project <https://github.com/kjd/idna/issues/18>`_ for more discussion.
+
+* **Transitional processing**. Unicode 16.0.0 removed transitional
+  processing so the `transitional` argument for the encode() method
+  no longer has any effect and will be removed at a later date.

idna/idnadata.py

@@ -1,6 +1,7 @@
 # This file is automatically generated by tools/idna-data
 
-__version__ = "15.1.0"
+__version__ = "16.0.0"
+
 scripts = {
     "Greek": (
         0x37000000374,
@@ -727,6 +728,7 @@
     0x88C: 68,
     0x88D: 68,
     0x88E: 82,
+    0x897: 84,
     0x898: 84,
     0x899: 84,
     0x89A: 84,
@@ -1873,8 +1875,17 @@
     0x10D25: 84,
     0x10D26: 84,
     0x10D27: 84,
+    0x10D69: 84,
+    0x10D6A: 84,
+    0x10D6B: 84,
+    0x10D6C: 84,
+    0x10D6D: 84,
     0x10EAB: 84,
     0x10EAC: 84,
+    0x10EC2: 82,
+    0x10EC3: 68,
+    0x10EC4: 68,
+    0x10EFC: 84,
     0x10EFD: 84,
     0x10EFE: 84,
     0x10EFF: 84,
@@ -2053,6 +2064,17 @@
     0x11372: 84,
     0x11373: 84,
     0x11374: 84,
+    0x113BB: 84,
+    0x113BC: 84,
+    0x113BD: 84,
+    0x113BE: 84,
+    0x113BF: 84,
+    0x113C0: 84,
+    0x113CE: 84,
+    0x113D0: 84,
+    0x113D2: 84,
+    0x113E1: 84,
+    0x113E2: 84,
     0x11438: 84,
     0x11439: 84,
     0x1143A: 84,
@@ -2108,7 +2130,6 @@
     0x116B5: 84,
     0x116B7: 84,
     0x1171D: 84,
-    0x1171E: 84,
     0x1171F: 84,
     0x11722: 84,
     0x11723: 84,
@@ -2265,6 +2286,7 @@
     0x11F3A: 84,
     0x11F40: 84,
     0x11F42: 84,
+    0x11F5A: 84,
     0x13430: 84,
     0x13431: 84,
     0x13432: 84,
@@ -2297,6 +2319,21 @@
     0x13453: 84,
     0x13454: 84,
     0x13455: 84,
+    0x1611E: 84,
+    0x1611F: 84,
+    0x16120: 84,
+    0x16121: 84,
+    0x16122: 84,
+    0x16123: 84,
+    0x16124: 84,
+    0x16125: 84,
+    0x16126: 84,
+    0x16127: 84,
+    0x16128: 84,
+    0x16129: 84,
+    0x1612D: 84,
+    0x1612E: 84,
+    0x1612F: 84,
     0x16AF0: 84,
     0x16AF1: 84,
     0x16AF2: 84,
@@ -2605,6 +2642,8 @@
     0x1E4ED: 84,
     0x1E4EE: 84,
     0x1E4EF: 84,
+    0x1E5EE: 84,
+    0x1E5EF: 84,
     0x1E8D0: 84,
     0x1E8D1: 84,
     0x1E8D2: 84,
@@ -3328,7 +3367,7 @@
         0x8600000086B,
         0x87000000888,
         0x8890000088F,
-        0x898000008E2,
+        0x897000008E2,
         0x8E300000958,
         0x96000000964,
         0x96600000970,
@@ -3563,6 +3602,7 @@
         0x1C0000001C38,
         0x1C4000001C4A,
         0x1C4D00001C7E,
+        0x1C8A00001C8B,
         0x1CD000001CD3,
         0x1CD400001CFB,
         0x1D0000001D2C,
@@ -3926,11 +3966,13 @@
         0xA7C30000A7C4,
         0xA7C80000A7C9,
         0xA7CA0000A7CB,
+        0xA7CD0000A7CE,
         0xA7D10000A7D2,
         0xA7D30000A7D4,
         0xA7D50000A7D6,
         0xA7D70000A7D8,
         0xA7D90000A7DA,
+        0xA7DB0000A7DC,
         0xA7F60000A7F8,
         0xA7FA0000A828,
         0xA82C0000A82D,
@@ -4000,6 +4042,7 @@
         0x105A3000105B2,
         0x105B3000105BA,
         0x105BB000105BD,
+        0x105C0000105F4,
         0x1060000010737,
         0x1074000010756,
         0x1076000010768,
@@ -4037,10 +4080,14 @@
         0x10CC000010CF3,
         0x10D0000010D28,
         0x10D3000010D3A,
+        0x10D4000010D50,
+        0x10D6900010D6E,
+        0x10D6F00010D86,
         0x10E8000010EAA,
         0x10EAB00010EAD,
         0x10EB000010EB2,
-        0x10EFD00010F1D,
+        0x10EC200010EC5,
+        0x10EFC00010F1D,
         0x10F2700010F28,
         0x10F3000010F51,
         0x10F7000010F86,
@@ -4086,6 +4133,16 @@
         0x1135D00011364,
         0x113660001136D,
         0x1137000011375,
+        0x113800001138A,
+        0x1138B0001138C,
+        0x1138E0001138F,
+        0x11390000113B6,
+        0x113B7000113C1,
+        0x113C2000113C3,
+        0x113C5000113C6,
+        0x113C7000113CB,
+        0x113CC000113D4,
+        0x113E1000113E3,
         0x114000001144B,
         0x114500001145A,
         0x1145E00011462,
@@ -4100,6 +4157,7 @@
         0x116500001165A,
         0x11680000116B9,
         0x116C0000116CA,
+        0x116D0000116E4,
         0x117000001171B,
         0x1171D0001172C,
         0x117300001173A,
@@ -4123,6 +4181,8 @@
         0x11A5000011A9A,
         0x11A9D00011A9E,
         0x11AB000011AF9,
+        0x11BC000011BE1,
+        0x11BF000011BFA,
         0x11C0000011C09,
         0x11C0A00011C37,
         0x11C3800011C41,
@@ -4147,14 +4207,16 @@
         0x11F0000011F11,
         0x11F1200011F3B,
         0x11F3E00011F43,
-        0x11F5000011F5A,
+        0x11F5000011F5B,
         0x11FB000011FB1,
         0x120000001239A,
         0x1248000012544,
         0x12F9000012FF1,
         0x1300000013430,
         0x1344000013456,
+        0x13460000143FB,
         0x1440000014647,
+        0x161000001613A,
         0x1680000016A39,
         0x16A4000016A5F,
         0x16A6000016A6A,
@@ -4167,6 +4229,8 @@
         0x16B5000016B5A,
         0x16B6300016B78,
         0x16B7D00016B90,
+        0x16D4000016D6D,
+        0x16D7000016D7A,
         0x16E6000016E80,
         0x16F0000016F4B,
         0x16F4F00016F88,
@@ -4176,7 +4240,7 @@
         0x16FF000016FF2,
         0x17000000187F8,
         0x1880000018CD6,
-        0x18D0000018D09,
+        0x18CFF00018D09,
         0x1AFF00001AFF4,
         0x1AFF50001AFFC,
         0x1AFFD0001AFFF,
@@ -4191,6 +4255,7 @@
         0x1BC800001BC89,
         0x1BC900001BC9A,
         0x1BC9D0001BC9F,
+        0x1CCF00001CCFA,
         0x1CF000001CF2E,
         0x1CF300001CF47,
         0x1DA000001DA37,
@@ -4214,6 +4279,7 @@
         0x1E2900001E2AF,
         0x1E2C00001E2FA,
         0x1E4D00001E4FA,
+        0x1E5D00001E5FB,
         0x1E7E00001E7E7,
         0x1E7E80001E7EC,
         0x1E7ED0001E7EF,