Skip to content

Commit 5a952af

Browse files
author
spirit
committed
More future-proof language code lookup
1 parent e2f3723 commit 5a952af

2 files changed

Lines changed: 239 additions & 18 deletions

File tree

‎language_tool/__init__.py‎

Lines changed: 19 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import sys
2828
import urllib.parse
2929
import urllib.request
30+
import warnings
3031
from collections import namedtuple
3132
from contextlib import closing
3233
from functools import total_ordering
@@ -37,6 +38,7 @@
3738
except ImportError:
3839
from xml.etree import ElementTree
3940

41+
from .country_codes import get_country_code
4042
from .which import which
4143

4244

@@ -46,19 +48,7 @@
4648

4749
FAILSAFE_LANGUAGE = "en"
4850
LANGUAGE_RE = re.compile(r"^([a-z]{2,3})(?:[_-]([a-z]{2}))?$", re.I)
49-
LANGUAGE_TAGS_MAPPING = {
50-
"English (Australian)": "en-AU",
51-
"English (Canadian)": "en-CA",
52-
"English (GB)": "en-GB",
53-
"English (New Zealand)": "en-NZ",
54-
"English (South African)": "en-ZA",
55-
"English (US)": "en-US",
56-
"German (Austria)": "de-AT",
57-
"German (Germany)": "de-DE",
58-
"German (Swiss)": "de-CH",
59-
"Portuguese (Brazil)": "pt-BR",
60-
"Portuguese (Portugal)": "pt-PT",
61-
}
51+
6252

6353
if os.name == "nt":
6454
startupinfo = subprocess.STARTUPINFO()
@@ -269,11 +259,22 @@ def _get_languages(cls):
269259
if not cls._server_is_alive():
270260
cls._start_server_on_free_port()
271261
url = urllib.parse.urljoin(cls.url, "Languages")
272-
root = cls._get_root(url, num_tries=1)
273-
return {
274-
LANGUAGE_TAGS_MAPPING.get(e.get("name"), e.get("abbr"))
275-
for e in root
276-
}
262+
languages = set()
263+
for e in cls._get_root(url, num_tries=1):
264+
language = e.get("abbr")
265+
if len(re.split(r"[_-]", language)) < 2:
266+
match = re.search(r"\((.*?)\)", e.get("name"))
267+
if match:
268+
country_name = match.group(1)
269+
try:
270+
country_code = get_country_code(country_name)
271+
except KeyError:
272+
warnings.warn(
273+
"unknown language: {!r}".format(e.get("name")))
274+
else:
275+
language += "-" + country_code
276+
languages.add(language)
277+
return languages
277278

278279
@classmethod
279280
def _get_root(cls, url, data=None, num_tries=2):

‎language_tool/country_codes.py‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
import difflib
2+
3+
__all__ = ["get_country_code"]
4+
5+
COUNTRY_CODES = {
6+
"Afghanistan": "AF",
7+
"Albania": "AL",
8+
"Algeria": "DZ",
9+
"Andorra": "AD",
10+
"Angola": "AO",
11+
"Antigua and Barbuda": "AG",
12+
"Argentina": "AR",
13+
"Armenia": "AM",
14+
"Australia": "AU",
15+
"Austria": "AT",
16+
"Azerbaijan": "AZ",
17+
"Bahamas": "BS",
18+
"Bahrain": "BH",
19+
"Bangladesh": "BD",
20+
"Barbados": "BB",
21+
"Belarus": "BY",
22+
"Belgium": "BE",
23+
"Belize": "BZ",
24+
"Benin": "BJ",
25+
"Bhutan": "BT",
26+
"Bolivia": "BO",
27+
"Bosnia and Herzegovina": "BA",
28+
"Botswana": "BW",
29+
"Brazil": "BR",
30+
"Brunei Darussalam": "BN",
31+
"Bulgaria": "BG",
32+
"Burkina Faso": "BF",
33+
"Burundi": "BI",
34+
"Cambodia": "KH",
35+
"Cameroon": "CM",
36+
"Canada": "CA",
37+
"Cape Verde": "CV",
38+
"Central African Republic": "CF",
39+
"Chad": "TD",
40+
"Chile": "CL",
41+
"Colombia": "CO",
42+
"Comoros": "KM",
43+
"Costa Rica": "CR",
44+
"Croatia": "HR",
45+
"Cuba": "CU",
46+
"Cyprus": "CY",
47+
"Czech Republic": "CZ",
48+
"Côte d'Ivoire": "CI",
49+
"Democratic Republic of the Congo": "CD",
50+
"Denmark": "DK",
51+
"Djibouti": "DJ",
52+
"Dominica": "DM",
53+
"Dominican Republic": "DO",
54+
"East Timor": "TL",
55+
"Ecuador": "EC",
56+
"Egypt": "EG",
57+
"El Salvador": "SV",
58+
"Equatorial Guinea": "GQ",
59+
"Eritrea": "ER",
60+
"Estonia": "EE",
61+
"Ethiopia": "ET",
62+
"Federated States of Micronesia": "FM",
63+
"Fiji": "FJ",
64+
"Finland": "FI",
65+
"France": "FR",
66+
"Gabon": "GA",
67+
"Georgia": "GE",
68+
"Germany": "DE",
69+
"Ghana": "GH",
70+
"Greece": "GR",
71+
"Grenada": "GD",
72+
"Guatemala": "GT",
73+
"Guinea": "GN",
74+
"Guinea-Bissau": "GW",
75+
"Guyana": "GY",
76+
"Haiti": "GT",
77+
"Honduras": "HN",
78+
"Hungary": "HU",
79+
"Iceland": "IS",
80+
"India": "IN",
81+
"Indonesia": "ID",
82+
"Iran": "IR",
83+
"Iraq": "IQ",
84+
"Israel": "IL",
85+
"Italy": "IT",
86+
"Jamaica": "JM",
87+
"Japan": "JP",
88+
"Jordan": "JO",
89+
"Kazakhstan": "KZ",
90+
"Kenya": "KE",
91+
"Kingdom of the Netherlands": "NL",
92+
"Kiribati": "KI",
93+
"Kuwait": "KW",
94+
"Kyrgyzstan": "KG",
95+
"Laos": "LA",
96+
"Latvia": "LV",
97+
"Lebanon": "LB",
98+
"Lesotho": "LS",
99+
"Liberia": "LR",
100+
"Libya": "LY",
101+
"Liechtenstein": "LI",
102+
"Lithuania": "LT",
103+
"Luxembourg": "LU",
104+
"Macedonia": "MK",
105+
"Madagascar": "MG",
106+
"Malawi": "MW",
107+
"Malaysia": "MY",
108+
"Maldives": "MV",
109+
"Mali": "ML",
110+
"Malta": "MT",
111+
"Marshall Islands": "MH",
112+
"Mauritania": "MR",
113+
"Mauritius": "MU",
114+
"Mexico": "MX",
115+
"Moldova": "MD",
116+
"Monaco": "MC",
117+
"Mongolia": "MN",
118+
"Montenegro": "ME",
119+
"Morocco": "MA",
120+
"Mozambique": "MZ",
121+
"Myanmar": "MM",
122+
"Namibia": "NA",
123+
"Nauru": "NR",
124+
"Nepal": "NP",
125+
"New Zealand": "NZ",
126+
"Nicaragua": "NI",
127+
"Niger": "NE",
128+
"Nigeria": "NG",
129+
"North Korea": "KP",
130+
"Norway": "NO",
131+
"Oman": "OM",
132+
"Pakistan": "PK",
133+
"Palau": "PW",
134+
"Panama": "PA",
135+
"Papua New Guinea": "PG",
136+
"Paraguay": "PY",
137+
"People's Republic of China": "CN",
138+
"Peru": "PE",
139+
"Philippines": "PH",
140+
"Poland": "PL",
141+
"Portugal": "PT",
142+
"Qatar": "QA",
143+
"Republic of Ireland": "IE",
144+
"Republic of the Congo": "CG",
145+
"Romania": "RO",
146+
"Russia": "RU",
147+
"Rwanda": "RW",
148+
"Saint Kitts and Nevis": "KN",
149+
"Saint Lucia": "LC",
150+
"Saint Vincent and the Grenadines": "VC",
151+
"Samoa": "WS",
152+
"San Marino": "SM",
153+
"Saudi Arabia": "SA",
154+
"Senegal": "SN",
155+
"Serbia": "RS",
156+
"Seychelles": "SC",
157+
"Sierra Leone": "SL",
158+
"Singapore": "SG",
159+
"Slovakia": "SK",
160+
"Slovenia": "SI",
161+
"Solomon Islands": "SB",
162+
"Somalia": "SO",
163+
"South Africa": "ZA",
164+
"South Korea": "KR",
165+
"Spain": "ES",
166+
"Sri Lanka": "LK",
167+
"Sudan": "SD",
168+
"Suriname": "SR",
169+
"Swaziland": "SZ",
170+
"Sweden": "SE",
171+
"Switzerland": "CH",
172+
"Syria": "SY",
173+
"São Tomé and Príncipe": "ST",
174+
"Tajikistan": "TJ",
175+
"Tanzania": "TZ",
176+
"Thailand": "TH",
177+
"The Gambia": "GM",
178+
"Togo": "TG",
179+
"Tonga": "TO",
180+
"Trinidad and Tobago": "TT",
181+
"Tunisia": "TN",
182+
"Turkey": "TR",
183+
"Turkmenistan": "TM",
184+
"Tuvalu": "TV",
185+
"Uganda": "UG",
186+
"Ukraine": "UA",
187+
"United Arab Emirates": "AE",
188+
"United Kingdom": "GB",
189+
"United States": "US",
190+
"Uruguay": "UY",
191+
"Uzbekistan": "UZ",
192+
"Vanuatu": "VU",
193+
"Vatican City": "VA",
194+
"Venezuela": "VE",
195+
"Vietnam": "VN",
196+
"Yemen": "YE",
197+
"Zambia": "ZM",
198+
"Zimbabwe": "ZW",
199+
}
200+
201+
LANGUAGE_TO_COUNTRY = {
202+
"Swiss": "CH",
203+
}
204+
205+
206+
def get_country_code(country_name):
207+
if country_name in COUNTRY_CODES.values():
208+
return country_name
209+
210+
codes = difflib.get_close_matches(
211+
country_name, COUNTRY_CODES.keys(), 1)
212+
if codes:
213+
return COUNTRY_CODES[codes[0]]
214+
215+
codes = difflib.get_close_matches(
216+
country_name, LANGUAGE_TO_COUNTRY.keys(), 1)
217+
if codes:
218+
return LANGUAGE_TO_COUNTRY[codes[0]]
219+
220+
raise KeyError(country_name)

0 commit comments

Comments
 (0)