From ed404f46c2eaff5005912a05609466b57cb47caf Mon Sep 17 00:00:00 2001 From: Saif Ali <128289649+SAKMZ@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:15:44 +0530 Subject: [PATCH] Improve device and bot detection for 3.1.0 - Add is_tv for smart TVs and streaming devices, which were reported as touch capable tablets, phones or PCs - Catch bots ua-parser misses, using product tokens, contact URLs, contact emails and headless browsers - Treat Generic Tablet as a tablet and Dalvik clients as phones, and add KaiOS to the mobile OS families - Accept None and bytes in parse() instead of raising TypeError - Move to the current ua-parser API, requiring ua-parser>=1.0.0, and add a fast extra for the Rust-backed parser - Run CI against both parsers and update the publish workflow actions --- .github/workflows/ci.yml | 5 +++ .github/workflows/publish.yml | 4 +- README.md | 25 ++++++++++-- pyproject.toml | 9 ++++- user_agents/__init__.py | 2 +- user_agents/parsers.py | 67 +++++++++++++++++++++++++------- user_agents/tests.py | 73 +++++++++++++++++++++++++++++++++++ 7 files changed, 163 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14dfff8..d0483d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,3 +29,8 @@ jobs: - name: Run tests run: python -m unittest -v user_agents.tests + + - name: Run tests with the fast parser + run: | + python -m pip install -e ".[fast]" + python -m unittest -v user_agents.tests diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 20e5505..ba7cd2b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,8 +12,8 @@ jobs: name: Build distributions runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.12" - name: Build sdist and wheel diff --git a/README.md b/README.md index 9b2b50b..f551be8 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,11 @@ to do the actual parsing of the raw user agent string. pip install user-agents-ng +For a roughly 20x parsing speedup on uncached user agents, install the optional +Rust-backed parser too. It's a drop-in change, no code needs updating: + + pip install "user-agents-ng[fast]" + ### Migrating from `user-agents` `user-agents-ng` is a drop-in replacement. Swap the dependency and your code @@ -81,8 +86,13 @@ Currently these attributes are supported: * `is_mobile`: whether user agent is identified as a mobile phone (iPhone, Android phones, Blackberry, Windows Phone devices etc) * `is_tablet`: whether user agent is identified as a tablet device (iPad, Kindle Fire, Nexus 7 etc) * `is_pc`: whether user agent is identified to be running a traditional "desktop" OS (Windows, OS X, Linux) +* `is_tv`: whether user agent is identified as a smart TV or streaming device (Samsung Tizen, LG webOS, Android TV, Fire TV, Roku, Chromecast etc) * `is_touch_capable`: whether user agent has touch capabilities -* `is_bot`: whether user agent is a search engine crawler/spider +* `is_bot`: whether user agent is a crawler, spider or headless browser. Besides the bots ua-parser knows about, this catches any user agent with a bot-style product token (`Applebot/0.1`), a contact URL (`+http://example.com/bot`) or a contact email address +* `is_email_client`: whether user agent is an email client (Outlook, Thunderbird, Apple Mail etc) + +`parse()` also accepts `bytes` and `None`, so a raw header or a request with no +`User-Agent` header at all won't raise an exception. For example: @@ -97,7 +107,7 @@ user_agent.is_tablet # returns False user_agent.is_touch_capable # returns False user_agent.is_pc # returns False user_agent.is_bot # returns False -str(user_agent) # returns "BlackBerry 9700 / BlackBerry OS 5 / BlackBerry 9700" +str(user_agent) # returns "BlackBerry 9700 / BlackBerry OS 5.0.0 / BlackBerry 9700" # Now a Samsung Galaxy S3 ua_string = 'Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30' @@ -117,7 +127,7 @@ user_agent.is_tablet # returns False user_agent.is_touch_capable # returns True user_agent.is_pc # returns True user_agent.is_bot # returns False -str(user_agent) # returns "PC / Windows 8 / IE 10" +str(user_agent) # returns "PC / Windows 8 / IE 10.0" ``` ## Running Tests @@ -126,6 +136,15 @@ str(user_agent) # returns "PC / Windows 8 / IE 10" ## Changelog +### Version 3.1.0 + +* Added `is_tv` for smart TVs and streaming devices, which no longer count as touch capable tablets, phones or PCs. +* `is_bot` now catches bots missed by ua-parser, like Applebot, Amazonbot, Bytespider and headless Chrome. +* Added the `user-agents-ng[fast]` extra, which parses about 20x faster. +* `parse()` accepts `None` and `bytes` instead of raising `TypeError`. +* Firefox on Android tablets is now a tablet, Dalvik clients are no longer tablets, and KaiOS is mobile. +* Now uses the current `ua-parser` API and requires `ua-parser>=1.0.0`, which keeps detection data up to date. + ### Version 3.0.0 * Released as `user-agents-ng`, a maintained successor to `user-agents`. diff --git a/pyproject.toml b/pyproject.toml index 761dc29..656daac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "user-agents-ng" -version = "3.0.0" +version = "3.1.0" description = "A maintained successor to user-agents: identify devices (phones, tablets) and their capabilities by parsing browser user agent strings." readme = "README.md" requires-python = ">=3.9" @@ -18,7 +18,7 @@ maintainers = [ ] keywords = ["user agent", "user-agents", "browser", "device", "detection", "parser", "mobile", "tablet", "bot"] dependencies = [ - "ua-parser>=0.10.0", + "ua-parser>=1.0.0", ] classifiers = [ "Development Status :: 5 - Production/Stable", @@ -38,6 +38,11 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] +[project.optional-dependencies] +fast = [ + "ua-parser[regex]>=1.0.0", +] + [project.urls] Homepage = "https://github.com/SAKMZ/python-user-agents" Repository = "https://github.com/SAKMZ/python-user-agents" diff --git a/user_agents/__init__.py b/user_agents/__init__.py index beeb266..43d305e 100644 --- a/user_agents/__init__.py +++ b/user_agents/__init__.py @@ -1,4 +1,4 @@ -VERSION = (3, 0, 0) +VERSION = (3, 1, 0) from .parsers import UserAgent, parse diff --git a/user_agents/parsers.py b/user_agents/parsers.py index 2de0e5d..592a6d2 100644 --- a/user_agents/parsers.py +++ b/user_agents/parsers.py @@ -1,6 +1,7 @@ -from typing import Any, NamedTuple, Optional +import re +from typing import Any, NamedTuple, Optional, Union -from ua_parser import user_agent_parser +import ua_parser MOBILE_DEVICE_FAMILIES = ( @@ -26,6 +27,7 @@ 'Windows CE', 'Windows Mobile', 'Maemo', + 'KaiOS', ) MOBILE_BROWSER_FAMILIES = ( @@ -47,6 +49,7 @@ 'Galaxy Tab', 'Xoom', 'Dell Streak', + 'Generic Tablet', ) TABLET_DEVICE_BRANDS = ( @@ -89,6 +92,23 @@ 'YahooMobileMail', } +# Catches bots ua-parser doesn't know about yet +BOT_UA_PATTERN = re.compile( + r'\+https?://' # Contact URL, e.g. (+http://example.com/bot.html) + r'|@[\w-]+\.[a-z]{2,}' # Contact email address + r'|(?:bot|crawler|spider)/\d' # Product token, e.g. Applebot/0.1 + r'|compatible; [\w.-]*(?:bot|crawler|spider)\b' # e.g. compatible; Bytespider + r'|HeadlessChrome/|PhantomJS/', + re.IGNORECASE +) + +TV_UA_PATTERN = re.compile( + r'smart[- ]?tv|hbbtv|google ?tv|android ?tv|apple ?tv|web0s|netcast|vidaa' + r'|\broku\b|bravia|crkey' + r'|(?-i:\bAFT[A-Z0-9]{1,6}\b)', # Amazon Fire TV models, e.g. AFTKA + re.IGNORECASE +) + def verify_attribute(attribute: Any) -> Any: if isinstance(attribute, str) and attribute.isdigit(): @@ -163,12 +183,19 @@ def parse_device(family: str, brand: Optional[str], model: Optional[str]) -> Dev class UserAgent: - def __init__(self, user_agent_string: str): - ua_dict = user_agent_parser.Parse(user_agent_string) + def __init__(self, user_agent_string: Union[str, bytes, None]): + # Frameworks return None for a missing header, raw headers are bytes + if user_agent_string is None: + user_agent_string = '' + elif isinstance(user_agent_string, bytes): + user_agent_string = user_agent_string.decode('latin-1') + + result = ua_parser.parse(user_agent_string).with_defaults() + browser, os = result.user_agent, result.os self.ua_string = user_agent_string - self.os = parse_operating_system(**ua_dict['os']) - self.browser = parse_browser(**ua_dict['user_agent']) - self.device = parse_device(**ua_dict['device']) + self.os = parse_operating_system(os.family, os.major, os.minor, os.patch, os.patch_minor) + self.browser = parse_browser(browser.family, browser.major, browser.minor, browser.patch, browser.patch_minor) + self.device = parse_device(result.device.family, result.device.brand, result.device.model) def __str__(self) -> str: return "{device} / {os} / {browser}".format( @@ -182,9 +209,9 @@ def __repr__(self) -> str: def _is_android_tablet(self) -> bool: # Newer Android tablets don't have "Mobile" in their user agent string, - # older ones like Galaxy Tab still have "Mobile" though they're not - return ('Mobile Safari' not in self.ua_string and - self.browser.family != "Firefox Mobile") + # older ones like Galaxy Tab still have "Mobile" though they're not. + # Dalvik never says either way, so assume the far more common phone. + return 'Mobile' not in self.ua_string and 'Dalvik' not in self.ua_string def _is_blackberry_touch_capable_device(self) -> bool: # A helper to determine whether a BB phone has touch capabilities @@ -206,6 +233,9 @@ def get_browser(self) -> str: @property def is_tablet(self) -> bool: + # ua-parser reports Android TVs as Generic_Android_Tablet + if self.is_tv: + return False if self.device.family in TABLET_DEVICE_FAMILIES: return True if self.device.brand in TABLET_DEVICE_BRANDS: @@ -225,7 +255,7 @@ def is_mobile(self) -> bool: # First check for mobile device and mobile browser families if self.device.family in MOBILE_DEVICE_FAMILIES: return True - if self.is_tablet or self.is_pc: + if self.is_tablet or self.is_pc or self.is_tv: return False if self.browser.family in MOBILE_BROWSER_FAMILIES: return True @@ -254,6 +284,8 @@ def is_mobile(self) -> bool: @property def is_touch_capable(self) -> bool: # TODO: detect touch capable Nokia devices + if self.is_tv: + return False if self.os.family in TOUCH_CAPABLE_OS_FAMILIES: return True if self.device.family in TOUCH_CAPABLE_DEVICE_FAMILIES: @@ -271,7 +303,8 @@ def is_touch_capable(self) -> bool: def is_pc(self) -> bool: if self.device.family in MOBILE_DEVICE_FAMILIES or \ self.device.family in TABLET_DEVICE_FAMILIES or \ - self.device.brand in TABLET_DEVICE_BRANDS: + self.device.brand in TABLET_DEVICE_BRANDS or \ + self.is_tv: return False # Returns True for "PC" devices (Windows, Mac and Linux) if 'Windows NT' in self.ua_string or self.os.family in PC_OS_FAMILIES or \ @@ -291,14 +324,20 @@ def is_pc(self) -> bool: return True return False + @property + def is_tv(self) -> bool: + return TV_UA_PATTERN.search(self.ua_string) is not None + @property def is_bot(self) -> bool: - return self.device.family == 'Spider' + if self.device.family == 'Spider': + return True + return BOT_UA_PATTERN.search(self.ua_string) is not None @property def is_email_client(self) -> bool: return self.browser.family in EMAIL_PROGRAM_FAMILIES -def parse(user_agent_string: str) -> UserAgent: +def parse(user_agent_string: Union[str, bytes, None]) -> UserAgent: return UserAgent(user_agent_string) diff --git a/user_agents/tests.py b/user_agents/tests.py index ae9647b..d606280 100644 --- a/user_agents/tests.py +++ b/user_agents/tests.py @@ -245,6 +245,78 @@ def test_unicode_strings(self): self.assertEqual(ua_str, "iPhone / iOS 5.1 / Mobile Safari 5.1") self.assertIsInstance(ua_str, str) + def test_missing_user_agent(self): + for ua_string in (None, '', b''): + ua = parse(ua_string) + self.assertEqual(ua.ua_string, '') + self.assertEqual(str(ua), 'Other / Other / Other') + self.assertFalse(ua.is_bot) + + def test_bytes_user_agent(self): + ua = parse(iphone_ua_string.encode('latin-1')) + self.assertEqual(ua.ua_string, iphone_ua_string) + self.assertTrue(ua.is_mobile) + + def test_is_bot_undetected_by_ua_parser(self): + for ua_string in ( + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)', + 'Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/140.0.0.0 Safari/537.36', + 'ExampleFetcher/1.0 (contact: ops@example.com)', + 'Mozilla/5.0 (compatible; ExampleCrawler; http://example.com)', + 'SomeBot/2.3', + ): + self.assertTrue(parse(ua_string).is_bot, msg=ua_string) + + def test_is_bot_ignores_real_browsers(self): + for ua_string in ( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0', + 'Mozilla/5.0 (Linux; Android 11; CUBOT_X50) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 9; CUBOT P20 Build/PPR1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36', + iphone_ua_string, + 'curl/8.7.1', + 'okhttp/4.12.0', + ): + self.assertFalse(parse(ua_string).is_bot, msg=ua_string) + + def test_is_tv(self): + for ua_string in ( + 'Mozilla/5.0 (SMART-TV; Linux; Tizen 7.0) AppleWebKit/537.36 (KHTML, like Gecko) 94.0.4606.31/7.0 TV Safari/537.36', + 'Mozilla/5.0 (SMART-TV; X11; Linux armv7l) AppleWebKit/537.42 (KHTML, like Gecko) Safari/537.42', + 'Mozilla/5.0 (Web0S; Linux/SmartTV) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 WebAppManager', + 'Mozilla/5.0 (Linux; Android 12; BRAVIA 4K UR3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Linux; Android 9; AFTKA) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux aarch64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 CrKey/1.56.500000', + 'Roku/DVP-12.5 (12.5.0.4130-AE)', + ): + ua = parse(ua_string) + self.assertTrue(ua.is_tv, msg=ua_string) + self.assertFalse(ua.is_tablet, msg=ua_string) + self.assertFalse(ua.is_mobile, msg=ua_string) + self.assertFalse(ua.is_pc, msg=ua_string) + self.assertFalse(ua.is_touch_capable, msg=ua_string) + + def test_is_not_tv(self): + for ua in (iphone_ua, ipad_ua, galaxy_s3_ua, nexus_7_ua, ie_ua, mac_safari_ua, ubuntu_firefox_ua): + self.assertFalse(ua.is_tv, msg=ua.ua_string) + + def test_android_firefox_tablet(self): + ua = parse('Mozilla/5.0 (Android 14; Tablet; rv:140.0) Gecko/140.0 Firefox/140.0') + self.assertTrue(ua.is_tablet) + self.assertFalse(ua.is_mobile) + + def test_android_dalvik_is_mobile(self): + ua = parse('Dalvik/2.1.0 (Linux; U; Android 7.1.2; Nexus 6P Build/N2G47O)') + self.assertFalse(ua.is_tablet) + self.assertTrue(ua.is_mobile) + + def test_kaios_is_mobile(self): + ua = parse('Mozilla/5.0 (Mobile; Nokia 8110 4G; rv:48.0) Gecko/48.0 Firefox/48.0 KAIOS/2.5') + self.assertTrue(ua.is_mobile) + self.assertFalse(ua.is_tablet) + with open(os.path.join(os.path.dirname(__file__), 'devices.json')) as f: devices = json.load(f) @@ -254,6 +326,7 @@ def test_wrapper(items): def test_func(self): attrs = ('is_bot', 'is_mobile', 'is_pc', 'is_tablet', 'is_touch_capable') + self.assertFalse(items['user_agent'].is_tv) for attr in attrs: self.assertEqual( getattr(items['user_agent'], attr), items[attr], msg=attr)