Improve device and bot detection for 3.1.0

.github/workflows/ci.yml

@@ -29,3 +29,8 @@ jobs:
 
       - name: Run tests
         run: python -m unittest -v user_agents.tests
+
+      - name: Run tests with the fast parser
+        run: |
+          python -m pip install -e ".[fast]"
+          python -m unittest -v user_agents.tests

.github/workflows/publish.yml

@@ -12,8 +12,8 @@ jobs:
     name: Build distributions
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: actions/setup-python@v6
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
         with:
           python-version: "3.12"
       - name: Build sdist and wheel

README.md

@@ -25,6 +25,11 @@ to do the actual parsing of the raw user agent string.
 
     pip install user-agents-ng
 
+For a roughly 20x parsing speedup on uncached user agents, install the optional
+Rust-backed parser too. It's a drop-in change, no code needs updating:
+
+    pip install "user-agents-ng[fast]"
+
 ### Migrating from `user-agents`
 
 `user-agents-ng` is a drop-in replacement. Swap the dependency and your code
@@ -81,8 +86,13 @@ Currently these attributes are supported:
 * `is_mobile`: whether user agent is identified as a mobile phone (iPhone, Android phones, Blackberry, Windows Phone devices etc)
 * `is_tablet`: whether user agent is identified as a tablet device (iPad, Kindle Fire, Nexus 7 etc)
 * `is_pc`: whether user agent is identified to be running a traditional "desktop" OS (Windows, OS X, Linux)
+* `is_tv`: whether user agent is identified as a smart TV or streaming device (Samsung Tizen, LG webOS, Android TV, Fire TV, Roku, Chromecast etc)
 * `is_touch_capable`: whether user agent has touch capabilities
-* `is_bot`: whether user agent is a search engine crawler/spider
+* `is_bot`: whether user agent is a crawler, spider or headless browser. Besides the bots ua-parser knows about, this catches any user agent with a bot-style product token (`Applebot/0.1`), a contact URL (`+http://example.com/bot`) or a contact email address
+* `is_email_client`: whether user agent is an email client (Outlook, Thunderbird, Apple Mail etc)
+
+`parse()` also accepts `bytes` and `None`, so a raw header or a request with no
+`User-Agent` header at all won't raise an exception.
 
 For example:
 
@@ -97,7 +107,7 @@ user_agent.is_tablet # returns False
 user_agent.is_touch_capable # returns False
 user_agent.is_pc # returns False
 user_agent.is_bot # returns False
-str(user_agent) # returns "BlackBerry 9700 / BlackBerry OS 5 / BlackBerry 9700"
+str(user_agent) # returns "BlackBerry 9700 / BlackBerry OS 5.0.0 / BlackBerry 9700"
 
 # Now a Samsung Galaxy S3
 ua_string = 'Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30'
@@ -117,7 +127,7 @@ user_agent.is_tablet # returns False
 user_agent.is_touch_capable # returns True
 user_agent.is_pc # returns True
 user_agent.is_bot # returns False
-str(user_agent) # returns "PC / Windows 8 / IE 10"
+str(user_agent) # returns "PC / Windows 8 / IE 10.0"
 ```
 
 ## Running Tests
@@ -126,6 +136,15 @@ str(user_agent) # returns "PC / Windows 8 / IE 10"
 
 ## Changelog
 
+### Version 3.1.0
+
+* Added `is_tv` for smart TVs and streaming devices, which no longer count as touch capable tablets, phones or PCs.
+* `is_bot` now catches bots missed by ua-parser, like Applebot, Amazonbot, Bytespider and headless Chrome.
+* Added the `user-agents-ng[fast]` extra, which parses about 20x faster.
+* `parse()` accepts `None` and `bytes` instead of raising `TypeError`.
+* Firefox on Android tablets is now a tablet, Dalvik clients are no longer tablets, and KaiOS is mobile.
+* Now uses the current `ua-parser` API and requires `ua-parser>=1.0.0`, which keeps detection data up to date.
+
 ### Version 3.0.0
 
 * Released as `user-agents-ng`, a maintained successor to `user-agents`.

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "user-agents-ng"
-version = "3.0.0"
+version = "3.1.0"
 description = "A maintained successor to user-agents: identify devices (phones, tablets) and their capabilities by parsing browser user agent strings."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -18,7 +18,7 @@ maintainers = [
 ]
 keywords = ["user agent", "user-agents", "browser", "device", "detection", "parser", "mobile", "tablet", "bot"]
 dependencies = [
-    "ua-parser>=0.10.0",
+    "ua-parser>=1.0.0",
 ]
 classifiers = [
     "Development Status :: 5 - Production/Stable",
@@ -38,6 +38,11 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 
+[project.optional-dependencies]
+fast = [
+    "ua-parser[regex]>=1.0.0",
+]
+
 [project.urls]
 Homepage = "https://github.com/SAKMZ/python-user-agents"
 Repository = "https://github.com/SAKMZ/python-user-agents"

user_agents/__init__.py

@@ -1,4 +1,4 @@
-VERSION = (3, 0, 0)
+VERSION = (3, 1, 0)
 
 from .parsers import UserAgent, parse
 

user_agents/parsers.py

@@ -1,6 +1,7 @@
-from typing import Any, NamedTuple, Optional
+import re
+from typing import Any, NamedTuple, Optional, Union
 
-from ua_parser import user_agent_parser
+import ua_parser
 
 
 MOBILE_DEVICE_FAMILIES = (
@@ -26,6 +27,7 @@
     'Windows CE',
     'Windows Mobile',
     'Maemo',
+    'KaiOS',
 )
 
 MOBILE_BROWSER_FAMILIES = (
@@ -47,6 +49,7 @@
     'Galaxy Tab',
     'Xoom',
     'Dell Streak',
+    'Generic Tablet',
 )
 
 TABLET_DEVICE_BRANDS = (
@@ -89,6 +92,23 @@
     'YahooMobileMail',
 }
 
+# Catches bots ua-parser doesn't know about yet
+BOT_UA_PATTERN = re.compile(
+    r'\+https?://'  # Contact URL, e.g. (+http://example.com/bot.html)
+    r'|@[\w-]+\.[a-z]{2,}'  # Contact email address
+    r'|(?:bot|crawler|spider)/\d'  # Product token, e.g. Applebot/0.1
+    r'|compatible; [\w.-]*(?:bot|crawler|spider)\b'  # e.g. compatible; Bytespider
+    r'|HeadlessChrome/|PhantomJS/',
+    re.IGNORECASE
+)
+
+TV_UA_PATTERN = re.compile(
+    r'smart[- ]?tv|hbbtv|google ?tv|android ?tv|apple ?tv|web0s|netcast|vidaa'
+    r'|\broku\b|bravia|crkey'
+    r'|(?-i:\bAFT[A-Z0-9]{1,6}\b)',  # Amazon Fire TV models, e.g. AFTKA
+    re.IGNORECASE
+)
+
 
 def verify_attribute(attribute: Any) -> Any:
     if isinstance(attribute, str) and attribute.isdigit():
@@ -163,12 +183,19 @@ def parse_device(family: str, brand: Optional[str], model: Optional[str]) -> Dev
 
 class UserAgent:
 
-    def __init__(self, user_agent_string: str):
-        ua_dict = user_agent_parser.Parse(user_agent_string)
+    def __init__(self, user_agent_string: Union[str, bytes, None]):
+        # Frameworks return None for a missing header, raw headers are bytes
+        if user_agent_string is None:
+            user_agent_string = ''
+        elif isinstance(user_agent_string, bytes):
+            user_agent_string = user_agent_string.decode('latin-1')
+
+        result = ua_parser.parse(user_agent_string).with_defaults()
+        browser, os = result.user_agent, result.os
         self.ua_string = user_agent_string
-        self.os = parse_operating_system(**ua_dict['os'])
-        self.browser = parse_browser(**ua_dict['user_agent'])
-        self.device = parse_device(**ua_dict['device'])
+        self.os = parse_operating_system(os.family, os.major, os.minor, os.patch, os.patch_minor)
+        self.browser = parse_browser(browser.family, browser.major, browser.minor, browser.patch, browser.patch_minor)
+        self.device = parse_device(result.device.family, result.device.brand, result.device.model)
 
     def __str__(self) -> str:
         return "{device} / {os} / {browser}".format(
@@ -182,9 +209,9 @@ def __repr__(self) -> str:
 
     def _is_android_tablet(self) -> bool:
         # Newer Android tablets don't have "Mobile" in their user agent string,
-        # older ones like Galaxy Tab still have "Mobile" though they're not
-        return ('Mobile Safari' not in self.ua_string and
-                self.browser.family != "Firefox Mobile")
+        # older ones like Galaxy Tab still have "Mobile" though they're not.
+        # Dalvik never says either way, so assume the far more common phone.
+        return 'Mobile' not in self.ua_string and 'Dalvik' not in self.ua_string
 
     def _is_blackberry_touch_capable_device(self) -> bool:
         # A helper to determine whether a BB phone has touch capabilities
@@ -206,6 +233,9 @@ def get_browser(self) -> str:
 
     @property
     def is_tablet(self) -> bool:
+        # ua-parser reports Android TVs as Generic_Android_Tablet
+        if self.is_tv:
+            return False
         if self.device.family in TABLET_DEVICE_FAMILIES:
             return True
         if self.device.brand in TABLET_DEVICE_BRANDS:
@@ -225,7 +255,7 @@ def is_mobile(self) -> bool:
         # First check for mobile device and mobile browser families
         if self.device.family in MOBILE_DEVICE_FAMILIES:
             return True
-        if self.is_tablet or self.is_pc:
+        if self.is_tablet or self.is_pc or self.is_tv:
             return False
         if self.browser.family in MOBILE_BROWSER_FAMILIES:
             return True
@@ -254,6 +284,8 @@ def is_mobile(self) -> bool:
     @property
     def is_touch_capable(self) -> bool:
         # TODO: detect touch capable Nokia devices
+        if self.is_tv:
+            return False
         if self.os.family in TOUCH_CAPABLE_OS_FAMILIES:
             return True
         if self.device.family in TOUCH_CAPABLE_DEVICE_FAMILIES:
@@ -271,7 +303,8 @@ def is_touch_capable(self) -> bool:
     def is_pc(self) -> bool:
         if self.device.family in MOBILE_DEVICE_FAMILIES or \
            self.device.family in TABLET_DEVICE_FAMILIES or \
-           self.device.brand in TABLET_DEVICE_BRANDS:
+           self.device.brand in TABLET_DEVICE_BRANDS or \
+           self.is_tv:
             return False
         # Returns True for "PC" devices (Windows, Mac and Linux)
         if 'Windows NT' in self.ua_string or self.os.family in PC_OS_FAMILIES or \
@@ -291,14 +324,20 @@ def is_pc(self) -> bool:
             return True
         return False
 
+    @property
+    def is_tv(self) -> bool:
+        return TV_UA_PATTERN.search(self.ua_string) is not None
+
     @property
     def is_bot(self) -> bool:
-        return self.device.family == 'Spider'
+        if self.device.family == 'Spider':
+            return True
+        return BOT_UA_PATTERN.search(self.ua_string) is not None
 
     @property
     def is_email_client(self) -> bool:
         return self.browser.family in EMAIL_PROGRAM_FAMILIES
 
 
-def parse(user_agent_string: str) -> UserAgent:
+def parse(user_agent_string: Union[str, bytes, None]) -> UserAgent:
     return UserAgent(user_agent_string)

user_agents/tests.py

@@ -245,6 +245,78 @@ def test_unicode_strings(self):
         self.assertEqual(ua_str, "iPhone / iOS 5.1 / Mobile Safari 5.1")
         self.assertIsInstance(ua_str, str)
 
+    def test_missing_user_agent(self):
+        for ua_string in (None, '', b''):
+            ua = parse(ua_string)
+            self.assertEqual(ua.ua_string, '')
+            self.assertEqual(str(ua), 'Other / Other / Other')
+            self.assertFalse(ua.is_bot)
+
+    def test_bytes_user_agent(self):
+        ua = parse(iphone_ua_string.encode('latin-1'))
+        self.assertEqual(ua.ua_string, iphone_ua_string)
+        self.assertTrue(ua.is_mobile)
+
+    def test_is_bot_undetected_by_ua_parser(self):
+        for ua_string in (
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)',
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
+            'Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)',
+            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/140.0.0.0 Safari/537.36',
+            'ExampleFetcher/1.0 (contact: ops@example.com)',
+            'Mozilla/5.0 (compatible; ExampleCrawler; http://example.com)',
+            'SomeBot/2.3',
+        ):
+            self.assertTrue(parse(ua_string).is_bot, msg=ua_string)
+
+    def test_is_bot_ignores_real_browsers(self):
+        for ua_string in (
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36',
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0',
+            'Mozilla/5.0 (Linux; Android 11; CUBOT_X50) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36',
+            'Mozilla/5.0 (Linux; Android 9; CUBOT P20 Build/PPR1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36',
+            iphone_ua_string,
+            'curl/8.7.1',
+            'okhttp/4.12.0',
+        ):
+            self.assertFalse(parse(ua_string).is_bot, msg=ua_string)
+
+    def test_is_tv(self):
+        for ua_string in (
+            'Mozilla/5.0 (SMART-TV; Linux; Tizen 7.0) AppleWebKit/537.36 (KHTML, like Gecko) 94.0.4606.31/7.0 TV Safari/537.36',
+            'Mozilla/5.0 (SMART-TV; X11; Linux armv7l) AppleWebKit/537.42 (KHTML, like Gecko) Safari/537.42',
+            'Mozilla/5.0 (Web0S; Linux/SmartTV) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 WebAppManager',
+            'Mozilla/5.0 (Linux; Android 12; BRAVIA 4K UR3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Mozilla/5.0 (Linux; Android 9; AFTKA) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Mozilla/5.0 (X11; Linux aarch64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 CrKey/1.56.500000',
+            'Roku/DVP-12.5 (12.5.0.4130-AE)',
+        ):
+            ua = parse(ua_string)
+            self.assertTrue(ua.is_tv, msg=ua_string)
+            self.assertFalse(ua.is_tablet, msg=ua_string)
+            self.assertFalse(ua.is_mobile, msg=ua_string)
+            self.assertFalse(ua.is_pc, msg=ua_string)
+            self.assertFalse(ua.is_touch_capable, msg=ua_string)
+
+    def test_is_not_tv(self):
+        for ua in (iphone_ua, ipad_ua, galaxy_s3_ua, nexus_7_ua, ie_ua, mac_safari_ua, ubuntu_firefox_ua):
+            self.assertFalse(ua.is_tv, msg=ua.ua_string)
+
+    def test_android_firefox_tablet(self):
+        ua = parse('Mozilla/5.0 (Android 14; Tablet; rv:140.0) Gecko/140.0 Firefox/140.0')
+        self.assertTrue(ua.is_tablet)
+        self.assertFalse(ua.is_mobile)
+
+    def test_android_dalvik_is_mobile(self):
+        ua = parse('Dalvik/2.1.0 (Linux; U; Android 7.1.2; Nexus 6P Build/N2G47O)')
+        self.assertFalse(ua.is_tablet)
+        self.assertTrue(ua.is_mobile)
+
+    def test_kaios_is_mobile(self):
+        ua = parse('Mozilla/5.0 (Mobile; Nokia 8110 4G; rv:48.0) Gecko/48.0 Firefox/48.0 KAIOS/2.5')
+        self.assertTrue(ua.is_mobile)
+        self.assertFalse(ua.is_tablet)
+
 
 with open(os.path.join(os.path.dirname(__file__), 'devices.json')) as f:
     devices = json.load(f)
@@ -254,6 +326,7 @@ def test_wrapper(items):
     def test_func(self):
         attrs = ('is_bot', 'is_mobile',
                  'is_pc', 'is_tablet', 'is_touch_capable')
+        self.assertFalse(items['user_agent'].is_tv)
         for attr in attrs:
             self.assertEqual(
                 getattr(items['user_agent'], attr), items[attr], msg=attr)