Match URL paths containing an equal sign (#82)

Author: AdrianAtZyte

src/protego/_utils.py

@@ -58,7 +58,7 @@ def _quote_path(path: str) -> str:
     """Return percent encoded path."""
     parts = urlparse(path)
     path = _unquote(parts.path, ignore="/%")
-    path = quote(path, safe="/%")
+    path = quote(path, safe="/%=")
 
     parts = ParseResult("", "", path, parts.params, parts.query, parts.fragment)
     path = urlunparse(parts)

tests/test_protego.py

@@ -1357,6 +1357,20 @@ def test_disallow_query_wildcard(self):
         assert not rp.can_fetch("https://www.site.local/?s=asd", "*")
 
 
+@pytest.mark.parametrize(
+    ("url", "allowed"),
+    [
+        ("https://example.com/1/filter/page=5/", True),
+        ("https://example.com/1/filter/page%3D5/", True),
+        ("https://example.com/1/filter/page=5/x", False),
+    ],
+)
+def test_equal_sign_in_path(url, allowed):
+    content = "User-agent: *\nAllow: /*/filter/page=*/$\nDisallow: /\n"
+    rp = Protego.parse(content)
+    assert rp.can_fetch(url, "*") == allowed
+
+
 @pytest.mark.parametrize(
     ("allow", "disallow", "url", "allowed"),
     [