Skip unnecessary regex matching for emails and URLs

AUTHORS.rst

@@ -84,6 +84,7 @@ The following organizations or individuals have contributed to ScanCode:
 - Sebastian Schuberth @sschuberth
 - Shankhadeep Dey @Iamshankhadeep
 - Sharikzama @ZamaSharik
+- Shrashti @Shrashti1234
 - Shrijal Acharya @OctoPie23
 - Shivam Chauhan @chashiv
 - Shivam Sandbhor @sbs2001

CHANGELOG.rst

@@ -4,6 +4,15 @@ Changelog
 Next release
 --------------
 
+- Add the missing ignorable template URL to two FSL license reference rules.
+
+- Update the pinned parallel test runner to execute worker tests in the main
+  thread, as required by the POSIX signal-based timeout tests.
+
+- Skip email and URL regular expression matching on lines without the
+  required markers, while preserving detections and source line numbers.
+  https://github.com/aboutcode-org/scancode-toolkit/issues/595
+
 - Fix the optional ``licenses`` extra dependency typo to install
   ``licensedcode-data``.
   https://github.com/aboutcode-org/scancode-toolkit/pull/5056

requirements-dev.txt

@@ -8,7 +8,7 @@ commonmark==0.9.1
 docutils==0.19
 doc8==2.0.0
 et-xmlfile==1.1.0
-execnet==1.9.0
+execnet==2.1.1
 flot==0.7.3
 iniconfig==1.1.1
 isort==5.10.1
@@ -24,7 +24,7 @@ py==1.11.0
 pytest==9.0.3
 pytest-forked==1.4.0
 pytest-rerunfailures==16.1
-pytest-xdist==2.5.0
+pytest-xdist==3.8.0
 readme-renderer==35.0
 requests==2.32.5
 requests-toolbelt==0.9.1

src/cluecode/finder.py

@@ -47,12 +47,15 @@ def logger_debug(*args):
 """
 
 
-def find(location, patterns):
+def find(location, patterns, prefilter=None):
     """
     Yield match and matched lines for patterns found in file at location as a
     tuple of (key, found text, text line). `patterns` is a list of tuples (key,
     compiled regex).
 
+    If provided, ``prefilter`` is a callable that takes a text line and returns
+    False to skip matching patterns against that line.
+
     Note: the location can be a list of lines for testing convenience.
     """
     if TRACE:
@@ -61,6 +64,8 @@ def find(location, patterns):
         logger_debug('find(location=%(loc)r,\n  patterns=%(patterns)r)' % locals())
 
     for line_number, line in analysis.numbered_text_lines(location, demarkup=False):
+        if prefilter is not None and not prefilter(line):
+            continue
         for key, pattern in patterns:
             for match in pattern.findall(line):
 
@@ -130,7 +135,7 @@ def find_emails(location, unique=True):
     Only return unique items if ``unique`` is True.
     """
     patterns = [('emails', emails_regex(),)]
-    matches = find(location, patterns)
+    matches = find(location, patterns, prefilter=lambda line: '@' in line)
 
     if TRACE_EMAIL:
         matches = list(matches)
@@ -233,7 +238,9 @@ def find_urls(location, unique=True):
     `location` can be a list of strings for testing.
     """
     patterns = [('urls', urls_regex(),)]
-    matches = find(location, patterns)
+    # Every supported URL contains either a scheme separator or a dot:
+    # www./ftp. prefixes and git-style URLs ending in .git contain a dot.
+    matches = find(location, patterns, prefilter=lambda line: '://' in line or '.' in line)
     if TRACE:
         matches = list(matches)
         for m in matches:

src/licensedcode/data/rules/fsl-1.1_1.RULE

@@ -4,6 +4,8 @@ is_license_reference: yes
 is_continuous: yes
 relevance: 100
 minimum_coverage: 100
+ignorable_urls:
+    - https://fsl.software/FSL-1.1-ALv2.template.md
 notes: Rule based on the manifest file of the liquidbase source jar. 
 ---
 

src/licensedcode/data/rules/fsl-1.1_2.RULE

@@ -4,6 +4,8 @@ is_license_reference: yes
 is_continuous: yes
 relevance: 100
 minimum_coverage: 100
+ignorable_urls:
+    - https://fsl.software/FSL-1.1-ALv2.template.md
 notes: Rule based on the manifest file of the liquidbase source jar. 
 ---
 

tests/cluecode/test_finder.py

@@ -11,6 +11,9 @@
 import os
 import re
 from unittest.case import expectedFailure
+from unittest.mock import Mock
+from unittest.mock import call
+from unittest.mock import patch
 
 from commoncode.testcase import FileBasedTesting
 
@@ -41,6 +44,20 @@ def _find_tester(func, lines_or_location, with_lineno=False, unique=True):
 class TestEmail(FileBasedTesting):
     test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
 
+    def test_find_emails_skips_regex_for_lines_without_at_sign(self):
+        lines = [
+            'This line has no email address.',
+            'Contact alice@python.org',
+            '',
+            'Contact alice@python.org',
+        ]
+        pattern = Mock(wraps=finder.emails_regex())
+        with patch.object(finder, 'emails_regex', return_value=pattern):
+            result = list(finder.find_emails(lines, unique=False))
+
+        assert result == [('alice@python.org', 2), ('alice@python.org', 4)]
+        assert pattern.findall.call_args_list == [call(lines[1]), call(lines[3])]
+
     def test_emails_regex(self):
         test_file = self.get_test_loc('finder/email/3w-xxxx.c')
         test_input = open(test_file).read()
@@ -193,6 +210,34 @@ def test_emails_for_ignored_hosts(self):
 class TestUrl(FileBasedTesting):
     test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
 
+    def test_find_urls_prefilter_preserves_supported_url_forms(self):
+        lines = [
+            'No URL on this line',
+            '',
+            'http://python.org',
+            'WWW.python.org',
+            'FTP.python.org/pub',
+            'git@github.com:aboutcode-org/scancode-toolkit.git',
+            'http://8.8.8.8',
+            'http://[2001:4860:4860::8888]',
+            'http://localhost',
+        ]
+        pattern = Mock(wraps=finder.urls_regex())
+        expected = [
+            ('http://python.org/', 3),
+            ('http://www.python.org/', 4),
+            ('http://ftp.python.org/pub', 5),
+            ('git@github.com:aboutcode-org/scancode-toolkit.git', 6),
+            ('http://8.8.8.8/', 7),
+        ]
+        with patch.object(finder, 'urls_regex', return_value=pattern):
+            result = list(finder.find_urls(lines, unique=False))
+
+        assert result == expected
+        # IP literals and dotless hosts must still reach the regex even when
+        # the existing downstream URL filters exclude them from the results.
+        assert pattern.findall.call_args_list == [call(line) for line in lines[2:]]
+
     def test_urls_regex_without_http(self):
         result = re.match(finder.urls_regex(),
                         u'www.something.domain.tld').group()
@@ -761,6 +806,15 @@ def test_find_in_go_does_not_crash_with_unicode_error(self):
 class TestSearch(FileBasedTesting):
     test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
 
+    def test_find_without_prefilter_preserves_arbitrary_patterns(self):
+        lines = ['Copyright Alice', 'Copyright Bob']
+        pattern = re.compile('Copyright')
+        result = list(find(lines, [('emails', pattern)]))
+        assert result == [
+            ('emails', 'Copyright', lines[0], 1),
+            ('emails', 'Copyright', lines[1], 2),
+        ]
+
     def test_search_is_non_unique_by_default(self):
         test_dir = self.get_test_loc('finder/search', copy=True)
         pattern = 'Copyright'