From df67d00aa5b565aabf400ef0e844b89eb89989f0 Mon Sep 17 00:00:00 2001 From: Shrashti1234 Date: Tue, 3 Nov 2026 00:15:27 +0530 Subject: [PATCH 1/3] Skip regex matching on clue-free lines Add an optional line prefilter to the shared finder. Skip email matching without @ and URL matching without a scheme separator or dot, preserving generic pattern searches and source line numbers. Fixes #595 Signed-off-by: Shrashti1234 --- AUTHORS.rst | 1 + CHANGELOG.rst | 4 +++ src/cluecode/finder.py | 13 +++++++-- tests/cluecode/test_finder.py | 54 +++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 9a8224dc7da..72a9863d2b1 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -84,6 +84,7 @@ The following organizations or individuals have contributed to ScanCode: - Sebastian Schuberth @sschuberth - Shankhadeep Dey @Iamshankhadeep - Sharikzama @ZamaSharik +- Shrashti @Shrashti1234 - Shrijal Acharya @OctoPie23 - Shivam Chauhan @chashiv - Shivam Sandbhor @sbs2001 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index abc3ae41ad2..929732dfea3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,10 @@ Changelog Next release -------------- +- Skip email and URL regular expression matching on lines without the + required markers, while preserving detections and source line numbers. + https://github.com/aboutcode-org/scancode-toolkit/issues/595 + - Fix the optional ``licenses`` extra dependency typo to install ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 diff --git a/src/cluecode/finder.py b/src/cluecode/finder.py index 4364b494551..24cc09fc87a 100644 --- a/src/cluecode/finder.py +++ b/src/cluecode/finder.py @@ -47,12 +47,15 @@ def logger_debug(*args): """ -def find(location, patterns): +def find(location, patterns, prefilter=None): """ Yield match and matched lines for patterns found in file at location as a tuple of (key, found text, text line). `patterns` is a list of tuples (key, compiled regex). + If provided, ``prefilter`` is a callable that takes a text line and returns + False to skip matching patterns against that line. + Note: the location can be a list of lines for testing convenience. """ if TRACE: @@ -61,6 +64,8 @@ def find(location, patterns): logger_debug('find(location=%(loc)r,\n patterns=%(patterns)r)' % locals()) for line_number, line in analysis.numbered_text_lines(location, demarkup=False): + if prefilter is not None and not prefilter(line): + continue for key, pattern in patterns: for match in pattern.findall(line): @@ -130,7 +135,7 @@ def find_emails(location, unique=True): Only return unique items if ``unique`` is True. """ patterns = [('emails', emails_regex(),)] - matches = find(location, patterns) + matches = find(location, patterns, prefilter=lambda line: '@' in line) if TRACE_EMAIL: matches = list(matches) @@ -233,7 +238,9 @@ def find_urls(location, unique=True): `location` can be a list of strings for testing. """ patterns = [('urls', urls_regex(),)] - matches = find(location, patterns) + # Every supported URL contains either a scheme separator or a dot: + # www./ftp. prefixes and git-style URLs ending in .git contain a dot. + matches = find(location, patterns, prefilter=lambda line: '://' in line or '.' in line) if TRACE: matches = list(matches) for m in matches: diff --git a/tests/cluecode/test_finder.py b/tests/cluecode/test_finder.py index 8625bba4fac..d583e1fac03 100644 --- a/tests/cluecode/test_finder.py +++ b/tests/cluecode/test_finder.py @@ -11,6 +11,9 @@ import os import re from unittest.case import expectedFailure +from unittest.mock import Mock +from unittest.mock import call +from unittest.mock import patch from commoncode.testcase import FileBasedTesting @@ -41,6 +44,20 @@ def _find_tester(func, lines_or_location, with_lineno=False, unique=True): class TestEmail(FileBasedTesting): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + def test_find_emails_skips_regex_for_lines_without_at_sign(self): + lines = [ + 'This line has no email address.', + 'Contact alice@python.org', + '', + 'Contact alice@python.org', + ] + pattern = Mock(wraps=finder.emails_regex()) + with patch.object(finder, 'emails_regex', return_value=pattern): + result = list(finder.find_emails(lines, unique=False)) + + assert result == [('alice@python.org', 2), ('alice@python.org', 4)] + assert pattern.findall.call_args_list == [call(lines[1]), call(lines[3])] + def test_emails_regex(self): test_file = self.get_test_loc('finder/email/3w-xxxx.c') test_input = open(test_file).read() @@ -193,6 +210,34 @@ def test_emails_for_ignored_hosts(self): class TestUrl(FileBasedTesting): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + def test_find_urls_prefilter_preserves_supported_url_forms(self): + lines = [ + 'No URL on this line', + '', + 'http://python.org', + 'WWW.python.org', + 'FTP.python.org/pub', + 'git@github.com:aboutcode-org/scancode-toolkit.git', + 'http://8.8.8.8', + 'http://[2001:4860:4860::8888]', + 'http://localhost', + ] + pattern = Mock(wraps=finder.urls_regex()) + expected = [ + ('http://python.org/', 3), + ('http://www.python.org/', 4), + ('http://ftp.python.org/pub', 5), + ('git@github.com:aboutcode-org/scancode-toolkit.git', 6), + ('http://8.8.8.8/', 7), + ] + with patch.object(finder, 'urls_regex', return_value=pattern): + result = list(finder.find_urls(lines, unique=False)) + + assert result == expected + # IP literals and dotless hosts must still reach the regex even when + # the existing downstream URL filters exclude them from the results. + assert pattern.findall.call_args_list == [call(line) for line in lines[2:]] + def test_urls_regex_without_http(self): result = re.match(finder.urls_regex(), u'www.something.domain.tld').group() @@ -761,6 +806,15 @@ def test_find_in_go_does_not_crash_with_unicode_error(self): class TestSearch(FileBasedTesting): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + def test_find_without_prefilter_preserves_arbitrary_patterns(self): + lines = ['Copyright Alice', 'Copyright Bob'] + pattern = re.compile('Copyright') + result = list(find(lines, [('emails', pattern)])) + assert result == [ + ('emails', 'Copyright', lines[0], 1), + ('emails', 'Copyright', lines[1], 2), + ] + def test_search_is_non_unique_by_default(self): test_dir = self.get_test_loc('finder/search', copy=True) pattern = 'Copyright' From 18a9220f2dba218730015c644d485257cda868f2 Mon Sep 17 00:00:00 2001 From: Shrashti1234 Date: Wed, 4 Nov 2026 11:14:51 +0530 Subject: [PATCH 2/3] Record ignorable URLs in FSL reference rules Both reference rules contain the FSL template URL but did not declare it in ignorable_urls. This caused the license_validate_ignorables_2 CI job to fail before the finder optimization as well. Register the detected URL so the metadata agrees with the existing validator. Signed-off-by: Shrashti1234 --- CHANGELOG.rst | 2 ++ src/licensedcode/data/rules/fsl-1.1_1.RULE | 2 ++ src/licensedcode/data/rules/fsl-1.1_2.RULE | 2 ++ 3 files changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 929732dfea3..48d144bce7a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,8 @@ Changelog Next release -------------- +- Add the missing ignorable template URL to two FSL license reference rules. + - Skip email and URL regular expression matching on lines without the required markers, while preserving detections and source line numbers. https://github.com/aboutcode-org/scancode-toolkit/issues/595 diff --git a/src/licensedcode/data/rules/fsl-1.1_1.RULE b/src/licensedcode/data/rules/fsl-1.1_1.RULE index 004881548a3..10481bc4645 100644 --- a/src/licensedcode/data/rules/fsl-1.1_1.RULE +++ b/src/licensedcode/data/rules/fsl-1.1_1.RULE @@ -4,6 +4,8 @@ is_license_reference: yes is_continuous: yes relevance: 100 minimum_coverage: 100 +ignorable_urls: + - https://fsl.software/FSL-1.1-ALv2.template.md notes: Rule based on the manifest file of the liquidbase source jar. --- diff --git a/src/licensedcode/data/rules/fsl-1.1_2.RULE b/src/licensedcode/data/rules/fsl-1.1_2.RULE index 2e4d72ff45c..92f4c87ed26 100644 --- a/src/licensedcode/data/rules/fsl-1.1_2.RULE +++ b/src/licensedcode/data/rules/fsl-1.1_2.RULE @@ -4,6 +4,8 @@ is_license_reference: yes is_continuous: yes relevance: 100 minimum_coverage: 100 +ignorable_urls: + - https://fsl.software/FSL-1.1-ALv2.template.md notes: Rule based on the manifest file of the liquidbase source jar. --- From 6ee343d522611ac0d65df56ea6a16dbc7b66f700 Mon Sep 17 00:00:00 2001 From: Shrashti1234 Date: Wed, 4 Nov 2026 11:19:51 +0530 Subject: [PATCH 3/3] Run parallel tests on the worker main thread Update pytest-xdist to 3.8.0 and execnet to 2.1.1. The older pinned runner can schedule tests outside the interpreter main thread, causing POSIX SIGALRM timeout tests to fail with ValueError. xdist 3.6 introduced the main_thread_only execution model; 3.8 retains that fix and supports all Python versions used here. Validated the finder, scanner integration, API email/URL, and interrupt tests with three workers: 388 passed and two expected failures on Windows/Python 3.14.2. POSIX validation is left to the hosted CI jobs. Signed-off-by: Shrashti1234 --- CHANGELOG.rst | 3 +++ requirements-dev.txt | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 48d144bce7a..eb391db9350 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,9 @@ Next release - Add the missing ignorable template URL to two FSL license reference rules. +- Update the pinned parallel test runner to execute worker tests in the main + thread, as required by the POSIX signal-based timeout tests. + - Skip email and URL regular expression matching on lines without the required markers, while preserving detections and source line numbers. https://github.com/aboutcode-org/scancode-toolkit/issues/595 diff --git a/requirements-dev.txt b/requirements-dev.txt index aec3a678c6d..d7c79acb3bb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ commonmark==0.9.1 docutils==0.19 doc8==2.0.0 et-xmlfile==1.1.0 -execnet==1.9.0 +execnet==2.1.1 flot==0.7.3 iniconfig==1.1.1 isort==5.10.1 @@ -24,7 +24,7 @@ py==1.11.0 pytest==9.0.3 pytest-forked==1.4.0 pytest-rerunfailures==16.1 -pytest-xdist==2.5.0 +pytest-xdist==3.8.0 readme-renderer==35.0 requests==2.32.5 requests-toolbelt==0.9.1