diff --git a/AUTHORS.rst b/AUTHORS.rst index 9a8224dc7da..72a9863d2b1 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -84,6 +84,7 @@ The following organizations or individuals have contributed to ScanCode: - Sebastian Schuberth @sschuberth - Shankhadeep Dey @Iamshankhadeep - Sharikzama @ZamaSharik +- Shrashti @Shrashti1234 - Shrijal Acharya @OctoPie23 - Shivam Chauhan @chashiv - Shivam Sandbhor @sbs2001 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index abc3ae41ad2..eb391db9350 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,15 @@ Changelog Next release -------------- +- Add the missing ignorable template URL to two FSL license reference rules. + +- Update the pinned parallel test runner to execute worker tests in the main + thread, as required by the POSIX signal-based timeout tests. + +- Skip email and URL regular expression matching on lines without the + required markers, while preserving detections and source line numbers. + https://github.com/aboutcode-org/scancode-toolkit/issues/595 + - Fix the optional ``licenses`` extra dependency typo to install ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 diff --git a/requirements-dev.txt b/requirements-dev.txt index aec3a678c6d..d7c79acb3bb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ commonmark==0.9.1 docutils==0.19 doc8==2.0.0 et-xmlfile==1.1.0 -execnet==1.9.0 +execnet==2.1.1 flot==0.7.3 iniconfig==1.1.1 isort==5.10.1 @@ -24,7 +24,7 @@ py==1.11.0 pytest==9.0.3 pytest-forked==1.4.0 pytest-rerunfailures==16.1 -pytest-xdist==2.5.0 +pytest-xdist==3.8.0 readme-renderer==35.0 requests==2.32.5 requests-toolbelt==0.9.1 diff --git a/src/cluecode/finder.py b/src/cluecode/finder.py index 4364b494551..24cc09fc87a 100644 --- a/src/cluecode/finder.py +++ b/src/cluecode/finder.py @@ -47,12 +47,15 @@ def logger_debug(*args): """ -def find(location, patterns): +def find(location, patterns, prefilter=None): """ Yield match and matched lines for patterns found in file at location as a tuple of (key, found text, text line). `patterns` is a list of tuples (key, compiled regex). + If provided, ``prefilter`` is a callable that takes a text line and returns + False to skip matching patterns against that line. + Note: the location can be a list of lines for testing convenience. """ if TRACE: @@ -61,6 +64,8 @@ def find(location, patterns): logger_debug('find(location=%(loc)r,\n patterns=%(patterns)r)' % locals()) for line_number, line in analysis.numbered_text_lines(location, demarkup=False): + if prefilter is not None and not prefilter(line): + continue for key, pattern in patterns: for match in pattern.findall(line): @@ -130,7 +135,7 @@ def find_emails(location, unique=True): Only return unique items if ``unique`` is True. """ patterns = [('emails', emails_regex(),)] - matches = find(location, patterns) + matches = find(location, patterns, prefilter=lambda line: '@' in line) if TRACE_EMAIL: matches = list(matches) @@ -233,7 +238,9 @@ def find_urls(location, unique=True): `location` can be a list of strings for testing. """ patterns = [('urls', urls_regex(),)] - matches = find(location, patterns) + # Every supported URL contains either a scheme separator or a dot: + # www./ftp. prefixes and git-style URLs ending in .git contain a dot. + matches = find(location, patterns, prefilter=lambda line: '://' in line or '.' in line) if TRACE: matches = list(matches) for m in matches: diff --git a/src/licensedcode/data/rules/fsl-1.1_1.RULE b/src/licensedcode/data/rules/fsl-1.1_1.RULE index 004881548a3..10481bc4645 100644 --- a/src/licensedcode/data/rules/fsl-1.1_1.RULE +++ b/src/licensedcode/data/rules/fsl-1.1_1.RULE @@ -4,6 +4,8 @@ is_license_reference: yes is_continuous: yes relevance: 100 minimum_coverage: 100 +ignorable_urls: + - https://fsl.software/FSL-1.1-ALv2.template.md notes: Rule based on the manifest file of the liquidbase source jar. --- diff --git a/src/licensedcode/data/rules/fsl-1.1_2.RULE b/src/licensedcode/data/rules/fsl-1.1_2.RULE index 2e4d72ff45c..92f4c87ed26 100644 --- a/src/licensedcode/data/rules/fsl-1.1_2.RULE +++ b/src/licensedcode/data/rules/fsl-1.1_2.RULE @@ -4,6 +4,8 @@ is_license_reference: yes is_continuous: yes relevance: 100 minimum_coverage: 100 +ignorable_urls: + - https://fsl.software/FSL-1.1-ALv2.template.md notes: Rule based on the manifest file of the liquidbase source jar. --- diff --git a/tests/cluecode/test_finder.py b/tests/cluecode/test_finder.py index 8625bba4fac..d583e1fac03 100644 --- a/tests/cluecode/test_finder.py +++ b/tests/cluecode/test_finder.py @@ -11,6 +11,9 @@ import os import re from unittest.case import expectedFailure +from unittest.mock import Mock +from unittest.mock import call +from unittest.mock import patch from commoncode.testcase import FileBasedTesting @@ -41,6 +44,20 @@ def _find_tester(func, lines_or_location, with_lineno=False, unique=True): class TestEmail(FileBasedTesting): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + def test_find_emails_skips_regex_for_lines_without_at_sign(self): + lines = [ + 'This line has no email address.', + 'Contact alice@python.org', + '', + 'Contact alice@python.org', + ] + pattern = Mock(wraps=finder.emails_regex()) + with patch.object(finder, 'emails_regex', return_value=pattern): + result = list(finder.find_emails(lines, unique=False)) + + assert result == [('alice@python.org', 2), ('alice@python.org', 4)] + assert pattern.findall.call_args_list == [call(lines[1]), call(lines[3])] + def test_emails_regex(self): test_file = self.get_test_loc('finder/email/3w-xxxx.c') test_input = open(test_file).read() @@ -193,6 +210,34 @@ def test_emails_for_ignored_hosts(self): class TestUrl(FileBasedTesting): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + def test_find_urls_prefilter_preserves_supported_url_forms(self): + lines = [ + 'No URL on this line', + '', + 'http://python.org', + 'WWW.python.org', + 'FTP.python.org/pub', + 'git@github.com:aboutcode-org/scancode-toolkit.git', + 'http://8.8.8.8', + 'http://[2001:4860:4860::8888]', + 'http://localhost', + ] + pattern = Mock(wraps=finder.urls_regex()) + expected = [ + ('http://python.org/', 3), + ('http://www.python.org/', 4), + ('http://ftp.python.org/pub', 5), + ('git@github.com:aboutcode-org/scancode-toolkit.git', 6), + ('http://8.8.8.8/', 7), + ] + with patch.object(finder, 'urls_regex', return_value=pattern): + result = list(finder.find_urls(lines, unique=False)) + + assert result == expected + # IP literals and dotless hosts must still reach the regex even when + # the existing downstream URL filters exclude them from the results. + assert pattern.findall.call_args_list == [call(line) for line in lines[2:]] + def test_urls_regex_without_http(self): result = re.match(finder.urls_regex(), u'www.something.domain.tld').group() @@ -761,6 +806,15 @@ def test_find_in_go_does_not_crash_with_unicode_error(self): class TestSearch(FileBasedTesting): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + def test_find_without_prefilter_preserves_arbitrary_patterns(self): + lines = ['Copyright Alice', 'Copyright Bob'] + pattern = re.compile('Copyright') + result = list(find(lines, [('emails', pattern)])) + assert result == [ + ('emails', 'Copyright', lines[0], 1), + ('emails', 'Copyright', lines[1], 2), + ] + def test_search_is_non_unique_by_default(self): test_dir = self.get_test_loc('finder/search', copy=True) pattern = 'Copyright'