diff --git a/AUTHORS.rst b/AUTHORS.rst index 9a8224dc7da..72a9863d2b1 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -84,6 +84,7 @@ The following organizations or individuals have contributed to ScanCode: - Sebastian Schuberth @sschuberth - Shankhadeep Dey @Iamshankhadeep - Sharikzama @ZamaSharik +- Shrashti @Shrashti1234 - Shrijal Acharya @OctoPie23 - Shivam Chauhan @chashiv - Shivam Sandbhor @sbs2001 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index abc3ae41ad2..84cdceaff7e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,10 @@ Changelog Next release -------------- +- Reject SPDX output for non-empty files without a SHA1 checksum instead + of substituting the empty-file hash. Scan with ``--info`` to collect checksums. + https://github.com/aboutcode-org/scancode-toolkit/issues/3453 + - Fix the optional ``licenses`` extra dependency typo to install ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 diff --git a/docs/source/reference/scancode-cli/cli-output-format-options.rst b/docs/source/reference/scancode-cli/cli-output-format-options.rst index 1f3a4bb852c..3bc96b4bd81 100644 --- a/docs/source/reference/scancode-cli/cli-output-format-options.rst +++ b/docs/source/reference/scancode-cli/cli-output-format-options.rst @@ -330,6 +330,14 @@ Comparing different ``json`` output formats ``--spdx-rdf FILE`` ------------------- + .. note:: + + SPDX RDF and Tag/Value exports require file SHA1 checksums. Include + ``--info`` (``-i``) when scanning to collect them. When exporting an existing + scan with ``--from-json``, that scan must already contain the checksums. + Export fails with an error identifying the file if a non-empty file's + checksum is missing; an unknown checksum is not replaced by the empty-file hash. + `SPDX `_ stands for "Software Package and Data Exchange" and is an open standard for communicating software bill of material information (including components, licenses, copyrights, and security references). diff --git a/src/formattedcode/output_spdx.py b/src/formattedcode/output_spdx.py index 88039c7f2e9..943fc5756cb 100644 --- a/src/formattedcode/output_spdx.py +++ b/src/formattedcode/output_spdx.py @@ -172,7 +172,7 @@ def check_sha1(codebase): click.secho( 'WARNING: Files are missing a SHA1 attribute. ' - 'Incomplete SPDX document created.', + 'Include --info when scanning to collect file checksums.', err=True, fg='red', ) @@ -261,8 +261,13 @@ def write_spdx( if file_data.get('file_type') == 'empty': checksum = Checksum(ChecksumAlgorithm.SHA1, "da39a3ee5e6b4b0d3255bfef95601890afd80709") else: - # FIXME: this sets the checksum of a file to the empty string hash if unknown; tracked in https://github.com/nexB/scancode-toolkit/issues/3453 - checksum = Checksum(ChecksumAlgorithm.SHA1, file_data.get('sha1') or 'da39a3ee5e6b4b0d3255bfef95601890afd80709') + sha1 = file_data.get('sha1') + if not sha1: + raise ValueError( + f'Cannot create SPDX output: missing SHA1 for file {name!r}. ' + 'Run a scan with --info to collect file checksums.' + ) + checksum = Checksum(ChecksumAlgorithm.SHA1, sha1) file_entry = File( spdx_id=f'SPDXRef-{sid}', diff --git a/tests/formattedcode/data/spdx/unicode.json b/tests/formattedcode/data/spdx/unicode.json index b1aa8e67f12..c646ab36007 100644 --- a/tests/formattedcode/data/spdx/unicode.json +++ b/tests/formattedcode/data/spdx/unicode.json @@ -16,7 +16,8 @@ { "path": "han/\u636e.svg", "type": "file", - "scan_errors": [] + "scan_errors": [], + "sha1": "c26d7e350178d93f90d9b22482d3d86f1da67180" } ] } \ No newline at end of file diff --git "a/tests/formattedcode/data/spdx/unicode/han/\346\215\256.svg" "b/tests/formattedcode/data/spdx/unicode/han/\346\215\256.svg" new file mode 100644 index 00000000000..0c7be4e2da4 --- /dev/null +++ "b/tests/formattedcode/data/spdx/unicode/han/\346\215\256.svg" @@ -0,0 +1 @@ + diff --git a/tests/formattedcode/test_output_spdx.py b/tests/formattedcode/test_output_spdx.py index 8aa1fda02f3..9d41451d630 100644 --- a/tests/formattedcode/test_output_spdx.py +++ b/tests/formattedcode/test_output_spdx.py @@ -9,6 +9,7 @@ # import io +import hashlib import os import re @@ -26,6 +27,100 @@ test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data') +def write_spdx_for_checksum_test(file_data, as_tagvalue, monkeypatch): + from formattedcode.output_spdx import write_spdx + from licensedcode import cache + + # These inputs contain no license detections and need no license database. + monkeypatch.setattr(cache, 'get_licenses_db', lambda: {}) + output = io.StringIO() + write_spdx( + codebase=None, + output_file=output, + files=[file_data], + tool_name='ScanCode', + tool_version='test', + notice='test', + package_name='sample', + as_tagvalue=as_tagvalue, + ) + return output.getvalue() + + +@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf']) +@pytest.mark.parametrize('checksum_data', [{}, {'sha1': None}, {'sha1': ''}]) +def test_spdx_missing_sha1_rejected(as_tagvalue, checksum_data, monkeypatch): + file_data = {'type': 'file', 'path': 'sample.txt', 'file_type': 'ASCII text'} + file_data.update(checksum_data) + with pytest.raises(ValueError, match='missing SHA1.*sample.txt'): + write_spdx_for_checksum_test(file_data, as_tagvalue, monkeypatch) + + +@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf']) +def test_spdx_file_checksum_preserves_known_sha1(as_tagvalue, monkeypatch): + checksum = hashlib.sha1(b'non-empty contents').hexdigest() + result = write_spdx_for_checksum_test( + {'type': 'file', 'path': 'sample.txt', 'sha1': checksum}, + as_tagvalue, + monkeypatch, + ) + assert checksum in result + assert hashlib.sha1(b'').hexdigest() not in result + + +@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf']) +def test_spdx_file_checksum_preserves_empty_file_hash(as_tagvalue, monkeypatch): + result = write_spdx_for_checksum_test( + {'type': 'file', 'path': 'empty.txt', 'file_type': 'empty'}, + as_tagvalue, + monkeypatch, + ) + assert hashlib.sha1(b'').hexdigest() in result + + +@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf']) +def test_spdx_file_checksum_does_not_require_directory_hash(as_tagvalue, monkeypatch): + result = write_spdx_for_checksum_test( + {'type': 'directory', 'path': 'directory'}, + as_tagvalue, + monkeypatch, + ) + assert "No results for package 'sample'." in result + + +@pytest.mark.parametrize('output_option', ['--spdx-tv', '--spdx-rdf']) +def test_spdx_missing_sha1_cli_error(output_option, tmp_path, monkeypatch): + from licensedcode import cache + + monkeypatch.setattr(cache, 'get_licenses_db', lambda: {}) + source = tmp_path / 'sample.txt' + source.write_text('non-empty contents', encoding='utf-8') + output = tmp_path / 'output.spdx' + result = run_scan_click( + ['--email', str(source), output_option, str(output)], + expected_rc=1, + retry=False, + ) + assert 'missing SHA1' in result.output + assert 'sample.txt' in result.output + assert '--info' in result.output + if output.exists(): + assert not output.read_text(encoding='utf-8') + + +@pytest.mark.parametrize('output_option', ['--spdx-tv', '--spdx-rdf']) +@pytest.mark.parametrize('content', [b'non-empty contents', b'']) +def test_spdx_file_checksum_cli_with_info(output_option, content, tmp_path, monkeypatch): + from licensedcode import cache + + monkeypatch.setattr(cache, 'get_licenses_db', lambda: {}) + source = tmp_path / 'sample.txt' + source.write_bytes(content) + output = tmp_path / 'output.spdx' + run_scan_click(['--info', str(source), output_option, str(output)]) + assert hashlib.sha1(content).hexdigest() in output.read_text(encoding='utf-8') + + def strip_variable_text(rdf_text): """ Return rdf_text stripped from variable parts such as rdf nodeids @@ -339,7 +434,10 @@ def test_spdx_rdf_with_empty_scan(): @pytest.mark.scanslow -def test_output_spdx_rdf_can_handle_non_ascii_paths(): +def test_output_spdx_rdf_can_handle_non_ascii_paths(monkeypatch): + from licensedcode import cache + + monkeypatch.setattr(cache, 'get_licenses_db', lambda: {}) test_file = test_env.get_test_loc('spdx/unicode.json') result_file = test_env.get_temp_file(extension='spdx', file_name='test_spdx') run_scan_click(['--from-json', test_file, '--spdx-rdf', result_file]) @@ -347,7 +445,10 @@ def test_output_spdx_rdf_can_handle_non_ascii_paths(): results = res.read() assert 'han/据.svg' in results -def test_output_spdx_tv_can_handle_non_ascii_paths(): +def test_output_spdx_tv_can_handle_non_ascii_paths(monkeypatch): + from licensedcode import cache + + monkeypatch.setattr(cache, 'get_licenses_db', lambda: {}) test_file = test_env.get_test_loc('spdx/unicode.json') result_file = test_env.get_temp_file(extension='spdx', file_name='test_spdx') run_scan_click(['--from-json', test_file, '--spdx-tv', result_file])