From c5e84a48b3ef0114802ecd6576f9b53b62b9c386 Mon Sep 17 00:00:00 2001 From: Shrashti1234 Date: Wed, 4 Nov 2026 13:48:04 +0530 Subject: [PATCH] Reject missing file checksums in SPDX exports Stop substituting the empty-file SHA1 for files whose checksum is unknown. Raise a clear error directing users to collect checksums with --info, while preserving known checksums and the empty-file special case. Add writer and CLI regressions for both SPDX formats, document the requirement, and give the Unicode scan fixture the checksum of its included synthetic SVG. The full SPDX suite passes: 38 tests on Windows/Python 3.14.2. Fixes #3453 Signed-off-by: Shrashti1234 --- AUTHORS.rst | 1 + CHANGELOG.rst | 4 + .../cli-output-format-options.rst | 8 ++ src/formattedcode/output_spdx.py | 11 +- tests/formattedcode/data/spdx/unicode.json | 3 +- .../data/spdx/unicode/han/\346\215\256.svg" | 1 + tests/formattedcode/test_output_spdx.py | 105 +++++++++++++++++- 7 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 "tests/formattedcode/data/spdx/unicode/han/\346\215\256.svg" diff --git a/AUTHORS.rst b/AUTHORS.rst index 9a8224dc7da..72a9863d2b1 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -84,6 +84,7 @@ The following organizations or individuals have contributed to ScanCode: - Sebastian Schuberth @sschuberth - Shankhadeep Dey @Iamshankhadeep - Sharikzama @ZamaSharik +- Shrashti @Shrashti1234 - Shrijal Acharya @OctoPie23 - Shivam Chauhan @chashiv - Shivam Sandbhor @sbs2001 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index abc3ae41ad2..84cdceaff7e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,10 @@ Changelog Next release -------------- +- Reject SPDX output for non-empty files without a SHA1 checksum instead + of substituting the empty-file hash. Scan with ``--info`` to collect checksums. + https://github.com/aboutcode-org/scancode-toolkit/issues/3453 + - Fix the optional ``licenses`` extra dependency typo to install ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 diff --git a/docs/source/reference/scancode-cli/cli-output-format-options.rst b/docs/source/reference/scancode-cli/cli-output-format-options.rst index 1f3a4bb852c..3bc96b4bd81 100644 --- a/docs/source/reference/scancode-cli/cli-output-format-options.rst +++ b/docs/source/reference/scancode-cli/cli-output-format-options.rst @@ -330,6 +330,14 @@ Comparing different ``json`` output formats ``--spdx-rdf FILE`` ------------------- + .. note:: + + SPDX RDF and Tag/Value exports require file SHA1 checksums. Include + ``--info`` (``-i``) when scanning to collect them. When exporting an existing + scan with ``--from-json``, that scan must already contain the checksums. + Export fails with an error identifying the file if a non-empty file's + checksum is missing; an unknown checksum is not replaced by the empty-file hash. + `SPDX `_ stands for "Software Package and Data Exchange" and is an open standard for communicating software bill of material information (including components, licenses, copyrights, and security references). diff --git a/src/formattedcode/output_spdx.py b/src/formattedcode/output_spdx.py index 88039c7f2e9..943fc5756cb 100644 --- a/src/formattedcode/output_spdx.py +++ b/src/formattedcode/output_spdx.py @@ -172,7 +172,7 @@ def check_sha1(codebase): click.secho( 'WARNING: Files are missing a SHA1 attribute. ' - 'Incomplete SPDX document created.', + 'Include --info when scanning to collect file checksums.', err=True, fg='red', ) @@ -261,8 +261,13 @@ def write_spdx( if file_data.get('file_type') == 'empty': checksum = Checksum(ChecksumAlgorithm.SHA1, "da39a3ee5e6b4b0d3255bfef95601890afd80709") else: - # FIXME: this sets the checksum of a file to the empty string hash if unknown; tracked in https://github.com/nexB/scancode-toolkit/issues/3453 - checksum = Checksum(ChecksumAlgorithm.SHA1, file_data.get('sha1') or 'da39a3ee5e6b4b0d3255bfef95601890afd80709') + sha1 = file_data.get('sha1') + if not sha1: + raise ValueError( + f'Cannot create SPDX output: missing SHA1 for file {name!r}. ' + 'Run a scan with --info to collect file checksums.' + ) + checksum = Checksum(ChecksumAlgorithm.SHA1, sha1) file_entry = File( spdx_id=f'SPDXRef-{sid}', diff --git a/tests/formattedcode/data/spdx/unicode.json b/tests/formattedcode/data/spdx/unicode.json index b1aa8e67f12..c646ab36007 100644 --- a/tests/formattedcode/data/spdx/unicode.json +++ b/tests/formattedcode/data/spdx/unicode.json @@ -16,7 +16,8 @@ { "path": "han/\u636e.svg", "type": "file", - "scan_errors": [] + "scan_errors": [], + "sha1": "c26d7e350178d93f90d9b22482d3d86f1da67180" } ] } \ No newline at end of file diff --git "a/tests/formattedcode/data/spdx/unicode/han/\346\215\256.svg" "b/tests/formattedcode/data/spdx/unicode/han/\346\215\256.svg" new file mode 100644 index 00000000000..0c7be4e2da4 --- /dev/null +++ "b/tests/formattedcode/data/spdx/unicode/han/\346\215\256.svg" @@ -0,0 +1 @@ + diff --git a/tests/formattedcode/test_output_spdx.py b/tests/formattedcode/test_output_spdx.py index 8aa1fda02f3..9d41451d630 100644 --- a/tests/formattedcode/test_output_spdx.py +++ b/tests/formattedcode/test_output_spdx.py @@ -9,6 +9,7 @@ # import io +import hashlib import os import re @@ -26,6 +27,100 @@ test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data') +def write_spdx_for_checksum_test(file_data, as_tagvalue, monkeypatch): + from formattedcode.output_spdx import write_spdx + from licensedcode import cache + + # These inputs contain no license detections and need no license database. + monkeypatch.setattr(cache, 'get_licenses_db', lambda: {}) + output = io.StringIO() + write_spdx( + codebase=None, + output_file=output, + files=[file_data], + tool_name='ScanCode', + tool_version='test', + notice='test', + package_name='sample', + as_tagvalue=as_tagvalue, + ) + return output.getvalue() + + +@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf']) +@pytest.mark.parametrize('checksum_data', [{}, {'sha1': None}, {'sha1': ''}]) +def test_spdx_missing_sha1_rejected(as_tagvalue, checksum_data, monkeypatch): + file_data = {'type': 'file', 'path': 'sample.txt', 'file_type': 'ASCII text'} + file_data.update(checksum_data) + with pytest.raises(ValueError, match='missing SHA1.*sample.txt'): + write_spdx_for_checksum_test(file_data, as_tagvalue, monkeypatch) + + +@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf']) +def test_spdx_file_checksum_preserves_known_sha1(as_tagvalue, monkeypatch): + checksum = hashlib.sha1(b'non-empty contents').hexdigest() + result = write_spdx_for_checksum_test( + {'type': 'file', 'path': 'sample.txt', 'sha1': checksum}, + as_tagvalue, + monkeypatch, + ) + assert checksum in result + assert hashlib.sha1(b'').hexdigest() not in result + + +@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf']) +def test_spdx_file_checksum_preserves_empty_file_hash(as_tagvalue, monkeypatch): + result = write_spdx_for_checksum_test( + {'type': 'file', 'path': 'empty.txt', 'file_type': 'empty'}, + as_tagvalue, + monkeypatch, + ) + assert hashlib.sha1(b'').hexdigest() in result + + +@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf']) +def test_spdx_file_checksum_does_not_require_directory_hash(as_tagvalue, monkeypatch): + result = write_spdx_for_checksum_test( + {'type': 'directory', 'path': 'directory'}, + as_tagvalue, + monkeypatch, + ) + assert "No results for package 'sample'." in result + + +@pytest.mark.parametrize('output_option', ['--spdx-tv', '--spdx-rdf']) +def test_spdx_missing_sha1_cli_error(output_option, tmp_path, monkeypatch): + from licensedcode import cache + + monkeypatch.setattr(cache, 'get_licenses_db', lambda: {}) + source = tmp_path / 'sample.txt' + source.write_text('non-empty contents', encoding='utf-8') + output = tmp_path / 'output.spdx' + result = run_scan_click( + ['--email', str(source), output_option, str(output)], + expected_rc=1, + retry=False, + ) + assert 'missing SHA1' in result.output + assert 'sample.txt' in result.output + assert '--info' in result.output + if output.exists(): + assert not output.read_text(encoding='utf-8') + + +@pytest.mark.parametrize('output_option', ['--spdx-tv', '--spdx-rdf']) +@pytest.mark.parametrize('content', [b'non-empty contents', b'']) +def test_spdx_file_checksum_cli_with_info(output_option, content, tmp_path, monkeypatch): + from licensedcode import cache + + monkeypatch.setattr(cache, 'get_licenses_db', lambda: {}) + source = tmp_path / 'sample.txt' + source.write_bytes(content) + output = tmp_path / 'output.spdx' + run_scan_click(['--info', str(source), output_option, str(output)]) + assert hashlib.sha1(content).hexdigest() in output.read_text(encoding='utf-8') + + def strip_variable_text(rdf_text): """ Return rdf_text stripped from variable parts such as rdf nodeids @@ -339,7 +434,10 @@ def test_spdx_rdf_with_empty_scan(): @pytest.mark.scanslow -def test_output_spdx_rdf_can_handle_non_ascii_paths(): +def test_output_spdx_rdf_can_handle_non_ascii_paths(monkeypatch): + from licensedcode import cache + + monkeypatch.setattr(cache, 'get_licenses_db', lambda: {}) test_file = test_env.get_test_loc('spdx/unicode.json') result_file = test_env.get_temp_file(extension='spdx', file_name='test_spdx') run_scan_click(['--from-json', test_file, '--spdx-rdf', result_file]) @@ -347,7 +445,10 @@ def test_output_spdx_rdf_can_handle_non_ascii_paths(): results = res.read() assert 'han/据.svg' in results -def test_output_spdx_tv_can_handle_non_ascii_paths(): +def test_output_spdx_tv_can_handle_non_ascii_paths(monkeypatch): + from licensedcode import cache + + monkeypatch.setattr(cache, 'get_licenses_db', lambda: {}) test_file = test_env.get_test_loc('spdx/unicode.json') result_file = test_env.get_temp_file(extension='spdx', file_name='test_spdx') run_scan_click(['--from-json', test_file, '--spdx-tv', result_file])