diff --git a/AUTHORS.rst b/AUTHORS.rst
index 9a8224dc7da..72a9863d2b1 100644
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@ -84,6 +84,7 @@ The following organizations or individuals have contributed to ScanCode:
- Sebastian Schuberth @sschuberth
- Shankhadeep Dey @Iamshankhadeep
- Sharikzama @ZamaSharik
+- Shrashti @Shrashti1234
- Shrijal Acharya @OctoPie23
- Shivam Chauhan @chashiv
- Shivam Sandbhor @sbs2001
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index abc3ae41ad2..84cdceaff7e 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -4,6 +4,10 @@ Changelog
Next release
--------------
+- Reject SPDX output for non-empty files without a SHA1 checksum instead
+ of substituting the empty-file hash. Scan with ``--info`` to collect checksums.
+ https://github.com/aboutcode-org/scancode-toolkit/issues/3453
+
- Fix the optional ``licenses`` extra dependency typo to install
``licensedcode-data``.
https://github.com/aboutcode-org/scancode-toolkit/pull/5056
diff --git a/docs/source/reference/scancode-cli/cli-output-format-options.rst b/docs/source/reference/scancode-cli/cli-output-format-options.rst
index 1f3a4bb852c..3bc96b4bd81 100644
--- a/docs/source/reference/scancode-cli/cli-output-format-options.rst
+++ b/docs/source/reference/scancode-cli/cli-output-format-options.rst
@@ -330,6 +330,14 @@ Comparing different ``json`` output formats
``--spdx-rdf FILE``
-------------------
+ .. note::
+
+ SPDX RDF and Tag/Value exports require file SHA1 checksums. Include
+ ``--info`` (``-i``) when scanning to collect them. When exporting an existing
+ scan with ``--from-json``, that scan must already contain the checksums.
+ Export fails with an error identifying the file if a non-empty file's
+ checksum is missing; an unknown checksum is not replaced by the empty-file hash.
+
`SPDX `_ stands for "Software Package and Data Exchange" and is an open standard
for communicating software bill of material information (including components, licenses,
copyrights, and security references).
diff --git a/src/formattedcode/output_spdx.py b/src/formattedcode/output_spdx.py
index 88039c7f2e9..943fc5756cb 100644
--- a/src/formattedcode/output_spdx.py
+++ b/src/formattedcode/output_spdx.py
@@ -172,7 +172,7 @@ def check_sha1(codebase):
click.secho(
'WARNING: Files are missing a SHA1 attribute. '
- 'Incomplete SPDX document created.',
+ 'Include --info when scanning to collect file checksums.',
err=True,
fg='red',
)
@@ -261,8 +261,13 @@ def write_spdx(
if file_data.get('file_type') == 'empty':
checksum = Checksum(ChecksumAlgorithm.SHA1, "da39a3ee5e6b4b0d3255bfef95601890afd80709")
else:
- # FIXME: this sets the checksum of a file to the empty string hash if unknown; tracked in https://github.com/nexB/scancode-toolkit/issues/3453
- checksum = Checksum(ChecksumAlgorithm.SHA1, file_data.get('sha1') or 'da39a3ee5e6b4b0d3255bfef95601890afd80709')
+ sha1 = file_data.get('sha1')
+ if not sha1:
+ raise ValueError(
+ f'Cannot create SPDX output: missing SHA1 for file {name!r}. '
+ 'Run a scan with --info to collect file checksums.'
+ )
+ checksum = Checksum(ChecksumAlgorithm.SHA1, sha1)
file_entry = File(
spdx_id=f'SPDXRef-{sid}',
diff --git a/tests/formattedcode/data/spdx/unicode.json b/tests/formattedcode/data/spdx/unicode.json
index b1aa8e67f12..c646ab36007 100644
--- a/tests/formattedcode/data/spdx/unicode.json
+++ b/tests/formattedcode/data/spdx/unicode.json
@@ -16,7 +16,8 @@
{
"path": "han/\u636e.svg",
"type": "file",
- "scan_errors": []
+ "scan_errors": [],
+ "sha1": "c26d7e350178d93f90d9b22482d3d86f1da67180"
}
]
}
\ No newline at end of file
diff --git "a/tests/formattedcode/data/spdx/unicode/han/\346\215\256.svg" "b/tests/formattedcode/data/spdx/unicode/han/\346\215\256.svg"
new file mode 100644
index 00000000000..0c7be4e2da4
--- /dev/null
+++ "b/tests/formattedcode/data/spdx/unicode/han/\346\215\256.svg"
@@ -0,0 +1 @@
+
diff --git a/tests/formattedcode/test_output_spdx.py b/tests/formattedcode/test_output_spdx.py
index 8aa1fda02f3..9d41451d630 100644
--- a/tests/formattedcode/test_output_spdx.py
+++ b/tests/formattedcode/test_output_spdx.py
@@ -9,6 +9,7 @@
#
import io
+import hashlib
import os
import re
@@ -26,6 +27,100 @@
test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
+def write_spdx_for_checksum_test(file_data, as_tagvalue, monkeypatch):
+ from formattedcode.output_spdx import write_spdx
+ from licensedcode import cache
+
+ # These inputs contain no license detections and need no license database.
+ monkeypatch.setattr(cache, 'get_licenses_db', lambda: {})
+ output = io.StringIO()
+ write_spdx(
+ codebase=None,
+ output_file=output,
+ files=[file_data],
+ tool_name='ScanCode',
+ tool_version='test',
+ notice='test',
+ package_name='sample',
+ as_tagvalue=as_tagvalue,
+ )
+ return output.getvalue()
+
+
+@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf'])
+@pytest.mark.parametrize('checksum_data', [{}, {'sha1': None}, {'sha1': ''}])
+def test_spdx_missing_sha1_rejected(as_tagvalue, checksum_data, monkeypatch):
+ file_data = {'type': 'file', 'path': 'sample.txt', 'file_type': 'ASCII text'}
+ file_data.update(checksum_data)
+ with pytest.raises(ValueError, match='missing SHA1.*sample.txt'):
+ write_spdx_for_checksum_test(file_data, as_tagvalue, monkeypatch)
+
+
+@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf'])
+def test_spdx_file_checksum_preserves_known_sha1(as_tagvalue, monkeypatch):
+ checksum = hashlib.sha1(b'non-empty contents').hexdigest()
+ result = write_spdx_for_checksum_test(
+ {'type': 'file', 'path': 'sample.txt', 'sha1': checksum},
+ as_tagvalue,
+ monkeypatch,
+ )
+ assert checksum in result
+ assert hashlib.sha1(b'').hexdigest() not in result
+
+
+@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf'])
+def test_spdx_file_checksum_preserves_empty_file_hash(as_tagvalue, monkeypatch):
+ result = write_spdx_for_checksum_test(
+ {'type': 'file', 'path': 'empty.txt', 'file_type': 'empty'},
+ as_tagvalue,
+ monkeypatch,
+ )
+ assert hashlib.sha1(b'').hexdigest() in result
+
+
+@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf'])
+def test_spdx_file_checksum_does_not_require_directory_hash(as_tagvalue, monkeypatch):
+ result = write_spdx_for_checksum_test(
+ {'type': 'directory', 'path': 'directory'},
+ as_tagvalue,
+ monkeypatch,
+ )
+ assert "No results for package 'sample'." in result
+
+
+@pytest.mark.parametrize('output_option', ['--spdx-tv', '--spdx-rdf'])
+def test_spdx_missing_sha1_cli_error(output_option, tmp_path, monkeypatch):
+ from licensedcode import cache
+
+ monkeypatch.setattr(cache, 'get_licenses_db', lambda: {})
+ source = tmp_path / 'sample.txt'
+ source.write_text('non-empty contents', encoding='utf-8')
+ output = tmp_path / 'output.spdx'
+ result = run_scan_click(
+ ['--email', str(source), output_option, str(output)],
+ expected_rc=1,
+ retry=False,
+ )
+ assert 'missing SHA1' in result.output
+ assert 'sample.txt' in result.output
+ assert '--info' in result.output
+ if output.exists():
+ assert not output.read_text(encoding='utf-8')
+
+
+@pytest.mark.parametrize('output_option', ['--spdx-tv', '--spdx-rdf'])
+@pytest.mark.parametrize('content', [b'non-empty contents', b''])
+def test_spdx_file_checksum_cli_with_info(output_option, content, tmp_path, monkeypatch):
+ from licensedcode import cache
+
+ monkeypatch.setattr(cache, 'get_licenses_db', lambda: {})
+ source = tmp_path / 'sample.txt'
+ source.write_bytes(content)
+ output = tmp_path / 'output.spdx'
+ run_scan_click(['--info', str(source), output_option, str(output)])
+ assert hashlib.sha1(content).hexdigest() in output.read_text(encoding='utf-8')
+
+
def strip_variable_text(rdf_text):
"""
Return rdf_text stripped from variable parts such as rdf nodeids
@@ -339,7 +434,10 @@ def test_spdx_rdf_with_empty_scan():
@pytest.mark.scanslow
-def test_output_spdx_rdf_can_handle_non_ascii_paths():
+def test_output_spdx_rdf_can_handle_non_ascii_paths(monkeypatch):
+ from licensedcode import cache
+
+ monkeypatch.setattr(cache, 'get_licenses_db', lambda: {})
test_file = test_env.get_test_loc('spdx/unicode.json')
result_file = test_env.get_temp_file(extension='spdx', file_name='test_spdx')
run_scan_click(['--from-json', test_file, '--spdx-rdf', result_file])
@@ -347,7 +445,10 @@ def test_output_spdx_rdf_can_handle_non_ascii_paths():
results = res.read()
assert 'han/据.svg' in results
-def test_output_spdx_tv_can_handle_non_ascii_paths():
+def test_output_spdx_tv_can_handle_non_ascii_paths(monkeypatch):
+ from licensedcode import cache
+
+ monkeypatch.setattr(cache, 'get_licenses_db', lambda: {})
test_file = test_env.get_test_loc('spdx/unicode.json')
result_file = test_env.get_temp_file(extension='spdx', file_name='test_spdx')
run_scan_click(['--from-json', test_file, '--spdx-tv', result_file])