Reject SPDX exports when file checksums are missing

AUTHORS.rst

@@ -84,6 +84,7 @@ The following organizations or individuals have contributed to ScanCode:
 - Sebastian Schuberth @sschuberth
 - Shankhadeep Dey @Iamshankhadeep
 - Sharikzama @ZamaSharik
+- Shrashti @Shrashti1234
 - Shrijal Acharya @OctoPie23
 - Shivam Chauhan @chashiv
 - Shivam Sandbhor @sbs2001

CHANGELOG.rst

@@ -4,6 +4,10 @@ Changelog
 Next release
 --------------
 
+- Reject SPDX output for non-empty files without a SHA1 checksum instead
+  of substituting the empty-file hash. Scan with ``--info`` to collect checksums.
+  https://github.com/aboutcode-org/scancode-toolkit/issues/3453
+
 - Fix the optional ``licenses`` extra dependency typo to install
   ``licensedcode-data``.
   https://github.com/aboutcode-org/scancode-toolkit/pull/5056

docs/source/reference/scancode-cli/cli-output-format-options.rst

@@ -330,6 +330,14 @@ Comparing different ``json`` output formats
 ``--spdx-rdf FILE``
 -------------------
 
+    .. note::
+
+        SPDX RDF and Tag/Value exports require file SHA1 checksums. Include
+        ``--info`` (``-i``) when scanning to collect them. When exporting an existing
+        scan with ``--from-json``, that scan must already contain the checksums.
+        Export fails with an error identifying the file if a non-empty file's
+        checksum is missing; an unknown checksum is not replaced by the empty-file hash.
+
     `SPDX <https://spdx.org/>`_ stands for "Software Package and Data Exchange" and is an open standard
     for communicating software bill of material information (including components, licenses,
     copyrights, and security references).

src/formattedcode/output_spdx.py

@@ -172,7 +172,7 @@ def check_sha1(codebase):
 
         click.secho(
             'WARNING: Files are missing a SHA1 attribute. '
-            'Incomplete SPDX document created.',
+            'Include --info when scanning to collect file checksums.',
             err=True,
             fg='red',
         )
@@ -261,8 +261,13 @@ def write_spdx(
         if file_data.get('file_type') == 'empty':
             checksum = Checksum(ChecksumAlgorithm.SHA1, "da39a3ee5e6b4b0d3255bfef95601890afd80709")
         else:
-            # FIXME: this sets the checksum of a file to the empty string hash if unknown; tracked in https://github.com/nexB/scancode-toolkit/issues/3453
-            checksum = Checksum(ChecksumAlgorithm.SHA1, file_data.get('sha1') or 'da39a3ee5e6b4b0d3255bfef95601890afd80709')
+            sha1 = file_data.get('sha1')
+            if not sha1:
+                raise ValueError(
+                    f'Cannot create SPDX output: missing SHA1 for file {name!r}. '
+                    'Run a scan with --info to collect file checksums.'
+                )
+            checksum = Checksum(ChecksumAlgorithm.SHA1, sha1)
 
         file_entry = File(
             spdx_id=f'SPDXRef-{sid}',

tests/formattedcode/data/spdx/unicode.json

@@ -16,7 +16,8 @@
     {
       "path": "han/\u636e.svg",
       "type": "file",
-      "scan_errors": []
+      "scan_errors": [],
+      "sha1": "c26d7e350178d93f90d9b22482d3d86f1da67180"
     }
   ]
 }

tests/formattedcode/test_output_spdx.py

@@ -9,6 +9,7 @@
 #
 
 import io
+import hashlib
 import os
 import re
 
@@ -26,6 +27,100 @@
 test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
 
 
+def write_spdx_for_checksum_test(file_data, as_tagvalue, monkeypatch):
+    from formattedcode.output_spdx import write_spdx
+    from licensedcode import cache
+
+    # These inputs contain no license detections and need no license database.
+    monkeypatch.setattr(cache, 'get_licenses_db', lambda: {})
+    output = io.StringIO()
+    write_spdx(
+        codebase=None,
+        output_file=output,
+        files=[file_data],
+        tool_name='ScanCode',
+        tool_version='test',
+        notice='test',
+        package_name='sample',
+        as_tagvalue=as_tagvalue,
+    )
+    return output.getvalue()
+
+
+@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf'])
+@pytest.mark.parametrize('checksum_data', [{}, {'sha1': None}, {'sha1': ''}])
+def test_spdx_missing_sha1_rejected(as_tagvalue, checksum_data, monkeypatch):
+    file_data = {'type': 'file', 'path': 'sample.txt', 'file_type': 'ASCII text'}
+    file_data.update(checksum_data)
+    with pytest.raises(ValueError, match='missing SHA1.*sample.txt'):
+        write_spdx_for_checksum_test(file_data, as_tagvalue, monkeypatch)
+
+
+@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf'])
+def test_spdx_file_checksum_preserves_known_sha1(as_tagvalue, monkeypatch):
+    checksum = hashlib.sha1(b'non-empty contents').hexdigest()
+    result = write_spdx_for_checksum_test(
+        {'type': 'file', 'path': 'sample.txt', 'sha1': checksum},
+        as_tagvalue,
+        monkeypatch,
+    )
+    assert checksum in result
+    assert hashlib.sha1(b'').hexdigest() not in result
+
+
+@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf'])
+def test_spdx_file_checksum_preserves_empty_file_hash(as_tagvalue, monkeypatch):
+    result = write_spdx_for_checksum_test(
+        {'type': 'file', 'path': 'empty.txt', 'file_type': 'empty'},
+        as_tagvalue,
+        monkeypatch,
+    )
+    assert hashlib.sha1(b'').hexdigest() in result
+
+
+@pytest.mark.parametrize('as_tagvalue', [True, False], ids=['tagvalue', 'rdf'])
+def test_spdx_file_checksum_does_not_require_directory_hash(as_tagvalue, monkeypatch):
+    result = write_spdx_for_checksum_test(
+        {'type': 'directory', 'path': 'directory'},
+        as_tagvalue,
+        monkeypatch,
+    )
+    assert "No results for package 'sample'." in result
+
+
+@pytest.mark.parametrize('output_option', ['--spdx-tv', '--spdx-rdf'])
+def test_spdx_missing_sha1_cli_error(output_option, tmp_path, monkeypatch):
+    from licensedcode import cache
+
+    monkeypatch.setattr(cache, 'get_licenses_db', lambda: {})
+    source = tmp_path / 'sample.txt'
+    source.write_text('non-empty contents', encoding='utf-8')
+    output = tmp_path / 'output.spdx'
+    result = run_scan_click(
+        ['--email', str(source), output_option, str(output)],
+        expected_rc=1,
+        retry=False,
+    )
+    assert 'missing SHA1' in result.output
+    assert 'sample.txt' in result.output
+    assert '--info' in result.output
+    if output.exists():
+        assert not output.read_text(encoding='utf-8')
+
+
+@pytest.mark.parametrize('output_option', ['--spdx-tv', '--spdx-rdf'])
+@pytest.mark.parametrize('content', [b'non-empty contents', b''])
+def test_spdx_file_checksum_cli_with_info(output_option, content, tmp_path, monkeypatch):
+    from licensedcode import cache
+
+    monkeypatch.setattr(cache, 'get_licenses_db', lambda: {})
+    source = tmp_path / 'sample.txt'
+    source.write_bytes(content)
+    output = tmp_path / 'output.spdx'
+    run_scan_click(['--info', str(source), output_option, str(output)])
+    assert hashlib.sha1(content).hexdigest() in output.read_text(encoding='utf-8')
+
+
 def strip_variable_text(rdf_text):
     """
     Return rdf_text stripped from variable parts such as rdf nodeids
@@ -339,15 +434,21 @@ def test_spdx_rdf_with_empty_scan():
 
 
 @pytest.mark.scanslow
-def test_output_spdx_rdf_can_handle_non_ascii_paths():
+def test_output_spdx_rdf_can_handle_non_ascii_paths(monkeypatch):
+    from licensedcode import cache
+
+    monkeypatch.setattr(cache, 'get_licenses_db', lambda: {})
     test_file = test_env.get_test_loc('spdx/unicode.json')
     result_file = test_env.get_temp_file(extension='spdx', file_name='test_spdx')
     run_scan_click(['--from-json', test_file, '--spdx-rdf', result_file])
     with io.open(result_file, encoding='utf-8') as res:
         results = res.read()
     assert 'han/据.svg' in results
 
-def test_output_spdx_tv_can_handle_non_ascii_paths():
+def test_output_spdx_tv_can_handle_non_ascii_paths(monkeypatch):
+    from licensedcode import cache
+
+    monkeypatch.setattr(cache, 'get_licenses_db', lambda: {})
     test_file = test_env.get_test_loc('spdx/unicode.json')
     result_file = test_env.get_temp_file(extension='spdx', file_name='test_spdx')
     run_scan_click(['--from-json', test_file, '--spdx-tv', result_file])