Handle Debian copyright paragraphs without a License field

AUTHORS.rst

@@ -36,6 +36,7 @@ The following organizations or individuals have contributed to ScanCode:
 - Hanif Ali @hanif-ali
 - Horie Issei @is2ei
 - James Ward @jamesward
+- Jatin Pandey @satiricalguru
 - Jelmer Vernooij @jelmer
 - Jens Keim @pepper-jk
 - Kunal Chhabra @kunalchhabra37

CHANGELOG.rst

@@ -4,6 +4,9 @@ Changelog
 Next release
 --------------
 
+- Handle Debian copyright paragraphs without a License field and detect misplaced license notices in their copyright text.
+  https://github.com/aboutcode-org/scancode-toolkit/issues/2977
+
 - Fix the optional ``licenses`` extra dependency typo to install
   ``licensedcode-data``.
   https://github.com/aboutcode-org/scancode-toolkit/pull/5056

src/licensedcode/data/rules/fsl-1.1_1.RULE

@@ -5,6 +5,8 @@ is_continuous: yes
 relevance: 100
 minimum_coverage: 100
 notes: Rule based on the manifest file of the liquidbase source jar. 
+ignorable_urls:
+    - https://fsl.software/FSL-1.1-ALv2.template.md
 ---
 
 Bundle-License: https://fsl.software/FSL-1.1-ALv2.template.md

src/licensedcode/data/rules/fsl-1.1_2.RULE

@@ -5,6 +5,8 @@ is_continuous: yes
 relevance: 100
 minimum_coverage: 100
 notes: Rule based on the manifest file of the liquidbase source jar. 
+ignorable_urls:
+    - https://fsl.software/FSL-1.1-ALv2.template.md
 ---
 
 https://fsl.software/FSL-1.1-ALv2.template.md

src/packagedcode/debian_copyright.py

@@ -949,6 +949,11 @@ def get_license_detections(paragraph, debian_licensing):
         name = paragraph.license.name
 
         if not name:
+            if 'license' not in paragraph.line_numbers_by_field:
+                return get_license_detections_from_other_fields(
+                    paragraph=paragraph,
+                    check_copyright=True,
+                ) or []
             license_detections.append(
                 get_license_detection_from_nameless_paragraph(paragraph=paragraph)
             )
@@ -1815,10 +1820,12 @@ def add_undetected_debian_matches(name, text):
     return get_undetected_matches(query_string)
 
 
-def get_license_detections_from_other_fields(paragraph):
+def get_license_detections_from_other_fields(paragraph, check_copyright=False):
     """
     Return a list of DebianLicenseDetection found in a ``paragraph`` comment,
-    disclaimer or extra_data fields, e.g. all but the "license" field
+    disclaimer or extra_data fields, e.g. all but the "license" field.
+    Also check the copyright field for misplaced notices when ``check_copyright``
+    is True (for paragraphs without a License field).
     """
 
     def detect_license_and_save(fname, fvalue):
@@ -1837,6 +1844,12 @@ def detect_license_and_save(fname, fvalue):
                 license_detections.append(fdetection)
 
     license_detections = []
+    if check_copyright and getattr(paragraph, 'copyright', None):
+        detect_license_and_save(
+            fname='copyright',
+            fvalue=paragraph.copyright.dumps(),
+        )
+
     for field_name in ('comment', 'disclaimer',):
         field = getattr(paragraph, field_name, None)
         if not field:

tests/packagedcode/test_debian_copyright.py

@@ -225,6 +225,62 @@ def test_add_unknown_matches(self):
         match = matches[0]
         assert match.matched_text() == 'license foo\nbar'
 
+    def test_parse_copyright_file_missing_license_field_with_license(self):
+        content = (
+            'Format: https://www.debian.org/doc/packaging-manuals/'
+            'copyright-format/1.0/\n'
+            '\n'
+            'Files: *\n'
+            'Copyright: 2020 ACME\n'
+            'License: Apache-2.0\n'
+            ' Licensed under Apache 2.0\n'
+            '\n'
+            'Files: clang/lib/Headers/cuda_wrappers/*\n'
+            'Copyright: 2016-2020, Google Inc.\n'
+            ' Permission is hereby granted, free of charge, to any person '
+            'obtaining a copy\n'
+            ' of this software and associated documentation files (the '
+            '"Software"), to deal\n'
+            ' in the Software without restriction, including without '
+            'limitation the rights\n'
+            ' to use, copy, modify, merge, publish, distribute, sublicense, '
+            'and/or sell\n'
+            ' copies of the Software, and to permit persons to whom the '
+            'Software is\n'
+            ' furnished to do so, subject to the following conditions:\n'
+        )
+        test_loc = self.get_temp_file(file_name='copyright', extension='')
+        with open(test_loc, 'w') as f:
+            f.write(content)
+
+        dc = debian_copyright.parse_copyright_file(location=test_loc)
+        assert dc is not None
+        lic_exp = dc.get_license_expression()
+        assert 'apache-2.0' in lic_exp
+        assert 'x11' in lic_exp or 'mit' in lic_exp
+
+    def test_parse_copyright_file_missing_license_field_without_license(self):
+        content = (
+            'Format: https://www.debian.org/doc/packaging-manuals/'
+            'copyright-format/1.0/\n'
+            '\n'
+            'Files: *\n'
+            'Copyright: 2020 ACME\n'
+            'License: Apache-2.0\n'
+            ' Licensed under Apache 2.0\n'
+            '\n'
+            'Files: no-license/*\n'
+            'Copyright: 2020 Bob\n'
+        )
+        test_loc = self.get_temp_file(file_name='copyright', extension='')
+        with open(test_loc, 'w') as f:
+            f.write(content)
+
+        dc = debian_copyright.parse_copyright_file(location=test_loc)
+        assert dc is not None
+        lic_exp = dc.get_license_expression()
+        assert 'apache-2.0' in lic_exp
+
 
 class TestMetadataDebianCopyright(PackageTester):
     test_data_dir = path.join(path.dirname(__file__), 'data/debian/copyright/')

tests/scancode/test_cli.py

@@ -11,6 +11,7 @@
 import json
 import os
 
+import click
 import pytest
 from unittest import skipIf
 
@@ -722,6 +723,16 @@ def no_spaces(s):
     with open(expected_file) as ef:
         expected = ef.read()
 
+    # Click 8.5.1 also lists arguments without help text.
+    input_argument = click.Argument(['input'])
+    if input_argument.get_help_record(click.Context(click.Command('scancode'))):
+        expected = expected.replace(
+            '  documentation:\n',
+            '  miscellaneous:\n'
+            '    <OUTPUT FORMAT OPTION(s)> <input>...\n\n'
+            '  documentation:\n',
+        )
+
     if no_spaces(result) != no_spaces(expected):
         assert result == expected