Merge pull request #5289 from aboutcode-org/system-resource-cache

System resource cache

Author: JonoYang

CHANGELOG.rst

@@ -10,6 +10,12 @@ Next release
 
 - Improve copyright detection for statements with parens or trailing "authors"
 
+- Add experimental option for using cached results during scan time. When the
+  ``--use-cached-results`` option is enabled in the ScanCode CLI, during scan
+  time for a given Resource, we iterate through the active scanners and see if we
+  have cached results for those already. If we do, we update our results with the
+  cached data. If not, we add those scanners to a list of scanners to be run.
+  After scanning, the cache is updated.
 
 v33.0.0rc1 - 2026-05-14
 ------------------------

commoncode-CHANGELOG.rst

@@ -1,6 +1,13 @@
 Release notes
 =============
 
+Version 33.0.0rc1 - (2026-xx-xx)
+--------------------------------
+
+- Split logic from `commoncode.hash.checksum_from_chunks` into new function
+  `commoncode.hash.hasher_from_chunks`. This returns a hasher of a specified
+  checksum that contains the contents of an iterator `chunks`.
+
 Version 32.5.2 - (2026-06-11)
 -----------------------------
 

configure

@@ -261,8 +261,8 @@ install_packages_with_local() {
     # we need to install commoncode from local source first so this
     # is tested and not the released commoncode
     "$CFG_BIN_DIR/flot" --pyproject pyproject-commoncode.toml
-    "$CFG_BIN_DIR/pip" install ./dist/commoncode*.whl
     install_packages "$CFG_REQUIREMENTS"
+    "$CFG_BIN_DIR/pip" install --force-reinstall ./dist/commoncode*.whl
 }
 
 

docs/source/reference/scancode-cli/cli-core-options.rst

@@ -196,6 +196,32 @@ Comparing progress message options
 
 ----
 
+.. _cli-use-cached-results-option:
+
+``--use-cached-results``
+------------------------
+
+    When enabled, during scan time for a given Resource, we iterate through the
+    active scanners and see if we have cached results for those already. If we
+    do, we update our results with the cached data. If not, we add those
+    scanners to a list of scanners to be run. After scanning, the cache is
+    updated.
+
+    **Example**
+
+    .. code-block:: shell
+
+        scancode -clipeu --use-cached-results samples samples.json
+
+    On the first run, the cache will be created for the scanned Resources in
+    `samples`. Subsequent runs on `samples` will return cached results.
+
+    When run as a stand-alone release, the results cache is tied to specific
+    ScanCode toolkit versions. When run in development mode, the cache is stored
+    in the `scancode-toolkit/.cache` directory.
+
+----
+
 .. _glob-pattern-matching:
 
 Glob Pattern Matching

pyproject.toml

@@ -164,7 +164,6 @@ files = [
     { filename = "pyproject.toml" },
     { filename = "pyproject-scancode-toolkit-mini.toml" },
     { filename = "pyproject-packagedcode.toml" },
-    
 ]
 
 

requirements.txt

@@ -3,7 +3,7 @@ babel==2.17.0
 banal==1.0.6
 beartype==0.22.9
 beautifulsoup4==4.14.3
-binaryornot==0.4.4
+binaryornot==0.6.0
 boolean.py==5.0
 certifi==2026.1.4
 cffi==2.0.0

src/commoncode/hash.py

@@ -229,15 +229,24 @@ def checksum(location, name, base64=False):
     return checksum_from_chunks(chunks=chunks, total_length=total_length, name=name, base64=base64)
 
 
+def hasher_from_chunks(chunks, name, total_length=0):
+    """
+    Return a hasher of ``name`` checksum algorithm of contains the contents of
+    the iterator of byte strings ``chunks`` of length ``total_length``.
+    """
+    hasher = get_hasher_instance_by_name(name=name, total_length=total_length)
+    for chunk in chunks:
+        hasher.update(chunk)
+    return hasher
+
+
 def checksum_from_chunks(chunks, name, total_length=0, base64=False):
     """
     Return a checksum from the content of the iterator of byte strings ``chunks`` with a
     ``total_length`` combined length using the ``name`` checksum algorithm. The returned checksum is
     a string as a hexdigest or is base64-encoded is ``base64`` is True.
     """
-    hasher = get_hasher_instance_by_name(name=name, total_length=total_length)
-    for chunk in chunks:
-        hasher.update(chunk)
+    hasher = hasher_from_chunks(chunks=chunks, name=name, total_length=total_length)
     if base64:
         return hasher.b64digest()
     return hasher.hexdigest()

src/scancode/cli.py

@@ -67,6 +67,7 @@ class WindowsError(Exception):
 from scancode import notice
 from scancode import print_about
 from scancode import Scanner
+from scancode import results_cache
 from scancode.help import epilog_text
 from scancode.help import examples_text
 from scancode.interrupt import DEFAULT_TIMEOUT
@@ -407,12 +408,18 @@ def default_processes():
     help_group=cliutils.MISC_GROUP, sort_order=1000, cls=PluggableCommandLineOption)
 
 @click.option(
-    "--check-version/--no-check-version",
-    help="Whether to check for new versions. Defaults to true.",
+    '--check-version/--no-check-version',
+    help='Whether to check for new versions. Defaults to true.',
     default=True,
     # not yet supported in Click 6.7 but added in PluggableCommandLineOption
     hidden=True,
     help_group=cliutils.MISC_GROUP, sort_order=1000, cls=PluggableCommandLineOption)
+
+@click.option('--use-cached-results',
+    is_flag=True,
+    default=False,
+    help='(EXPERIMENTAL) Enables the creation, update, and usage of cached results during scan time.',
+    help_group=cliutils.CORE_GROUP, sort_order=250, cls=PluggableCommandLineOption)
 def scancode(
     ctx,
     input,  # NOQA
@@ -433,6 +440,7 @@ def scancode(
     test_error_mode,
     keep_temp_files,
     check_version,
+    use_cached_results,
     echo_func=echo_stderr,
     *args,
     **kwargs,
@@ -549,6 +557,7 @@ def scancode(
             return_results=False,
             echo_func=echo_func,
             outdated=outdated,
+            use_cached_results=use_cached_results,
             *args,
             **kwargs
         )
@@ -570,7 +579,7 @@ def scancode(
 
 
 def run_scan(
-    input,  # 
+    input,  #
     config_file=None,
     ignore=[],
     from_json=False,
@@ -594,6 +603,7 @@ def run_scan(
     pretty_params=None,
     plugin_options=plugin_options,
     outdated=None,
+    use_cached_results=False,
     *args,
     **kwargs
 ):
@@ -1006,6 +1016,7 @@ def echo_func(*_args, **_kwargs):
             verbose=verbose,
             kwargs=requested_options,
             echo_func=echo_func,
+            use_cached_results=use_cached_results,
         )
         success = success and scan_success
 
@@ -1141,7 +1152,7 @@ def load_configuration_file(path):
 
     click.echo(f"Loading env from {path}")
     try:
-        
+
         config_values = saneyaml.load(path.read())
         ignores = config_values.get("ignored_patterns", [])
     except (saneyaml.YAMLError, Exception):
@@ -1226,6 +1237,7 @@ def run_scanners(
     verbose=False,
     kwargs=None,
     echo_func=echo_stderr,
+    use_cached_results=False,
 ):
     """
     Run the list of `stage` ScanPlugin `plugins` on `codebase`.
@@ -1267,7 +1279,8 @@ def run_scanners(
     # TODO: add CLI option to bypass cache entirely?
     scan_success = scan_codebase(
         codebase, scanners, processes, timeout,
-        with_timing=timing, progress_manager=progress_manager)
+        with_timing=timing, progress_manager=progress_manager,
+        use_cached_results=use_cached_results)
 
     # TODO: add progress indicator
     # run the process codebase of each scan plugin (most often a no-op)
@@ -1305,6 +1318,7 @@ def scan_codebase(
     with_timing=False,
     progress_manager=None,
     echo_func=echo_stderr,
+    use_cached_results=False,
 ):
     """
     Run the `scanners` Scanner objects on the `codebase` Codebase. Return True
@@ -1326,15 +1340,16 @@ def scan_codebase(
     """
 
     # NOTE: we never scan directories
-    resources = ((r.location, r.path) for r in codebase.walk() if r.is_file)
+    resources = ((r.location, r.path, r.name) for r in codebase.walk() if r.is_file)
 
     use_threading = processes >= 0
     runner = partial(
         scan_resource,
         scanners=scanners,
         timeout=timeout,
         with_timing=with_timing,
-        with_threading=use_threading
+        with_threading=use_threading,
+        use_cached_results=use_cached_results,
     )
 
     if TRACE:
@@ -1462,11 +1477,12 @@ def terminate_pool_with_backoff(pool, number_of_trials=3):
 
 
 def scan_resource(
-    location_path,
+    location_path_name,
     scanners,
     timeout=DEFAULT_TIMEOUT,
     with_timing=False,
     with_threading=True,
+    use_cached_results=False,
 ):
     """
     Given a ``location_path`` tuple pf (location, path), return a tuple of:
@@ -1489,10 +1505,11 @@ def scan_resource(
     processing and threading works.
     """
     scan_time = time()
-    location, path = location_path
+    location, path, name = location_path_name
     results = {}
     scan_errors = []
     timings = {} if with_timing else None
+    scanners_to_run = []
 
     if not with_threading:
         interruptor = fake_interruptible
@@ -1503,8 +1520,28 @@ def scan_resource(
     # and start returning values. The kill timeout is otherwise there
     # as a gatekeeper for runaway processes.
 
+    results_cache_index = ''
+    if use_cached_results:
+        # compute results_cache_index
+        results_cache_index = results_cache.compute_results_cache_index(location=location, filename=name)
+
+        # update `results` with cached data or add scanner to scanners_to_run if no
+        # cache data is available
+        for scanner in scanners:
+            # get resource_cache_data
+            resource_cache_data = results_cache.get_results_cache_data(
+                results_cache_index=results_cache_index,
+                plugin_name=scanner.name
+            )
+            if resource_cache_data:
+                results.update(resource_cache_data)
+            else:
+                scanners_to_run.append(scanner)
+    else:
+        scanners_to_run = scanners
+
     # run each scanner in sequence in its own interruptible
-    for scanner in scanners:
+    for scanner in scanners_to_run:
         if with_timing:
             start = time()
 
@@ -1523,6 +1560,12 @@ def scan_resource(
             # the return value of a scanner fun MUST be a mapping
             if values_mapping:
                 results.update(values_mapping)
+                if use_cached_results:
+                    results_cache.update_results_cache_data(
+                        results_cache_index=results_cache_index,
+                        plugin_name=scanner.name,
+                        results=values_mapping,
+                    )
 
         except Exception:
             msg = 'ERROR: for scanner: ' + scanner.name + ':\n' + traceback.format_exc()