Fix Raqm licensing issue by building against SheenBiDi and simplify the build

.ci/install.sh

@@ -19,7 +19,7 @@ set -e
 
 sudo apt-get -qq install libfreetype6-dev liblcms2-dev libtiff-dev python3-tk\
                          ghostscript libjpeg-turbo8-dev libopenjp2-7-dev\
-                         cmake meson imagemagick libharfbuzz-dev libfribidi-dev\
+                         cmake meson imagemagick libharfbuzz-dev\
                          sway wl-clipboard libopenblas-dev nasm
 
 python3 -m pip install --upgrade pip
@@ -49,7 +49,7 @@ pushd depends && ./install_webp.sh && popd
 pushd depends && ./install_imagequant.sh && popd
 
 # raqm
-pushd depends && sudo ./install_raqm.sh && popd
+pushd depends && sudo ./install_sheenbidi.sh && sudo ./install_raqm.sh && popd
 
 # libavif
 pushd depends && ./install_libavif.sh && popd

.github/INCIDENT_RESPONSE.md

@@ -241,7 +241,7 @@ require a Pillow point release even if Pillow's own code is unchanged.
 | [lcms2](https://www.littlecms.com/) | ICC color management | [GitHub](https://github.com/mm2/Little-CMS/security) |
 | [harfbuzz](https://harfbuzz.github.io/) | Text shaping (via raqm) | [GitHub](https://github.com/harfbuzz/harfbuzz/security) |
 | [raqm](https://github.com/HOST-Oman/libraqm) | Complex text layout | [GitHub](https://github.com/HOST-Oman/libraqm) |
-| [fribidi](https://github.com/fribidi/fribidi) | Unicode bidi (via raqm) | [GitHub](https://github.com/fribidi/fribidi) |
+| [sheenbidi](https://github.com/Tehreer/SheenBidi) | Unicode bidi (via raqm) | [GitHub](https://github.com/Tehreer/SheenBidi) |
 | [zlib](https://zlib.net/) | Deflate compression | [zlib.net](https://zlib.net/) |
 | [liblzma / xz-utils](https://tukaani.org/xz/) | XZ/LZMA compression | [GitHub](https://github.com/tukaani-project/xz/security) |
 | [bzip2](https://gitlab.com/bzip2/bzip2) | BZ2 compression | [GitLab](https://gitlab.com/bzip2/bzip2/-/work_items) |

.github/dependencies.json

@@ -2,7 +2,6 @@
   "brotli": "1.2.0",
   "bzip2": "1.0.8",
   "freetype": "2.14.3",
-  "fribidi": "1.0.16",
   "harfbuzz": "14.2.1",
   "jpegturbo": "3.2.0",
   "lcms2": "2.19.1",
@@ -12,6 +11,8 @@
   "libwebp": "1.6.0",
   "libxcb": "1.17.0",
   "openjpeg": "2.5.4",
+  "raqm": "0.11.0",
+  "sheenbidi": "3.0.0",
   "tiff": "4.7.2",
   "xz": "5.8.3",
   "zlib-ng": "2.3.3",

.github/generate-sbom.py

@@ -11,12 +11,9 @@
 from __future__ import annotations
 
 import argparse
-import base64
 import datetime as dt
-import difflib
 import hashlib
 import json
-import urllib.request
 import uuid
 from pathlib import Path
 
@@ -35,28 +32,6 @@ def sha256_file(path: Path) -> str:
     return hashlib.sha256(path.read_bytes()).hexdigest()
 
 
-def upstream_diff_b64(
-    upstream_url: str,
-    upstream_display: bytes,
-    local_path: Path,
-    local_display: bytes,
-) -> str:
-    """
-    Fetch an upstream file and return a base64-encoded unified diff vs the local copy.
-    """
-    with urllib.request.urlopen(upstream_url) as resp:
-        upstream_text = resp.read()
-    local_text = local_path.read_bytes()
-    diff_lines = difflib.diff_bytes(
-        difflib.unified_diff,
-        upstream_text.splitlines(keepends=True),
-        local_text.splitlines(keepends=True),
-        fromfile=b"a/" + upstream_display,
-        tofile=b"b/" + local_display,
-    )
-    return base64.b64encode(b"".join(diff_lines)).decode()
-
-
 def generate(version: str) -> dict:
     serial = str(uuid.uuid4())
     now = dt.datetime.now(dt.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -115,28 +90,6 @@ def generate(version: str) -> dict:
     ]
 
     vendored_components = [
-        {
-            "bom-ref": f"{purl}#thirdparty/fribidi-shim",
-            "type": "library",
-            "name": "fribidi-shim",
-            "version": "1.x",
-            "description": "FriBiDi runtime-loading shim "
-            "(vendored in src/thirdparty/fribidi-shim/); "
-            "loads libfribidi dynamically",
-            "licenses": [{"license": {"id": "LGPL-2.1-or-later"}}],
-            "hashes": [
-                {
-                    "alg": "SHA-256",
-                    "content": sha256_file(thirdparty / "fribidi-shim" / "fribidi.c"),
-                }
-            ],
-            "pedigree": {
-                "notes": "Pillow-authored shim; not taken from an upstream project."
-            },
-            "externalReferences": [
-                {"type": "website", "url": "https://github.com/fribidi/fribidi"},
-            ],
-        },
         {
             "bom-ref": "pkg:github/python/pythoncapi-compat",
             "type": "library",
@@ -160,90 +113,6 @@ def generate(version: str) -> dict:
                 },
             ],
         },
-        {
-            "bom-ref": f"{purl}#thirdparty/raqm",
-            "type": "library",
-            "name": "raqm",
-            "version": "0.11.0",
-            "description": "Complex text layout library "
-            "(vendored in src/thirdparty/raqm/)",
-            "licenses": [{"license": {"id": "MIT"}}],
-            "hashes": [
-                {
-                    "alg": "SHA-256",
-                    "content": sha256_file(thirdparty / "raqm" / "raqm.c"),
-                }
-            ],
-            "pedigree": {
-                "ancestors": [
-                    {
-                        "bom-ref": "pkg:github/HOST-Oman/libraqm@0.11.0#upstream",
-                        "type": "library",
-                        "name": "raqm",
-                        "version": "0.11.0",
-                        "purl": "pkg:github/HOST-Oman/libraqm@0.11.0",
-                        "externalReferences": [
-                            {
-                                "type": "distribution",
-                                "url": "https://github.com/HOST-Oman/libraqm/releases/tag/v0.11.0",
-                            }
-                        ],
-                    }
-                ],
-                "patches": [
-                    {
-                        "type": "unofficial",
-                        "diff": {
-                            "text": {
-                                # raqm-version.h.in → raqm-version.h:
-                                # template @RAQM_VERSION_*@ placeholders replaced
-                                # with literal 0.11.0 values; filename changed to
-                                # drop the .in suffix.
-                                "content": upstream_diff_b64(
-                                    "https://raw.githubusercontent.com/HOST-Oman/libraqm/v0.11.0/src/raqm-version.h.in",
-                                    b"src/raqm-version.h.in",
-                                    thirdparty / "raqm" / "raqm-version.h",
-                                    b"src/raqm-version.h",
-                                ),
-                                "encoding": "base64",
-                            }
-                        },
-                    },
-                    {
-                        "type": "unofficial",
-                        "diff": {
-                            "text": {
-                                # raqm.c: wrap the <fribidi.h> include in an
-                                # #ifdef HAVE_FRIBIDI_SYSTEM guard so that when
-                                # building without a system FriBiDi Pillow's own
-                                # fribidi-shim is used instead.
-                                "content": upstream_diff_b64(
-                                    "https://raw.githubusercontent.com/HOST-Oman/libraqm/v0.11.0/src/raqm.c",
-                                    b"src/raqm.c",
-                                    thirdparty / "raqm" / "raqm.c",
-                                    b"src/raqm.c",
-                                ),
-                                "encoding": "base64",
-                            }
-                        },
-                    },
-                ],
-                "notes": (
-                    "Vendored from upstream HOST-Oman/libraqm v0.11.0 with two "
-                    "Pillow-specific modifications: (1) raqm-version.h.in was "
-                    "pre-processed into raqm-version.h with version placeholders "
-                    "replaced by literal values; (2) raqm.c wraps the <fribidi.h> "
-                    "include in an #ifdef HAVE_FRIBIDI_SYSTEM guard so Pillow's "
-                    "bundled fribidi-shim is used when a system FriBiDi is absent."
-                ),
-            },
-            "externalReferences": [
-                {
-                    "type": "vcs",
-                    "url": "https://github.com/python-pillow/Pillow/tree/main/src/thirdparty/raqm",
-                },
-            ],
-        },
     ]
 
     native_deps = [
@@ -264,23 +133,6 @@ def generate(version: str) -> dict:
                 },
             ],
         },
-        {
-            "bom-ref": "pkg:generic/fribidi",
-            "type": "library",
-            "name": "FriBiDi",
-            "version": versions["fribidi"],
-            "scope": "optional",
-            "description": "Unicode bidi algorithm library (optional, "
-            "loaded at runtime by fribidi-shim).",
-            "licenses": [{"license": {"id": "LGPL-2.1-or-later"}}],
-            "externalReferences": [
-                {"type": "website", "url": "https://github.com/fribidi/fribidi"},
-                {
-                    "type": "distribution",
-                    "url": "https://github.com/fribidi/fribidi/releases",
-                },
-            ],
-        },
         {
             "bom-ref": "pkg:generic/harfbuzz",
             "type": "library",
@@ -434,6 +286,39 @@ def generate(version: str) -> dict:
                 },
             ],
         },
+        {
+            "bom-ref": "pkg:generic/raqm",
+            "type": "library",
+            "name": "Raqm",
+            "version": versions["raqm"],
+            "scope": "optional",
+            "description": "Complex text layout (optional, used by PIL._imagingft).",
+            "licenses": [{"license": {"id": "MIT"}}],
+            "externalReferences": [
+                {"type": "website", "url": "https://github.com/HOST-Oman/libraqm"},
+                {
+                    "type": "distribution",
+                    "url": "https://github.com/HOST-Oman/libraqm/releases",
+                },
+            ],
+        },
+        {
+            "bom-ref": "pkg:generic/sheenbidi",
+            "type": "library",
+            "name": "SheenBidi",
+            "version": versions["sheenbidi"],
+            "scope": "optional",
+            "description": "Unicode bidirectional algorithm (optional, required "
+            "by libraqm for bidirectional text).",
+            "licenses": [{"license": {"id": "Apache-2.0"}}],
+            "externalReferences": [
+                {"type": "website", "url": "https://github.com/Tehreer/SheenBidi"},
+                {
+                    "type": "distribution",
+                    "url": "https://github.com/Tehreer/SheenBidi/releases",
+                },
+            ],
+        },
         {
             "bom-ref": "pkg:pypi/pybind11",
             "type": "library",
@@ -492,21 +377,19 @@ def generate(version: str) -> dict:
             "ref": f"{purl}#c-ext/PIL._imagingft",
             "dependsOn": [
                 "pkg:generic/freetype2",
-                "pkg:generic/fribidi",
-                "pkg:generic/harfbuzz",
-                f"{purl}#thirdparty/fribidi-shim",
-                f"{purl}#thirdparty/raqm",
+                "pkg:generic/raqm",
             ],
         },
         {
             "ref": f"{purl}#c-ext/PIL._webp",
             "dependsOn": ["pkg:generic/libwebp"],
         },
         {
-            "ref": f"{purl}#thirdparty/raqm",
+            "ref": "pkg:generic/raqm",
             "dependsOn": [
+                "pkg:generic/freetype2",
                 "pkg:generic/harfbuzz",
-                f"{purl}#thirdparty/fribidi-shim",
+                "pkg:generic/sheenbidi",
             ],
         },
     ]

.github/renovate.json

@@ -41,15 +41,6 @@
             "extractVersionTemplate": "^VER-(?<version>[\\d-]+)$",
             "versioningTemplate": "regex:^(?<major>\\d+)[.-](?<minor>\\d+)[.-](?<patch>\\d+)$"
         },
-        {
-            "customType": "regex",
-            "managerFilePatterns": ["/^\\.github/dependencies\\.json$/"],
-            "matchStrings": ["\"fribidi\":\\s*\"(?<currentValue>\\d+[^\"]*)\""],
-            "depNameTemplate": "fribidi",
-            "packageNameTemplate": "fribidi/fribidi",
-            "datasourceTemplate": "github-releases",
-            "extractVersionTemplate": "^v(?<version>.+)$"
-        },
         {
             "customType": "regex",
             "managerFilePatterns": ["/^\\.github/dependencies\\.json$/"],
@@ -129,6 +120,24 @@
             "datasourceTemplate": "github-releases",
             "extractVersionTemplate": "^v(?<version>.+)$"
         },
+        {
+            "customType": "regex",
+            "managerFilePatterns": ["/^\\.github/dependencies\\.json$/"],
+            "matchStrings": ["\"raqm\":\\s*\"(?<currentValue>\\d+[^\"]*)\""],
+            "depNameTemplate": "raqm",
+            "packageNameTemplate": "HOST-Oman/libraqm",
+            "datasourceTemplate": "github-releases",
+            "extractVersionTemplate": "^v(?<version>.+)$"
+        },
+        {
+            "customType": "regex",
+            "managerFilePatterns": ["/^\\.github/dependencies\\.json$/"],
+            "matchStrings": ["\"sheenbidi\":\\s*\"(?<currentValue>\\d+[^\"]*)\""],
+            "depNameTemplate": "sheenbidi",
+            "packageNameTemplate": "Tehreer/SheenBidi",
+            "datasourceTemplate": "github-releases",
+            "extractVersionTemplate": "^v(?<version>.+)$"
+        },
         {
             "customType": "regex",
             "managerFilePatterns": ["/^\\.github/dependencies\\.json$/"],

.github/workflows/test-windows.yml

@@ -176,9 +176,13 @@ jobs:
       run: "& winbuild\\build\\build_dep_harfbuzz.cmd"
 
     # Raqm dependencies
-    - name: Build dependencies / FriBidi
+    - name: Build dependencies / SheenBidi
       if: steps.build-cache.outputs.cache-hit != 'true'
-      run: "& winbuild\\build\\build_dep_fribidi.cmd"
+      run: "& winbuild\\build\\build_dep_sheenbidi.cmd"
+
+    - name: Build dependencies / Raqm
+      if: steps.build-cache.outputs.cache-hit != 'true'
+      run: "& winbuild\\build\\build_dep_raqm.cmd"
 
     # trim ~150MB for each job
     - name: Optimize build cache
@@ -189,7 +193,7 @@ jobs:
 
     - name: Build Pillow
       run: |
-        $FLAGS="-C raqm=vendor -C fribidi=vendor"
+        $FLAGS="-C raqm=enable"
         cmd /c "winbuild\build\build_env.cmd && $env:pythonLocation\python.exe -m pip install -v $FLAGS .[tests]"
         & $env:pythonLocation\python.exe selftest.py --installed
       shell: pwsh