Fix PyInstaller LD_LIBRARY_PATH contamination breaking FFmpeg on Linux (#766)

* Implement clean_env to manage environment variables

Add clean_env function to remove LD_LIBRARY_PATH from environment.

* Add clean_env to subprocess environment

* Use clean_env() in run_check_features function

Added clean_env() to subprocess calls for feature checks.

* Add clean_env to subprocess calls in background_tasks

Updated the run method to include a clean environment for subprocesses.

* Update subprocess call to include cleaned environment

Pass cleaned environment to subprocess run for thumbnail generation.

* Use clean_env for thumbnail generation

Updated thumbnail generation to use a cleaned environment.

* Use clean environment for thumbnail generation

Updated thumbnail generation to use a cleaned environment.

* Add clean_env function to subprocess environment

---------

Co-authored-by: Chris Griffith <chris@cdgriffith.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Author: 3 people

CHANGES

@@ -25,6 +25,7 @@
 * Adding warning in Data & Attachments tab and reason text when tracks are incompatible with the output format (e.g. data streams in MKV)
 * Adding Go-based Windows installer, launcher and uninstaller using an embeddable Python distribution (per-user and all-users installs, Add/Remove Programs integration, upgrades from previous NSIS installs) replacing the PyInstaller and NSIS Windows builds, plus a portable zip release
 * Changing Advanced panel layout to use QGroupBox sections (Video Details, Frame Rate, Video Processing, Color & Appearance, Color, Output) with restoration filters separated from color correction filters
+* Fixing #766 FFmpeg and other external tools failing on Linux PyInstaller builds (e.g. AUR fastflix-bin) with library symbol errors due to inherited bundled `LD_LIBRARY_PATH` (thanks to anne-o-pixel)
 * Fixing #741 VideoToolbox HEVC and H264 encoders passing invalid numeric profile index instead of profile name, causing "incorrect parameters" errors on macOS (thanks to enaveso)
 * Fixing #560 queue items could not be removed while encoding — now allows removing non-running items and grays out unavailable controls (move, reload, load queue) during encoding (thanks to 54x1)
 * Fixing #736 thumbnail and crop preview generation failing with FFmpeg 8.0+ by adding `-update 1` flag required by the image2 muxer for single-image output (thanks to kliffgomel)

fastflix/command_runner.py

@@ -14,6 +14,8 @@
 
 from psutil import Popen
 
+from fastflix.resources import clean_env
+
 try:
     from psutil import (
         HIGH_PRIORITY_CLASS,
@@ -93,6 +95,7 @@ def start_exec(self, command, work_dir: str = None, shell: bool = False, errors=
                 stderr=stderr_handle,
                 stdin=PIPE,  # FFmpeg can try to read stdin and wrecks havoc on linux
                 encoding="utf-8",
+                env=clean_env(),
             )
         except PermissionError:
             logger.error(

fastflix/flix.py

@@ -16,6 +16,7 @@
 from fastflix.language import t
 from fastflix.models.config import Config
 from fastflix.models.fastflix_app import FastFlixApp
+from fastflix.shared import clean_env
 
 here = os.path.abspath(os.path.dirname(__file__))
 re_tff = re.compile(r"TFF:\s+(\d+)")
@@ -151,6 +152,7 @@ def execute(command: List, work_dir: Union[Path, str] = None, timeout: int = Non
         cwd=work_dir,
         timeout=timeout,
         encoding="utf-8",
+        env=clean_env(),
     )
 
 
@@ -361,6 +363,7 @@ def extract_attachment(ffmpeg: Path, source: Path, stream: int, work_dir: Path,
             stderr=PIPE,
             stdin=PIPE,
             cwd=work_dir,
+            env=clean_env(),
         )
         try:
             stdout, _ = proc.communicate(timeout=3)
@@ -695,7 +698,9 @@ def get_hdr10_parser_version(config: Config) -> version:
     global HDR10_parser_version
     if HDR10_parser_version:
         return HDR10_parser_version
-    HDR10_parser_version_output = check_output([str(config.hdr10plus_parser), "--version"], encoding="utf-8")
+    HDR10_parser_version_output = check_output(
+        [str(config.hdr10plus_parser), "--version"], encoding="utf-8", env=clean_env()
+    )
 
     _, version_string = HDR10_parser_version_output.rsplit(sep=" ", maxsplit=1)
     HDR10_parser_version = version.parse(version_string)
@@ -759,6 +764,7 @@ def _detect_hdr10_plus_tool(app: FastFlixApp, config: Config, stream) -> bool:
         stdout=PIPE,
         stderr=PIPE,
         stdin=PIPE,  # FFmpeg can try to read stdin and wrecks havoc
+        env=clean_env(),
     )
 
     parser_version = get_hdr10_parser_version(config)
@@ -772,6 +778,7 @@ def _detect_hdr10_plus_tool(app: FastFlixApp, config: Config, stream) -> bool:
         stderr=PIPE,
         stdin=process.stdout,
         encoding="utf-8",
+        env=clean_env(),
     )
 
     try:

fastflix/resources.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from pathlib import Path
 import os
+import sys
 from functools import lru_cache
 
 import importlib.resources
@@ -84,3 +85,21 @@ def get_bool_env(variable: str):
     if var.lower() in ("1", "on", "yes", "true", "t"):
         return True
     return False
+
+
+def clean_env() -> dict:
+    """Return a copy of os.environ that is safe for launching external tools (FFmpeg, FFprobe, rigaya, etc).
+
+    A PyInstaller onefile build points LD_LIBRARY_PATH at its bundled libraries, so system binaries such as
+    /usr/bin/ffmpeg would load those (e.g. an incompatible libass) instead of the system ones. PyInstaller keeps
+    the user's original value in LD_LIBRARY_PATH_ORIG, which is restored here (or the variable is removed).
+    Kept in this Qt-free module so the conversion worker process can use it without importing PySide6.
+    """
+    env = os.environ.copy()
+    if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
+        original = env.pop("LD_LIBRARY_PATH_ORIG", None)
+        if original is not None:
+            env["LD_LIBRARY_PATH"] = original
+        else:
+            env.pop("LD_LIBRARY_PATH", None)
+    return env

fastflix/rigaya_helpers.py

@@ -2,6 +2,8 @@
 from dataclasses import dataclass, field
 from subprocess import run, PIPE
 
+from fastflix.resources import clean_env
+
 
 @dataclass
 class Encoder:
@@ -72,11 +74,13 @@ def parse_qsv_devices(split_text: list[str]) -> QSVEncoder:
 def run_check_features(executable, is_qsv=False):
     outputs = []
     if is_qsv:
-        result = run([executable, "--check-features"], stdout=PIPE, stderr=PIPE, encoding="utf-8")
+        result = run([executable, "--check-features"], stdout=PIPE, stderr=PIPE, encoding="utf-8", env=clean_env())
         outputs.append(result.stdout.splitlines())
     else:
         for i in range(10):
-            result = run([executable, "--check-features", str(i)], stdout=PIPE, stderr=PIPE, encoding="utf-8")
+            result = run(
+                [executable, "--check-features", str(i)], stdout=PIPE, stderr=PIPE, encoding="utf-8", env=clean_env()
+            )
             if result.stderr:
                 break
             outputs.append(result.stdout.splitlines())

fastflix/shared.py

@@ -19,7 +19,7 @@
 from PySide6 import QtCore, QtGui, QtWidgets
 
 from fastflix.language import t
-from fastflix.resources import get_bool_env
+from fastflix.resources import clean_env, get_bool_env  # noqa: F401 (clean_env re-exported)
 
 try:
     # PyInstaller creates a temp folder and stores path in _MEIPASS
@@ -337,7 +337,7 @@ def open_folder(path):
         elif sys.platform == "darwin":
             run(["open", path])
         else:
-            run(["xdg-open", path])
+            run(["xdg-open", path], env=clean_env())
     except FileNotFoundError:
         logger.error(f"Do not know which command to use to open: {path}")
 

fastflix/widgets/background_tasks.py

@@ -11,6 +11,7 @@
 from ffmpeg_normalize import FFmpegNormalize
 
 from fastflix.flix import extract_attachments
+from fastflix.shared import clean_env
 from fastflix.language import t
 from fastflix.models.fastflix_app import FastFlixApp
 from fastflix.shared import clean_file_string
@@ -42,7 +43,7 @@ def __init__(self, main, command=""):
 
     def run(self):
         self.main.thread_logging_signal.emit(f"DEBUG:{t('Generating thumbnail')}: {_format_command(self.command)}")
-        result = run(self.command, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
+        result = run(self.command, stdin=PIPE, stdout=PIPE, stderr=STDOUT, env=clean_env())
         if result.returncode > 0:
             output_text = result.stdout.decode(encoding="utf-8", errors="ignore")
             if "No such filter: 'zscale'" in output_text:
@@ -165,6 +166,7 @@ def run(self):
                 command,
                 stdout=PIPE,
                 stderr=STDOUT,
+                env=clean_env(),
             )
             stdout, _ = self._process.communicate()
             returncode = self._process.returncode
@@ -233,6 +235,7 @@ def _get_subtitle_format(self):
                 stdout=PIPE,
                 stderr=STDOUT,
                 text=True,
+                env=clean_env(),
             )
 
             if result.returncode != 0:
@@ -504,7 +507,9 @@ def run(self):
         self.ffmpeg_signal.emit("Extracting HDR10+ metadata")
 
         hdr10_parser_version_output = check_output(
-            [str(self.app.fastflix.config.hdr10plus_parser), "--version"], encoding="utf-8"
+            [str(self.app.fastflix.config.hdr10plus_parser), "--version"],
+            encoding="utf-8",
+            env=clean_env(),
         )
         _, version_string = hdr10_parser_version_output.rsplit(sep=" ", maxsplit=1)
         hdr10_parser_version = version.parse(version_string)
@@ -539,6 +544,7 @@ def run(self):
             stdout=PIPE,
             stderr=open(self.app.fastflix.current_video.work_path / "hdr10extract_out.txt", "wb"),
             # stdin=PIPE,  # FFmpeg can try to read stdin and wrecks havoc
+            env=clean_env(),
         )
 
         process_two = Popen(
@@ -548,6 +554,7 @@ def run(self):
             stdin=process.stdout,
             encoding="utf-8",
             cwd=str(self.app.fastflix.current_video.work_path),
+            env=clean_env(),
         )
 
         with open(self.app.fastflix.current_video.work_path / "hdr10extract_out.txt", "r", encoding="utf-8") as f:

fastflix/widgets/container.py

@@ -12,6 +12,7 @@
 from PySide6 import QtCore, QtGui, QtWidgets
 from PySide6.QtGui import QAction
 
+from fastflix.resources import clean_env
 from fastflix.exceptions import FastFlixInternalException
 from fastflix.language import t
 from fastflix.version import __version__
@@ -732,7 +733,7 @@ def run(self):
             elif sys.platform == "darwin":
                 run(["open", self.path])
             else:
-                run(["xdg-open", self.path])
+                run(["xdg-open", self.path], env=clean_env())
         except FileNotFoundError:
             logger.error(f"Do not know which command to use to open: {self.path}")
 

fastflix/widgets/main_post_encode.py

@@ -141,6 +141,7 @@ def _record_history(self, video: Video, success: bool = True):
             from subprocess import run as subprocess_run
 
             from fastflix.flix import generate_thumbnail_command
+            from fastflix.shared import clean_env
 
             thumb_command = generate_thumbnail_command(
                 config=self.app.fastflix.config,
@@ -150,7 +151,7 @@ def _record_history(self, video: Video, success: bool = True):
                 start_time=video.video_settings.start_time or 0,
                 input_track=video.video_settings.selected_track,
             )
-            result = subprocess_run(thumb_command, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
+            result = subprocess_run(thumb_command, stdin=PIPE, stdout=PIPE, stderr=STDOUT, env=clean_env())
             if result.returncode != 0 or not thumb_output.exists():
                 logger.warning("Failed to generate thumbnail for history entry")
                 entry.thumbnail_filename = ""

fastflix/widgets/windows/crop_window.py

@@ -9,6 +9,7 @@
 
 from fastflix.encoders.common import helpers
 from fastflix.flix import generate_thumbnail_command
+from fastflix.shared import clean_env
 from fastflix.language import t
 from fastflix.shared import time_to_number
 from fastflix.ui_scale import scaler
@@ -691,7 +692,7 @@ def generate_image(self, with_crop=False):
 
         logger.info(f"Generating crop preview: {thumb_command}")
 
-        thumb_run = run(thumb_command, shell=True, stderr=PIPE, stdout=PIPE)
+        thumb_run = run(thumb_command, shell=True, stderr=PIPE, stdout=PIPE, env=clean_env())
         if thumb_run.returncode > 0:
             stderr_text = thumb_run.stderr.decode(encoding="utf-8", errors="ignore")
             if settings.get("remove_hdr") and "no path between colorspaces" in stderr_text:
@@ -710,7 +711,7 @@ def generate_image(self, with_crop=False):
                     start_time=self._get_preview_time(),
                     input_track=video.video_settings.selected_track,
                 )
-                thumb_run = run(thumb_command, shell=True, stderr=PIPE, stdout=PIPE)
+                thumb_run = run(thumb_command, shell=True, stderr=PIPE, stdout=PIPE, env=clean_env())
             if thumb_run.returncode > 0:
                 logger.warning(f"Could not generate crop preview: {thumb_run.stdout} |----| {thumb_run.stderr}")
                 return