Remove trt-llm (#662)

Signed-off-by: Onur Yilmaz <oyilmaz@nvidia.com>
Signed-off-by: oliver könig <okoenig@nvidia.com>
Signed-off-by: Charlie Truong <chtruong@nvidia.com>
Co-authored-by: oliver könig <okoenig@nvidia.com>
Co-authored-by: Charlie Truong <chtruong@nvidia.com>

Author: 3 people

.github/labeler.yml

@@ -1,6 +1,5 @@
 export:
 - nemo_export/**/*
-- tests/functional_tests/tests_trtllm/test_export.py
 - tests/functional_tests/tests_onnx_trt/test_export.py
 - tests/unit_tests/export/**/*
 - docs/llm/**/*gen*ckpt.md
@@ -9,7 +8,6 @@ export:
 deploy:
 - nemo_deploy/**/*
 - scripts/deploy/**/*
-- tests/functional_tests/tests_trtllm/test_deploy*.py
 - tests/unit_tests/deploy/**/*
 - docs/llm/**/in-framework*.md
 - docs/mm/**/in-framework*.md
@@ -27,13 +25,6 @@ LLM:
 common:
 - nemo_export_deploy_common/**/*
 
-TensorRT-LLM:
-- nemo_export/trt_llm/**/*
-- nemo_export/tensorrt_llm*.py
-- tests/functional_tests/tests_trtllm/**/*
-- docs/llm/**/optimized/**/*
-- docs/mm/**/optimized/**/*
-
 vLLM:
 - nemo_export/vllm_exporter.py
 - tests/functional_tests/tests_vllm/**/*

.github/workflows/cicd-main.yml

@@ -144,43 +144,6 @@ jobs:
           secrets: |
             GH_TOKEN=${{ secrets.PAT }}
 
-  cicd-unit-tests-trtllm:
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - script: L0_Unit_Tests_GPU
-            runner: ${{ needs.pre-flight.outputs.runner_prefix }}
-          - script: L0_Unit_Tests_CPU
-            runner: ${{ needs.pre-flight.outputs.runner_prefix }}
-            cpu-only: true
-    needs: [pre-flight, cicd-wait-in-queue, cicd-container-build]
-    if: |
-      (
-        success()
-        || needs.pre-flight.outputs.is_ci_workload == 'true'
-        || needs.pre-flight.outputs.force_run_all == 'true'
-      )
-      && !cancelled()
-    runs-on: ${{ matrix.runner }}
-    name: ${{ matrix.script }}
-    environment: nemo-ci
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-      - name: main
-        uses: ./.github/actions/test-template
-        with:
-          script: ${{ matrix.script }}
-          timeout: 60
-          is_unit_test: "true"
-          cpu-only: ${{ matrix.cpu-only || false }}
-          PAT: ${{ secrets.PAT }}
-          container-image: ${{ needs.pre-flight.outputs.registry }}/export-deploy:${{ github.sha }}
-          inference-framework: trtllm
-          test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
-          runner: ${{ matrix.runner }}
-
   cicd-unit-tests-vllm:
     strategy:
       fail-fast: false
@@ -218,44 +181,8 @@ jobs:
           test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
           runner: ${{ matrix.runner }}
 
-  cicd-e2e-tests-trtllm:
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - script: L2_Launch_TRTLLM
-            runner: ${{ needs.pre-flight.outputs.runner_prefix }}
-          - script: L2_TRTLLM_API_Deploy_Query
-            runner: ${{ needs.pre-flight.outputs.runner_prefix }}
-    needs: [pre-flight, cicd-unit-tests-trtllm]
-    runs-on: ${{ matrix.runner }}
-    name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
-    if: |
-      (
-        success()
-        || needs.pre-flight.outputs.is_ci_workload == 'true'
-        || needs.pre-flight.outputs.force_run_all == 'true'
-      )
-      && !cancelled()
-    environment: nemo-ci
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-      - name: main
-        uses: ./.github/actions/test-template
-        with:
-          script: ${{ matrix.script }}
-          is_optional: ${{ matrix.is_optional || false }}
-          is_unit_test: "false"
-          timeout: 60
-          PAT: ${{ secrets.PAT }}
-          container-image: ${{ needs.pre-flight.outputs.registry }}/export-deploy:${{ github.sha }}
-          inference-framework: trtllm
-          test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
-          runner: ${{ matrix.runner }}
-
   cicd-e2e-tests-trt-onnx:
-    needs: [cicd-unit-tests-trtllm, pre-flight]
+    needs: [pre-flight, cicd-unit-tests-vllm]
     runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}
     name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
     environment: nemo-ci
@@ -313,7 +240,7 @@ jobs:
           runner: ${{ needs.pre-flight.outputs.runner_prefix }}
 
   cicd-e2e-tests-inframework:
-    needs: [pre-flight, cicd-unit-tests-trtllm, cicd-unit-tests-vllm]
+    needs: [pre-flight, cicd-unit-tests-vllm]
     runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}
     name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
     environment: nemo-ci
@@ -343,9 +270,7 @@ jobs:
   Nemo_CICD_Test:
     needs:
       - pre-flight
-      - cicd-unit-tests-trtllm
       - cicd-unit-tests-vllm
-      - cicd-e2e-tests-trtllm
       - cicd-e2e-tests-vllm
       - cicd-e2e-tests-inframework
     if: |

CONTRIBUTING.md

@@ -15,13 +15,9 @@ uv sync --only-group build  # Installs build dependencies required by Transforme
 uv sync
 ```
 
-On a machine with CUDA, you can additionally sync TRT-LLM or vLLM:
+On a machine with CUDA, you can additionally sync vLLM:
 
 ```bash
-# TRT-LLM
-uv sync --extra te --extra trtllm
-
-# or vLLM
 uv sync --extra te --extra vllm
 ```
 
@@ -35,7 +31,7 @@ New required dependencies can be added by `uv add $DEPENDENCY`.
 New optional dependencies can be added by `uv add --optional --extra $EXTRA $DEPENDENCY`.
 
 `EXTRA` refers to the subgroup of extra-dependencies to which you're adding the new dependency.
-Example: For adding a TRT-LLM specific dependency, run `uv add --optional --extra trtllm $DEPENDENCY`.
+Example: For adding a vLLM specific dependency, run `uv add --optional --extra vllm $DEPENDENCY`.
 
 Alternatively, the `pyproject.toml` file can also be modified directly.
 

README.md

@@ -21,7 +21,7 @@
 
 </div>
 
-The **Export-Deploy library ("NeMo Export-Deploy")** provides tools and APIs for exporting and deploying NeMo and 🤗Hugging Face models to production environments. It supports various deployment paths including TensorRT, TensorRT-LLM, and vLLM deployment through NVIDIA Triton Inference Server and Ray Serve.
+The **Export-Deploy library ("NeMo Export-Deploy")** provides tools and APIs for exporting and deploying NeMo and 🤗Hugging Face models to production environments. It supports various deployment paths including TensorRT and vLLM deployment through NVIDIA Triton Inference Server and Ray Serve.
 
 ![image](docs/NeMo_Repo_Overview_ExportDeploy.png)
 
@@ -32,7 +32,7 @@ The **Export-Deploy library ("NeMo Export-Deploy")** provides tools and APIs for
 ## 🚀 Key Features
 
 - Support for Large Language Models (LLMs) and Multimodal Models (MMs)
-- Export Megatron-Brdige and Hugging Face models to optimized inference formats including TensorRT-LLM and vLLM
+- Export Megatron-Brdige and Hugging Face models to optimized inference formats including vLLM
 - Deploy Megatron-Brdige and Hugging Face models using Ray Serve or NVIDIA Triton Inference Server
 - Multi-GPU and distributed inference capabilities
 - Multi-instance deployment options
@@ -41,11 +41,11 @@ The **Export-Deploy library ("NeMo Export-Deploy")** provides tools and APIs for
 
 ### Model Export Capabilities
 
-| Model / Checkpoint                                                                              | TensorRT-LLM                                   | vLLM      | ONNX                        | TensorRT               |
-|-------------------------------------------------------------------------------------------------|:----------------------------------------------:|:---------:|:--------------------------:|:----------------------:|
-| [Hugging Face](https://huggingface.co/docs/transformers/en/index)                               | bf16                                           | bf16      | N/A                      | N/A                    |
-| [NIM Embedding](https://docs.nvidia.com/nim/nemo-retriever/text-embedding/latest/overview.html) | N/A                                            | N/A       | bf16, fp8, int8 (PTQ)      | bf16, fp8, int8 (PTQ)  |
-| [NIM Reranking](https://docs.nvidia.com/nim/nemo-retriever/text-reranking/latest/overview.html) | N/A                                            | N/A       | Coming Soon                | Coming Soon            |
+| Model / Checkpoint                                                                              | vLLM      | ONNX                        | TensorRT               |
+|-------------------------------------------------------------------------------------------------|:---------:|:--------------------------:|:----------------------:|
+| [Hugging Face](https://huggingface.co/docs/transformers/en/index)                               | bf16      | N/A                      | N/A                    |
+| [NIM Embedding](https://docs.nvidia.com/nim/nemo-retriever/text-embedding/latest/overview.html) | N/A       | bf16, fp8, int8 (PTQ)      | bf16, fp8, int8 (PTQ)  |
+| [NIM Reranking](https://docs.nvidia.com/nim/nemo-retriever/text-reranking/latest/overview.html) | N/A       | Coming Soon                | Coming Soon            |
 
 The support matrix above outlines the export capabilities for each model or checkpoint, including the supported precision options across various inference-optimized libraries. The export module enables exporting models that have been quantized using post-training quantization (PTQ) with the [TensorRT Model Optimizer](https://github.com/NVIDIA/TensorRT-Model-Optimizer) library, as shown above. Models trained with low precision or quantization-aware training are also supported, as indicated in the table.
 
@@ -59,16 +59,15 @@ Please note that not all large language models (LLMs) and multimodal models (MMs
 |-------------------------------------------------------------------------------------------|------------------------------------------|-------------------------|
 | [Megatron-LM](https://github.com/NVIDIA/Megatron-LM)                                      | Limited                                  | Limited              |
 | [Hugging Face](https://huggingface.co/docs/transformers/en/index)                         | Single-Node Multi-GPU,<br>Multi-instance | Single-Node Multi-GPU   |
-| [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM)                                    | Single-Node Multi-GPU,<br>Multi-instance | Multi-Node Multi-GPU    |
 | [vLLM](https://github.com/vllm-project/vllm)                                              | N/A                                      | Single-Node Multi-GPU   |
 
 The support matrix above outlines the available deployment options for each model or checkpoint, highlighting multi-node and multi-GPU support where applicable. For comprehensive details, please refer to the [documentation](https://docs.nvidia.com/nemo/export-deploy/latest/index.html).
 
 Refer to the table below for an overview of optimized inference and deployment support for NeMo Framework and Hugging Face models with Triton Inference Server.
 
-| Model / Checkpoint           | TensorRT-LLM + Triton Inference Server | vLLM + Triton Inference Server | Direct Triton Inference Server |
-|------------------------------|:--------------------------------------:|:------------------------------:|:------------------------------:|
-| Hugging Face                 | &#x2611;                              | &#x2611;                      | &#x2611;                      |
+| Model / Checkpoint           | vLLM + Triton Inference Server | Direct Triton Inference Server |
+|------------------------------|:------------------------------:|:------------------------------:|
+| Hugging Face                 | &#x2611;                      | &#x2611;                      |
 
 ## 🔧 Install
 
@@ -78,7 +77,7 @@ For quick exploration of NeMo Export-Deploy, we recommend installing our pip pac
 pip install nemo-export-deploy
 ```
 
-This installation comes without extra dependencies like [TransformerEngine](https://github.com/NVIDIA/TransformerEngine/), [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) or [vLLM](https://github.com/vllm-project/vllm). The installation serves for navigating around and for exploring the project.
+This installation comes without extra dependencies like [TransformerEngine](https://github.com/NVIDIA/TransformerEngine/) or [vLLM](https://github.com/vllm-project/vllm). The installation serves for navigating around and for exploring the project.
 
 For a feature-complete install, please refer to the following sections.
 
@@ -95,7 +94,7 @@ docker run --rm -it -w /workdir -v $(pwd):/workdir \
 
 ### Build with Dockerfile
 
-For containerized development, use our Dockerfile for building your own container. There are three flavors: `INFERENCE_FRAMEWORK=inframework`, `INFERENCE_FRAMEWORK=trtllm` and `INFERENCE_FRAMEWORK=vllm`:
+For containerized development, use our Dockerfile for building your own container. There are two flavors: `INFERENCE_FRAMEWORK=inframework` and `INFERENCE_FRAMEWORK=vllm`:
 
 ```bash
 docker build \
@@ -116,7 +115,7 @@ docker run --rm -it -w /workdir -v $(pwd):/workdir \
 
 ### Install from Source
 
-For complete feature coverage, we recommend to install [TransformerEngine](https://github.com/NVIDIA/TransformerEngine/?tab=readme-ov-file#pip-installation) and additionally either [TensorRT-LLM](https://nvidia.github.io/TensorRT-LLM/0.20.0/installation/linux.html) or [vLLM](https://docs.vllm.ai/en/latest/getting_started/installation/gpu.html#pre-built-wheels).
+For complete feature coverage, we recommend to install [TransformerEngine](https://github.com/NVIDIA/TransformerEngine/?tab=readme-ov-file#pip-installation) and additionally [vLLM](https://docs.vllm.ai/en/latest/getting_started/installation/gpu.html#pre-built-wheels).
 
 #### Recommended Requirements
 
@@ -141,21 +140,6 @@ cd Export-Deploy/
 pip install --no-build-isolation .
 ```
 
-#### Install TransformerEngine + TensorRT-LLM
-
-For highly optimized TransformerEngine path with TensorRT-LLM backend, please make sure to install the following prerequisites first:
-
-```bash
-sudo apt-get -y install libopenmpi-dev  # Required for TensorRT-LLM
-pip install torch==2.7.0 setuptools pybind11 wheel_stub  # Required for TE
-```
-
-Now proceed with the main installation:
-
-```bash
-pip install --no-build-isolation .[trtllm]
-```
-
 #### Install TransformerEngine + vLLM
 
 For highly optimized TransformerEngine path with vLLM backend, please make sure to install the following prerequisites first:

docker/common/install.sh

@@ -22,7 +22,7 @@ while [[ $# -gt 0 ]]; do
         ;;
     *)
         echo "Unknown option: $1"
-        echo "Usage: $0 --base-image {pytorch|ubuntu} --inference-framework {trtllm|vllm|inframework|trt-onnx} [--python-version] [--use-uv]"
+        echo "Usage: $0 --base-image {pytorch|ubuntu} --inference-framework {vllm|inframework|trt-onnx} [--python-version] [--use-uv]"
         exit 1
         ;;
     esac
@@ -39,7 +39,7 @@ fi
 # Validate base image argument
 if [[ -z "${BASE_IMAGE:-}" || -z "${INFERENCE_FRAMEWORK:-}" ]]; then
     echo "Error: --base-image and --inference-framework arguments are required"
-    echo "Usage: $0 --base-image {pytorch|ubuntu} --inference-framework {trtllm|vllm|inframework}"
+    echo "Usage: $0 --base-image {pytorch|ubuntu} --inference-framework {vllm|inframework|trt-onnx}"
     exit 1
 fi
 
@@ -49,9 +49,9 @@ if [[ "$BASE_IMAGE" != "pytorch" && "$BASE_IMAGE" != "ubuntu" ]]; then
     exit 1
 fi
 
-if [[ "$INFERENCE_FRAMEWORK" != "trtllm" && "$INFERENCE_FRAMEWORK" != "vllm" && "$INFERENCE_FRAMEWORK" != "inframework" && "$INFERENCE_FRAMEWORK" != "trt-onnx" ]]; then
-    echo "Error: --inference-framework must be either 'trtllm' or 'vllm' or 'inframework' or 'trt-onnx'"
-    echo "Usage: $0 --inference-framework {trtllm|vllm|inframework}"
+if [[ "$INFERENCE_FRAMEWORK" != "vllm" && "$INFERENCE_FRAMEWORK" != "inframework" && "$INFERENCE_FRAMEWORK" != "trt-onnx" ]]; then
+    echo "Error: --inference-framework must be either 'vllm' or 'inframework' or 'trt-onnx'"
+    echo "Usage: $0 --inference-framework {vllm|inframework|trt-onnx}"
     exit 1
 fi
 
@@ -133,7 +133,6 @@ main() {
                 "--no-install-package" "nvidia-cusparse"
                 "--no-install-package" "nvidia-cusparselt-cu13"
                 "--no-install-package" "nvidia-nccl-cu13"
-                "--no-install-package" "tensorrt-llm"
                 "--no-install-package" "vllm"
             )
         else

nemo_deploy/deploy_pytriton.py

@@ -30,39 +30,7 @@
 
 
 class DeployPyTriton(DeployBase):
-    """Deploys any models to Triton Inference Server that implements ITritonDeployable interface in nemo_deploy.
-
-    Example:
-        from nemo_deploy import DeployPyTriton, NemoQueryLLM
-        from nemo_export.tensorrt_llm import TensorRTLLM
-
-        trt_llm_exporter = TensorRTLLM(model_dir="/path/for/model/files")
-        trt_llm_exporter.export(
-            nemo_checkpoint_path="/path/for/nemo/checkpoint",
-            model_type="llama",
-            tensor_parallelism_size=1,
-        )
-
-        nm = DeployPyTriton(model=trt_llm_exporter, triton_model_name="model_name", http_port=8000)
-        nm.deploy()
-        nm.run()
-        nq = NemoQueryLLM(url="localhost", model_name="model_name")
-
-        prompts = ["hello, testing GPT inference", "another GPT inference test?"]
-        output = nq.query_llm(prompts=prompts, max_output_len=100)
-        print("prompts: ", prompts)
-        print("")
-        print("output: ", output)
-        print("")
-
-        prompts = ["Give me some info about Paris", "Do you think Londan is a good city to visit?", "What do you think about Rome?"]
-        output = nq.query_llm(prompts=prompts, max_output_len=250)
-        print("prompts: ", prompts)
-        print("")
-        print("output: ", output)
-        print("")
-
-    """
+    """Deploys any models to Triton Inference Server that implements ITritonDeployable interface in nemo_deploy."""
 
     def __init__(
         self,