From 58080295b5312c768b14d5ee3e796ce9eca925bb Mon Sep 17 00:00:00 2001 From: Ayantika Nandi Date: Tue, 1 Sep 2026 18:59:18 -0400 Subject: [PATCH 1/2] Add framework-parity harness + CrewAI example Backs the "framework-agnostic" claim with an actual nightly check instead of a static example list: - scripts/framework_parity/: installs each framework example's own package into an isolated `uv` venv, runs the real `nat validate` CLI against it (structural tier, no credentials needed), and for entries with a single-credential canonical config, actually runs the workflow and inspects the IntermediateStepManager event stream to assert a real LLM span with non-zero token usage was emitted (live tier). Two-tier design and its rationale documented in scripts/framework_parity/README.md. - .github/workflows/framework_parity.yml: runs the matrix nightly (and on PRs touching these paths, structural-only), publishes results as artifacts, and regenerates the badge table in examples/frameworks/README.md on the scheduled/manual run only. - examples/frameworks/crewai_demo/: new CrewAI example, structured as the direct counterpart to semantic_kernel_demo (same two tools, same task, different framework) so the two are comparable side by side. Verified with `nat validate` and a live `nat run` (auth-failing without a real key, which is the expected/correct failure point). ADK already has a full example (adk_demo) contrary to the original brief; left as-is and added to the parity matrix rather than duplicated. Signed-off-by: Ayantika Nandi --- .github/workflows/framework_parity.yml | 123 +++++++++++++ .gitignore | 3 + examples/frameworks/README.md | 50 ++++++ examples/frameworks/crewai_demo/README.md | 67 +++++++ .../frameworks/crewai_demo/pyproject.toml | 44 +++++ .../src/nat_crewai_demo/__init__.py | 14 ++ .../src/nat_crewai_demo/configs/config.yml | 58 ++++++ .../nat_crewai_demo/data/hotel_prices.json | 3 + .../nat_crewai_demo/data/local_events.json | 3 + .../src/nat_crewai_demo/hotel_price_tool.py | 77 ++++++++ .../src/nat_crewai_demo/local_events_tool.py | 51 ++++++ .../src/nat_crewai_demo/register.py | 137 +++++++++++++++ .../tests/test_crewai_demo_workflow.py | 34 ++++ scripts/framework_parity/README.md | 79 +++++++++ scripts/framework_parity/__init__.py | 14 ++ scripts/framework_parity/matrix.py | 144 +++++++++++++++ scripts/framework_parity/orchestrate.py | 128 ++++++++++++++ scripts/framework_parity/render_table.py | 107 ++++++++++++ scripts/framework_parity/run_check.py | 165 ++++++++++++++++++ 19 files changed, 1301 insertions(+) create mode 100644 .github/workflows/framework_parity.yml create mode 100644 examples/frameworks/README.md create mode 100644 examples/frameworks/crewai_demo/README.md create mode 100644 examples/frameworks/crewai_demo/pyproject.toml create mode 100644 examples/frameworks/crewai_demo/src/nat_crewai_demo/__init__.py create mode 100644 examples/frameworks/crewai_demo/src/nat_crewai_demo/configs/config.yml create mode 100644 examples/frameworks/crewai_demo/src/nat_crewai_demo/data/hotel_prices.json create mode 100644 examples/frameworks/crewai_demo/src/nat_crewai_demo/data/local_events.json create mode 100644 examples/frameworks/crewai_demo/src/nat_crewai_demo/hotel_price_tool.py create mode 100644 examples/frameworks/crewai_demo/src/nat_crewai_demo/local_events_tool.py create mode 100644 examples/frameworks/crewai_demo/src/nat_crewai_demo/register.py create mode 100644 examples/frameworks/crewai_demo/tests/test_crewai_demo_workflow.py create mode 100644 scripts/framework_parity/README.md create mode 100644 scripts/framework_parity/__init__.py create mode 100644 scripts/framework_parity/matrix.py create mode 100644 scripts/framework_parity/orchestrate.py create mode 100644 scripts/framework_parity/render_table.py create mode 100644 scripts/framework_parity/run_check.py diff --git a/.github/workflows/framework_parity.yml b/.github/workflows/framework_parity.yml new file mode 100644 index 0000000000..5d3ae709b4 --- /dev/null +++ b/.github/workflows/framework_parity.yml @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Framework Parity +run-name: Framework Parity + +# NAT claims to work alongside every framework listed in scripts/framework_parity/matrix.py. +# This workflow is the proof: each one gets installed into its own isolated environment, +# nightly, and checked for real -- not just "the example folder still exists". +on: + schedule: + - cron: "0 9 * * *" # 09:00 UTC nightly + workflow_dispatch: {} + pull_request: + paths: + - "examples/frameworks/**" + - "scripts/framework_parity/**" + - ".github/workflows/framework_parity.yml" + +defaults: + run: + shell: bash + +permissions: + contents: read + +jobs: + check: + name: ${{ matrix.entry }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + entry: + - crewai + - semantic_kernel + - adk + - langchain_llama_index + - agno + - autogen + - strands + - haystack + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + + - name: Run framework-parity check for ${{ matrix.entry }} + # Only the scheduled nightly run and manual dispatch get the live-tier + # credentials; PRs from forks (including this harness's own PRs while it's + # under review) never see repo secrets, so they naturally exercise the + # structural tier only -- which needs no credentials at all. + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + MEM0_API_KEY: ${{ secrets.MEM0_API_KEY }} + run: | + LIVE_FLAG="" + if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + LIVE_FLAG="--live" + fi + python scripts/framework_parity/orchestrate.py --only ${{ matrix.entry }} $LIVE_FLAG + + - name: Upload result + if: always() + uses: actions/upload-artifact@v4 + with: + name: framework-parity-result-${{ matrix.entry }} + path: .framework_parity_results/${{ matrix.entry }}.json + retention-days: 90 + + publish-table: + name: Publish parity table + needs: check + if: always() + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Download all results + uses: actions/download-artifact@v4 + with: + pattern: framework-parity-result-* + path: .framework_parity_results + merge-multiple: true + + - name: Render badge table + run: | + python scripts/framework_parity/render_table.py \ + --results-dir .framework_parity_results \ + --readme examples/frameworks/README.md + + - name: Commit updated table + # Only push the regenerated table back on the nightly/manual run against this + # repo's own branches -- never from a pull_request event, so an external + # contributor's PR can't be used to trigger a commit back to the branch. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + if git diff --quiet -- examples/frameworks/README.md; then + echo "Table unchanged, nothing to commit." + exit 0 + fi + git add examples/frameworks/README.md + git commit -m "chore: update framework-parity table [skip ci]" + git push diff --git a/.gitignore b/.gitignore index 152e7c7536..dd4b1e838f 100644 --- a/.gitignore +++ b/.gitignore @@ -85,6 +85,9 @@ install_manifest.txt compile_commands.json CTestTestfile.cmake +### Framework parity harness ### +.framework_parity_results/ + ### Python ### # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/examples/frameworks/README.md b/examples/frameworks/README.md new file mode 100644 index 0000000000..36b23aa4ad --- /dev/null +++ b/examples/frameworks/README.md @@ -0,0 +1,50 @@ + + +# Framework Examples + +NeMo Agent Toolkit is framework-agnostic: it can wrap and orchestrate agents built with +LangChain, LlamaIndex, CrewAI, Semantic Kernel, Google ADK, AutoGen, Agno, Strands +Agents, and Haystack, among others. Each subdirectory here is a maintained, runnable +reference example for one of those frameworks. + +The table below is generated nightly by the [framework-parity +harness](../../.github/workflows/framework_parity.yml) +([source](../../scripts/framework_parity)), which installs each example's own package +into a fresh, isolated environment and validates it there — the same thing a new user +does when they follow one of the READMEs below. For frameworks with a single-credential +canonical config, it goes a step further and actually runs the workflow, then checks +the profiler's own event stream for a real LLM span with non-zero token usage, rather +than only checking that the YAML parses. + + +_Not yet generated. Run `python scripts/framework_parity/orchestrate.py` and then +`scripts/framework_parity/render_table.py` to populate this table._ + + + +## Examples + +- [`crewai_demo`](crewai_demo/README.md) — CrewAI +- [`semantic_kernel_demo`](semantic_kernel_demo/README.md) — Semantic Kernel +- [`adk_demo`](adk_demo/README.md) — Google ADK +- [`multi_frameworks`](multi_frameworks/README.md) — LangChain + LlamaIndex in one workflow +- [`agno_personal_finance`](agno_personal_finance/README.md) — Agno +- [`nat_autogen_demo`](nat_autogen_demo/README.md) — AutoGen +- [`strands_demo`](strands_demo/README.md) — Strands Agents +- [`haystack_deep_research_agent`](haystack_deep_research_agent/README.md) — Haystack +- [`auto_wrapper/langchain_deep_research`](auto_wrapper/langchain_deep_research/README.md) — wrapping an existing LangChain workflow without rewriting it diff --git a/examples/frameworks/crewai_demo/README.md b/examples/frameworks/crewai_demo/README.md new file mode 100644 index 0000000000..bec3bc21cb --- /dev/null +++ b/examples/frameworks/crewai_demo/README.md @@ -0,0 +1,67 @@ + + +# CrewAI Example + +**Complexity:** 🟢 Beginner + +A minimal example using CrewAI showcasing a multi-agent travel planning system where an Itinerary Expert plans activities, a Budget Advisor prices lodging and totals the trip cost, and a Summarizer compiles the final plan. This is the CrewAI counterpart to the [Semantic Kernel example](../semantic_kernel_demo/README.md) — same task, same two tools, different framework — intended to be run side by side with it as a framework-parity check. + +## Table of Contents + +- [Key Features](#key-features) +- [Installation and Setup](#installation-and-setup) + - [Install this Workflow](#install-this-workflow) + - [Set Up API Keys](#set-up-api-keys) +- [Run the Workflow](#run-the-workflow) + +## Key Features + +- **CrewAI Framework Integration:** Demonstrates NeMo Agent Toolkit support for CrewAI alongside other frameworks like LangChain/LangGraph and Semantic Kernel. +- **Multi-Agent Travel Planning:** Three role-based agents (Itinerary Expert, Budget Advisor, Summarizer) running as a sequential `Crew`, each contributing its own specialized `Task`. +- **Shared Tools Across Agents:** Both the itinerary and budget agents draw on the same two NAT-registered tools (`hotel_price`, `local_events`), demonstrating that a single tool implementation is reusable across every supported framework, not rewritten per integration. +- **Task Context Chaining:** The budget and summary tasks receive prior tasks' output via CrewAI's `context=[...]`, so later agents build on earlier agents' work rather than repeating it. + +## Installation and Setup + +If you have not already done so, follow the instructions in the [Install Guide](../../../docs/source/get-started/installation.md#install-from-source) to create the development environment and install NeMo Agent Toolkit. + +### Install this Workflow + +From the root directory of the NeMo Agent Toolkit library, run the following commands: + +```bash +uv pip install -e examples/frameworks/crewai_demo +``` + +### Set Up API Keys + +You need to set your OpenAI API key as an environment variable to access OpenAI AI services: + +```bash +export OPENAI_API_KEY= +``` + +## Run the Workflow + +```bash +nat run --config_file examples/frameworks/crewai_demo/configs/config.yml --input "Create a 3-day travel itinerary for Tokyo in April, covering hotels and activities within a USD 2000 budget." +``` + +**Expected Workflow Output** + +The workflow produces a day-by-day itinerary, a lodging + activity cost breakdown, and a total estimated cost, compiled by the Summarizer agent into a single final plan — structurally the same kind of output the Semantic Kernel example produces for the same prompt, which is the point: the two examples are meant to be comparable, not just individually correct. diff --git a/examples/frameworks/crewai_demo/pyproject.toml b/examples/frameworks/crewai_demo/pyproject.toml new file mode 100644 index 0000000000..12ef7eed0c --- /dev/null +++ b/examples/frameworks/crewai_demo/pyproject.toml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +build-backend = "setuptools.build_meta" +requires = ["setuptools>=64", "setuptools-scm>=8", "setuptools_dynamic_dependencies>=1.0.0"] + +[tool.setuptools_scm] +git_describe_command = "git describe --long --first-parent" +root = "../../.." + +[tool.setuptools.packages.find] +where = ["src"] + +[project] +name = "nat_crewai_demo" +dynamic = ["version", "dependencies"] +requires-python = ">=3.11,<3.14" +description = "CrewAI Example" +keywords = ["ai", "rag", "agents"] +classifiers = ["Programming Language :: Python"] + +[tool.setuptools_dynamic_dependencies] +dependencies = [ + "nvidia-nat[crewai,test] == {version}", +] + +[tool.uv.sources] +nvidia-nat = { path = "../../..", editable = true } + +[project.entry-points.'nat.components'] +nat_crewai_demo = "nat_crewai_demo.register" diff --git a/examples/frameworks/crewai_demo/src/nat_crewai_demo/__init__.py b/examples/frameworks/crewai_demo/src/nat_crewai_demo/__init__.py new file mode 100644 index 0000000000..3bcc1c39bb --- /dev/null +++ b/examples/frameworks/crewai_demo/src/nat_crewai_demo/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/examples/frameworks/crewai_demo/src/nat_crewai_demo/configs/config.yml b/examples/frameworks/crewai_demo/src/nat_crewai_demo/configs/config.yml new file mode 100644 index 0000000000..08203ab3ed --- /dev/null +++ b/examples/frameworks/crewai_demo/src/nat_crewai_demo/configs/config.yml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +general: + front_end: + _type: fastapi + cors: + allow_origins: ['*'] + step_adaptor: + mode: default + +functions: + hotel_price: + _type: hotel_price + local_events: + _type: local_events + +llms: + openai_llm: + _type: openai + model_name: gpt-5.4-mini + temperature: 0.0 + +workflow: + _type: crewai + tool_names: [hotel_price, local_events] + llm_name: openai_llm + verbose: true + itinerary_expert_role: Itinerary Expert + itinerary_expert_goal: Build detailed, well-paced day-by-day travel itineraries. + itinerary_expert_backstory: | + You specialize in attractions, local events, and trip logistics. You focus purely on what to do and when; + you leave hotel pricing and overall budgeting to the Budget Advisor. + + budget_advisor_role: Budget Advisor + budget_advisor_goal: Produce an accurate, itemized cost estimate for the whole trip. + budget_advisor_backstory: | + You specialize in estimating and optimizing travel costs. You use the hotel price tool to find lodging costs + and combine them with the itinerary's activity costs into one clear total. + + summarize_agent_role: Summarizer + summarize_agent_goal: Turn the itinerary and the budget into one polished, user-facing travel plan. + summarize_agent_backstory: | + You specialize in taking detailed notes from other specialists and compiling them into a clear, + well-structured final document, without duplicating information or losing any details. diff --git a/examples/frameworks/crewai_demo/src/nat_crewai_demo/data/hotel_prices.json b/examples/frameworks/crewai_demo/src/nat_crewai_demo/data/hotel_prices.json new file mode 100644 index 0000000000..73b7cd05b3 --- /dev/null +++ b/examples/frameworks/crewai_demo/src/nat_crewai_demo/data/hotel_prices.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bdc639461909b2c298528db842d8bf148f1fdd60dab55785e2ca009df6f6c77 +size 280 diff --git a/examples/frameworks/crewai_demo/src/nat_crewai_demo/data/local_events.json b/examples/frameworks/crewai_demo/src/nat_crewai_demo/data/local_events.json new file mode 100644 index 0000000000..cb5d3f5ab1 --- /dev/null +++ b/examples/frameworks/crewai_demo/src/nat_crewai_demo/data/local_events.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9fe6cd6942cfb26412ca3e176e85cd9760a0339763f0480f69597cb0bcbc44d +size 422 diff --git a/examples/frameworks/crewai_demo/src/nat_crewai_demo/hotel_price_tool.py b/examples/frameworks/crewai_demo/src/nat_crewai_demo/hotel_price_tool.py new file mode 100644 index 0000000000..01b9d4dea9 --- /dev/null +++ b/examples/frameworks/crewai_demo/src/nat_crewai_demo/hotel_price_tool.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pydantic import BaseModel + +from nat.builder.builder import Builder +from nat.builder.function_info import FunctionInfo +from nat.cli.register_workflow import register_function +from nat.data_models.function import FunctionBaseConfig + + +class HotelPriceToolConfig(FunctionBaseConfig, name="hotel_price"): + data_path: str = "examples/frameworks/crewai_demo/src/nat_crewai_demo/data/hotel_prices.json" + date_format: str = "%Y-%m-%d" + + +class HotelOffer(BaseModel): + name: str + price_per_night: float + total_price: float + city: str + checkin: str + checkout: str + + +class HotelOffersResponse(BaseModel): + offers: list[HotelOffer] + + +@register_function(config_type=HotelPriceToolConfig) +async def hotel_price(tool_config: HotelPriceToolConfig, builder: Builder): + + import json + + with open(tool_config.data_path, encoding='utf-8') as f: + hotel_prices = json.load(f) + + search_date_format = tool_config.date_format + + async def _get_hotel_price(city: str, checkin: str, checkout: str) -> HotelOffersResponse: + from datetime import datetime + + base_hotels = hotel_prices + + checkin_dt = datetime.strptime(checkin, search_date_format) + checkout_dt = datetime.strptime(checkout, search_date_format) + nights = (checkout_dt - checkin_dt).days + + offers = [] + for hotel in base_hotels: + total_price = hotel["price_per_night"] * nights + offers.append( + HotelOffer(name=hotel["name"], + price_per_night=hotel["price_per_night"], + total_price=total_price, + city=city, + checkin=checkin, + checkout=checkout)) + + return HotelOffersResponse(offers=offers) + + yield FunctionInfo.from_fn( + _get_hotel_price, + description=( + "This tool returns a list of hotels and nightly prices for the given city and checkin/checkout dates.")) diff --git a/examples/frameworks/crewai_demo/src/nat_crewai_demo/local_events_tool.py b/examples/frameworks/crewai_demo/src/nat_crewai_demo/local_events_tool.py new file mode 100644 index 0000000000..04d31d7de8 --- /dev/null +++ b/examples/frameworks/crewai_demo/src/nat_crewai_demo/local_events_tool.py @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pydantic import BaseModel + +from nat.builder.builder import Builder +from nat.builder.function_info import FunctionInfo +from nat.cli.register_workflow import register_function +from nat.data_models.function import FunctionBaseConfig + + +class LocalEvent(BaseModel): + name: str + cost: float + city: str + + +class LocalEventsResponse(BaseModel): + events: list[LocalEvent] + + +class LocalEventsToolConfig(FunctionBaseConfig, name="local_events"): + data_path: str = "examples/frameworks/crewai_demo/src/nat_crewai_demo/data/local_events.json" + + +@register_function(config_type=LocalEventsToolConfig) +async def local_events(tool_config: LocalEventsToolConfig, builder: Builder): + + import json + + with open(tool_config.data_path) as f: + events = LocalEventsResponse.model_validate({"events": json.load(f)}).events + + async def _local_events(city: str) -> LocalEventsResponse: + return LocalEventsResponse(events=[e for e in events if e.city == city]) + + yield FunctionInfo.from_fn( + _local_events, + description=("This tool can provide information and cost of local events and activities in a city")) diff --git a/examples/frameworks/crewai_demo/src/nat_crewai_demo/register.py b/examples/frameworks/crewai_demo/src/nat_crewai_demo/register.py new file mode 100644 index 0000000000..c3ec4b08c5 --- /dev/null +++ b/examples/frameworks/crewai_demo/src/nat_crewai_demo/register.py @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from pydantic import Field + +from nat.builder.builder import Builder +from nat.builder.framework_enum import LLMFrameworkEnum +from nat.builder.function_info import FunctionInfo +from nat.cli.register_workflow import register_function +from nat.data_models.component_ref import FunctionRef +from nat.data_models.component_ref import LLMRef +from nat.data_models.function import FunctionBaseConfig + +from . import hotel_price_tool # noqa: F401, pylint: disable=unused-import +from . import local_events_tool # noqa: F401, pylint: disable=unused-import + +logger = logging.getLogger(__name__) + + +class CrewAITravelPlanningWorkflowConfig(FunctionBaseConfig, name="crewai"): + """ + Structural companion to the semantic_kernel_demo example: the same travel-planning task (an itinerary expert, + a budget advisor, and a summarizer, each backed by the same two tools), reimplemented against CrewAI instead of + Semantic Kernel, so the two examples can be run through a framework-parity harness as comparable canonical + workflows. + """ + tool_names: list[FunctionRef] = Field(default_factory=list, + description="The list of tools to provide to the crew.") + llm_name: LLMRef = Field(description="The LLM model to use for every agent in the crew.") + verbose: bool = Field(default=False, description="Set the verbosity of CrewAI's logging.") + itinerary_expert_role: str = Field(description="The role of the itinerary expert agent.") + itinerary_expert_goal: str = Field(description="The goal of the itinerary expert agent.") + itinerary_expert_backstory: str = Field(description="The backstory of the itinerary expert agent.") + budget_advisor_role: str = Field(description="The role of the budget advisor agent.") + budget_advisor_goal: str = Field(description="The goal of the budget advisor agent.") + budget_advisor_backstory: str = Field(description="The backstory of the budget advisor agent.") + summarize_agent_role: str = Field(description="The role of the summarizer agent.") + summarize_agent_goal: str = Field(description="The goal of the summarizer agent.") + summarize_agent_backstory: str = Field(description="The backstory of the summarizer agent.") + + +@register_function(config_type=CrewAITravelPlanningWorkflowConfig, framework_wrappers=[LLMFrameworkEnum.CREWAI]) +async def crewai_travel_planning_workflow(config: CrewAITravelPlanningWorkflowConfig, builder: Builder): + + import asyncio + + from crewai import Agent + from crewai import Crew + from crewai import Process + from crewai import Task + + llm = await builder.get_llm(config.llm_name, wrapper_type=LLMFrameworkEnum.CREWAI) + tools = await builder.get_tools(config.tool_names, wrapper_type=LLMFrameworkEnum.CREWAI) + + itinerary_expert = Agent( + role=config.itinerary_expert_role, + goal=config.itinerary_expert_goal, + backstory=config.itinerary_expert_backstory, + tools=tools, + llm=llm, + verbose=config.verbose, + ) + + budget_advisor = Agent( + role=config.budget_advisor_role, + goal=config.budget_advisor_goal, + backstory=config.budget_advisor_backstory, + tools=tools, + llm=llm, + verbose=config.verbose, + ) + + summarizer = Agent( + role=config.summarize_agent_role, + goal=config.summarize_agent_goal, + backstory=config.summarize_agent_backstory, + llm=llm, + verbose=config.verbose, + ) + + itinerary_task = Task( + description=("Using the local events tool, put together a day-by-day itinerary for this trip: {input}. " + "Focus on attractions and activities and their cost; leave hotel pricing to the Budget " + "Advisor."), + expected_output="A day-by-day itinerary listing activities and their individual costs.", + agent=itinerary_expert, + ) + + budget_task = Task( + description=("Using the hotel price tool, estimate the total lodging cost for this trip: {input}. " + "Combine it with the itinerary's activity costs to produce a total estimated trip cost."), + expected_output="A cost breakdown covering lodging and activities, plus a total estimated cost.", + agent=budget_advisor, + context=[itinerary_task], + ) + + summary_task = Task( + description=("Compile the itinerary and the budget breakdown into a single, clear, well-structured travel " + "plan for: {input}."), + expected_output="A final travel plan with sections for the itinerary and the cost breakdown.", + agent=summarizer, + context=[itinerary_task, budget_task], + ) + + crew = Crew( + agents=[itinerary_expert, budget_advisor, summarizer], + tasks=[itinerary_task, budget_task, summary_task], + process=Process.sequential, + verbose=config.verbose, + ) + + async def _response_fn(input_message: str) -> str: + # CrewAI's kickoff() is synchronous; run it off the event loop thread so this + # coroutine doesn't block NAT's async runtime while the crew executes. + result = await asyncio.to_thread(crew.kickoff, inputs={"input": input_message}) + return str(result) + + try: + yield FunctionInfo.from_fn(_response_fn, description="Plans a trip using a crew of specialized agents.") + except GeneratorExit: + logger.exception("Exited early!") + finally: + logger.debug("Cleaning up") diff --git a/examples/frameworks/crewai_demo/tests/test_crewai_demo_workflow.py b/examples/frameworks/crewai_demo/tests/test_crewai_demo_workflow.py new file mode 100644 index 0000000000..eaef9a114d --- /dev/null +++ b/examples/frameworks/crewai_demo/tests/test_crewai_demo_workflow.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +import pytest + + +@pytest.mark.usefixtures("openai_api_key") +@pytest.mark.integration +async def test_full_workflow(): + from nat.test.utils import locate_example_config + from nat.test.utils import run_workflow + from nat_crewai_demo.register import CrewAITravelPlanningWorkflowConfig + + config_file: Path = locate_example_config(CrewAITravelPlanningWorkflowConfig) + + await run_workflow( + config_file=config_file, + question=("Create a 3-day travel itinerary for Tokyo in April, covering hotels and activities within a USD " + "2000 budget."), + expected_answer="budget") diff --git a/scripts/framework_parity/README.md b/scripts/framework_parity/README.md new file mode 100644 index 0000000000..ad843b2282 --- /dev/null +++ b/scripts/framework_parity/README.md @@ -0,0 +1,79 @@ + + +# Framework Parity Harness + +Backs the badge table in [`examples/frameworks/README.md`](../../examples/frameworks/README.md). +Runs nightly via [`.github/workflows/framework_parity.yml`](../../.github/workflows/framework_parity.yml). + +## What it actually checks + +Each entry in [`matrix.py`](matrix.py) gets **two** independent tiers, run in its own +throwaway `uv` venv so one framework's dependencies can never leak into another's +result: + +1. **Structural** (always runs, no credentials): `uv pip install -e ` into a + fresh venv, then the real `nat validate` CLI against the example's shipped config + file. Confirms the example installs the way its own README says to, and that its + config/registration graph is valid. + +2. **Live** (only for entries with `required_live_env` set, and only when those env + vars are actually present): loads the workflow for real, runs its canonical + question through the real framework and a real LLM, subscribes to the + `IntermediateStepManager` event stream, and asserts a `WORKFLOW_START`/`WORKFLOW_END` + pair and at least one `LLM_END` span with non-zero `usage_info.token_usage.total_tokens` + were actually emitted. This is the part that backs the "framework-agnostic" claim -- + a config that parses is not proof an integration executes. + +## Why most rows are structural-only today + +`agno_personal_finance`, `nat_autogen_demo`, `strands_demo`, `multi_frameworks`, and +`haystack_deep_research_agent` each reach across several external services in their +*normal, realistic* configuration (NIM, SerpAPI, Tavily, AWS Bedrock, MCP servers). +Wiring a live check for those means either provisioning that whole service list as CI +secrets, or forking a second "minimal" config that no longer matches what a user +actually installs -- both worse than being honest that the live tier isn't wired yet. +`crewai_demo`, `semantic_kernel_demo`, and `adk_demo`'s `config_oai.yml` were verified +by hand to build and run to the point of a real (auth-failing, without a key) LLM call +using only `OPENAI_API_KEY` (`semantic_kernel_demo` additionally needs `MEM0_API_KEY` +for its memory tools), so those three are live-wired now. + +Notably, NAT's own built-in `nat_test_llm` mock provider +(`nvidia_nat_test/src/nat/test/llm.py`) was **not** used for the live tier: its +per-framework client classes are bare stand-ins that don't inherit from each +framework's real base LLM class, so they don't go through that framework's real +callback/instrumentation path -- using it would let this harness pass without ever +proving the profiler's span/token wiring works. + +## Running it locally + +```bash +# One framework: +python scripts/framework_parity/orchestrate.py --only crewai --live + +# The whole matrix: +python scripts/framework_parity/orchestrate.py --live + +# Regenerate the table from whatever's in .framework_parity_results/: +python scripts/framework_parity/render_table.py \ + --results-dir .framework_parity_results \ + --readme examples/frameworks/README.md +``` + +`--live` is a no-op for an entry unless every env var in its `required_live_env` is +already set -- it degrades to the structural tier and reports `live_status: skipped` +rather than failing. diff --git a/scripts/framework_parity/__init__.py b/scripts/framework_parity/__init__.py new file mode 100644 index 0000000000..3bcc1c39bb --- /dev/null +++ b/scripts/framework_parity/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/scripts/framework_parity/matrix.py b/scripts/framework_parity/matrix.py new file mode 100644 index 0000000000..647bb358f9 --- /dev/null +++ b/scripts/framework_parity/matrix.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +The framework-parity matrix. + +NAT claims to be framework-agnostic: it ships a maintained reference example for each +of its supported agent frameworks. This module is the single source of truth for what +"maintained" is actually checked against for each one: + + - ``structural``: install the example's own package (with its own declared extras, + exactly as a user would) into an isolated environment and run the real ``nat + validate`` CLI against its shipped config file. This requires no external + credentials and runs on every scheduled harness invocation. + + - ``live``: in addition to the structural check, actually execute the example's + canonical question through the real framework and a real LLM, then inspect the + emitted IntermediateStep event stream to confirm the profiler attached at least + one LLM span with non-zero token usage. This is the one that actually backs the + "framework-agnostic" claim -- a config that merely parses is not proof a + framework integration works end to end. + +Only entries whose `required_live_env` is fully satisfiable by a single, cheap +credential are wired for the live tier today. Several real examples in this repo +(agno_personal_finance, nat_autogen_demo, strands_demo, multi_frameworks, +haystack_deep_research_agent) reach across multiple external services (NIM, SerpAPI, +Tavily, AWS Bedrock, MCP servers) as part of their normal, realistic configuration -- +wiring live checks for those means either provisioning that whole service list as CI +secrets, or forking a second "minimal" config that no longer matches what a user +actually installs. Neither is worth doing to turn a yellow cell green; they run the +structural tier only, honestly labeled, until that changes. +""" + +import dataclasses + + +@dataclasses.dataclass(frozen=True) +class FrameworkEntry: + # Short key, matches the GitHub Actions matrix value and the result filename. + key: str + # Human-readable name for the badge table. + display_name: str + # Path to the example's directory, relative to the repo root. + example_dir: str + # Path to the config file to validate/run, relative to the repo root. + config_file: str + # PyPI distribution name of the underlying framework, for `importlib.metadata` + # version reporting in the badge table. + framework_package: str + # Env vars that must ALL be set for the live tier to run. If any are missing, the + # entry falls back to the structural-only tier and is reported as "skipped (live)" + # rather than failed. + required_live_env: tuple[str, ...] + # The question to run through the workflow during the live tier. + question: str = "Plan a 1-day trip to Tokyo, covering one hotel and one activity." + + +FRAMEWORK_MATRIX: tuple[FrameworkEntry, ...] = ( + FrameworkEntry( + key="crewai", + display_name="CrewAI", + example_dir="examples/frameworks/crewai_demo", + config_file="examples/frameworks/crewai_demo/src/nat_crewai_demo/configs/config.yml", + framework_package="crewai", + required_live_env=("OPENAI_API_KEY", ), + ), + FrameworkEntry( + key="semantic_kernel", + display_name="Semantic Kernel", + example_dir="examples/frameworks/semantic_kernel_demo", + config_file="examples/frameworks/semantic_kernel_demo/src/nat_semantic_kernel_demo/configs/config.yml", + framework_package="semantic-kernel", + required_live_env=("OPENAI_API_KEY", "MEM0_API_KEY"), + ), + FrameworkEntry( + key="adk", + display_name="Google ADK", + example_dir="examples/frameworks/adk_demo", + config_file="examples/frameworks/adk_demo/src/nat_adk_demo/configs/config_oai.yml", + framework_package="google-adk", + required_live_env=("OPENAI_API_KEY", ), + question="What is the weather in Tokyo right now?", + ), + FrameworkEntry( + key="langchain_llama_index", + display_name="LangChain + LlamaIndex", + example_dir="examples/frameworks/multi_frameworks", + config_file="examples/frameworks/multi_frameworks/src/nat_multi_frameworks/configs/config.yml", + framework_package="langchain", + required_live_env=(), # needs NVIDIA_API_KEY + TAVILY_API_KEY; structural only for now. + ), + FrameworkEntry( + key="agno", + display_name="Agno", + example_dir="examples/frameworks/agno_personal_finance", + config_file="examples/frameworks/agno_personal_finance/src/nat_agno_personal_finance/configs/config.yml", + framework_package="agno", + required_live_env=(), # needs NVIDIA_API_KEY + SERP_API_KEY; structural only for now. + ), + FrameworkEntry( + key="autogen", + display_name="AutoGen", + example_dir="examples/frameworks/nat_autogen_demo", + config_file="examples/frameworks/nat_autogen_demo/src/nat_autogen_demo/configs/config.yml", + framework_package="autogen-agentchat", + required_live_env=(), # mixes NIM/OpenAI/Azure + an MCP client; structural only for now. + ), + FrameworkEntry( + key="strands", + display_name="Strands Agents", + example_dir="examples/frameworks/strands_demo", + config_file="examples/frameworks/strands_demo/src/nat_strands_demo/configs/config.yml", + framework_package="strands-agents", + required_live_env=(), # mixes NIM/AWS Bedrock/OpenAI; structural only for now. + ), + FrameworkEntry( + key="haystack", + display_name="Haystack", + example_dir="examples/frameworks/haystack_deep_research_agent", + config_file=("examples/frameworks/haystack_deep_research_agent/src/" + "nat_haystack_deep_research_agent/configs/config.yml"), + framework_package="haystack-ai", + required_live_env=(), # needs NVIDIA_API_KEY; structural only for now. + ), +) + + +def get_entry(key: str) -> FrameworkEntry: + for entry in FRAMEWORK_MATRIX: + if entry.key == key: + return entry + raise KeyError(f"No framework-parity entry named {key!r}. " + f"Known keys: {[e.key for e in FRAMEWORK_MATRIX]}") diff --git a/scripts/framework_parity/orchestrate.py b/scripts/framework_parity/orchestrate.py new file mode 100644 index 0000000000..d4cf7983ec --- /dev/null +++ b/scripts/framework_parity/orchestrate.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Drives the full framework-parity matrix: for each entry, creates a throwaway `uv` +venv, installs that example's own package into it (exactly as the example's own +README tells a user to), and runs `run_check.py` inside that venv's interpreter. + +This is the piece that makes the check honest: every framework gets its own +environment, so a dependency pulled in by the CrewAI example can't silently make the +ADK example's import succeed (or fail) for the wrong reason. + +Usage: + python scripts/framework_parity/orchestrate.py [--live] [--only KEY [KEY ...]] + +Writes one /.framework_parity_results/.json per entry and a summary +to stdout. Exits 1 if any entry's structural or (attempted) live tier failed. +""" + +import argparse +import json +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from matrix import FRAMEWORK_MATRIX # noqa: E402 +from matrix import FrameworkEntry # noqa: E402 + +REPO_ROOT = Path(__file__).resolve().parents[2] +RESULTS_DIR = REPO_ROOT / ".framework_parity_results" + + +def _venv_python(venv_dir: Path) -> Path: + if sys.platform == "win32": + return venv_dir / "Scripts" / "python.exe" + return venv_dir / "bin" / "python" + + +def run_entry(entry: FrameworkEntry, live: bool, keep_venvs: bool) -> dict: + print(f"== {entry.display_name} ({entry.key}) ==", file=sys.stderr) + + work_dir = Path(tempfile.mkdtemp(prefix=f"nat_parity_{entry.key}_")) + venv_dir = work_dir / "venv" + + try: + subprocess.run(["uv", "venv", str(venv_dir)], check=True, cwd=REPO_ROOT) + python = _venv_python(venv_dir) + + install = subprocess.run( + ["uv", "pip", "install", "-e", entry.example_dir, "--python", str(python)], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + if install.returncode != 0: + return { + "key": entry.key, + "display_name": entry.display_name, + "framework_package": entry.framework_package, + "framework_version": None, + "structural_status": "fail", + "structural_detail": f"uv pip install failed:\n{install.stdout}\n{install.stderr}"[-4000:], + "live_status": "skipped", + "live_detail": "install failed before the live tier could run", + } + + check_cmd = [str(python), str(Path(__file__).resolve().parent / "run_check.py"), entry.key, "--repo-root", + str(REPO_ROOT)] + if live: + check_cmd.append("--live") + + check = subprocess.run(check_cmd, cwd=REPO_ROOT, capture_output=True, text=True, check=False) + if not check.stdout.strip(): + return { + "key": entry.key, + "display_name": entry.display_name, + "framework_package": entry.framework_package, + "framework_version": None, + "structural_status": "fail", + "structural_detail": f"run_check.py produced no output:\n{check.stderr}"[-4000:], + "live_status": "skipped", + "live_detail": "", + } + return json.loads(check.stdout.strip().splitlines()[-1]) + finally: + if not keep_venvs: + shutil.rmtree(work_dir, ignore_errors=True) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--live", action="store_true", help="Attempt the live tier for entries that support it") + parser.add_argument("--only", nargs="*", default=None, help="Restrict to these matrix keys") + parser.add_argument("--keep-venvs", action="store_true", help="Don't delete the per-framework venvs (debugging)") + args = parser.parse_args() + + entries = [e for e in FRAMEWORK_MATRIX if args.only is None or e.key in args.only] + + RESULTS_DIR.mkdir(exist_ok=True) + results = [] + for entry in entries: + result = run_entry(entry, live=args.live, keep_venvs=args.keep_venvs) + results.append(result) + (RESULTS_DIR / f"{entry.key}.json").write_text(json.dumps(result, indent=2)) + print(json.dumps(result)) + + any_failed = any(r["structural_status"] == "fail" or r["live_status"] == "fail" for r in results) + return 1 if any_failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/framework_parity/render_table.py b/scripts/framework_parity/render_table.py new file mode 100644 index 0000000000..e74d08c9d6 --- /dev/null +++ b/scripts/framework_parity/render_table.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Renders the framework-parity result JSON files into the Markdown table between the +`` / `:END` markers in a target README, and +rewrites the file in place. Re-running this is idempotent -- everything outside the +markers is left untouched. + +Usage: + python scripts/framework_parity/render_table.py --results-dir --readme +""" + +import argparse +import datetime +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from matrix import FRAMEWORK_MATRIX # noqa: E402 + +START_MARKER = "" +END_MARKER = "" + +_STATUS_BADGE = { + "pass": "✅ pass", + "fail": "❌ fail", + "skipped": "⚪ skipped (no credentials)", + "not_wired": "⚪ not wired (multi-credential example)", +} + + +def _row(result: dict) -> str: + structural = _STATUS_BADGE.get(result["structural_status"], result["structural_status"]) + live = _STATUS_BADGE.get(result["live_status"], result["live_status"]) + version = result.get("framework_version") or "not installed" + return (f"| {result['display_name']} | {result['framework_package']} `{version}` " + f"| {structural} | {live} |") + + +def render(results_dir: Path) -> str: + rows = [] + for entry in FRAMEWORK_MATRIX: + result_file = results_dir / f"{entry.key}.json" + if result_file.exists(): + result = json.loads(result_file.read_text()) + else: + result = { + "display_name": entry.display_name, + "framework_package": entry.framework_package, + "framework_version": None, + "structural_status": "fail", + "live_status": "skipped", + } + rows.append(_row(result)) + + timestamp = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M UTC") + lines = [ + START_MARKER, + f"_Last generated {timestamp} by [`framework_parity.yml`]" + f"(../../.github/workflows/framework_parity.yml)._", + "", + "| Framework | Version tested | Install + validate | Live workflow run |", + "| --- | --- | --- | --- |", + *rows, + "", + END_MARKER, + ] + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results-dir", required=True, type=Path) + parser.add_argument("--readme", required=True, type=Path) + args = parser.parse_args() + + table = render(args.results_dir) + content = args.readme.read_text(encoding="utf-8") + + if START_MARKER not in content or END_MARKER not in content: + raise SystemExit(f"{args.readme} is missing the {START_MARKER}/{END_MARKER} markers") + + before, _, rest = content.partition(START_MARKER) + _, _, after = rest.partition(END_MARKER) + new_content = before + table + after + + args.readme.write_text(new_content, encoding="utf-8") + print(f"Updated {args.readme}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/framework_parity/run_check.py b/scripts/framework_parity/run_check.py new file mode 100644 index 0000000000..0eb5cca506 --- /dev/null +++ b/scripts/framework_parity/run_check.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Runs the framework-parity check for a single entry in ``matrix.FRAMEWORK_MATRIX``. + +Intended to be invoked with the *target example's own isolated interpreter* (i.e. the +venv that `uv pip install -e ` was run into), so that `import nat` and +`import ` resolve to exactly what that example depends on -- +not whatever happens to be on the orchestrator's own Python path. + +Usage: + python run_check.py --repo-root [--live] + +Run directly (not with `-m`) so Python's automatic sys.path[0] makes the sibling +`matrix` module importable regardless of the caller's working directory. + +Prints one JSON object to stdout and exits 0 on pass, 1 on fail. A missing live +credential is not a failure: the entry is reported with live_status="skipped" and the +process still exits 0, since the structural tier is what CI enforces on every run. +""" + +import argparse +import asyncio +import importlib.metadata +import json +import os +import subprocess +import sys +import traceback +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from matrix import get_entry # noqa: E402 + + +def _framework_version(package_name: str) -> str | None: + try: + return importlib.metadata.version(package_name) + except importlib.metadata.PackageNotFoundError: + return None + + +def _run_structural(repo_root: Path, config_file: Path) -> tuple[bool, str]: + """Runs the real `nat validate` CLI against the example's config file.""" + env = dict(os.environ) + env["PYTHONIOENCODING"] = "utf-8" # nat validate prints unicode check/cross marks + proc = subprocess.run( + [sys.executable, "-m", "nat.cli.main", "validate", "--config_file", str(config_file)], + cwd=repo_root, + env=env, + capture_output=True, + encoding="utf-8", # the child writes utf-8 (PYTHONIOENCODING above); decode it as such, + errors="replace", # not the parent's locale-preferred encoding, or checkmarks get mangled + check=False, + ) + if proc.returncode != 0: + return False, (proc.stdout + proc.stderr).strip()[-4000:] + return True, proc.stdout.strip()[-2000:] + + +async def _run_live(config_file: Path, question: str) -> dict: + """ + Runs the workflow for real and inspects the emitted IntermediateStep stream. + + Returns a dict with the raw counts so the caller can decide pass/fail -- kept + separate from the assertions themselves so a failure here always comes with the + actual span/token counts observed, not just a bare "it didn't work". + """ + from nat.builder.context import Context + from nat.data_models.intermediate_step import IntermediateStepType + from nat.runtime.loader import load_workflow + + collected = [] + ctx = Context.get() + subscription = ctx.intermediate_step_manager.subscribe(on_next=collected.append) + + try: + async with load_workflow(config_file) as workflow: + async with workflow.run(question) as runner: + answer = await runner.result() + finally: + subscription.unsubscribe() + + has_workflow_start = any(s.event_type == IntermediateStepType.WORKFLOW_START for s in collected) + has_workflow_end = any(s.event_type == IntermediateStepType.WORKFLOW_END for s in collected) + llm_end_steps = [s for s in collected if s.event_type == IntermediateStepType.LLM_END] + total_tokens = sum( + s.usage_info.token_usage.total_tokens for s in llm_end_steps if s.usage_info is not None) + + return { + "answer_preview": str(answer)[:200], + "total_steps": len(collected), + "has_workflow_start": has_workflow_start, + "has_workflow_end": has_workflow_end, + "llm_end_span_count": len(llm_end_steps), + "total_tokens": total_tokens, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("key", help="Framework matrix key, e.g. 'crewai'") + parser.add_argument("--repo-root", required=True, type=Path) + parser.add_argument("--live", action="store_true", help="Attempt the live tier if credentials are present") + args = parser.parse_args() + + entry = get_entry(args.key) + repo_root: Path = args.repo_root.resolve() + config_file = repo_root / entry.config_file + + result = { + "key": entry.key, + "display_name": entry.display_name, + "framework_package": entry.framework_package, + "framework_version": _framework_version(entry.framework_package), + "structural_status": "fail", + "structural_detail": "", + "live_status": "skipped", + "live_detail": "", + } + + structural_ok, structural_detail = _run_structural(repo_root, config_file) + result["structural_status"] = "pass" if structural_ok else "fail" + result["structural_detail"] = structural_detail + + missing_env = [v for v in entry.required_live_env if not os.environ.get(v)] + + if args.live and entry.required_live_env and not missing_env: + try: + live_result = asyncio.run(_run_live(config_file, entry.question)) + ok = (live_result["has_workflow_start"] and live_result["has_workflow_end"] + and live_result["llm_end_span_count"] > 0 and live_result["total_tokens"] > 0) + result["live_status"] = "pass" if ok else "fail" + result["live_detail"] = json.dumps(live_result) + except Exception: # noqa: BLE001 - report the real failure, don't swallow it + result["live_status"] = "fail" + result["live_detail"] = traceback.format_exc()[-4000:] + elif entry.required_live_env and missing_env: + result["live_status"] = "skipped" + result["live_detail"] = f"missing env vars: {', '.join(missing_env)}" + elif not entry.required_live_env: + result["live_status"] = "not_wired" + result["live_detail"] = "no single-credential live path yet; see matrix.py" + + print(json.dumps(result)) + + failed = result["structural_status"] == "fail" or result["live_status"] == "fail" + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 353c1ad74054113950c426b88c94d432f0210773 Mon Sep 17 00:00:00 2001 From: Ayantika Nandi Date: Fri, 11 Sep 2026 19:25:21 -0400 Subject: [PATCH 2/2] Remove framework parity harness per review; list demo in examples/README Addresses review feedback on PR #2201: - Removed scripts/framework_parity/ and .github/workflows/framework_parity.yml. An internal nightly CI already runs all examples, so the harness was superfluous. Also reverted the .gitignore entry it needed. - Removed the new examples/frameworks/README.md. examples/README.md already lists every framework example, so crewai_demo is added there instead, in alphabetical order and matching the existing entry format. - Dropped the framework-parity framing from the demo's own README and config docstring, since the harness it referred to no longer exists. Text only; no workflow, tool, or config logic changed. The PR is now just the CrewAI example. Co-Authored-By: Claude Opus 5 Signed-off-by: Ayantika Nandi --- .github/workflows/framework_parity.yml | 123 ------------- .gitignore | 3 - examples/README.md | 1 + examples/frameworks/README.md | 50 ------ examples/frameworks/crewai_demo/README.md | 2 +- .../src/nat_crewai_demo/register.py | 6 +- scripts/framework_parity/README.md | 79 --------- scripts/framework_parity/__init__.py | 14 -- scripts/framework_parity/matrix.py | 144 --------------- scripts/framework_parity/orchestrate.py | 128 -------------- scripts/framework_parity/render_table.py | 107 ------------ scripts/framework_parity/run_check.py | 165 ------------------ 12 files changed, 4 insertions(+), 818 deletions(-) delete mode 100644 .github/workflows/framework_parity.yml delete mode 100644 examples/frameworks/README.md delete mode 100644 scripts/framework_parity/README.md delete mode 100644 scripts/framework_parity/__init__.py delete mode 100644 scripts/framework_parity/matrix.py delete mode 100644 scripts/framework_parity/orchestrate.py delete mode 100644 scripts/framework_parity/render_table.py delete mode 100644 scripts/framework_parity/run_check.py diff --git a/.github/workflows/framework_parity.yml b/.github/workflows/framework_parity.yml deleted file mode 100644 index 5d3ae709b4..0000000000 --- a/.github/workflows/framework_parity.yml +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Framework Parity -run-name: Framework Parity - -# NAT claims to work alongside every framework listed in scripts/framework_parity/matrix.py. -# This workflow is the proof: each one gets installed into its own isolated environment, -# nightly, and checked for real -- not just "the example folder still exists". -on: - schedule: - - cron: "0 9 * * *" # 09:00 UTC nightly - workflow_dispatch: {} - pull_request: - paths: - - "examples/frameworks/**" - - "scripts/framework_parity/**" - - ".github/workflows/framework_parity.yml" - -defaults: - run: - shell: bash - -permissions: - contents: read - -jobs: - check: - name: ${{ matrix.entry }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - entry: - - crewai - - semantic_kernel - - adk - - langchain_llama_index - - agno - - autogen - - strands - - haystack - steps: - - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v3 - with: - enable-cache: true - - - name: Run framework-parity check for ${{ matrix.entry }} - # Only the scheduled nightly run and manual dispatch get the live-tier - # credentials; PRs from forks (including this harness's own PRs while it's - # under review) never see repo secrets, so they naturally exercise the - # structural tier only -- which needs no credentials at all. - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - MEM0_API_KEY: ${{ secrets.MEM0_API_KEY }} - run: | - LIVE_FLAG="" - if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - LIVE_FLAG="--live" - fi - python scripts/framework_parity/orchestrate.py --only ${{ matrix.entry }} $LIVE_FLAG - - - name: Upload result - if: always() - uses: actions/upload-artifact@v4 - with: - name: framework-parity-result-${{ matrix.entry }} - path: .framework_parity_results/${{ matrix.entry }}.json - retention-days: 90 - - publish-table: - name: Publish parity table - needs: check - if: always() - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - - name: Download all results - uses: actions/download-artifact@v4 - with: - pattern: framework-parity-result-* - path: .framework_parity_results - merge-multiple: true - - - name: Render badge table - run: | - python scripts/framework_parity/render_table.py \ - --results-dir .framework_parity_results \ - --readme examples/frameworks/README.md - - - name: Commit updated table - # Only push the regenerated table back on the nightly/manual run against this - # repo's own branches -- never from a pull_request event, so an external - # contributor's PR can't be used to trigger a commit back to the branch. - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - if git diff --quiet -- examples/frameworks/README.md; then - echo "Table unchanged, nothing to commit." - exit 0 - fi - git add examples/frameworks/README.md - git commit -m "chore: update framework-parity table [skip ci]" - git push diff --git a/.gitignore b/.gitignore index dd4b1e838f..152e7c7536 100644 --- a/.gitignore +++ b/.gitignore @@ -85,9 +85,6 @@ install_manifest.txt compile_commands.json CTestTestfile.cmake -### Framework parity harness ### -.framework_parity_results/ - ### Python ### # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/examples/README.md b/examples/README.md index 451271bbe9..b74baee714 100644 --- a/examples/README.md +++ b/examples/README.md @@ -129,6 +129,7 @@ _Additional information can be found in the [Agents README](./agents/README.md). - **[`adk_demo`](frameworks/adk_demo/README.md)**: Minimal example using Google Agent Development Kit showcasing a simple weather time agent that can call tools (a function tool and an MCP tool) **[🟢 Beginner]** - **[`agno_personal_finance`](frameworks/agno_personal_finance/README.md)**: Personal finance planning agent built with Agno framework that researches and creates tailored financial plans **[🟢 Beginner]** - **[`autogen_demo`](frameworks/nat_autogen_demo/README.md)**: Minimal example using Microsoft AutoGen showcasing a traffic estimator agent that can call tools (a function tool and an MCP tool) and orchestrate multiple AutoGen agents **[🟨 Intermediate]** +- **[`crewai_demo`](frameworks/crewai_demo/README.md)**: Multi-agent travel planning system using CrewAI with role-based agents for itinerary planning, budget analysis, and final plan summarization **[🟢 Beginner]** - **[`haystack_deep_research_agent`](frameworks/haystack_deep_research_agent/README.md)**: Deep research agent using Haystack framework that combines web search and Retrieval Augmented Generation (RAG) capabilities with SerperDev API and OpenSearch **[🟨 Intermediate]** - **[`langchain_deep_research`](frameworks/auto_wrapper/langchain_deep_research/README.md)**: An example that integrates any existing LangGraph agent with NeMo Agent Toolkit using the `langgraph_wrapper` workflow type **[🟨 Intermediate]** - **[`multi_frameworks`](frameworks/multi_frameworks/README.md)**: Supervisor agent coordinating LangChain/LangGraph, LlamaIndex, and Haystack agents for research, RAG, and chitchat tasks **[🟨 Intermediate]** diff --git a/examples/frameworks/README.md b/examples/frameworks/README.md deleted file mode 100644 index 36b23aa4ad..0000000000 --- a/examples/frameworks/README.md +++ /dev/null @@ -1,50 +0,0 @@ - - -# Framework Examples - -NeMo Agent Toolkit is framework-agnostic: it can wrap and orchestrate agents built with -LangChain, LlamaIndex, CrewAI, Semantic Kernel, Google ADK, AutoGen, Agno, Strands -Agents, and Haystack, among others. Each subdirectory here is a maintained, runnable -reference example for one of those frameworks. - -The table below is generated nightly by the [framework-parity -harness](../../.github/workflows/framework_parity.yml) -([source](../../scripts/framework_parity)), which installs each example's own package -into a fresh, isolated environment and validates it there — the same thing a new user -does when they follow one of the READMEs below. For frameworks with a single-credential -canonical config, it goes a step further and actually runs the workflow, then checks -the profiler's own event stream for a real LLM span with non-zero token usage, rather -than only checking that the YAML parses. - - -_Not yet generated. Run `python scripts/framework_parity/orchestrate.py` and then -`scripts/framework_parity/render_table.py` to populate this table._ - - - -## Examples - -- [`crewai_demo`](crewai_demo/README.md) — CrewAI -- [`semantic_kernel_demo`](semantic_kernel_demo/README.md) — Semantic Kernel -- [`adk_demo`](adk_demo/README.md) — Google ADK -- [`multi_frameworks`](multi_frameworks/README.md) — LangChain + LlamaIndex in one workflow -- [`agno_personal_finance`](agno_personal_finance/README.md) — Agno -- [`nat_autogen_demo`](nat_autogen_demo/README.md) — AutoGen -- [`strands_demo`](strands_demo/README.md) — Strands Agents -- [`haystack_deep_research_agent`](haystack_deep_research_agent/README.md) — Haystack -- [`auto_wrapper/langchain_deep_research`](auto_wrapper/langchain_deep_research/README.md) — wrapping an existing LangChain workflow without rewriting it diff --git a/examples/frameworks/crewai_demo/README.md b/examples/frameworks/crewai_demo/README.md index bec3bc21cb..6e02f77623 100644 --- a/examples/frameworks/crewai_demo/README.md +++ b/examples/frameworks/crewai_demo/README.md @@ -19,7 +19,7 @@ limitations under the License. **Complexity:** 🟢 Beginner -A minimal example using CrewAI showcasing a multi-agent travel planning system where an Itinerary Expert plans activities, a Budget Advisor prices lodging and totals the trip cost, and a Summarizer compiles the final plan. This is the CrewAI counterpart to the [Semantic Kernel example](../semantic_kernel_demo/README.md) — same task, same two tools, different framework — intended to be run side by side with it as a framework-parity check. +A minimal example using CrewAI showcasing a multi-agent travel planning system where an Itinerary Expert plans activities, a Budget Advisor prices lodging and totals the trip cost, and a Summarizer compiles the final plan. ## Table of Contents diff --git a/examples/frameworks/crewai_demo/src/nat_crewai_demo/register.py b/examples/frameworks/crewai_demo/src/nat_crewai_demo/register.py index c3ec4b08c5..24945c0f1c 100644 --- a/examples/frameworks/crewai_demo/src/nat_crewai_demo/register.py +++ b/examples/frameworks/crewai_demo/src/nat_crewai_demo/register.py @@ -33,10 +33,8 @@ class CrewAITravelPlanningWorkflowConfig(FunctionBaseConfig, name="crewai"): """ - Structural companion to the semantic_kernel_demo example: the same travel-planning task (an itinerary expert, - a budget advisor, and a summarizer, each backed by the same two tools), reimplemented against CrewAI instead of - Semantic Kernel, so the two examples can be run through a framework-parity harness as comparable canonical - workflows. + A travel-planning crew: an itinerary expert, a budget advisor, and a summarizer, each backed by the same two + NAT-registered tools. """ tool_names: list[FunctionRef] = Field(default_factory=list, description="The list of tools to provide to the crew.") diff --git a/scripts/framework_parity/README.md b/scripts/framework_parity/README.md deleted file mode 100644 index ad843b2282..0000000000 --- a/scripts/framework_parity/README.md +++ /dev/null @@ -1,79 +0,0 @@ - - -# Framework Parity Harness - -Backs the badge table in [`examples/frameworks/README.md`](../../examples/frameworks/README.md). -Runs nightly via [`.github/workflows/framework_parity.yml`](../../.github/workflows/framework_parity.yml). - -## What it actually checks - -Each entry in [`matrix.py`](matrix.py) gets **two** independent tiers, run in its own -throwaway `uv` venv so one framework's dependencies can never leak into another's -result: - -1. **Structural** (always runs, no credentials): `uv pip install -e ` into a - fresh venv, then the real `nat validate` CLI against the example's shipped config - file. Confirms the example installs the way its own README says to, and that its - config/registration graph is valid. - -2. **Live** (only for entries with `required_live_env` set, and only when those env - vars are actually present): loads the workflow for real, runs its canonical - question through the real framework and a real LLM, subscribes to the - `IntermediateStepManager` event stream, and asserts a `WORKFLOW_START`/`WORKFLOW_END` - pair and at least one `LLM_END` span with non-zero `usage_info.token_usage.total_tokens` - were actually emitted. This is the part that backs the "framework-agnostic" claim -- - a config that parses is not proof an integration executes. - -## Why most rows are structural-only today - -`agno_personal_finance`, `nat_autogen_demo`, `strands_demo`, `multi_frameworks`, and -`haystack_deep_research_agent` each reach across several external services in their -*normal, realistic* configuration (NIM, SerpAPI, Tavily, AWS Bedrock, MCP servers). -Wiring a live check for those means either provisioning that whole service list as CI -secrets, or forking a second "minimal" config that no longer matches what a user -actually installs -- both worse than being honest that the live tier isn't wired yet. -`crewai_demo`, `semantic_kernel_demo`, and `adk_demo`'s `config_oai.yml` were verified -by hand to build and run to the point of a real (auth-failing, without a key) LLM call -using only `OPENAI_API_KEY` (`semantic_kernel_demo` additionally needs `MEM0_API_KEY` -for its memory tools), so those three are live-wired now. - -Notably, NAT's own built-in `nat_test_llm` mock provider -(`nvidia_nat_test/src/nat/test/llm.py`) was **not** used for the live tier: its -per-framework client classes are bare stand-ins that don't inherit from each -framework's real base LLM class, so they don't go through that framework's real -callback/instrumentation path -- using it would let this harness pass without ever -proving the profiler's span/token wiring works. - -## Running it locally - -```bash -# One framework: -python scripts/framework_parity/orchestrate.py --only crewai --live - -# The whole matrix: -python scripts/framework_parity/orchestrate.py --live - -# Regenerate the table from whatever's in .framework_parity_results/: -python scripts/framework_parity/render_table.py \ - --results-dir .framework_parity_results \ - --readme examples/frameworks/README.md -``` - -`--live` is a no-op for an entry unless every env var in its `required_live_env` is -already set -- it degrades to the structural tier and reports `live_status: skipped` -rather than failing. diff --git a/scripts/framework_parity/__init__.py b/scripts/framework_parity/__init__.py deleted file mode 100644 index 3bcc1c39bb..0000000000 --- a/scripts/framework_parity/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/scripts/framework_parity/matrix.py b/scripts/framework_parity/matrix.py deleted file mode 100644 index 647bb358f9..0000000000 --- a/scripts/framework_parity/matrix.py +++ /dev/null @@ -1,144 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -The framework-parity matrix. - -NAT claims to be framework-agnostic: it ships a maintained reference example for each -of its supported agent frameworks. This module is the single source of truth for what -"maintained" is actually checked against for each one: - - - ``structural``: install the example's own package (with its own declared extras, - exactly as a user would) into an isolated environment and run the real ``nat - validate`` CLI against its shipped config file. This requires no external - credentials and runs on every scheduled harness invocation. - - - ``live``: in addition to the structural check, actually execute the example's - canonical question through the real framework and a real LLM, then inspect the - emitted IntermediateStep event stream to confirm the profiler attached at least - one LLM span with non-zero token usage. This is the one that actually backs the - "framework-agnostic" claim -- a config that merely parses is not proof a - framework integration works end to end. - -Only entries whose `required_live_env` is fully satisfiable by a single, cheap -credential are wired for the live tier today. Several real examples in this repo -(agno_personal_finance, nat_autogen_demo, strands_demo, multi_frameworks, -haystack_deep_research_agent) reach across multiple external services (NIM, SerpAPI, -Tavily, AWS Bedrock, MCP servers) as part of their normal, realistic configuration -- -wiring live checks for those means either provisioning that whole service list as CI -secrets, or forking a second "minimal" config that no longer matches what a user -actually installs. Neither is worth doing to turn a yellow cell green; they run the -structural tier only, honestly labeled, until that changes. -""" - -import dataclasses - - -@dataclasses.dataclass(frozen=True) -class FrameworkEntry: - # Short key, matches the GitHub Actions matrix value and the result filename. - key: str - # Human-readable name for the badge table. - display_name: str - # Path to the example's directory, relative to the repo root. - example_dir: str - # Path to the config file to validate/run, relative to the repo root. - config_file: str - # PyPI distribution name of the underlying framework, for `importlib.metadata` - # version reporting in the badge table. - framework_package: str - # Env vars that must ALL be set for the live tier to run. If any are missing, the - # entry falls back to the structural-only tier and is reported as "skipped (live)" - # rather than failed. - required_live_env: tuple[str, ...] - # The question to run through the workflow during the live tier. - question: str = "Plan a 1-day trip to Tokyo, covering one hotel and one activity." - - -FRAMEWORK_MATRIX: tuple[FrameworkEntry, ...] = ( - FrameworkEntry( - key="crewai", - display_name="CrewAI", - example_dir="examples/frameworks/crewai_demo", - config_file="examples/frameworks/crewai_demo/src/nat_crewai_demo/configs/config.yml", - framework_package="crewai", - required_live_env=("OPENAI_API_KEY", ), - ), - FrameworkEntry( - key="semantic_kernel", - display_name="Semantic Kernel", - example_dir="examples/frameworks/semantic_kernel_demo", - config_file="examples/frameworks/semantic_kernel_demo/src/nat_semantic_kernel_demo/configs/config.yml", - framework_package="semantic-kernel", - required_live_env=("OPENAI_API_KEY", "MEM0_API_KEY"), - ), - FrameworkEntry( - key="adk", - display_name="Google ADK", - example_dir="examples/frameworks/adk_demo", - config_file="examples/frameworks/adk_demo/src/nat_adk_demo/configs/config_oai.yml", - framework_package="google-adk", - required_live_env=("OPENAI_API_KEY", ), - question="What is the weather in Tokyo right now?", - ), - FrameworkEntry( - key="langchain_llama_index", - display_name="LangChain + LlamaIndex", - example_dir="examples/frameworks/multi_frameworks", - config_file="examples/frameworks/multi_frameworks/src/nat_multi_frameworks/configs/config.yml", - framework_package="langchain", - required_live_env=(), # needs NVIDIA_API_KEY + TAVILY_API_KEY; structural only for now. - ), - FrameworkEntry( - key="agno", - display_name="Agno", - example_dir="examples/frameworks/agno_personal_finance", - config_file="examples/frameworks/agno_personal_finance/src/nat_agno_personal_finance/configs/config.yml", - framework_package="agno", - required_live_env=(), # needs NVIDIA_API_KEY + SERP_API_KEY; structural only for now. - ), - FrameworkEntry( - key="autogen", - display_name="AutoGen", - example_dir="examples/frameworks/nat_autogen_demo", - config_file="examples/frameworks/nat_autogen_demo/src/nat_autogen_demo/configs/config.yml", - framework_package="autogen-agentchat", - required_live_env=(), # mixes NIM/OpenAI/Azure + an MCP client; structural only for now. - ), - FrameworkEntry( - key="strands", - display_name="Strands Agents", - example_dir="examples/frameworks/strands_demo", - config_file="examples/frameworks/strands_demo/src/nat_strands_demo/configs/config.yml", - framework_package="strands-agents", - required_live_env=(), # mixes NIM/AWS Bedrock/OpenAI; structural only for now. - ), - FrameworkEntry( - key="haystack", - display_name="Haystack", - example_dir="examples/frameworks/haystack_deep_research_agent", - config_file=("examples/frameworks/haystack_deep_research_agent/src/" - "nat_haystack_deep_research_agent/configs/config.yml"), - framework_package="haystack-ai", - required_live_env=(), # needs NVIDIA_API_KEY; structural only for now. - ), -) - - -def get_entry(key: str) -> FrameworkEntry: - for entry in FRAMEWORK_MATRIX: - if entry.key == key: - return entry - raise KeyError(f"No framework-parity entry named {key!r}. " - f"Known keys: {[e.key for e in FRAMEWORK_MATRIX]}") diff --git a/scripts/framework_parity/orchestrate.py b/scripts/framework_parity/orchestrate.py deleted file mode 100644 index d4cf7983ec..0000000000 --- a/scripts/framework_parity/orchestrate.py +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Drives the full framework-parity matrix: for each entry, creates a throwaway `uv` -venv, installs that example's own package into it (exactly as the example's own -README tells a user to), and runs `run_check.py` inside that venv's interpreter. - -This is the piece that makes the check honest: every framework gets its own -environment, so a dependency pulled in by the CrewAI example can't silently make the -ADK example's import succeed (or fail) for the wrong reason. - -Usage: - python scripts/framework_parity/orchestrate.py [--live] [--only KEY [KEY ...]] - -Writes one /.framework_parity_results/.json per entry and a summary -to stdout. Exits 1 if any entry's structural or (attempted) live tier failed. -""" - -import argparse -import json -import shutil -import subprocess -import sys -import tempfile -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -from matrix import FRAMEWORK_MATRIX # noqa: E402 -from matrix import FrameworkEntry # noqa: E402 - -REPO_ROOT = Path(__file__).resolve().parents[2] -RESULTS_DIR = REPO_ROOT / ".framework_parity_results" - - -def _venv_python(venv_dir: Path) -> Path: - if sys.platform == "win32": - return venv_dir / "Scripts" / "python.exe" - return venv_dir / "bin" / "python" - - -def run_entry(entry: FrameworkEntry, live: bool, keep_venvs: bool) -> dict: - print(f"== {entry.display_name} ({entry.key}) ==", file=sys.stderr) - - work_dir = Path(tempfile.mkdtemp(prefix=f"nat_parity_{entry.key}_")) - venv_dir = work_dir / "venv" - - try: - subprocess.run(["uv", "venv", str(venv_dir)], check=True, cwd=REPO_ROOT) - python = _venv_python(venv_dir) - - install = subprocess.run( - ["uv", "pip", "install", "-e", entry.example_dir, "--python", str(python)], - cwd=REPO_ROOT, - capture_output=True, - text=True, - check=False, - ) - if install.returncode != 0: - return { - "key": entry.key, - "display_name": entry.display_name, - "framework_package": entry.framework_package, - "framework_version": None, - "structural_status": "fail", - "structural_detail": f"uv pip install failed:\n{install.stdout}\n{install.stderr}"[-4000:], - "live_status": "skipped", - "live_detail": "install failed before the live tier could run", - } - - check_cmd = [str(python), str(Path(__file__).resolve().parent / "run_check.py"), entry.key, "--repo-root", - str(REPO_ROOT)] - if live: - check_cmd.append("--live") - - check = subprocess.run(check_cmd, cwd=REPO_ROOT, capture_output=True, text=True, check=False) - if not check.stdout.strip(): - return { - "key": entry.key, - "display_name": entry.display_name, - "framework_package": entry.framework_package, - "framework_version": None, - "structural_status": "fail", - "structural_detail": f"run_check.py produced no output:\n{check.stderr}"[-4000:], - "live_status": "skipped", - "live_detail": "", - } - return json.loads(check.stdout.strip().splitlines()[-1]) - finally: - if not keep_venvs: - shutil.rmtree(work_dir, ignore_errors=True) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--live", action="store_true", help="Attempt the live tier for entries that support it") - parser.add_argument("--only", nargs="*", default=None, help="Restrict to these matrix keys") - parser.add_argument("--keep-venvs", action="store_true", help="Don't delete the per-framework venvs (debugging)") - args = parser.parse_args() - - entries = [e for e in FRAMEWORK_MATRIX if args.only is None or e.key in args.only] - - RESULTS_DIR.mkdir(exist_ok=True) - results = [] - for entry in entries: - result = run_entry(entry, live=args.live, keep_venvs=args.keep_venvs) - results.append(result) - (RESULTS_DIR / f"{entry.key}.json").write_text(json.dumps(result, indent=2)) - print(json.dumps(result)) - - any_failed = any(r["structural_status"] == "fail" or r["live_status"] == "fail" for r in results) - return 1 if any_failed else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/framework_parity/render_table.py b/scripts/framework_parity/render_table.py deleted file mode 100644 index e74d08c9d6..0000000000 --- a/scripts/framework_parity/render_table.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Renders the framework-parity result JSON files into the Markdown table between the -`` / `:END` markers in a target README, and -rewrites the file in place. Re-running this is idempotent -- everything outside the -markers is left untouched. - -Usage: - python scripts/framework_parity/render_table.py --results-dir --readme -""" - -import argparse -import datetime -import json -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -from matrix import FRAMEWORK_MATRIX # noqa: E402 - -START_MARKER = "" -END_MARKER = "" - -_STATUS_BADGE = { - "pass": "✅ pass", - "fail": "❌ fail", - "skipped": "⚪ skipped (no credentials)", - "not_wired": "⚪ not wired (multi-credential example)", -} - - -def _row(result: dict) -> str: - structural = _STATUS_BADGE.get(result["structural_status"], result["structural_status"]) - live = _STATUS_BADGE.get(result["live_status"], result["live_status"]) - version = result.get("framework_version") or "not installed" - return (f"| {result['display_name']} | {result['framework_package']} `{version}` " - f"| {structural} | {live} |") - - -def render(results_dir: Path) -> str: - rows = [] - for entry in FRAMEWORK_MATRIX: - result_file = results_dir / f"{entry.key}.json" - if result_file.exists(): - result = json.loads(result_file.read_text()) - else: - result = { - "display_name": entry.display_name, - "framework_package": entry.framework_package, - "framework_version": None, - "structural_status": "fail", - "live_status": "skipped", - } - rows.append(_row(result)) - - timestamp = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M UTC") - lines = [ - START_MARKER, - f"_Last generated {timestamp} by [`framework_parity.yml`]" - f"(../../.github/workflows/framework_parity.yml)._", - "", - "| Framework | Version tested | Install + validate | Live workflow run |", - "| --- | --- | --- | --- |", - *rows, - "", - END_MARKER, - ] - return "\n".join(lines) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--results-dir", required=True, type=Path) - parser.add_argument("--readme", required=True, type=Path) - args = parser.parse_args() - - table = render(args.results_dir) - content = args.readme.read_text(encoding="utf-8") - - if START_MARKER not in content or END_MARKER not in content: - raise SystemExit(f"{args.readme} is missing the {START_MARKER}/{END_MARKER} markers") - - before, _, rest = content.partition(START_MARKER) - _, _, after = rest.partition(END_MARKER) - new_content = before + table + after - - args.readme.write_text(new_content, encoding="utf-8") - print(f"Updated {args.readme}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/framework_parity/run_check.py b/scripts/framework_parity/run_check.py deleted file mode 100644 index 0eb5cca506..0000000000 --- a/scripts/framework_parity/run_check.py +++ /dev/null @@ -1,165 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Runs the framework-parity check for a single entry in ``matrix.FRAMEWORK_MATRIX``. - -Intended to be invoked with the *target example's own isolated interpreter* (i.e. the -venv that `uv pip install -e ` was run into), so that `import nat` and -`import ` resolve to exactly what that example depends on -- -not whatever happens to be on the orchestrator's own Python path. - -Usage: - python run_check.py --repo-root [--live] - -Run directly (not with `-m`) so Python's automatic sys.path[0] makes the sibling -`matrix` module importable regardless of the caller's working directory. - -Prints one JSON object to stdout and exits 0 on pass, 1 on fail. A missing live -credential is not a failure: the entry is reported with live_status="skipped" and the -process still exits 0, since the structural tier is what CI enforces on every run. -""" - -import argparse -import asyncio -import importlib.metadata -import json -import os -import subprocess -import sys -import traceback -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -from matrix import get_entry # noqa: E402 - - -def _framework_version(package_name: str) -> str | None: - try: - return importlib.metadata.version(package_name) - except importlib.metadata.PackageNotFoundError: - return None - - -def _run_structural(repo_root: Path, config_file: Path) -> tuple[bool, str]: - """Runs the real `nat validate` CLI against the example's config file.""" - env = dict(os.environ) - env["PYTHONIOENCODING"] = "utf-8" # nat validate prints unicode check/cross marks - proc = subprocess.run( - [sys.executable, "-m", "nat.cli.main", "validate", "--config_file", str(config_file)], - cwd=repo_root, - env=env, - capture_output=True, - encoding="utf-8", # the child writes utf-8 (PYTHONIOENCODING above); decode it as such, - errors="replace", # not the parent's locale-preferred encoding, or checkmarks get mangled - check=False, - ) - if proc.returncode != 0: - return False, (proc.stdout + proc.stderr).strip()[-4000:] - return True, proc.stdout.strip()[-2000:] - - -async def _run_live(config_file: Path, question: str) -> dict: - """ - Runs the workflow for real and inspects the emitted IntermediateStep stream. - - Returns a dict with the raw counts so the caller can decide pass/fail -- kept - separate from the assertions themselves so a failure here always comes with the - actual span/token counts observed, not just a bare "it didn't work". - """ - from nat.builder.context import Context - from nat.data_models.intermediate_step import IntermediateStepType - from nat.runtime.loader import load_workflow - - collected = [] - ctx = Context.get() - subscription = ctx.intermediate_step_manager.subscribe(on_next=collected.append) - - try: - async with load_workflow(config_file) as workflow: - async with workflow.run(question) as runner: - answer = await runner.result() - finally: - subscription.unsubscribe() - - has_workflow_start = any(s.event_type == IntermediateStepType.WORKFLOW_START for s in collected) - has_workflow_end = any(s.event_type == IntermediateStepType.WORKFLOW_END for s in collected) - llm_end_steps = [s for s in collected if s.event_type == IntermediateStepType.LLM_END] - total_tokens = sum( - s.usage_info.token_usage.total_tokens for s in llm_end_steps if s.usage_info is not None) - - return { - "answer_preview": str(answer)[:200], - "total_steps": len(collected), - "has_workflow_start": has_workflow_start, - "has_workflow_end": has_workflow_end, - "llm_end_span_count": len(llm_end_steps), - "total_tokens": total_tokens, - } - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("key", help="Framework matrix key, e.g. 'crewai'") - parser.add_argument("--repo-root", required=True, type=Path) - parser.add_argument("--live", action="store_true", help="Attempt the live tier if credentials are present") - args = parser.parse_args() - - entry = get_entry(args.key) - repo_root: Path = args.repo_root.resolve() - config_file = repo_root / entry.config_file - - result = { - "key": entry.key, - "display_name": entry.display_name, - "framework_package": entry.framework_package, - "framework_version": _framework_version(entry.framework_package), - "structural_status": "fail", - "structural_detail": "", - "live_status": "skipped", - "live_detail": "", - } - - structural_ok, structural_detail = _run_structural(repo_root, config_file) - result["structural_status"] = "pass" if structural_ok else "fail" - result["structural_detail"] = structural_detail - - missing_env = [v for v in entry.required_live_env if not os.environ.get(v)] - - if args.live and entry.required_live_env and not missing_env: - try: - live_result = asyncio.run(_run_live(config_file, entry.question)) - ok = (live_result["has_workflow_start"] and live_result["has_workflow_end"] - and live_result["llm_end_span_count"] > 0 and live_result["total_tokens"] > 0) - result["live_status"] = "pass" if ok else "fail" - result["live_detail"] = json.dumps(live_result) - except Exception: # noqa: BLE001 - report the real failure, don't swallow it - result["live_status"] = "fail" - result["live_detail"] = traceback.format_exc()[-4000:] - elif entry.required_live_env and missing_env: - result["live_status"] = "skipped" - result["live_detail"] = f"missing env vars: {', '.join(missing_env)}" - elif not entry.required_live_env: - result["live_status"] = "not_wired" - result["live_detail"] = "no single-credential live path yet; see matrix.py" - - print(json.dumps(result)) - - failed = result["structural_status"] == "fail" or result["live_status"] == "fail" - return 1 if failed else 0 - - -if __name__ == "__main__": - raise SystemExit(main())