Merge pull request #1 from context-dot-dev/feat/context-dev-components

feat: add Context.dev Haystack integration

Author: aadithyanr

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2026-present AUTHOR <your@email.com>
+# SPDX-FileCopyrightText: 2026-present Context.dev
 #
 # SPDX-License-Identifier: Apache-2.0
 

.github/workflows/test.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2026-present AUTHOR <your@email.com>
+# SPDX-FileCopyrightText: 2026-present Context.dev
 #
 # SPDX-License-Identifier: Apache-2.0
 
@@ -44,5 +44,9 @@ jobs:
         if: matrix.python-version == '3.10' && runner.os == 'Linux'
         run: hatch run fmt-check
 
+      - name: Type check
+        if: matrix.python-version == '3.10' && runner.os == 'Linux'
+        run: hatch run test:types
+
       - name: Run tests
         run: hatch run test:all

CONTRIBUTING.md

@@ -0,0 +1,38 @@
+# Contributing
+
+Thanks for contributing to the Context.dev Haystack integration.
+
+## Setup
+
+This project uses [Hatch](https://hatch.pypa.io/) for environments, formatting, tests, and builds.
+
+```bash
+pip install hatch
+hatch --version
+```
+
+## Checks
+
+Run the same checks used by CI before opening a pull request:
+
+```bash
+hatch run fmt-check
+hatch run test:types
+hatch run test:unit
+hatch run test:cov
+```
+
+Integration tests call the live Context.dev API and consume credits. They are skipped unless `CONTEXT_API_KEY` is set:
+
+```bash
+export CONTEXT_API_KEY="your-api-key"
+hatch run test:integration
+```
+
+## Pull requests
+
+Keep changes focused, add tests for behavior changes, and use Conventional Commit titles such as `feat: add a component option` or `fix: preserve response metadata`.
+
+## Releases
+
+Maintainers publish releases by pushing a semantic version tag such as `v0.1.0`. The release workflow builds the source distribution and wheel, then publishes both to PyPI.

README.md

@@ -1,80 +1,92 @@
-# Custom Component Template
+# Context.dev for Haystack
 
-A template repository for creating custom [Haystack](https://haystack.deepset.ai/) components and publishing them as standalone Python packages.
+[![PyPI](https://img.shields.io/pypi/v/context-dev-haystack)](https://pypi.org/project/context-dev-haystack/)
+[![Python](https://img.shields.io/pypi/pyversions/context-dev-haystack)](https://pypi.org/project/context-dev-haystack/)
+[![Test](https://github.com/context-dot-dev/context-haystack/actions/workflows/test.yml/badge.svg)](https://github.com/context-dot-dev/context-haystack/actions/workflows/test.yml)
+[![License](https://img.shields.io/github/license/context-dot-dev/context-haystack)](LICENSE)
 
-For more details, see the Haystack documentation on [creating custom components](https://docs.haystack.deepset.ai/docs/custom-components) and [creating custom document stores](https://docs.haystack.deepset.ai/docs/creating-custom-document-stores).
+Haystack components for live web search, webpage and YouTube transcript retrieval, and bounded website crawling with [Context.dev](https://context.dev).
 
-## How to use this template
+## Installation
 
-1. Click **[Use this template](https://github.com/deepset-ai/custom-component/generate)** to create a new repository.
+```bash
+pip install context-dev-haystack
+```
 
-2. **Rename the package directory** from `src/haystack_integrations/components/example/` to match your integration. See [Namespace convention](#namespace-convention) below for the correct path.
+Create an API key in the [Context.dev dashboard](https://context.dev/dashboard/api-keys), then export it:
 
-3. **Update `pyproject.toml`** — search for `TODO` comments and replace:
-   - `name`: your package name, following the `<technology>-haystack` convention (e.g. `opensearch-haystack`)
-   - `description`, `authors`, `keywords`, `project.urls`
-   - `dependencies`: add your integration-specific dependencies
-   - `tool.hatch.version.raw-options`: if you renamed directories, the version path is still derived from git tags so no change is needed here
+```bash
+export CONTEXT_API_KEY="your-api-key"
+```
 
-4. **Add your component code** in the renamed directory and export your classes from `__init__.py`.
+## Components
 
-5. **Add tests** in `tests/` — see the skeleton in `tests/test_example.py`.
+| Component | Purpose | Import |
+| --- | --- | --- |
+| `ContextWebSearch` | Search the live web and return ranked Haystack Documents and source links | `haystack_integrations.components.websearch.context` |
+| `ContextFetcher` | Fetch webpages or YouTube videos as clean Markdown Documents | `haystack_integrations.components.fetchers.context` |
+| `ContextCrawler` | Crawl websites into Documents with explicit page and depth limits | `haystack_integrations.components.fetchers.context` |
 
-6. **Search for all `TODO` comments** across the project and address them.
+All components support both `run()` and `run_async()`, Haystack serialization, custom timeouts, and retry configuration.
 
-Check out the [video walkthrough](https://www.youtube.com/watch?v=SWC0QecAMcI) for a step-by-step guide on how to use this template.
+## Search the live web
 
-## Namespace convention
+```python
+from haystack_integrations.components.websearch.context import ContextWebSearch
 
-Haystack integrations use the `haystack_integrations` namespace package. The directory structure under `src/` determines the import path for your component.
+search = ContextWebSearch(top_k=5, include_markdown=True)
+result = search.run(query="Recent advances in retrieval-augmented generation")
 
-**Components** (converters, embedders, generators, rankers, etc.) use:
+documents = result["documents"]
+links = result["links"]
 ```
-src/haystack_integrations/components/<type>/<name>/
-```
-Import path: `from haystack_integrations.components.<type>.<name> import MyComponent`
 
-Common component types: `converters`, `embedders`, `generators`, `rankers`, `retrievers`, `connectors`, `tools`, `websearch`
+Use `include_domains`, `exclude_domains`, `freshness`, and `country` to constrain results. Extra Context.dev Search API fields can be supplied through `search_params`.
 
-**Document stores** use a separate namespace:
-```
-src/haystack_integrations/document_stores/<name>/
+## Fetch webpages or YouTube transcripts
+
+```python
+from haystack_integrations.components.fetchers.context import ContextFetcher
+
+fetcher = ContextFetcher()
+result = fetcher.run(
+    urls=[
+        "https://haystack.deepset.ai",
+        "https://www.youtube.com/watch?v=UF8uR6Z6KLc",
+    ]
+)
+
+documents = result["documents"]
 ```
-Import path: `from haystack_integrations.document_stores.<name> import MyDocumentStore`
 
-## Development
+Each URL becomes a Haystack `Document`. Webpages contain clean Markdown and page metadata; supported YouTube URLs return timestamped transcript Markdown.
 
-This project uses [Hatch](https://hatch.pypa.io/) for build and environment management.
+## Crawl a website
 
-```bash
-# Install Hatch
-pip install hatch
-
-# Format and lint
-hatch run fmt        # auto-fix
-hatch run fmt-check  # check only
-
-# Run tests
-hatch run test:unit         # unit tests only
-hatch run test:integration  # integration tests only
-hatch run test:all          # all tests
-hatch run test:cov          # with coverage
+```python
+from haystack_integrations.components.fetchers.context import ContextCrawler
+
+crawler = ContextCrawler(crawl_params={"maxPages": 25, "maxDepth": 2})
+result = crawler.run(urls=["https://docs.haystack.deepset.ai"])
+
+documents = result["documents"]
 ```
 
-## Publishing to PyPI
+`ContextCrawler` defaults to one page to prevent accidental credit consumption. Set `maxPages` explicitly for larger crawls.
+
+## Async usage
 
-This template includes a GitHub Actions workflow that publishes your package to PyPI when you push a version tag.
+```python
+result = await search.run_async(query="Haystack agents")
+documents = result["documents"]
+```
 
-1. **Add a `PYPI_API_TOKEN` secret** to your repository settings (Settings > Secrets and variables > Actions).
+The fetcher and crawler process multiple input URLs concurrently in their async methods.
 
-2. **Create a version tag** and push it:
-   ```bash
-   git tag v0.1.0
-   git push origin v0.1.0
-   ```
+## Development
 
-The release workflow will build and publish the package automatically.
+See [CONTRIBUTING.md](CONTRIBUTING.md) for the Hatch-based development and release workflow.
 
 ## License
 
-`Apache-2.0` - See [LICENSE](LICENSE) for details.
+Apache-2.0. See [LICENSE](LICENSE).

examples/context_pipeline.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: 2026-present Context.dev
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from haystack import Pipeline
+
+from haystack_integrations.components.websearch.context import ContextWebSearch
+
+pipeline = Pipeline()
+pipeline.add_component("search", ContextWebSearch(top_k=5, include_markdown=True))
+
+result = pipeline.run({"search": {"query": "What is Haystack by deepset?"}})
+for document in result["search"]["documents"]:
+    print(document.meta["url"])

examples/example.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2026-present AUTHOR <your@email.com>
+# SPDX-FileCopyrightText: 2026-present Context.dev
 #
 # SPDX-License-Identifier: Apache-2.0
 
@@ -7,19 +7,22 @@ requires = ["hatchling", "hatch-vcs"]
 build-backend = "hatchling.build"
 
 [project]
-name = "example-haystack"  # TODO: Replace with your package name, e.g. "deepset-ai-haystack"
+name = "context-dev-haystack"
 dynamic = ["version"]
-description = "A custom Haystack component"  # TODO: Replace with your description
+description = "Haystack components for Context.dev live web search, scraping, and crawling"
 readme = "README.md"
 requires-python = ">=3.10"
 license = "Apache-2.0"
 keywords = [
     "haystack",
-    # TODO: Add relevant keywords for your integration
+    "context-dev",
+    "web-search",
+    "web-scraping",
+    "web-crawling",
+    "rag",
 ]
 authors = [
-    # TODO: Replace with your name and email
-    { name = "AUTHOR", email = "your@email.com" },
+    { name = "Context.dev" },
 ]
 classifiers = [
     "Development Status :: 4 - Beta",
@@ -35,15 +38,16 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
 dependencies = [
-    "haystack-ai",
-    # TODO: Add your integration-specific dependencies here
+    "haystack-ai>=2.24.1",
+    "httpx>=0.27.0",
+    "requests>=2.32.0",
 ]
 
 [project.urls]
-# TODO: Replace with your repository URL
-Documentation = "https://github.com/your-org/example-haystack#readme"
-Issues = "https://github.com/your-org/example-haystack/issues"
-Source = "https://github.com/your-org/example-haystack"
+Homepage = "https://context.dev"
+Documentation = "https://github.com/context-dot-dev/context-haystack#readme"
+Issues = "https://github.com/context-dot-dev/context-haystack/issues"
+Source = "https://github.com/context-dot-dev/context-haystack"
 
 [tool.hatch.version]
 source = "vcs"
@@ -65,15 +69,20 @@ fmt-check = "ruff check {args:.} && ruff format --check {args:.}"
 
 [tool.hatch.envs.test]
 dependencies = [
+    "mypy",
     "pytest",
+    "pytest-asyncio",
     "pytest-cov",
+    "pytest-mock",
+    "types-requests",
 ]
 
 [tool.hatch.envs.test.scripts]
 unit = 'pytest -m "not integration" {args:tests}'
 integration = 'pytest -m "integration" {args:tests}'
 all = "pytest {args:tests}"
 cov = "pytest --cov=haystack_integrations {args:tests}"
+types = "mypy src tests"
 
 [tool.ruff]
 line-length = 120
@@ -138,10 +147,10 @@ ban-relative-imports = "parents"
 "examples/**/*" = ["D", "T201", "ANN"]
 
 [tool.mypy]
-install_types = true
-non_interactive = true
 check_untyped_defs = true
 disallow_incomplete_defs = true
+explicit_package_bases = true
+mypy_path = "src"
 
 [[tool.mypy.overrides]]
 module = ["haystack.*"]

pyproject.toml

@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: 2026-present Context.dev
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from haystack_integrations.components.fetchers.context.context_crawler import ContextCrawler
+from haystack_integrations.components.fetchers.context.context_fetcher import ContextFetcher
+from haystack_integrations.context import ContextError
+
+__all__ = ["ContextCrawler", "ContextError", "ContextFetcher"]