Merge pull request #34 from KRLabsOrg/code_hallucination

Code hallucination

Author: adaamko

.github/workflows/docs.yml

@@ -0,0 +1,43 @@
+name: Deploy Documentation
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docs/**'
+      - 'mkdocs.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: pip install mkdocs-material "mkdocstrings[python]>=0.24"
+      - run: pip install -e .
+      - run: mkdocs build
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: site/
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/python-tests.yml

@@ -1,8 +1,10 @@
 name: Python Tests
 
 on:
+  push:
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
 
 jobs:
   test:
@@ -12,24 +14,24 @@ jobs:
         python-version: ["3.12"]
 
     steps:
-    - uses: actions/checkout@v3
-    
+    - uses: actions/checkout@v4
+
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
-    
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -e ".[dev]"
-    
+
     - name: Lint with ruff
       run: |
-        ruff format --check --diff .
-        ruff check --select I .
-    
+        ruff format --check --diff lettucedetect/ tests/
+        ruff check lettucedetect/ tests/ --extend-exclude lettucedetect/integrations/
+
     - name: Test with pytest
       run: |
-        pytest tests/test_inference_pytest.py -v 
+        pytest tests/test_inference_pytest.py -v

.gitignore

@@ -170,6 +170,9 @@ cython_debug/
 # PyPI configuration file
 .pypirc
 
+# macOS
+.DS_Store
+
 # data/
 data/
 
@@ -178,4 +181,5 @@ output/
 temp/
 
 # cache/
-lettucedetect/cache/
+lettucedetect/cache/
+testing/

docs/EUROBERT.md

@@ -1,7 +1,7 @@
 # 🥬 LettuceDetect Goes Multilingual: Fine-tuning EuroBERT on Synthetic RAGTruth Translations
 
 <p align="center">
-  <img src="https://github.com/KRLabsOrg/LettuceDetect/blob/feature/cn_llm_eval/assets/lettuce_detective_multi.png?raw=true" alt="LettuceDetect Multilingual Task Force" width="520"/>
+  <img src="https://github.com/KRLabsOrg/LettuceDetect/blob/main/assets/lettuce_detective_multi.png?raw=true" alt="LettuceDetect Multilingual Task Force" width="520"/>
   <br>
   <em>Expanding hallucination detection across languages for RAG pipelines.</em>
 </p>

docs/EVALUATION.md

@@ -0,0 +1,13 @@
+# Datasets
+
+## HallucinationSample
+
+::: lettucedetect.datasets.hallucination_dataset.HallucinationSample
+
+## HallucinationData
+
+::: lettucedetect.datasets.hallucination_dataset.HallucinationData
+
+## HallucinationDataset
+
+::: lettucedetect.datasets.hallucination_dataset.HallucinationDataset

docs/api/datasets.md

@@ -0,0 +1,13 @@
+# Detectors
+
+## Factory
+
+::: lettucedetect.detectors.factory.make_detector
+
+## Base Detector
+
+::: lettucedetect.detectors.base.BaseDetector
+
+## Transformer Detector
+
+::: lettucedetect.detectors.transformer.TransformerDetector

docs/api/detectors.md

@@ -0,0 +1,5 @@
+# Inference
+
+The main entry point for hallucination detection.
+
+::: lettucedetect.models.inference.HallucinationDetector

docs/api/inference.md

@@ -0,0 +1,9 @@
+# Training
+
+## Trainer
+
+::: lettucedetect.models.trainer.Trainer
+
+## Evaluator
+
+::: lettucedetect.models.evaluator

docs/api/training.md

@@ -0,0 +1,43 @@
+# Benchmarks
+
+## RAGTruth (English)
+
+Evaluated on the [RAGTruth](https://aclanthology.org/2024.acl-long.585/) test set. This benchmark measures how well models detect hallucinations in LLM-generated text across QA, summarization, and data-to-text tasks.
+
+### Example-Level Detection
+
+Binary classification: does the answer contain any hallucination?
+
+| Model | Type | Overall F1 |
+|-------|------|-----------|
+| GPT-4 | LLM (zero-shot) | 63.4% |
+| Luna | Encoder | 65.4% |
+| **lettucedetect-base-v1** | **Encoder (149M)** | **76.8%** |
+| Llama-2-13B (fine-tuned) | LLM | 78.7% |
+| **lettucedetect-large-v1** | **Encoder (395M)** | **79.2%** |
+| RAG-HAT (Llama-3-8B) | LLM | 83.9% |
+
+### Span-Level Detection
+
+LettuceDetect achieves state-of-the-art span-level results among models that report this metric, outperforming fine-tuned Llama-2-13B. Span-level evaluation measures how precisely the model can locate the exact hallucinated text within an answer.
+
+## What These Numbers Mean
+
+- **Example F1** — Can the model tell if an answer has *any* hallucination? Higher is better.
+- **Span F1** — Can the model point to *exactly which parts* are hallucinated? This is the harder task and where LettuceDetect excels relative to its size.
+
+LettuceDetect models are 50-500x smaller than LLM-based detectors while achieving competitive or better accuracy.
+
+## Citation
+
+```bibtex
+@misc{Kovacs:2025,
+      title={LettuceDetect: A Hallucination Detection Framework for RAG Applications},
+      author={Ádám Kovács and Gábor Recski},
+      year={2025},
+      eprint={2502.17125},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2502.17125},
+}
+```