mkdocs.yml

site_name: Ragas
site_description: Evaluation framework for your AI Application
site_url: !ENV READTHEDOCS_CANONICAL_URL
repo_name: explodinggradients/ragas
repo_url: https://github.com/explodinggradients/ragas
watch:
  - src

# Navigation
nav:
  - "": index.md
  - 🚀 Get Started:
      - getstarted/index.md
      - Installation: getstarted/install.md
      - Evaluate your first LLM App: getstarted/evals.md
      - Evaluate a simple RAG: getstarted/rag_eval.md
      - Generate Synthetic Testset for RAG: getstarted/rag_testset_generation.md
      - Experiments:
          - Run your first experiment: getstarted/experiments_quickstart.md
          - Evaluate a prompt: tutorials/prompt.md
          - Evaluate a simple RAG system: tutorials/rag.md
          - Evaluate an AI Workflow: tutorials/workflow.md
          - Evaluate an AI Agent: tutorials/agent.md
  - 📚 Core Concepts:
      - concepts/index.md
      - Components:
          - concepts/components/index.md
          - General:
              - Prompt: concepts/components/prompt.md
          - Evaluation:
              - Evaluation Sample: concepts/components/eval_sample.md
              - Evaluation Dataset: concepts/components/eval_dataset.md
      - Metrics:
          - concepts/metrics/index.md
          - Overview: concepts/metrics/overview/index.md
          - Available Metrics:
              - concepts/metrics/available_metrics/index.md
              - Retrieval Augmented Generation:
                  - Context Precision: concepts/metrics/available_metrics/context_precision.md
                  - Context Recall: concepts/metrics/available_metrics/context_recall.md
                  - Context Entities Recall: concepts/metrics/available_metrics/context_entities_recall.md
                  - Noise Sensitivity: concepts/metrics/available_metrics/noise_sensitivity.md
                  - Response Relevancy: concepts/metrics/available_metrics/answer_relevance.md
                  - Faithfulness: concepts/metrics/available_metrics/faithfulness.md
              - Nvidia Metrics:
                  - Answer Accuracy: concepts/metrics/available_metrics/nvidia_metrics/#answer-accuracy
                  - Context Relevance: concepts/metrics/available_metrics/nvidia_metrics/#context-relevance
                  - Response Groundedness: concepts/metrics/available_metrics/nvidia_metrics/#response-groundedness
              - Agents or Tool Use Cases:
                    - concepts/metrics/available_metrics/agents.md
                    - Topic Adherence: concepts/metrics/available_metrics/agents/#topic-adherence
                    - Tool Call Accuracy: concepts/metrics/available_metrics/agents/#tool-call-accuracy
                    - Agent Goal Accuracy: concepts/metrics/available_metrics/agents/#agent-goal-accuracy
              - Natural Language Comparison:
                  - Factual Correctness: concepts/metrics/available_metrics/factual_correctness.md
                  - Semantic Similarity: concepts/metrics/available_metrics/semantic_similarity.md
                  - Traditional non LLM metrics:
                    - concepts/metrics/available_metrics/traditional.md
                    - Non LLM String Similarity: concepts/metrics/available_metrics/traditional/#non-llm-string-similarity
                    - BLEU Score: concepts/metrics/available_metrics/traditional/#bleu-score
                    - ROUGE Score: concepts/metrics/available_metrics/traditional/#rouge-score
                    - String Presence: concepts/metrics/available_metrics/traditional/#string-presence
                    - Exact Match: concepts/metrics/available_metrics/traditional/#exact-match
              - SQL:
                  - concepts/metrics/available_metrics/sql.md
                  - Execution based Datacompy Score: concepts/metrics/available_metrics/sql/#execution-based-metrics
                  - SQL Query Equivalence: concepts/metrics/available_metrics/sql/#sql-query-semantic-equivalence
              - General Purpose:
                  - concepts/metrics/available_metrics/general_purpose.md
                  - Aspect Critic: concepts/metrics/available_metrics/general_purpose/#aspect-critic
                  - Simple Criteria Scoring: concepts/metrics/available_metrics/general_purpose/#simple-criteria-scoring
                  - Rubrics Based Scoring: concepts/metrics/available_metrics/general_purpose/#rubrics-based-criteria-scoring
                  - Instance Specific Rubrics Scoring: concepts/metrics/available_metrics/general_purpose/#instance-specific-rubrics-criteria-scoring
              - Other Tasks:
                  - Summarization: concepts/metrics/available_metrics/summarization_score.md
      - Test Data Generation:
          - concepts/test_data_generation/index.md
          - RAG:
              - concepts/test_data_generation/rag.md
              - KG Building: concepts/test_data_generation/rag/#knowledge-graph-creation
              - Scenario Generation: concepts/test_data_generation/rag/#scenario-generation
          - Agents or tool use:
              - concepts/test_data_generation/agents.md
      - Feedback Intelligence:
          - concepts/feedback/index.md
      - Datasets: concepts/datasets.md
      - Experimentation: concepts/experimentation.md
  
  - 🛠️ How-to Guides:
      - howtos/index.md
      - Customizations:
          - howtos/customizations/index.md
          - General:
              - Customise models: howtos/customizations/customize_models.md
              - Run Config: howtos/customizations/_run_config.md
              - Caching: howtos/customizations/_caching.md
              - Cancelling Tasks: howtos/customizations/cancellation.md
          - Metrics:
              - Modify Prompts: howtos/customizations/metrics/_modifying-prompts-metrics.md
              - Adapt Metrics to Languages: howtos/customizations/metrics/_metrics_language_adaptation.md
              - Write your own Metrics: howtos/customizations/metrics/_write_your_own_metric.md
              - Write your own Metrics - (advanced): howtos/customizations/metrics/_write_your_own_metric_advanced.md
              - Train and Align Metrics: howtos/customizations/metrics/train_your_own_metric.md
          - Testset Generation:
              - Non-English Testset Generation: howtos/customizations/testgenerator/_language_adaptation.md
              - Persona Generation: howtos/customizations/testgenerator/_persona_generator.md
              - Custom Single-hop Query: howtos/customizations/testgenerator/_testgen-custom-single-hop.md
              - Custom Multi-hop Query: howtos/customizations/testgenerator/_testgen-customisation.md

      - Applications:
          - howtos/applications/index.md
          - Prompt Evaluation:
            - Iterate and Improve Prompts: howtos/applications/iterate_prompt.md
            - Systematic Prompt Optimization: howtos/applications/prompt_optimization.md
          - Metrics:
            - Cost Analysis: howtos/applications/_cost.md
            - Evaluating Multi-turn Conversations: howtos/applications/evaluating_multi_turn_conversations.md
            - Evaluations with Vertex AI models: howtos/applications/vertexai_x_ragas.md
          - Testset Generation:
            - Single-hop Query Testset: howtos/applications/singlehop_testset_gen.md
          - Benchmarking:
            - Evaluate a New LLM: howtos/applications/benchmark_llm.md
          - Agent Evaluation:
            - Evaluate a Text-to-SQL Agent: howtos/applications/text2sql.md
          - RAG Evaluation:
            - Evaluate and Improve a RAG App: howtos/applications/evaluate-and-improve-rag.md
      - Integrations:
          - howtos/integrations/index.md
          - Arize: howtos/integrations/_arize.md
          - Amazon Bedrock: howtos/integrations/amazon_bedrock.md
          - Haystack: howtos/integrations/haystack.md
          - Griptape: howtos/integrations/griptape.md
          - LangChain: howtos/integrations/langchain.md
          - OCI Gen AI: howtos/integrations/oci_genai.md
          - LangGraph: howtos/integrations/_langgraph_agent_evaluation.md
          - LangSmith: howtos/integrations/langsmith.md
          - LlamaIndex RAG: howtos/integrations/_llamaindex.md
          - LlamaIndex Agents: howtos/integrations/llamaindex_agents.md
          - LlamaStack: howtos/integrations/llama_stack.md
          - R2R: howtos/integrations/r2r.md
          - Swarm: howtos/integrations/swarm_agent_evaluation.md
      - Migrations:
          - From v0.1 to v0.2: howtos/migrations/migrate_from_v01_to_v02.md
  - 📖 References:
    - references/index.md
    - Core:
      - Prompt: references/prompt.md
      - LLMs: references/llms.md
      - Embeddings: references/embeddings.md
      - RunConfig: references/run_config.md
      - Executor: references/executor.md
      - Cache: references/cache.md
    - Evaluation:
      - Schemas: references/evaluation_schema.md
      - Metrics: references/metrics.md
      - evaluate(): references/evaluate.md
      - aevaluate(): references/aevaluate.md
    - Testset Generation:
      - Schemas: references/testset_schema.md
      - Graph: references/graph.md
      - Transforms: references/transforms.md
      - Synthesizers: references/synthesizers.md
      - Generation: references/generate.md
    - Integrations: references/integrations.md
  - ❤️ Community: community/index.md

# https://www.mkdocs.org/user-guide/configuration/#validation
validation:
  omitted_files: warn
  absolute_links: warn
  unrecognized_links: warn

# Material-Docs Theme
theme:
  name: material
  custom_dir: docs/extra/overrides
  logo: _static/imgs/ragas-logo.png
  favicon: _static/favicon.ico
  features:
    - announce.dismiss
    - content.tabs.link
    - content.code.annotate
    - content.code.copy
    - announce.dismiss
    - navigation.tabs
    - navigation.path
    - navigation.instant
    - navigation.instant.prefetch
    - navigation.instant.preview
    - navigation.sections
    - navigation.top
    - navigation.tracking
    - navigation.indexes
    - navigation.footer
    - search.suggest
    - search.highlight
  palette:
    - media: "(prefers-color-scheme)"
      toggle:
        icon: material/brightness-auto
        name: Switch to light mode
    - media: "(prefers-color-scheme: light)"
      scheme: default
      primary: "#bd8526"
      accent: "#bd8526"
      toggle:
        icon: material/brightness-7
        name: Switch to dark mode
    - media: "(prefers-color-scheme: dark)"
      scheme: slate
      primary: "#bd8526"
      accent: "#bd8526"
      toggle:
        icon: material/brightness-4
        name: Switch to system preference

markdown_extensions:
  - pymdownx.highlight:
      anchor_linenums: true
      line_spans: __span
      pygments_lang_class: true
  - admonition
  - pymdownx.inlinehilite
  - pymdownx.details
  - pymdownx.tabbed:
      alternate_style: true
  - pymdownx.emoji:
      emoji_index: !!python/name:material.extensions.emoji.twemoji
      emoji_generator: !!python/name:material.extensions.emoji.to_svg
  - attr_list
  - md_in_html
  - pymdownx.arithmatex:
      generic: true
  - pymdownx.superfences:
      custom_fences:
        - name: mermaid
          class: mermaid
          format: !!python/name:pymdownx.superfences.fence_code_format
  - pymdownx.snippets:
      base_path: ["./docs/extra/components/"]

# Extra CSS
extra_css:
  - extra/ragas-modern.css

# Plugins
extra:
  version:
    provider: mike
  analytics:
    provider: google
    property: !ENV GOOGLE_ANALYTICS_KEY
plugins:
  - social
  - search
  - git-revision-date-localized:
      enabled: !ENV [MKDOCS_CI, false]
      enable_creation_date: true
  - git-committers:
      enabled: !ENV [MKDOCS_CI, false]
      repository: explodinggradients/ragas
      branch: main
  - mkdocstrings:
      handlers:
        python:
          paths: [src]
          options:
            docstring_style: numpy
            members_order: source
            separate_signature: true
            filters: ["!^_"]
            docstring_options:
              ignore_init_summary: true
            merge_init_into_class: true
            show_signature_annotations: true
            signature_crossrefs: true
  - glightbox
  # - gen-files:
  #     scripts:
  #       - docs/ipynb_to_md.py
extra_javascript:
  - _static/js/mathjax.js
  - _static/js/header_border.js
  - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js
  - _static/js/toggle.js
  - https://cdn.octolane.com/tag.js?pk=c7c9b2b863bf7eaf4e2a # octolane for analytics
  - _static/js/commonroom.js # commonroom analytics