mkdocs.yml

site_name: Splink
site_url: https://moj-analytical-services.github.io/splink/
use_directory_urls: false
repo_url: https://github.com/moj-analytical-services/splink
edit_uri: edit/master/docs/
theme:
  icon:
    repo: fontawesome/brands/github
  name: "material"
  features:
    - content.code.annotate
    - content.code.copy
    - content.tabs.link
    - content.tooltips
    - header.autohide
    - search.highlight
    - search.share
    - search.suggest
    - navigation.indexes
    - navigation.footer
    - content.action.edit
    - navigation.tabs
    - navigation.tabs.sticky
    - navigation.top
    - toc.follow
  logo: "img/favicon.ico"
  favicon: "img/favicon.ico"
  palette:
    - scheme: default
      primary: teal
      accent: purple
      toggle:
        icon: material/toggle-switch
        name: Switch to dark mode
    - scheme: slate
      primary: teal
      accent: purple
      toggle:
        icon: material/toggle-switch-off-outline
        name: Switch to light mode
  custom_dir: docs/overrides
plugins:
  - blog:
      post_date_format: long
      archive: false
  - search
  - mknotebooks:
      tag_remove_configs:
        remove_input_tags:
          - hide_input
        remove_all_outputs_tags:
          - hide_output
  - tags
  # automatic documentation of python objects from docstrings
  - mkdocstrings:
      default_handler: python
      handlers:
        python:
          options:
            show_source: false
  # embedding videos, see e.g. docs/dev_guides/charts/understanding_and_editing_charts.md
  - mkdocs-video:
      is_video: True
      video_autoplay: false
  - rss:
      match_path: blog/posts/.*
      date_from_meta:
        as_creation: date
  - llmstxt:
      markdown_description: |
        Splink is a Python library for probabilistic record linkage, deduplication,
        and entity resolution across backends such as DuckDB, Spark, SQLite, and
        PostgreSQL. It is aimed at data scientists, analysts, researchers, and
        engineers building or maintaining linkage pipelines.

        Agents should fetch /llms.txt first as a compact map of the documentation,
        then fetch only the linked Markdown pages that match the task. Start with
        the overview and getting started pages for orientation, the core concepts
        pages to understand settings, blocking, comparisons, and theory, the
        tutorials and examples for end-to-end workflows, and the API reference for
        method-level detail.
      full_output: llms-full.txt
      sections:
        Overview:
          - index.md: Product overview, capabilities, and high-level positioning.
          - topic_guides/topic_guides_index.md: Map of the conceptual user guide.
          - api_docs/api_docs_index.md: Map of the API reference.
        Getting started:
          - getting_started.md: Installation and the first end-to-end Splink workflow.
        Tutorials and examples:
          - demos/examples/examples_index.md: Index of runnable examples by backend and problem type.
        "User guide: record linkage theory":
          - topic_guides/theory/*.md
        "User guide: Splink fundamentals":
          - topic_guides/splink_fundamentals/*.md
          - topic_guides/splink_fundamentals/backends/*.md
        "User guide: data preparation":
          - topic_guides/data_preparation/*.md
        "User guide: blocking":
          - topic_guides/blocking/*.md
        "User guide: comparisons":
          - topic_guides/comparisons/*.md
        "User guide: training":
          - topic_guides/training/*.md
        "User guide: evaluation":
          - topic_guides/evaluation/*.md
          - topic_guides/evaluation/clusters/*.md
        "User guide: performance":
          - topic_guides/performance/*.md
        API reference:
          - api_docs/training.md: Training methods on the Linker.
          - api_docs/inference.md: Inference and prediction methods.
          - api_docs/linker_clustering.md: Clustering APIs.
          - api_docs/evaluation.md: Evaluation APIs.
          - api_docs/comparison_library.md: Built-in comparison helpers.
          - api_docs/comparison_level_library.md: Built-in comparison level helpers.
          - api_docs/blocking.md: Blocking API reference.
          - api_docs/datasets.md: Built-in datasets for demos and tests.
        Contributor and debugging guides:
          - dev_guides/changing_splink/development_quickstart.md: Contributor setup when the task involves changing Splink itself.
          - dev_guides/debug_modes.md: Debugging and inspection modes when behavior is unclear.
hooks:
  - docs/hooks/__init__.py

markdown_extensions:
  # add id/style etc to markdown elements - see getting started button in docs/index.md
  # markdown
  - attr_list
  # for frontmatter in --- blocks, for e.g. tags
  # mkdocs-material
  - meta
  # for admonition blocks e.g. !!! warning
  # markdown
  - admonition
  # markdown in html blocks, e.g. <figure> in docs/topic_guides/comparisons/term-frequency.md
  # markdown
  - md_in_html
  # for expandables with ???
  # pymdown-extensions
  - pymdownx.details
  # for inline latex - see e.g. docs/topic_guides/theory/fellegi_sunter.md
  # pymdown-extensions
  - pymdownx.arithmatex:
      generic: true
  # code markdown in admonitions, e.g. docs/getting_started.md
  - pymdownx.superfences
  # tabbed sections with === - e.g. docs/index.md
  # pymdown-extensions
  - pymdownx.tabbed:
      alternate_style: true
  # table of contents
  # markdown
  - toc:
      permalink: True
      toc_depth: 3
  # emojis specified as e.g. :link: - see e.g. docs/blog/posts/2024-01-25-ethics.md
  # pymdown-extensions
  - pymdownx.emoji:
      emoji_index: !!python/name:material.extensions.emoji.twemoji
      emoji_generator: !!python/name:material.extensions.emoji.to_svg
        # footnotes see e.g. docs/topic_guides/theory/fellegi_sunter.md
        # mkdocs-material

  - footnotes
  # ::cards:: - see e.g. docs/charts/index.md
  # neoteroi-mkdocs
  - neoteroi.cards
nav:
  - Getting Started: "getting_started.md"
  - Tutorial:
      - Introduction: "demos/tutorials/00_Tutorial_Introduction.ipynb"
      - 1. Data prep prerequisites: "demos/tutorials/01_Prerequisites.ipynb"
      - 2. Exploratory analysis: "demos/tutorials/02_Exploratory_analysis.ipynb"
      - 3. Blocking: "demos/tutorials/03_Blocking.ipynb"
      - 4. Estimating model parameters: "demos/tutorials/04_Estimating_model_parameters.ipynb"
      - 5. Predicting results: "demos/tutorials/05_Predicting_results.ipynb"
      - 6. Visualising predictions: "demos/tutorials/06_Visualising_predictions.ipynb"
      - 7. Evaluation: "demos/tutorials/07_Evaluation.ipynb"
      - 8. Tips for building your own model: "demos/tutorials/08_building_your_own_model.md"
      - 9. Scaling up to large datasets: "demos/tutorials/09_scaling_up_techniques.ipynb"
  - Examples:
      - Introduction: "demos/examples/examples_index.md"
      - DuckDB:
        - Deduplicate 50k rows historical persons: "demos/examples/duckdb/deduplicate_50k_synthetic.ipynb"
        - Linking financial transactions: "demos/examples/duckdb/transactions.ipynb"
        - Linking businesses: "demos/examples/duckdb_no_test/business_rates_match.ipynb"
        - Linking two tables of persons: "demos/examples/duckdb/link_only.ipynb"
        - Real time record linkage: "demos/examples/duckdb/real_time_record_linkage.ipynb"
        - Evaluation from ground truth column: "demos/examples/duckdb/accuracy_analysis_from_labels_column.ipynb"
        - Estimating m probabilities from labels: "demos/examples/duckdb/pairwise_labels.ipynb"
        - Quick and dirty persons model: "demos/examples/duckdb/quick_and_dirty_persons.ipynb"
        - Deterministic dedupe: "demos/examples/duckdb/deterministic_dedupe.ipynb"
        - Febrl3 Dedupe: "demos/examples/duckdb/febrl3.ipynb"
        - Febrl4 link-only: "demos/examples/duckdb/febrl4.ipynb"
        - Cookbook: "demos/examples/duckdb_no_test/cookbook.ipynb"
        - Investigating Bias in a Splink Model: "demos/examples/duckdb_no_test/bias_eval.ipynb"
        - Comparison playground: "demos/examples/duckdb_no_test/comparison_playground.ipynb"
        - Pseudopeople Census to ACS link: "demos/examples/duckdb_no_test/pseudopeople-acs.ipynb"
      - PySpark:
        - Deduplication using Pyspark: "demos/examples/spark/deduplicate_1k_synthetic.ipynb"
      - SQLite:
        - Deduplicate 50k rows historical persons: "demos/examples/sqlite/deduplicate_50k_synthetic.ipynb"
  - API Docs:
    # This should align with docs/api_docs/api_docs_index.md
    - Introduction: "api_docs/api_docs_index.md"
    - Linker:
      - Training: "api_docs/training.md"
      - Visualisations: "api_docs/visualisations.md"
      - Inference: "api_docs/inference.md"
      - Clustering: "api_docs/linker_clustering.md"
      - Evaluation: "api_docs/evaluation.md"
      - Table Management: "api_docs/table_management.md"
      - Miscellaneous functions: "api_docs/misc.md"
    - Comparisons Library:
        - Comparison Library: "api_docs/comparison_library.md"
        - Comparison Level Library: "api_docs/comparison_level_library.md"
    - Other:
        - Exploratory: "api_docs/exploratory.md"
        - Blocking analysis: "api_docs/blocking_analysis.md"
        - Blocking: "api_docs/blocking.md"
        - Clustering: "api_docs/clustering.md"
        - SplinkDataFrame: "api_docs/splink_dataframe.md"
        - EM Training Session API: "api_docs/em_training_session.md"
        - Column Expressions: "api_docs/column_expression.md"
    - In-build datasets:
        - SplinkDatasets: "api_docs/datasets.md"
    - Splink Settings:
        - Settings Dict: "api_docs/settings_dict_guide.md"
  - User Guide:
    - Introduction: "topic_guides/topic_guides_index.md"
    - Record Linkage Theory:
      - Why do we need record linkage?: "topic_guides/theory/record_linkage.md"
      - Probabilistic vs Deterministic linkage: "topic_guides/theory/probabilistic_vs_deterministic.md"
      - The Fellegi-Sunter Model: "topic_guides/theory/fellegi_sunter.md"
      - Linked Data as Graphs: "topic_guides/theory/linked_data_as_graphs.md"
    - Linkage Models in Splink:
      - Defining Splink models: "topic_guides/splink_fundamentals/settings.md"
      - Retrieving and querying Splink results: "topic_guides/splink_fundamentals/querying_splink_results.md"
      - Link type - linking vs deduping: "topic_guides/splink_fundamentals/link_type.md"
      - Splink's SQL backends - Spark, DuckDB etc:
        - Backends overview: "topic_guides/splink_fundamentals/backends/backends.md"
        - PostgreSQL: "topic_guides/splink_fundamentals/backends/postgres.md"
    - Data Preparation:
      - Feature Engineering: "topic_guides/data_preparation/feature_engineering.md"
    - Blocking:
      - What are Blocking Rules?: "topic_guides/blocking/blocking_rules.md"
      - Computational Performance: "topic_guides/blocking/performance.md"
      - Model Training Blocking Rules: "topic_guides/blocking/model_training.md"
    - Comparing Records:
      - Comparisons and comparison levels: "topic_guides/comparisons/comparisons_and_comparison_levels.md"
      - Defining and customising comparisons: "topic_guides/comparisons/customising_comparisons.ipynb"
      - Out-of-the-box comparisons: "topic_guides/comparisons/out_of_the_box_comparisons.ipynb"
      - Term frequency adjustments: "topic_guides/comparisons/term-frequency.md"
      - Comparing strings:
        - String comparators: "topic_guides/comparisons/comparators.md"
        - Choosing string comparators: "topic_guides/comparisons/choosing_comparators.ipynb"
        - Phonetic algorithms: "topic_guides/comparisons/phonetic.md"
        - Regular expressions: "topic_guides/comparisons/regular_expressions.ipynb"
    - Training:
      - Training rationale: "topic_guides/training/training_rationale.md"
    - Evaluation:
      - Overview: "topic_guides/evaluation/overview.md"
      - Model: "topic_guides/evaluation/model.md"
      - Edges (Links):
        - Overview: "topic_guides/evaluation/edge_overview.md"
        - Edge Metrics: "topic_guides/evaluation/edge_metrics.md"
        - Clerical Labelling: "topic_guides/evaluation/labelling.md"
      - Clusters:
        - Overview: "topic_guides/evaluation/clusters/overview.md"
        - Graph metrics: "topic_guides/evaluation/clusters/graph_metrics.md"
        - How to compute graph metrics: "topic_guides/evaluation/clusters/how_to_compute_metrics.ipynb"
    - Performance:
      - Run times, performance and linking large data: "topic_guides/performance/drivers_of_performance.md"
      - Performance of comparison functions: "topic_guides/performance/performance_of_comparison_functions.ipynb"
      - Spark Performance:
          - Optimising Spark performance: "topic_guides/performance/optimising_spark.md"
      - DuckDB Performance:
          - Optimising DuckDB performance: "topic_guides/performance/optimising_duckdb.md"
    - Charts Gallery:
      - "charts/index.md"
      - Exploratory Analysis:
        - completeness chart: "charts/completeness_chart.ipynb"
        - profile columns: "charts/profile_columns.ipynb"
      - Blocking:
        - cumulative num comparisons from blocking rules chart: "charts/cumulative_comparisons_to_be_scored_from_blocking_rules_chart.ipynb"
      - Similarity analysis:
        - Comparator score chart: "topic_guides/comparisons/choosing_comparators.html#comparing-string-similarity-and-distance-scores"
        - Comparator score threshold chart: "topic_guides/comparisons/choosing_comparators.html#choosing-thresholds"
        - Phonetic match chart: "topic_guides/comparisons/choosing_comparators.html#phonetic-matching"
      - Model Training:
        - comparison viewer dashboard: "charts/comparison_viewer_dashboard.ipynb"
        - match weights chart: "charts/match_weights_chart.ipynb"
        - m u parameters chart: "charts/m_u_parameters_chart.ipynb"
        - parameter estimate comparisons chart: "charts/parameter_estimate_comparisons_chart.ipynb"
        - tf adjustment chart: "charts/tf_adjustment_chart.ipynb"
        - unlinkables chart: "charts/unlinkables_chart.ipynb"
        - waterfall chart: "charts/waterfall_chart.ipynb"
      - Clustering:
        - cluster studio dashboard: "charts/cluster_studio_dashboard.ipynb"
      - Model Evaluation:
        - accuracy chart from labels table: "charts/accuracy_analysis_from_labels_table.ipynb"
        - threshold selection tool: charts/threshold_selection_tool_from_labels_table.ipynb
  - Contributing:
      - "dev_guides/index.md"
      - Contributing to Splink:
        - Contributor Guide: "dev_guides/CONTRIBUTING.md"
        - Development Quickstart: "dev_guides/changing_splink/development_quickstart.md"
        - Linting and Formatting: "dev_guides/changing_splink/lint_and_format.md"
        - Testing: "dev_guides/changing_splink/testing.md"
        - Contributing to Documentation: "dev_guides/changing_splink/contributing_to_docs.md"
        - Managing Environment and Dependencies: "dev_guides/changing_splink/managing_dependencies_with_uv.md"
        - Releasing a Package Version: "dev_guides/changing_splink/releases.md"
        - Contributing to the Splink Blog: "dev_guides/changing_splink/blog_posts.md"
      - How Splink works:
        - Understanding and debugging Splink: "dev_guides/debug_modes.md"
        - Transpilation using sqlglot: "dev_guides/transpilation.md"
        - Performance and caching:
          - Caching and pipelining: "dev_guides/caching.md"
          - Spark caching: "dev_guides/spark_pipelining_and_caching.md"
        - Charts:
          - Understanding and editing charts: "dev_guides/charts/understanding_and_editing_charts.md"
          - Building new charts: "dev_guides/charts/building_charts.ipynb"
        - User-Defined Functions: "dev_guides/udfs.md"
        - Dependency Compatibility Policy: "dev_guides/dependency_compatibility_policy.md"
  - Blog:
    - blog/index.md
extra_css:
- css/custom.css
- css/neoteroi-mkdocs.css
extra_javascript:
  - javascripts/mathjax.js
  - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js
extra:
  analytics:
    provider: google
    property: G-L6JT8NK528
  social:
    - icon: fontawesome/brands/github
      link: https://github.com/moj-analytical-services/splink
    - icon: fontawesome/brands/python
      link: https://pypi.org/project/splink/
    - icon: fontawesome/solid/chevron-right
      link: https://www.robinlinacre.com/
    - icon: fontawesome/solid/rss
      link: https://moj-analytical-services.github.io/splink/feed_rss_created.xml
  new: Recently added