.gitlab-ci.yml

# GitLab CI/CD Pipeline for n8n-deploy
# Python CLI tool for n8n workflow management
# Optimized for parallel stage execution

stages:
  - security
  - quality
  - test
  - build-matrix
  - release

# Docker services for caching (only used by Docker-specific jobs)
# Uncomment and configure for jobs that need Docker-in-Docker:
# services:
#   - name: docker:20.10.16-dind
#     command: [
#       "--mtu=1300",  # Fix TLS handshake timeout issues
#       "--registry-mirror", "http://registry-mirror.pirouter.dev:5000"  # Use local mirror (when available)
#     ]

variables:
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  PYTHON_VERSION: "3.9"
  # Docker caching configuration
  DOCKER_DRIVER: overlay2
  DOCKER_TLS_CERTDIR: "/certs"
  DOCKER_BUILDKIT: 1
  # Docker Registry Configuration:
  # - pull_policy: if-not-present (configured in GitLab Runner config.toml)
  # - registry_mirror: http://registry-mirror.pirouter.dev:5000 (configured in GitLab Runner config.toml)
  # - MTU: 1300 for DinD (see services comment above)
  # APT proxy for faster package installation
  APT_PROXY: "192.168.76.5:3142"
  # PyPI deployment configuration (set in CI/CD variables)
  # PYPI_TOKEN: <set in GitLab CI/CD settings>
  # TEST_PYPI_TOKEN: <set in GitLab CI/CD settings>
  # Git configuration for proper versioning
  GIT_DEPTH: 0  # Full clone with all tags for setuptools_scm version detection

# Global defaults for all jobs
default:
  image: python:3.9-slim
  tags:
    - python
  retry:
    max: 2
    when:
      - runner_system_failure
      - stuck_or_timeout_failure
      - scheduler_failure
  interruptible: true

# Workflow rules - define when pipeline runs
# Push to protected branches (master/develop): full pipeline
# MR targeting protected branches: full pipeline
# Conventional commit branches (feat/, fix/, etc.): security stage only
workflow:
  rules:
    # Run on master/develop (protected branches)
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "develop"'
      when: always
    # Run on all tags
    - if: '$CI_COMMIT_TAG'
      when: always
    # Run on merge requests targeting master or develop
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop")'
      when: always
    # Manual workflow dispatch
    - if: '$CI_PIPELINE_SOURCE == "web"'
      when: always
    # Run on conventional commit branches (security checks only)
    - if: '$CI_COMMIT_BRANCH =~ /^(feat|fix|chore|refactor|docs|test|style|perf|ci|build)\//'
      when: always
    # Skip for all other cases
    - when: never

# Cache for pip dependencies (default: pull-push)
cache:
  key:
    files:
      - requirements.txt
    prefix: ${CI_COMMIT_REF_SLUG}
  fallback_keys:
    - develop
    - master
  paths:
    - .cache/pip/
    - .venv/

# Cache template for read-only jobs (policy: pull)
.cache_pull: &cache_pull
  key:
    files:
      - requirements.txt
    prefix: ${CI_COMMIT_REF_SLUG}
  fallback_keys:
    - develop
    - master
  paths:
    - .cache/pip/
    - .venv/
  policy: pull

# Template for Python environment setup (uses cached venv from warm-cache job)
.python_setup: &python_setup
  before_script:
    - echo "Acquire::http::Proxy \"http://${APT_PROXY}\";" > /etc/apt/apt.conf.d/01proxy
    - apt-get -q update && apt-get -qy install git
    - python -m pip install --quiet --upgrade pip
    - pip install --quiet uv
    - test -f .venv/bin/activate && echo "✓ Using cached venv" || echo "⚠ Cache miss - creating venv"
    - test -f .venv/bin/activate || uv venv --python python3 .venv
    - source .venv/bin/activate
    - test -f .venv/lib/python*/site-packages/n8n_deploy.egg-info || uv pip install --quiet -e ".[test,dev]"

# ============================================
# PRE STAGE - Cache warming
# ============================================

warm-cache:
  stage: .pre
  # Use dualbuntu image for better performance
  tags:
    - cache
  rules:
    - when: on_success
  cache:
    key:
      files:
        - requirements.txt
      prefix: ${CI_COMMIT_REF_SLUG}
    fallback_keys:
      - develop
      - master
    paths:
      - .cache/pip/
      - .venv/
    policy: pull-push
  before_script:
    - echo "Acquire::http::Proxy \"http://${APT_PROXY}\";" > /etc/apt/apt.conf.d/01proxy
    - apt-get -q update && apt-get -qy install git
    - python -m pip install --quiet --upgrade pip
    - pip install --quiet uv
  script:
    - echo "Warming cache..."
    - uv venv --python python3 .venv
    - uv pip install --quiet -e ".[test,dev]"
    - echo "Cache populated successfully"

# ============================================
# SECURITY STAGE
# ============================================

# Secret detection with gitleaks
secret_detection:
  stage: security
  tags:
    - linux
    - python
  rules:
    - if: '$CI_COMMIT_TAG'
      when: never
    # Skip on MR pipelines (already ran on feature branch push)
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
    # Run on conventional commit branches (feature branch push)
    - if: '$CI_COMMIT_BRANCH =~ /^(feat|fix|chore|refactor|docs|test|style|perf|ci|build)\//'
      when: on_success
    # Run on protected branches (post-merge)
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "develop"'
      when: on_success
  cache:
    <<: *cache_pull
  needs:
    - warm-cache
  variables:
    SECRET_DETECTION_EXCLUDED_PATHS: "tests/"
  before_script:
    - echo "Acquire::http::Proxy \"http://${APT_PROXY}\";" > /etc/apt/apt.conf.d/01proxy
    - apt-get -q update && apt-get -qy install git wget
    - wget -qO- https://github.com/gitleaks/gitleaks/releases/download/v8.29.1/gitleaks_8.29.1_linux_x64.tar.gz | tar xz -C /usr/local/bin
  script:
    - echo "🔍 Running secret detection with gitleaks..."
    - gitleaks git . --verbose --redact --config .gitleaks.toml
    - echo "✅ Secret detection completed"

# Dependency vulnerability scanning
dependency_scanning:
  stage: security
  tags:
    - linux
    - python
  needs:
    - warm-cache
  rules:
    - if: '$CI_COMMIT_TAG'
      when: never
    # Skip on MR pipelines (already ran on feature branch push)
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
    # Run on conventional commit branches (feature branch push)
    - if: '$CI_COMMIT_BRANCH =~ /^(feat|fix|chore|refactor|docs|test|style|perf|ci|build)\//'
      when: on_success
    # Run on protected branches (post-merge)
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "develop"'
      when: on_success
  cache:
    <<: *cache_pull
  before_script:
    - echo "Acquire::http::Proxy \"http://${APT_PROXY}\";" > /etc/apt/apt.conf.d/01proxy
    - apt-get -q update && apt-get -qy install git
    - python -m pip install --quiet --upgrade pip
    - pip install --quiet uv
    - test -f .venv/bin/activate && echo "✓ Using cached venv" || echo "⚠ Cache miss - creating venv"
    - test -f .venv/bin/activate || uv venv --python python3 .venv
    - source .venv/bin/activate
    - test -f .venv/lib/python*/site-packages/n8n_deploy.egg-info || uv pip install --quiet -e ".[test,dev]"
    - uv pip install --quiet pip-audit
  script:
    - echo "🔍 Running dependency vulnerability scanning..."
    - pip-audit -r requirements.txt
    - echo "✅ Dependency scanning completed"

# Python security scanning with Bandit
security:bandit:
  stage: security
  tags:
    - linux
    - python
  needs:
    - warm-cache
  rules:
    - if: '$CI_COMMIT_TAG'
      when: never
    # Skip on MR pipelines (already ran on feature branch push)
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
    # Run on conventional commit branches (feature branch push)
    - if: '$CI_COMMIT_BRANCH =~ /^(feat|fix|chore|refactor|docs|test|style|perf|ci|build)\//'
      when: on_success
    # Run on protected branches (post-merge)
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "develop"'
      when: on_success
  cache:
    <<: *cache_pull
  variables:
    SAST_EXCLUDED_PATHS: "tests/, .venv/"
  before_script:
    - echo "Acquire::http::Proxy \"http://${APT_PROXY}\";" > /etc/apt/apt.conf.d/01proxy
    - apt-get -q update && apt-get -qy install git
    - python -m pip install --quiet --upgrade pip
    - pip install --quiet uv
    - test -f .venv/bin/activate && echo "✓ Using cached venv" || echo "⚠ Cache miss - creating venv"
    - test -f .venv/bin/activate || uv venv --python python3 .venv
    - source .venv/bin/activate
    - test -f .venv/lib/python*/site-packages/n8n_deploy.egg-info || uv pip install --quiet -e ".[test,dev]"
    - pip install --quiet bandit
  script:
    - echo "🔍 Running Python security analysis with Bandit..."
    - bandit -r api/ -f json --exclude tests/
    - echo "✅ Bandit analysis completed"

# Pattern-based security scanning with Semgrep
security:semgrep:
  stage: security
  tags:
    - linux
    - python
  needs:
    - warm-cache
  rules:
    - if: '$CI_COMMIT_TAG'
      when: never
    # Skip on MR pipelines (already ran on feature branch push)
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
    # Run on conventional commit branches (feature branch push)
    - if: '$CI_COMMIT_BRANCH =~ /^(feat|fix|chore|refactor|docs|test|style|perf|ci|build)\//'
      when: on_success
    # Run on protected branches (post-merge)
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "develop"'
      when: on_success
  cache:
    <<: *cache_pull
  variables:
    SAST_EXCLUDED_PATHS: "tests/, .venv/"
  before_script:
    - echo "Acquire::http::Proxy \"http://${APT_PROXY}\";" > /etc/apt/apt.conf.d/01proxy
    - apt-get -q update && apt-get -qy install git
    - python -m pip install --quiet --upgrade pip
    - pip install --quiet uv
    - test -f .venv/bin/activate && echo "✓ Using cached venv" || echo "⚠ Cache miss - creating venv"
    - test -f .venv/bin/activate || uv venv --python python3 .venv
    - source .venv/bin/activate
    - test -f .venv/lib/python*/site-packages/n8n_deploy.egg-info || uv pip install --quiet -e ".[test,dev]"
    - pip install --quiet semgrep
  script:
    - echo "🔍 Running pattern-based security analysis with Semgrep..."
    - semgrep --config=auto api/ --json --error
    - echo "✅ Semgrep analysis completed"

# ============================================
# QUALITY STAGE
# ============================================

# Type checking with mypy
quality:mypy:
  stage: quality
  tags:
    - alpine
    - linux
  needs:
    - warm-cache
  rules:
    - if: '$CI_COMMIT_TAG'
      when: never
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
    - if: '$CI_COMMIT_BRANCH =~ /^(feat|fix|chore|refactor|docs|test|style|perf|ci|build)\//'
      when: on_success
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "develop"'
      when: on_success
  <<: *python_setup
  cache:
    <<: *cache_pull
  script:
    - mypy api/ --strict --show-error-codes
    - echo "Type checking passed"

# Code formatting check with black
quality:black:
  stage: quality
  tags:
    - linux
    - python
  needs:
    - warm-cache
  rules:
    - if: '$CI_COMMIT_TAG'
      when: never
    # Skip on MR pipelines (already ran on feature branch push)
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
    # Run on conventional commit branches (feature branch push)
    - if: '$CI_COMMIT_BRANCH =~ /^(feat|fix|chore|refactor|docs|test|style|perf|ci|build)\//'
      when: on_success
    # Run on protected branches (post-merge)
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "develop"'
      when: on_success
  <<: *python_setup
  cache:
    <<: *cache_pull
  script:
    - black --check api/
    - echo "Code formatting check passed"

# ============================================
# AFFECTED TESTING (MR pipelines only)
# Uses pytest-testmon for coverage-based test selection
# ============================================

# Affected tests for merge request pipelines
test:affected:
  stage: test
  tags:
    - heavy
    - tests
  cache:
    # Use multiple caches: venv (branch-specific) + testmondata (shared)
    - key:
        files:
          - requirements.txt
        prefix: ${CI_COMMIT_REF_SLUG}
      fallback_keys:
        - develop
        - master
      paths:
        - .cache/pip/
        - .venv/
      policy: pull
    - key: testmondata-baseline
      paths:
        - .testmondata
        #- .pytest_cache
        - .hypothesis
      policy: pull-push
  needs:
    - warm-cache
  rules:
    - if: '$CI_COMMIT_TAG'
      when: on_success
    # Run on conventional commit branches (feature branch push)
    - if: '$CI_COMMIT_BRANCH =~ /^(feat|fix|chore|refactor|docs|test|style|perf|ci|build)\//'
      when: on_success
    # Run affected tests on MR pipelines (fast validation)
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop")'
      when: on_success
    # Run on protected branches (post-merge)
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "develop"'
      when: on_success
  before_script:
    - echo "Acquire::http::Proxy \"http://${APT_PROXY}\";" > /etc/apt/apt.conf.d/01proxy
    - apt-get -q update && apt-get -qy install git sqlite3
    - python -m pip install --quiet --upgrade pip
    - pip install --quiet uv
    - test -f .venv/bin/activate && echo "✓ Using cached venv" || echo "⚠ Cache miss - creating venv"
    - test -f .venv/bin/activate || uv venv --python python3 .venv
    - source .venv/bin/activate
    - test -f .venv/lib/python*/site-packages/n8n_deploy.egg-info || uv pip install --quiet -e ".[test,dev]"
  script:
    - export N8N_DEPLOY_TESTING=1
    - test -f .testmondata && echo "Using cached testmon database" || echo "No testmon baseline found"
    - ./run_tests.py --affected --no-deps-check
    - echo "Affected tests completed"
  after_script:
    # Flush SQLite WAL to main database before caching
    # This ensures .testmondata contains all data without needing -wal/-shm files
    - sqlite3 .testmondata "PRAGMA wal_checkpoint(TRUNCATE);" 2>/dev/null || true
  artifacts:
    paths:
      - .testmondata
    expire_in: 7 days
    when: always

# ============================================
# BUILD-MATRIX STAGE (runs on merge/push to protected branches)
# ============================================

# Multi-version package builds (Python 3.9-3.13)
build:python-matrix:
  stage: build-matrix
  image: python:${PYTHON_VERSION}-slim
  interruptible: false
  tags:
    - linux
    - python
  cache:
    key:
      files:
        - requirements.txt
      prefix: py${PYTHON_VERSION}-${CI_COMMIT_REF_SLUG}
    paths:
      - .cache/pip/
      - .venv/
    policy: pull-push
  needs:
    - job: test:affected
      optional: true
  rules:
    - if: '$CI_COMMIT_TAG'
      when: on_success
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "develop"'
      when: on_success
  parallel:
    matrix:
      - PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12", "3.13"]
  before_script:
    - echo "Acquire::http::Proxy \"http://${APT_PROXY}\";" > /etc/apt/apt.conf.d/01proxy
    - apt-get -q update && apt-get -qy install git
    - python -m pip install --quiet --upgrade pip
    - pip install --quiet uv
    - test -f .venv/bin/activate && echo "✓ Using cached venv for Python ${PYTHON_VERSION}" || echo "⚠ Cache miss - creating venv for Python ${PYTHON_VERSION}"
    - test -f .venv/bin/activate || uv venv --python python3 .venv
    - source .venv/bin/activate
    - test -f .venv/lib/python*/site-packages/n8n_deploy.egg-info || uv pip install --quiet -e ".[test,dev]"
    - uv pip install --quiet build
  script:
    - echo "Building package for Python ${PYTHON_VERSION}..."
    - python -m build --wheel --sdist
    - pip install dist/*.whl
    - n8n-deploy --version
    - echo "Build successful for Python ${PYTHON_VERSION}"
  artifacts:
    paths:
      - dist/
    expire_in: 1 hour

# ============================================
# RELEASE STAGE (automatic versioning with go-semrel-gitlab)
# ============================================

# Release candidate tags on develop branch
release:rc:
  stage: release
  image: registry.gitlab.com/juhani/go-semrel-gitlab:v0.21.1
  tags:
    - linux
    - python
  cache: {}  # Disable cache - not needed for release
  needs:
    - job: build:python-matrix
      artifacts: false
  rules:
    # Run after MR merge to develop (not on tags or MR events)
    - if: '$CI_COMMIT_TAG'
      when: never
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
    - if: '$CI_COMMIT_BRANCH == "develop"'
      when: on_success
    - when: never
  variables:
    GIT_SSL_NO_VERIFY: "1"
    GITLAB_URL: "https://192.168.76.5:8443/api/v4"
    GSG_TOKEN: $GITLAB_TOKEN
    GSG_RELEASE_BRANCHES: master
    GSG_PRE_TMPL: 'rc,{{ seq }}'
  script:
    - echo "Calculating next RC version..."
    - NEXT_VERSION=$(release --skip-ssl-verify next-version --allow-current)
    - echo "Next version - ${NEXT_VERSION}"
    - |
      if [ -n "$NEXT_VERSION" ]; then
        echo "Creating RC tag v${NEXT_VERSION}..."
        if ! release --skip-ssl-verify tag 2>&1 | tee /tmp/release.log; then
          if grep -q "no changes found" /tmp/release.log; then
            echo "No releasable changes (ci/docs/chore commits only) - skipping tag"
          else
            echo "Release failed with error:"
            cat /tmp/release.log
            exit 1
          fi
        fi
      else
        echo "No version bump needed"
      fi

# Production release tags on master branch
release:production:
  stage: release
  image: registry.gitlab.com/juhani/go-semrel-gitlab:v0.21.1
  tags:
    - linux
    - python
  cache: {}  # Disable cache - not needed for release
  needs:
    - job: build:python-matrix
      artifacts: false
  rules:
    # Run after MR merge to master (not on tags or MR events)
    - if: '$CI_COMMIT_TAG'
      when: never
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    - when: never
  variables:
    GIT_SSL_NO_VERIFY: "1"
    GITLAB_URL: "https://192.168.76.5:8443/api/v4"
    GSG_TOKEN: $GITLAB_TOKEN
  script:
    - echo "Calculating next production version..."
    - NEXT_VERSION=$(release --skip-ssl-verify next-version --allow-current)
    - echo "Next version - ${NEXT_VERSION}"
    - |
      if [ -n "$NEXT_VERSION" ]; then
        echo "Creating release tag v${NEXT_VERSION}..."
        if ! release --skip-ssl-verify tag 2>&1 | tee /tmp/release.log; then
          if grep -q "no changes found" /tmp/release.log; then
            echo "No releasable changes (ci/docs/chore commits only) - skipping tag"
          else
            echo "Release failed with error:"
            cat /tmp/release.log
            exit 1
          fi
        fi
      else
        echo "No version bump needed"
      fi