.gitlab-ci.yml

include:
  # Docker Scout CVE scanning (shared DevOps template, publishes SBOM to the CVE dashboard)
  - project: 'devops-public/pipelines'
    ref: '${X_DEVOPS_SHARED_PIPELINES_STABLE}'
    file: '/applications/common/tests/docker-scout.yaml'
    inputs:
      scout_image: '${REGISTRY}/ingress-controller:${TAG}'
      # bucket sub-path organizes the CVE dashboard tree: <node-type>/<artifact>/<release line>
      scout_sbom_bucket: 'wlrm-devops-cve-dashboard/nginx-node/ingress/6.x'
      # only release pipelines publish SBOMs; they set SCOUT_PUBLISH_SBOM in the
      # workflow rules below, dev/MR/api/web pipelines do not
      scout_publish_sbom: 'false'
      stage: 'scan'

default:
  id_tokens:
    VAULT_ID_TOKEN:
      aud: https://gl.wallarm.com
  image: europe-docker.pkg.dev/wallarm-infra/wallarm-cloud-europe/wallarm-node/node-gitlab-runner/common:v0.5.0
  tags: [node-c05r1-stable-amd]

stages:
  - build
  - scan
  - test
  - sign
  - publish

variables:
  VAULT_SERVER_URL: https://vault-common.i.gcp.wallarm.space
  VAULT_AUTH_ROLE: gitlab-ci_node-team_wallarm-node
  VAULT_AUTH_PATH: jwt-gitlab
  X_CI_BUILD_KIND:
    description: "What we want to run, usually 'develop' or 'smokeonly' flow"
    value: 'develop'
    options:
      - 'develop'
      - 'production-local'
      - 'chart'
      - 'smokeonly'
      - 'cve-scan'
  ARCH:
    description: default architecture (mandatory var in test scripts)
    value: amd64
  WALLARM_API_PRESET:
    description: "Wallarm api preset"
    value: us1
    options:
      - audit
      - eu1
      - us1
  SKIP_TESTS:
    description: "Set to true to skip all test jobs"
    value: "false"

  ### For doc purposes only ###
  ## Variables used implicitly, only in upstream-triggered pipelines (KINDs production|release-candidate)
  # AIO_VERSION:
  #   description: AIO version to extract the nginx module from, and is also used as appVersion in chart; do not confuse with TAG, which in dynamically calculated


workflow:
  rules:
    - if: $AIO_VERSION =~ /^([0-9]+.[0-9]+.[0-9]+-rc).*$/
      variables:
        REGISTRY: $CI_REGISTRY_IMAGE # using private registry to build img and run tests before publishing images
        HELM_ARGS: --set controller.wallarm.helpers.tag=$AIO_VERSION --set controller.wallarm.helpers.image=dkr.wallarm.com/wallarm-node/node-helpers
        CHART_TYPE: "release-candidate"
        SCOUT_PUBLISH_SBOM: "true" # release artifacts must land on the CVE dashboard
    - if: $AIO_VERSION
      variables:
        REGISTRY: $CI_REGISTRY_IMAGE # using private registry to build img and run tests before publishing images
        HELM_ARGS: --set controller.wallarm.helpers.tag=$AIO_VERSION --set controller.wallarm.helpers.image=dkr.wallarm.com/wallarm-node/node-helpers
        CHART_TYPE: "release"
        SCOUT_PUBLISH_SBOM: "true" # release artifacts must land on the CVE dashboard
    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH =~ /^stable\/.*$/) && $CI_PIPELINE_SOURCE == "push"
      changes:
        - TAG
      variables:
        X_CI_BUILD_KIND: "production-local"
        REGISTRY: docker.io/wallarm
    - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /^stable\/.*$/ || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH
      variables:
        X_CI_BUILD_KIND: "develop"
        REGISTRY: $CI_REGISTRY_IMAGE
        TAG: "$CI_COMMIT_SHORT_SHA"
    - if: $CI_COMMIT_TAG =~ /^([0-9]+.[0-9]+.[0-9]+\+upstream).*$/
      variables:
        X_CI_BUILD_KIND: "chart"
        CHART_TYPE: "release"
    - if: $CI_COMMIT_TAG =~ /^([0-9]+.[0-9]+.[0-9]+-rc).*$/
      variables:
        X_CI_BUILD_KIND: "chart"
        CHART_TYPE: "release-candidate"
    - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "smoke"
      variables:
        X_CI_BUILD_KIND: "smokeonly"
        REGISTRY: docker.io/wallarm
    - if: $X_CI_BUILD_KIND == "cve-scan"
      variables:
        REGISTRY: $CI_REGISTRY_IMAGE
        TAG: "$CI_COMMIT_SHORT_SHA"
    - if: $CI_PIPELINE_SOURCE == "api"
      variables:
        REGISTRY: $CI_REGISTRY_IMAGE
        TAG: "$CI_COMMIT_SHORT_SHA"
    - if: $CI_PIPELINE_SOURCE == "web"


.x-docker-login-private: &x_docker_login_private
  - | # GitLab Container Registry
    docker login "${CI_REGISTRY}" \
        -u "${CI_REGISTRY_USER}" \
        -p "${CI_REGISTRY_PASSWORD}"
  - | # GitLab Dependency Proxy
    docker login "${CI_DEPENDENCY_PROXY_SERVER}" \
        -u "${CI_DEPENDENCY_PROXY_USER}" \
        -p "${CI_DEPENDENCY_PROXY_PASSWORD}"

.x-docker-login-public: &x_docker_login_public
  - | # DockerHub
    docker login \
        -u "${X_CREDS_DOCKERHUB_WALLARMCICD_USER}" \
        -p "${X_CREDS_DOCKERHUB_WALLARMCICD_PASSWORD}"

.before-build:
  before_script:
    - *x_docker_login_private
    - *x_docker_login_public

.before-test:
  before_script:
    - *x_docker_login_private
  variables:
    CLIENT_ID: $CLIENT_ID_${WALLARM_API_PRESET} # because fetching as vault secret leads to masking all id-like numbers in logs
  secrets:
    WALLARM_API_HOST: {vault: "pipelines/common/smoke-test/${WALLARM_API_PRESET}/api_host@node-team", file: false}
    WALLARM_API_TOKEN: {vault: "pipelines/common/smoke-test/${WALLARM_API_PRESET}/user_token@node-team", file: false}

    USER_TOKEN: {vault: "pipelines/common/smoke-test/${WALLARM_API_PRESET}/user_token@node-team", file: false}
    USER_UUID: {vault: "pipelines/common/smoke-test/${WALLARM_API_PRESET}/user_uuid@node-team", file: false}

    ALLURE_TOKEN: {vault: "pipelines/common/smoke-test/${WALLARM_API_PRESET}/allure_server_token@node-team", file: false}
    WEBHOOK_UUID: {vault: "pipelines/common/smoke-test/${WALLARM_API_PRESET}/webhook_uuid@node-team", file: false}
    WEBHOOK_API_KEY: {vault: "pipelines/common/smoke-test/${WALLARM_API_PRESET}/webhook_api_key@node-team", file: false}

.before-sign-docker:
  before_script:
    - *x_docker_login_private
    - *x_docker_login_public

.update_kind: &update-kind
  - LATEST_KIND_VERSION=$(curl -s https://api.github.com/repos/kubernetes-sigs/kind/releases/latest | jq -r .tag_name)
  - curl -s -Lo ./kind_latest https://kind.sigs.k8s.io/dl/${LATEST_KIND_VERSION}/kind-linux-${ARCH}
  - chmod +x ./kind_latest
  - sudo mv ./kind_latest /usr/local/bin/kind


### DEV & PROD jobs

IMAGE_BUILD_IC:
  stage: build
  image: europe-docker.pkg.dev/wallarm-infra/wallarm-cloud-europe/wallarm-node/node-gitlab-runner/tests-k8s:v0.5.0
  tags: [node-c2r4-stable-dind-amd]
  rules:
    - if: $X_CI_BUILD_KIND =~ /^(develop|production-local|production|release-candidate|cve-scan)$/
  extends:
    - .before-build
  variables:
    PLATFORMS: amd64 arm64
    BUILDX_PLATFORMS: linux/amd64,linux/arm64
  script: |
    if [[ ! -z $AIO_VERSION ]]; then
      .gitlab/calculate-version.sh
      source version.env && export TAG
    fi
    # make sure TAG always reaches downstream jobs (Docker Scout scan) via dotenv,
    # also for flows where it comes from the TAG file instead of calculate-version.sh
    if [[ -z "${TAG}" ]]; then
      echo "TAG=$(cat TAG)" >> version.env
    fi
    echo "Calculated node version: ${TAG:-from TAG}"
    echo "Building for registry: $REGISTRY"

    echo "Enabling docker buildx..."
    docker buildx create --name multi-arch --platform $BUILDX_PLATFORMS --driver docker-container --use

    echo "Building ingress and ingress-chroot images (multi-platform)..."
    make release
  artifacts:
    reports:
      dotenv: version.env

IMAGE_BUILD_E2E:
  stage: build
  tags: [node-c2r4-stable-dind-amd]
  image: europe-docker.pkg.dev/wallarm-infra/wallarm-cloud-europe/wallarm-node/node-gitlab-runner/tests-k8s:v0.5.0
  rules:
    - if: $X_CI_BUILD_KIND =~ /^(develop|production|release-candidate)$/
  extends:
    - .before-build
  script: |
    if [[ ! -z $AIO_VERSION ]]; then
      .gitlab/calculate-version.sh
      source version.env && export TAG
    fi
    echo "Calculated node version: ${TAG:-from TAG}"
    echo "Building for registry: $REGISTRY"

    echo "Building e2e test image (amd64)..."
    make -C test/e2e-image image
    echo "Pushing built E2E test image to private registry..."
    LOCAL_E2E_IMAGE="${REGISTRY}/nginx-ingress-controller-e2e:${TAG}"
    docker image tag nginx-ingress-controller:e2e $LOCAL_E2E_IMAGE
    docker push $LOCAL_E2E_IMAGE

.SMOKE_BASE:
  image: europe-docker.pkg.dev/wallarm-infra/wallarm-cloud-europe/wallarm-node/node-gitlab-runner/tests-k8s:v0.5.0
  extends:
    - .before-test
  variables:
    SKIP_CLUSTER_CREATION: true
    SKIP_IMAGE_CREATION: true
    DOCKERHUB_USER: $X_CREDS_DOCKERHUB_WALLARMCICD_USER
    DOCKERHUB_PASSWORD: $X_CREDS_DOCKERHUB_WALLARMCICD_PASSWORD
    SMOKE_REGISTRY_TOKEN: $X_CREDS_GITLAB_READONLY_USER
    SMOKE_REGISTRY_SECRET: $X_CREDS_GITLAB_READONLY_PASSWORD
    ALLURE_UPLOAD_REPORT: true
    ALLURE_GENERATE_REPORT: true
    ALLURE_ENVIRONMENT_K8S: $K8S
    ALLURE_ENVIRONMENT_ARCH: $ARCH
  script:
    #- *update-kind
    - |
      export KINDEST_NODE_IMAGE=kindest/node:$(curl -s "https://registry.hub.docker.com/v2/repositories/kindest/node/tags?page_size=1000" \
      | jq -r --arg k "${K8S}." '.results[].name | select(startswith($k))' | sort -V | tail -n 1)
    - kind --version
    - |
      export KIND_CLUSTER_NAME=kind-$K8S
      export KUBECONFIG=$HOME/.kube/kind-config-$K8S
      export ALLURE_ENVIRONMENT_K8S=$K8S
    - |
      if [ -z "$TAG" ]; then
        export TAG="$(cat TAG)"
        export REGISTRY=docker.io/wallarm
      fi
    - docker pull -q ${REGISTRY}/ingress-controller:${TAG}
    - kind create cluster --image=$KINDEST_NODE_IMAGE
    - make kind-smoke-test

SMOKE:
  stage: test
  needs: [IMAGE_BUILD_IC]
  tags: [$RUNNER]
  rules:
    - if: '$SKIP_TESTS == "true"'
      when: never
    - if: $X_CI_BUILD_KIND =~ /^(develop|production|release-candidate)$/
  extends:
    - .SMOKE_BASE
  parallel:
    matrix:
      - ARCH: amd64
        RUNNER: node-c4r8-stable-dind-amd
        K8S: [v1.26, v1.30] # v1.30 is the latest supported version, v1.26 is the oldest supported version, check README.md Supported Versions table
      - ARCH: arm64
        RUNNER: node-c4r8-stable-dind-arm
        K8S: [v1.26, v1.30] # v1.30 is the latest supported version, v1.26 is the oldest supported version, check README.md Supported Versions table

SMOKE_ONLY:
  stage: test
  tags: [$RUNNER]
  rules:
    - if: '$SKIP_TESTS == "true"'
      when: never
    - if: $X_CI_BUILD_KIND == "smokeonly"
  extends:
    - .SMOKE_BASE
  parallel:
    matrix:
      - ARCH: amd64
        RUNNER: node-c4r8-stable-dind-amd
        K8S: [v1.26, v1.30] # v1.30 is the latest supported version, v1.26 is the oldest supported version, check README.md Supported Versions table
      - ARCH: arm64
        RUNNER: node-c4r8-stable-dind-arm
        K8S: [v1.26, v1.30] # v1.30 is the latest supported version, v1.26 is the oldest supported version, check README.md Supported Versions table

E2E_UPSTREAM:
  stage: test
  image: europe-docker.pkg.dev/wallarm-infra/wallarm-cloud-europe/wallarm-node/node-gitlab-runner/tests-k8s:v0.5.0
  tags: [node-c6r12-stable-dind-amd]
  timeout: 3h
  allow_failure: true
  needs: [IMAGE_BUILD_IC, IMAGE_BUILD_E2E]
  rules:
    - if: '$SKIP_TESTS == "true"'
      when: never
    - if: $X_CI_BUILD_KIND =~ /^(develop|production|release-candidate)$/
  extends:
    - .before-test
  variables:
    K8S: 1.30.0
    KIND_CLUSTER_NAME: kind
    SKIP_CLUSTER_CREATION: "true"
    SKIP_INGRESS_IMAGE_CREATION: "true"
    SKIP_E2E_IMAGE_CREATION: "true"
    DOCKERHUB_USER: $X_CREDS_DOCKERHUB_WALLARMCICD_USER
    DOCKERHUB_PASSWORD: $X_CREDS_DOCKERHUB_WALLARMCICD_PASSWORD
  script: |
    docker pull -q ${REGISTRY}/ingress-controller:${TAG}
    docker pull -q ${REGISTRY}/nginx-ingress-controller-e2e:${TAG}

    kind create cluster --image=kindest/node:v$K8S --config test/e2e/kind.yaml

    kind get kubeconfig > $HOME/.kube/kind-config-kind
    make E2E_NODES=15 kind-e2e-test

E2E_WALLARM:
  stage: test
  image: europe-docker.pkg.dev/wallarm-infra/wallarm-cloud-europe/wallarm-node/node-gitlab-runner/tests-k8s:v0.5.0
  tags: [$RUNNER]
  timeout: 3h
  allow_failure: true
  needs: [IMAGE_BUILD_IC, IMAGE_BUILD_E2E]
  rules:
    - if: '$SKIP_TESTS == "true"'
      when: never
    - if: $X_CI_BUILD_KIND =~ /^(develop|production|release-candidate)$/
  parallel:
    matrix:
      - ARCH: amd64
        RUNNER: node-c6r12-stable-dind-amd
        K8S: [1.26.15, 1.30.0]
  extends:
    - .before-test
  variables:
    KIND_CLUSTER_NAME: kind
    SKIP_CLUSTER_CREATION: "true"
    SKIP_INGRESS_IMAGE_CREATION: "true"
    SKIP_E2E_IMAGE_CREATION: "true"
    WALLARM_ENABLED: "true"
    DOCKERHUB_USER: $X_CREDS_DOCKERHUB_WALLARMCICD_USER
    DOCKERHUB_PASSWORD: $X_CREDS_DOCKERHUB_WALLARMCICD_PASSWORD
  script: |
    docker pull -q ${REGISTRY}/ingress-controller:${TAG}
    docker pull -q ${REGISTRY}/nginx-ingress-controller-e2e:${TAG}

    kind create cluster --image=kindest/node:v$K8S --config test/e2e/kind.yaml

    kind get kubeconfig > $HOME/.kube/kind-config-kind
    make E2E_NODES=7 kind-e2e-test

CHART_TESTING:
  stage: test
  image: europe-docker.pkg.dev/wallarm-infra/wallarm-cloud-europe/wallarm-node/node-gitlab-runner/tests-k8s:v0.5.0
  tags: [node-c4r8-stable-dind-amd]
  needs: [IMAGE_BUILD_IC]
  rules:
    - if: '$SKIP_TESTS == "true"'
      when: never
    - if: $X_CI_BUILD_KIND =~ /^(develop|production|release-candidate)$/
  extends:
    - .before-test
  variables:
    K8S: 1.26.15
    KIND_CLUSTER_NAME: kind
    SKIP_CLUSTER_CREATION: "true"
    SKIP_IMAGE_CREATION: "true"
    CT_MODE: install
    DOCKERHUB_USER: $X_CREDS_DOCKERHUB_WALLARMCICD_USER
    DOCKERHUB_PASSWORD: $X_CREDS_DOCKERHUB_WALLARMCICD_PASSWORD
  script: |
    docker pull -q ${REGISTRY}/ingress-controller:${TAG}

    kind create cluster --image=kindest/node:v$K8S --config test/e2e/kind.yaml
    kind get kubeconfig > $HOME/.kube/kind-config-$KIND_CLUSTER_NAME
    export CT_CONFIG="${CI_PROJECT_DIR}/kind-config-ct-${KIND_CLUSTER_NAME}"

    make kind-e2e-chart-tests

# Vulnerability scan (docker_scout_scan comes from the shared docker-scout
# template, see include; only needs/rules are customized here)
docker_scout_scan:
  needs:
    - job: IMAGE_BUILD_IC
      optional: true
  rules:
    - if: '$SKIP_TESTS == "true"'
      when: never
    - if: $X_CI_BUILD_KIND =~ /^(develop|cve-scan|production|release-candidate)$/
    - when: never



### PROD related jobs

PUBLISH_GUARD:
  stage: sign
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline" && $X_CI_BUILD_KIND == "production"
      when: manual
      allow_failure: true
    - if: $X_CI_BUILD_KIND =~ /^(production-local|production|release-candidate)$/
      when: on_success
  script:
    - echo "Publish gate opened"

PUBL_SIGN_DOCKER:
  stage: sign
  needs:
    - PUBLISH_GUARD
    - IMAGE_BUILD_IC
  tags: [node-c4r8-stable-dind-amd]
  image: europe-docker.pkg.dev/wallarm-infra/wallarm-cloud-europe/wallarm-node/node-gitlab-runner/docker:v0.5.0
  rules:
    - if: $X_CI_BUILD_KIND =~ /^(production-local|production|release-candidate)$/
  parallel:
    matrix:
      - IMAGE: [ingress-controller, ingress-controller-chroot]
  extends:
    - .before-sign-docker
  secrets:
    COSIGN_PASSWORD: {vault: "pipelines/cosign/password@node-team", file: false}
    COSIGN_PRIVATE: {vault: "pipelines/cosign/private_key@node-team", file: false}
  script:
    - |
      set -euo pipefail
      if [[ ! -z $AIO_VERSION ]]; then
        echo "Will push ${CI_REGISTRY_IMAGE}/${IMAGE}:${TAG} (private) to wallarm/${IMAGE}:${TAG} (public)..."
        docker buildx imagetools create -t wallarm/${IMAGE}:${TAG} ${CI_REGISTRY_IMAGE}/${IMAGE}:${TAG}
      fi

      if [[ "$CHART_TYPE" == "release" ]]; then
        make sign
      else
        echo "Skipping signing step (CHART_TYPE=$CHART_TYPE)"
      fi
  artifacts:
    expire_in: 30 days
    paths:
      - $CI_PROJECT_DIR/*.json

HELM_PUBLISH:
  stage: publish
  needs:
    - job: PUBLISH_GUARD
      optional: true
    - job: IMAGE_BUILD_IC  # optional since it doesn't run for X_CI_BUILD_KIND=chart
      optional: true
    - job: "PUBL_SIGN_DOCKER: [ingress-controller]"
      optional: true
    - job: "PUBL_SIGN_DOCKER: [ingress-controller-chroot]"
      optional: true
  rules:
   - if: $X_CI_BUILD_KIND =~ /^(chart|production|release-candidate)$/
  secrets:
    GITHUB_TOKEN: {vault: "pipelines/common/github_token/token@node-team", file: false}
  variables:
    GITHUB_REPOSITORY: wallarm/helm-charts
    CHARTS_DIR: charts
    CHARTS_URL: https://charts.wallarm.com
    LINTING: off
    BRANCH: main
    TARGET_DIR: wallarm-ingress
    INDEX_DIR: .
  ## Original flow (with CHART_TYPE pre-set by CI based on git tag)
  # Git tags used for (final) releases are long (include two versions) and we only need first part for helm chart version, so we crop it
  # Git tags used for RC contain only one version and there is no need to change them, so we pass them as is
  ## New flow (upstream triggered, CHART_TYPE empty in CI)
  # Use upstream-set vars (TAG and AIO_VERSION) to set CHART_TYPE and set docker image versions in the chart (using TAG "as is")
  script: |
    echo "Testing IC image version calc: $IMAGE_VERSION"

    if [[ $X_CI_BUILD_KIND == "chart" ]]; then
      echo "Classic(tag) chart publishing flow..."
      if [[ $CHART_TYPE =~ "release-candidate" ]]; then
        CHART_VERSION=$CI_COMMIT_TAG
      else
        CHART_VERSION=$(echo $CI_COMMIT_TAG | sed 's/[+-].*$//g')
      fi
    else
      echo "Upstream-based chart publishing flow..."
      CHART_VERSION=$TAG
      curl -Lo /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.2.0/yq_linux_amd64 && chmod +x /usr/bin/yq
      yq e -i '.appVersion = "'$AIO_VERSION'"' charts/ingress-nginx/Chart.yaml
      yq e -i '.controller.wallarm.helpers.tag = "'$AIO_VERSION'"' charts/ingress-nginx/values.yaml
      yq e -i '.controller.image.tag = "'$TAG'"' charts/ingress-nginx/values.yaml
    fi
    export CHART_VERSION
    echo "CHART_VERSION=$CHART_VERSION" >> chart.env

    echo "Chart type: $CHART_TYPE"
    echo "Publishing Helm chart version: $CHART_VERSION"

    .gitlab/chart-publish.sh
  artifacts:
    reports:
      dotenv: chart.env

UPDATE_VERSION:
  stage: publish
  image: europe-docker.pkg.dev/wallarm-infra/wallarm-cloud-europe/wallarm-node/node-gitlab-runner/gitlab:v0.5.0
  needs: [HELM_PUBLISH]
  rules:
    - if: $CHART_TYPE == "release"
  secrets:
    GITLAB_TOKEN: {vault: "pipelines/gl_version_repo_creds/token_secret@node-team", file: false}
    GITLAB_TOKEN_NAME: {vault: "pipelines/gl_version_repo_creds/token_name@node-team", file: false}
    GITLAB_HOST: {vault: "pipelines/gl_version_repo_creds/host@node-team", file: false}
    GITLAB_REPO: {vault: "pipelines/gl_version_repo_creds/repo@node-team", file: false}
  variables:
    COMPONENT_NAME: wallarm-ingress-controller
    COMPONENT_VERSION: $CHART_VERSION
  script: |
    .gitlab/update-version.sh