test-integration.yaml

name: integration
# To debug locally on a branch, and when needing secrets
# change this to include `push` so the build is ran on
# the main repository.
on: [pull_request]
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true
defaults:
  run:
    shell: nix develop --fallback --command bash -e {0}
jobs:
  # build: Builds binaries and Docker images once, uploads as artifacts for reuse.
  # build-postgres: Pulls postgres image separately to avoid Docker Hub rate limits.
  # build-tailscale-released: Pre-pulls released Tailscale images from ghcr.io
  # so fork PRs (no DOCKERHUB_USERNAME secret) don't hit Docker Hub rate
  # limits at test time.
  # sqlite: Runs all integration tests with SQLite backend.
  # postgres: Runs a subset of tests with PostgreSQL to verify database compatibility.
  build:
    runs-on: ubuntu-24.04-arm
    outputs:
      files-changed: ${{ steps.changed-files.outputs.files }}
    steps:
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          fetch-depth: 2
      - name: Get changed files
        id: changed-files
        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        with:
          filters: |
            files:
              - '*.nix'
              - 'go.*'
              - '**/*.go'
              - 'integration/**'
              - 'config-example.yaml'
              - '.github/workflows/test-integration.yaml'
              - '.github/workflows/integration-test-template.yml'
              - 'Dockerfile.*'
      - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main
        if: steps.changed-files.outputs.files == 'true'
      - uses: Mic92/hestia@fb239a2f72d4b6e26eec5425f289dea23b27a527 # v2.0.0
        if: steps.changed-files.outputs.files == 'true'
      - name: Build binaries and warm Go cache
        if: steps.changed-files.outputs.files == 'true'
        run: |
          # Build all Go binaries in one nix shell to maximize cache reuse
          go build -o hi ./cmd/hi
          CGO_ENABLED=0 GOOS=linux go build -o headscale ./cmd/headscale
          # Build integration test binary to warm the cache with all dependencies
          go test -c ./integration -o /dev/null 2>/dev/null || true
      - name: Upload hi binary
        if: steps.changed-files.outputs.files == 'true'
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: hi-binary
          path: hi
          retention-days: 10
      - name: Package Go cache
        if: steps.changed-files.outputs.files == 'true'
        run: |
          # Package Go module cache and build cache
          tar -czf go-cache.tar.gz -C ~ go .cache/go-build
      - name: Upload Go cache
        if: steps.changed-files.outputs.files == 'true'
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: go-cache
          path: go-cache.tar.gz
          retention-days: 10
      - name: Force overlay2 storage driver
        if: steps.changed-files.outputs.files == 'true'
        run: |
          # Docker 29 runner images default to overlayfs, which breaks
          # docker build via Go SDK libraries and docker save/load
          # tarball formats. overlay2 is the long-standing default.
          # https://github.com/actions/runner-images/issues/13474
          sudo mkdir -p /etc/docker
          echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json
          sudo systemctl restart docker
          docker version
      - name: Login to Docker Hub
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
        if: env.DOCKERHUB_USERNAME != ''
        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
        with:
          username: ${{ env.DOCKERHUB_USERNAME }}
          password: ${{ env.DOCKERHUB_TOKEN }}
      - name: Build headscale image
        if: steps.changed-files.outputs.files == 'true'
        run: |
          docker build \
            --file Dockerfile.integration-ci \
            --tag headscale:${{ github.sha }} \
            .
          docker save headscale:${{ github.sha }} | gzip > headscale-image.tar.gz
      - name: Build tailscale HEAD image
        if: steps.changed-files.outputs.files == 'true'
        run: |
          docker build \
            --file Dockerfile.tailscale-HEAD \
            --tag tailscale-head:${{ github.sha }} \
            .
          docker save tailscale-head:${{ github.sha }} | gzip > tailscale-head-image.tar.gz
      - name: Upload headscale image
        if: steps.changed-files.outputs.files == 'true'
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: headscale-image
          path: headscale-image.tar.gz
          retention-days: 10
      - name: Upload tailscale HEAD image
        if: steps.changed-files.outputs.files == 'true'
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: tailscale-head-image
          path: tailscale-head-image.tar.gz
          retention-days: 10
  build-postgres:
    runs-on: ubuntu-24.04-arm
    needs: build
    if: needs.build.outputs.files-changed == 'true'
    steps:
      - name: Force overlay2 storage driver
        shell: bash
        run: |
          sudo mkdir -p /etc/docker
          echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json
          sudo systemctl restart docker
          docker version
      - name: Login to Docker Hub
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
        if: env.DOCKERHUB_USERNAME != ''
        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
        with:
          username: ${{ env.DOCKERHUB_USERNAME }}
          password: ${{ env.DOCKERHUB_TOKEN }}
      - name: Pull and save postgres image
        shell: bash
        run: |
          docker pull postgres:latest
          docker tag postgres:latest postgres:${{ github.sha }}
          docker save postgres:${{ github.sha }} | gzip > postgres-image.tar.gz
      - name: Upload postgres image
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: postgres-image
          path: postgres-image.tar.gz
          retention-days: 10
  build-tailscale-released:
    runs-on: ubuntu-24.04-arm
    needs: build
    if: needs.build.outputs.files-changed == 'true'
    steps:
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
      - uses: NixOS/nix-installer-action@6b8548fe06acfb0155a50ab5d561accb215764cc # main
      - uses: Mic92/hestia@fb239a2f72d4b6e26eec5425f289dea23b27a527 # v2.0.0
      - name: Force overlay2 storage driver
        run: |
          sudo mkdir -p /etc/docker
          echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json
          sudo systemctl restart docker
          docker version
      - name: Login to Docker Hub
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
        if: env.DOCKERHUB_USERNAME != ''
        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
        with:
          username: ${{ env.DOCKERHUB_USERNAME }}
          password: ${{ env.DOCKERHUB_TOKEN }}
      - name: List Tailscale versions to pre-pull
        id: versions
        run: |
          versions=$(go run ./cmd/hi list-versions --set=must --exclude=head)
          echo "versions=${versions}" >> "$GITHUB_OUTPUT"
          echo "Pre-pulling: ${versions}"
      - name: Pull Tailscale images
        run: |
          # Releases come from ghcr.io (anonymous, unmetered). The
          # "unstable" floating tag on ghcr.io has been stale since 2022,
          # so it still needs to come from Docker Hub. xargs -P 0 fans
          # out one process per tag and returns non-zero if any pull
          # fails.
          refs=""
          for v in ${{ steps.versions.outputs.versions }}; do
            if [ "${v}" = "unstable" ]; then
              refs="${refs} tailscale/tailscale:${v}"
            else
              refs="${refs} ghcr.io/tailscale/tailscale:${v}"
            fi
          done
          echo "${refs}" | tr ' ' '\n' | grep -v '^$' \
            | xargs -P 0 -I{} docker pull "{}"
          echo "REFS=${refs}" >> "$GITHUB_ENV"
      - name: Save Tailscale images to tarball
        run: |
          # Single docker save with all refs: one consistent snapshot, no
          # parallel-daemon race.
          docker save ${REFS} | gzip > tailscale-released-images.tar.gz
          ls -lh tailscale-released-images.tar.gz
      - name: Upload Tailscale released images
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: tailscale-released-images
          path: tailscale-released-images.tar.gz
          retention-days: 10
  sqlite:
    needs: [build, build-tailscale-released]
    if: needs.build.outputs.files-changed == 'true'
    strategy:
      fail-fast: false
      matrix:
        test:
          - TestACLHostsInNetMapTable
          - TestACLAllowUser80Dst
          - TestACLDenyAllPort80
          - TestACLAllowUserDst
          - TestACLAllowStarDst
          - TestACLNamedHostsCanReachBySubnet
          - TestACLNamedHostsCanReach
          - TestACLDevice1CanAccessDevice2
          - TestPolicyUpdateWhileRunningWithCLIInDatabase
          - TestACLAutogroupMember
          - TestACLAutogroupTagged
          - TestACLAutogroupSelf
          - TestACLPolicyPropagationOverTime
          - TestACLTagPropagation
          - TestACLTagPropagationPortSpecific
          - TestACLGroupWithUnknownUser
          - TestACLGroupAfterUserDeletion
          - TestACLGroupDeletionExactReproduction
          - TestACLDynamicUnknownUserAddition
          - TestACLDynamicUnknownUserRemoval
          - TestAPIAuthenticationBypass
          - TestAPIAuthenticationBypassCurl
          - TestRemoteCLIAuthenticationBypass
          - TestCLIWithConfigAuthenticationBypass
          - TestAuthKeyLogoutAndReloginSameUser
          - TestAuthKeyLogoutAndReloginNewUser
          - TestAuthKeyLogoutAndReloginSameUserExpiredKey
          - TestAuthKeyDeleteKey
          - TestAuthKeyLogoutAndReloginRoutesPreserved
          - TestOIDCAuthenticationPingAll
          - TestOIDCExpireNodesBasedOnTokenExpiry
          - TestOIDC024UserCreation
          - TestOIDCAuthenticationWithPKCE
          - TestOIDCReloginSameNodeNewUser
          - TestOIDCFollowUpUrl
          - TestOIDCMultipleOpenedLoginUrls
          - TestOIDCReloginSameNodeSameUser
          - TestOIDCExpiryAfterRestart
          - TestOIDCACLPolicyOnJoin
          - TestOIDCReloginSameUserRoutesPreserved
          - TestAuthWebFlowAuthenticationPingAll
          - TestAuthWebFlowLogoutAndReloginSameUser
          - TestAuthWebFlowLogoutAndReloginNewUser
          - TestApiKeyCommand
          - TestApiKeyCommandValidation
          - TestAuthCommandValidation
          - TestNodeCommand
          - TestNodeExpireCommand
          - TestNodeRenameCommand
          - TestPreAuthKeyCorrectUserLoggedInCommand
          - TestTaggedNodesCLIOutput
          - TestNodeExpireFlagsCommand
          - TestNodeCommandValidation
          - TestNodeTagCommand
          - TestNodeRouteCommands
          - TestNodeBackfillIPsCommand
          - TestOAuthClientCommand
          - TestOAuthClientCommandValidation
          - TestPolicyCheckCommand
          - TestSSHTestsRejectFailingPolicy
          - TestPolicyCommand
          - TestPolicyBrokenConfigCommand
          - TestPreAuthKeyCommand
          - TestPreAuthKeyCommandWithoutExpiry
          - TestPreAuthKeyCommandReusableEphemeral
          - TestPreAuthKeyDeleteCommand
          - TestPreAuthKeyCommandValidation
          - TestServerInfoCommands
          - TestUserCommand
          - TestUserCreateCommand
          - TestUserCommandValidation
          - TestDERPVerifyEndpoint
          - TestResolveMagicDNS
          - TestResolveMagicDNSExtraRecordsPath
          - TestDERPServerScenario
          - TestDERPServerWebsocketScenario
          - TestPingAllByIP
          - TestPingAllByIPPublicDERP
          - TestEphemeral
          - TestEphemeralInAlternateTimezone
          - TestEphemeral2006DeletedTooQuickly
          - TestPingAllByHostname
          - TestTaildrop
          - TestUpdateHostnameFromClient
          - TestExpireNode
          - TestSetNodeExpiryInFuture
          - TestDisableNodeExpiry
          - TestNodeOnlineStatus
          - TestPingAllByIPManyUpDown
          - Test2118DeletingOnlineNodePanics
          - TestGrantCapRelay
          - TestGrantCapDrive
          - TestK8sOperator
          - TestEnablingRoutes
          - TestHASubnetRouterFailover
          - TestSubnetRouteACL
          - TestEnablingExitRoutes
          - TestExitRoutesWithAutogroupInternetACL
          - TestSubnetRouterMultiNetwork
          - TestSubnetRouterMultiNetworkExitNode
          - TestAutoApproveMultiNetwork/authkey-tag.*
          - TestAutoApproveMultiNetwork/authkey-user.*
          - TestAutoApproveMultiNetwork/authkey-group.*
          - TestAutoApproveMultiNetwork/webauth-tag.*
          - TestAutoApproveMultiNetwork/webauth-user.*
          - TestAutoApproveMultiNetwork/webauth-group.*
          - TestSubnetRouteACLFiltering
          - TestGrantViaSubnetSteering
          - TestHASubnetRouterPingFailover
          - TestHASubnetRouterFailoverBothOffline
          - TestHASubnetRouterFailoverBothOfflineCablePull
          - TestHASubnetRouterFailoverDockerDisconnect
          - TestHeadscale
          - TestTailscaleNodesJoiningHeadcale
          - TestSSHOneUserToAll
          - TestSSHMultipleUsersAllToAll
          - TestSSHNoSSHConfigured
          - TestSSHIsBlockedInACL
          - TestSSHUserOnlyIsolation
          - TestSSHAutogroupSelf
          - TestSSHOneUserToOneCheckModeCLI
          - TestSSHOneUserToOneCheckModeOIDC
          - TestSSHCheckModeUnapprovedTimeout
          - TestSSHCheckModeCheckPeriodCLI
          - TestSSHCheckModeAutoApprove
          - TestSSHCheckModeSessionLossReDelegates
          - TestSSHCheckModeNegativeCLI
          - TestSSHLocalpart
          - TestTagsAuthKeyWithTagRequestDifferentTag
          - TestTagsAuthKeyWithTagNoAdvertiseFlag
          - TestTagsAuthKeyWithTagCannotAddViaCLI
          - TestTagsAuthKeyWithTagCannotChangeViaCLI
          - TestTagsAuthKeyWithTagAdminOverrideReauthPreserves
          - TestTagsReauthDifferentKeyRetagsNode
          - TestTagsReauthDifferentKeyRemovesTag
          - TestTagsAuthKeyWithTagCLICannotModifyAdminTags
          - TestTagsAuthKeyWithoutTagCannotRequestTags
          - TestTagsAuthKeyWithoutTagRegisterNoTags
          - TestTagsAuthKeyWithoutTagCannotAddViaCLI
          - TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithReset
          - TestTagsAuthKeyWithoutTagCLINoOpAfterAdminWithEmptyAdvertise
          - TestTagsAuthKeyWithoutTagCLICannotReduceAdminMultiTag
          - TestTagsUserLoginOwnedTagAtRegistration
          - TestTagsUserLoginNonExistentTagAtRegistration
          - TestTagsUserLoginUnownedTagAtRegistration
          - TestTagsUserLoginAddTagViaCLIReauth
          - TestTagsUserLoginRemoveTagViaCLIReauth
          - TestTagsUserLoginCLINoOpAfterAdminAssignment
          - TestTagsUserLoginCLICannotRemoveAdminTags
          - TestTagsAuthKeyWithTagRequestNonExistentTag
          - TestTagsAuthKeyWithTagRequestUnownedTag
          - TestTagsAuthKeyWithoutTagRequestNonExistentTag
          - TestTagsAuthKeyWithoutTagRequestUnownedTag
          - TestTagsAdminAPICannotSetNonExistentTag
          - TestTagsAdminAPICanSetUnownedTag
          - TestTagsAdminAPICannotRemoveAllTags
          - TestTagsIssue2978ReproTagReplacement
          - TestTagsAdminAPICannotSetInvalidFormat
          - TestTagsUserLoginReauthWithEmptyTagsRemovesAllTags
          - TestTagsAuthKeyWithoutUserInheritsTags
          - TestTagsAuthKeyWithoutUserRejectsAdvertisedTags
          - TestTagsAuthKeyConvertToUserViaCLIRegister
          - TestTaggedNodeLogoutReloginSingleUseKeyOnline
          - TestTaggedNodeLogoutReloginReusableKeyOnline
          - TestTagsOIDCReauthAddOwnedTag
          - TestTagsReauthEmptyTagsReturnsToUserSurvives
          - TestTS2021WebSocketGET
          - TestTS2021WASMClientUnderNode
          - TestTailscaleRustAxum
    uses: ./.github/workflows/integration-test-template.yml
    secrets: inherit
    with:
      test: ${{ matrix.test }}
      postgres_flag: "--postgres=0"
      database_name: "sqlite"
  postgres:
    needs: [build, build-postgres, build-tailscale-released]
    if: needs.build.outputs.files-changed == 'true'
    strategy:
      fail-fast: false
      matrix:
        test:
          - TestACLAllowUserDst
          - TestPingAllByIP
          - TestEphemeral2006DeletedTooQuickly
          - TestPingAllByIPManyUpDown
          - TestSubnetRouterMultiNetwork
    uses: ./.github/workflows/integration-test-template.yml
    secrets: inherit
    with:
      test: ${{ matrix.test }}
      postgres_flag: "--postgres=1"
      database_name: "postgres"