Setup CI for adapter with service dependencies

Run the Kafka adapter integration tests in CI against a real broker. Adds a test_adapters job
that stands up ci/kafka/docker-compose.yml, sets CSP_TEST_<ADAPTER>, and runs the matching
tests, plus dockerup/dockerps/dockerdown targets for doing the same locally.

The compose stack is trimmed to zookeeper and a single broker, with healthchecks so
`docker compose up --wait` blocks until the broker accepts connections rather than relying on a
fixed sleep. Only 9092 is published, bound to loopback. Broker-side topic auto-creation is
disabled so test_invalid_topic can exercise the failure path, and tests create their topics
explicitly through the Kafka AdminClient.

Test changes target the startup race where a subscriber misses the first few records while its
consumer group is being assigned. Rather than loosening the assertions, the affected tests align
on the first record the subscriber saw and then require an exact contiguous run, so loss,
duplication and reordering are still caught.

Also drops curl from the Windows chocolatey install: the package fails whenever a new version is
approved on the community feed before it is downloadable, and curl.exe ships with Windows.

Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com>

Author: timkpaine

.github/workflows/build.yml

@@ -601,7 +601,6 @@ jobs:
           env:
             CSP_TEST_SKIP_EXAMPLES: "1"
 
-
     ####################################################
     #..................................................#
     #..|########|..|########|..../####\....|########|..#
@@ -713,7 +712,75 @@ jobs:
     ###########################################################################################################
     #  Test Service Adapters                                                                                  #
     ###########################################################################################################
-    # Coming soon!
+    test_adapters:
+      needs:
+        - initialize
+        - build
+
+      permissions:
+        contents: read
+
+      timeout-minutes: 30
+
+      strategy:
+        matrix:
+          os:
+            - ubuntu-24.04
+          python-version:
+            - 3.11
+          adapter:
+            - kafka
+
+      runs-on: ${{ matrix.os }}
+
+      steps:
+        - name: Checkout
+          uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+          with:
+              submodules: recursive
+              persist-credentials: false
+
+        - name: Set up Python ${{ matrix.python-version }}
+          uses: ./.github/actions/setup-python
+          with:
+            version: '${{ matrix.python-version }}'
+            cibuildwheel: false
+
+        - name: Install python dependencies
+          run: make requirements
+
+        - name: Install test dependencies
+          shell: bash
+          run: sudo apt-get update && sudo apt-get install -y graphviz
+
+        # Download artifact
+        - name: Download wheel
+          uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+          with:
+            name: csp-dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}
+
+        - name: Install wheel
+          run: |
+            python -m pip install -U *manylinux*.whl
+            python -m pip install -U --no-deps *manylinux*.whl --target .
+
+        # Services declare healthchecks, so --wait blocks until the broker actually accepts connections
+        - name: Spin up adapter service
+          run: make dockerup ADAPTER=${{ matrix.adapter }} DOCKERARGS="--wait --wait-timeout 180"
+
+        # Run tests
+        - name: Setup test flags
+          shell: bash
+          env:
+            ADAPTER: ${{ matrix.adapter }}
+          run: echo "CSP_TEST_${ADAPTER^^}=1" >> "$GITHUB_ENV"
+
+        - name: Python Test Steps
+          run: make test-py TEST_ARGS="-k ${{ matrix.adapter }}"
+
+        - name: Spin down adapter service
+          run: make dockerdown ADAPTER=${{ matrix.adapter }}
+          if: ${{ always() }}
 
     ############################################################################################
     #..........................................................................................#
@@ -728,7 +795,6 @@ jobs:
     ############################################################################################
     # Upload Release Artifacts                                                                 #
     ############################################################################################
-
     # only publish artifacts on tags, but otherwise this always runs
     # Note this whole workflow only triggers on release tags (e.g. "v0.1.0")
     publish_release_artifacts:
@@ -740,6 +806,7 @@ jobs:
         - test
         - test_sdist
         - test_dependencies
+        - test_adapters
 
       if: startsWith(github.ref, 'refs/tags/v')
       runs-on: ubuntu-24.04

Makefile

@@ -117,21 +117,23 @@ tests: test
 
 .PHONY: dockerup dockerps dockerdown initpodmanmac
 ADAPTER := kafka
-DOCKER := podman
+# Prefer docker, fall back to podman-compose; override with DOCKER_COMPOSE=...
+DOCKER_COMPOSE := $(shell command -v docker >/dev/null 2>&1 && echo "docker compose" || echo "podman-compose")
+DOCKERARGS :=
 
 initpodmanmac:
 	podman machine stop
 	podman machine set --cpus 4 --memory 8096
 	podman machine start
 
 dockerup:  ## spin up docker compose services for adapter testing
-	$(DOCKER) compose -f ci/$(ADAPTER)/docker-compose.yml up -d
+	$(DOCKER_COMPOSE) -f ci/$(ADAPTER)/docker-compose.yml up -d $(DOCKERARGS)
 
-dockerps:  ## spin up docker compose services for adapter testing
-	$(DOCKER) compose -f ci/$(ADAPTER)/docker-compose.yml ps
+dockerps:  ## get status of current docker compose services
+	$(DOCKER_COMPOSE) -f ci/$(ADAPTER)/docker-compose.yml ps
 
-dockerdown:  ## spin up docker compose services for adapter testing
-	$(DOCKER) compose -f ci/$(ADAPTER)/docker-compose.yml down
+dockerdown:  ## spin down docker compose services for adapter testing
+	$(DOCKER_COMPOSE) -f ci/$(ADAPTER)/docker-compose.yml down
 
 ###########
 # VERSION #
@@ -222,9 +224,11 @@ dependencies-fedora:  ## install dependencies for linux - note that zip is neede
 dependencies-vcpkg:  ## install dependencies via vcpkg
 	cd vcpkg && ./bootstrap-vcpkg.sh && ./vcpkg install
 
+# curl.exe ships with Windows; installing it from choco breaks whenever a new version is
+# approved on the feed before the package is actually downloadable
 dependencies-win:  ## install dependencies via windows
 	choco install cmake --version=3.31.6 --allow-downgrade
-	choco install curl winflexbison ninja unzip --no-progress -y
+	choco install winflexbison ninja unzip --no-progress -y
 
 ############################################################################################
 # Thanks to Francoise at marmelab.com for this

ci/kafka/docker-compose.yml

@@ -1,181 +1,47 @@
-# https://docs.confluent.io/platform/current/platform-quickstart.html
-# https://raw.githubusercontent.com/confluentinc/cp-all-in-one/7.5.3-post/cp-all-in-one-kraft/docker-compose.yml
+# https://github.com/conduktor/kafka-stack-docker-compose
 ---
-version: '2'
 services:
-  zookeeper:
+  zoo1:
     image: confluentinc/cp-zookeeper:7.5.3
-    hostname: zookeeper
-    container_name: zookeeper
+    hostname: zoo1
+    container_name: zoo1
     ports:
-      - "2181:2181"
+      - "127.0.0.1:2181:2181"
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
-
-  broker:
-    image: confluentinc/cp-server:7.5.3
-    hostname: broker
-    container_name: broker
-    depends_on:
-      - zookeeper
+      ZOOKEEPER_SERVER_ID: 1
+      ZOOKEEPER_SERVERS: zoo1:2888:3888
+    healthcheck:
+      test: ["CMD-SHELL", "echo ruok | nc -w 2 localhost 2181 | grep imok"]
+      interval: 5s
+      timeout: 5s
+      retries: 24
+
+  kafka1:
+    image: confluentinc/cp-kafka:7.5.3
+    hostname: kafka1
+    container_name: kafka1
     ports:
-      - "9092:9092"
-      - "9101:9101"
+      - "127.0.0.1:9092:9092"
     environment:
+      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
-      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
+      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
-      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
-      KAFKA_JMX_PORT: 9101
-      KAFKA_JMX_HOSTNAME: localhost
-      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081
-      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092
-      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
-      CONFLUENT_METRICS_ENABLE: 'true'
-      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
-      TOPIC_AUTO_CREATE: 'true'
-
-  schema-registry:
-    image: confluentinc/cp-schema-registry:7.5.3
-    hostname: schema-registry
-    container_name: schema-registry
-    depends_on:
-      - broker
-    ports:
-      - "8081:8081"
-    environment:
-      SCHEMA_REGISTRY_HOST_NAME: schema-registry
-      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092'
-      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
-
-  connect:
-    image: cnfldemos/cp-server-connect-datagen:0.6.2-7.5.0
-    hostname: connect
-    container_name: connect
-    depends_on:
-      - broker
-      - schema-registry
-    ports:
-      - "8083:8083"
-    environment:
-      CONNECT_BOOTSTRAP_SERVERS: 'broker:29092'
-      CONNECT_REST_ADVERTISED_HOST_NAME: connect
-      CONNECT_GROUP_ID: compose-connect-group
-      CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
-      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
-      CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
-      CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
-      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
-      CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
-      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
-      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
-      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
-      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
-      # CLASSPATH required due to CC-2422
-      CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.5.3.jar
-      CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
-      CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
-      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
-      CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
-
-  control-center:
-    image: confluentinc/cp-enterprise-control-center:7.5.3
-    hostname: control-center
-    container_name: control-center
-    depends_on:
-      - broker
-      - schema-registry
-      - connect
-      - ksqldb-server
-    ports:
-      - "9021:9021"
-    environment:
-      CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092'
-      CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'connect:8083'
-      CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksqldb-server:8088"
-      CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://localhost:8088"
-      CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
-      CONTROL_CENTER_REPLICATION_FACTOR: 1
-      CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
-      CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
-      CONFLUENT_METRICS_TOPIC_REPLICATION: 1
-      PORT: 9021
-
-  ksqldb-server:
-    image: confluentinc/cp-ksqldb-server:7.5.3
-    hostname: ksqldb-server
-    container_name: ksqldb-server
-    depends_on:
-      - broker
-      - connect
-    ports:
-      - "8088:8088"
-    environment:
-      KSQL_CONFIG_DIR: "/etc/ksql"
-      KSQL_BOOTSTRAP_SERVERS: "broker:29092"
-      KSQL_HOST_NAME: ksqldb-server
-      KSQL_LISTENERS: "http://0.0.0.0:8088"
-      KSQL_CACHE_MAX_BYTES_BUFFERING: 0
-      KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
-      KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
-      KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
-      KSQL_KSQL_CONNECT_URL: "http://connect:8083"
-      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: 1
-      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: 'true'
-      KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: 'true'
-
-  # ksqldb-cli:
-  #   image: confluentinc/cp-ksqldb-cli:7.5.3
-  #   container_name: ksqldb-cli
-  #   depends_on:
-  #     - broker
-  #     - connect
-  #     - ksqldb-server
-  #   entrypoint: /bin/sh
-  #   tty: true
-
-  # ksql-datagen:
-  #   image: confluentinc/ksqldb-examples:7.5.3
-  #   hostname: ksql-datagen
-  #   container_name: ksql-datagen
-  #   depends_on:
-  #     - ksqldb-server
-  #     - broker
-  #     - schema-registry
-  #     - connect
-  #   command: "bash -c 'echo Waiting for Kafka to be ready... && \
-  #                      cub kafka-ready -b broker:29092 1 40 && \
-  #                      echo Waiting for Confluent Schema Registry to be ready... && \
-  #                      cub sr-ready schema-registry 8081 40 && \
-  #                      echo Waiting a few seconds for topic creation to finish... && \
-  #                      sleep 11 && \
-  #                      tail -f /dev/null'"
-  #   environment:
-  #     KSQL_CONFIG_DIR: "/etc/ksql"
-  #     STREAMS_BOOTSTRAP_SERVERS: broker:29092
-  #     STREAMS_SCHEMA_REGISTRY_HOST: schema-registry
-  #     STREAMS_SCHEMA_REGISTRY_PORT: 8081
-
-  rest-proxy:
-    image: confluentinc/cp-kafka-rest:7.5.3
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      # Consumer groups are created per-run, so skip the 3s default rebalance debounce
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      # Tests create their topics explicitly; test_invalid_topic depends on this being off
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
+    healthcheck:
+      test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:9092"]
+      interval: 5s
+      timeout: 10s
+      retries: 24
     depends_on:
-      - broker
-      - schema-registry
-    ports:
-      - 8082:8082
-    hostname: rest-proxy
-    container_name: rest-proxy
-    environment:
-      KAFKA_REST_HOST_NAME: rest-proxy
-      KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092'
-      KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
-      KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
+      zoo1:
+        condition: service_healthy

conda/dev-environment-unix.yml

@@ -17,7 +17,6 @@ dependencies:
  - flex
  - graphviz
  - gtest
- - httpx>=0.20,<1
  - libarrow<24
  - libboost>=1.80.0
  - libboost-headers>=1.80.0
@@ -41,6 +40,7 @@ dependencies:
  - pytest-sugar
  - python>=3.10,<3.15
  - python-build
+ - python-confluent-kafka
  - python-graphviz
  - python-rapidjson
  - pytz

conda/dev-environment-win.yml

@@ -17,7 +17,6 @@ dependencies:
 #  - flex  # not available on windows
  - graphviz
  - gtest
- - httpx>=0.20,<1
  - libarrow<24
  - libboost>=1.80.0
  - libboost-headers>=1.80.0
@@ -41,6 +40,7 @@ dependencies:
  - pytest-sugar
  - python>=3.10,<3.14
  - python-build
+ - python-confluent-kafka
  - python-graphviz
  - python-rapidjson
  - pytz