docker-compose.yml

name: kafka-cdc-jdbc-sink

x-postgres-env: &postgres-env
  POSTGRES_USER: ${POSTGRES_USER:-postgres}
  POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
  POSTGRES_DB: ${POSTGRES_DB:-cdc_db}

x-clickhouse-password: &clickhouse-password ${CLICKHOUSE_PASSWORD:-clickhouse}

networks:
  cdc-network:
    driver: bridge

volumes:
  postgres-data:
  clickhouse-data:
  kafka-data:

services:
  postgres:
    image: postgres:16
    container_name: postgres
    networks: [cdc-network]
    ports:
      - "${POSTGRES_PORT:-5432}:5432"
    environment:
      <<: *postgres-env
    command:
      - "postgres"
      - "-c"
      - "wal_level=logical"
      - "-c"
      - "max_wal_senders=10"
      - "-c"
      - "max_replication_slots=10"
    volumes:
      - postgres-data:/var/lib/postgresql/data
      - ./postgres/init:/docker-entrypoint-initdb.d
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-cdc_db}"]
      interval: 5s
      timeout: 5s
      retries: 10

  kafka:
    image: confluentinc/cp-kafka:7.6.1
    container_name: kafka
    networks: [cdc-network]
    ports:
      - "${KAFKA_PORT:-9092}:9092"
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:29093
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENERS: PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:${KAFKA_PORT:-9092}
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      CLUSTER_ID: ${KAFKA_CLUSTER_ID:-MkU3OEVBNTcwNTJENDM2Qk}
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
    volumes:
      - kafka-data:/var/lib/kafka/data
    healthcheck:
      test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:9092"]
      interval: 5s
      timeout: 5s
      retries: 10

  schema-registry:
    image: confluentinc/cp-schema-registry:7.6.1
    container_name: schema-registry
    networks: [cdc-network]
    depends_on:
      kafka:
        condition: service_healthy
    ports:
      - "${SCHEMA_REGISTRY_PORT:-8081}:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:29092
      SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
    healthcheck:
      test: ["CMD-SHELL", "curl -sf http://localhost:8081/subjects"]
      interval: 5s
      timeout: 5s
      retries: 15

  connect:
    build:
      context: ./connect
    container_name: connect
    networks: [cdc-network]
    depends_on:
      kafka:
        condition: service_healthy
      postgres:
        condition: service_healthy
      clickhouse:
        condition: service_healthy
      schema-registry:
        condition: service_healthy
    ports:
      - "${CONNECT_PORT:-8083}:8083"
    environment:
      CONNECT_BOOTSTRAP_SERVERS: kafka:29092
      CONNECT_REST_ADVERTISED_HOST_NAME: connect
      CONNECT_GROUP_ID: cdc-connect-cluster
      CONNECT_CONFIG_STORAGE_TOPIC: _connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: _connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: _connect-status
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      CONNECT_LOG4J_ROOT_LOGLEVEL: INFO
      CONNECT_CONFIG_PROVIDERS: env
      CONNECT_CONFIG_PROVIDERS_ENV_CLASS: org.apache.kafka.common.config.provider.EnvVarConfigProvider
      <<: *postgres-env
      CLICKHOUSE_DB: ${CLICKHOUSE_DB:-cdc_db}
      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default}
      CLICKHOUSE_PASSWORD: *clickhouse-password
    healthcheck:
      test: ["CMD-SHELL", "curl -sf http://localhost:8083/connectors"]
      interval: 5s
      timeout: 5s
      retries: 20

  clickhouse:
    image: clickhouse/clickhouse-server:24.3
    container_name: clickhouse
    networks: [cdc-network]
    ports:
      - "${CLICKHOUSE_HTTP_PORT:-8123}:8123"
      - "${CLICKHOUSE_NATIVE_PORT:-9000}:9000"
    environment:
      CLICKHOUSE_PASSWORD: *clickhouse-password
    ulimits:
      nofile:
        soft: 262144
        hard: 262144
    volumes:
      - clickhouse-data:/var/lib/clickhouse
      - ./clickhouse/init:/docker-entrypoint-initdb.d
    healthcheck:
      test: ["CMD-SHELL", "wget --spider -q http://localhost:8123/ping"]
      interval: 5s
      timeout: 5s
      retries: 10

  akhq:
    image: tchiotludo/akhq:0.24.0
    container_name: akhq
    networks: [cdc-network]
    depends_on:
      kafka:
        condition: service_healthy
      connect:
        condition: service_healthy
      schema-registry:
        condition: service_healthy
    ports:
      - "${AKHQ_PORT:-8080}:8080"
    volumes:
      - ./akhq/application.yml:/app/application.yml

  connect-init:
    image: curlimages/curl:8.9.1
    container_name: connect-init
    networks: [cdc-network]
    depends_on:
      connect:
        condition: service_healthy
    volumes:
      - ./connectors:/connectors
      - ./scripts/register-connectors.sh:/register-connectors.sh
    entrypoint: ["sh", "/register-connectors.sh"]