docker-compose.yml

services:
  postgres:
    image: postgres:16
    container_name: ai-observability-postgres
    command:
      - "postgres"
      - "-c"
      - "wal_level=logical"
      - "-c"
      - "max_replication_slots=10"
      - "-c"
      - "max_wal_senders=10"
    ports:
      - "5432:5432"
    environment:
      POSTGRES_DB: ai_observability
      POSTGRES_USER: ai_observability
      POSTGRES_PASSWORD: ai_observability
    volumes:
      - postgres_data:/var/lib/postgresql/data
      - ./sql/source_postgres_schema.sql:/docker-entrypoint-initdb.d/01_source_postgres_schema.sql:ro

  gravitino:
    image: apache/gravitino:1.2.0
    container_name: ai-observability-gravitino
    restart: unless-stopped
    ports:
      - "8090:8090"
    environment:
      GRAVITINO_MEM: "-Xms256m -Xmx512m -XX:MaxMetaspaceSize=256m"
      GRAVITINO_USE_WEB_V2: "true"
    volumes:
      - gravitino_data:/root/gravitino/data
      - paimon_warehouse:/workspace/data/paimon
    healthcheck:
      test:
        - CMD-SHELL
        - >-
          curl -fsS
          -H 'Accept: application/vnd.gravitino.v1+json'
          http://localhost:8090/api/version >/dev/null
      interval: 5s
      timeout: 3s
      retries: 24
      start_period: 20s
    depends_on:
      - postgres

  flink-jobmanager:
    build:
      context: .
      dockerfile: docker/flink/Dockerfile
    image: ai-observability-flink:1.20-paimon
    container_name: ai-observability-flink-jobmanager
    user: root
    command: jobmanager
    ports:
      - "8081:8081"
    environment:
      FLINK_PROPERTIES: |
        jobmanager.rpc.address: flink-jobmanager
        rest.address: 0.0.0.0
        parallelism.default: 1
        restart-strategy.type: fixed-delay
        restart-strategy.fixed-delay.attempts: 3
        restart-strategy.fixed-delay.delay: 10s
        execution.checkpointing.interval: 10s
        execution.checkpointing.mode: EXACTLY_ONCE
        execution.checkpointing.externalized-checkpoint-retention: RETAIN_ON_CANCELLATION
        state.checkpoints.dir: file:///workspace/data/paimon/_checkpoints
        state.savepoints.dir: file:///workspace/data/paimon/_savepoints
        classloader.parent-first-patterns.additional: org.apache.commons.collections;io.debezium
    volumes:
      - ./flink:/workspace/flink:ro
      - paimon_warehouse:/workspace/data/paimon
    depends_on:
      - postgres
      - kafka
      - gravitino

  flink-taskmanager:
    image: ai-observability-flink:1.20-paimon
    container_name: ai-observability-flink-taskmanager
    user: root
    command: taskmanager
    environment:
      FLINK_PROPERTIES: |
        jobmanager.rpc.address: flink-jobmanager
        taskmanager.numberOfTaskSlots: 4
        parallelism.default: 1
        restart-strategy.type: fixed-delay
        restart-strategy.fixed-delay.attempts: 3
        restart-strategy.fixed-delay.delay: 10s
        execution.checkpointing.interval: 10s
        execution.checkpointing.mode: EXACTLY_ONCE
        execution.checkpointing.externalized-checkpoint-retention: RETAIN_ON_CANCELLATION
        state.checkpoints.dir: file:///workspace/data/paimon/_checkpoints
        state.savepoints.dir: file:///workspace/data/paimon/_savepoints
        classloader.parent-first-patterns.additional: org.apache.commons.collections;io.debezium
    volumes:
      - ./flink:/workspace/flink:ro
      - paimon_warehouse:/workspace/data/paimon
    depends_on:
      - flink-jobmanager
      - gravitino

  flink-sql-client:
    image: ai-observability-flink:1.20-paimon
    user: root
    command: bin/sql-client.sh
    environment:
      FLINK_PROPERTIES: |
        jobmanager.rpc.address: flink-jobmanager
        rest.address: flink-jobmanager
        parallelism.default: 1
        restart-strategy.type: fixed-delay
        restart-strategy.fixed-delay.attempts: 3
        restart-strategy.fixed-delay.delay: 10s
        execution.checkpointing.interval: 10s
        execution.checkpointing.mode: EXACTLY_ONCE
        execution.checkpointing.externalized-checkpoint-retention: RETAIN_ON_CANCELLATION
        state.checkpoints.dir: file:///workspace/data/paimon/_checkpoints
        state.savepoints.dir: file:///workspace/data/paimon/_savepoints
        classloader.parent-first-patterns.additional: org.apache.commons.collections;io.debezium
    volumes:
      - ./flink:/workspace/flink:ro
      - paimon_warehouse:/workspace/data/paimon
    depends_on:
      - flink-jobmanager

  kafka:
    image: apache/kafka:3.9.0
    container_name: ai-observability-kafka
    ports:
      - "9092:9092"
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_LOG_RETENTION_HOURS: 48
      CLUSTER_ID: 'ai-observability-kafka-cluster-001'
    volumes:
      - kafka_data:/var/lib/kafka/data

  doris-fe:
    image: apache/doris:fe-2.1.11
    container_name: ai-observability-doris-fe
    entrypoint: ["/opt/apache-doris/fe/bin/start_fe.sh", "--console"]
    ports:
      - "8030:8030"
      - "9030:9030"
    volumes:
      - doris_fe_data:/opt/apache-doris/fe/doris-meta
      - paimon_warehouse:/workspace/data/paimon:ro

  doris-be:
    image: apache/doris:be-2.1.11
    container_name: ai-observability-doris-be
    entrypoint: ["bash", "-c", "export SKIP_CHECK_ULIMIT=true && /opt/apache-doris/be/bin/start_be.sh --console"]
    ports:
      - "8040:8040"
      - "9050:9050"
    volumes:
      - doris_be_data:/opt/apache-doris/be/storage
      - paimon_warehouse:/workspace/data/paimon:ro
    depends_on:
      - doris-fe

  doris-init:
    image: mysql:8.4
    container_name: ai-observability-doris-init
    entrypoint:
      - /bin/sh
      - /init_fe.sh
    restart: "no"
    volumes:
      - ./config/doris/init_fe.sh:/init_fe.sh:ro
    depends_on:
      - doris-fe
      - doris-be

  superset-metadata:
    image: postgres:16
    container_name: ai-observability-superset-db
    environment:
      POSTGRES_DB: superset
      POSTGRES_USER: superset
      POSTGRES_PASSWORD: superset
    volumes:
      - superset_metadata:/var/lib/postgresql/data

  superset-redis:
    image: redis:7-alpine
    container_name: ai-observability-superset-redis

  superset:
    build:
      context: .
      dockerfile: docker/superset/Dockerfile
    image: ai-observability-superset:local
    container_name: ai-observability-superset
    ports:
      - "8088:8088"
    environment:
      SUPERSET_SECRET_KEY: "ai-observability-local-dev-secret-key"
      SQLALCHEMY_DATABASE_URI: "postgresql://superset:superset@superset-metadata:5432/superset"
      REDIS_URL: "redis://superset-redis:6379/0"
      SUPERSET_LOAD_EXAMPLES: "no"
    volumes:
      - superset_home:/app/superset_home
      - ./config/superset:/app/config/superset:ro
      - ./scripts:/app/bootstrap-scripts:ro
    depends_on:
      - superset-metadata
      - superset-redis
      - doris-fe

  grafana:
    image: grafana/grafana-oss:11.6.0
    container_name: ai-observability-grafana
    ports:
      - "3001:3000"
    environment:
      GF_SECURITY_ADMIN_USER: admin
      GF_SECURITY_ADMIN_PASSWORD: admin
      GF_INSTALL_PLUGINS: ""
    volumes:
      - grafana_data:/var/lib/grafana
      - ./config/grafana/provisioning:/etc/grafana/provisioning:ro
  
volumes:
  postgres_data:
  gravitino_data:
  paimon_warehouse:
  doris_fe_data:
  doris_be_data:
  kafka_data:
  superset_metadata:
  superset_home:
  grafana_data: