docker-compose.yml

# Stack autônomo/reprodutível (HDFS próprio: namenode/datanode). O deploy no
# servidor do curso usa um overlay sobre a infra existente — ver
# deploy/server-lakehouse/. O hostname do HDFS é parametrizável por HDFS_HOST
# (default `namenode` aqui; `hadoop` no servidor), permitindo um único conjunto
# de arquivos/configs servir as duas topologias.
services:
  postgres:
    image: postgres:15
    container_name: datalab_postgres
    environment:
      # Defaults só valem para o stack local/descartável (`docker compose up`
      # sem configurar nada); em qualquer deploy real, sobrescreva via .env
      # (nunca commitado) — mesmo padrão de OPENAI_API_KEY.
      POSTGRES_USER: ${POSTGRES_USER:-dlab}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-d123}
      POSTGRES_DB: ${POSTGRES_DB:-datalab}
      # Lido por docker/postgres-init/01-metastore.sh (usuário do backing
      # store do Hive Metastore) — precisa bater com os args do build do
      # serviço hive-metastore, abaixo.
      METASTORE_DB_USER: ${METASTORE_DB_USER:-hive}
      METASTORE_DB_PASSWORD: ${METASTORE_DB_PASSWORD:-hive}
      # Lido por docker/postgres-init/02-superset.sh (metadados do Superset).
      SUPERSET_DB_USER: ${SUPERSET_DB_USER:-superset}
      SUPERSET_DB_PASSWORD: ${SUPERSET_DB_PASSWORD:-superset}
    ports:
      - "5432:5432"
    volumes:
      - postgres_data:/var/lib/postgresql/data
      # Provisiona o DB `metastore` do Hive (só em volume novo — ver
      # docker/postgres-init/01-metastore.sh para o fallback manual).
      - ./docker/postgres-init:/docker-entrypoint-initdb.d:ro
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-dlab} -d ${POSTGRES_DB:-datalab}"]
      interval: 10s
      retries: 5
      start_period: 5s

  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    container_name: datalab_hadoop_namenode
    environment:
      - CLUSTER_NAME=datalab
      - CORE_CONF_fs_defaultFS=hdfs://namenode:9000
    ports:
      - "9870:9870"
      - "9000:9000"
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
    restart: unless-stopped

  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: datalab_hadoop_datanode
    environment:
      - CORE_CONF_fs_defaultFS=hdfs://namenode:9000
      - SERVICE_PRECONDITION=namenode:9870
    depends_on:
      - namenode
    ports:
      - "9864:9864"
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
    restart: unless-stopped

  airflow-init:
    build:
      context: ./docker/airflow
      network: host  # necessário em hosts sem egress para as redes bridge do Docker (ver docker/airflow/README.md)
    image: datalab-airflow:local
    container_name: datalab_airflow_init
    depends_on:
      postgres:
        condition: service_healthy
    environment: &airflow-env
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER:-dlab}:${POSTGRES_PASSWORD:-d123}@postgres:5432/${POSTGRES_DB:-datalab}
      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
      SOURCE_POSTGRES_URL: ${SOURCE_POSTGRES_URL:-postgresql://usuario:senha@host:5432/ceara_transparente}
      BRONZE_STORAGE_BACKEND: hdfs
      BRONZE_BASE_PATH: /bronze
      SILVER_STORAGE_BACKEND: hdfs
      SILVER_BASE_PATH: /silver
      HDFS_WEBHDFS_URL: http://${HDFS_HOST:-namenode}:9870
      HDFS_USER: root
      # Conexão usada pelo SparkSubmitOperator (DAG 2) — cluster standalone.
      AIRFLOW_CONN_SPARK_DEFAULT: "spark://spark-master:7077"
      # DAG 4 (ml_inference) lê Silver/Gold e grava score/previsão via Trino.
      TRINO_HOST: trino
      TRINO_PORT: "8080"
      TRINO_USER: airflow
      TRINO_CATALOG: iceberg
      # Componente de IA Generativa (models/narrative_report.py, tarefa 25) —
      # lido do .env na raiz (nunca commitado); vazio aqui não quebra o build,
      # só faz a task falhar em runtime se ninguém configurar a chave.
      OPENAI_API_KEY: ${OPENAI_API_KEY:-}
      OPENAI_MODEL: ${OPENAI_MODEL:-gpt-4o-mini}
    entrypoint: /bin/bash
    command:
      - -c
      - "airflow db migrate && airflow users create --username ${AIRFLOW_ADMIN_USER:-admin} --password ${AIRFLOW_ADMIN_PASSWORD:-admin} --firstname Data --lastname Admin --role Admin --email admin@datalab.com || true"
    volumes:
      - ./dags:/opt/airflow/dags
      - ./src:/opt/airflow/src
      - ./models:/opt/airflow/models
      - airflow_logs:/opt/airflow/logs

  airflow-webserver:
    build:
      context: ./docker/airflow
      network: host
    image: datalab-airflow:local
    container_name: datalab_airflow_webserver
    command: webserver
    ports:
      - "8080:8080"
    depends_on:
      postgres:
        condition: service_healthy
      airflow-init:
        condition: service_completed_successfully
    environment: *airflow-env
    # Ver documentacao/workaround-egress-ipv4-api.md: o host não tem saída IPv4
    # (só IPv6), mas a API do Ceará Transparente é IPv4-only. Isto força o
    # hostname da API a resolver para um relay TCP alcançável via Tailscale.
    extra_hosts: &api-relay-host
      - "api-dados-abertos.cearatransparente.ce.gov.br:${API_EGRESS_RELAY_IP:-100.101.236.119}"
    volumes:
      - ./dags:/opt/airflow/dags
      - ./src:/opt/airflow/src
      - ./models:/opt/airflow/models
      - airflow_logs:/opt/airflow/logs
    restart: unless-stopped

  airflow-scheduler:
    build:
      context: ./docker/airflow
      network: host
    image: datalab-airflow:local
    container_name: datalab_airflow_scheduler
    command: scheduler
    depends_on:
      postgres:
        condition: service_healthy
      airflow-init:
        condition: service_completed_successfully
    environment: *airflow-env
    extra_hosts: *api-relay-host
    volumes:
      - ./dags:/opt/airflow/dags
      - ./src:/opt/airflow/src
      - ./models:/opt/airflow/models
      - airflow_logs:/opt/airflow/logs
      # DAG 3 dispara o dbt via DockerOperator (imagem datalab-dbt) -> precisa do
      # socket do Docker. Ver documentacao/gold-dbt-trino.md (permissão do socket).
      - /var/run/docker.sock:/var/run/docker.sock
    restart: unless-stopped

  jupyter:
    image: jupyter/scipy-notebook:python-3.11
    container_name: datalab_jupyter
    ports:
      - "8888:8888"
    environment:
      - JUPYTER_TOKEN=${JUPYTER_TOKEN:-datalab}
      - SOURCE_POSTGRES_URL=${SOURCE_POSTGRES_URL:-postgresql://usuario:senha@host:5432/ceara_transparente}
      - BRONZE_STORAGE_BACKEND=hdfs
      - BRONZE_BASE_PATH=/bronze
      - HDFS_WEBHDFS_URL=http://${HDFS_HOST:-namenode}:9870
      - HDFS_USER=root
    volumes:
      - ./notebooks:/home/jovyan/work/notebooks
      - ./src:/home/jovyan/work/src
    depends_on:
      - namenode
    restart: unless-stopped

  hive-metastore:
    build:
      context: ./docker/hive
      network: host  # host só tem IPv6; ver docker/airflow/README.md
      args:
        HDFS_HOST: ${HDFS_HOST:-namenode}
        METASTORE_DB_HOST: ${METASTORE_DB_HOST:-postgres}
        METASTORE_DB_USER: ${METASTORE_DB_USER:-hive}
        METASTORE_DB_PASSWORD: ${METASTORE_DB_PASSWORD:-hive}
    image: datalab-hive-metastore:local
    container_name: datalab_hive_metastore
    mem_limit: 2g
    cpus: 1.0
    healthcheck:
      test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/9083' 2>/dev/null || exit 1"]
      interval: 20s
      timeout: 5s
      retries: 5
      start_period: 40s
    depends_on:
      postgres:
        condition: service_healthy
      namenode:
        condition: service_started
    ports:
      - "9083:9083"
    restart: unless-stopped

  spark-master:
    build:
      context: ./docker/spark
      network: host
    image: datalab-spark:local
    container_name: datalab_spark_master
    mem_limit: 8g
    cpus: 3.0
    entrypoint: ["/opt/spark/bin/spark-class"]
    command: ["org.apache.spark.deploy.master.Master", "--host", "spark-master", "--port", "7077", "--webui-port", "8081"]
    environment: &spark-env
      HIVE_METASTORE_URI: thrift://hive-metastore:9083
      ICEBERG_WAREHOUSE: hdfs://${HDFS_HOST:-namenode}:9000/warehouse
      HDFS_DEFAULT_FS: hdfs://${HDFS_HOST:-namenode}:9000
      BRONZE_BASE_PATH: /bronze
      # HDFS roda como root/superusuário; sem isto, escrever no /warehouse falha.
      HADOOP_USER_NAME: root
    ports:
      - "7077:7077"
      - "8081:8081"
    depends_on:
      - namenode
      - hive-metastore
    # Repo montado para execução manual/debug de jobs direto no cluster (local[*]).
    volumes:
      - ./src:/opt/datalab/src
    restart: unless-stopped

  spark-worker:
    build:
      context: ./docker/spark
      network: host
    image: datalab-spark:local
    container_name: datalab_spark_worker
    mem_limit: 10g
    cpus: 4.0
    entrypoint: ["/opt/spark/bin/spark-class"]
    command: ["org.apache.spark.deploy.worker.Worker", "spark://spark-master:7077", "--webui-port", "8081"]
    environment: *spark-env
    ports:
      - "8082:8081"
    depends_on:
      - spark-master
    volumes:
      - ./src:/opt/datalab/src
    restart: unless-stopped

  trino:
    build:
      context: ./docker/trino
      network: host
      args:
        HDFS_HOST: ${HDFS_HOST:-namenode}
    image: datalab-trino:local
    container_name: datalab_trino
    mem_limit: 6g
    cpus: 3.0
    environment:
      HADOOP_USER_NAME: root  # escrever a Gold Iceberg no HDFS como superusuário
    depends_on:
      - hive-metastore
      - namenode
    ports:
      - "8085:8080"  # UI/HTTP do Trino (8080 já é do Airflow no host)
    restart: unless-stopped

  superset-init:
    build:
      context: ./docker/superset
      network: host
    image: datalab-superset:local
    container_name: datalab_superset_init
    depends_on:
      postgres:
        condition: service_healthy
    environment: &superset-env
      SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY:-changeme-dev-only-secret-key}
      DATABASE_DIALECT: postgresql
      DATABASE_USER: ${SUPERSET_DB_USER:-superset}
      DATABASE_PASSWORD: ${SUPERSET_DB_PASSWORD:-superset}
      DATABASE_HOST: postgres
      DATABASE_PORT: "5432"
      DATABASE_DB: superset
    entrypoint: /bin/bash
    command:
      - -c
      - "superset db upgrade && superset fab create-admin --username ${SUPERSET_ADMIN_USER:-admin} --firstname Superset --lastname Admin --email admin@datalab.com --password ${SUPERSET_ADMIN_PASSWORD:-admin} || true && superset init"
    volumes:
      - superset_home:/app/superset_home

  superset:
    image: datalab-superset:local
    container_name: datalab_superset
    mem_limit: 2g
    cpus: 1.0
    depends_on:
      superset-init:
        condition: service_completed_successfully
    environment: *superset-env
    ports:
      # 8089, não 8088: no servidor real esse host já usa 8088 pra UI do YARN
      # do Hadoop (aula_hadoop, container do curso) — mantido igual aqui pra
      # não ter porta divergente entre os dois ambientes.
      - "8089:8088"
    volumes:
      - superset_home:/app/superset_home
    restart: unless-stopped

  # dbt (adapter Trino) — não sobe no `up` (profile "tools"); é buildado e rodado
  # sob demanda: `docker compose run --rm dbt build`, ou pela DAG 3 via DockerOperator.
  dbt:
    build:
      context: ./dbt
      network: host
    image: datalab-dbt:local
    container_name: datalab_dbt
    profiles: ["tools"]
    depends_on:
      - trino
    volumes:
      - ./dbt:/dbt
    command: ["build"]

# Nome de rede fixo para o DockerOperator da DAG 3 referenciar de forma
# determinística (network_mode="datalab_net").
networks:
  default:
    name: datalab_net

volumes:
  postgres_data:
  hadoop_namenode:
  hadoop_datanode:
  airflow_logs:
  superset_home: