ingest-file/docker-compose.yml at main · openaleph/ingest-file · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
x-ingest-env: &ingest-env
  OPENALEPH_DB_URI: postgresql://ingest:ingest@postgres/ingest
  FTM_FRAGMENTS_URI: postgresql://ingest:ingest@postgres/ingest
  # the lakehouse journal defaults to an in-memory sqlite, which is per-process
  # and so invisible to any other reader. the driver has to be spelled out: the
  # journal hands the uri to `create_engine` as-is and bare `postgresql://`
  # resolves to psycopg2, which we don't ship (only psycopg 3)
  LAKEHOUSE_JOURNAL_URI: postgresql+psycopg://ingest:ingest@postgres/ingest
  LOG_FORMAT: TEXT # TEXT or JSON
  PYTHONWARNINGS: "ignore"
  TYPER_STANDARD_TRACEBACK: True

services:
  postgres:
    image: postgres:17
    environment:
      POSTGRES_USER: ingest
      POSTGRES_PASSWORD: ingest
      POSTGRES_DATABASE: ingest
    # over tcp, not the socket: during initdb the entrypoint runs a temporary
    # server that only listens on the socket, so a socket check reports ready
    # before the database actually accepts connections
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U ingest -d ingest"]
      interval: 2s
      timeout: 5s
      retries: 30

  redis:
    image: redis:alpine
    command: ["redis-server", "--save", "3600", "10"]
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 2s
      timeout: 5s
      retries: 30

  ingest-file:
    build:
      context: .
      target: runtime
    hostname: ingest
    tmpfs:
      - /tmp:mode=777
      - /data:mode=777
    environment:
      <<: *ingest-env
    volumes:
      - "./ingestors:/ingestors/ingestors"
      - "./data:/ingestors/data"
    depends_on:
      redis:
        condition: service_healthy
      postgres:
        condition: service_healthy

  test-ingest-file:
    build:
      context: .
      target: test
    image: test-ingest-file
    hostname: ingest
    tmpfs:
      - /tmp:mode=777
      # LAKEHOUSE_URI points here, so lakehouse output is ephemeral per run
      - /data:mode=777
    environment:
      <<: *ingest-env
      DEBUG: 1
      PYTHONDEVMODE: 1
      PYTHONTRACEMALLOC: 1
    volumes:
      - "./ingestors:/ingestors/ingestors"
      - "./tests:/ingestors/tests"
    # both backends talk to postgres: the fragments store, the lakehouse journal
    # and, either way, the tags cache. `run` brings it up for us, so `make
    # test-*` works without a preceding `make services` (CI relies on that).
    depends_on:
      redis:
        condition: service_healthy
      postgres:
        condition: service_healthy