ai_price_radar/docker-compose.pricememo.yml at main · BeterXie/ai_price_radar · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
services:
  api:
    build:
      context: ./apps/api
      args:
        PIP_INDEX_URL: https://mirrors.aliyun.com/pypi/simple/
    networks:
      default:
      detector_control:
      pricememo_frontend:
        aliases:
          - ai-radar-api

  web:
    build:
      context: ./apps/web
      dockerfile: Dockerfile.prebuilt
      args:
        ALPINE_REPOSITORY_HOST: mirrors.aliyun.com
    environment:
      INDEXNOW_KEY: ${INDEXNOW_KEY:-}
    volumes:
      - ./data/support:/app/public/support:ro
    networks:
      default:
      pricememo_frontend:
        aliases:
          - ai-radar-web

  notification-worker:
    build:
      context: ./apps/api
      args:
        PIP_INDEX_URL: https://mirrors.aliyun.com/pypi/simple/

  importer:
    build:
      context: .
      dockerfile: pipeline/Dockerfile
      args:
        PIP_INDEX_URL: https://mirrors.aliyun.com/pypi/simple/

  crawler:
    profiles: ["crawler"]
    build:
      context: .
      dockerfile: crawler/ldxp/Dockerfile.remote
      args:
        PIP_INDEX_URL: https://mirrors.aliyun.com/pypi/simple/
    init: true
    mem_limit: 1g
    shm_size: 1gb
    environment:
      HOME: /home/crawler
      PLAYWRIGHT_BROWSERS_PATH: /ms-playwright
      INTAKE_API_URL: ${INTAKE_API_URL:-http://api:8000}
      INTAKE_WORKER_KEY: ${INTAKE_WORKER_KEY:-replace-with-a-separate-intake-worker-key}
      DISCOVERY_API_URL: ${DISCOVERY_API_URL:-http://api:8000}
      DISCOVERY_WORKER_KEY: ${DISCOVERY_WORKER_KEY:-}
      GITHUB_TOKEN: ${GITHUB_TOKEN:-}
      DISCOVERY_SOURCES: ${DISCOVERY_SOURCES:-seed,16688,commoncrawl,bing,github}
      DISCOVERY_DUJIAO_SOURCES: ${DISCOVERY_DUJIAO_SOURCES:-seed,bing,github}
      DISCOVERY_MAX_RAW_URLS: ${DISCOVERY_MAX_RAW_URLS:-2000}
      DISCOVERY_MAX_NEW_CANDIDATES: ${DISCOVERY_MAX_NEW_CANDIDATES:-1000}
      DISCOVERY_BING_PAGES: ${DISCOVERY_BING_PAGES:-5}
      DISCOVERY_BING_COUNT: ${DISCOVERY_BING_COUNT:-30}
      DISCOVERY_BING_DELAY_SECONDS: ${DISCOVERY_BING_DELAY_SECONDS:-2}
      DISCOVERY_GITHUB_PAGES: ${DISCOVERY_GITHUB_PAGES:-3}
      DISCOVERY_GITHUB_COUNT: ${DISCOVERY_GITHUB_COUNT:-100}
      DISCOVERY_GITHUB_MAX_CANDIDATES: ${DISCOVERY_GITHUB_MAX_CANDIDATES:-300}
      DISCOVERY_MAX_PROCESSED_CANDIDATES: ${DISCOVERY_MAX_PROCESSED_CANDIDATES:-3000}
      DISCOVERY_REVERIFY_STALE_HOURS: ${DISCOVERY_REVERIFY_STALE_HOURS:-24}
      DISCOVERY_REQUEST_INTERVAL_SECONDS: ${DISCOVERY_REQUEST_INTERVAL_SECONDS:-2}
      DISCOVERY_COMMONCRAWL_INDEXES: ${DISCOVERY_COMMONCRAWL_INDEXES:-2}
      DISCOVERY_COMMONCRAWL_MAX_URLS: ${DISCOVERY_COMMONCRAWL_MAX_URLS:-500}
      DISCOVERY_16688_SOURCE_PAGES: ${DISCOVERY_16688_SOURCE_PAGES:-10}
    volumes:
      - ./crawler/ldxp/seeds.txt:/config/seeds.txt:ro
      - ./config/discovery/general_seeds.txt:/config/general_seeds.txt:ro
      - ./config/discovery/dujiao_seeds.txt:/config/dujiao_seeds.txt:ro
      - ./data/crawler:/data

  crawler-bootstrap:
    profiles: ["bootstrap"]
    build:
      context: .
      dockerfile: crawler/ldxp/Dockerfile.bootstrap
      args:
        PIP_INDEX_URL: https://mirrors.aliyun.com/pypi/simple/
    init: true
    shm_size: 1gb
    environment:
      HOME: /home/crawler
      PLAYWRIGHT_BROWSERS_PATH: /ms-playwright
      INTAKE_API_URL: ${INTAKE_API_URL:-http://api:8000}
      INTAKE_WORKER_KEY: ${INTAKE_WORKER_KEY:-replace-with-a-separate-intake-worker-key}
    ports:
      - "127.0.0.1:6080:6080"
    volumes:
      - ./data/crawler:/data

networks:
  pricememo_frontend:
    external: true
    name: pricememo_frontend