Skip to content

Commit 9b526f4

Browse files
hwdsl2hwdsl2
committed
Add workflows
1 parent 857c215 commit 9b526f4

6 files changed

Lines changed: 414 additions & 0 deletions

File tree

‎.github/workflows/buildx.yml‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
name: buildx
2+
3+
on:
4+
workflow_call:
5+
secrets:
6+
CACHE_NAME:
7+
required: true
8+
DOCKER_TOKEN:
9+
required: true
10+
QUAY_USER:
11+
required: true
12+
QUAY_TOKEN:
13+
required: true
14+
BUILD_ONLY:
15+
required: true
16+
17+
jobs:
18+
buildx:
19+
runs-on: ubuntu-24.04
20+
if: github.repository_owner == 'hwdsl2'
21+
steps:
22+
- uses: actions/checkout@v6
23+
with:
24+
persist-credentials: false
25+
- name: Cache
26+
uses: actions/cache@v5
27+
with:
28+
path: |
29+
${{ runner.temp }}/.buildx-bin
30+
${{ runner.temp }}/.buildx-cache
31+
${{ runner.temp }}/.docker-images
32+
key: ${{ secrets.CACHE_NAME }}-${{ github.sha }}-${{ github.run_id }}
33+
restore-keys: |
34+
${{ secrets.CACHE_NAME }}-
35+
- name: Set up Buildx
36+
env:
37+
RUNNER_TEMP: ${{ runner.temp }}
38+
run: |
39+
[ ! -x /usr/bin/docker ] && exit 1
40+
if [ ! -x /usr/bin/wget ] || [ ! -x /usr/bin/jq ]; then
41+
export DEBIAN_FRONTEND=noninteractive
42+
sudo apt-get -yq update
43+
sudo apt-get -yq install wget jq
44+
fi
45+
46+
BUILDX_VER=$(wget -t 3 -T 30 -qO- "https://api.github.com/repos/docker/buildx/releases/latest" | jq -r ".tag_name")
47+
[ -z "${BUILDX_VER}" ] && exit 1
48+
echo "Buildx version: ${BUILDX_VER}"
49+
50+
DOCKER_DIR="${HOME}/.docker/cli-plugins"
51+
CACHE_DIR="${RUNNER_TEMP}/.buildx-bin"
52+
mkdir -p "${DOCKER_DIR}"
53+
if [ -s "${CACHE_DIR}/docker-buildx-${BUILDX_VER}" ]; then
54+
echo "Using buildx binary from cache..."
55+
cp -f "${CACHE_DIR}/docker-buildx-${BUILDX_VER}" "${DOCKER_DIR}/docker-buildx"
56+
else
57+
echo "Downloading buildx..."
58+
BUILDX_URL="https://github.com/docker/buildx/releases/download/${BUILDX_VER}/buildx-${BUILDX_VER}.linux-amd64"
59+
wget -t 3 -T 30 -nv -O "${DOCKER_DIR}/docker-buildx" "${BUILDX_URL}"
60+
[ ! -s "${DOCKER_DIR}/docker-buildx" ] && exit 1
61+
/bin/rm -rf "${CACHE_DIR}"
62+
mkdir -p "${CACHE_DIR}"
63+
/bin/cp -f "${DOCKER_DIR}/docker-buildx" "${CACHE_DIR}/docker-buildx-${BUILDX_VER}"
64+
fi
65+
chmod a+x "${DOCKER_DIR}/docker-buildx"
66+
67+
docker info
68+
docker buildx version
69+
70+
image_cache_dir="${RUNNER_TEMP}/.docker-images"
71+
for image in "quay.io/hwdsl2/qemu-user-static:latest" "moby/buildkit:buildx-stable-1"; do
72+
image_file=$(printf '%s' "${image}.tar.gz" | tr '/' '-' | tr ':' '-')
73+
if [ -s "${image_cache_dir}/${image_file}" ]; then
74+
echo "Using ${image} from cache..."
75+
docker load -i "${image_cache_dir}/${image_file}"
76+
else
77+
echo "Downloading ${image}..."
78+
docker pull -q "${image}"
79+
mkdir -p "${image_cache_dir}"
80+
docker save "${image}" | gzip > "${image_cache_dir}/${image_file}"
81+
fi
82+
done
83+
84+
docker run --rm --privileged quay.io/hwdsl2/qemu-user-static:latest --reset -p yes --credential yes
85+
docker buildx create --name "builder-${GITHUB_SHA::8}" --driver docker-container --use
86+
docker buildx inspect --bootstrap
87+
- name: Quay.io Login
88+
env:
89+
QUAY_USER: ${{ secrets.QUAY_USER }}
90+
QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }}
91+
run: echo "$QUAY_TOKEN" | docker login quay.io --username "$QUAY_USER" --password-stdin 2>&1
92+
- name: Docker Hub Login
93+
env:
94+
DOCKER_USER: ${{ github.repository_owner }}
95+
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
96+
run: echo "$DOCKER_TOKEN" | docker login --username "$DOCKER_USER" --password-stdin 2>&1
97+
- name: Docker Buildx
98+
env:
99+
DOCKER_USER: ${{ github.repository_owner }}
100+
BUILD_ONLY: ${{ secrets.BUILD_ONLY }}
101+
RUNNER_TEMP: ${{ runner.temp }}
102+
run: |
103+
cd "$GITHUB_WORKSPACE"
104+
pwd
105+
ls -ld embed.env.example
106+
107+
[ "$BUILD_ONLY" = "true" ] && BUILD_ONLY=""
108+
ts_now=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
109+
110+
docker buildx build \
111+
--progress plain \
112+
--platform=linux/amd64 \
113+
--build-arg BUILD_DATE=$ts_now \
114+
--build-arg VCS_REF=${GITHUB_SHA::8} \
115+
--build-arg VERSION=latest \
116+
--cache-from type=local,src="${RUNNER_TEMP}/.buildx-cache" \
117+
--cache-to type=local,dest="${RUNNER_TEMP}/.buildx-cache-new" \
118+
-t "$DOCKER_USER/embeddings-server:latest" \
119+
--pull \
120+
${BUILD_ONLY:+--push} \
121+
.
122+
123+
docker buildx build \
124+
--progress plain \
125+
--platform=linux/amd64 \
126+
--build-arg BUILD_DATE=$ts_now \
127+
--build-arg VCS_REF=${GITHUB_SHA::8} \
128+
--build-arg VERSION=latest \
129+
--cache-from type=local,src="${RUNNER_TEMP}/.buildx-cache-new" \
130+
-t "quay.io/$DOCKER_USER/embeddings-server:latest" \
131+
--pull \
132+
${BUILD_ONLY:+--push} \
133+
.
134+
135+
/bin/rm -rf "${RUNNER_TEMP}/.buildx-cache"
136+
/bin/mv -f "${RUNNER_TEMP}/.buildx-cache-new" "${RUNNER_TEMP}/.buildx-cache"
137+
- name: Clear
138+
if: always()
139+
run: |
140+
shred -u "${HOME}/.docker/config.json"
141+
rm -f "${HOME}/.docker/config.json"
142+
docker buildx rm "builder-${GITHUB_SHA::8}" || true

‎.github/workflows/check_update.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: check_update
2+
3+
on:
4+
workflow_call:
5+
secrets:
6+
CACHE_NAME:
7+
required: true
8+
outputs:
9+
should_test:
10+
value: ${{ jobs.check_update.outputs.should_test }}
11+
should_update:
12+
value: ${{ jobs.check_update.outputs.should_update }}
13+
14+
jobs:
15+
check_update:
16+
runs-on: ubuntu-24.04
17+
if: github.repository_owner == 'hwdsl2'
18+
env:
19+
DOCKER_USER: ${{ github.repository_owner }}
20+
outputs:
21+
should_test: ${{ steps.check.outputs.should_test }}
22+
should_update: ${{ steps.check.outputs.should_update }}
23+
steps:
24+
- name: Cache
25+
uses: actions/cache@v5
26+
with:
27+
path: |
28+
${{ runner.temp }}/.buildx-bin
29+
${{ runner.temp }}/.buildx-cache
30+
${{ runner.temp }}/.docker-images
31+
key: ${{ secrets.CACHE_NAME }}-${{ github.sha }}-${{ github.run_id }}-check
32+
restore-keys: |
33+
${{ secrets.CACHE_NAME }}-
34+
- name: Prepare
35+
run: |
36+
docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-latest
37+
docker pull "$DOCKER_USER/embeddings-server"
38+
- name: Check
39+
id: check
40+
run: |
41+
BASE_UPDATED=false
42+
43+
base_ts=$(docker inspect --format='{{.Created}}' ghcr.io/huggingface/text-embeddings-inference:cpu-latest)
44+
image_ts=$(docker inspect --format='{{.Created}}' "$DOCKER_USER/embeddings-server")
45+
46+
if [ -n "$base_ts" ] && [ -n "$image_ts" ]; then
47+
base_ts_s=$(date -d "$base_ts" +%s)
48+
image_ts_s=$(date -d "$image_ts" +%s)
49+
ts_now=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
50+
ts_now_s=$(date -d "$ts_now" +%s)
51+
diff_s=$((ts_now_s - base_ts_s))
52+
diff=$(printf '%dd %dh:%dm:%ds\n' $((diff_s/86400)) $((diff_s%86400/3600)) $((diff_s%3600/60)) $((diff_s%60)))
53+
54+
echo "Base update time: $base_ts"
55+
echo "Image update time: $image_ts"
56+
echo "Current time: $ts_now"
57+
echo "Time diff (cur-base): $diff (${diff_s}s)"
58+
59+
if [ -n "$base_ts_s" ] && [ -n "$image_ts_s" ] \
60+
&& [ "$base_ts_s" -ge "$image_ts_s" ] \
61+
&& [ "$diff_s" -ge 14400 ]; then
62+
echo "Starting build..."
63+
BASE_UPDATED=true
64+
else
65+
echo "Not starting build."
66+
fi
67+
fi
68+
echo "should_test=${BASE_UPDATED}" >> "$GITHUB_OUTPUT"
69+
echo "should_update=${BASE_UPDATED}" >> "$GITHUB_OUTPUT"

‎.github/workflows/cron.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: build cron
2+
3+
on:
4+
schedule:
5+
- cron: '30 3 * * 1,4'
6+
7+
jobs:
8+
check_update:
9+
if: github.repository_owner == 'hwdsl2'
10+
uses: ./.github/workflows/check_update.yml
11+
secrets:
12+
CACHE_NAME: ${{ secrets.CACHE_NAME }}
13+
14+
embed_test:
15+
needs: check_update
16+
if: needs.check_update.outputs.should_test == 'true'
17+
uses: ./.github/workflows/embed_test.yml
18+
19+
buildx:
20+
needs: [check_update, embed_test]
21+
if: needs.check_update.outputs.should_update == 'true'
22+
uses: ./.github/workflows/buildx.yml
23+
secrets:
24+
CACHE_NAME: ${{ secrets.CACHE_NAME }}
25+
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
26+
QUAY_USER: ${{ secrets.QUAY_USER }}
27+
QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }}
28+
BUILD_ONLY: ${{ secrets.BUILD_ONLY }}

‎.github/workflows/embed_test.yml‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
name: embed_test
2+
3+
on: workflow_call
4+
5+
jobs:
6+
embed_test:
7+
runs-on: ubuntu-24.04
8+
if: github.repository_owner == 'hwdsl2'
9+
strategy:
10+
matrix:
11+
test_id: [no-env, with-env]
12+
fail-fast: false
13+
env:
14+
TEST_ID: ${{ matrix.test_id }}
15+
steps:
16+
- uses: actions/checkout@v6
17+
with:
18+
persist-credentials: false
19+
- name: Test
20+
run: |
21+
[ ! -x /usr/bin/docker ] && exit 1
22+
23+
cd "$GITHUB_WORKSPACE"
24+
pwd
25+
ls -ld embed.env.example
26+
27+
set -x
28+
29+
docker build -t embed-test .
30+
31+
if [ "$TEST_ID" = "with-env" ]; then
32+
docker run \
33+
--name "$TEST_ID" \
34+
--restart=always \
35+
-v embed-test-data:/var/lib/embeddings \
36+
-v ./embed.env.example:/embed.env:ro \
37+
-e EMBED_MODEL=BAAI/bge-small-en-v1.5 \
38+
-p 8000:8000/tcp \
39+
-d embed-test
40+
elif [ "$TEST_ID" = "no-env" ]; then
41+
docker run \
42+
--name "$TEST_ID" \
43+
--restart=always \
44+
-v embed-test-data:/var/lib/embeddings \
45+
-e EMBED_MODEL=BAAI/bge-small-en-v1.5 \
46+
-p 8000:8000/tcp \
47+
-d embed-test
48+
fi
49+
50+
# Wait for server to be ready (up to 300 seconds for model download)
51+
i=0
52+
while [ "$i" -lt 300 ]; do
53+
if docker exec "$TEST_ID" curl -sf http://127.0.0.1:8000/health >/dev/null 2>&1; then
54+
break
55+
fi
56+
sleep 1
57+
i=$((i + 1))
58+
done
59+
60+
docker ps | grep "$TEST_ID"
61+
docker logs "$TEST_ID"
62+
63+
# Verify health endpoint
64+
docker exec "$TEST_ID" curl -sf http://127.0.0.1:8000/health
65+
66+
# Verify info endpoint
67+
docker exec "$TEST_ID" curl -sf http://127.0.0.1:8000/info \
68+
| grep -q '"model_id"'
69+
70+
# Test embed_manage commands
71+
docker exec "$TEST_ID" embed_manage --showinfo
72+
docker exec "$TEST_ID" embed_manage --listmodels
73+
74+
# Test embeddings endpoint (single input)
75+
curl -sf http://127.0.0.1:8000/v1/embeddings \
76+
-H "Content-Type: application/json" \
77+
-d '{"input": "The quick brown fox", "model": "text-embedding-ada-002"}' \
78+
| grep -q '"embedding"'
79+
80+
# Test embeddings endpoint (batch input)
81+
curl -sf http://127.0.0.1:8000/v1/embeddings \
82+
-H "Content-Type: application/json" \
83+
-d '{"input": ["First sentence", "Second sentence"], "model": "text-embedding-ada-002"}' \
84+
| grep -q '"embedding"'
85+
86+
docker restart "$TEST_ID"
87+
88+
sleep 15
89+
90+
# Wait for server to come back (up to 120 seconds)
91+
i=0
92+
while [ "$i" -lt 120 ]; do
93+
if docker exec "$TEST_ID" curl -sf http://127.0.0.1:8000/health >/dev/null 2>&1; then
94+
break
95+
fi
96+
sleep 1
97+
i=$((i + 1))
98+
done
99+
100+
docker ps | grep "$TEST_ID"
101+
docker logs "$TEST_ID"
102+
103+
# Verify server is healthy after restart
104+
docker exec "$TEST_ID" curl -sf http://127.0.0.1:8000/health
105+
106+
exit 0
107+
- name: Clear
108+
if: always()
109+
run: |
110+
docker rm -f "$TEST_ID" || true
111+
docker volume rm embed-test-data || true
112+
docker rmi embed-test || true

‎.github/workflows/main.yml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
name: build
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- '.github/workflows/main.yml'
8+
- '.github/workflows/buildx.yml'
9+
- '.github/workflows/shellcheck.yml'
10+
- '.github/workflows/embed_test.yml'
11+
- '.dockerignore'
12+
- 'Dockerfile'
13+
- 'manage.sh'
14+
- 'run.sh'
15+
- 'embed.env.example'
16+
17+
jobs:
18+
shellcheck:
19+
if: github.repository_owner == 'hwdsl2'
20+
uses: ./.github/workflows/shellcheck.yml
21+
22+
embed_test:
23+
if: github.repository_owner == 'hwdsl2'
24+
uses: ./.github/workflows/embed_test.yml
25+
26+
buildx:
27+
needs: [shellcheck, embed_test]
28+
uses: ./.github/workflows/buildx.yml
29+
secrets:
30+
CACHE_NAME: ${{ secrets.CACHE_NAME }}
31+
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
32+
QUAY_USER: ${{ secrets.QUAY_USER }}
33+
QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }}
34+
BUILD_ONLY: ${{ secrets.BUILD_ONLY }}

0 commit comments

Comments
 (0)