daily-paper-reader #15
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: daily-paper-reader | |
| on: | |
| schedule: | |
| - cron: "30 18 * * *" # 北京时间次日 02:30（UTC 18:30） | |
| workflow_dispatch: | |
| inputs: | |
| run_enrich: | |
| description: "Run 0.enrich_config_queries.py（默认关闭，按需开启）" | |
| required: false | |
| default: "false" | |
| fetch_days: | |
| description: "回溯抓取天数（例如 30）；留空则使用默认配置" | |
| required: false | |
| default: "10" | |
| fetch_mode: | |
| description: "抓取模式：auto/standard/skims（auto 按 fetch-days 阈值决定，standard/ skims 强制）" | |
| required: false | |
| default: "auto" | |
| permissions: | |
| contents: write | |
| concurrency: | |
| group: daily-paper-reader | |
| cancel-in-progress: false | |
| jobs: | |
| run: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| env: | |
| BLT_API_KEY: ${{ secrets.BLT_API_KEY }} | |
| BLT_REWRITE_MODEL: gemini-3-flash-preview | |
| BLT_RERANK_MODEL: qwen3-reranker-4b | |
| BLT_FILTER_MODEL: gemini-3-flash-preview-nothinking | |
| PYTHONUNBUFFERED: "1" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Cache pip + torch | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/torch | |
| key: ${{ runner.os }}-dpr-hf-v2-${{ hashFiles('requirements.txt') }} | |
| - name: Install deps (skip sqlite3) | |
| run: | | |
| python - <<'PY' | |
| import re | |
| lines = open("requirements.txt", "r", encoding="utf-8").read().splitlines() | |
| lines = [l for l in lines if l.strip() and not re.match(r"^sqlite3\\b", l)] | |
| open("/tmp/req.txt", "w", encoding="utf-8").write("\n".join(lines)) | |
| PY | |
| python -m pip install --upgrade pip | |
| python -m pip install uv | |
| uv pip install --system -r /tmp/req.txt | |
| - name: Run Retrieval and Embedding | |
| env: | |
| HF_HOME: ${{ github.workspace }}/hf_cache | |
| HUGGINGFACE_HUB_CACHE: ${{ github.workspace }}/hf_cache | |
| HF_HUB_DISABLE_SYMLINKS: "1" | |
| run: | | |
| echo "Cleaning any previous local cache..." | |
| rm -rf ${{ github.workspace }}/hf_cache | |
| echo "=== Debug: runtime env ===" | |
| echo "PWD=$(pwd)" | |
| echo "HOME=$HOME" | |
| echo "GITHUB_WORKSPACE=$GITHUB_WORKSPACE" | |
| echo "HF_HOME=$HF_HOME" | |
| echo "HUGGINGFACE_HUB_CACHE=$HUGGINGFACE_HUB_CACHE" | |
| echo "HF_HUB_DISABLE_SYMLINKS=$HF_HUB_DISABLE_SYMLINKS" | |
| echo "TRANSFORMERS_CACHE=${TRANSFORMERS_CACHE:-<unset>}" | |
| echo "XDG_CACHE_HOME=${XDG_CACHE_HOME:-<unset>}" | |
| echo "--- env (filtered) ---" | |
| env | grep -E '^(HF_|HUGGINGFACE_|TRANSFORMERS_|XDG_|HOME=|GITHUB_WORKSPACE=)' | sort || true | |
| echo "--- python/hf versions ---" | |
| /opt/hostedtoolcache/Python/3.11.14/x64/bin/python - <<'PY' | |
| import os | |
| def v(mod): | |
| try: | |
| m = __import__(mod) | |
| return getattr(m, "__version__", "<no __version__>") | |
| except Exception as e: | |
| return f"<import failed: {e}>" | |
| print("python_exe:", os.environ.get("PYTHON", "")) | |
| print("HF_HOME env:", os.environ.get("HF_HOME")) | |
| print("HUGGINGFACE_HUB_CACHE env:", os.environ.get("HUGGINGFACE_HUB_CACHE")) | |
| print("HF_HUB_DISABLE_SYMLINKS env:", os.environ.get("HF_HUB_DISABLE_SYMLINKS")) | |
| print("TRANSFORMERS_CACHE env:", os.environ.get("TRANSFORMERS_CACHE")) | |
| print("versions:", { | |
| "huggingface_hub": v("huggingface_hub"), | |
| "transformers": v("transformers"), | |
| "sentence_transformers": v("sentence_transformers"), | |
| }) | |
| try: | |
| from huggingface_hub import constants as c | |
| print("hub.constants:", { | |
| "HF_HOME": getattr(c, "HF_HOME", None), | |
| "HUGGINGFACE_HUB_CACHE": getattr(c, "HUGGINGFACE_HUB_CACHE", None), | |
| "HF_HUB_DISABLE_SYMLINKS": getattr(c, "HF_HUB_DISABLE_SYMLINKS", None), | |
| }) | |
| except Exception as e: | |
| print("hub.constants import failed:", e) | |
| PY | |
| echo "--- cache dir check ---" | |
| echo "[legacy] ~/.cache/huggingface:" | |
| ls -la ~/.cache/huggingface || true | |
| echo "[current] $HF_HOME:" | |
| ls -la "$HF_HOME" || true | |
| echo "Running main script..." | |
| RUN_ENRICH="${{ github.event.inputs.run_enrich }}" | |
| FETCH_DAYS="${{ github.event.inputs.fetch_days }}" | |
| FETCH_MODE="${{ github.event.inputs.fetch_mode }}" | |
| ARGS="" | |
| if [ "$RUN_ENRICH" = "true" ]; then | |
| ARGS="--run-enrich" | |
| fi | |
| if [ -n "$FETCH_DAYS" ]; then | |
| ARGS="$ARGS --fetch-days $FETCH_DAYS" | |
| fi | |
| if [ -n "$FETCH_MODE" ]; then | |
| ARGS="$ARGS --fetch-mode $FETCH_MODE" | |
| fi | |
| /opt/hostedtoolcache/Python/3.11.14/x64/bin/python src/main.py $ARGS --embedding-device cpu --embedding-batch-size 8 | |
| # 让前端可以在不配置 GitHub Token 的情况下直接读取配置（只读快照） | |
| cp -f config.yaml docs/config.yaml | |
| echo "--- post-run cache symlink check ---" | |
| find "$HF_HOME" -xtype l | head -n 50 || true | |
| - name: Cleanup non-recommend archives | |
| run: | | |
| find archive -mindepth 2 -maxdepth 2 -type d \( -name raw -o -name filtered -o -name rank \) -exec rm -rf {} + | |
| - name: Commit results | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| branch="${GITHUB_REF_NAME:-main}" | |
| shopt -s nullglob | |
| paths=(docs config.yaml) | |
| if [ -f archive/arxiv_seen.json ]; then | |
| paths+=(archive/arxiv_seen.json) | |
| fi | |
| if [ -f archive/crawl_state.json ]; then | |
| paths+=(archive/crawl_state.json) | |
| fi | |
| if [ -f archive/carryover.json ]; then | |
| paths+=(archive/carryover.json) | |
| fi | |
| for d in archive/*/recommend; do | |
| paths+=("$d") | |
| done | |
| git add "${paths[@]}" | |
| if git diff --cached --quiet; then | |
| echo "No changes to commit." | |
| exit 0 | |
| fi | |
| git commit -m "[chore] daily pipeline" | |
| git fetch origin "$branch" | |
| git rebase -X theirs "origin/$branch" | |
| git push origin HEAD:"$branch" |