ScoutBot — Scrape (Weekly, Sundays) #128
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ScoutBot — Scrape (Weekly, Sundays) | |
| # Scrapes all sources and removes expired entries once per week on Sunday at 07:00 WAT. | |
| # Does NOT send the email digest — that runs separately (also Sundays, digest.yml). | |
| # | |
| # The spider refuses to add any opportunity whose source post is older than | |
| # 3 days (MAX_POST_AGE_DAYS), ensuring only fresh listings enter the sheet. | |
| # cleanup.py then removes entries that have: | |
| # • a deadline that has already passed, OR | |
| # • been in the sheet for more than 33 days (hard cap, no exceptions) | |
| # Together these ensure every entry in the sheet is open and current. | |
| on: | |
| schedule: | |
| - cron: '0 6 * * 0' # 07:00 WAT every Sunday (UTC+1 → 06:00 UTC) | |
| workflow_dispatch: | |
| jobs: | |
| scrape: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt | |
| - name: Restore Google service account JSON | |
| env: | |
| SA_JSON: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON }} | |
| SA_JSON_B64: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON_B64 }} | |
| run: | | |
| if [ -n "$SA_JSON_B64" ]; then | |
| echo "$SA_JSON_B64" | base64 -d > service_account.json | |
| echo "Restored from base64-encoded secret." | |
| elif [ -n "$SA_JSON" ]; then | |
| printf '%s' "$SA_JSON" > service_account.json | |
| echo "Restored from raw JSON secret." | |
| else | |
| echo "ERROR: Neither GOOGLE_SERVICE_ACCOUNT_JSON nor GOOGLE_SERVICE_ACCOUNT_JSON_B64 is set in repo secrets!" >&2 | |
| exit 1 | |
| fi | |
| python -c "import json; json.load(open('service_account.json')); print('service_account.json is valid.')" | |
| - name: Build .env file from secrets | |
| run: | | |
| cat > .env <<EOF | |
| SENDER_EMAIL=${{ secrets.SENDER_EMAIL }} | |
| GMAIL_APP_PASSWORD=${{ secrets.GMAIL_APP_PASSWORD }} | |
| SPREADSHEET_ID=${{ secrets.SPREADSHEET_ID }} | |
| FORM_SHEET_ID=1dFcnVvQjWkuYhN1rplICTY0j88KgvGqQ3FzYId2ru4s | |
| GOOGLE_SERVICE_ACCOUNT_JSON=service_account.json | |
| RECIPIENT_EMAILS=${{ secrets.RECIPIENT_EMAILS }} | |
| GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }} | |
| EOF | |
| - name: Scrape new opportunities | |
| run: python run.py --scrape | |
| - name: Print Scrapy stats | |
| if: always() | |
| run: | | |
| echo "=== scrapy.log (last 120 lines) ===" | |
| tail -n 120 scrapy.log 2>/dev/null || echo "(no scrapy.log)" | |
| - name: Clean expired entries (23-day cap) | |
| run: python run.py --cleanup | |
| - name: Upload logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: scrape-logs-${{ github.run_number }} | |
| path: | | |
| scoutbot.log | |
| scrapy.log | |
| if-no-files-found: ignore | |
| retention-days: 3 |