Make the daily scraper survive refused connections from govjobs.publi…

.github/workflows/deploy-job-php.yml

@@ -9,7 +9,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v7
 
     - name: Deploy via SFTP
       uses: wlixcc/SFTP-Deploy-Action@v1.2.4

.github/workflows/deploy-to-db.yml

@@ -20,10 +20,10 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v7
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v7
         with:
           python-version: '3.10'
           cache: 'pip'

.github/workflows/python-package-conda.yml

@@ -2,27 +2,42 @@ name: Job Scraping and Firebase Processing Pipeline
 
 on:
   schedule:
-    - cron: '0 0 * * *'  # Run daily at midnight UTC
+    # Not on the full hour: GitHub delays or drops scheduled runs at :00 when it is busy.
+    # The second run catches up if the first one failed or was dropped.
+    - cron: '17 5 * * *'
+    - cron: '47 15 * * *'
   workflow_dispatch:  # Allow manual triggering
 
+permissions:
+  contents: write
+
+# Never run two scrapers at once (e.g. a manual run while the scheduled one is running)
+concurrency:
+  group: job-scraper
+  cancel-in-progress: false
+
 jobs:
   scrape-and-process:
     runs-on: ubuntu-latest
+    timeout-minutes: 45
+    env:
+      PYTHONUNBUFFERED: '1'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v7
         with:
-          fetch-depth: 0
-          persist-credentials: true
+          # Latest commit of the branch, not the trigger commit (matters when a run was queued)
+          ref: ${{ github.ref_name }}
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v7
         with:
-          python-version: '3.x'
+          python-version: '3.14'
+          cache: pip
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install requests beautifulsoup4 firebase-admin
+          pip install -r requirements.txt
 
       - name: Run job scraper
         run: python job_scraper.py
@@ -35,21 +50,32 @@ jobs:
       #    FIREBASE_SERVICE_ACCOUNT: ${{ secrets.FIREBASE_SERVICE_ACCOUNT }}
       #  run: python firebase_importer.py
 
+      # Notify before committing: if Telegram fails, nothing is committed and the next run
+      # sends these jobs again. Only on main: test runs on other branches neither notify nor commit.
       - name: Send Telegram Notification
+        if: github.ref == 'refs/heads/main'
         env:
           TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
           TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }}
         run: python telegram.py
 
       - name: Commit and push if changes
+        if: github.ref == 'refs/heads/main'
         run: |
           git config --local user.email "action@github.com"
           git config --local user.name "GitHub Action"
           git add jobs_all_processed.json new_jobs.json
-          git diff --quiet && git diff --staged --quiet || (git commit -m "Update job data" && git push)
-
+          git diff --staged --quiet && exit 0
+          git commit -m "Update job data"
+          for attempt in 1 2 3; do
+            git push && exit 0
+            sleep 10
+            git pull --rebase origin main || { git rebase --abort; exit 1; }
+          done
+          exit 1
+
             # Trigger the database import workflow
-            
+
       #- name: Trigger DB Import Workflow
       #  if: success()
       #  uses: peter-evans/repository-dispatch@v2