Update and speed up GitHub workflows (#1368)

* Update actions and switch to GitHub runners

* Bump workflow Python version to 3.14

* Reuse nitter build for integration test

* Add missing libpcre3 installation to workflow

* Consolidate workflow runtime deps install

* Make nitter binary executable

* Run nimble md and scss simultaneously in workflow

* Run tests with 4 workers in workflow

* Rerun failing integration tests

* Bump integration test workers to 5

* Improve python dep install and run less workers

* Use native GitHub Actions Redis service

* Lower integration test workers to 2

* Switch to poetry to cache venv

* Ensure poetry is installed before setup-python

* Fix poetry sync command

* Switch back to 3 workers

* Cache poetry install

* WIP

* WIP

* Fix poetry/pipx caching

* Speed up integration test significantly

* WIP

* Cleanup

.github/workflows/build-docker.yml

@@ -11,20 +11,19 @@ jobs:
   tests:
     uses: ./.github/workflows/run-tests.yml
     secrets: inherit
+
   build-docker-amd64:
     needs: [tests]
-    runs-on: buildjet-2vcpu-ubuntu-2204
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           version: latest
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -36,20 +35,19 @@ jobs:
           platforms: linux/amd64
           push: true
           tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
+
   build-docker-arm64:
     needs: [tests]
-    runs-on: buildjet-2vcpu-ubuntu-2204-arm
+    runs-on: ubuntu-24.04-arm
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           version: latest
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/run-tests.yml

@@ -20,19 +20,17 @@ defaults:
 jobs:
   build-test:
     name: Build and test
-    runs-on: buildjet-2vcpu-ubuntu-2204
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         nim: ["2.0.x", "2.2.x", "devel"]
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
 
       - name: Cache Nimble Dependencies
         id: cache-nimble
-        uses: buildjet/cache@v4
+        uses: actions/cache@v5
         with:
           path: ~/.nimble
           key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }}
@@ -47,62 +45,100 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build Project
-        run: nimble build -d:release -Y
+        run: nimble build -Y
+
+      - name: Upload 2.2.x build artifact
+        if: matrix.nim == '2.2.x'
+        uses: actions/upload-artifact@v6
+        with:
+          name: nitter-linux-nim-2.2.x-${{ github.sha }}
+          path: |
+            ./nitter
+          if-no-files-found: error
 
   integration-test:
     needs: [build-test]
     name: Integration test
-    runs-on: buildjet-2vcpu-ubuntu-2204
+    runs-on: ubuntu-24.04
+
+    services:
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+
     steps:
+      - name: Install runtime deps
+        run: |
+          sudo apt-get install -y --no-install-recommends libsass-dev libpcre3
+
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
+
+      - name: Cache pipx (poetry)
+        uses: actions/cache@v5
         with:
-          fetch-depth: 0
+          path: |
+            ~/.local/pipx
+            ~/.local/bin
+          key: pipx-poetry-${{ runner.os }}
+
+      - name: Install poetry
+        env:
+          PIPX_HOME: ~/.local/pipx
+          PIPX_BIN_DIR: ~/.local/bin
+        run: command -v poetry >/dev/null 2>&1 || pipx install poetry
+
+      - name: Setup Python (3.14) with Poetry cache
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+          cache: poetry
+          cache-dependency-path: tests/poetry.lock
+
+      - name: Install Python deps
+        working-directory: tests
+        run: poetry sync
 
       - name: Cache Nimble Dependencies
-        id: cache-nimble
+        uses: actions/cache@v5
-        uses: buildjet/cache@v4
         with:
           path: ~/.nimble
-          key: devel-nimble-v2-${{ hashFiles('*.nimble') }}
+          key: 2.2.x-nimble-v2-${{ hashFiles('*.nimble') }}
           restore-keys: |
-            devel-nimble-v2-
+            2.2.x-nimble-v2-
-
-      - name: Setup Python (3.10) with pip cache
-        uses: buildjet/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: pip
 
       - name: Setup Nim
         uses: jiro4989/setup-nim-action@v2
         with:
-          nim-version: devel
+          nim-version: 2.2.x
           use-nightlies: true
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build Project
+      - name: Download 2.2.x build artifact
-        run: nimble build -d:release -Y
+        uses: actions/download-artifact@v4
+        with:
+          name: nitter-linux-nim-2.2.x-${{ github.sha }}
+          path: .
 
-      - name: Install SeleniumBase and Chromedriver
+      - name: Make nitter binary executable
-        run: |
+        run: chmod +x ./nitter
-          pip install seleniumbase
-          seleniumbase install chromedriver
-
-      - name: Start Redis Service
-        uses: supercharge/redis-github-action@1.5.0
 
       - name: Prepare Nitter Environment
         run: |
-          sudo apt-get update && sudo apt-get install -y libsass-dev
           cp nitter.example.conf nitter.conf
           sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf
-          nimble md
+
-          nimble scss
+          # Run both Nimble tasks concurrently
+          nim r tools/rendermd.nim &
+          nim r tools/gencss.nim &
+          wait
+
           echo '${{ secrets.SESSIONS }}' | head -n1
           echo '${{ secrets.SESSIONS }}' > ./sessions.jsonl
 
       - name: Run Tests
         run: |
           ./nitter &
-          pytest -n1 tests
+          cd tests
+          poetry run pytest -n3 --reruns=3 --rs .

nitter.nimble

@@ -28,7 +28,7 @@ requires "oauth#b8c163b"
 # Tasks
 
 task scss, "Generate css":
-  exec "nimble c --hint[Processing]:off -d:danger -r tools/gencss"
+  exec "nim r --hint[Processing]:off tools/gencss"
 
 task md, "Render md":
-  exec "nimble c --hint[Processing]:off -d:danger -r tools/rendermd"
+  exec "nim r --hint[Processing]:off tools/rendermd"

tests/poetry.toml

@@ -0,0 +1,2 @@
+[virtualenvs]
+in-project = true

tests/pyproject.toml

@@ -0,0 +1,8 @@
+[tool.poetry]
+name = "nitter-tests"
+version = "0.0.0"
+package-mode = false
+
+[tool.poetry.dependencies]
+python = "^3.14"
+seleniumbase = "4.46.5"

tests/requirements.txt

@@ -1 +1 @@
-seleniumbase
+seleniumbase==4.46.5