zotero/.github/workflows/ci.yml

name: CI
on:
  push:
  pull_request:
  workflow_dispatch:
  schedule:
    # Monthly, to catch runner image and OS changes
    - cron: '0 6 1 * *'
concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true
jobs:
  # Decide whether to run the network-filesystem tests: always for manual and scheduled
  # runs, and otherwise only when the changes touch the Gecko version or the files
  # responsible for network-filesystem database handling
  changes:
    name: Detect changes
    runs-on: ubuntu-latest
    if: >
      github.event_name != 'pull_request'
      || github.event.pull_request.head.repo.full_name != github.repository
    outputs:
      network: ${{ steps.check.outputs.network }}
    steps:
      - name: Check for network-filesystem-related changes
        id: check
        env:
          EVENT: ${{ github.event_name }}
          REPO: ${{ github.repository }}
          SHA: ${{ github.sha }}
          BEFORE: ${{ github.event.before }}
          BASE: ${{ github.event.pull_request.base.sha }}
        run: |
          network=false
          if [[ "$EVENT" == "workflow_dispatch" || "$EVENT" == "schedule" ]]; then
            network=true
          else
            base="$BASE"
            [[ -z "$base" ]] && base="$BEFORE"
            git init -q repo && cd repo
            git remote add origin "https://github.com/$REPO"
            git fetch -q --depth 1 origin "$SHA"
            # Run the tests if the base of the change can't be determined (e.g., a new
            # branch or an unfetchable pre-force-push commit)
            if [[ -z "$base" || "$base" == 0000000000000000000000000000000000000000 ]] \
                || ! git fetch -q --depth 1 origin "$base" 2>/dev/null; then
              network=true
            elif git diff --name-only "$base" "$SHA" \
                | grep -qE '^(app/config\.sh|chrome/content/zotero/xpcom/(db|file)\.js|\.github/workflows/ci\.yml)$'; then
              network=true
            fi
          fi
          echo "network=$network" >> "$GITHUB_OUTPUT"

  test:
    name: Test (${{ matrix.shard }})
    runs-on: ubuntu-latest
    timeout-minutes: 30
    needs: changes
    # Same-repo PRs are covered by the push run, so run pull_request jobs only for forks
    if: >
      github.event_name != 'pull_request'
      || github.event.pull_request.head.repo.full_name != github.repository
    strategy:
      matrix:
        # Numbered shards split the full suite. 'smb' and 'nfs' run the db and file tests
        # with the test data directory on a loopback CIFS or NFS mount, so that connection
        # initialization itself runs against a network filesystem, which is where
        # network-filesystem failures (startup crashes and hangs) occur. The network shards
        # run only when the 'changes' job detects relevant changes.
        shard: ${{ fromJSON(needs.changes.outputs.network == 'true'
          && '["1", "2", "3", "4", "smb", "nfs"]'
          || '["1", "2", "3", "4"]') }}
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
          lfs: true

      - name: Install Node
        uses: actions/setup-node@v4
        with:
          node-version: 18
          #cache: npm

      # On GitHub -- xvfb is preinstalled on the runner image, so this is only a fallback,
      # since the apt mirrors regularly hang
      - name: Install xvfb
        if: env.ACT != 'true'
        run: command -v xvfb-run > /dev/null || (sudo apt update && sudo apt install -y xvfb)

      # Local via act
      - name: Install packages for act
        if: env.ACT == 'true'
        run: apt update && apt install -y zstd xvfb dbus-x11 libgtk-3-0 libx11-xcb1 libdbus-glib-1-2 libxt6

      - name: Cache xulrunner
        id: xulrunner-cache
        uses: actions/cache@v4
        with:
          path: app/xulrunner/firefox-x86_64
          key: xulrunner-${{ hashFiles('app/config.sh', 'app/scripts/fetch_xulrunner') }}

      - name: Fetch xulrunner
        if: steps.xulrunner-cache.outputs.cache-hit != 'true'
        run: app/scripts/fetch_xulrunner -p l

      - name: Cache Node modules
        id: node-cache
        uses: actions/cache@v4
        with:
          path: node_modules
          key: node-modules-${{ hashFiles('package-lock.json') }}

      - name: Install Node modules
        if: steps.node-cache.outputs.cache-hit != 'true'
        run: npm install

      - name: Build Zotero
        run: npm run build
        # Currently necessary for document-worker Webpack: https://stackoverflow.com/a/69746937
        env:
          NODE_OPTIONS: --openssl-legacy-provider

      # Create deployment ZIP from the build output, then replace build/ with the
      # unzipped contents so that tests run against the same artifact that gets
      # deployed. This catches problems that only manifest after a zip round-trip
      # (e.g., resolved symlinks, missing files).
      - name: Create deployment ZIP
        run: |
          cd build
          zip -r ../build.zip *
          cd ..

      - name: Replace build with ZIP contents
        run: |
          rm -rf build
          mkdir build
          cd build
          unzip ../build.zip

      - name: Set up SMB share
        if: matrix.shard == 'smb'
        timeout-minutes: 10
        run: |
          sudo apt install -y --no-install-recommends samba cifs-utils
          sudo mkdir -p /srv/zotero-smb /mnt/zotero-smb
          sudo chown $(whoami) /srv/zotero-smb
          sudo tee -a /etc/samba/smb.conf > /dev/null <<EOF
          [ztest]
          path = /srv/zotero-smb
          read only = no
          EOF
          (echo smbtestpass; echo smbtestpass) | sudo smbpasswd -s -a $(whoami)
          sudo systemctl restart smbd
          sudo mount -t cifs //127.0.0.1/ztest /mnt/zotero-smb \
            -o user=$(whoami),pass=smbtestpass,vers=3.0,uid=$(id -u),gid=$(id -g)

      - name: Set up NFS share
        if: matrix.shard == 'nfs'
        timeout-minutes: 10
        run: |
          sudo apt install -y --no-install-recommends nfs-kernel-server
          sudo mkdir -p /srv/zotero-nfs /mnt/zotero-nfs
          sudo chown $(whoami) /srv/zotero-nfs
          echo "/srv/zotero-nfs localhost(rw,no_subtree_check,insecure)" | sudo tee -a /etc/exports
          sudo exportfs -ra
          sudo mount -t nfs4 localhost:/srv/zotero-nfs /mnt/zotero-nfs

      - name: Run tests
        if: matrix.shard != 'smb' && matrix.shard != 'nfs'
        run: xvfb-run test/runtests.sh -f -r 3 -p ${{ matrix.shard }}/4

      # The step timeout turns a startup hang into a failure
      - name: Run tests on network share
        if: matrix.shard == 'smb' || matrix.shard == 'nfs'
        timeout-minutes: 10
        run: TMPDIR=/mnt/zotero-${{ matrix.shard }} xvfb-run test/runtests.sh -f -r 3 -b db file

  # Run the db and file tests on macOS with the test data directory on a loopback NFS
  # mount. On macOS, SQLite selects locking methods without shared-memory support on
  # network filesystems, which is what the network-filesystem database handling has to
  # detect and handle, so this runs the real code path that crashed on SMB/NFS/sshfs
  # data directories.
  test-mac:
    name: Test (macOS NFS)
    runs-on: macos-15
    timeout-minutes: 30
    needs: changes
    if: >
      needs.changes.outputs.network == 'true'
      && (github.event_name != 'pull_request'
      || github.event.pull_request.head.repo.full_name != github.repository)
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
          lfs: true

      - name: Install Node
        uses: actions/setup-node@v4
        with:
          node-version: 18

      - name: Cache xulrunner
        id: xulrunner-cache
        uses: actions/cache@v4
        with:
          path: app/xulrunner/Firefox.app
          key: xulrunner-mac-${{ hashFiles('app/config.sh', 'app/scripts/fetch_xulrunner') }}

      - name: Fetch xulrunner
        if: steps.xulrunner-cache.outputs.cache-hit != 'true'
        run: app/scripts/fetch_xulrunner -p m

      - name: Cache Node modules
        id: node-cache
        uses: actions/cache@v4
        with:
          path: node_modules
          key: node-modules-mac-${{ hashFiles('package-lock.json') }}

      - name: Install Node modules
        if: steps.node-cache.outputs.cache-hit != 'true'
        run: npm install

      - name: Build Zotero
        run: npm run build
        # Currently necessary for document-worker Webpack: https://stackoverflow.com/a/69746937
        env:
          NODE_OPTIONS: --openssl-legacy-provider

      - name: Set up NFS share
        run: |
          sudo mkdir -p /private/var/zotero-nfs
          sudo chown $(whoami) /private/var/zotero-nfs
          echo "/private/var/zotero-nfs -mapall=$(whoami) localhost" | sudo tee /etc/exports
          sudo nfsd enable || sudo nfsd start
          sleep 2
          showmount -e localhost
          mkdir -p "$HOME/zotero-nfs"
          sudo mount_nfs -o vers=3 localhost:/private/var/zotero-nfs "$HOME/zotero-nfs"

      # The step timeout turns a startup hang into a failure
      - name: Run tests on network share
        timeout-minutes: 10
        run: TMPDIR="$HOME/zotero-nfs" test/runtests.sh -f -r 3 -b db file

  # Boot the Windows build -- which uses custom-built Firefox components (xul.dll) rather
  # than stock Mozilla binaries -- and run the db and file tests. A Gecko bump commit
  # updates the custom-component hashes in config.sh along with the version, so this
  # tests each new set of custom components.
  test-win:
    name: Test (Windows ${{ matrix.arch }})
    runs-on: ${{ matrix.os }}
    timeout-minutes: 40
    needs: changes
    if: >
      needs.changes.outputs.network == 'true'
      && (github.event_name != 'pull_request'
      || github.event.pull_request.head.repo.full_name != github.repository)
    strategy:
      fail-fast: false
      matrix:
        include:
          - arch: x64
            os: windows-latest
          - arch: arm64
            os: windows-11-arm
    defaults:
      run:
        shell: bash
    steps:
      # The repository contains symlinks, which Git checks out as plain files on Windows
      # by default, breaking the build
      - name: Enable symlinks
        run: git config --global core.symlinks true

      - uses: actions/checkout@v4
        with:
          submodules: recursive
          lfs: true

      - name: Install Node
        uses: actions/setup-node@v4
        with:
          node-version: 18
          # Node 18 has no Windows arm64 build, so use the x64 build, which runs emulated
          architecture: x64

      # Git Bash has unzip but not zip, which fetch_xulrunner needs to repack omni.ja
      - name: Install zip
        run: choco install zip --no-progress -y

      - name: Cache xulrunner
        id: xulrunner-cache
        uses: actions/cache@v4
        with:
          path: app/xulrunner/firefox-win-${{ matrix.arch }}
          key: xulrunner-win-${{ matrix.arch }}-${{ hashFiles('app/config.sh', 'app/scripts/fetch_xulrunner') }}

      - name: Fetch xulrunner
        if: steps.xulrunner-cache.outputs.cache-hit != 'true'
        run: app/scripts/fetch_xulrunner -p w -a ${{ matrix.arch }}

      - name: Cache Node modules
        id: node-cache
        uses: actions/cache@v4
        with:
          path: node_modules
          key: node-modules-win-${{ hashFiles('package-lock.json') }}

      - name: Install Node modules
        if: steps.node-cache.outputs.cache-hit != 'true'
        run: npm install

      # Committed symlinks point into node_modules, which doesn't exist at checkout time,
      # so Git creates the directory ones with the wrong symlink type on Windows. Recreate
      # them now that the targets exist.
      - name: Recreate symlinks
        run: |
          git ls-files -s | awk '$1 == 120000 { print $4 }' | xargs rm
          git checkout -- .

      - name: Build Zotero
        run: npm run build
        # Currently necessary for document-worker Webpack: https://stackoverflow.com/a/69746937
        env:
          NODE_OPTIONS: --openssl-legacy-provider

      # The step timeout turns a startup hang into a failure
      - name: Run tests
        timeout-minutes: 15
        run: test/runtests.sh -f -r 3 -b -x app/staging/Zotero_win-${{ matrix.arch }}/zotero.exe db file

  utilities:
    name: Utilities Tests
    runs-on: ubuntu-latest
    if: >
      github.event_name != 'pull_request'
      || github.event.pull_request.head.repo.full_name != github.repository
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Install Node
        uses: actions/setup-node@v4
        with:
          node-version: 18

      - name: Cache utilities Node modules
        id: utilities-node-cache
        uses: actions/cache@v4
        with:
          path: chrome/content/zotero/xpcom/utilities/node_modules
          key: utilities-node-modules-${{ hashFiles('chrome/content/zotero/xpcom/utilities/package-lock.json') }}

      - name: Install utilities Node modules
        if: steps.utilities-node-cache.outputs.cache-hit != 'true'
        run: npm install --prefix chrome/content/zotero/xpcom/utilities

      - name: Run utilities tests
        run: |
          npm test --prefix chrome/content/zotero/xpcom/utilities -- -j resource/schema/global/schema.json

  # Build deployment ZIPs from main, version branches (e.g., 9.0, 10.0), and *-hotfix branches
  deploy:
    name: Build, Upload
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && github.repository == 'zotero/zotero'
    steps:
      - name: Check if deployment branch
        id: check-deploy
        if: env.ACT != 'true'
        run: |
          branch="${GITHUB_REF#refs/heads/}"
          if [[ "$branch" == "main" || "$branch" =~ ^[0-9]+\.[0-9]+$ || "$branch" == *-hotfix ]]; then
            echo "deploy=true" >> "$GITHUB_OUTPUT"
          fi

      - uses: actions/checkout@v4
        if: steps.check-deploy.outputs.deploy == 'true'
        with:
          submodules: recursive
          lfs: true

      - name: Install Node
        if: steps.check-deploy.outputs.deploy == 'true'
        uses: actions/setup-node@v4
        with:
          node-version: 18

      - name: Cache Node modules
        id: node-cache
        if: steps.check-deploy.outputs.deploy == 'true'
        uses: actions/cache@v4
        with:
          path: node_modules
          key: node-modules-${{ hashFiles('package-lock.json') }}

      - name: Install Node modules
        if: steps.check-deploy.outputs.deploy == 'true' && steps.node-cache.outputs.cache-hit != 'true'
        run: npm install

      - name: Build Zotero
        if: steps.check-deploy.outputs.deploy == 'true'
        run: npm run build
        # Currently necessary for document-worker Webpack: https://stackoverflow.com/a/69746937
        env:
          NODE_OPTIONS: --openssl-legacy-provider

      - name: Create deployment ZIP
        if: steps.check-deploy.outputs.deploy == 'true'
        run: |
          cd build
          zip -r ../build.zip *
          cd ..

      - uses: ruby/setup-ruby@v1
        if: steps.check-deploy.outputs.deploy == 'true'
        with:
          ruby-version: '3.3'
          bundler-cache: true # runs 'bundle install' and caches installed gems automatically

      - name: Upload deployment ZIP
        if: steps.check-deploy.outputs.deploy == 'true'
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: |
          mkdir build-zip
          cp build.zip build-zip/$GITHUB_SHA.zip
          gem install --no-document dpl -v '>= 2.0'
          dpl s3 --bucket zotero-download --local_dir build-zip --upload_dir ci/client --acl public_read