mirror of
https://github.com/zotero/zotero.git
synced 2026-08-28 05:25:31 +00:00
7f074b1924
Some checks are pending
Run the smb/nfs test shards and a new macOS NFS job only when the
changes touch the Gecko version or the network-filesystem database
handling, plus on manual and monthly scheduled runs, via a 'changes'
job feeding the test matrix. The macOS job uses the native NFS server
with a loopback mount and runs the real code path that crashed on
network data directories on macOS.
(cherry picked from commit ed8049dffc)
345 lines
12 KiB
YAML
345 lines
12 KiB
YAML
name: CI
|
|
on:
|
|
push:
|
|
pull_request:
|
|
workflow_dispatch:
|
|
schedule:
|
|
# Monthly, to catch runner image and OS changes
|
|
- cron: '0 6 1 * *'
|
|
concurrency:
|
|
group: ${{ github.ref }}
|
|
cancel-in-progress: true
|
|
jobs:
|
|
# Decide whether to run the network-filesystem tests: always for manual and scheduled
|
|
# runs, and otherwise only when the changes touch the Gecko version or the files
|
|
# responsible for network-filesystem database handling
|
|
changes:
|
|
name: Detect changes
|
|
runs-on: ubuntu-latest
|
|
if: >
|
|
github.event_name != 'pull_request'
|
|
|| github.event.pull_request.head.repo.full_name != github.repository
|
|
outputs:
|
|
network: ${{ steps.check.outputs.network }}
|
|
steps:
|
|
- name: Check for network-filesystem-related changes
|
|
id: check
|
|
env:
|
|
EVENT: ${{ github.event_name }}
|
|
REPO: ${{ github.repository }}
|
|
SHA: ${{ github.sha }}
|
|
BEFORE: ${{ github.event.before }}
|
|
BASE: ${{ github.event.pull_request.base.sha }}
|
|
run: |
|
|
network=false
|
|
if [[ "$EVENT" == "workflow_dispatch" || "$EVENT" == "schedule" ]]; then
|
|
network=true
|
|
else
|
|
base="$BASE"
|
|
[[ -z "$base" ]] && base="$BEFORE"
|
|
git init -q repo && cd repo
|
|
git remote add origin "https://github.com/$REPO"
|
|
git fetch -q --depth 1 origin "$SHA"
|
|
# Run the tests if the base of the change can't be determined (e.g., a new
|
|
# branch or an unfetchable pre-force-push commit)
|
|
if [[ -z "$base" || "$base" == 0000000000000000000000000000000000000000 ]] \
|
|
|| ! git fetch -q --depth 1 origin "$base" 2>/dev/null; then
|
|
network=true
|
|
elif git diff --name-only "$base" "$SHA" \
|
|
| grep -qE '^(app/config\.sh|chrome/content/zotero/xpcom/(db|file)\.js|\.github/workflows/ci\.yml)$'; then
|
|
network=true
|
|
fi
|
|
fi
|
|
echo "network=$network" >> "$GITHUB_OUTPUT"
|
|
|
|
test:
|
|
name: Test (${{ matrix.shard }})
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
needs: changes
|
|
# Same-repo PRs are covered by the push run, so run pull_request jobs only for forks
|
|
if: >
|
|
github.event_name != 'pull_request'
|
|
|| github.event.pull_request.head.repo.full_name != github.repository
|
|
strategy:
|
|
matrix:
|
|
# Numbered shards split the full suite. 'smb' and 'nfs' run the db and file tests
|
|
# with the test data directory on a loopback CIFS or NFS mount, so that connection
|
|
# initialization itself runs against a network filesystem, which is where
|
|
# network-filesystem failures (startup crashes and hangs) occur. The network shards
|
|
# run only when the 'changes' job detects relevant changes.
|
|
shard: ${{ fromJSON(needs.changes.outputs.network == 'true'
|
|
&& '["1", "2", "3", "4", "smb", "nfs"]'
|
|
|| '["1", "2", "3", "4"]') }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
submodules: recursive
|
|
lfs: true
|
|
|
|
- name: Install Node
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: 18
|
|
#cache: npm
|
|
|
|
# On GitHub
|
|
- name: Install xvfb
|
|
if: env.ACT != 'true'
|
|
run: sudo apt update && sudo apt install -y xvfb
|
|
|
|
# Local via act
|
|
- name: Install packages for act
|
|
if: env.ACT == 'true'
|
|
run: apt update && apt install -y zstd xvfb dbus-x11 libgtk-3-0 libx11-xcb1 libdbus-glib-1-2 libxt6
|
|
|
|
- name: Cache xulrunner
|
|
id: xulrunner-cache
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: app/xulrunner/firefox-x86_64
|
|
key: xulrunner-${{ hashFiles('app/config.sh', 'app/scripts/fetch_xulrunner') }}
|
|
|
|
- name: Fetch xulrunner
|
|
if: steps.xulrunner-cache.outputs.cache-hit != 'true'
|
|
run: app/scripts/fetch_xulrunner -p l
|
|
|
|
- name: Cache Node modules
|
|
id: node-cache
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: node_modules
|
|
key: node-modules-${{ hashFiles('package-lock.json') }}
|
|
|
|
- name: Install Node modules
|
|
if: steps.node-cache.outputs.cache-hit != 'true'
|
|
run: npm install
|
|
|
|
- name: Build Zotero
|
|
run: npm run build
|
|
# Currently necessary for document-worker Webpack: https://stackoverflow.com/a/69746937
|
|
env:
|
|
NODE_OPTIONS: --openssl-legacy-provider
|
|
|
|
# Create deployment ZIP from the build output, then replace build/ with the
|
|
# unzipped contents so that tests run against the same artifact that gets
|
|
# deployed. This catches problems that only manifest after a zip round-trip
|
|
# (e.g., resolved symlinks, missing files).
|
|
- name: Create deployment ZIP
|
|
run: |
|
|
cd build
|
|
zip -r ../build.zip *
|
|
cd ..
|
|
|
|
- name: Replace build with ZIP contents
|
|
run: |
|
|
rm -rf build
|
|
mkdir build
|
|
cd build
|
|
unzip ../build.zip
|
|
|
|
- name: Set up SMB share
|
|
if: matrix.shard == 'smb'
|
|
run: |
|
|
sudo apt install -y --no-install-recommends samba cifs-utils
|
|
sudo mkdir -p /srv/zotero-smb /mnt/zotero-smb
|
|
sudo chown $(whoami) /srv/zotero-smb
|
|
sudo tee -a /etc/samba/smb.conf > /dev/null <<EOF
|
|
[ztest]
|
|
path = /srv/zotero-smb
|
|
read only = no
|
|
EOF
|
|
(echo smbtestpass; echo smbtestpass) | sudo smbpasswd -s -a $(whoami)
|
|
sudo systemctl restart smbd
|
|
sudo mount -t cifs //127.0.0.1/ztest /mnt/zotero-smb \
|
|
-o user=$(whoami),pass=smbtestpass,vers=3.0,uid=$(id -u),gid=$(id -g)
|
|
|
|
- name: Set up NFS share
|
|
if: matrix.shard == 'nfs'
|
|
run: |
|
|
sudo apt install -y --no-install-recommends nfs-kernel-server
|
|
sudo mkdir -p /srv/zotero-nfs /mnt/zotero-nfs
|
|
sudo chown $(whoami) /srv/zotero-nfs
|
|
echo "/srv/zotero-nfs localhost(rw,no_subtree_check,insecure)" | sudo tee -a /etc/exports
|
|
sudo exportfs -ra
|
|
sudo mount -t nfs4 localhost:/srv/zotero-nfs /mnt/zotero-nfs
|
|
|
|
- name: Run tests
|
|
if: matrix.shard != 'smb' && matrix.shard != 'nfs'
|
|
run: xvfb-run test/runtests.sh -f -r 3 -p ${{ matrix.shard }}/4
|
|
|
|
# The step timeout turns a startup hang into a failure
|
|
- name: Run tests on network share
|
|
if: matrix.shard == 'smb' || matrix.shard == 'nfs'
|
|
timeout-minutes: 10
|
|
run: TMPDIR=/mnt/zotero-${{ matrix.shard }} xvfb-run test/runtests.sh -f -r 3 -b db file
|
|
|
|
# Run the db and file tests on macOS with the test data directory on a loopback NFS
|
|
# mount. On macOS, SQLite selects locking methods without shared-memory support on
|
|
# network filesystems, which is what the network-filesystem database handling has to
|
|
# detect and handle, so this runs the real code path that crashed on SMB/NFS/sshfs
|
|
# data directories.
|
|
test-mac:
|
|
name: Test (macOS NFS)
|
|
runs-on: macos-15
|
|
timeout-minutes: 30
|
|
needs: changes
|
|
if: >
|
|
needs.changes.outputs.network == 'true'
|
|
&& (github.event_name != 'pull_request'
|
|
|| github.event.pull_request.head.repo.full_name != github.repository)
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
submodules: recursive
|
|
lfs: true
|
|
|
|
- name: Install Node
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: 18
|
|
|
|
- name: Cache xulrunner
|
|
id: xulrunner-cache
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: app/xulrunner/Firefox.app
|
|
key: xulrunner-mac-${{ hashFiles('app/config.sh', 'app/scripts/fetch_xulrunner') }}
|
|
|
|
- name: Fetch xulrunner
|
|
if: steps.xulrunner-cache.outputs.cache-hit != 'true'
|
|
run: app/scripts/fetch_xulrunner -p m
|
|
|
|
- name: Cache Node modules
|
|
id: node-cache
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: node_modules
|
|
key: node-modules-mac-${{ hashFiles('package-lock.json') }}
|
|
|
|
- name: Install Node modules
|
|
if: steps.node-cache.outputs.cache-hit != 'true'
|
|
run: npm install
|
|
|
|
- name: Build Zotero
|
|
run: npm run build
|
|
# Currently necessary for document-worker Webpack: https://stackoverflow.com/a/69746937
|
|
env:
|
|
NODE_OPTIONS: --openssl-legacy-provider
|
|
|
|
- name: Set up NFS share
|
|
run: |
|
|
sudo mkdir -p /private/var/zotero-nfs
|
|
sudo chown $(whoami) /private/var/zotero-nfs
|
|
echo "/private/var/zotero-nfs -mapall=$(whoami) localhost" | sudo tee /etc/exports
|
|
sudo nfsd enable || sudo nfsd start
|
|
sleep 2
|
|
showmount -e localhost
|
|
mkdir -p "$HOME/zotero-nfs"
|
|
sudo mount_nfs -o vers=3 localhost:/private/var/zotero-nfs "$HOME/zotero-nfs"
|
|
|
|
# The step timeout turns a startup hang into a failure
|
|
- name: Run tests on network share
|
|
timeout-minutes: 10
|
|
run: TMPDIR="$HOME/zotero-nfs" test/runtests.sh -f -r 3 -b db file
|
|
|
|
utilities:
|
|
name: Utilities Tests
|
|
runs-on: ubuntu-latest
|
|
if: >
|
|
github.event_name != 'pull_request'
|
|
|| github.event.pull_request.head.repo.full_name != github.repository
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
submodules: recursive
|
|
|
|
- name: Install Node
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: 18
|
|
|
|
- name: Cache utilities Node modules
|
|
id: utilities-node-cache
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: chrome/content/zotero/xpcom/utilities/node_modules
|
|
key: utilities-node-modules-${{ hashFiles('chrome/content/zotero/xpcom/utilities/package-lock.json') }}
|
|
|
|
- name: Install utilities Node modules
|
|
if: steps.utilities-node-cache.outputs.cache-hit != 'true'
|
|
run: npm install --prefix chrome/content/zotero/xpcom/utilities
|
|
|
|
- name: Run utilities tests
|
|
run: |
|
|
npm test --prefix chrome/content/zotero/xpcom/utilities -- -j resource/schema/global/schema.json
|
|
|
|
# Build deployment ZIPs from main, version branches (e.g., 9.0, 10.0), and *-hotfix branches
|
|
deploy:
|
|
name: Build, Upload
|
|
runs-on: ubuntu-latest
|
|
if: github.event_name == 'push' && github.repository == 'zotero/zotero'
|
|
steps:
|
|
- name: Check if deployment branch
|
|
id: check-deploy
|
|
if: env.ACT != 'true'
|
|
run: |
|
|
branch="${GITHUB_REF#refs/heads/}"
|
|
if [[ "$branch" == "main" || "$branch" =~ ^[0-9]+\.[0-9]+$ || "$branch" == *-hotfix ]]; then
|
|
echo "deploy=true" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
- uses: actions/checkout@v4
|
|
if: steps.check-deploy.outputs.deploy == 'true'
|
|
with:
|
|
submodules: recursive
|
|
lfs: true
|
|
|
|
- name: Install Node
|
|
if: steps.check-deploy.outputs.deploy == 'true'
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: 18
|
|
|
|
- name: Cache Node modules
|
|
id: node-cache
|
|
if: steps.check-deploy.outputs.deploy == 'true'
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: node_modules
|
|
key: node-modules-${{ hashFiles('package-lock.json') }}
|
|
|
|
- name: Install Node modules
|
|
if: steps.check-deploy.outputs.deploy == 'true' && steps.node-cache.outputs.cache-hit != 'true'
|
|
run: npm install
|
|
|
|
- name: Build Zotero
|
|
if: steps.check-deploy.outputs.deploy == 'true'
|
|
run: npm run build
|
|
# Currently necessary for document-worker Webpack: https://stackoverflow.com/a/69746937
|
|
env:
|
|
NODE_OPTIONS: --openssl-legacy-provider
|
|
|
|
- name: Create deployment ZIP
|
|
if: steps.check-deploy.outputs.deploy == 'true'
|
|
run: |
|
|
cd build
|
|
zip -r ../build.zip *
|
|
cd ..
|
|
|
|
- uses: ruby/setup-ruby@v1
|
|
if: steps.check-deploy.outputs.deploy == 'true'
|
|
with:
|
|
ruby-version: '3.3'
|
|
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
|
|
|
- name: Upload deployment ZIP
|
|
if: steps.check-deploy.outputs.deploy == 'true'
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
run: |
|
|
mkdir build-zip
|
|
cp build.zip build-zip/$GITHUB_SHA.zip
|
|
gem install --no-document dpl -v '>= 2.0'
|
|
dpl s3 --bucket zotero-download --local_dir build-zip --upload_dir ci/client --acl public_read
|