diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1c5ee240c..11b72ccc3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,62 +1,121 @@ name: CI -on: [push, pull_request] +on: + push: + pull_request: + workflow_dispatch: + schedule: + # Monthly, to catch runner image and OS changes + - cron: '0 6 1 * *' concurrency: group: ${{ github.ref }} cancel-in-progress: true jobs: - build: - name: Build, Upload, Test + # Decide whether to run the network-filesystem tests: always for manual and scheduled + # runs, and otherwise only when the changes touch the Gecko version or the files + # responsible for network-filesystem database handling + changes: + name: Detect changes runs-on: ubuntu-latest + if: > + github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name != github.repository + outputs: + network: ${{ steps.check.outputs.network }} + steps: + - name: Check for network-filesystem-related changes + id: check + env: + EVENT: ${{ github.event_name }} + REPO: ${{ github.repository }} + SHA: ${{ github.sha }} + BEFORE: ${{ github.event.before }} + BASE: ${{ github.event.pull_request.base.sha }} + run: | + network=false + if [[ "$EVENT" == "workflow_dispatch" || "$EVENT" == "schedule" ]]; then + network=true + else + base="$BASE" + [[ -z "$base" ]] && base="$BEFORE" + git init -q repo && cd repo + git remote add origin "https://github.com/$REPO" + git fetch -q --depth 1 origin "$SHA" + # Run the tests if the base of the change can't be determined (e.g., a new + # branch or an unfetchable pre-force-push commit) + if [[ -z "$base" || "$base" == 0000000000000000000000000000000000000000 ]] \ + || ! git fetch -q --depth 1 origin "$base" 2>/dev/null; then + network=true + elif git diff --name-only "$base" "$SHA" \ + | grep -qE '^(app/config\.sh|chrome/content/zotero/xpcom/(db|file)\.js|\.github/workflows/ci\.yml)$'; then + network=true + fi + fi + echo "network=$network" >> "$GITHUB_OUTPUT" + + test: + name: Test (${{ matrix.shard }}) + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: changes + # Same-repo PRs are covered by the push run, so run pull_request jobs only for forks + if: > + github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name != github.repository + strategy: + matrix: + # Numbered shards split the full suite. 'smb' and 'nfs' run the db and file tests + # with the test data directory on a loopback CIFS or NFS mount, so that connection + # initialization itself runs against a network filesystem, which is where + # network-filesystem failures (startup crashes and hangs) occur. The network shards + # run only when the 'changes' job detects relevant changes. + shard: ${{ fromJSON(needs.changes.outputs.network == 'true' + && '["1", "2", "3", "4", "smb", "nfs"]' + || '["1", "2", "3", "4"]') }} steps: - uses: actions/checkout@v4 with: submodules: recursive lfs: true - + - name: Install Node uses: actions/setup-node@v4 with: - node-version: 18 + node-version: 24 #cache: npm - - # On GitHub - - name: Install xvfb - if: env.ACT != 'true' - run: sudo apt update && sudo apt install -y xvfb - + # Local via act - name: Install packages for act if: env.ACT == 'true' run: apt update && apt install -y zstd xvfb dbus-x11 libgtk-3-0 libx11-xcb1 libdbus-glib-1-2 libxt6 - + - name: Cache xulrunner id: xulrunner-cache uses: actions/cache@v4 with: path: app/xulrunner/firefox-x86_64 key: xulrunner-${{ hashFiles('app/config.sh', 'app/scripts/fetch_xulrunner') }} - + - name: Fetch xulrunner if: steps.xulrunner-cache.outputs.cache-hit != 'true' run: app/scripts/fetch_xulrunner -p l - + - name: Cache Node modules id: node-cache uses: actions/cache@v4 with: path: node_modules - key: node-modules-${{ hashFiles('package-lock.json') }} - + key: node-modules-24-${{ hashFiles('package-lock.json') }} + - name: Install Node modules if: steps.node-cache.outputs.cache-hit != 'true' run: npm install - + - name: Build Zotero run: npm run build # Currently necessary for document-worker Webpack: https://stackoverflow.com/a/69746937 env: NODE_OPTIONS: --openssl-legacy-provider - + # Create deployment ZIP from the build output, then replace build/ with the # unzipped contents so that tests run against the same artifact that gets # deployed. This catches problems that only manifest after a zip round-trip @@ -74,19 +133,282 @@ jobs: cd build unzip ../build.zip - # Build deployment ZIPs from main, version branches (e.g., 9.0, 10.0), and *-hotfix branches + - name: Set up SMB share + if: matrix.shard == 'smb' + timeout-minutes: 10 + run: | + sudo apt install -y --no-install-recommends samba cifs-utils + sudo mkdir -p /srv/zotero-smb /mnt/zotero-smb + sudo chown $(whoami) /srv/zotero-smb + sudo tee -a /etc/samba/smb.conf > /dev/null < + needs.changes.outputs.network == 'true' + && (github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name != github.repository) + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + lfs: true + + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Cache xulrunner + id: xulrunner-cache + uses: actions/cache@v4 + with: + path: app/xulrunner/Firefox.app + key: xulrunner-mac-${{ hashFiles('app/config.sh', 'app/scripts/fetch_xulrunner') }} + + - name: Fetch xulrunner + if: steps.xulrunner-cache.outputs.cache-hit != 'true' + run: app/scripts/fetch_xulrunner -p m + + - name: Cache Node modules + id: node-cache + uses: actions/cache@v4 + with: + path: node_modules + key: node-modules-mac-24-${{ hashFiles('package-lock.json') }} + + - name: Install Node modules + if: steps.node-cache.outputs.cache-hit != 'true' + run: npm install + + - name: Build Zotero + run: npm run build + # Currently necessary for document-worker Webpack: https://stackoverflow.com/a/69746937 + env: + NODE_OPTIONS: --openssl-legacy-provider + + - name: Set up NFS share + run: | + sudo mkdir -p /private/var/zotero-nfs + sudo chown $(whoami) /private/var/zotero-nfs + echo "/private/var/zotero-nfs -mapall=$(whoami) localhost" | sudo tee /etc/exports + sudo nfsd enable || sudo nfsd start + sleep 2 + showmount -e localhost + mkdir -p "$HOME/zotero-nfs" + sudo mount_nfs -o vers=3 localhost:/private/var/zotero-nfs "$HOME/zotero-nfs" + + # The step timeout turns a startup hang into a failure + - name: Run tests on network share + timeout-minutes: 10 + run: TMPDIR="$HOME/zotero-nfs" test/runtests.sh -f -r 3 -b db file + + # Boot the Windows build -- which uses custom-built Firefox components (xul.dll) rather + # than stock Mozilla binaries -- and run the db and file tests. A Gecko bump commit + # updates the custom-component hashes in config.sh along with the version, so this + # tests each new set of custom components. + test-win: + name: Test (Windows ${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + needs: changes + if: > + needs.changes.outputs.network == 'true' + && (github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name != github.repository) + strategy: + fail-fast: false + matrix: + include: + - arch: x64 + os: windows-latest + - arch: arm64 + os: windows-11-arm + defaults: + run: + shell: bash + steps: + # The repository contains symlinks, which Git checks out as plain files on Windows + # by default, breaking the build + - name: Enable symlinks + run: git config --global core.symlinks true + + - uses: actions/checkout@v4 + with: + submodules: recursive + lfs: true + + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: 24 + + # Git Bash has unzip but not zip, which fetch_xulrunner needs to repack omni.ja + - name: Install zip + run: choco install zip --no-progress -y + + - name: Cache xulrunner + id: xulrunner-cache + uses: actions/cache@v4 + with: + path: app/xulrunner/firefox-win-${{ matrix.arch }} + key: xulrunner-win-${{ matrix.arch }}-${{ hashFiles('app/config.sh', 'app/scripts/fetch_xulrunner') }} + + - name: Fetch xulrunner + if: steps.xulrunner-cache.outputs.cache-hit != 'true' + run: app/scripts/fetch_xulrunner -p w -a ${{ matrix.arch }} + + - name: Cache Node modules + id: node-cache + uses: actions/cache@v4 + with: + path: node_modules + key: node-modules-win-24-${{ hashFiles('package-lock.json') }} + + - name: Install Node modules + if: steps.node-cache.outputs.cache-hit != 'true' + run: npm install + + # Committed symlinks point into node_modules, which doesn't exist at checkout time, + # so Git creates the directory ones with the wrong symlink type on Windows. Recreate + # them now that the targets exist. + - name: Recreate symlinks + run: | + git ls-files -s | awk '$1 == 120000 { print $4 }' | xargs rm + git checkout -- . + + - name: Build Zotero + run: npm run build + # Currently necessary for document-worker Webpack: https://stackoverflow.com/a/69746937 + env: + NODE_OPTIONS: --openssl-legacy-provider + + # The step timeout turns a startup hang into a failure + - name: Run tests + timeout-minutes: 15 + run: test/runtests.sh -f -r 3 -b -x app/staging/Zotero_win-${{ matrix.arch }}/zotero.exe db file + + utilities: + name: Utilities Tests + runs-on: ubuntu-latest + if: > + github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Cache utilities Node modules + id: utilities-node-cache + uses: actions/cache@v4 + with: + path: chrome/content/zotero/xpcom/utilities/node_modules + key: utilities-node-modules-24-${{ hashFiles('chrome/content/zotero/xpcom/utilities/package-lock.json') }} + + - name: Install utilities Node modules + if: steps.utilities-node-cache.outputs.cache-hit != 'true' + run: npm install --prefix chrome/content/zotero/xpcom/utilities + + - name: Run utilities tests + run: | + npm test --prefix chrome/content/zotero/xpcom/utilities -- -j resource/schema/global/schema.json + + # Build deployment ZIPs from main, version branches (e.g., 9.0, 10.0), and *-hotfix branches + deploy: + name: Build, Upload + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.repository == 'zotero/zotero' + steps: - name: Check if deployment branch id: check-deploy - if: | - env.ACT != 'true' - && github.repository == 'zotero/zotero' - && github.event_name == 'push' + if: env.ACT != 'true' run: | branch="${GITHUB_REF#refs/heads/}" if [[ "$branch" == "main" || "$branch" =~ ^[0-9]+\.[0-9]+$ || "$branch" == *-hotfix ]]; then echo "deploy=true" >> "$GITHUB_OUTPUT" fi + - uses: actions/checkout@v4 + if: steps.check-deploy.outputs.deploy == 'true' + with: + submodules: recursive + lfs: true + + - name: Install Node + if: steps.check-deploy.outputs.deploy == 'true' + uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Cache Node modules + id: node-cache + if: steps.check-deploy.outputs.deploy == 'true' + uses: actions/cache@v4 + with: + path: node_modules + key: node-modules-24-${{ hashFiles('package-lock.json') }} + + - name: Install Node modules + if: steps.check-deploy.outputs.deploy == 'true' && steps.node-cache.outputs.cache-hit != 'true' + run: npm install + + - name: Build Zotero + if: steps.check-deploy.outputs.deploy == 'true' + run: npm run build + # Currently necessary for document-worker Webpack: https://stackoverflow.com/a/69746937 + env: + NODE_OPTIONS: --openssl-legacy-provider + + - name: Create deployment ZIP + if: steps.check-deploy.outputs.deploy == 'true' + run: | + cd build + zip -r ../build.zip * + cd .. + - uses: ruby/setup-ruby@v1 if: steps.check-deploy.outputs.deploy == 'true' with: @@ -103,21 +425,3 @@ jobs: cp build.zip build-zip/$GITHUB_SHA.zip gem install --no-document dpl -v '>= 2.0' dpl s3 --bucket zotero-download --local_dir build-zip --upload_dir ci/client --acl public_read - - - name: Run tests - run: xvfb-run test/runtests.sh -f -r 3 - - - name: Cache utilities Node modules - id: utilities-node-cache - uses: actions/cache@v4 - with: - path: chrome/content/zotero/xpcom/utilities/node_modules - key: utilities-node-modules-${{ hashFiles('chrome/content/zotero/xpcom/utilities/package-lock.json') }} - - - name: Install utilities Node modules - if: steps.utilities-node-cache.outputs.cache-hit != 'true' - run: npm install --prefix chrome/content/zotero/xpcom/utilities - - - name: Run utilities tests - run: | - npm test --prefix chrome/content/zotero/xpcom/utilities -- -j resource/schema/global/schema.json diff --git a/app/assets/application.ini b/app/assets/application.ini index 802d2f666c..68599e32b6 100644 --- a/app/assets/application.ini +++ b/app/assets/application.ini @@ -7,8 +7,8 @@ Copyright=Copyright (c) 2006-2025 Contributors ID=zotero@zotero.org [Gecko] -MinVersion=140.0 -MaxVersion=140.99.* +MinVersion=153.0 +MaxVersion=153.99.* [XRE] EnableExtensionManager=1 diff --git a/app/assets/commandLineHandler.js b/app/assets/commandLineHandler.js index 8fe82c1a2f..e48ae49336 100644 --- a/app/assets/commandLineHandler.js +++ b/app/assets/commandLineHandler.js @@ -50,6 +50,7 @@ if (processTestOptions) { TestOptions.bail = cmdLine.handleFlag("bail", false); TestOptions.startAt = cmdLine.handleFlagWithParam("startAtTestFile", false); TestOptions.stopAt = cmdLine.handleFlagWithParam("stopAtTestFile", false); + TestOptions.shard = cmdLine.handleFlagWithParam("shard", false); TestOptions.grep = cmdLine.handleFlagWithParam("grep", false); TestOptions.timeout = cmdLine.handleFlagWithParam("ZoteroTestTimeout", false); TestOptions.retries = cmdLine.handleFlagWithParam("retries", false) || 0; diff --git a/app/build.sh b/app/build.sh index ff1e862e16..576a2721d6 100755 --- a/app/build.sh +++ b/app/build.sh @@ -42,7 +42,7 @@ Options -f FILE ZIP file to build from (cannot be used with -d) -t add devtools -p PLATFORMS build for platforms PLATFORMS (m=Mac, w=Windows, l=Linux) - -a ARCH architecture to build (arm64, x64, i686, win32) + -a ARCH architecture to build (arm64, x64, win32) * Ignored for Mac (always universal) * If omitted on Windows/Linux, all standard archs are built -c CHANNEL use update channel CHANNEL @@ -59,6 +59,17 @@ function cleanup { } trap cleanup EXIT +# Copy the contents of a directory into another directory, which may already exist, +# using rsync if it's available (not on Windows) +function copy_dir { + if command -v rsync > /dev/null; then + rsync -a "$1/" "$2/" + else + mkdir -p "$2" + cp -a "$1/." "$2/" + fi +} + function abspath { echo $(cd $(dirname $1); pwd)/$(basename $1); } @@ -210,7 +221,7 @@ if [ $BUILD_LINUX == 1 ]; then if [[ -n $arch ]]; then check_xulrunner_hash l $(get_canonical_arch l $arch) else - for _a in x64 arm64 i686; do + for _a in x64 arm64; do check_xulrunner_hash l "$_a" done fi @@ -296,7 +307,7 @@ elif [[ $BUILD_LINUX == 1 ]]; then if [[ -n $arch ]]; then omni_arch=$(get_canonical_arch l $arch) else - for cand in x86_64 arm64 i686; do + for cand in x86_64 arm64; do [[ -d "${LINUX_RUNTIME_PATH_PREFIX}${cand}" ]] && { omni_arch="$cand"; break; } done fi @@ -311,7 +322,6 @@ cd $omni_dir rm actors/AboutLogins{Parent,Child}.sys.mjs rm actors/AboutMessagePreview{Parent,Child}.sys.mjs rm actors/AboutNewTab{Parent,Child}.sys.mjs -rm actors/AboutPocket{Parent,Child}.sys.mjs rm actors/AboutPrivateBrowsing{Parent,Child}.sys.mjs rm actors/AboutProtections{Parent,Child}.sys.mjs rm actors/AboutReader{Parent,Child}.sys.mjs @@ -350,11 +360,10 @@ browser_keep=( content/browser/parent/ext-browser.js # For spellchecking content/browser/built_in_addons.json + # Statically imported by BackupService, which SelectableProfileService pulls in when it's + # instantiated as a command-line handler at startup + content/browser/backup/backup-constants.mjs ) -if [ $BUILD_WIN == 1 ]; then - # Windows window controls - browser_keep+=(skin/classic/browser/window-controls) -fi for file in "${browser_keep[@]}"; do mkdir -p "$(dirname "chrome/browser-fx/$file")" mv "chrome/browser/$file" "chrome/browser-fx/$file" @@ -376,11 +385,18 @@ if [ -n "$ZIP_FILE" ]; then echo "Building from $ZIP_FILE" unzip -q $ZIP_FILE -d "$omni_dir" else - rsync_params="" - if [ $include_tests -eq 0 ]; then - rsync_params="--exclude /test" + if command -v rsync > /dev/null; then + rsync_params="" + if [ $include_tests -eq 0 ]; then + rsync_params="--exclude /test" + fi + rsync -a $rsync_params "$SOURCE_DIR/" ./ + else + copy_dir "$SOURCE_DIR" . + if [ $include_tests -eq 0 ]; then + rm -rf ./test + fi fi - rsync -a $rsync_params "$SOURCE_DIR/" ./ fi mv defaults defaults-z @@ -406,6 +422,8 @@ echo "" >> $prefs_file echo "# Zotero extension prefs" >> $prefs_file echo "" >> $prefs_file cat defaults-z/preferences/zotero.js >> $prefs_file +# Babel strips the trailing newline, so anything appended below would share a line +echo "" >> $prefs_file rm -rf defaults-z @@ -574,11 +592,11 @@ fi # Copy platform-specific assets if [ $BUILD_MAC == 1 ]; then - rsync -a "$CALLDIR/assets/mac/" ./ + copy_dir "$CALLDIR/assets/mac" . elif [ $BUILD_WIN == 1 ]; then - rsync -a "$CALLDIR/assets/win/" ./ + copy_dir "$CALLDIR/assets/win" . elif [ $BUILD_LINUX == 1 ]; then - rsync -a "$CALLDIR/assets/unix/" ./ + copy_dir "$CALLDIR/assets/unix" . fi # Add word processor plug-ins @@ -676,8 +694,8 @@ if [ $BUILD_MAC == 1 ]; then # Merge relevant assets from Firefox mkdir "$CONTENTSDIR/MacOS" - cp -r "$MAC_RUNTIME_PATH/Contents/MacOS/"!(firefox|firefox-bin|crashreporter.app|minidump-analyzer|nmhproxy|pingsender|updater.app) "$CONTENTSDIR/MacOS" - cp -r "$MAC_RUNTIME_PATH/Contents/Resources/"!(application.ini|browser|defaults|precomplete|removed-files|updater.ini|update-settings.ini|webapprt*|*.icns|*.lproj) "$CONTENTSDIR/Resources" + cp -r "$MAC_RUNTIME_PATH/Contents/MacOS/"!(firefox|firefox-bin|crashhelper|crashreporter.app|minidump-analyzer|nmhproxy|pingsender|updater.app) "$CONTENTSDIR/MacOS" + cp -r "$MAC_RUNTIME_PATH/Contents/Resources/"!(application.ini|Assets.car|browser|defaults|precomplete|removed-files|updater.ini|update-settings.ini|webapprt*|*.icns|*.lproj) "$CONTENTSDIR/Resources" # Add our custom ChannelPrefs.framework and change channel if not a source build mkdir "$CONTENTSDIR/Frameworks" @@ -721,7 +739,7 @@ if [ $BUILD_MAC == 1 ]; then echo # Copy app files - rsync -a "$base_dir/" "$CONTENTSDIR/Resources/" + copy_dir "$base_dir" "$CONTENTSDIR/Resources" # Add word processor plug-ins mkdir "$CONTENTSDIR/Resources/integration" @@ -732,12 +750,66 @@ if [ $BUILD_MAC == 1 ]; then find "$CONTENTSDIR" -depth -type d -name .git -exec rm -rf {} \; find "$CONTENTSDIR" \( -name .DS_Store -or -name update.rdf \) -exec rm -f {} \; - # Add Safari App Extension -- this depends on signing but needs to be done before generating + # Add Safari extensions -- this depends on signing but needs to be done before generating # the precomplete file + # + # $SAFARI_APPEX is a stub appex built from the safari-web-extension wrapper project. The web + # extension itself comes from $SAFARI_EXT_RESOURCES (a zotero-connectors build/safari + # directory), which replaces the stub's placeholder resources here before signing. + # + # $SAFARI_APP_EXTENSION is an optional prebuilt legacy Safari App Extension, embedded + # alongside the web extension for Safari versions that can't load Developer ID web + # extensions (supported in Safari 18.4 and later). On Safari versions that can load the + # web extension, the SFSafariAppExtensionBundleIdentifiersToReplace key causes it to + # replace the App Extension. if [[ $SIGN == 1 ]] && [[ -n "$SAFARI_APPEX" ]] && [[ -d "$SAFARI_APPEX" ]]; then + if [[ -z "${SAFARI_EXT_RESOURCES:-}" ]] || [[ ! -f "$SAFARI_EXT_RESOURCES/manifest.json" ]]; then + echo "SAFARI_EXT_RESOURCES doesn't contain a web extension -- aborting" 2>&1 + exit 1 + fi + bundle_identifier=$(/usr/libexec/PlistBuddy -c "Print CFBundleIdentifier" "$APPDIR/Contents/Info.plist") mkdir "$APPDIR/Contents/PlugIns" - cp -R $SAFARI_APPEX "$APPDIR/Contents/PlugIns/ZoteroSafariExtension.appex" - rm -rf "$APPDIR/Contents/PlugIns/ZoteroSafariExtension.appex/Contents/Resources/safari/test/" + + webext_appex="$APPDIR/Contents/PlugIns/ZoteroSafariWebExtension.appex" + cp -R "$SAFARI_APPEX" "$webext_appex" + appex_resources="$webext_appex/Contents/Resources" + rm -rf "$appex_resources" + mkdir "$appex_resources" + cp -R "$SAFARI_EXT_RESOURCES/." "$appex_resources/" + + # Show the connector version in Safari + connector_version=$(python3 -c "import json, sys; print(json.load(open(sys.argv[1]))['version'])" "$appex_resources/manifest.json") + if [[ $connector_version == *999* ]] && [ "$UPDATE_CHANNEL" != "test" ]; then + echo "Placeholder connector version $connector_version not allowed for '$UPDATE_CHANNEL' channel -- aborting" 2>&1 + exit 1 + fi + /usr/libexec/PlistBuddy -c "Set CFBundleShortVersionString $connector_version" \ + "$webext_appex/Contents/Info.plist" + /usr/libexec/PlistBuddy -c "Set CFBundleVersion $connector_version" \ + "$webext_appex/Contents/Info.plist" + + # Give the appex the same bundle identifier prefix as the parent app + /usr/libexec/PlistBuddy -c "Set CFBundleIdentifier $bundle_identifier.SafariWebExtension" \ + "$webext_appex/Contents/Info.plist" + + # Replace the legacy App Extension on Safari versions that can load the web extension + /usr/libexec/PlistBuddy -c "Add :NSExtension:SFSafariAppExtensionBundleIdentifiersToReplace array" \ + "$webext_appex/Contents/Info.plist" + /usr/libexec/PlistBuddy -c "Add :NSExtension:SFSafariAppExtensionBundleIdentifiersToReplace:0 string $bundle_identifier.SafariExtension" \ + "$webext_appex/Contents/Info.plist" + + # Add legacy Safari App Extension + if [[ -n "${SAFARI_APP_EXTENSION:-}" ]]; then + if [[ ! -d "$SAFARI_APP_EXTENSION" ]]; then + echo "SAFARI_APP_EXTENSION not found at $SAFARI_APP_EXTENSION -- aborting" 2>&1 + exit 1 + fi + appext_appex="$APPDIR/Contents/PlugIns/ZoteroSafariExtension.appex" + cp -R "$SAFARI_APP_EXTENSION" "$appext_appex" + rm -rf "$appext_appex/Contents/Resources/safari/test" + /usr/libexec/PlistBuddy -c "Set CFBundleIdentifier $bundle_identifier.SafariExtension" \ + "$appext_appex/Contents/Info.plist" + fi fi # Copy over removed-files and make a precomplete file @@ -773,34 +845,36 @@ if [ $BUILD_MAC == 1 ]; then # Sign .jnilib (Java native shared library) within LibreOffice extension, since notarization # started failing without this. The .jnilib is within a .jar within the .oxt, so we have to - # extract both, sign the library, and then update each ZIP. + # extract both, sign the libraries, and then update each ZIP. + # + # TODO: Remove this block once the plugin ships a jna.jar without the macOS native + # libraries, which are never loaded (JNA is used only on Windows) pushd "$BUILD_DIR" mkdir libreoffice-repack cd libreoffice-repack unzip -q "$APPDIR/Contents/Resources/integration/libreoffice/Zotero_LibreOffice_Integration.oxt" external_jars/jna.jar - unzip -q external_jars/jna.jar com/sun/jna/darwin/libjnidispatch.jnilib - /usr/bin/codesign --force --options runtime --sign "$DEVELOPER_ID" com/sun/jna/darwin/libjnidispatch.jnilib - zip -u external_jars/jna.jar com/sun/jna/darwin/libjnidispatch.jnilib + unzip -q external_jars/jna.jar 'com/sun/jna/darwin*/libjnidispatch.jnilib' + /usr/bin/codesign --force --options runtime --sign "$DEVELOPER_ID" com/sun/jna/darwin*/libjnidispatch.jnilib + zip -u external_jars/jna.jar com/sun/jna/darwin*/libjnidispatch.jnilib zip -u "$APPDIR/Contents/Resources/integration/libreoffice/Zotero_LibreOffice_Integration.oxt" external_jars/jna.jar cd .. rm -rf libreoffice-repack popd - # Sign Safari App Extension + # Sign Safari extensions # - # Even though it's signed by Xcode, we sign it again to make sure it matches the parent app signature - if [ -d "$APPDIR/Contents/PlugIns/ZoteroSafariExtension.appex" ]; then + # Even though they're signed by Xcode, we sign them again to make sure they match the parent app signature + for appex in "$APPDIR"/Contents/PlugIns/*.appex; do + if [ ! -d "$appex" ]; then + continue + fi echo # Extract entitlements, which differ from parent app - /usr/bin/codesign -d --entitlements "$BUILD_DIR/safari-entitlements.plist" --xml "$SAFARI_APPEX" + /usr/bin/codesign -d --entitlements "$BUILD_DIR/safari-entitlements.plist" --xml "$appex" - # Change appex bundle identifier to have same prefix as parent app - bundle_identifier=$(/usr/libexec/PlistBuddy -c "Print CFBundleIdentifier" "$APPDIR/Contents/Info.plist") - perl -pi -e "s/org\.zotero\.SafariExtensionApp\.SafariExtension/$bundle_identifier.SafariExtension/" "$APPDIR/Contents/PlugIns/ZoteroSafariExtension.appex/Contents/Info.plist" - - find "$APPDIR/Contents/PlugIns/ZoteroSafariExtension.appex/Contents" -name '*.dylib' -exec /usr/bin/codesign --force --options runtime --entitlements "$entitlements_file" --sign "$DEVELOPER_ID" {} \; - /usr/bin/codesign --force --options runtime --entitlements "$BUILD_DIR/safari-entitlements.plist" --sign "$DEVELOPER_ID" "$APPDIR/Contents/PlugIns/ZoteroSafariExtension.appex" - fi + find "$appex/Contents" -name '*.dylib' -exec /usr/bin/codesign --force --options runtime --entitlements "$entitlements_file" --sign "$DEVELOPER_ID" {} \; + /usr/bin/codesign --force --options runtime --entitlements "$BUILD_DIR/safari-entitlements.plist" --sign "$DEVELOPER_ID" "$appex" + done # Sign final app package echo @@ -808,11 +882,14 @@ if [ $BUILD_MAC == 1 ]; then # Verify app /usr/bin/codesign --verify -vvvv "$APPDIR" - # Verify Safari App Extension - if [[ -n "$SAFARI_APPEX" ]] && [[ -d "$SAFARI_APPEX" ]]; then + # Verify Safari extensions + for appex in "$APPDIR"/Contents/PlugIns/*.appex; do + if [ ! -d "$appex" ]; then + continue + fi echo - /usr/bin/codesign --verify -vvvv "$APPDIR/Contents/PlugIns/ZoteroSafariExtension.appex" - fi + /usr/bin/codesign --verify -vvvv "$appex" + done fi # Build and notarize disk image @@ -885,7 +962,7 @@ if [ $BUILD_WIN == 1 ]; then # # 'i686' is a huge directory containing x86 versions of xul.dll and other files in # Firefox ARM64 builds for use with the EME DRM plugins - cp -R "$runtime_path"/!(application.ini|browser|crashreporter*|default-browser-agent.exe|defaultagent*|defaults|devtools-files|firefox*|i686|maintenanceservice*|minidump-analyzer.exe|pingsender.exe|private_browsing*|precomplete|removed-files|uninstall|update*) "$APPDIR" + cp -R "$runtime_path"/!(application.ini|browser|crashhelper.exe|crashreporter*|default-browser-agent.exe|defaultagent*|defaults|desktop-launcher|devtools-files|firefox*|i686|maintenanceservice*|minidump-analyzer.exe|nmhproxy.exe|pingsender.exe|private_browsing*|precomplete|removed-files|uninstall|update*) "$APPDIR" # Copy zotero.exe, which is built directly from Firefox source and then modified by # ResourceHacker to add icons @@ -910,11 +987,11 @@ if [ $BUILD_WIN == 1 ]; then fi # Copy app files - rsync -a "$base_dir/" "$APPDIR/" + copy_dir "$base_dir" "$APPDIR" #mv "$APPDIR/app/application.ini" "$APPDIR/" # Copy in common files - rsync -a "$COMMON_APPDIR/" "$APPDIR/" + copy_dir "$COMMON_APPDIR" "$APPDIR" cat "$CALLDIR/win/installer/updater_append.ini" >> "$APPDIR/updater.ini" @@ -1056,7 +1133,7 @@ if [ $BUILD_LINUX == 1 ]; then if [[ -n $arch ]]; then archs=("$(get_canonical_arch l $arch)") else - archs=(x64 arm64 i686) + archs=(x64 arm64) fi for arch in "${archs[@]}"; do [[ $arch == x64 ]] && arch="x86_64" @@ -1070,7 +1147,7 @@ if [ $BUILD_LINUX == 1 ]; then mkdir "$APPDIR" # Merge relevant assets from Firefox - cp -r "$runtime_path/"!(application.ini|browser|defaults|devtools-files|crashreporter|crashreporter.ini|firefox|pingsender|precomplete|removed-files|run-mozilla.sh|update-settings.ini|updater|updater.ini) "$APPDIR" + cp -r "$runtime_path/"!(application.ini|browser|crashhelper|crashreporter|crashreporter.ini|defaults|devtools-files|firefox|pingsender|precomplete|removed-files|run-mozilla.sh|update-settings.ini|updater|updater.ini) "$APPDIR" # Use our own launcher that calls the original Firefox executable with -app mv "$APPDIR"/firefox-bin "$APPDIR"/zotero-bin @@ -1086,7 +1163,7 @@ if [ $BUILD_LINUX == 1 ]; then chmod 755 "$APPDIR/updater" # Copy app files - rsync -a "$base_dir/" "$APPDIR/" + copy_dir "$base_dir" "$APPDIR" # Add word processor plug-ins mkdir "$APPDIR/integration" diff --git a/app/config.sh b/app/config.sh index 072e075242..fd2b0e5a35 100644 --- a/app/config.sh +++ b/app/config.sh @@ -1,17 +1,17 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # Version of Gecko to build with -GECKO_VERSION_MAC="140.11.0esr" -GECKO_VERSION_LINUX="140.11.0esr" -GECKO_VERSION_WIN="140.10.0esr" -RUST_VERSION=1.86.0 +GECKO_VERSION_MAC="153.1.0esr" +GECKO_VERSION_LINUX="153.1.0esr" +GECKO_VERSION_WIN="153.1.0esr" +RUST_VERSION=1.94.0 # URL prefix for custom builds of Firefox components custom_components_url="https://download.zotero.org/dev/firefox-components/" custom_components_hash_mac="" -custom_components_hash_win_x64="d4f13db655004401c114fb937b1535a74919b6516fae637e35847734bde4d602" -custom_components_hash_win_arm64="69390facb8502fc40127f1ad80012858a3717935d5806331f1b17f54b05c2ca8" -custom_components_hash_win32="163e98eecf2c17579f47e1c2cfa1eb44e204cc26bcb220bb7c31168ab866e2ef" +custom_components_hash_win_x64="ac2cd5e2d4190c07af1649ca0372ff07dae70afd880d309441dafb575ccdc61a" +custom_components_hash_win_arm64="2369eb75912fb91633fb16d0f756c78d41f533522a3415bb4746c2a911547083" +custom_components_hash_win32="8eea54a3754b5f5331fc9147a6e7741976d70ddb48c8759f3b1a77e8729d8019" APP_NAME="Zotero" APP_ID="zotero\@zotero.org" diff --git a/app/linux/build b/app/linux/build index ee14b1dd36..2d4f74f883 100755 --- a/app/linux/build +++ b/app/linux/build @@ -6,7 +6,7 @@ APP_ROOT_DIR="$(dirname "$SCRIPT_DIR")" . "$APP_ROOT_DIR/config.sh" if [ -z "${1:-}" ]; then - echo "Usage: $0 x64|arm64|i686" >&2 + echo "Usage: $0 x64|arm64" >&2 exit 1 fi @@ -24,8 +24,6 @@ if [ $arch = "x64" ]; then rust_target=x86_64 elif [ $arch = "arm64" ]; then rust_target=aarch64 -elif [ $arch = "i686" ]; then - rust_target=i686 else echo "Unknown architecture $arch" >&2 exit 1 @@ -37,8 +35,6 @@ touch "$gecko_path/mozconfig" if [ $arch == "arm64" ]; then echo "ac_add_options --target=aarch64-linux-gnu" >> "$gecko_path/mozconfig" -elif [ $arch == "i686" ]; then - echo "ac_add_options --target=i686" >> "$gecko_path/mozconfig" fi cat "$SCRIPT_DIR/mozconfig" >> "$gecko_path/mozconfig" diff --git a/app/linux/update_updater b/app/linux/update_updater index cf8f6aa342..e68bdd872d 100755 --- a/app/linux/update_updater +++ b/app/linux/update_updater @@ -11,7 +11,7 @@ fi version="$1" -for arch in x86_64 aarch64 i686; do +for arch in x86_64 aarch64; do package="firefox-${version}.en-US.linux-${arch}.tar.xz" if [ $arch = 'aarch64' ]; then arch="arm64" @@ -36,8 +36,8 @@ if [ `uname` = "Darwin" ]; then else tar=tar fi -$tar --owner=0 --group=0 --numeric-owner --no-xattrs --mode=0755 -cJvf updater.tar.xz updater-x86_64 updater-arm64 updater-i686 +$tar --owner=0 --group=0 --numeric-owner --no-xattrs --mode=0755 -cJvf updater.tar.xz updater-x86_64 updater-arm64 -rm updater-x86_64 updater-arm64 updater-i686 +rm updater-x86_64 updater-arm64 echo "Done: updater.tar.xz created" diff --git a/app/mac/pkg-dmg b/app/mac/pkg-dmg index f84ba962ca..6d5581861c 100755 --- a/app/mac/pkg-dmg +++ b/app/mac/pkg-dmg @@ -303,6 +303,7 @@ my(@gCleanup, %gConfig, $gDarwinMajor, $gDryRun, $gVerbosity); 'cmd_chmod' => 'chmod', 'cmd_diskutil' => 'diskutil', 'cmd_du' => 'du', + 'cmd_find' => 'find', 'cmd_hdid' => 'hdid', 'cmd_hdiutil' => 'hdiutil', 'cmd_mkdir' => 'mkdir', @@ -311,6 +312,7 @@ my(@gCleanup, %gConfig, $gDarwinMajor, $gDryRun, $gVerbosity); 'cmd_rm' => 'rm', 'cmd_rsync' => 'rsync', 'cmd_SetFile' => '/Developer/Tools/SetFile', + 'cmd_xattr' => 'xattr', # create_directly indicates whether hdiutil create supports # -srcfolder and -srcdevice. It does on >= 10.3 (Panther). @@ -996,6 +998,65 @@ sub diskImageMaker($$$$$$$$) { cleanupDie('unlink hybridImage failed: '.$!); } } + + # hdiutil makehybrid gives every file on the image non-empty + # Finder info (it sets an icon location), which appears as a + # com.apple.FinderInfo extended attribute that "codesign + # --verify --strict" rejects as detritus and that can cause + # Safari to silently ignore a bundled web extension (observed + # with Safari 17.6 on macOS 12; the exact trigger conditions + # are unknown). Mount a read-write copy of the image and strip + # the attributes before compressing. Note that Finder flags set + # with --attribute are stored in the same Finder info, so if + # --attribute is ever used, the affected files would need to be + # excluded from the strip. + if($uncompressedImage eq $hybridImage) { + my($udrwImage); + $udrwImage = giveExtension($tempDir.'/udrw', '.dmg'); + + if(command($gConfig{'cmd_hdiutil'}, 'convert', '-format', 'UDRW', + '-ov', $hybridImage, '-o', $udrwImage) != 0) { + cleanupDie('hdiutil convert to UDRW failed'); + } + + push(@gCleanup, + sub {commandInternalVerbosity(0, 'unlink', $udrwImage);}); + + $uncompressedImage = $udrwImage; + + # $hybridImage is no longer needed. Remove it and its cleanup + # entry, which is below the entry for $udrwImage. + my(@tempCleanup) = splice(@gCleanup, -2); + push(@gCleanup, $tempCleanup[1]); + + if(commandInternal('unlink', $hybridImage) != 1) { + cleanupDie('unlink hybridImage failed: '.$!); + } + } + + my($rootDevice, $partitionDevice, $partitionMountPoint); + if(!(($rootDevice, $partitionDevice, $partitionMountPoint) = + hdidMountImage($tempMount, $uncompressedImage))) { + cleanupDie('hdid mount failed'); + } + + push(@gCleanup, sub {commandVerbosity(0, + $gConfig{'cmd_diskutil'}, 'eject', $rootDevice);}); + + if(command($gConfig{'cmd_find'}, $partitionMountPoint, + '-mindepth', '1', '-not', '-type', 'l', + '-xattrname', 'com.apple.FinderInfo', + '-exec', $gConfig{'cmd_xattr'}, '-d', 'com.apple.FinderInfo', + '{}', '+') != 0) { + cleanupDie('stripping com.apple.FinderInfo failed'); + } + + # Pop diskutil eject + pop(@gCleanup); + + if(command($gConfig{'cmd_diskutil'}, 'eject', $rootDevice) != 0) { + cleanupDie('diskutil eject failed'); + } } else { # makehybrid is not available, fall back to making a UDRW and diff --git a/app/mac/set-channel-prefs-channel b/app/mac/set-channel-prefs-channel index 2431837584..fc1ccc1335 100755 --- a/app/mac/set-channel-prefs-channel +++ b/app/mac/set-channel-prefs-channel @@ -19,7 +19,8 @@ binary=$1 from_channel=$2 to_channel=$3 # `strings` has a 4-character minimum by default, but we need 3 for 'dev' -strings_cmd="strings -n 3" +# `-arch all` scans all slices of the universal binary rather than just the host architecture +strings_cmd="strings -n 3 -arch all" if [ ${#to_channel} -gt 7 ]; then echo "Channel length cannot exceed 7 characters -- aborting" >&2 diff --git a/app/modules/zotero-libreoffice-integration b/app/modules/zotero-libreoffice-integration index c216d787c4..227551c164 160000 --- a/app/modules/zotero-libreoffice-integration +++ b/app/modules/zotero-libreoffice-integration @@ -1 +1 @@ -Subproject commit c216d787c4a3dfbb440b5db620fb14f2f185ad74 +Subproject commit 227551c1644c6cb1156226d4db5d1b293fdb271d diff --git a/app/modules/zotero-word-for-mac-integration b/app/modules/zotero-word-for-mac-integration index 084f16d78b..300886d198 160000 --- a/app/modules/zotero-word-for-mac-integration +++ b/app/modules/zotero-word-for-mac-integration @@ -1 +1 @@ -Subproject commit 084f16d78b4924b15ad24c83dd5d8aaaeb9998d8 +Subproject commit 300886d1984de7a0006d7a48310baa0c8c99bea0 diff --git a/app/modules/zotero-word-for-windows-integration b/app/modules/zotero-word-for-windows-integration index 72e8364775..c0aa6e4bef 160000 --- a/app/modules/zotero-word-for-windows-integration +++ b/app/modules/zotero-word-for-windows-integration @@ -1 +1 @@ -Subproject commit 72e83647756b759ec80923922218fff8bf8b0343 +Subproject commit c0aa6e4bef039d94e17e81cb28b1fe9170c45b96 diff --git a/app/scripts/8.0_release_build_and_deploy b/app/scripts/8.0_release_build_and_deploy deleted file mode 100755 index 3f18495ce0..0000000000 --- a/app/scripts/8.0_release_build_and_deploy +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -set -euo pipefail - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -APP_ROOT_DIR="$(dirname "$SCRIPT_DIR")" -ROOT_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" -. "$APP_ROOT_DIR/config.sh" - -CHANNEL="release" -BRANCH="8.0" -BUILD_ONLY=0 -if [ "`uname`" = "Darwin" ]; then - export SAFARI_APPEX="$ROOT_DIR/../safari-app-extension-builds/release/ZoteroSafariExtension.appex" -fi - -while getopts "b" opt; do - case $opt in - b) - BUILD_ONLY=1 - ;; - esac - shift $((OPTIND-1)); OPTIND=1 -done - -cd "$SCRIPT_DIR" -./check_requirements - -hash=`./get_repo_branch_hash $BRANCH` -source_dir=`./get_commit_files $hash` -build_dir=`mktemp -d` - -function cleanup { - rm -rf "$source_dir" - rm -rf "$build_dir" -} -trap cleanup EXIT - -./prepare_build -s "$source_dir" -o "$build_dir" -c $CHANNEL -m $hash -VERSION="`cat \"$build_dir/version\"`" -./build_for_deploy -d "$build_dir" -p $BUILD_PLATFORMS -c $CHANNEL - -if [ $BUILD_ONLY -eq 1 ]; then - echo - echo "Build only -- skipping deploy." - echo "To deploy, run on the deploy server:" - echo " $DEPLOY_PATH/deploy $CHANNEL $VERSION $BUILD_PLATFORMS" -else - ssh $DEPLOY_HOST "$DEPLOY_PATH/deploy" $CHANNEL "$VERSION" $BUILD_PLATFORMS -fi diff --git a/app/scripts/9.0_release_build_and_deploy b/app/scripts/9.0_release_build_and_deploy index 72c7b0db09..79d325b770 100755 --- a/app/scripts/9.0_release_build_and_deploy +++ b/app/scripts/9.0_release_build_and_deploy @@ -10,7 +10,16 @@ CHANNEL="release" BRANCH="9.0" BUILD_ONLY=0 if [ "`uname`" = "Darwin" ]; then - export SAFARI_APPEX="$ROOT_DIR/../safari-app-extension-builds/release/ZoteroSafariExtension.appex" + export SAFARI_APPEX="$ROOT_DIR/../safari-web-extension-builds/ZoteroSafariExtension.appex" + export SAFARI_EXT_RESOURCES="$ROOT_DIR/../safari-web-extension-builds/release/safari" + if [ ! -d "$SAFARI_APPEX" ]; then + echo "Safari extension stub not found at $SAFARI_APPEX -- aborting" >&2 + exit 1 + fi + if [ ! -f "$SAFARI_EXT_RESOURCES/manifest.json" ]; then + echo "Safari web extension not found in $SAFARI_EXT_RESOURCES -- aborting" >&2 + exit 1 + fi fi while getopts "b" opt; do @@ -26,6 +35,7 @@ cd "$SCRIPT_DIR" ./check_requirements hash=`./get_repo_branch_hash $BRANCH` +./check_app_matches_commit $hash source_dir=`./get_commit_files $hash` build_dir=`mktemp -d` diff --git a/app/scripts/add_omni_file b/app/scripts/add_omni_file index cc309e14da..8b3006e04a 100755 --- a/app/scripts/add_omni_file +++ b/app/scripts/add_omni_file @@ -37,11 +37,14 @@ win_path="$STAGE_DIR/Zotero_win-x64" linux_path="$STAGE_DIR/Zotero_linux-x86_64" added=0 +build_id=$(date +%Y%m%d%H%M%S) for path in "$mac_path" "$win_path" "$linux_path"; do if [ -d "$path" ]; then echo "$path/app/omni.ja" echo "Updating $(basename $(dirname $(dirname $path)))" zip "$path/app/omni.ja" $files + # Bump BuildID so that startup caches are invalidated + perl -pi -e "s/^BuildID=.*/BuildID=$build_id/" "$path/app/application.ini" added=1 fi done diff --git a/app/scripts/add_version_info b/app/scripts/add_version_info index 68320b108f..1d5690aa15 100755 --- a/app/scripts/add_version_info +++ b/app/scripts/add_version_info @@ -4,6 +4,7 @@ Update a builds manifest with info on a given build """ import argparse import os +import re import sys import shutil import json @@ -25,7 +26,7 @@ def main(): try: file = args.file version = args.version - short_version = version[0:3] + short_version = re.match(r'\d+\.\d+', version).group(0) # Back up JSON file shutil.copy2(file, file + '.bak') diff --git a/app/scripts/beta_build_and_deploy b/app/scripts/beta_build_and_deploy index 92676beeee..f2087968d0 100755 --- a/app/scripts/beta_build_and_deploy +++ b/app/scripts/beta_build_and_deploy @@ -8,15 +8,39 @@ ROOT_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" CHANNEL="beta" BRANCH="main" -export SAFARI_APPEX="$ROOT_DIR/../safari-app-extension-builds/beta/ZoteroSafariExtension.appex" +BUILD_ONLY=0 +if [ "`uname`" = "Darwin" ]; then + export SAFARI_APPEX="$ROOT_DIR/../safari-web-extension-builds/ZoteroSafariExtension.appex" + export SAFARI_EXT_RESOURCES="$ROOT_DIR/../safari-web-extension-builds/beta/safari" + export SAFARI_APP_EXTENSION="$ROOT_DIR/../safari-app-extension-builds/beta/ZoteroSafariExtension.appex" + if [ ! -d "$SAFARI_APPEX" ]; then + echo "Safari extension stub not found at $SAFARI_APPEX -- aborting" >&2 + exit 1 + fi + if [ ! -f "$SAFARI_EXT_RESOURCES/manifest.json" ]; then + echo "Safari web extension not found in $SAFARI_EXT_RESOURCES -- aborting" >&2 + exit 1 + fi + if [ ! -d "$SAFARI_APP_EXTENSION" ]; then + echo "Safari App Extension not found at $SAFARI_APP_EXTENSION -- aborting" >&2 + exit 1 + fi +fi -# Set Safari extension LSMinimumSystemVersion to Mojave for betas -perl -pi -e 's/11\.0<\/string>/10.14<\/string>/' "$SAFARI_APPEX"/Contents/Info.plist +while getopts "b" opt; do + case $opt in + b) + BUILD_ONLY=1 + ;; + esac + shift $((OPTIND-1)); OPTIND=1 +done cd "$SCRIPT_DIR" ./check_requirements hash=`./get_repo_branch_hash $BRANCH` +./check_app_matches_commit $hash source_dir=`./get_commit_files $hash` build_dir=`mktemp -d` @@ -29,4 +53,12 @@ trap cleanup EXIT ./prepare_build -s "$source_dir" -o "$build_dir" -c $CHANNEL -m $hash VERSION="`cat \"$build_dir/version\"`" ./build_for_deploy -d "$build_dir" -p $BUILD_PLATFORMS -c $CHANNEL -ssh $DEPLOY_HOST "$DEPLOY_PATH/deploy" $CHANNEL "$VERSION" $BUILD_PLATFORMS + +if [ $BUILD_ONLY -eq 1 ]; then + echo + echo "Build only -- skipping deploy." + echo "To deploy, run on the deploy server:" + echo " $DEPLOY_PATH/deploy $CHANNEL $VERSION $BUILD_PLATFORMS" +else + ssh $DEPLOY_HOST "$DEPLOY_PATH/deploy" $CHANNEL "$VERSION" $BUILD_PLATFORMS +fi diff --git a/app/scripts/build_and_run b/app/scripts/build_and_run index 154ad95670..2cffaa24da 100755 --- a/app/scripts/build_and_run +++ b/app/scripts/build_and_run @@ -10,13 +10,23 @@ if [ -n "${ZOTERO_PROFILE:-}" ]; then profile_args=(-p "$ZOTERO_PROFILE") fi -REBUILD=0 +FORCE_FULL=0 +NO_REBUILD=0 SKIP_BUNDLED_FILES=0 DEBUGGER=0 -while getopts "rbd" opt; do +while getopts "rfnbd" opt; do case $opt in r) - REBUILD=1 + # Deprecated -- rebuilding is now the default + echo "-r is deprecated -- rebuilding is now the default (use -n to skip)" >&2 + ;; + + f) + FORCE_FULL=1 + ;; + + n) + NO_REBUILD=1 ;; b) @@ -28,7 +38,6 @@ while getopts "rbd" opt; do ;; \?) - echo "Invalid option: -$OPTARG" >&2 exit 1 ;; esac @@ -37,12 +46,15 @@ done # Remove options from $@ shift $((OPTIND-1)) -if [ $REBUILD -eq 1 ]; then +if [ $NO_REBUILD -eq 0 ]; then PARAMS="" if [ $DEBUGGER -eq 1 ]; then PARAMS="-t" fi - + if [ $FORCE_FULL -eq 1 ]; then + PARAMS="$PARAMS -f" + fi + # Check if build watch is running # If not, run now if ! ps u | grep js-build/build.js | grep -v grep > /dev/null; then @@ -50,16 +62,11 @@ if [ $REBUILD -eq 1 ]; then echo cd $ROOT_DIR # TEMP: --openssl-legacy-provider avoids a build error in pdf.js - NODE_OPTIONS=--openssl-legacy-provider npm run build + NODE_OPTIONS=--openssl-legacy-provider node js-build/build.js echo fi - - "$SCRIPT_DIR/dir_build" -q $PARAMS - - if [ "`uname`" = "Darwin" ]; then - # Sign the Word dylib so it works on Apple Silicon - "$SCRIPT_DIR/codesign_local" "$APP_ROOT_DIR/staging/Zotero.app" - fi + + "$SCRIPT_DIR/dir_build" $PARAMS fi PARAMS="" @@ -87,4 +94,5 @@ else exit 1 fi -"$APP_ROOT_DIR/staging/$command" "${profile_args[@]}" -ZoteroDebugText -purgecaches $PARAMS "$@" +echo +"$APP_ROOT_DIR/staging/$command" "${profile_args[@]}" -ZoteroDebugText $PARAMS "$@" diff --git a/app/scripts/build_for_deploy b/app/scripts/build_for_deploy index f61591f0cc..1f9db44ade 100755 --- a/app/scripts/build_for_deploy +++ b/app/scripts/build_for_deploy @@ -121,7 +121,7 @@ do fi # Combine and deduplicate - INCREMENTALS="`echo -e "$INCREMENTALS\n$PINNED" | sort -u | grep -v '^$'`" + INCREMENTALS="`echo -e "$INCREMENTALS\n$PINNED" | sort -u | grep -v '^$' || true`" echo "$INCREMENTALS" echo @@ -144,7 +144,7 @@ chmod g+ws "$TEMP_DIR/version_info" cp "$DIST_DIR"/files-* "$TEMP_DIR/version_info" chmod g+w "$TEMP_DIR"/version_info/files-* # Generate build-{os}.json for each platform -SHORT_VERSION="${VERSION:0:3}" +SHORT_VERSION=$(echo "$VERSION" | grep -oE '^[0-9]+\.[0-9]+') DETAILS_URL="https://www.zotero.org/support/${SHORT_VERSION}_changelog" for i in `seq 0 1 $((${#PLATFORMS}-1))` do diff --git a/app/scripts/check_app_matches_commit b/app/scripts/check_app_matches_commit new file mode 100755 index 0000000000..236ca7744c --- /dev/null +++ b/app/scripts/check_app_matches_commit @@ -0,0 +1,37 @@ +#!/bin/bash +set -euo pipefail + +# Check that app/ in this checkout matches app/ in the commit being built. +# +# The build-and-deploy scripts run the build scripts and config from the local +# checkout but build source files from the tip of the remote branch, so a stale +# checkout can silently build with the wrong Gecko version or omni patches. + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" + +if [ -z "${1:-}" ]; then + echo "Usage: $0 commit-hash" >&2 + exit 1 +fi +hash=$1 + +cd "$ROOT_DIR" + +if ! git cat-file -e "$hash^{commit}" 2>/dev/null; then + git fetch -q origin +fi +if ! git cat-file -e "$hash^{commit}" 2>/dev/null; then + echo "Commit $hash not found locally after fetching -- aborting" >&2 + exit 1 +fi + +if ! git diff --quiet HEAD "$hash" -- app/; then + echo >&2 + echo "app/ files in this checkout differ from app/ in $hash:" >&2 + echo >&2 + git --no-pager diff --stat HEAD "$hash" -- app/ >&2 + echo >&2 + echo "Update this checkout (e.g., 'git pull') and try again" >&2 + exit 1 +fi diff --git a/app/scripts/dev_build_and_deploy b/app/scripts/dev_build_and_deploy index 0bc3cc01cd..98923fe1d6 100755 --- a/app/scripts/dev_build_and_deploy +++ b/app/scripts/dev_build_and_deploy @@ -8,12 +8,24 @@ ROOT_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" CHANNEL="dev" BRANCH="main" -export SAFARI_APPEX="$ROOT_DIR/../safari-app-extension-builds/dev/ZoteroSafariExtension.appex" +if [ "`uname`" = "Darwin" ]; then + export SAFARI_APPEX="$ROOT_DIR/../safari-web-extension-builds/ZoteroSafariExtension.appex" + export SAFARI_EXT_RESOURCES="$ROOT_DIR/../safari-web-extension-builds/dev/safari" + if [ ! -d "$SAFARI_APPEX" ]; then + echo "Safari extension stub not found at $SAFARI_APPEX -- aborting" >&2 + exit 1 + fi + if [ ! -f "$SAFARI_EXT_RESOURCES/manifest.json" ]; then + echo "Safari web extension not found in $SAFARI_EXT_RESOURCES -- aborting" >&2 + exit 1 + fi +fi cd "$SCRIPT_DIR" ./check_requirements hash=`./get_repo_branch_hash $BRANCH` +./check_app_matches_commit $hash source_dir=`./get_commit_files $hash` build_dir=`mktemp -d` diff --git a/app/scripts/dir_build b/app/scripts/dir_build index 493288ad47..20f5ef0590 100755 --- a/app/scripts/dir_build +++ b/app/scripts/dir_build @@ -5,6 +5,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" APP_ROOT_DIR="$(dirname "$SCRIPT_DIR")" ROOT_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" . "$APP_ROOT_DIR/config.sh" +. "$SCRIPT_DIR/utils.sh" function usage { cat >&2 <&2 - exit 1 + usage ;; esac done @@ -76,29 +76,100 @@ if [[ $platform = "m" ]]; then else # Windows / Linux: derive arch if not supplied if [[ -z $arch ]]; then - case "$(uname -m)" in - arm64|aarch64) arch="arm64" ;; - x86_64) arch="x64" ;; - i?86) arch="i686" ;; - esac + # An x64 build of Git for Windows runs emulated on Windows on ARM, where it + # reports x86_64 from `uname -m`, while `uname -s` reports the native + # architecture + if [[ $platform = "w" && "$(uname -s)" == *ARM64* ]]; then + arch="arm64" + else + case "$(uname -m)" in + arm64|aarch64) arch="arm64" ;; + x86_64) arch="x64" ;; + esac + fi fi fi CHANNEL="source" +# Remove stale symlinks left behind in build/ when source files are deleted, +# which would otherwise break the build +if [ -d "$ROOT_DIR/build" ]; then + # With -L, -type l matches only broken symlinks + stale_links=$(find -L "$ROOT_DIR/build" -type l -print) + if [ -n "$stale_links" ]; then + echo "Removing stale symlinks from build/:" >&2 + echo "$stale_links" >&2 + while IFS= read -r link; do + rm "$link" + done <<< "$stale_links" + fi +fi + +# Try to update the staged build in place instead of doing a full rebuild +if [[ $force_full -eq 0 ]]; then + incr_cmd=("$SCRIPT_DIR/incremental_update" -p "$platform") + [[ -n $arch ]] && incr_cmd+=( -a "$arch" ) + [[ $devtools -eq 1 ]] && incr_cmd+=( -t ) + set +e + "${incr_cmd[@]}" + incr_status=$? + set -e + if [ $incr_status -ne 2 ]; then + exit $incr_status + fi +fi + hash=$(git -C "$ROOT_DIR" rev-parse --short HEAD) build_dir=$(mktemp -d) -cleanup() { rm -rf "$build_dir"; } +manifest_tmp="" +cleanup() { + rm -rf "$build_dir" + if [ -n "$manifest_tmp" ]; then + rm -f "$manifest_tmp" + fi +} trap cleanup EXIT +# Snapshot the state of build/ and app/ before the build for later incremental +# updates +app_hash="" +if [ -d "$ROOT_DIR/build" ]; then + manifest_tmp=$(mktemp) + generate_build_manifest "$ROOT_DIR/build" > "$manifest_tmp" + app_hash=$(generate_app_hash "$APP_ROOT_DIR") +fi + "$SCRIPT_DIR/prepare_build" -s "$ROOT_DIR/build" -o "$build_dir" -c "$CHANNEL" -m "$hash" -build_cmd=("$APP_ROOT_DIR/build.sh" -d "$build_dir" -p "$platform" -c "$CHANNEL" -s) +build_cmd=("$APP_ROOT_DIR/build.sh" -d "$build_dir" -p "$platform" -c "$CHANNEL" -s -q) [[ -n $arch ]] && build_cmd+=( -a "$arch" ) [[ $devtools -eq 1 ]] && build_cmd+=( -t ) -[[ $quick_build -eq 1 ]] && build_cmd+=( -q ) "${build_cmd[@]}" +# Save manifest for future incremental updates +if [ -n "$manifest_tmp" ]; then + if [[ -z "${ZOTERO_TEST:-}" ]] || [[ "${ZOTERO_TEST:-}" == "0" ]]; then + include_tests=0 + else + include_tests=1 + fi + { + echo "#format=2" + echo "#include_tests=$include_tests" + echo "#devtools=$devtools" + echo "#app_hash=$app_hash" + cat "$manifest_tmp" + } > "$STAGE_DIR/.build-manifest" +fi + +# Ad-hoc-sign the Word dylib so it works on Apple Silicon. This only needs to +# happen when the staged build is fully rebuilt -- incremental updates don't +# touch the dylib. +if [[ $platform = "m" ]] && [[ "$(uname -s)" = "Darwin" ]]; then + "$SCRIPT_DIR/codesign_local" "$STAGE_DIR/Zotero.app" +fi + echo Done diff --git a/app/scripts/fetch_xulrunner b/app/scripts/fetch_xulrunner index 7ec257f506..065970ab22 100755 --- a/app/scripts/fetch_xulrunner +++ b/app/scripts/fetch_xulrunner @@ -35,7 +35,7 @@ Required Optional / Conditional -a ARCH - Mac: disallowed (builds are universal) - Windows: x64 | arm64 | win32 - - Linux: x64 (or x86_64) | arm64 | i686 + - Linux: x64 (or x86_64) | arm64 If omitted on Windows/Linux, all standard archs are fetched. DONE exit 1 @@ -87,7 +87,7 @@ if [[ $BUILD_WIN == 1 || $BUILD_LINUX == 1 ]]; then fi if [[ $BUILD_LINUX == 1 ]]; then [[ $arch == x64 ]] && arch="x86_64" - case $arch in x86_64|i686|arm64) ;; *) echo "Invalid Linux arch: $arch" >&2; echo; usage;; esac + case $arch in x86_64|arm64) ;; *) echo "Invalid Linux arch: $arch" >&2; echo; usage;; esac fi fi fi @@ -133,6 +133,12 @@ function modify_omni { rm actors/AudioPlayback{Parent,Child}.sys.mjs replace_line 'BROWSER_CHROME_URL:.+' 'BROWSER_CHROME_URL: "chrome:\/\/zotero\/content\/zoteroPane.xhtml",' modules/AppConstants.sys.mjs + # Used by OSKeyStore as the master-key label, visible in macOS Keychain Access. + # Verify that OSKeyStore still derives the label from MOZ_APP_BASENAME, so a + # future Mozilla change to a hardcoded string doesn't silently rebrand the + # keychain entry back to "Firefox Encrypted Storage". + replace_line 'MOZ_APP_BASENAME: "Firefox"' 'MOZ_APP_BASENAME: "Zotero"' modules/AppConstants.sys.mjs + check_line 'STORE_LABEL: AppConstants\.MOZ_APP_BASENAME \+ " Encrypted Storage"' modules/OSKeyStore.sys.mjs # https://firefox-source-docs.mozilla.org/toolkit/components/telemetry/internals/preferences.html # @@ -195,15 +201,17 @@ function modify_omni { file="modules/ActorManagerParent.sys.mjs" # Remove deleted actors - remove_between 'AboutTranslations: \{' '^ },' $file - remove_between 'CookieBanner: \{' '^ },' $file - remove_between 'PictureInPictureLauncher: \{' '^ },' $file - remove_between 'PictureInPictureToggle: \{' '^ },' $file - remove_between 'PictureInPicture: \{' '^ },' $file - remove_between 'Thumbnails: \{' '^ },' $file - remove_between 'Translations: \{' '^ },' $file - remove_between 'TranslationsEngine: \{' '^ },' $file - remove_between 'AudioPlayback: \{' '^ },' $file + # Entries in the JSWINDOWACTORS/JSPROCESSACTORS literals: + remove_between '^ AudioPlayback: \{' '^ },' $file + remove_between '^ CookieBanner: \{' '^ },' $file + remove_between '^ Thumbnails: \{' '^ },' $file + remove_between '^ Translations: \{' '^ },' $file + remove_between '^ TranslationsEngine: \{' '^ },' $file + # Entries added conditionally after the literals: + remove_between '^ JSWINDOWACTORS\.AboutTranslations = \{' '^ \};' $file + remove_between '^ JSWINDOWACTORS\.PictureInPictureLauncher = \{' '^ \};' $file + remove_between '^ JSWINDOWACTORS\.PictureInPictureToggle = \{' '^ \};' $file + remove_between '^ JSWINDOWACTORS\.PictureInPicture = \{' '^ \};' $file # Do not trigger LoginManager event that logs an error on autocomplete submission remove_line 'DOMInputPasswordAdded: \{\},' $file @@ -387,8 +395,12 @@ function modify_omni { file="chrome/toolkit/content/mozapps/extensions/aboutaddons.css" echo >> $file - # Hide search bar, Themes and Plugins tabs, and sidebar footer - echo '.main-search, button[name="theme"], button[name="plugin"], sidebar-footer { display: none; }' >> $file + # Hide the search bar and the whole sidebar, since we only ever show plugins and the main pane + # is already headed "Manage Your Plugins" + echo '.main-search, #sidebar { display: none; }' >> $file + # Center the content in the window now that it isn't offset by the sidebar + echo '#full { grid-template-columns: 1fr; }' >> $file + echo '#content { max-width: calc(var(--page-main-content-width) + var(--main-margin-start)); margin-inline: auto; }' >> $file echo '.main-heading { margin-top: 2em; }' >> $file # Hide Details/Permissions tabs in addon details so we only show details echo 'addon-details > button-group { display: none !important; }' >> $file @@ -404,46 +416,46 @@ function modify_omni { echo '.addon-detail-row-homepage .text-link { cursor: pointer; color: LinkText; }' >> $file echo '.addon-detail-row-homepage .text-link:hover { text-decoration: underline; }' >> $file - file="chrome/toolkit/content/mozapps/extensions/aboutaddons.js" + file="chrome/toolkit/content/mozapps/extensions/aboutaddons-utils.mjs" # Hide unsigned-addon warning - replace_line 'if \(!isCorrectlySigned\(addon\)\) \{' 'if (!isCorrectlySigned(addon)) {return {};' $file + replace_line 'export function isUnsignedWarningMessageDisabled\(\) \{' \ + 'export function isUnsignedWarningMessageDisabled() {if (true) return true;' $file + # Use our own localized string for blocked plugin notification + replace_line 'details-notification-hard-blocked-\$\{typeSuffix\}' 'plugins-blocked-plugin' $file + # Hide Recommendations tab in sidebar and recommendations in main pane + replace_line 'function isDiscoverEnabled\(\) \{' 'function isDiscoverEnabled() {return false;' $file + + file="chrome/toolkit/content/mozapps/extensions/components/addon-details.mjs" # Hide Private Browsing setting in addon details replace_line 'pbRow\.' '\/\/pbRow.' $file replace_line 'let isAllowed = await isAllowedInPrivateBrowsing' '\/\/let isAllowed = await isAllowedInPrivateBrowsing' $file - # Use our own strings for the removal prompt - replace_line 'let \{ BrowserAddonUI \} = windowRoot.ownerGlobal;' '' $file - replace_line 'await BrowserAddonUI.promptRemoveExtension' 'promptRemoveExtension' $file - - # Customize empty-list message - replace_line 'createEmptyListMessage\(\) {' 'createEmptyListMessage() { - var p = document.createElement("p"); - p.id = "empty-list-message"; - return p;' $file - # Swap in include.js, which we need for Zotero.getString(), for abuse-reports.js, which we don't need - # Open plugin links in external browser replace_line 'let homepageURL = homepageRow.querySelector\(\"a\"\);' 'let homepageURL = homepageRow.querySelector(\"\.text-link\");' $file replace_line 'homepageURL.href = addon.homepageURL;' 'homepageURL.setAttribute("href", addon.homepageURL);' $file replace_line '<\/a>' \ - '