From 711b3361fe0bceb05bd24372a6019778dc427dd4 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Mon, 14 Jul 2025 03:16:59 -0400 Subject: [PATCH] Support arch-specific builds on Windows and Linux `fetch_xulrunner`, `dir_build`, and `build.sh` now take an optional `-a` argument on Windows and Linux to specify the architecture to fetch/build. If `-a` is omitted for `dir_build` (as in `build_and_run`), it defaults to the current architecture. xulrunner hashes are now computed for each architecture. This changes some filenames in xulrunner/, so you may want to clean out some old folders/files. This removes the custom handling for CI, since it will now default to the architecture of the test runner. --- app/build.sh | 159 ++++++++++++++++++++++++++---------- app/config.sh | 3 + app/scripts/dir_build | 98 ++++++++++++---------- app/scripts/fetch_xulrunner | 92 ++++++++++++++------- app/scripts/utils.sh | 26 ++++++ app/scripts/xulrunner_hash | 93 ++++++++++++++++----- test/runtests.sh | 2 +- 7 files changed, 334 insertions(+), 139 deletions(-) diff --git a/app/build.sh b/app/build.sh index 65caa38810..b353ebb419 100755 --- a/app/build.sh +++ b/app/build.sh @@ -35,12 +35,16 @@ fi function usage { cat >&2 <&2 usage ;; esac done ;; + a) + arch="$OPTARG" + ;; c) UPDATE_CHANNEL="$OPTARG" ;; @@ -118,31 +125,46 @@ while getopts "d:f:p:c:tseq" opt; do shift $((OPTIND-1)); OPTIND=1 done +xul_hash_file() { + local _platform=$1 + local _arch=$2 + + case $_platform in + m) + echo "hash-mac";; + + w) + [[ $_arch == x64 ]] && _arch="win-x64" + [[ $_arch == arm64 ]] && _arch="win-arm64" + echo "hash-${_arch}";; + + l) + [[ $arch == x64 ]] && arch="x86_64" + echo "hash-linux-${_arch}";; + + *) + echo "Invalid platform '$_platform'" 2>&1 + exit 1 + esac +} + function check_xulrunner_hash { - platform=$1 - - if [ $platform == "m" ]; then - platform_hash_file="hash-mac" - elif [ $platform == "w" ]; then - platform_hash_file="hash-win" - elif [ $platform == "l" ]; then - platform_hash_file="hash-linux" - else - echo "Platform parameter incorrect. Acceptable values: m/w/l" - exit 1 - fi - + local _platform=$1 + local _arch=$2 + + local platform_hash_file=$(xul_hash_file "$_platform" "${_arch:-universal}") + if [ ! -e "$CALLDIR/xulrunner/$platform_hash_file" ]; then - echo "xulrunner not found -- downloading" + echo "$platform_hash_file not found -- downloading" echo - $CALLDIR/scripts/fetch_xulrunner -p $platform + $CALLDIR/scripts/fetch_xulrunner -p $_platform ${_arch:+-a $_arch} else - recalculated_xulrunner_hash=$("$CALLDIR/scripts/xulrunner_hash" -p $platform) + recalculated_xulrunner_hash=$("$CALLDIR/scripts/xulrunner_hash" -p $_platform ${_arch:+-a $_arch}) current_xulrunner_hash=$(< "$CALLDIR/xulrunner/$platform_hash_file") - if [ "$current_xulrunner_hash" != "$recalculated_xulrunner_hash" ]; then + if [ "$current_xulrunner_hash" != "$recalculated_xulrunner_hash" ]; then echo "xulrunner hashes don't match -- redownloading" echo - "$CALLDIR/scripts/fetch_xulrunner" -p $platform + "$CALLDIR/scripts/fetch_xulrunner" -p $_platform ${_arch:+-a $_arch} current_xulrunner_hash=$(< "$CALLDIR/xulrunner/$platform_hash_file") if [ "$current_xulrunner_hash" != "$recalculated_xulrunner_hash" ]; then echo "xulrunner hashes don't match after running fetch_xulrunner!" @@ -152,15 +174,36 @@ function check_xulrunner_hash { fi } -#Check if xulrunner and GECKO_VERSION for each platform match +# Derive platform flags from host if none provided +if [[ $BUILD_MAC == 0 ]] && [[ $BUILD_WIN == 0 ]] && [[ $BUILD_LINUX == 0 ]]; then + case "$(uname -s)" in + Darwin) BUILD_MAC=1 ;; + Linux) BUILD_LINUX=1 ;; + CYGWIN*|MINGW*) BUILD_WIN=1 ;; + esac +fi + +# Check xulrunner hashes for requested builds if [ $BUILD_MAC == 1 ]; then check_xulrunner_hash m fi if [ $BUILD_WIN == 1 ]; then - check_xulrunner_hash w + if [[ -n $arch ]]; then + check_xulrunner_hash w $(get_canonical_arch w $arch) + else + for _a in win-x64 win-arm64 win32; do + check_xulrunner_hash w "$_a" + done + fi fi if [ $BUILD_LINUX == 1 ]; then - check_xulrunner_hash l + if [[ -n $arch ]]; then + check_xulrunner_hash l $(get_canonical_arch l $arch) + else + for _a in x64 arm64 i686; do + check_xulrunner_hash l "$_a" + done + fi fi @@ -193,7 +236,7 @@ BUILD_ID=`date +%Y%m%d%H%M%S` # Paths to Gecko runtimes MAC_RUNTIME_PATH="$CALLDIR/xulrunner/Firefox.app" WIN_RUNTIME_PATH_PREFIX="$CALLDIR/xulrunner/firefox-" -LINUX_RUNTIME_PATH_PREFIX="$CALLDIR/xulrunner/firefox-" +LINUX_RUNTIME_PATH_PREFIX="$CALLDIR/xulrunner/firefox-linux-" base_dir="$BUILD_DIR/base" app_dir="$BUILD_DIR/base/app" @@ -221,14 +264,35 @@ set +e if [ $BUILD_MAC == 1 ]; then cp -Rp "$MAC_RUNTIME_PATH"/Contents/Resources/browser/omni "$app_dir" unzip -qj "$MAC_RUNTIME_PATH"/Contents/Resources/omni.ja "hyphenation/*" -d "$app_dir"/hyphenation/ -elif [ $BUILD_WIN == 1 ]; then - # Non-arch-specific files, so just use 64-bit version - cp -Rp "${WIN_RUNTIME_PATH_PREFIX}win-x64"/browser/omni "$app_dir" - unzip -qj "${WIN_RUNTIME_PATH_PREFIX}win-x64"/omni.ja "hyphenation/*" -d "$app_dir"/hyphenation/ -elif [ $BUILD_LINUX == 1 ]; then - # Non-arch-specific files, so just use 64-bit version - cp -Rp "${LINUX_RUNTIME_PATH_PREFIX}x86_64"/browser/omni "$app_dir" - unzip -qj "${LINUX_RUNTIME_PATH_PREFIX}x86_64"/omni.ja "hyphenation/*" -d "$app_dir"/hyphenation/ + +elif [[ $BUILD_WIN == 1 ]]; then + # These are non-arch-specific files, so just find a runtime we fetched: + # - If the caller specified -a, that arch is guaranteed present. + # - Otherwise prefer win-x64 → win-arm64 → win32, falling back to the first + # directory that exists under xulrunner/. + if [[ -n $arch ]]; then + omni_arch=$(get_canonical_arch w $arch) + else + for cand in win-x64 win-arm64 win32; do + [[ -d "${WIN_RUNTIME_PATH_PREFIX}${cand}" ]] && { omni_arch="$cand"; break; } + done + fi + cp -Rp "${WIN_RUNTIME_PATH_PREFIX}${omni_arch}"/browser/omni "$app_dir" + unzip -qj "${WIN_RUNTIME_PATH_PREFIX}${omni_arch}"/omni.ja "hyphenation/*" \ + -d "$app_dir"/hyphenation/ + +elif [[ $BUILD_LINUX == 1 ]]; then + # Same as above + if [[ -n $arch ]]; then + omni_arch=$(get_canonical_arch l $arch) + else + for cand in x86_64 arm64 i686; do + [[ -d "${LINUX_RUNTIME_PATH_PREFIX}${cand}" ]] && { omni_arch="$cand"; break; } + done + fi + cp -Rp "${LINUX_RUNTIME_PATH_PREFIX}${omni_arch}"/browser/omni "$app_dir" + unzip -qj "${LINUX_RUNTIME_PATH_PREFIX}${omni_arch}"/omni.ja "hyphenation/*" \ + -d "$app_dir"/hyphenation/ fi set -e cd $omni_dir @@ -741,10 +805,15 @@ if [ $BUILD_WIN == 1 ]; then fi - for arch in win32 win-x64 win-arm64; do + if [[ -n $arch ]]; then + archs=("$(get_canonical_arch w $arch)"); + else + archs=(win-x64 win-arm64 win32); + fi + for arch in "${archs[@]}"; do echo "Building Zotero_$arch" - runtime_path="${WIN_RUNTIME_PATH_PREFIX}${arch}" + runtime_path="${WIN_RUNTIME_PATH_PREFIX}$arch" # Set up directory APPDIR="$STAGE_DIR/Zotero_$arch" @@ -922,14 +991,14 @@ fi # Linux if [ $BUILD_LINUX == 1 ]; then - # Skip 32-bit build in tests - if [[ "${ZOTERO_TEST:-}" = "1" ]] || [[ "${SKIP_32:-}" = "1" ]]; then - archs="x86_64" + if [[ -n $arch ]]; then + archs=("$(get_canonical_arch w $arch)") else - archs="i686 x86_64 arm64" + archs=(x64 arm64 i686) fi - for arch in $archs; do + [[ $arch == x64 ]] && arch="x86_64" + runtime_path="${LINUX_RUNTIME_PATH_PREFIX}${arch}" # Set up directory diff --git a/app/config.sh b/app/config.sh index 9f975fb68a..55442dba4e 100644 --- a/app/config.sh +++ b/app/config.sh @@ -9,6 +9,9 @@ RUST_VERSION=1.86.0 # URL prefix for custom builds of Firefox components custom_components_url="https://download.zotero.org/dev/firefox-components/" custom_components_hash_mac="" +custom_components_hash_win_x64="" +custom_components_hash_win_arm64="" +custom_components_hash_win32="" APP_NAME="Zotero" APP_ID="zotero\@zotero.org" diff --git a/app/scripts/dir_build b/app/scripts/dir_build index 1bf90415c6..493288ad47 100755 --- a/app/scripts/dir_build +++ b/app/scripts/dir_build @@ -8,37 +8,45 @@ ROOT_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" function usage { cat >&2 <&2 usage ;; esac done ;; + a) + arch="$OPTARG" + ;; q) quick_build=1 ;; @@ -49,40 +57,48 @@ while getopts "tp:q" opt; do esac done -if [[ -z $PLATFORM ]]; then - if [ "`uname`" = "Darwin" ]; then - PLATFORM="m" - elif [ "`uname`" = "Linux" ]; then - PLATFORM="l" - - # If platform not given explicitly, skip 32-bit build if 64-bit system - if [ "$(uname -m)" = "x86_64" ]; then - export SKIP_32=1 - fi - elif [ "`uname -o 2> /dev/null`" = "Cygwin" ]; then - PLATFORM="w" +# Derive platform from host if not provided +if [[ -z $platform ]]; then + case "$(uname -s)" in + Darwin) platform="m" ;; + Linux) platform="l" ;; + CYGWIN*|MINGW*) platform="w" ;; + esac +fi + +# Handle architecture rules +if [[ $platform = "m" ]]; then + # No arch for Mac — ignore anything provided + if [[ -n $arch ]]; then + echo "Ignoring -a $arch for Mac build (universal binary)" >&2 + fi + arch="" +else + # Windows / Linux: derive arch if not supplied + if [[ -z $arch ]]; then + case "$(uname -m)" in + arm64|aarch64) arch="arm64" ;; + x86_64) arch="x64" ;; + i?86) arch="i686" ;; + esac fi fi CHANNEL="source" -PARAMS="" -if [ $DEVTOOLS -eq 1 ]; then - PARAMS+=" -t" -fi -if [ $quick_build -eq 1 ]; then - PARAMS+=" -q" -fi +hash=$(git -C "$ROOT_DIR" rev-parse --short HEAD) -hash=`git -C "$ROOT_DIR" rev-parse --short HEAD` - -build_dir=`mktemp -d` -function cleanup { - rm -rf $build_dir -} +build_dir=$(mktemp -d) +cleanup() { rm -rf "$build_dir"; } trap cleanup EXIT -"$SCRIPT_DIR/prepare_build" -s "$ROOT_DIR/build" -o "$build_dir" -c $CHANNEL -m $hash -"$APP_ROOT_DIR/build.sh" -d "$build_dir" -p $PLATFORM -c $CHANNEL -s $PARAMS +"$SCRIPT_DIR/prepare_build" -s "$ROOT_DIR/build" -o "$build_dir" -c "$CHANNEL" -m "$hash" + +build_cmd=("$APP_ROOT_DIR/build.sh" -d "$build_dir" -p "$platform" -c "$CHANNEL" -s) +[[ -n $arch ]] && build_cmd+=( -a "$arch" ) +[[ $devtools -eq 1 ]] && build_cmd+=( -t ) +[[ $quick_build -eq 1 ]] && build_cmd+=( -q ) + +"${build_cmd[@]}" echo Done diff --git a/app/scripts/fetch_xulrunner b/app/scripts/fetch_xulrunner index bc56e6c338..aedba986ee 100755 --- a/app/scripts/fetch_xulrunner +++ b/app/scripts/fetch_xulrunner @@ -25,11 +25,18 @@ APP_ROOT_DIR="$(dirname "$SCRIPT_DIR")" . "$SCRIPT_DIR/utils.sh" cd "$APP_ROOT_DIR" -function usage { - cat >&2 <&2 <<'DONE' +Usage: fetch_xulrunner.sh -p [-a ARCH] + +Required + -p PLATFORM m (Mac), w (Windows), l (Linux) + +Optional / Conditional + -a ARCH - Mac: disallowed (builds are universal) + - Windows: x64 | arm64 | win32 + - Linux: x64 (or x86_64) | arm64 | i686 + If omitted on Windows/Linux, all standard archs are fetched. DONE exit 1 } @@ -37,22 +44,20 @@ DONE BUILD_MAC=0 BUILD_WIN=0 BUILD_LINUX=0 -while getopts "p:s" opt; do +arch="" +while getopts 'p:a:' opt; do case $opt in p) - for i in `seq 0 1 $((${#OPTARG}-1))` - do + for (( i=0; i<${#OPTARG}; i++ )); do case ${OPTARG:i:1} in m) BUILD_MAC=1;; w) BUILD_WIN=1;; l) BUILD_LINUX=1;; - *) - echo "$0: Invalid platform option ${OPTARG:i:1}" - usage - ;; + *) echo "Invalid platform: ${OPTARG:i:1}"; echo; usage;; esac - done - ;; + done;; + a) arch="$OPTARG";; + *) usage;; esac shift $((OPTIND-1)); OPTIND=1 done @@ -62,6 +67,31 @@ if [[ $BUILD_MAC == 0 ]] && [[ $BUILD_WIN == 0 ]] && [[ $BUILD_LINUX == 0 ]]; th usage fi +# Sanity‑check arch flag vs platform selections +if [[ $BUILD_MAC == 1 ]]; then + [[ -n $arch ]] && { echo "-a is not allowed for Mac" >&2; echo; usage; } +fi +if [[ $BUILD_WIN == 1 || $BUILD_LINUX == 1 ]]; then + # arch optional; if provided validate & map aliases now + if [[ -n $arch ]]; then + if [[ $BUILD_WIN == 1 && $BUILD_LINUX == 1 ]]; then + echo "-a cannot be used when building both Windows and Linux simultaneously" >&2; echo; usage + fi + if [[ $BUILD_WIN == 1 ]]; then + case $arch in + x64) arch="win-x64" ;; + arm64) arch="win-arm64" ;; + win32|win-x64|win-arm64) ;; + *) echo "Invalid Windows arch: $arch" >&2; echo; usage;; + esac + fi + if [[ $BUILD_LINUX == 1 ]]; then + [[ $arch == x64 ]] && arch="x86_64" + case $arch in x86_64|i686|arm64) ;; *) echo "Invalid Linux arch: $arch" >&2; echo; usage;; esac + fi + fi +fi + function get_utf16_chars { str=$(echo -n "$1" | xxd -p | fold -w 2 | sed -r 's/(.+)/\\\\x{\1}\\\\x{00}/') # Add NUL padding @@ -546,7 +576,12 @@ if [ $BUILD_WIN == 1 ]; then GECKO_VERSION="$GECKO_VERSION_WIN" DOWNLOAD_URL="https://ftp.mozilla.org/pub/firefox/releases/$GECKO_VERSION" - for arch in win32 win-x64 win-arm64; do + if [[ -n $arch ]]; then + arch_list=("$(get_canonical_arch w $arch)") + else + arch_list=(win-x64 win-arm64 win32) + fi + for arch in "${arch_list[@]}"; do xdir=firefox-$arch rm -rf $xdir @@ -612,26 +647,21 @@ if [ $BUILD_WIN == 1 ]; then rm "Firefox%20Setup%20$GECKO_VERSION.exe" echo echo + + echo "$( "$SCRIPT_DIR/xulrunner_hash" -p w -a "$arch" )" > "hash-$arch" done - echo $("$SCRIPT_DIR/xulrunner_hash" -p w) > hash-win fi if [ $BUILD_LINUX == 1 ]; then GECKO_VERSION="$GECKO_VERSION_LINUX" DOWNLOAD_URL="https://ftp.mozilla.org/pub/firefox/releases/$GECKO_VERSION" - - - # CI only needs x86_64 - if [[ "${CI:-}" = "1" ]]; then - arches="x86_64" - # Specific architecture (used by dir_build) - elif [[ -n "${ARCH:-}" ]]; then - arches="$ARCH" - # Not specified; fetch all + + if [[ -n $arch ]]; then + arch_list=("$arch") else - arches="i686 x86_64 arm64" + arch_list=(x86_64 arm64 i686) fi - for arch in $arches; do + for arch in "${arch_list[@]}"; do xdir="firefox-$arch" rm -rf $xdir @@ -651,13 +681,15 @@ if [ $BUILD_LINUX == 1 ]; then fi tar xvf firefox-$GECKO_VERSION.tar.xz - mv firefox firefox-$arch + mv firefox firefox-linux-$arch - pushd firefox-$arch + pushd firefox-linux-$arch modify_omni $arch popd - echo $($SCRIPT_DIR/xulrunner_hash -p l) > hash-linux + rm "firefox-$GECKO_VERSION.tar.xz" + + echo "$( $SCRIPT_DIR/xulrunner_hash -p l -a "$arch" )" > "hash-linux-${arch}" done fi diff --git a/app/scripts/utils.sh b/app/scripts/utils.sh index e46f38b7df..215f6ded20 100644 --- a/app/scripts/utils.sh +++ b/app/scripts/utils.sh @@ -13,6 +13,32 @@ get_current_platform() { fi } +get_canonical_arch() { + local _platform=$1 + local _arch=$2 + + case $_platform in + w) + case $_arch in + x64) _arch="win-x64" ;; + arm64) _arch="win-arm64" ;; + win32|win-x64|win-arm64) ;; + *) echo "Invalid Windows archicture: $_arch" >&2;; + esac + echo $_arch + ;; + + l) + [[ $_arch == x64 ]] && _arch="x86_64" + echo $_arch + ;; + + *) + echo "Invalid platform '$platform'" 2>&1 + exit 1 + esac +} + function check_line { pattern=$1 if ! egrep -q "$pattern" "$file"; then diff --git a/app/scripts/xulrunner_hash b/app/scripts/xulrunner_hash index c149d0775d..9daf3ce2b8 100755 --- a/app/scripts/xulrunner_hash +++ b/app/scripts/xulrunner_hash @@ -1,4 +1,15 @@ #!/bin/bash +# Calculates a hash for the processed Firefox package for a given platform and architecture +# +# Usage: +# xulrunner_hash.sh -p [-a ARCH] +# +# * **Mac (m)** ‑a is disallowed (universal build). +# * **Windows (w)** ‑a is **required**. Accepted values: +# win32 | x64 | arm64 +# * **Linux (l)** ‑a is **required**. Accepted values: +# x86_64 | i686 | arm64 (alias x64 → x86_64) + set -euo pipefail SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" @@ -6,40 +17,78 @@ APP_ROOT_DIR="$(dirname "$SCRIPT_DIR")" . "$APP_ROOT_DIR/config.sh" cd "$APP_ROOT_DIR" -platform="" -while getopts ":p:" opt; do +platform=""; arch="" + +usage() { + cat >&2 <<'DONE' +Usage: xulrunner_hash.sh -p [-a ARCH] + + -p PLATFORM m (mac), w (windows), l (linux) [required] + -a ARCH *required* for Windows & Linux builds + Windows: x64 | arm64 | win32 + Linux: x64 (or x86_64) | arm64 | i686 + Mac: disallowed – always universal +DONE + exit 1 +} + +while getopts ':p:a:' opt; do case $opt in - p) - platform="$OPTARG" - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "Option -$OPTARG requires an argument." >&2 - exit 1 - ;; + p) platform="$OPTARG";; + a) arch="$OPTARG";; + :) echo "Option -$OPTARG requires an argument" >&2; usage;; + ? ) echo "Invalid option: -$OPTARG" >&2; echo; usage;; esac + shift $((OPTIND-1)); OPTIND=1 done +[[ -z $platform ]] && { echo "-p is required" >&2; echo; usage; } -if [ $platform == "m" ]; then +case $platform in + m) + [[ -n $arch ]] && { echo "-a is not allowed for Mac (universal build)" >&2; echo; usage; } + arch="universal" GECKO_VERSION="$GECKO_VERSION_MAC" -elif [ $platform == "w" ]; then + ;; + w) + [[ -z $arch ]] && { echo "-a is required for Windows builds" >&2; echo; usage; } + # Map CLI-friendly aliases → canonical form + case $arch in + x64) arch="win-x64" ;; + arm64) arch="win-arm64" ;; + win32|win-x64|win-arm64) ;; # already canonical + *) echo "Invalid Windows arch: $arch" >&2; echo; usage;; + esac GECKO_VERSION="$GECKO_VERSION_WIN" -elif [ $platform == "l" ]; then + ;; + l) + [[ -z $arch ]] && { echo "-a is required for Linux builds" >&2; echo; usage; } + [[ $arch == x64 ]] && arch="x86_64" + case $arch in + x86_64|i686|arm64) ;; + *) echo "Invalid Linux arch: $arch" >&2; echo; usage;; + esac GECKO_VERSION="$GECKO_VERSION_LINUX" -else - echo "Platform parameter incorrect. Usage: -p m(mac)/w(windows)/l(linux)" - exit 1 -fi + ;; + *) + echo "Invalid platform: $platform" >&2; echo; usage;; +esac + +# Concatenate scripts that affect the build xulrunner_content=$(< "$SCRIPT_DIR/fetch_xulrunner") xulrunner_content+=$(< "$SCRIPT_DIR/utils.sh") xulrunner_content+=$(< "$APP_ROOT_DIR/assets/multilocale.txt") -xulrunner_gecko_hash=$(echo -n "$GECKO_VERSION - '$xulrunner_content'" | openssl dgst -sha256) -echo "$xulrunner_gecko_hash" +# Get custom components hash +if [[ $platform = "m" ]]; then + components_hash="$custom_components_hash_mac" +else + hash_arch_name="${arch/-/_}" + arch_var_name="custom_components_hash_${hash_arch_name}" + components_hash=${!arch_var_name:-} +fi +input_string="${platform}-${arch}-${components_hash}|${GECKO_VERSION}|${xulrunner_content}" +printf "%s" "$input_string" | openssl dgst -sha256 | awk '{print $2}' diff --git a/test/runtests.sh b/test/runtests.sh index a7dcd6b43f..4f0fadf0a5 100755 --- a/test/runtests.sh +++ b/test/runtests.sh @@ -166,7 +166,7 @@ if [[ -z "$CI" ]] && ! ps | grep js-build/build.js | grep -v grep > /dev/null; t echo fi -ZOTERO_TEST=1 "$ROOT_DIR/app/scripts/dir_build" -q +"$ROOT_DIR/app/scripts/dir_build" -q makePath FX_PROFILE "$PROFILE" MOZ_NO_REMOTE=1 NO_EM_RESTART=1 "$Z_EXECUTABLE" -profile "$FX_PROFILE" \