mirror of
https://github.com/zotero/zotero.git
synced 2026-09-12 23:01:22 +00:00
39a1c56f6c
Cache patches and compressed files by content hash so that they're reused across FROM versions and releases. Cache entries are touched when used and expire after 60 days. The cache key includes a tag covering the xz/mbsdiff versions and the compression options, so entries are invalidated when the toolchain changes.
450 lines
14 KiB
Bash
450 lines
14 KiB
Bash
#!/bin/bash
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
#
|
|
# Code shared by update packaging scripts.
|
|
# Author: Darin Fisher
|
|
#
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Preserve a value set by the parent so it survives in the child processes
|
|
# spawned by run_parallel_tasks()
|
|
QUIET=${QUIET:-0}
|
|
|
|
# By default just assume that these tools exist on our path
|
|
MAR=${MAR:-mar}
|
|
MBSDIFF=${MBSDIFF:-mbsdiff}
|
|
XZ=${XZ:-xz}
|
|
$XZ --version > /dev/null 2>&1
|
|
if [ $? -ne 0 ]; then
|
|
# If $XZ is not set and not found on the path then this is probably
|
|
# running on a windows buildbot. Some of the Windows build systems have
|
|
# xz.exe in topsrcdir/xz/. Look in the places this would be in both a
|
|
# mozilla-central and comm-central build.
|
|
XZ="$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe"
|
|
$XZ --version > /dev/null 2>&1
|
|
if [ $? -ne 0 ]; then
|
|
XZ="$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe"
|
|
$XZ --version > /dev/null 2>&1
|
|
if [ $? -ne 0 ]; then
|
|
echo "xz was not found on this system!"
|
|
echo "exiting"
|
|
exit 1
|
|
fi
|
|
fi
|
|
fi
|
|
# Ensure that we're always using the right compression settings.
|
|
# -T1 (single-threaded) is required: multi-threaded xz produces non-deterministic
|
|
# output, which would break both the reproducibility of MARs and the content-hash
|
|
# cache in process_update_task(). See get_cache_tag() for how changes here are
|
|
# accounted for in the cache.
|
|
export XZ_OPT="-T1 -7e"
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Helper routines
|
|
|
|
notice() {
|
|
echo "$*" 1>&2
|
|
}
|
|
|
|
verbose_notice() {
|
|
if [ $QUIET -eq 0 ]; then
|
|
notice "$*"
|
|
fi
|
|
}
|
|
|
|
get_file_size() {
|
|
info=($(ls -ln "$1"))
|
|
echo ${info[4]}
|
|
}
|
|
|
|
copy_perm() {
|
|
reference="$1"
|
|
target="$2"
|
|
|
|
if [ -x "$reference" ]; then
|
|
chmod 0755 "$target"
|
|
else
|
|
chmod 0644 "$target"
|
|
fi
|
|
}
|
|
|
|
make_add_instruction() {
|
|
f="$1"
|
|
filev3="$2"
|
|
|
|
# Used to log to the console
|
|
if [ $4 ]; then
|
|
forced=" (forced)"
|
|
else
|
|
forced=
|
|
fi
|
|
|
|
# Changed by Zotero for -e
|
|
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') || true
|
|
if [ $is_extension = "1" ]; then
|
|
# Use the subdirectory of the extensions folder as the file to test
|
|
# before performing this add instruction.
|
|
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
|
|
verbose_notice " add-if \"$testdir\" \"$f\""
|
|
echo "add-if \"$testdir\" \"$f\"" >> "$filev3"
|
|
else
|
|
verbose_notice " add \"$f\"$forced"
|
|
echo "add \"$f\"" >> "$filev3"
|
|
fi
|
|
}
|
|
|
|
check_for_add_if_not_update() {
|
|
add_if_not_file_chk="$1"
|
|
|
|
if [[ "$(basename "$add_if_not_file_chk")" = "channel-prefs.js" || \
|
|
"$add_if_not_file_chk" =~ (^|/)ChannelPrefs\.framework/ || \
|
|
"$(basename "$add_if_not_file_chk")" = "update-settings.ini" || \
|
|
"$add_if_not_file_chk" =~ (^|/)UpdateSettings\.framework/ ]]; then
|
|
## "true"
|
|
return 0;
|
|
fi
|
|
## "false"
|
|
return 1;
|
|
}
|
|
|
|
make_add_if_not_instruction() {
|
|
f="$1"
|
|
filev3="$2"
|
|
|
|
verbose_notice " add-if-not \"$f\" \"$f\""
|
|
echo "add-if-not \"$f\" \"$f\"" >> "$filev3"
|
|
}
|
|
|
|
make_patch_instruction() {
|
|
f="$1"
|
|
filev3="$2"
|
|
|
|
# Changed by Zotero for -e
|
|
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') || true
|
|
if [ $is_extension = "1" ]; then
|
|
# Use the subdirectory of the extensions folder as the file to test
|
|
# before performing this add instruction.
|
|
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
|
|
verbose_notice " patch-if \"$testdir\" \"$f.patch\" \"$f\""
|
|
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev3"
|
|
else
|
|
verbose_notice " patch \"$f.patch\" \"$f\""
|
|
echo "patch \"$f.patch\" \"$f\"" >> "$filev3"
|
|
fi
|
|
}
|
|
|
|
append_remove_instructions() {
|
|
dir="$1"
|
|
filev3="$2"
|
|
|
|
if [ -f "$dir/removed-files" ]; then
|
|
listfile="$dir/removed-files"
|
|
elif [ -f "$dir/Contents/Resources/removed-files" ]; then
|
|
listfile="$dir/Contents/Resources/removed-files"
|
|
fi
|
|
if [ -n "$listfile" ]; then
|
|
# Changed by Zotero: Use subshell and disable filename globbing to prevent bash from expanding
|
|
# entries in removed-files with paths from the root (e.g., 'xulrunner/*')
|
|
(
|
|
set -f
|
|
# Map spaces to pipes so that we correctly handle filenames with spaces.
|
|
files=($(cat "$listfile" | tr " " "|" | sort -r))
|
|
num_files=${#files[*]}
|
|
for ((i=0; $i<$num_files; i=$i+1)); do
|
|
# Map pipes back to whitespace and remove carriage returns
|
|
f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
|
|
# Trim whitespace
|
|
f=$(echo $f)
|
|
# Exclude blank lines.
|
|
if [ -n "$f" ]; then
|
|
# Exclude comments
|
|
if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
|
|
if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
|
|
verbose_notice " rmdir \"$f\""
|
|
echo "rmdir \"$f\"" >> "$filev3"
|
|
elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
|
|
# Remove the *
|
|
f=$(echo "$f" | sed -e 's:\*$::')
|
|
verbose_notice " rmrfdir \"$f\""
|
|
echo "rmrfdir \"$f\"" >> "$filev3"
|
|
else
|
|
verbose_notice " remove \"$f\""
|
|
echo "remove \"$f\"" >> "$filev3"
|
|
fi
|
|
fi
|
|
fi
|
|
done
|
|
)
|
|
fi
|
|
}
|
|
|
|
# Return the number of parallel jobs to use for diffing/compression, from
|
|
# UPDATE_PACKAGING_JOBS or the number of cores
|
|
get_parallel_jobs() {
|
|
if [ -n "${UPDATE_PACKAGING_JOBS:-}" ]; then
|
|
echo "$UPDATE_PACKAGING_JOBS"
|
|
return 0
|
|
fi
|
|
nproc 2>/dev/null && return 0
|
|
sysctl -n hw.ncpu 2>/dev/null && return 0
|
|
echo 1
|
|
}
|
|
|
|
# Run per-file tasks from a task file in parallel by re-invoking the calling
|
|
# script with --run-task for each line. Tasks are ordered largest-file-first so
|
|
# that long-running jobs (e.g., diffing xul) start as early as possible.
|
|
#
|
|
# $1 - task file, with one "<type><TAB><relative path>" task per line
|
|
# $2 - directory to measure file sizes against for job ordering
|
|
# $3 - script to re-invoke (callers pass "$0" and handle --run-task before
|
|
# option parsing by calling process_update_task)
|
|
#
|
|
# Callers must export the variables needed by process_update_task().
|
|
run_parallel_tasks() {
|
|
local taskfile="$1"
|
|
local basedir="$2"
|
|
local script="$3"
|
|
local jobs
|
|
|
|
if [ ! -s "$taskfile" ]; then
|
|
return 0
|
|
fi
|
|
|
|
# xargs requires a resolvable command path
|
|
script="$(cd "$(dirname "$script")" && pwd)/$(basename "$script")"
|
|
|
|
# Compute the cache tag once here and export it so the per-file child
|
|
# processes inherit it rather than recomputing it for every file
|
|
if [ -n "${UPDATE_CACHE_DIR:-}" ] && [ -z "${UPDATE_CACHE_TAG:-}" ]; then
|
|
UPDATE_CACHE_TAG="$(get_cache_tag)"
|
|
export UPDATE_CACHE_TAG
|
|
fi
|
|
|
|
jobs=$(get_parallel_jobs)
|
|
notice "Processing $(wc -l < "$taskfile" | tr -d ' ') files with $jobs parallel jobs"
|
|
|
|
(cd "$basedir" && du -ak .) > "$taskfile.sizes"
|
|
awk -F'\t' 'NR==FNR { sizes[substr($2, 3)] = $1; next } { print sizes[$2] "\t" $0 }' \
|
|
"$taskfile.sizes" "$taskfile" \
|
|
| sort -rn \
|
|
| cut -f 2- \
|
|
| tr '\n' '\000' \
|
|
| xargs -0 -n 1 -P "$jobs" "$script" --run-task
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Diff and compression cache
|
|
#
|
|
# Building updates repeats the same expensive work: the same file contents get
|
|
# xz-compressed once per architecture, per FROM version, and per release, and
|
|
# the same before/after file pair gets binary-diffed again and again. When
|
|
# UPDATE_CACHE_DIR is set, we cache the results by content hash and reuse them
|
|
# across builds. Two kinds of entry are stored:
|
|
#
|
|
# full-<tag>-<hash>.xz A single file compressed with xz, keyed
|
|
# on the file's contents. Reused whenever
|
|
# the same content shows up again, in any
|
|
# architecture, version, or release.
|
|
#
|
|
# patch-<tag>-<oldhash>-<newhash>.xz The compressed binary diff between a
|
|
# file's old and new contents, keyed on
|
|
# both. Reused whenever the same
|
|
# before/after pair recurs (e.g., a file
|
|
# unchanged across several FROM versions
|
|
# being patched to the same TO).
|
|
#
|
|
# Both entries are written even when only one is ultimately packaged, so a file
|
|
# that loses the patch-vs-full size comparison in one build still seeds the
|
|
# cache for the next. <tag> is get_cache_tag(), so a toolchain/settings change
|
|
# starts a fresh namespace rather than reusing stale output. Entries are touched
|
|
# on use and expire via the time-based cleanup in build_autoupdate.sh.
|
|
# -----------------------------------------------------------------------------
|
|
|
|
# Print a SHA-256 content hash to use as a cache key
|
|
hash_file() {
|
|
if command -v sha256sum > /dev/null 2>&1; then
|
|
sha256sum "$1" | awk '{print $1}'
|
|
else
|
|
shasum -a 256 "$1" | awk '{print $1}'
|
|
fi
|
|
}
|
|
|
|
# Print a SHA-256 hash of stdin
|
|
hash_stdin() {
|
|
if command -v sha256sum > /dev/null 2>&1; then
|
|
sha256sum | awk '{print $1}'
|
|
else
|
|
shasum -a 256 | awk '{print $1}'
|
|
fi
|
|
}
|
|
|
|
# Print a short tag identifying the toolchain and options that produce cached
|
|
# files, so that entries are invalidated when any of them change. The cache
|
|
# keys files by input content only, on the assumption that the same inputs
|
|
# always compress/diff to the same output; this tag makes that assumption hold
|
|
# across xz/mbsdiff upgrades and XZ_OPT/BCJ_OPTIONS changes. (A change to any
|
|
# input invalidates both the full and patch caches, which is more than strictly
|
|
# necessary but keeps this simple; a stale build is just a cold rebuild.)
|
|
# Stale-tag entries are removed by the time-based cleanup in build_autoupdate.sh.
|
|
get_cache_tag() {
|
|
local mbsdiff_bin
|
|
mbsdiff_bin="$(command -v "$MBSDIFF" 2>/dev/null)"
|
|
{
|
|
$XZ --version 2>/dev/null | head -1
|
|
echo "XZ_OPT=$XZ_OPT"
|
|
echo "BCJ_OPTIONS=${BCJ_OPTIONS:-}"
|
|
# mbsdiff has no --version, so identify it by its binary contents
|
|
if [ -n "$mbsdiff_bin" ]; then
|
|
hash_file "$mbsdiff_bin"
|
|
else
|
|
echo "MBSDIFF=$MBSDIFF"
|
|
fi
|
|
} | hash_stdin | cut -c 1-16
|
|
}
|
|
|
|
# Atomically store file $2 in the cache as $1
|
|
cache_store() {
|
|
mkdir -p "$(dirname "$1")"
|
|
cp "$2" "$1.tmp.$$" && mv -f "$1.tmp.$$" "$1"
|
|
}
|
|
|
|
# xz-compress $1 to $2, reusing a cached result when one exists (see the cache
|
|
# overview above)
|
|
compress_full_file() {
|
|
local src="$1"
|
|
local dest="$2"
|
|
local key=""
|
|
|
|
if [ -n "${UPDATE_CACHE_DIR:-}" ]; then
|
|
key="$UPDATE_CACHE_DIR/full-${UPDATE_CACHE_TAG:-}-$(hash_file "$src").xz"
|
|
if [ -f "$key" ]; then
|
|
# Keep entries that are still in use from expiring
|
|
touch -c "$key" 2>/dev/null || true
|
|
cp "$key" "$dest"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$src" > "$dest"
|
|
|
|
if [ -n "$key" ]; then
|
|
cache_store "$key" "$dest"
|
|
fi
|
|
}
|
|
|
|
# Process a single task line from run_parallel_tasks():
|
|
#
|
|
# full - compress the new file into the work directory
|
|
# diff - if the file changed, generate a binary diff and keep the smaller of
|
|
# the compressed patch ($f.patch) and the compressed file ($f)
|
|
#
|
|
# Expects newdir and workdir in the environment, plus olddir for diff tasks.
|
|
# The results are assembled into the manifest serially by the caller based on
|
|
# which files exist in the work directory. Diffs and compressed files are
|
|
# cached when enabled -- see the cache overview above.
|
|
process_update_task() {
|
|
local task="$1"
|
|
local tab=$(printf '\t')
|
|
local type="${task%%"$tab"*}"
|
|
local f="${task#*"$tab"}"
|
|
local oldfile_path newfile_path patch_path patchsize fullsize full_pid patchkey
|
|
|
|
if [ "$type" = "diff" ]; then
|
|
if diff "$olddir/$f" "$newdir/$f" > /dev/null; then
|
|
# Unchanged
|
|
return 0
|
|
fi
|
|
|
|
mkdir -p "$(dirname "$workdir/$f")"
|
|
verbose_notice "diffing \"$f\""
|
|
|
|
# Compress the full new file in the background while diffing, since both
|
|
# are needed to decide which to package
|
|
compress_full_file "$newdir/$f" "$workdir/$f" &
|
|
full_pid=$!
|
|
|
|
patchkey=""
|
|
if [ -n "${UPDATE_CACHE_DIR:-}" ]; then
|
|
patchkey="$UPDATE_CACHE_DIR/patch-${UPDATE_CACHE_TAG:-}-$(hash_file "$olddir/$f")-$(hash_file "$newdir/$f").xz"
|
|
fi
|
|
if [ -n "$patchkey" ] && [ -f "$patchkey" ]; then
|
|
# Keep entries that are still in use from expiring
|
|
touch -c "$patchkey" 2>/dev/null || true
|
|
cp "$patchkey" "$workdir/$f.patch.xz"
|
|
else
|
|
# mbsdiff doesn't like POSIX paths on Windows
|
|
if [ ${WIN_NATIVE:-0} -eq 1 ]; then
|
|
oldfile_path=$(cygpath -m "$olddir/$f")
|
|
newfile_path=$(cygpath -m "$newdir/$f")
|
|
patch_path=$(cygpath -m "$workdir/$f.patch")
|
|
else
|
|
oldfile_path="$olddir/$f"
|
|
newfile_path="$newdir/$f"
|
|
patch_path="$workdir/$f.patch"
|
|
fi
|
|
$MBSDIFF "$oldfile_path" "$newfile_path" "$patch_path"
|
|
$XZ $XZ_OPT --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
|
|
if [ -n "$patchkey" ]; then
|
|
cache_store "$patchkey" "$workdir/$f.patch.xz"
|
|
fi
|
|
fi
|
|
|
|
wait $full_pid
|
|
copy_perm "$newdir/$f" "$workdir/$f"
|
|
|
|
patchsize=$(get_file_size "$workdir/$f.patch.xz")
|
|
fullsize=$(get_file_size "$workdir/$f")
|
|
|
|
if [ $patchsize -lt $fullsize ]; then
|
|
mv -f "$workdir/$f.patch.xz" "$workdir/$f.patch"
|
|
rm -f "$workdir/$f"
|
|
else
|
|
rm -f "$workdir/$f.patch.xz"
|
|
fi
|
|
else
|
|
mkdir -p "$(dirname "$workdir/$f")"
|
|
compress_full_file "$newdir/$f" "$workdir/$f"
|
|
copy_perm "$newdir/$f" "$workdir/$f"
|
|
fi
|
|
}
|
|
|
|
# List all files in the current directory, stripping leading "./"
|
|
# Pass a variable name and it will be filled as an array.
|
|
list_files() {
|
|
count=0
|
|
temp_filelist=$(mktemp)
|
|
find . -type f \
|
|
! -name "update.manifest" \
|
|
! -name "updatev2.manifest" \
|
|
! -name "updatev3.manifest" \
|
|
| sed 's/\.\/\(.*\)/\1/' \
|
|
| sort -r > "${temp_filelist}"
|
|
while read file; do
|
|
eval "${1}[$count]=\"$file\""
|
|
# Changed for Zotero to avoid eval as 1
|
|
#(( count++ ))
|
|
(( ++count ))
|
|
done < "${temp_filelist}"
|
|
rm "${temp_filelist}"
|
|
}
|
|
|
|
# List all directories in the current directory, stripping leading "./"
|
|
list_dirs() {
|
|
count=0
|
|
temp_dirlist=$(mktemp)
|
|
find . -type d \
|
|
! -name "." \
|
|
! -name ".." \
|
|
| sed 's/\.\/\(.*\)/\1/' \
|
|
| sort -r > "${temp_dirlist}"
|
|
while read dir; do
|
|
eval "${1}[$count]=\"$dir\""
|
|
# Changed for Zotero
|
|
#(( count++ ))
|
|
(( ++count ))
|
|
done < "${temp_dirlist}"
|
|
rm "${temp_dirlist}"
|
|
}
|