mirror of
https://github.com/fish-shell/fish-shell.git
synced 2024-11-22 11:47:04 +08:00
Release automation script
We provide releases in several different formats and channels. There are many manual steps involved in making a release. This makes it hard to learn or share improvements (especially share improvements that last once-and-for-all). Also there is potential for error -- for example our Debian PPAs don't include the latest release and our website doesn't link to the latest tarball. With our current approach, packaging tends to get more complicated, for example we'll want to add things like "Depends: librust-hexponent" to our Debian package, and set the version based on Cargo.toml. Add a script to automate the boring steps of the release process. This should obsolete most of https://github.com/fish-shell/fish-shell/wiki/Release-checklist It seems to work on my Linux system but it's obviously not battle-tested yet. The most essential TODO comments ones are for uploading Debian packages. Iteration on the script itself is usually quite fast. I run something like git add build_tools/release.sh git commit -m wip build_tools/release.sh 3.8.0 --dry-run --no-test # plus eventual --no-rebuild --no-rebuild-debs As a temporary hack, the Debian packages are currently built in a Docker container. It'd be easy to skip Docker when debmake is installed on the host system. We don't yet pass through gpg-agent into Docker, so the debuild process requires interactive password input. It takes a really long time (> 15 minutes) to build all Debian packages because we rebuild them sequentially for all Ubuntu releases (mantic, jammy, focal, bionic and xenial) even though only "debian/changelog" is different. For iterating on the script, we can use "--no-rebuild-debs" to reuse the the packages after they've been built once. Of course this is only valid if the tarball has not changed since. We could probably ask debuild to validate this, or speed up the process in a different way, at least parallelize. Future ideas: - we could run the script in CI instead, to keep us even more honest. - we should try to simplify the website release process (should not need to talk to the github API, the script already has enough information). - do something similar for the the macOS-specific parts
This commit is contained in:
Johannes Altmanninger
parent
aa26546088
commit
e5e8670bcb
|
|
@ -18,6 +18,10 @@ trim_trailing_whitespace = false
|
|||
[*.{sh,ac}]
|
||||
indent_size = 2
|
||||
|
||||
[{build_tools/release.sh,build_tools/release/*.sh}]
|
||||
indent_size = 4
|
||||
max_line_length = 72
|
||||
|
||||
[Dockerfile]
|
||||
indent_size = 2
|
||||
|
||||
|
|
|
|||
273
build_tools/release.sh
Executable file
273
build_tools/release.sh
Executable file
|
|
@ -0,0 +1,273 @@
|
|||
#!/bin/sh
|
||||
|
||||
{
|
||||
|
||||
set -ex
|
||||
|
||||
usage() {
|
||||
cat << EOF
|
||||
Usage: [FISH_GPG_KEY_ID=<KEY>] $(basename $0) [OPTIONS] <version>
|
||||
|
||||
Build and publish a release with the given version.
|
||||
|
||||
Release artifacts will be left behind in "./release" for debugging.
|
||||
|
||||
On failure, an attempted is made to restore the state.
|
||||
Same if --dry-run is given, even if there was no failure.
|
||||
|
||||
$FISH_GPG_KEY_ID defaults to your Git author identity.
|
||||
|
||||
Options:
|
||||
-n, --dry-run Do not publish the release
|
||||
--no-test Skip running tests.
|
||||
--no-rebuild Do not rebuild tarball
|
||||
--no-rebuild-debs Do not rebuild Debian packages
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
build_debian_packages() {
|
||||
dockerfile=$PWD/build_tools/release/build-debian-package.Dockerfile
|
||||
|
||||
export DOCKER_BUILDKIT=1
|
||||
|
||||
# Construct a docker image.
|
||||
img_tagname="ghcr.io/fish-shell/fish-ci/$(basename -s .Dockerfile "$dockerfile"):latest"
|
||||
sudo=sudo
|
||||
if groups | grep -q '\bdocker\b'; then
|
||||
sudo=
|
||||
fi
|
||||
$sudo docker build \
|
||||
-t "$img_tagname" \
|
||||
-f "$dockerfile" \
|
||||
"$PWD"/build_tools/release/
|
||||
|
||||
tmpdir=$(mktemp -d)
|
||||
$sudo docker run -it \
|
||||
--cidfile="$tmpdir/container_id" \
|
||||
--mount type=bind,source="$PWD",target=/fish-source,readonly \
|
||||
--mount type=bind,source="${GNUPGHOME:-"$HOME/.gnupg"}",target=/gnupg,readonly \
|
||||
--env "FISH_COMMITTER=$committer" \
|
||||
--env "FISH_GPG_KEY_ID=$gpg_key_id" \
|
||||
--env "FISH_VERSION=$version" \
|
||||
"$img_tagname"
|
||||
|
||||
container_id=$(cat "$tmpdir/container_id")
|
||||
|
||||
$sudo docker cp "$container_id":/home/fishuser/dpkgs release/dpkgs
|
||||
$sudo chown -R $USER release/dpkgs
|
||||
$sudo docker rm "$container_id"
|
||||
$sudo rm -r "$tmpdir"
|
||||
}
|
||||
|
||||
committer="$(git var GIT_AUTHOR_IDENT)"
|
||||
committer=${committer% *} # strip timezone
|
||||
committer=${committer% *} # strip timestamp
|
||||
gpg_key_id=${FISH_GPG_KEY_ID:-"$committer"}
|
||||
|
||||
# TODO Check for user errors here (in case HEAD is not pushed yet).
|
||||
is_latest_release=false
|
||||
if git merge-base --is-ancestor origin/master HEAD; then
|
||||
is_latest_release=true
|
||||
fi
|
||||
|
||||
version=
|
||||
dry_run=
|
||||
do_test=true
|
||||
do_rebuild=true
|
||||
do_rebuild_debs=true
|
||||
for arg
|
||||
do
|
||||
case "$arg" in
|
||||
(-n|--dry-run) dry_run=: ;;
|
||||
(--no-test) do_test=false ;;
|
||||
(--no-rebuild) do_rebuild=false ;;
|
||||
(--no-rebuild-debs) do_rebuild_debs=false ;;
|
||||
(-*) usage ;;
|
||||
(*)
|
||||
if [ -n "$version" ]; then
|
||||
usage
|
||||
fi
|
||||
version=$arg
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$version" ]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
for tool in \
|
||||
gh \
|
||||
gpg \
|
||||
osc \
|
||||
pip \
|
||||
sphinx-build \
|
||||
virtualenv \
|
||||
; do
|
||||
if ! command -v "$tool" >/dev/null; then
|
||||
echo >&2 "$0: missing command: $1"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if ! echo "$version" | grep '^3\.'; then
|
||||
echo >&2 "$0: major version bump needs changes to PPA code"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -e release ] && $do_rebuild ; then
|
||||
echo >&2 "$0: ./release exists, please remove it or pass --no-rebuild to reuse the tarball"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for repo in . ../fish-site
|
||||
do
|
||||
if ! git -C "$repo" diff HEAD --quiet; then
|
||||
echo >&2 "$0: index and worktree must be clean"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if git tag | grep -qxF "$version"; then
|
||||
echo >&2 "$0: tag $version already exists"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cleanup_done=false
|
||||
did_commit=false
|
||||
cleanup_after_error_or_dry_run() {
|
||||
printf '\n\n'
|
||||
if $cleanup_done; then
|
||||
return
|
||||
fi
|
||||
cleanup_done=true
|
||||
{
|
||||
set +e
|
||||
if $did_commit; then
|
||||
# Try to only undo our changelog changes.
|
||||
git revert --no-edit HEAD &&
|
||||
git reset HEAD~2
|
||||
fi
|
||||
git tag -d $version
|
||||
exit 1
|
||||
} >/dev/null
|
||||
}
|
||||
trap cleanup_after_error_or_dry_run INT EXIT
|
||||
|
||||
if $do_test; then
|
||||
ninja -Cbuild test
|
||||
fi
|
||||
|
||||
sed -i "1cfish $version (released $(date +'%B %d, %Y'))" CHANGELOG.rst
|
||||
git add CHANGELOG.rst
|
||||
git commit -m "Release $version"
|
||||
did_commit=true # TODO This is hacky.
|
||||
git -c "user.signingKey=$gpg_key_id" tag -a --message="Release $version" $version -s
|
||||
|
||||
if ! [ -d release ] || $do_rebuild; then
|
||||
mkdir -p release
|
||||
FISH_ARTEFACT_PATH=$PWD/release build_tools/make_tarball.sh
|
||||
gpg --local-user "$gpg_key_id" --sign --detach --armor "release/fish-$version.tar.xz"
|
||||
fi
|
||||
|
||||
if ! [ -d release/dpkgs ] || $do_rebuild_debs; then
|
||||
build_debian_packages
|
||||
fi
|
||||
|
||||
rm -rf release/obs
|
||||
mkdir -p release/obs
|
||||
(
|
||||
cd release/obs
|
||||
osc checkout shells:fish:release:3/fish
|
||||
cd shells:fish:release:3/fish
|
||||
rm -f *.tar.* *.dsc
|
||||
)
|
||||
ln release/dpkgs/fish_$version.orig.tar.xz release/obs/shells:fish:release:3/fish
|
||||
ln release/dpkgs/fish_$version-1.debian.tar.xz release/obs/shells:fish:release:3/fish
|
||||
ln release/dpkgs/fish_$version-1.dsc release/obs/shells:fish:release:3/fish
|
||||
rpmversion=$(echo $version | sed -e 's/-/+/' -e 's/-/./g')
|
||||
sed -e "s/@version@/$version/g" -e "s/@RPMVERSION@/$rpmversion/g" \
|
||||
< "release/dpkgs/fish-$version/fish.spec.in" > release/obs/shells:fish:release:3/fish/fish.spec
|
||||
|
||||
(
|
||||
cd release/obs/shells:fish:release:3/fish
|
||||
osc addremove
|
||||
)
|
||||
|
||||
relnotes_tmp=$(mktemp -d)
|
||||
(
|
||||
virtualenv release/.venv
|
||||
. release/.venv/bin/activate
|
||||
pip install sphinx-markdown-builder
|
||||
# We only need to build latest relnotes. I'm not sure if that's
|
||||
# possible, so resort to building everything.
|
||||
mkdir $relnotes_tmp/src $relnotes_tmp/out
|
||||
sphinx-build -j 8 -b markdown -n "release/dpkgs/fish-$version/doc_src" \
|
||||
-d "release/dpkgs/fish-$version/user_doc/doctrees" \
|
||||
$relnotes_tmp/markdown
|
||||
# Delete title
|
||||
sed -i 1,3d "$relnotes_tmp/markdown/relnotes.md"
|
||||
# Delete notes for prior releases.
|
||||
sed -i '/^## /,$d' "$relnotes_tmp/markdown/relnotes.md"
|
||||
)
|
||||
|
||||
if ! [ -n $dry_run ]; then
|
||||
trap '' INT EXIT
|
||||
fi
|
||||
$dry_run git push origin "$version" # Push the tag
|
||||
(
|
||||
cd release/obs
|
||||
$dry_run osc commit -m "New release: $version"
|
||||
)
|
||||
$dry_run gh release create --draft \
|
||||
--title=$(head -n 1 "release/dpkgs/fish-$version/CHANGELOG.rst") \
|
||||
--notes-file="$relnotes_tmp/markdown/relnotes.md" \
|
||||
--verify-tag \
|
||||
"$version" \
|
||||
"release/fish-$version.tar.xz" "release/fish-$version.tar.xz.asc"
|
||||
if $is_latest_release; then
|
||||
$dry_run git push origin $version:master
|
||||
fi
|
||||
rm -rf $relnotes_tmp
|
||||
|
||||
(set +x
|
||||
echo "\
|
||||
To update fish-site we currently query the github release API
|
||||
Please publish the draft at https://github.com/fish-shell/fish-shell/releases/tag/$version
|
||||
and hit Enter to continue with updating ../fish-site")
|
||||
read line
|
||||
|
||||
minor_version=${version%.*}
|
||||
if $is_latest_release; then
|
||||
rm -rf "../fish-site/site/docs/current"
|
||||
cp -r "release/dpkgs/fish-$version/user_doc/html" ../fish-site/site/docs/current
|
||||
fi
|
||||
rm -rf "../fish-site/site/docs/$minor_version"
|
||||
cp -r "release/dpkgs/fish-$version/user_doc/html" "../fish-site/site/docs/$minor_version"
|
||||
(
|
||||
cd ../fish-site
|
||||
make new-release
|
||||
git add -u
|
||||
git add site/docs/$minor_version docs/docs/$minor_version
|
||||
git add site/docs/current docs/docs/current
|
||||
git commit --message="Release $version"
|
||||
$dry_run git push
|
||||
if [ -n $dry_run ]; then
|
||||
git reset --hard HEAD~ >/dev/null
|
||||
fi
|
||||
)
|
||||
|
||||
cat <<EOF
|
||||
Remaining work:
|
||||
- Create macOS packages:
|
||||
./build_tools/make_pkg.sh
|
||||
./build_tools/mac_notarize.sh ~/fish_built/fish-$version.pkg <AC_USER>
|
||||
./build_tools/mac_notarize.sh ~/fish_built/fish-$version.app.zip <AC_USER>
|
||||
- Add macOS packages to https://github.com/fish-shell/fish-shell/releases/tag/$version
|
||||
- Send email to "fish-users Mailing List <fish-users@lists.sourceforge.net>"
|
||||
EOF
|
||||
|
||||
exit
|
||||
|
||||
}
|
||||
47
build_tools/release/build-debian-package.Dockerfile
Normal file
47
build_tools/release/build-debian-package.Dockerfile
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
# Currently need testing for rustc>=1.67
|
||||
FROM debian:testing
|
||||
LABEL org.opencontainers.image.source=https://github.com/fish-shell/fish-shell
|
||||
|
||||
ENV LANG C.UTF-8
|
||||
ENV LC_ALL C.UTF-8
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get -y install \
|
||||
build-essential \
|
||||
cargo \
|
||||
cmake \
|
||||
debhelper \
|
||||
debmake \
|
||||
file \
|
||||
g++ \
|
||||
gettext \
|
||||
git \
|
||||
libpcre2-dev \
|
||||
locales \
|
||||
locales-all \
|
||||
ninja-build \
|
||||
pkg-config \
|
||||
python3 \
|
||||
python3-pexpect \
|
||||
python3-launchpadlib \
|
||||
rustc \
|
||||
sudo \
|
||||
tmux \
|
||||
&& locale-gen en_US.UTF-8 \
|
||||
&& apt-get clean
|
||||
|
||||
RUN groupadd -g 1000 fishuser \
|
||||
&& useradd -p $(openssl passwd -1 fish) -d /home/fishuser -m -u 1000 -g 1000 fishuser \
|
||||
&& adduser fishuser sudo \
|
||||
&& mkdir /fish-source \
|
||||
&& chown -R fishuser:fishuser /home/fishuser /fish-source
|
||||
|
||||
USER fishuser
|
||||
WORKDIR /home/fishuser
|
||||
|
||||
COPY /build-debian-package.sh /
|
||||
|
||||
CMD cp -r /gnupg ~/.gnupg && \
|
||||
/build-debian-package.sh /fish-source /fish-source/release /home/fishuser/dpkgs || \
|
||||
bash
|
||||
75
build_tools/release/build-debian-package.sh
Executable file
75
build_tools/release/build-debian-package.sh
Executable file
|
|
@ -0,0 +1,75 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -eux
|
||||
|
||||
source=$1
|
||||
tarball_dir=$2
|
||||
output=$3
|
||||
|
||||
committer=$FISH_COMMITTER # TODO Make sure that this person is listed in debian/control ?
|
||||
gpg_key_id=$FISH_GPG_KEY_ID
|
||||
version=$FISH_VERSION
|
||||
|
||||
export DEBEMAIL=$committer
|
||||
export EDITOR=true
|
||||
|
||||
mkdir "$output"
|
||||
# Should be a hard link.
|
||||
cp "$tarball_dir/fish-$version.tar.xz" "$output/fish_$version.orig.tar.xz"
|
||||
(
|
||||
cd "$output"
|
||||
tar xf "fish_$version.orig.tar.xz"
|
||||
)
|
||||
tarball_sha256sum=$(sha256sum "$output/fish-$version.tar.xz" | cut -d' ' -f1)
|
||||
|
||||
# Copy in the Debian packaging information
|
||||
cp -r "$source/debian" "$output"/fish-$version/debian
|
||||
|
||||
# TODO Include old changes?
|
||||
echo > "$output"/fish-$version/debian/changelog "\
|
||||
fish ($version-1) testing; urgency=medium
|
||||
|
||||
* Upstream release.
|
||||
|
||||
-- $committer $(date)"
|
||||
|
||||
# Update the Debian changelog for the new version - interactive
|
||||
(
|
||||
cd "$output"/fish-$version
|
||||
dch --newversion "$version-1" --distribution testing
|
||||
)
|
||||
|
||||
(
|
||||
cd "$output"
|
||||
(
|
||||
cd fish-$version
|
||||
# Build the package for Debian on OBS.
|
||||
debuild -S -k"$gpg_key_id"
|
||||
# Set up an array for the Ubuntu packages
|
||||
ppa_series=$(
|
||||
echo '
|
||||
from launchpadlib.launchpad import Launchpad
|
||||
launchpad = Launchpad.login_anonymously("fish shell build script", "production", "~/.cache", version="devel")
|
||||
ubu = launchpad.projects("ubuntu")
|
||||
# dash requires a double backslash for unknown reasons.
|
||||
print("\\n".join(x["name"] for x in ubu.series.entries if x["supported"] == True and x["name"] not in ("trusty", "precise")))
|
||||
' |
|
||||
sed s,^[[:space:]]*,, |
|
||||
python3
|
||||
)
|
||||
# Build the Ubuntu source packages
|
||||
for series in $ppa_series
|
||||
do
|
||||
sed -i -e "s/^fish ($version-1)/fish ($version-1~$series)/" -e "s/testing/$series/" debian/changelog
|
||||
debuild -S -sa -k"$gpg_key_id"
|
||||
sed -i -e "s/^fish ($version-1~$series)/fish ($version-1)/" -e "s/$series/testing/" debian/changelog
|
||||
done
|
||||
)
|
||||
|
||||
# TODO populate the dput config file.
|
||||
# # Or other appropriate PPA defined in ~/.dput.cf
|
||||
# for i in fish_$version-1~*.changes
|
||||
# do
|
||||
# dput fish-release-3 "$i"
|
||||
# done
|
||||
)
|
||||
Loading…
Reference in New Issue
Block a user