From 950d06ad5b54abfc3f0d4dffb6b4b890b029e9e8 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 11:59:46 +0200 Subject: ci: Initial automatic release support See #3725 --- .github/workflows/release.yml | 137 ++++++++++++++++++++++++++++++++++++++++++ scripts/dev/build_release.py | 29 +++++++-- scripts/dev/update_version.py | 64 ++++++++++++++------ tox.ini | 8 +++ 4 files changed, 214 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..213078730 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,137 @@ +name: Release + +on: + workflow_dispatch: + inputs: + release_type: + description: 'Release type' + required: true + default: 'patch' + type: choice + options: + - 'patch' + - 'minor' + - 'major' + # FIXME do we want a possibility to do prereleases here? + python_version: + description: 'Python version' + required: true + default: '3.11' + type: choice + options: + - '3.8' + - '3.9' + - '3.10' + - '3.11' +jobs: + prepare: + runs-on: ubuntu-20.04 + timeout-minutes: 5 + outputs: + version: ${{ steps.bump.outputs.version }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + # Doesn't really matter what we prepare the release with, but let's + # use the same version for consistency. + python-version: ${{ github.event.inputs.python_version }} + - name: Install dependencies + run: | + python -m pip install -U pip + python -m pip install -U -r misc/requirements/requirements-tox.txt + - name: Configure git + run: | + git config --global user.name "qutebrowser bot" + git config --global user.email "bot@qutebrowser.org" + - name: Switch to release branch + if: "${{ github.event.inputs.release_type }} == 'patch'" + run: | + git checkout "$(git branch --format='%(refname:short)' --list 'v*.*.x' | sort -V | tail -n1)" + # FIXME set up GPG for signed tag + - name: Bump version + id: bump + run: "tox -e update-version -- ${{ github.event.inputs.release_type }}" + - name: Push release commit/tag + run: | + git push origin main + git push origin v${{ steps.bump.outputs.version }} + - name: Cherry-pick release commit + if: "${{ github.event.inputs.release_type }} == 'patch'" + run: | + git checkout main + git cherry-pick v${{ steps.bump.outputs.version }} + git push origin main + git checkout v${{ steps.bump.outputs.version_x }} + - name: Create release branch + if: "${{ github.event.inputs.release_type }} != 'patch'" + run: | + git checkout -b v${{ steps.bump.outputs.version_x }} + git push --set-upstream origin v${{ steps.bump.outputs.version_x }} + - name: Create GitHub draft release + uses: softprops/action-gh-release@v1 + with: + tag_name: v${{ steps.bump.outputs.version }} + draft: true + body: "*Release artifacts for this release are currently being uploaded...*" + release: + strategy: + matrix: + include: + - os: macos-11 + - os: windows-2019 + - os: ubuntu-20.04 + runs-on: "${{ matrix.os }}" + timeout-minutes: 45 + needs: [prepare] + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ github.event.inputs.python_version }} + # FIXME set up GPG for signed releases (at least on Ubuntu) + - name: Install dependencies + run: | + python -m pip install -U pip + python -m pip install -U -r misc/requirements/requirements-tox.txt + - name: Build and upload release + run: "tox -e build-release -- --upload --no-confirm --experimental --gh-token ${{ secrets.GITHUB_TOKEN }}" + finalize: + runs-on: ubuntu-20.04 + timeout-minutes: 5 + needs: [prepare, release] + steps: + - name: Publish final release + uses: softprops/action-gh-release@v1 + with: + tag_name: v${{ needs.prepare.outputs.version }} + draft: false + # FIXME automatically cut relevant changes from changelog and add them here? + body: | + Check the [changelog](https://github.com/qutebrowser/qutebrowser/blob/master/doc/changelog.asciidoc) for changes in this release. + irc: + timeout-minutes: 2 + continue-on-error: true + runs-on: ubuntu-20.04 + needs: [prepare, release, finalize] + if: "${{ always() }}" + steps: + - name: Send success IRC notification + uses: Gottox/irc-message-action@v2 + if: "${{ needs.finalize.result == 'success' }}" + with: + server: irc.libera.chat + channel: '#qutebrowser-bots' + nickname: qutebrowser-bot + message: "[${{ github.workflow }}] \u00033Success:\u0003 ${{ github.ref }} https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} (@${{ github.actor }})" + - name: Send non-success IRC notification + uses: Gottox/irc-message-action@v2 + if: "${{ needs.finalize.result != 'success' }}" + with: + server: irc.libera.chat + channel: '#qutebrowser-bots' + nickname: qutebrowser-bot + message: "[${{ github.workflow }}] \u00034FAIL:\u0003 ${{ github.ref }} https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} (@${{ github.actor }})\n + prepare: ${{ needs.prepare.result }}, release: ${{ needs.release.result}}, finalize: ${{ needs.finalize.result }}" diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index 55b3f5f1c..2038a7f67 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -544,13 +544,19 @@ def read_github_token( return token -def github_upload(artifacts: List[Artifact], tag: str, gh_token: str) -> None: +def github_upload( + artifacts: List[Artifact], + tag: str, + gh_token: str, + experimental: bool, +) -> None: """Upload the given artifacts to GitHub. Args: artifacts: A list of Artifacts to upload. tag: The name of the release tag gh_token: The GitHub token to use + experimental: Upload to the experiments repo """ # pylint: disable=broad-exception-raised import github3 @@ -558,7 +564,11 @@ def github_upload(artifacts: List[Artifact], tag: str, gh_token: str) -> None: utils.print_title("Uploading to github...") gh = github3.login(token=gh_token) - repo = gh.repository('qutebrowser', 'qutebrowser') + + if experimental: + repo = gh.repository('qutebrowser', 'experiments') + else: + repo = gh.repository('qutebrowser', 'qutebrowser') release = None # to satisfy pylint for release in repo.releases(): @@ -602,10 +612,13 @@ def github_upload(artifacts: List[Artifact], tag: str, gh_token: str) -> None: break -def pypi_upload(artifacts: List[Artifact]) -> None: +def pypi_upload(artifacts: List[Artifact], experimental: bool) -> None: """Upload the given artifacts to PyPI using twine.""" utils.print_title("Uploading to PyPI...") - run_twine('upload', artifacts) + if experimental: + run_twine('upload', artifacts, "-r", "testpypi") + else: + run_twine('upload', artifacts) def twine_check(artifacts: List[Artifact]) -> None: @@ -635,6 +648,8 @@ def main() -> None: help="Build a debug build.") parser.add_argument('--qt5', action='store_true', required=False, help="Build against PyQt5") + parser.add_argument('--experimental', action='store_true', required=False, + help="Upload to experiments repo and test PyPI") args = parser.parse_args() utils.change_cwd() @@ -647,6 +662,7 @@ def main() -> None: gh_token = read_github_token(args.gh_token) else: gh_token = read_github_token(args.gh_token, optional=True) + assert not args.experimental # makes no sense without upload if not misc_checks.check_git(): utils.print_error("Refusing to do a release with a dirty git tree") @@ -685,9 +701,10 @@ def main() -> None: input() assert gh_token is not None - github_upload(artifacts, version_tag, gh_token=gh_token) + github_upload( + artifacts, version_tag, gh_token=gh_token, experimental=args.experimental) if upload_to_pypi: - pypi_upload(artifacts) + pypi_upload(artifacts, experimental=args.experimental) else: print() utils.print_title("Artifacts") diff --git a/scripts/dev/update_version.py b/scripts/dev/update_version.py index ec1550414..c67873496 100644 --- a/scripts/dev/update_version.py +++ b/scripts/dev/update_version.py @@ -8,6 +8,7 @@ """Update version numbers using bump2version.""" +import re import sys import argparse import os.path @@ -19,6 +20,24 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, from scripts import utils +class Error(Exception): + """Base class for exceptions in this module.""" + + +def verify_branch(version_leap): + """Check that we're on the correct git branch.""" + proc = subprocess.run( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], + check=True, capture_output=True, text=True) + branch = proc.stdout.strip() + + if ( + version_leap == 'patch' and not re.fullmatch(r'v\d+\.\d+\.\*', branch) or + version_leap != 'patch' and branch != 'main' + ): + raise Error(f"Invalid branch for {version_leap} release: {branch}") + + def bump_version(version_leap="patch"): """Update qutebrowser release version. @@ -46,6 +65,7 @@ if __name__ == "__main__": utils.change_cwd() if not args.commands: + verify_branch(args.bump) bump_version(args.bump) show_commit() @@ -54,22 +74,30 @@ if __name__ == "__main__": x_version = '.'.join([str(p) for p in qutebrowser.__version_info__[:-1]] + ['x']) - print("Run the following commands to create a new release:") - print("* git push origin; git push origin v{v}".format(v=version)) - if args.bump == 'patch': - print("* git checkout main && git cherry-pick v{v} && " - "git push origin".format(v=version)) + if utils.ON_CI: + output_file = os.environ["GITHUB_OUTPUT"] + with open(output_file, "w", encoding="ascii") as f: + f.write(f"version={version}\n") + f.write(f"x_version={x_version}\n") + + print(f"Outputs for {version} written to GitHub Actions output file") else: - print("* git branch v{x} v{v} && git push --set-upstream origin v{x}" - .format(v=version, x=x_version)) - print("* Create new release via GitHub (required to upload release " - "artifacts)") - print("* Linux: git fetch && git checkout v{v} && " - "tox -e build-release -- --upload" - .format(v=version)) - print("* Windows: git fetch; git checkout v{v}; " - "py -3.9 -m tox -e build-release -- --upload" - .format(v=version)) - print("* macOS: git fetch && git checkout v{v} && " - "tox -e build-release -- --upload" - .format(v=version)) + print("Run the following commands to create a new release:") + print("* git push origin; git push origin v{v}".format(v=version)) + if args.bump == 'patch': + print("* git checkout main && git cherry-pick v{v} && " + "git push origin".format(v=version)) + else: + print("* git branch v{x} v{v} && git push --set-upstream origin v{x}" + .format(v=version, x=x_version)) + print("* Create new release via GitHub (required to upload release " + "artifacts)") + print("* Linux: git fetch && git checkout v{v} && " + "tox -e build-release -- --upload" + .format(v=version)) + print("* Windows: git fetch; git checkout v{v}; " + "py -3.9 -m tox -e build-release -- --upload" + .format(v=version)) + print("* macOS: git fetch && git checkout v{v} && " + "tox -e build-release -- --upload" + .format(v=version)) diff --git a/tox.ini b/tox.ini index 060dccaaa..06c96cdf1 100644 --- a/tox.ini +++ b/tox.ini @@ -267,6 +267,14 @@ deps = commands = {envpython} -m sphinx -jauto -W --color {posargs} {toxinidir}/doc/extapi/ {toxinidir}/doc/extapi/_build/ +[testenv:update-version] +basepython = {env:PYTHON:python3} +passenv = + GITHUB_OUTPUT + CI +deps = -r{toxinidir}/misc/requirements/requirements-dev.txt +commands = {envpython} scripts/dev/update_version.py {posargs} + [testenv:build-release{,-qt5}] basepython = {env:PYTHON:python3} passenv = * -- cgit v1.2.3-54-g00ecf From 9757fa79ee9e94d60f521269c675df16c33b79b0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 14:48:49 +0200 Subject: ci: Use GitHub Script to get release branch By default, we only get a narrow checkout, so we don't know about any other branches. Use the GitHub API and some JS to get the release branch instead. --- .github/workflows/release.yml | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 213078730..9b963bc9e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,6 +30,35 @@ jobs: outputs: version: ${{ steps.bump.outputs.version }} steps: + - name: Find release branch + uses: actions/github-script@v6 + id: find-branch + with: + script: | + if (context.payload.inputs.release_type != 'patch') { + return 'main'; + } + const branches = await github.paginate(github.rest.repos.listBranches, { + owner: context.repo.owner, + repo: context.repo.repo, + }); + const branch_names = branches.map(branch => branch.name); + console.log(`branches: ${branch_names}`); + const release_branches = branch_names.filter(branch => branch.match(/^v\d+\.\d+\.x$/)); + if (release_branches.length === 0) { + core.setFailed('No release branch found!'); + return ''; + } + console.log(`release_branches: ${release_branches}`); + // Get newest release branch (biggest version number) + const sorted = release_branches.sort( + function (a, b) { + return a.localeCompare(b, undefined, { numeric: true, sensitivity: 'base' }); + } + ); + console.log(`sorted: ${sorted}`); + return sorted.at(-1); + result-encoding: string - uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v4 @@ -46,9 +75,9 @@ jobs: git config --global user.name "qutebrowser bot" git config --global user.email "bot@qutebrowser.org" - name: Switch to release branch - if: "${{ github.event.inputs.release_type }} == 'patch'" - run: | - git checkout "$(git branch --format='%(refname:short)' --list 'v*.*.x' | sort -V | tail -n1)" + uses: actions/checkout@v3 + with: + ref: ${{ steps.find-branch.outputs.result }} # FIXME set up GPG for signed tag - name: Bump version id: bump -- cgit v1.2.3-54-g00ecf From 13ddb50170ed55f63ec3bee618d9c2475252791a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 16:34:03 +0200 Subject: ci: More automatic release improvements/fixes --- .github/workflows/release.yml | 35 ++++++++++++++++++++++++++++++----- scripts/dev/update_version.py | 14 +++++++++----- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b963bc9e..309b59392 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,6 +29,8 @@ jobs: timeout-minutes: 5 outputs: version: ${{ steps.bump.outputs.version }} + permissions: + contents: write # To push release commit/tag steps: - name: Find release branch uses: actions/github-script@v6 @@ -78,23 +80,25 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ steps.find-branch.outputs.result }} - # FIXME set up GPG for signed tag + - name: Import GPG Key + run: | + gpg --import <<< "${{ secrets.QUTEBROWSER_BOT_GPGKEY }}" - name: Bump version id: bump run: "tox -e update-version -- ${{ github.event.inputs.release_type }}" - name: Push release commit/tag run: | - git push origin main + git push origin ${{ steps.find-branch.outputs.result }} git push origin v${{ steps.bump.outputs.version }} - name: Cherry-pick release commit - if: "${{ github.event.inputs.release_type }} == 'patch'" + if: ${{ github.event.inputs.release_type == 'patch' }} run: | git checkout main git cherry-pick v${{ steps.bump.outputs.version }} git push origin main git checkout v${{ steps.bump.outputs.version_x }} - name: Create release branch - if: "${{ github.event.inputs.release_type }} != 'patch'" + if: ${{ github.event.inputs.release_type != 'patch' }} run: | git checkout -b v${{ steps.bump.outputs.version_x }} git push --set-upstream origin v${{ steps.bump.outputs.version_x }} @@ -114,13 +118,32 @@ jobs: runs-on: "${{ matrix.os }}" timeout-minutes: 45 needs: [prepare] + permissions: + contents: write # To upload release artifacts steps: - uses: actions/checkout@v3 + with: + ref: v${{ needs.prepare.outputs.version }} - name: Set up Python uses: actions/setup-python@v4 with: python-version: ${{ github.event.inputs.python_version }} - # FIXME set up GPG for signed releases (at least on Ubuntu) + - name: Import GPG Key + if: ${{ startsWith(matrix.os, 'ubuntu-') }} + run: | + gpg --import <<< "${{ secrets.QUTEBROWSER_BOT_GPGKEY }}" + # Needed because of the following import chain: + # - scripts/dev/build_release.py + # - scripts/dev/update_3rdparty.py + # - scripts/dictcli.py + # - qutebrowser/browser/webengine/spell.py + # - utils.message -> utils.usertypes -> utils.qtutils -> qt.gui + # - PyQt6.QtGui + - name: Install apt dependencies + if: ${{ startsWith(matrix.os, 'ubuntu-') }} + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends libegl1-mesa - name: Install dependencies run: | python -m pip install -U pip @@ -131,6 +154,8 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 5 needs: [prepare, release] + permissions: + contents: write # To change release steps: - name: Publish final release uses: softprops/action-gh-release@v1 diff --git a/scripts/dev/update_version.py b/scripts/dev/update_version.py index c67873496..1029fb29d 100644 --- a/scripts/dev/update_version.py +++ b/scripts/dev/update_version.py @@ -32,7 +32,7 @@ def verify_branch(version_leap): branch = proc.stdout.strip() if ( - version_leap == 'patch' and not re.fullmatch(r'v\d+\.\d+\.\*', branch) or + version_leap == 'patch' and not re.fullmatch(r'v\d+\.\d+\.x', branch) or version_leap != 'patch' and branch != 'main' ): raise Error(f"Invalid branch for {version_leap} release: {branch}") @@ -50,7 +50,11 @@ def bump_version(version_leap="patch"): def show_commit(): - subprocess.run(['git', 'show'], check=True) + """Show the latest git commit.""" + git_args = ['git', 'show'] + if utils.ON_CI: + git_args.append("--color") + subprocess.run(git_args, check=True) if __name__ == "__main__": @@ -71,14 +75,14 @@ if __name__ == "__main__": import qutebrowser version = qutebrowser.__version__ - x_version = '.'.join([str(p) for p in qutebrowser.__version_info__[:-1]] + + version_x = '.'.join([str(p) for p in qutebrowser.__version_info__[:-1]] + ['x']) if utils.ON_CI: output_file = os.environ["GITHUB_OUTPUT"] with open(output_file, "w", encoding="ascii") as f: f.write(f"version={version}\n") - f.write(f"x_version={x_version}\n") + f.write(f"version_x={version_x}\n") print(f"Outputs for {version} written to GitHub Actions output file") else: @@ -89,7 +93,7 @@ if __name__ == "__main__": "git push origin".format(v=version)) else: print("* git branch v{x} v{v} && git push --set-upstream origin v{x}" - .format(v=version, x=x_version)) + .format(v=version, x=version_x)) print("* Create new release via GitHub (required to upload release " "artifacts)") print("* Linux: git fetch && git checkout v{v} && " -- cgit v1.2.3-54-g00ecf From ab820fe8a0698231539992f3b4013c23dc42b259 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 17:02:02 +0200 Subject: ci: Set up asciidoc properly for releases --- .github/workflows/release.yml | 3 ++- misc/Makefile | 3 ++- scripts/dev/build_release.py | 12 ++++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 309b59392..1323ce605 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -139,11 +139,12 @@ jobs: # - qutebrowser/browser/webengine/spell.py # - utils.message -> utils.usertypes -> utils.qtutils -> qt.gui # - PyQt6.QtGui + # Some additional packages are needed for a2x to build manpage - name: Install apt dependencies if: ${{ startsWith(matrix.os, 'ubuntu-') }} run: | sudo apt-get update - sudo apt-get install --no-install-recommends libegl1-mesa + sudo apt-get install --no-install-recommends libegl1-mesa libxml2-utils docbook-xml xsltproc docbook-xsl - name: Install dependencies run: | python -m pip install -U pip diff --git a/misc/Makefile b/misc/Makefile index 62294ba61..39a7e005f 100644 --- a/misc/Makefile +++ b/misc/Makefile @@ -4,6 +4,7 @@ ICONSIZES = 16 24 32 48 64 128 256 512 DATAROOTDIR = $(PREFIX)/share DATADIR ?= $(DATAROOTDIR) MANDIR ?= $(DATAROOTDIR)/man +A2X ?= a2x ifdef DESTDIR SETUPTOOLSOPTS = --root="$(DESTDIR)" @@ -14,7 +15,7 @@ all: man man: doc/qutebrowser.1 doc/qutebrowser.1: doc/qutebrowser.1.asciidoc - a2x -f manpage $< + $(A2X) -f manpage $< install: man $(PYTHON) setup.py install --prefix="$(PREFIX)" --optimize=1 $(SETUPTOOLSOPTS) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index 2038a7f67..d16e24866 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -518,9 +518,17 @@ def build_sdist() -> List[Artifact]: def test_makefile() -> None: """Make sure the Makefile works correctly.""" utils.print_title("Testing makefile") + a2x_path = pathlib.Path(sys.executable).parent / 'a2x' + assert a2x_path.exists(), a2x_path with tempfile.TemporaryDirectory() as tmpdir: - subprocess.run(['make', '-f', 'misc/Makefile', - f'DESTDIR={tmpdir}', 'install'], check=True) + subprocess.run( + [ + 'make', '-f', 'misc/Makefile', + f'DESTDIR={tmpdir}', f'A2X={a2x_path}', + 'install' + ], + check=True, + ) def read_github_token( -- cgit v1.2.3-54-g00ecf From 0e07d3b7b600ae6d1757e6f5fd22b104ec30aacb Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 18:43:50 +0200 Subject: scripts: Show more info when no release was found --- scripts/dev/build_release.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index d16e24866..acbe31507 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -583,7 +583,9 @@ def github_upload( if release.tag_name == tag: break else: - raise Exception(f"No release found for {tag!r}!") + releases = ", ".join(r.tag_name for r in repo.releases()) + raise Exception( + f"No release found for {tag!r} in {repo.full_name}, found: {releases}") for artifact in artifacts: while True: -- cgit v1.2.3-54-g00ecf From a59bf7bc5bc9ec863fd359be453e326c727008ce Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 19:20:27 +0200 Subject: ci: Set twine PyPI token --- .github/workflows/release.yml | 8 +++++++- scripts/dev/build_release.py | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1323ce605..288186eb1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -149,8 +149,14 @@ jobs: run: | python -m pip install -U pip python -m pip install -U -r misc/requirements/requirements-tox.txt + # FIXME consider switching to trusted publishers: + # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/ - name: Build and upload release - run: "tox -e build-release -- --upload --no-confirm --experimental --gh-token ${{ secrets.GITHUB_TOKEN }}" + run: "tox -e build-release -- --upload --no-confirm --experimental" + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.QUTEBROWSER_BOT_TEST_PYPI_TOKEN }} # FIXME use real token + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} finalize: runs-on: ubuntu-20.04 timeout-minutes: 5 diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index acbe31507..fe2ec16c3 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -539,6 +539,9 @@ def read_github_token( if arg_token is not None: return arg_token + if "GITHUB_TOKEN" in os.environ: + return os.environ["GITHUB_TOKEN"] + token_path = pathlib.Path.home() / '.gh_token' if not token_path.exists(): if optional: -- cgit v1.2.3-54-g00ecf From 66f654b95ca4d270a38e1d2aa4e9f1d1dc02d994 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 19:28:04 +0200 Subject: scripts: Avoid inputs on CI when releasing --- scripts/dev/build_release.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index fe2ec16c3..64978aba2 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -598,6 +598,10 @@ def github_upload( if asset.name == artifact.path.name] if assets: print(f"Assets already exist: {assets}") + + if utils.ON_CI: + sys.exit(1) + print("Press enter to continue anyways or Ctrl-C to abort.") input() @@ -611,8 +615,13 @@ def github_upload( ) except github3.exceptions.ConnectionError as e: utils.print_error(f'Failed to upload: {e}') - print("Press Enter to retry...", file=sys.stderr) - input() + if utils.ON_CI: + print("Retrying in 30s...") + time.sleep(30) + else: + print("Press Enter to retry...", file=sys.stderr) + input() + print("Retrying!") assets = [asset for asset in release.assets() @@ -709,7 +718,7 @@ def main() -> None: if args.upload: version_tag = f"v{qutebrowser.__version__}" - if not args.no_confirm: + if not args.no_confirm and not utils.ON_CI: utils.print_title(f"Press enter to release {version_tag}...") input() -- cgit v1.2.3-54-g00ecf From 680a941e641343dfaf734ef7707a093671c738a1 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 19:41:31 +0200 Subject: scripts: Fix artifact list on Windows Regressed in c2210539a9e2be1deacf8df8f432e035d9b9b9f0: We don't want to return a list of lists. --- scripts/dev/build_release.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index 64978aba2..573db8539 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -400,13 +400,11 @@ def build_windows( utils.print_title("Updating VersionInfo file") gen_versioninfo.main() - artifacts = [ - _build_windows_single( - skip_packaging=skip_packaging, - debug=debug, - qt5=qt5, - ), - ] + artifacts = _build_windows_single( + skip_packaging=skip_packaging, + debug=debug, + qt5=qt5, + ) return artifacts -- cgit v1.2.3-54-g00ecf From ae150d3a9792e2b160f08653af138372e528e5c4 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 19:52:49 +0200 Subject: scripts: Don't upload PGP signature to PyPI See https://blog.pypi.org/posts/2023-05-23-removing-pgp/ --- scripts/dev/build_release.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index 573db8539..f9efa3ff5 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -634,6 +634,9 @@ def github_upload( def pypi_upload(artifacts: List[Artifact], experimental: bool) -> None: """Upload the given artifacts to PyPI using twine.""" + # https://blog.pypi.org/posts/2023-05-23-removing-pgp/ + artifacts = [a for a in artifacts if a.mimetype != 'application/pgp-signature'] + utils.print_title("Uploading to PyPI...") if experimental: run_twine('upload', artifacts, "-r", "testpypi") -- cgit v1.2.3-54-g00ecf From 60bb247f0d32368f7d7b56a7f7e28be868cdb917 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 20:08:05 +0200 Subject: scripts: Fix name for Windows installer Regressed in c2210539a9e2be1deacf8df8f432e035d9b9b9f0: The current NSIS installer still adds the suffix. Let's keep it there for now until we switch to the rewritten one. See #6050 --- scripts/dev/build_release.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index f9efa3ff5..65eef720c 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -432,6 +432,8 @@ def _package_windows_single( name_parts.append('debug') if qt5: name_parts.append('qt5') + + name_parts.append('amd64') # FIXME:qt6 temporary until new installer name = '-'.join(name_parts) + '.exe' artifacts.append(Artifact( -- cgit v1.2.3-54-g00ecf From c10cda7d4aeb473e2e1f4844731b177fa05907d9 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 20:57:42 +0200 Subject: ci: Use github script to update release See https://github.com/softprops/action-gh-release/issues/163 --- .github/workflows/release.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 288186eb1..ee4c7ee38 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -165,13 +165,22 @@ jobs: contents: write # To change release steps: - name: Publish final release - uses: softprops/action-gh-release@v1 + uses: actions/github-script@v6 with: - tag_name: v${{ needs.prepare.outputs.version }} - draft: false - # FIXME automatically cut relevant changes from changelog and add them here? - body: | - Check the [changelog](https://github.com/qutebrowser/qutebrowser/blob/master/doc/changelog.asciidoc) for changes in this release. + script: | + const tag = "v${{ needs.prepare.outputs.version }}"; + const release = await github.rest.repos.getReleaseByTag({ + owner: context.repo.owner, + repo: context.repo.repo, + tag: tag, + }) + await github.rest.repos.updateRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: release.data.id, + draft: false, + body: "Check the [changelog](https://github.com/qutebrowser/qutebrowser/blob/master/doc/changelog.asciidoc) for changes in this release." + }) irc: timeout-minutes: 2 continue-on-error: true -- cgit v1.2.3-54-g00ecf From fa74860417f82a922183e2fbe522b0b98d324baf Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 21:41:05 +0200 Subject: ci: Use earlier release ID --- .github/workflows/release.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee4c7ee38..9b9dde525 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,6 +29,7 @@ jobs: timeout-minutes: 5 outputs: version: ${{ steps.bump.outputs.version }} + release_id: ${{ steps.create-release.outputs.id }} permissions: contents: write # To push release commit/tag steps: @@ -103,6 +104,7 @@ jobs: git checkout -b v${{ steps.bump.outputs.version_x }} git push --set-upstream origin v${{ steps.bump.outputs.version_x }} - name: Create GitHub draft release + id: create-release uses: softprops/action-gh-release@v1 with: tag_name: v${{ steps.bump.outputs.version }} @@ -168,16 +170,10 @@ jobs: uses: actions/github-script@v6 with: script: | - const tag = "v${{ needs.prepare.outputs.version }}"; - const release = await github.rest.repos.getReleaseByTag({ - owner: context.repo.owner, - repo: context.repo.repo, - tag: tag, - }) await github.rest.repos.updateRelease({ owner: context.repo.owner, repo: context.repo.repo, - release_id: release.data.id, + release_id: "${{ needs.prepare.outputs.release_id }}", draft: false, body: "Check the [changelog](https://github.com/qutebrowser/qutebrowser/blob/master/doc/changelog.asciidoc) for changes in this release." }) -- cgit v1.2.3-54-g00ecf From 67ae39561a22bdd80ca7419fdd158a7f1b938b47 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 23:22:37 +0200 Subject: ci: Use -x for cherry picks --- .github/workflows/release.yml | 2 +- scripts/dev/update_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b9dde525..54f79f945 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -95,7 +95,7 @@ jobs: if: ${{ github.event.inputs.release_type == 'patch' }} run: | git checkout main - git cherry-pick v${{ steps.bump.outputs.version }} + git cherry-pick -x v${{ steps.bump.outputs.version }} git push origin main git checkout v${{ steps.bump.outputs.version_x }} - name: Create release branch diff --git a/scripts/dev/update_version.py b/scripts/dev/update_version.py index 1029fb29d..424a77559 100644 --- a/scripts/dev/update_version.py +++ b/scripts/dev/update_version.py @@ -89,7 +89,7 @@ if __name__ == "__main__": print("Run the following commands to create a new release:") print("* git push origin; git push origin v{v}".format(v=version)) if args.bump == 'patch': - print("* git checkout main && git cherry-pick v{v} && " + print("* git checkout main && git cherry-pick -x v{v} && " "git push origin".format(v=version)) else: print("* git branch v{x} v{v} && git push --set-upstream origin v{x}" -- cgit v1.2.3-54-g00ecf From bcbfc2b68cec5512dc7934452f4c260535f58f1f Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 16 Aug 2023 23:28:54 +0200 Subject: ci: Add IRC notification for main channel Releases are probably important enough --- .github/workflows/release.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 54f79f945..adabed2fb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -192,6 +192,14 @@ jobs: channel: '#qutebrowser-bots' nickname: qutebrowser-bot message: "[${{ github.workflow }}] \u00033Success:\u0003 ${{ github.ref }} https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} (@${{ github.actor }})" + - name: Send main channel IRC notification + uses: Gottox/irc-message-action@v2 + if: "${{ needs.finalize.result == 'success' && github.repository == 'qutebrowser/qutebrowser' }}" + with: + server: irc.libera.chat + channel: '#qutebrowser' + nickname: qutebrowser-bot + message: "qutebrowser v${{ needs.prepare.outputs.version }} has been released! https://github.com/${{ github.repository }}/releases/tag/v${{ needs.prepare.outputs.version }}" - name: Send non-success IRC notification uses: Gottox/irc-message-action@v2 if: "${{ needs.finalize.result != 'success' }}" -- cgit v1.2.3-54-g00ecf From 60eb849fad32b2e3d23d9f80bcbecc34df91c7f9 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 17 Aug 2023 11:56:03 +0200 Subject: ci: Check for open milestone before release --- .github/workflows/release.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index adabed2fb..0c5b6d4b1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -87,6 +87,21 @@ jobs: - name: Bump version id: bump run: "tox -e update-version -- ${{ github.event.inputs.release_type }}" + - name: Check milestone + uses: actions/github-script@v6 + with: + script: | + const milestones = await github.paginate(github.rest.issues.listMilestones, { + owner: context.repo.owner, + repo: context.repo.repo, + }); + const names = milestones.map(milestone => milestone.title); + console.log(`milestones: ${names}`); + + const milestone = milestones.find(milestone => milestone.title === "v${{ steps.bump.outputs.version }}"); + if (milestone !== undefined) { + core.setFailed(`Found open milestone ${milestone.title} with ${milestone.open_issues} open and ${milestone.closed_issues} closed issues!`); + } - name: Push release commit/tag run: | git push origin ${{ steps.find-branch.outputs.result }} -- cgit v1.2.3-54-g00ecf From 5567c9290b692ee4e0a938a5233dc3d4fc153329 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 17 Aug 2023 12:18:11 +0200 Subject: Update releasing docs --- doc/changelog.asciidoc | 7 +++++++ doc/contributing.asciidoc | 18 +++++++++++++++--- scripts/dev/update_version.py | 2 +- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 30fbfb8fd..53931f8b7 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -38,6 +38,13 @@ Major changes * `run-with-count` -> `cmd-run-with-count` The old names continue to work for the time being, but are deprecated and show a warning. +- Releases are now automated on CI, and GPG signed by + `qutebrowser bot `, fingerprint + `27F3 BB4F C217 EECB 8585 78AE EF7E E4D0 3969 0B7B`. + The key is available as follows: + * On https://qutebrowser.org/pubkey.gpg + * Via keys.openpgp.org + * Via WKD for bot@qutebrowser.org Added ~~~~~ diff --git a/doc/contributing.asciidoc b/doc/contributing.asciidoc index aa0c7516a..0be2655c5 100644 --- a/doc/contributing.asciidoc +++ b/doc/contributing.asciidoc @@ -758,18 +758,30 @@ qutebrowser release * Make sure there are no unstaged changes and the tests are green. * Make sure all issues with the related milestone are closed. +* Mark the https://github.com/qutebrowser/qutebrowser/milestones[milestone] as closed. * Consider updating the completions for `content.headers.user_agent` in `configdata.yml`. * Minor release: Consider updating some files from main: - `misc/requirements/` and `requirements.txt` - `scripts/` -* Make sure Python is up-to-date on build machines. -* Mark the milestone at https://github.com/qutebrowser/qutebrowser/milestones as closed. -* Update changelog in main branch +* Update changelog in main branch and ensure the correct version number has `(unreleased)` * If necessary: Update changelog in release branch from main. + +**Automatic release via GitHub Actions (starting with v3.0.0):** + +* Double check Python version in `.github/workflows/release.yml` +* Run the `release` workflow on the `main` branch, e.g. via `gh workflow run release -f release_type=major` (`release_type` can be `major`, `minor` or `patch`; you can also override `python_version`) + +**Manual release:** + +* Make sure Python is up-to-date on build machines. * Run `./.venv/bin/python3 scripts/dev/update_version.py {major,minor,patch}`. * Run the printed instructions accordingly. + +**Post release:** + * Update `qutebrowser-git` PKGBUILD if dependencies/install changed. * Add unreleased future versions to changelog * Update IRC topic * Announce to qutebrowser and qutebrowser-announce mailinglist. * Post announcement mail to subreddit +* Post on the website formerly known as Twitter diff --git a/scripts/dev/update_version.py b/scripts/dev/update_version.py index 424a77559..b0f48710e 100644 --- a/scripts/dev/update_version.py +++ b/scripts/dev/update_version.py @@ -100,7 +100,7 @@ if __name__ == "__main__": "tox -e build-release -- --upload" .format(v=version)) print("* Windows: git fetch; git checkout v{v}; " - "py -3.9 -m tox -e build-release -- --upload" + "py -3.X -m tox -e build-release -- --upload" .format(v=version)) print("* macOS: git fetch && git checkout v{v} && " "tox -e build-release -- --upload" -- cgit v1.2.3-54-g00ecf From 9dfe641263b67cce45b4ab850a318d86c10310c9 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 17 Aug 2023 14:10:33 +0200 Subject: ci: Switch from experimental to real releases --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0c5b6d4b1..8425530b2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -169,10 +169,10 @@ jobs: # FIXME consider switching to trusted publishers: # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/ - name: Build and upload release - run: "tox -e build-release -- --upload --no-confirm --experimental" + run: "tox -e build-release -- --upload --no-confirm" env: TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.QUTEBROWSER_BOT_TEST_PYPI_TOKEN }} # FIXME use real token + TWINE_PASSWORD: ${{ secrets.QUTEBROWSER_BOT_PYPI_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} finalize: runs-on: ubuntu-20.04 -- cgit v1.2.3-54-g00ecf From ffbbc3d9e75e82f2fd60bdd117496e9bf31ff74a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 17 Aug 2023 14:28:43 +0200 Subject: Remove old download_release.sh Releases aren't hosted on qutebrowser.org anymore --- scripts/dev/download_release.sh | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 scripts/dev/download_release.sh diff --git a/scripts/dev/download_release.sh b/scripts/dev/download_release.sh deleted file mode 100644 index 207da21c8..000000000 --- a/scripts/dev/download_release.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -set -e - -# This script downloads the given release from GitHub so we can mirror it on -# qutebrowser.org. - -tmpdir=$(mktemp -d) -oldpwd=$PWD - -if [[ $# != 1 ]]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -cd "$tmpdir" -mkdir windows - -base="https://github.com/qutebrowser/qutebrowser/releases/download/v$1" - -wget "$base/qutebrowser-$1.tar.gz" -wget "$base/qutebrowser-$1.tar.gz.asc" -wget "$base/qutebrowser-$1.dmg" -wget "$base/qutebrowser_${1}-1_all.deb" - -cd windows -wget "$base/qutebrowser-${1}-amd64.msi" -wget "$base/qutebrowser-${1}-win32.msi" -wget "$base/qutebrowser-${1}-windows-standalone-amd64.zip" -wget "$base/qutebrowser-${1}-windows-standalone-win32.zip" - -dest="/srv/http/qutebrowser/releases/v$1" -cd "$oldpwd" -sudo mv "$tmpdir" "$dest" -sudo chown -R http:http "$dest" -- cgit v1.2.3-54-g00ecf From 8795b88d358deec34e3c3829f01eacded2f53a6a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 17 Aug 2023 14:51:53 +0200 Subject: ci: Fix JS style issues --- .github/workflows/release.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8425530b2..65dec84e8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,11 +54,7 @@ jobs: } console.log(`release_branches: ${release_branches}`); // Get newest release branch (biggest version number) - const sorted = release_branches.sort( - function (a, b) { - return a.localeCompare(b, undefined, { numeric: true, sensitivity: 'base' }); - } - ); + const sorted = release_branches.sort((a, b) => a.localeCompare(b, undefined, { numeric: true, sensitivity: 'base' })); console.log(`sorted: ${sorted}`); return sorted.at(-1); result-encoding: string @@ -190,7 +186,7 @@ jobs: repo: context.repo.repo, release_id: "${{ needs.prepare.outputs.release_id }}", draft: false, - body: "Check the [changelog](https://github.com/qutebrowser/qutebrowser/blob/master/doc/changelog.asciidoc) for changes in this release." + body: "Check the [changelog](https://github.com/qutebrowser/qutebrowser/blob/master/doc/changelog.asciidoc) for changes in this release.", }) irc: timeout-minutes: 2 -- cgit v1.2.3-54-g00ecf