From 611a6d5cb2f15182e14c2249d1b5cedc44385878 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 16 Jun 2022 19:36:24 +0200 Subject: build-release: Modernize pathlib, type annotations, modern syntax, dataclasses --- scripts/dev/build_release.py | 307 ++++++++++++++++++++++++++----------------- 1 file changed, 184 insertions(+), 123 deletions(-) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index b078c66a2..0529645a5 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -22,7 +22,6 @@ import os -import os.path import sys import time import shutil @@ -32,22 +31,40 @@ import argparse import tarfile import tempfile import collections +import dataclasses import re +from typing import Iterable, List, Optional try: import winreg except ImportError: pass -sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, - os.pardir)) +REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) import qutebrowser from scripts import utils from scripts.dev import update_3rdparty, misc_checks -def call_script(name, *args, python=sys.executable): +SCRIPTS_PATH = REPO_ROOT / "scripts" + + +@dataclasses.dataclass +class Artifact: + + """A single file being uploaded to GitHub.""" + + path: pathlib.Path + mimetype: str + description: str + + def __str__(self): + return f"{self.path} ({self.mimetype}): {self.description}" + + +def call_script(name: str, *args: str, python: str = sys.executable) -> None: """Call a given shell script. Args: @@ -55,11 +72,15 @@ def call_script(name, *args, python=sys.executable): *args: The arguments to pass. python: The python interpreter to use. """ - path = os.path.join(os.path.dirname(__file__), os.pardir, name) - subprocess.run([python, path] + list(args), check=True) + subprocess.run([python, REPO_ROOT / "scripts" / name, *args], check=True) -def call_tox(toxenv, *args, python=sys.executable, debug=False): +def call_tox( + toxenv: str, + *args: str, + python: pathlib.Path = pathlib.Path(sys.executable), + debug: bool = False, +) -> None: """Call tox. Args: @@ -69,17 +90,17 @@ def call_tox(toxenv, *args, python=sys.executable, debug=False): debug: Turn on pyinstaller debugging """ env = os.environ.copy() - env['PYTHON'] = python - env['PATH'] = os.environ['PATH'] + os.pathsep + os.path.dirname(python) + env['PYTHON'] = str(python) + env['PATH'] = os.environ['PATH'] + os.pathsep + str(python.parent) if debug: env['PYINSTALLER_DEBUG'] = '1' subprocess.run( - [sys.executable, '-m', 'tox', '-vv', '-e', toxenv] + list(args), + [sys.executable, '-m', 'tox', '-vv', '-e', toxenv, *args], env=env, check=True) -def run_asciidoc2html(args): - """Common buildsteps used for all OS'.""" +def run_asciidoc2html(args: argparse.Namespace) -> None: + """Run the asciidoc2html script.""" utils.print_title("Running asciidoc2html.py") a2h_args = [] if args.asciidoc is not None: @@ -89,7 +110,7 @@ def run_asciidoc2html(args): call_script('asciidoc2html.py', *a2h_args) -def _maybe_remove(path): +def _maybe_remove(path: pathlib.Path) -> None: """Remove a path if it exists.""" try: shutil.rmtree(path) @@ -97,13 +118,17 @@ def _maybe_remove(path): pass -def _filter_whitelisted(output, patterns): +def _filter_whitelisted(output: bytes, patterns: Iterable[str]) -> Iterable[str]: + """Get all lines not matching any of the given regex patterns.""" for line in output.decode('utf-8').splitlines(): if not any(re.fullmatch(pattern, line) for pattern in patterns): yield line -def _smoke_test_run(executable, *args): +def _smoke_test_run( + executable: pathlib.Path, + *args: str, +) -> subprocess.CompletedProcess: """Get a subprocess to run a smoke test.""" argv = [ executable, @@ -117,7 +142,7 @@ def _smoke_test_run(executable, *args): return subprocess.run(argv, check=True, capture_output=True) -def smoke_test(executable, debug): +def smoke_test(executable: pathlib.Path, debug: bool) -> None: """Try starting the given qutebrowser executable.""" stdout_whitelist = [] stderr_whitelist = [ @@ -206,43 +231,50 @@ def smoke_test(executable, debug): raise Exception("\n".join(lines)) -def verify_windows_exe(exe_path): +def verify_windows_exe(exe_path: pathlib.Path) -> None: """Make sure the Windows .exe has a correct checksum.""" import pefile pe = pefile.PE(exe_path) assert pe.verify_checksum() -def patch_mac_app(): - """Patch .app to save some space.""" - app_path = os.path.join('dist', 'qutebrowser.app') +def patch_mac_app() -> None: + """Patch .adpp to save some space.""" + dist_path = pathlib.Path('dist') + app_path = dist_path / 'qutebrowser.app' # Replace some duplicate files by symlinks - framework_path = os.path.join(app_path, 'Contents', 'MacOS', 'PyQt5', - 'Qt5', 'lib', 'QtWebEngineCore.framework') - - core_lib = os.path.join(framework_path, 'Versions', '5', 'QtWebEngineCore') - os.remove(core_lib) - core_target = os.path.join(*[os.pardir] * 7, 'MacOS', 'QtWebEngineCore') - os.symlink(core_target, core_lib) - - framework_resource_path = os.path.join(framework_path, 'Resources') - for name in os.listdir(framework_resource_path): - file_path = os.path.join(framework_resource_path, name) - target = os.path.join(*[os.pardir] * 5, name) - if os.path.isdir(file_path): + framework_path = ( + app_path / 'Contents' / 'MacOS' / 'PyQt5' / 'Qt5' / 'lib' / + 'QtWebEngineCore.framework' + ) + + core_lib = framework_path / 'Versions' / '5' / 'QtWebEngineCore' + core_lib.unlink() + core_target = pathlib.Path(*[os.pardir] * 7, 'MacOS', 'QtWebEngineCore') + core_lib.symlink_to(core_target) + + framework_resource_path = framework_path / 'Resources' + for file_path in framework_resource_path.iterdir(): + target = pathlib.Path(*[os.pardir] * 5, file_path.name) + if file_path.is_dir(): shutil.rmtree(file_path) else: - os.remove(file_path) - os.symlink(target, file_path) + file_path.unlink() + file_path.symlink_to(target) -def _mac_bin_path(base): +def _mac_bin_path(base: pathlib.Path) -> pathlib.Path: """Get the macOS qutebrowser binary path.""" - return os.path.join(base, 'qutebrowser.app', 'Contents', 'MacOS', 'qutebrowser') + return pathlib.Path(base, 'qutebrowser.app', 'Contents', 'MacOS', 'qutebrowser') -def build_mac(*, gh_token, skip_packaging, debug): +def build_mac( + *, + gh_token: Optional[str], + skip_packaging: bool, + debug: bool, +) -> List[Artifact]: """Build macOS .dmg/.app.""" utils.print_title("Cleaning up...") for f in ['wc.dmg', 'template.dmg']: @@ -263,74 +295,87 @@ def build_mac(*, gh_token, skip_packaging, debug): utils.print_title("Patching .app") patch_mac_app() + dist_path = pathlib.Path("dist") + utils.print_title("Running pre-dmg smoke test") - smoke_test(_mac_bin_path('dist'), debug=debug) + smoke_test(_mac_bin_path(dist_path), debug=debug) if skip_packaging: return [] utils.print_title("Building .dmg") - subprocess.run(['make', '-f', 'scripts/dev/Makefile-dmg'], check=True) + dmg_makefile_path = REPO_ROOT / "scripts" / "dev" / "Makefile-dmg" + subprocess.run(['make', '-f', dmg_makefile_path], check=True) suffix = "-debug" if debug else "" - dmg_path = f'dist/qutebrowser-{qutebrowser.__version__}{suffix}.dmg' - os.rename('qutebrowser.dmg', dmg_path) + dmg_path = dist_path / f'qutebrowser-{qutebrowser.__version__}{suffix}.dmg' + pathlib.Path('qutebrowser.dmg').rename(dmg_path) utils.print_title("Running smoke test") try: with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = pathlib.Path(tmpdir) subprocess.run(['hdiutil', 'attach', dmg_path, - '-mountpoint', tmpdir], check=True) + '-mountpoint', tmp_path], check=True) try: - smoke_test(_mac_bin_path(tmpdir), debug=debug) + smoke_test(_mac_bin_path(tmp_path), debug=debug) finally: print("Waiting 10s for dmg to be detachable...") time.sleep(10) - subprocess.run(['hdiutil', 'detach', tmpdir], check=False) + subprocess.run(['hdiutil', 'detach', tmp_path], check=False) except PermissionError as e: - print("Failed to remove tempdir: {}".format(e)) + print(f"Failed to remove tempdir: {e}") - return [(dmg_path, 'application/x-apple-diskimage', 'macOS .dmg')] + return [ + Artifact( + path=dmg_path, + mimetype='application/x-apple-diskimage', + description='macOS .dmg' + ) + ] -def _get_windows_python_path(x64): +def _get_windows_python_path(x64: bool) -> pathlib.Path: """Get the path to Python.exe on Windows.""" parts = str(sys.version_info.major), str(sys.version_info.minor) ver = ''.join(parts) dot_ver = '.'.join(parts) if x64: - path = (r'SOFTWARE\Python\PythonCore\{}\InstallPath' - .format(dot_ver)) - fallback = r'C:\Python{}\python.exe'.format(ver) + path = rf'SOFTWARE\Python\PythonCore\{dot_ver}\InstallPath' + fallback = pathlib.Path('C:', f'Python{ver}', 'python.exe') else: - path = (r'SOFTWARE\WOW6432Node\Python\PythonCore\{}-32\InstallPath' - .format(dot_ver)) - fallback = r'C:\Python{}-32\python.exe'.format(ver) + path = rf'SOFTWARE\WOW6432Node\Python\PythonCore\{dot_ver}-32\InstallPath' + fallback = pathlib.Path('C:', f'Python{ver}-32', 'python.exe') try: key = winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE, path) - return winreg.QueryValueEx(key, 'ExecutablePath')[0] + return pathlib.Path(winreg.QueryValueEx(key, 'ExecutablePath')[0]) except FileNotFoundError: return fallback -def _build_windows_single(*, x64, skip_packaging, debug): +def _build_windows_single( + *, x64: bool, + skip_packaging: bool, + debug: bool, +) -> List[Artifact]: """Build on Windows for a single architecture.""" human_arch = '64-bit' if x64 else '32-bit' utils.print_title(f"Running pyinstaller {human_arch}") + dist_path = pathlib.Path("dist") - outdir = os.path.join( - 'dist', f'qutebrowser-{qutebrowser.__version__}-{"x64" if x64 else "x86"}') - _maybe_remove(outdir) + arch = "x64" if x64 else "x86" + out_path = dist_path / 'dist' / f'qutebrowser-{qutebrowser.__version__}-{arch}' + _maybe_remove(out_path) python = _get_windows_python_path(x64=x64) call_tox(f'pyinstaller-{"64" if x64 else "32"}', '-r', python=python, debug=debug) - out_pyinstaller = os.path.join('dist', 'qutebrowser') - shutil.move(out_pyinstaller, outdir) - exe_path = os.path.join(outdir, 'qutebrowser.exe') + out_pyinstaller = dist_path / "qutebrowser" + shutil.move(out_pyinstaller, out_path) + exe_path = out_path / 'qutebrowser.exe' utils.print_title(f"Verifying {human_arch} exe") verify_windows_exe(exe_path) @@ -344,7 +389,7 @@ def _build_windows_single(*, x64, skip_packaging, debug): utils.print_title(f"Packaging {human_arch}") return _package_windows_single( nsis_flags=[] if x64 else ['/DX86'], - outdir=outdir, + out_path=out_path, filename_arch='amd64' if x64 else 'win32', desc_arch=human_arch, desc_suffix='' if x64 else ' (only for 32-bit Windows!)', @@ -352,7 +397,13 @@ def _build_windows_single(*, x64, skip_packaging, debug): ) -def build_windows(*, gh_token, skip_packaging, only_32bit, only_64bit, debug): +def build_windows( + *, gh_token: str, + skip_packaging: bool, + only_32bit: bool, + only_64bit: bool, + debug: bool, +) -> List[Artifact]: """Build windows executables/setups.""" utils.print_title("Updating 3rdparty content") # FIXME:qt6 Use modern PDF.js version here @@ -385,16 +436,17 @@ def build_windows(*, gh_token, skip_packaging, only_32bit, only_64bit, debug): def _package_windows_single( *, - nsis_flags, - outdir, - desc_arch, - desc_suffix, - filename_arch, - debug, -): + nsis_flags: List[str], + out_path: pathlib.Path, + desc_arch: str, + desc_suffix: str, + filename_arch: str, + debug: bool, +) -> List[Artifact]: """Build the given installer/zip for windows.""" artifacts = [] + dist_path = pathlib.Path("dist") utils.print_subtitle(f"Building {desc_arch} installer...") subprocess.run(['makensis.exe', f'/DVERSION={qutebrowser.__version__}', *nsis_flags, @@ -409,10 +461,10 @@ def _package_windows_single( name_parts.append('debug') name = '-'.join(name_parts) + '.exe' - artifacts.append(( - os.path.join('dist', name), - 'application/vnd.microsoft.portable-executable', - f'Windows {desc_arch} installer{desc_suffix}', + artifacts.append(Artifact( + path=dist_path / name, + mimetype='application/vnd.microsoft.portable-executable', + description=f'Windows {desc_arch} installer{desc_suffix}', )) utils.print_subtitle(f"Zipping {desc_arch} standalone...") @@ -427,18 +479,18 @@ def _package_windows_single( zip_name_parts.append('debug') zip_name = '-'.join(zip_name_parts) - zip_path = os.path.join('dist', zip_name) - shutil.make_archive(zip_path, 'zip', 'dist', os.path.basename(outdir)) - artifacts.append(( - f'{zip_path}.zip', - 'application/zip', - f'Windows {desc_arch} standalone{desc_suffix}' + zip_path = dist_path / zip_name + shutil.make_archive(str(zip_path), 'zip', 'dist', out_path.name) + artifacts.append(Artifact( + path=zip_path.with_suffix(".zip"), + mimetype='application/zip', + description=f'Windows {desc_arch} standalone{desc_suffix}', )) return artifacts -def build_sdist(): +def build_sdist() -> List[Artifact]: """Build an sdist and list the contents.""" utils.print_title("Building sdist") @@ -448,7 +500,7 @@ def build_sdist(): subprocess.run([sys.executable, '-m', 'build'], check=True) dist_files = list(dist_path.glob('*.tar.gz')) - filename = 'qutebrowser-{}.tar.gz'.format(qutebrowser.__version__) + filename = f'qutebrowser-{qutebrowser.__version__}.tar.gz' assert dist_files == [dist_path / filename], dist_files dist_file = dist_files[0] @@ -460,45 +512,51 @@ def build_sdist(): for tarinfo in tar.getmembers(): if not tarinfo.isfile(): continue - name = os.sep.join(tarinfo.name.split(os.sep)[1:]) - _base, ext = os.path.splitext(name) - by_ext[ext].append(name) + path = pathlib.Path(*pathlib.Path(tarinfo.name).parts[1:]) + by_ext[path.suffix].append(path) assert '.pyc' not in by_ext utils.print_title("sdist contents") - for ext, files in sorted(by_ext.items()): + for ext, paths in sorted(by_ext.items()): utils.print_subtitle(ext) - print('\n'.join(files)) + print('\n'.join(str(p) for p in paths)) artifacts = [ - (str(dist_file), 'application/gzip', 'Source release'), - ( - str(dist_file.with_suffix(dist_file.suffix + '.asc')), - 'application/pgp-signature', - 'Source release - PGP signature', + Artifact( + path=dist_file, + mimetype='application/gzip', + description='Source release', + ), + Artifact( + path=dist_file.with_suffix(dist_file.suffix + '.asc'), + mimetype='application/pgp-signature', + description='Source release - PGP signature', ), ] return artifacts -def test_makefile(): +def test_makefile() -> None: """Make sure the Makefile works correctly.""" utils.print_title("Testing makefile") with tempfile.TemporaryDirectory() as tmpdir: subprocess.run(['make', '-f', 'misc/Makefile', - 'DESTDIR={}'.format(tmpdir), 'install'], check=True) + f'DESTDIR={tmpdir}', 'install'], check=True) -def read_github_token(arg_token, *, optional=False): +def read_github_token( + arg_token: Optional[str], *, + optional: bool = False, +) -> Optional[str]: """Read the GitHub API token from disk.""" if arg_token is not None: return arg_token - token_file = os.path.join(os.path.expanduser('~'), '.gh_token') - if not os.path.exists(token_file): + token_path = pathlib.Path.home() / '.gh_token' + if not token_path.exists(): if optional: return None else: @@ -506,16 +564,15 @@ def read_github_token(arg_token, *, optional=False): "GitHub token needed, but ~/.gh_token not found, " "and --gh-token not given.") - with open(token_file, encoding='ascii') as f: - token = f.read().strip() + token = token_path.read_text(encoding="ascii").strip() return token -def github_upload(artifacts, tag, gh_token): +def github_upload(artifacts: List[Artifact], tag: str, gh_token: str) -> None: """Upload the given artifacts to GitHub. Args: - artifacts: A list of (filename, mimetype, description) tuples + artifacts: A list of Artifacts to upload. tag: The name of the release tag gh_token: The GitHub token to use """ @@ -531,58 +588,61 @@ def github_upload(artifacts, tag, gh_token): if release.tag_name == tag: break else: - raise Exception("No release found for {!r}!".format(tag)) + raise Exception(f"No release found for {tag!r}!") - for filename, mimetype, description in artifacts: + for artifact in artifacts: while True: - print("Uploading {}".format(filename)) + print("Uploading {path}") - basename = os.path.basename(filename) assets = [asset for asset in release.assets() - if asset.name == basename] + if asset.name == artifact.path.name] if assets: - print("Assets already exist: {}".format(assets)) + print(f"Assets already exist: {assets}") print("Press enter to continue anyways or Ctrl-C to abort.") input() try: - with open(filename, 'rb') as f: - release.upload_asset(mimetype, basename, f, description) + with artifact.path.open('rb') as f: + release.upload_asset( + artifact.mimetype, + artifact.path.name, + f, + artifact.description, + ) except github3.exceptions.ConnectionError as e: - utils.print_error('Failed to upload: {}'.format(e)) + utils.print_error(f'Failed to upload: {e}') print("Press Enter to retry...", file=sys.stderr) input() print("Retrying!") assets = [asset for asset in release.assets() - if asset.name == basename] + if asset.name == artifact.path.name] if assets: stray_asset = assets[0] - print("Deleting stray asset {}".format(stray_asset.name)) + print(f"Deleting stray asset {stray_asset.name}") stray_asset.delete() else: break -def pypi_upload(artifacts): +def pypi_upload(artifacts: List[Artifact]) -> None: """Upload the given artifacts to PyPI using twine.""" utils.print_title("Uploading to PyPI...") run_twine('upload', artifacts) -def twine_check(artifacts): +def twine_check(artifacts: List[Artifact]) -> None: """Check packages using 'twine check'.""" utils.print_title("Running twine check...") run_twine('check', artifacts, '--strict') -def run_twine(command, artifacts, *args): - filenames = [a[0] for a in artifacts] - subprocess.run([sys.executable, '-m', 'twine', command] + list(args) + filenames, - check=True) +def run_twine(command: str, artifacts: List[Artifact], *args: str) -> None: + paths = [a.path for a in artifacts] + subprocess.run([sys.executable, '-m', 'twine', command, *args, *paths], check=True) -def main(): +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('--skip-docs', action='store_true', help="Don't generate docs") @@ -624,7 +684,7 @@ def main(): sys.exit(1) if args.skip_docs: - os.makedirs(os.path.join('qutebrowser', 'html', 'doc'), exist_ok=True) + pathlib.Path("qutebrowser", "html", "doc").mkdir(parents=True, exist_ok=True) else: run_asciidoc2html(args) @@ -649,12 +709,13 @@ def main(): upload_to_pypi = True if args.upload: - version_tag = "v" + qutebrowser.__version__ + version_tag = f"v{qutebrowser.__version__}" if not args.no_confirm: - utils.print_title("Press enter to release {}...".format(version_tag)) + utils.print_title(f"Press enter to release {version_tag}...") input() + assert gh_token is not None github_upload(artifacts, version_tag, gh_token=gh_token) if upload_to_pypi: pypi_upload(artifacts) -- cgit v1.2.3-54-g00ecf