path: root/scripts/dev/build_release.py

#!/usr/bin/env python3
# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:

# Copyright 2014-2021 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
#
# This file is part of qutebrowser.
#
# qutebrowser is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# qutebrowser is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with qutebrowser.  If not, see <https://www.gnu.org/licenses/>.

"""Build a new release."""


import os
import os.path
import sys
import time
import shutil
import plistlib
import subprocess
import argparse
import tarfile
import tempfile
import collections
import re

try:
    import winreg
except ImportError:
    pass

sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
                                os.pardir))

import qutebrowser
from scripts import utils
from scripts.dev import update_3rdparty, misc_checks


def call_script(name, *args, python=sys.executable):
    """Call a given shell script.

    Args:
        name: The script to call.
        *args: The arguments to pass.
        python: The python interpreter to use.
    """
    path = os.path.join(os.path.dirname(__file__), os.pardir, name)
    subprocess.run([python, path] + list(args), check=True)


def call_tox(toxenv, *args, python=sys.executable, debug=False):
    """Call tox.

    Args:
        toxenv: Which tox environment to use
        *args: The arguments to pass.
        python: The python interpreter to use.
        debug: Turn on pyinstaller debugging
    """
    env = os.environ.copy()
    env['PYTHON'] = python
    env['PATH'] = os.environ['PATH'] + os.pathsep + os.path.dirname(python)
    if debug:
        env['PYINSTALLER_DEBUG'] = '1'
    subprocess.run(
        [sys.executable, '-m', 'tox', '-vv', '-e', toxenv] + list(args),
        env=env, check=True)


def run_asciidoc2html(args):
    """Common buildsteps used for all OS'."""
    utils.print_title("Running asciidoc2html.py")
    a2h_args = []
    if args.asciidoc is not None:
        a2h_args += ['--asciidoc', args.asciidoc]
    if args.asciidoc_python is not None:
        a2h_args += ['--asciidoc-python', args.asciidoc_python]
    call_script('asciidoc2html.py', *a2h_args)


def _maybe_remove(path):
    """Remove a path if it exists."""
    try:
        shutil.rmtree(path)
    except FileNotFoundError:
        pass


def _filter_whitelisted(output, patterns):
    for line in output.decode('utf-8').splitlines():
        if not any(re.fullmatch(pattern, line) for pattern in patterns):
            yield line


def _smoke_test_run(executable, *args):
    """Get a subprocess to run a smoke test."""
    argv = [
        executable,
        '--no-err-windows',
        '--nowindow',
        '--temp-basedir',
        *args,
        'about:blank',
        ':later 500 quit',
    ]
    return subprocess.run(argv, check=True, capture_output=True)


def smoke_test(executable, debug):
    """Try starting the given qutebrowser executable."""
    stdout_whitelist = []
    stderr_whitelist = [
        # PyInstaller debug output
        r'\[.*\] PyInstaller Bootloader .*',
        r'\[.*\] LOADER: .*',

        # https://github.com/qutebrowser/qutebrowser/issues/4919
        (r'objc\[.*\]: .* One of the two will be used\. '
         r'Which one is undefined\.'),
        (r'QCoreApplication::applicationDirPath: Please instantiate the '
         r'QApplication object first'),
        (r'\[.*:ERROR:mach_port_broker.mm\(48\)\] bootstrap_look_up '
         r'org\.chromium\.Chromium\.rohitfork\.1: Permission denied \(1100\)'),
        (r'\[.*:ERROR:mach_port_broker.mm\(43\)\] bootstrap_look_up: '
         r'Unknown service name \(1102\)'),

        (r'[0-9:]* WARNING: The available OpenGL surface format was either not '
         r'version 3\.2 or higher or not a Core Profile\.'),
        r'Chromium on macOS will fall back to software rendering in this case\.',
        r'Hardware acceleration and features such as WebGL will not be available\.',
        r'Unable to create basic Accelerated OpenGL renderer\.',
        r'Core Image is now using the software OpenGL renderer\. This will be slow\.',

        # Windows N:
        # https://github.com/microsoft/playwright/issues/2901
        (r'\[.*:ERROR:dxva_video_decode_accelerator_win.cc\(\d+\)\] '
         r'DXVAVDA fatal error: could not LoadLibrary: .*: The specified '
         r'module could not be found. \(0x7E\)'),

        # https://github.com/qutebrowser/qutebrowser/issues/3719
        '[0-9:]* ERROR: Load error: ERR_FILE_NOT_FOUND',
    ]

    proc = _smoke_test_run(executable)
    if debug:
        print("Skipping output check for debug build")
        return

    stdout = '\n'.join(_filter_whitelisted(proc.stdout, stdout_whitelist))
    stderr = '\n'.join(_filter_whitelisted(proc.stderr, stderr_whitelist))

    if stdout or stderr:
        print("Unexpected output, running with --debug")
        proc = _smoke_test_run(executable, '--debug')
        debug_stdout = proc.stdout.decode('utf-8')
        debug_stderr = proc.stderr.decode('utf-8')

        lines = [
            "Unexpected output!",
            "",
        ]
        if stdout:
            lines += [
                "stdout",
                "------",
                "",
                stdout,
                "",
            ]
        if stderr:
            lines += [
                "stderr",
                "------",
                "",
                stderr,
                "",
            ]
        if debug_stdout:
            lines += [
                "debug rerun stdout",
                "------------------",
                "",
                debug_stdout,
                "",
            ]
        if debug_stderr:
            lines += [
                "debug rerun stderr",
                "------------------",
                "",
                debug_stderr,
                "",
            ]

        raise Exception("\n".join(lines))


def verify_windows_exe(exe_path):
    """Make sure the Windows .exe has a correct checksum."""
    import pefile
    pe = pefile.PE(exe_path)
    assert pe.verify_checksum()


def patch_mac_app():
    """Patch .app to use our Info.plist and save some space."""
    app_path = os.path.join('dist', 'qutebrowser.app')

    # Patch Info.plist - pyinstaller's options are too limiting
    plist_path = os.path.join(app_path, 'Contents', 'Info.plist')
    with open(plist_path, "rb") as f:
        plist_data = plistlib.load(f)
    plist_data.update(INFO_PLIST_UPDATES)
    with open(plist_path, "wb") as f:
        plistlib.dump(plist_data, f)

    # Replace some duplicate files by symlinks
    framework_path = os.path.join(app_path, 'Contents', 'MacOS', 'PyQt5',
                                  'Qt', 'lib', 'QtWebEngineCore.framework')

    core_lib = os.path.join(framework_path, 'Versions', '5', 'QtWebEngineCore')
    os.remove(core_lib)
    core_target = os.path.join(*[os.pardir] * 7, 'MacOS', 'QtWebEngineCore')
    os.symlink(core_target, core_lib)

    framework_resource_path = os.path.join(framework_path, 'Resources')
    for name in os.listdir(framework_resource_path):
        file_path = os.path.join(framework_resource_path, name)
        target = os.path.join(*[os.pardir] * 5, name)
        if os.path.isdir(file_path):
            shutil.rmtree(file_path)
        else:
            os.remove(file_path)
        os.symlink(target, file_path)


INFO_PLIST_UPDATES = {
    'CFBundleVersion': qutebrowser.__version__,
    'CFBundleShortVersionString': qutebrowser.__version__,
    'NSSupportsAutomaticGraphicsSwitching': True,
    'NSHighResolutionCapable': True,
    'NSRequiresAquaSystemAppearance': False,
    'CFBundleURLTypes': [{
        "CFBundleURLName": "http(s) URL",
        "CFBundleURLSchemes": ["http", "https"]
    }, {
        "CFBundleURLName": "local file URL",
        "CFBundleURLSchemes": ["file"]
    }],
    'CFBundleDocumentTypes': [{
        "CFBundleTypeExtensions": ["html", "htm"],
        "CFBundleTypeMIMETypes": ["text/html"],
        "CFBundleTypeName": "HTML document",
        "CFBundleTypeOSTypes": ["HTML"],
        "CFBundleTypeRole": "Viewer",
    }, {
        "CFBundleTypeExtensions": ["xhtml"],
        "CFBundleTypeMIMETypes": ["text/xhtml"],
        "CFBundleTypeName": "XHTML document",
        "CFBundleTypeRole": "Viewer",
    }],

    # https://developer.apple.com/documentation/avfoundation/cameras_and_media_capture/requesting_authorization_for_media_capture_on_macos
    #
    # Keys based on Google Chrome's .app, except Bluetooth keys which seem to
    # be iOS-only.
    #
    # If we don't do this, we get a SIGABRT from macOS when those permissions
    # are used, and even in some other situations (like logging into Google
    # accounts)...
    'NSCameraUsageDescription':
        'A website in qutebrowser wants to use the camera.',
    'NSLocationUsageDescription':
        'A website in qutebrowser wants to use your location information.',
    'NSMicrophoneUsageDescription':
        'A website in qutebrowser wants to use your microphone.',
}


def build_mac(*, gh_token, debug):
    """Build macOS .dmg/.app."""
    utils.print_title("Cleaning up...")
    for f in ['wc.dmg', 'template.dmg']:
        try:
            os.remove(f)
        except FileNotFoundError:
            pass
    for d in ['dist', 'build']:
        shutil.rmtree(d, ignore_errors=True)
    utils.print_title("Updating 3rdparty content")
    update_3rdparty.run(ace=False, pdfjs=True, fancy_dmg=False, gh_token=gh_token)
    utils.print_title("Building .app via pyinstaller")
    call_tox('pyinstaller-64', '-r', debug=debug)
    utils.print_title("Patching .app")
    patch_mac_app()
    utils.print_title("Building .dmg")
    subprocess.run(['make', '-f', 'scripts/dev/Makefile-dmg'], check=True)

    suffix = "-debug" if debug else ""
    dmg_path = f'dist/qutebrowser-{qutebrowser.__version__}{suffix}.dmg'
    os.rename('qutebrowser.dmg', dmg_path)

    utils.print_title("Running smoke test")

    try:
        with tempfile.TemporaryDirectory() as tmpdir:
            subprocess.run(['hdiutil', 'attach', dmg_path,
                            '-mountpoint', tmpdir], check=True)
            try:
                binary = os.path.join(tmpdir, 'qutebrowser.app', 'Contents',
                                      'MacOS', 'qutebrowser')
                smoke_test(binary, debug=debug)
            finally:
                print("Waiting 10s for dmg to be detachable...")
                time.sleep(10)
                subprocess.run(['hdiutil', 'detach', tmpdir], check=False)
    except PermissionError as e:
        print("Failed to remove tempdir: {}".format(e))

    return [(dmg_path, 'application/x-apple-diskimage', 'macOS .dmg')]


def _get_windows_python_path(x64):
    """Get the path to Python.exe on Windows."""
    parts = str(sys.version_info.major), str(sys.version_info.minor)
    ver = ''.join(parts)
    dot_ver = '.'.join(parts)

    if x64:
        path = (r'SOFTWARE\Python\PythonCore\{}\InstallPath'
                .format(dot_ver))
        fallback = r'C:\Python{}\python.exe'.format(ver)
    else:
        path = (r'SOFTWARE\WOW6432Node\Python\PythonCore\{}-32\InstallPath'
                .format(dot_ver))
        fallback = r'C:\Python{}-32\python.exe'.format(ver)

    try:
        key = winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE, path)
        return winreg.QueryValueEx(key, 'ExecutablePath')[0]
    except FileNotFoundError:
        return fallback


def _build_windows_single(*, x64, skip_packaging, debug):
    """Build on Windows for a single architecture."""
    human_arch = '64-bit' if x64 else '32-bit'
    utils.print_title(f"Running pyinstaller {human_arch}")

    outdir = os.path.join(
        'dist', f'qutebrowser-{qutebrowser.__version__}-{"x64" if x64 else "x86"}')
    _maybe_remove(outdir)

    python = _get_windows_python_path(x64=x64)
    call_tox(f'pyinstaller-{"64" if x64 else "32"}', '-r', python=python, debug=debug)

    out_pyinstaller = os.path.join('dist', 'qutebrowser')
    shutil.move(out_pyinstaller, outdir)
    exe_path = os.path.join(outdir, 'qutebrowser.exe')

    utils.print_title(f"Verifying {human_arch} exe")
    verify_windows_exe(exe_path)

    utils.print_title(f"Running {human_arch} smoke test")
    smoke_test(exe_path, debug=debug)

    if skip_packaging:
        return []

    utils.print_title(f"Packaging {human_arch}")
    return _package_windows_single(
        nsis_flags=[] if x64 else ['/DX86'],
        outdir=outdir,
        filename_arch='amd64' if x64 else 'win32',
        desc_arch=human_arch,
        desc_suffix='' if x64 else ' (only for 32-bit Windows!)',
        debug=debug,
    )


def build_windows(*, gh_token, skip_packaging, only_32bit, only_64bit, debug):
    """Build windows executables/setups."""
    utils.print_title("Updating 3rdparty content")
    update_3rdparty.run(nsis=True, ace=False, pdfjs=True, fancy_dmg=False,
                        gh_token=gh_token)

    utils.print_title("Building Windows binaries")

    artifacts = []

    from scripts.dev import gen_versioninfo
    utils.print_title("Updating VersionInfo file")
    gen_versioninfo.main()

    if not only_32bit:
        artifacts += _build_windows_single(
            x64=True,
            skip_packaging=skip_packaging,
            debug=debug,
        )
    if not only_64bit:
        artifacts += _build_windows_single(
            x64=False,
            skip_packaging=skip_packaging,
            debug=debug,
        )

    return artifacts


def _package_windows_single(
    *,
    nsis_flags,
    outdir,
    desc_arch,
    desc_suffix,
    filename_arch,
    debug,
):
    """Build the given installer/zip for windows."""
    artifacts = []

    utils.print_subtitle(f"Building {desc_arch} installer...")
    subprocess.run(['makensis.exe',
                    f'/DVERSION={qutebrowser.__version__}', *nsis_flags,
                    'misc/nsis/qutebrowser.nsi'], check=True)

    name_parts = [
        'qutebrowser',
        str(qutebrowser.__version__),
        filename_arch,
    ]
    if debug:
        name_parts.append('debug')
    name = '-'.join(name_parts) + '.exe'

    artifacts.append((
        os.path.join('dist', name),
        'application/vnd.microsoft.portable-executable',
        f'Windows {desc_arch} installer{desc_suffix}',
    ))

    utils.print_subtitle(f"Zipping {desc_arch} standalone...")
    zip_name_parts = [
        'qutebrowser',
        str(qutebrowser.__version__),
        'windows',
        'standalone',
        filename_arch,
    ]
    if debug:
        zip_name_parts.append('debug')
    zip_name = '-'.join(zip_name_parts)

    zip_path = os.path.join('dist', zip_name)
    shutil.make_archive(zip_path, 'zip', 'dist', os.path.basename(outdir))
    artifacts.append((
        f'{zip_path}.zip',
        'application/zip',
        f'Windows {desc_arch} standalone{desc_suffix}'
    ))

    return artifacts


def build_sdist():
    """Build an sdist and list the contents."""
    utils.print_title("Building sdist")

    _maybe_remove('dist')

    subprocess.run([sys.executable, 'setup.py', 'sdist'], check=True)
    dist_files = os.listdir(os.path.abspath('dist'))
    assert len(dist_files) == 1

    dist_file = os.path.join('dist', dist_files[0])
    subprocess.run(['gpg', '--detach-sign', '-a', dist_file], check=True)

    tar = tarfile.open(dist_file)
    by_ext = collections.defaultdict(list)

    for tarinfo in tar.getmembers():
        if not tarinfo.isfile():
            continue
        name = os.sep.join(tarinfo.name.split(os.sep)[1:])
        _base, ext = os.path.splitext(name)
        by_ext[ext].append(name)

    assert '.pyc' not in by_ext

    utils.print_title("sdist contents")

    for ext, files in sorted(by_ext.items()):
        utils.print_subtitle(ext)
        print('\n'.join(files))

    filename = 'qutebrowser-{}.tar.gz'.format(qutebrowser.__version__)
    artifacts = [
        (os.path.join('dist', filename), 'application/gzip', 'Source release'),
        (os.path.join('dist', filename + '.asc'), 'application/pgp-signature',
         'Source release - PGP signature'),
    ]

    return artifacts


def test_makefile():
    """Make sure the Makefile works correctly."""
    utils.print_title("Testing makefile")
    with tempfile.TemporaryDirectory() as tmpdir:
        subprocess.run(['make', '-f', 'misc/Makefile',
                        'DESTDIR={}'.format(tmpdir), 'install'], check=True)


def read_github_token(arg_token, *, optional=False):
    """Read the GitHub API token from disk."""
    if arg_token is not None:
        return arg_token

    token_file = os.path.join(os.path.expanduser('~'), '.gh_token')
    if not os.path.exists(token_file):
        if optional:
            return None
        else:
            raise Exception(
                "GitHub token needed, but ~/.gh_token not found, "
                "and --gh-token not given.")

    with open(token_file, encoding='ascii') as f:
        token = f.read().strip()
    return token


def github_upload(artifacts, tag, gh_token):
    """Upload the given artifacts to GitHub.

    Args:
        artifacts: A list of (filename, mimetype, description) tuples
        tag: The name of the release tag
    """
    import github3
    import github3.exceptions
    utils.print_title("Uploading to github...")

    gh = github3.login(token=gh_token)
    repo = gh.repository('qutebrowser', 'qutebrowser')

    release = None  # to satisfy pylint
    for release in repo.releases():
        if release.tag_name == tag:
            break
    else:
        raise Exception("No release found for {!r}!".format(tag))

    for filename, mimetype, description in artifacts:
        while True:
            print("Uploading {}".format(filename))

            basename = os.path.basename(filename)
            assets = [asset for asset in release.assets()
                      if asset.name == basename]
            if assets:
                print("Assets already exist: {}".format(assets))
                print("Press enter to continue anyways or Ctrl-C to abort.")
                input()

            try:
                with open(filename, 'rb') as f:
                    release.upload_asset(mimetype, basename, f, description)
            except github3.exceptions.ConnectionError as e:
                utils.print_error('Failed to upload: {}'.format(e))
                print("Press Enter to retry...", file=sys.stderr)
                input()
                print("Retrying!")

                assets = [asset for asset in release.assets()
                          if asset.name == basename]
                if assets:
                    asset = assets[0]
                    print("Deleting stray asset {}".format(asset.name))
                    asset.delete()
            else:
                break


def pypi_upload(artifacts):
    """Upload the given artifacts to PyPI using twine."""
    utils.print_title("Uploading to PyPI...")
    filenames = [a[0] for a in artifacts]
    subprocess.run([sys.executable, '-m', 'twine', 'upload'] + filenames,
                   check=True)


def upgrade_sdist_dependencies():
    """Make sure we have the latest tools for an sdist release."""
    subprocess.run([sys.executable, '-m', 'pip', 'install', '-U', 'twine',
                    'pip', 'wheel', 'setuptools'], check=True)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--skip-docs', action='store_true',
                        help="Don't generate docs")
    parser.add_argument('--asciidoc', help="Full path to asciidoc.py. "
                        "If not given, it's searched in PATH.",
                        nargs='?')
    parser.add_argument('--asciidoc-python', help="Python to use for asciidoc."
                        "If not given, the current Python interpreter is used.",
                        nargs='?')
    parser.add_argument('--gh-token', help="GitHub token to use.",
                        nargs='?')
    parser.add_argument('--upload', action='store_true', required=False,
                        help="Toggle to upload the release to GitHub.")
    parser.add_argument('--no-confirm', action='store_true', required=False,
                        help="Skip confirmation before uploading.")
    parser.add_argument('--skip-packaging', action='store_true', required=False,
                        help="Skip Windows installer/zip generation.")
    parser.add_argument('--32bit', action='store_true', required=False,
                        help="Skip Windows 64 bit build.", dest='only_32bit')
    parser.add_argument('--64bit', action='store_true', required=False,
                        help="Skip Windows 32 bit build.", dest='only_64bit')
    parser.add_argument('--debug', action='store_true', required=False,
                        help="Build a debug build.")
    args = parser.parse_args()
    utils.change_cwd()

    upload_to_pypi = False

    if args.upload:
        # Fail early when trying to upload without github3 installed
        # or without API token
        import github3  # pylint: disable=unused-import
        gh_token = read_github_token(args.gh_token)
    else:
        gh_token = read_github_token(args.gh_token, optional=True)

    if not misc_checks.check_git():
        utils.print_error("Refusing to do a release with a dirty git tree")
        sys.exit(1)

    if args.skip_docs:
        os.makedirs(os.path.join('qutebrowser', 'html', 'doc'), exist_ok=True)
    else:
        run_asciidoc2html(args)

    if os.name == 'nt':
        artifacts = build_windows(
            gh_token=gh_token,
            skip_packaging=args.skip_packaging,
            only_32bit=args.only_32bit,
            only_64bit=args.only_64bit,
            debug=args.debug,
        )
    elif sys.platform == 'darwin':
        artifacts = build_mac(gh_token=gh_token, debug=args.debug)
    else:
        upgrade_sdist_dependencies()
        test_makefile()
        artifacts = build_sdist()
        upload_to_pypi = True

    if args.upload:
        version_tag = "v" + qutebrowser.__version__

        if not args.no_confirm:
            utils.print_title("Press enter to release {}...".format(version_tag))
            input()

        github_upload(artifacts, version_tag, gh_token=gh_token)
        if upload_to_pypi:
            pypi_upload(artifacts)
    else:
        print()
        utils.print_title("Artifacts")
        for artifact in artifacts:
            print(artifact)


if __name__ == '__main__':
    main()