Merge branch 'master' into more-sophisticated-adblock

8 files changed, 78 insertions, 23 deletions

diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py
index 6044a1e18..479283a92 100755
--- a/scripts/dev/build_release.py
+++ b/scripts/dev/build_release.py
@@ -116,7 +116,13 @@ def smoke_test(executable):
         (r'\[.*:ERROR:mach_port_broker.mm\(48\)\] bootstrap_look_up '
          r'org\.chromium\.Chromium\.rohitfork\.1: Permission denied \(1100\)'),
         (r'\[.*:ERROR:mach_port_broker.mm\(43\)\] bootstrap_look_up: '
-         r'Unknown service name \(1102\)')
+         r'Unknown service name \(1102\)'),
+
+        # Windows N:
+        # https://github.com/microsoft/playwright/issues/2901
+        (r'\[.*:ERROR:dxva_video_decode_accelerator_win.cc\(\d+\)\] '
+         r'DXVAVDA fatal error: could not LoadLibrary: .*: The specified '
+         r'module could not be found. \(0x7E\)'),
     ]
 
     proc = subprocess.run([executable, '--no-err-windows', '--nowindow',
diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py
index 728a36873..aa88c97a3 100644
--- a/scripts/dev/check_coverage.py
+++ b/scripts/dev/check_coverage.py
@@ -339,7 +339,7 @@ def main_check():
         print("or check https://codecov.io/github/qutebrowser/qutebrowser")
         print()
 
-    if 'CI' in os.environ:
+    if scriptutils.ON_CI:
         print("Keeping coverage.xml on CI.")
     else:
         os.remove('coverage.xml')
diff --git a/scripts/dev/ci/docker/Dockerfile.j2 b/scripts/dev/ci/docker/Dockerfile.j2
index 412c42cf2..1835f0a2f 100644
--- a/scripts/dev/ci/docker/Dockerfile.j2
+++ b/scripts/dev/ci/docker/Dockerfile.j2
@@ -24,4 +24,4 @@ WORKDIR /home/user
 
 CMD git clone /outside qutebrowser.git && \
     cd qutebrowser.git && \
-    tox -e py38
+    tox -e py
diff --git a/scripts/dev/ci/problemmatchers.py b/scripts/dev/ci/problemmatchers.py
index 3e804af05..cc423f922 100644
--- a/scripts/dev/ci/problemmatchers.py
+++ b/scripts/dev/ci/problemmatchers.py
@@ -188,7 +188,7 @@ MATCHERS = {
             "severity": "error",
             "pattern": [
                 {
-                    "regexp": r'^([^:]+):(\d+): (Found .*)',
+                    "regexp": r'^([^:]+):(\d+): \033\[34m(Found .*)\033\[0m',
                     "file": 1,
                     "line": 2,
                     "message": 3,
diff --git a/scripts/dev/misc_checks.py b/scripts/dev/misc_checks.py
index 14373f94f..ad446412c 100644
--- a/scripts/dev/misc_checks.py
+++ b/scripts/dev/misc_checks.py
@@ -145,8 +145,8 @@ def _check_spelling_file(path, fobj, patterns):
         for pattern, explanation in patterns:
             if pattern.search(line):
                 ok = False
-                print(f'{path}:{num}: Found "{pattern.pattern}" - ', end='')
-                utils.print_col(explanation, 'blue')
+                print(f'{path}:{num}: ', end='')
+                utils.print_col(f'Found "{pattern.pattern}" - {explanation}', 'blue')
     return ok
 
 
@@ -185,7 +185,7 @@ def check_spelling(args: argparse.Namespace) -> Optional[bool]:
              "'type: ignore[error-code]' instead."),
         ),
         (
-            re.compile(r'# type: (?!ignore\[)'),
+            re.compile(r'# type: (?!ignore(\[|$))'),
             "Don't use type comments, use type annotations instead.",
         ),
         (
@@ -274,12 +274,35 @@ def check_userscripts_descriptions(_args: argparse.Namespace = None) -> bool:
     return ok
 
 
+def check_userscript_shebangs(_args: argparse.Namespace) -> bool:
+    """Check that we're using /usr/bin/env in shebangs."""
+    ok = True
+    folder = pathlib.Path('misc/userscripts')
+
+    for sub in folder.iterdir():
+        if sub.is_dir() or sub.name == 'README.md':
+            continue
+
+        with sub.open('r', encoding='utf-8') as f:
+            shebang = f.readline()
+        assert shebang.startswith('#!'), shebang
+        binary = shebang.split()[0][2:]
+
+        if binary not in ['/bin/sh', '/usr/bin/env']:
+            bin_name = pathlib.Path(binary).name
+            print(f"In {sub}, use #!/usr/bin/env {bin_name} instead of #!{binary}")
+            ok = False
+
+    return ok
+
+
 def main() -> int:
     checkers = {
         'git': check_git,
         'vcs': check_vcs_conflict,
         'spelling': check_spelling,
-        'userscripts': check_userscripts_descriptions,
+        'userscript-descriptions': check_userscripts_descriptions,
+        'userscript-shebangs': check_userscript_shebangs,
         'changelog-urls': check_changelog_urls,
     }
 
diff --git a/scripts/dev/recompile_requirements.py b/scripts/dev/recompile_requirements.py
index 3257b4734..5cda4b3d7 100644
--- a/scripts/dev/recompile_requirements.py
+++ b/scripts/dev/recompile_requirements.py
@@ -73,7 +73,7 @@ CHANGELOG_URLS = {
     'pytest-bdd': 'https://github.com/pytest-dev/pytest-bdd/blob/master/CHANGES.rst',
     'snowballstemmer': 'https://github.com/snowballstem/snowball/blob/master/NEWS',
     'virtualenv': 'https://virtualenv.pypa.io/en/latest/changelog.html',
-    'packaging': 'https://pypi.org/project/packaging/',
+    'packaging': 'https://packaging.pypa.io/en/latest/changelog.html',
     'build': 'https://github.com/pypa/build/commits/master',
     'attrs': 'http://www.attrs.org/en/stable/changelog.html',
     'Jinja2': 'https://github.com/pallets/jinja/blob/master/CHANGES.rst',
@@ -130,7 +130,7 @@ CHANGELOG_URLS = {
     'six': 'https://github.com/benjaminp/six/blob/master/CHANGES',
     'altgraph': 'https://github.com/ronaldoussoren/altgraph/blob/master/doc/changelog.rst',
     'urllib3': 'https://github.com/urllib3/urllib3/blob/master/CHANGES.rst',
-    'lxml': 'https://lxml.de/4.6/changes-4.6.0.html',
+    'lxml': 'https://lxml.de/index.html#old-versions',
     'jwcrypto': 'https://github.com/latchset/jwcrypto/commits/master',
     'wrapt': 'https://github.com/GrahamDumpleton/wrapt/blob/develop/docs/changes.rst',
     'pep517': 'https://github.com/pypa/pep517/blob/master/doc/changelog.rst',
@@ -175,6 +175,7 @@ CHANGELOG_URLS = {
     'pyroma': 'https://github.com/regebro/pyroma/blob/master/HISTORY.txt',
     'adblock': 'https://github.com/ArniDagur/python-adblock/blob/master/CHANGELOG.md',
     'pyPEG2': None,
+    'importlib-resources': 'https://importlib-resources.readthedocs.io/en/latest/changelog%20%28links%29.html',
 }
 
 
@@ -262,13 +263,17 @@ def get_all_names():
         yield basename[len('requirements-'):-len('.txt-raw')]
 
 
-def run_pip(venv_dir, *args, **kwargs):
+def run_pip(venv_dir, *args, quiet=False, **kwargs):
     """Run pip inside the virtualenv."""
+    args = list(args)
+    if quiet:
+        args.insert(1, '-q')
+
     arg_str = ' '.join(str(arg) for arg in args)
     utils.print_col('venv$ pip {}'.format(arg_str), 'blue')
+
     venv_python = os.path.join(venv_dir, 'bin', 'python')
-    return subprocess.run([venv_python, '-m', 'pip'] + list(args),
-                          check=True, **kwargs)
+    return subprocess.run([venv_python, '-m', 'pip'] + args, check=True, **kwargs)
 
 
 def init_venv(host_python, venv_dir, requirements, pre=False):
@@ -277,8 +282,8 @@ def init_venv(host_python, venv_dir, requirements, pre=False):
         utils.print_col('$ python3 -m venv {}'.format(venv_dir), 'blue')
         subprocess.run([host_python, '-m', 'venv', venv_dir], check=True)
 
-        run_pip(venv_dir, 'install', '-U', 'pip')
-        run_pip(venv_dir, 'install', '-U', 'setuptools', 'wheel')
+        run_pip(venv_dir, 'install', '-U', 'pip', quiet=not utils.ON_CI)
+        run_pip(venv_dir, 'install', '-U', 'setuptools', 'wheel', quiet=not utils.ON_CI)
 
     install_command = ['install', '-r', requirements]
     if pre:
@@ -292,6 +297,8 @@ def init_venv(host_python, venv_dir, requirements, pre=False):
 def parse_args():
     """Parse commandline arguments via argparse."""
     parser = argparse.ArgumentParser()
+    parser.add_argument('--force-test', help="Force running environment tests",
+                        action='store_true')
     parser.add_argument('names', nargs='*')
     return parser.parse_args()
 
@@ -358,6 +365,7 @@ def _get_changed_files():
 def parse_versioned_line(line):
     """Parse a requirements.txt line into name/version."""
     if '==' in line:
+        line = line.rsplit('#', maxsplit=1)[0]  # Strip comments
         name, version = line.split('==')
         if ';' in version:  # pip environment markers
             version = version.split(';')[0].strip()
@@ -412,7 +420,7 @@ def print_changed_files():
     utils.print_subtitle('Diff')
     print(diff_text)
 
-    if 'CI' in os.environ:
+    if utils.ON_CI:
         print()
         print('::set-output name=changed::' +
               files_text.replace('\n', '%0A'))
@@ -481,7 +489,6 @@ def build_requirements(name):
 
 def test_tox():
     """Test requirements via tox."""
-    utils.print_title('Testing via tox')
     host_python = get_host_python('tox')
     req_path = os.path.join(REQ_DIR, 'requirements-tox.txt')
 
@@ -506,11 +513,15 @@ def test_tox():
                                check=True)
 
 
-def test_requirements(name, outfile):
+def test_requirements(name, outfile, *, force=False):
     """Test a resulting requirements file."""
     print()
     utils.print_subtitle("Testing")
 
+    if name not in _get_changed_files() and not force:
+        print(f"Skipping test as there were no changes for {name}.")
+        return
+
     host_python = get_host_python(name)
     with tempfile.TemporaryDirectory() as tmpdir:
         init_venv(host_python, tmpdir, outfile)
@@ -528,11 +539,16 @@ def main():
     for name in names:
         utils.print_title(name)
         outfile = build_requirements(name)
-        test_requirements(name, outfile)
+        test_requirements(name, outfile, force=args.force_test)
 
-    if not args.names:
+    utils.print_title('Testing via tox')
+    if args.names and not args.force_test:
         # If we selected a subset, let's not go through the trouble of testing
         # via tox.
+        print("Skipping: Selected a subset only")
+    elif not _get_changed_files() and not args.force_test:
+        print("Skipping: No changes")
+    else:
         test_tox()
 
     print_changed_files()
diff --git a/scripts/dev/update_version.py b/scripts/dev/update_version.py
index 5f7df5cae..1f744d392 100644
--- a/scripts/dev/update_version.py
+++ b/scripts/dev/update_version.py
@@ -82,7 +82,7 @@ if __name__ == "__main__":
           .format(v=version))
     print("* Windows: git fetch; git checkout v{v}; "
           "py -3.7 -m tox -e build-release -- --asciidoc "
-          "$env:userprofile\\bin\\asciidoc-9.0.2\\asciidoc.py --upload"
+          "$env:userprofile\\bin\\asciidoc-9.0.4\\asciidoc.py --upload"
           .format(v=version))
     print("* macOS: git fetch && git checkout v{v} && "
           "tox -e build-release -- --upload"
diff --git a/scripts/mkvenv.py b/scripts/mkvenv.py
index ad5f2073e..897088539 100644
--- a/scripts/mkvenv.py
+++ b/scripts/mkvenv.py
@@ -87,6 +87,9 @@ def parse_args(argv: List[str] = None) -> argparse.Namespace:
     parser.add_argument('--skip-docs',
                         action='store_true',
                         help="Skip doc generation.")
+    parser.add_argument('--skip-smoke-test',
+                        action='store_true',
+                        help="Skip Qt smoke test.")
     parser.add_argument('--tox-error',
                         action='store_true',
                         help=argparse.SUPPRESS)
@@ -296,12 +299,19 @@ def apply_xcb_util_workaround(
 def _find_libs() -> Dict[Tuple[str, str], List[str]]:
     """Find all system-wide .so libraries."""
     all_libs: Dict[Tuple[str, str], List[str]] = {}
+
+    if pathlib.Path("/sbin/ldconfig").exists():
+        # /sbin might not be in PATH on e.g. Debian
+        ldconfig_bin = "/sbin/ldconfig"
+    else:
+        ldconfig_bin = "ldconfig"
     ldconfig_proc = subprocess.run(
-        ['ldconfig', '-p'],
+        [ldconfig_bin, '-p'],
         check=True,
         stdout=subprocess.PIPE,
         encoding=sys.getfilesystemencoding(),
     )
+
     pattern = re.compile(r'(?P<name>\S+) \((?P<abi_type>[^)]+)\) => (?P<path>.*)')
     for line in ldconfig_proc.stdout.splitlines():
         match = pattern.fullmatch(line.strip())
@@ -421,7 +431,7 @@ def run(args) -> None:
         raise AssertionError
 
     apply_xcb_util_workaround(venv_dir, args.pyqt_type, args.pyqt_version)
-    if args.pyqt_type != 'skip':
+    if args.pyqt_type != 'skip' and not args.skip_smoke_test:
         run_qt_smoke_test(venv_dir)
 
     install_requirements(venv_dir)