Merge branch 'master' into more-sophisticated-adblock

22 files changed, 87 insertions, 63 deletions

diff --git a/misc/Makefile b/misc/Makefile
index 985541fdd..b916a20d5 100644
--- a/misc/Makefile
+++ b/misc/Makefile
@@ -29,7 +29,7 @@ install: man
 	install -Dm644 icons/qutebrowser.svg \
 		"$(DESTDIR)$(DATADIR)/icons/hicolor/scalable/apps/qutebrowser.svg"
 	install -Dm755 -t "$(DESTDIR)$(DATADIR)/qutebrowser/userscripts/" \
-		$(wildcard misc/userscripts/*)
+		$(filter-out misc/userscripts/__pycache__,$(wildcard misc/userscripts/*))
 	install -Dm755 -t "$(DESTDIR)$(DATADIR)/qutebrowser/scripts/" \
 		$(filter-out scripts/__init__.py scripts/__pycache__ scripts/dev \
 		  scripts/testbrowser scripts/asciidoc2html.py scripts/setupcommon.py \
diff --git a/misc/nsis/qutebrowser.nsi b/misc/nsis/qutebrowser.nsi
index d9b8fbf8d..77fd373eb 100755
--- a/misc/nsis/qutebrowser.nsi
+++ b/misc/nsis/qutebrowser.nsi
@@ -1,4 +1,5 @@
 # Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>

+# encoding: iso-8859-1

 #

 # This file is part of qutebrowser.

 #

diff --git a/misc/requirements/requirements-dev.txt b/misc/requirements/requirements-dev.txt
index c543088c6..798fecad6 100644
--- a/misc/requirements/requirements-dev.txt
+++ b/misc/requirements/requirements-dev.txt
@@ -5,10 +5,10 @@ certifi==2020.6.20
 cffi==1.14.2
 chardet==3.0.4
 colorama==0.4.3
-cryptography==3.0
+cryptography==3.1
 cssutils==1.0.2
 github3.py==1.3.0
-hunter==3.2.1
+hunter==3.2.2
 idna==2.10
 jwcrypto==0.8
 manhole==1.6.0
@@ -16,10 +16,10 @@ packaging==20.4
 pycparser==2.20
 Pympler==0.8
 pyparsing==2.4.7
-PyQt-builder==1.4.0
+PyQt-builder==1.5.0
 python-dateutil==2.8.1
 requests==2.24.0
-sip==5.3.0
+sip==5.4.0
 six==1.15.0
 toml==0.10.1
 uritemplate==3.0.1
diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt
index 00d22b236..afd4f0bd6 100644
--- a/misc/requirements/requirements-flake8.txt
+++ b/misc/requirements/requirements-flake8.txt
@@ -1,6 +1,6 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-attrs==20.1.0
+attrs==20.2.0
 flake8==3.8.3
 flake8-bugbear==20.1.4
 flake8-builtins==1.5.3
@@ -18,7 +18,7 @@ flake8-tuple==0.4.1
 mccabe==0.6.1
 pep8-naming==0.11.1
 pycodestyle==2.6.0
-pydocstyle==5.1.0
+pydocstyle==5.1.1
 pyflakes==2.2.0
 six==1.15.0
 snowballstemmer==2.0.0
diff --git a/misc/requirements/requirements-mypy.txt b/misc/requirements/requirements-mypy.txt
index 09d11ea6c..863f48c6f 100644
--- a/misc/requirements/requirements-mypy.txt
+++ b/misc/requirements/requirements-mypy.txt
@@ -1,6 +1,6 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-diff-cover==3.0.1
+diff-cover==4.0.0
 inflect==4.1.0
 Jinja2==2.11.2
 jinja2-pluralize==0.3.0
@@ -9,8 +9,7 @@ MarkupSafe==1.1.1
 mypy==0.782
 mypy-extensions==0.4.3
 pluggy==0.13.1
-Pygments==2.6.1
+Pygments==2.7.1
 -e git+https://github.com/stlehmann/PyQt5-stubs.git@master#egg=PyQt5_stubs
-six==1.15.0
 typed-ast==1.4.1
 typing-extensions==3.7.4.3
diff --git a/misc/requirements/requirements-pyinstaller.txt b/misc/requirements/requirements-pyinstaller.txt
index 6de7b6fa8..06405d96a 100644
--- a/misc/requirements/requirements-pyinstaller.txt
+++ b/misc/requirements/requirements-pyinstaller.txt
@@ -2,4 +2,4 @@
 
 altgraph==0.17
 pyinstaller==4.0
-pyinstaller-hooks-contrib==2020.7
+pyinstaller-hooks-contrib==2020.8
diff --git a/misc/requirements/requirements-pylint.txt b/misc/requirements/requirements-pylint.txt
index 3c7440627..08c1d2c10 100644
--- a/misc/requirements/requirements-pylint.txt
+++ b/misc/requirements/requirements-pylint.txt
@@ -4,7 +4,7 @@ astroid==2.3.3  # rq.filter: < 2.4
 certifi==2020.6.20
 cffi==1.14.2
 chardet==3.0.4
-cryptography==3.0
+cryptography==3.1
 github3.py==1.3.0
 idna==2.10
 isort==4.3.21
diff --git a/misc/requirements/requirements-pyqt-5.12.txt b/misc/requirements/requirements-pyqt-5.12.txt
index 9b458cd98..80a700f09 100644
--- a/misc/requirements/requirements-pyqt-5.12.txt
+++ b/misc/requirements/requirements-pyqt-5.12.txt
@@ -1,5 +1,5 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 PyQt5==5.12.3  # rq.filter: < 5.13
-PyQt5-sip==12.8.0
+PyQt5-sip==12.8.1
 PyQtWebEngine==5.12.1  # rq.filter: < 5.13
diff --git a/misc/requirements/requirements-pyqt-5.13.txt b/misc/requirements/requirements-pyqt-5.13.txt
index 7c07eac3d..438c600da 100644
--- a/misc/requirements/requirements-pyqt-5.13.txt
+++ b/misc/requirements/requirements-pyqt-5.13.txt
@@ -1,5 +1,5 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 PyQt5==5.13.2  # rq.filter: < 5.14
-PyQt5-sip==12.8.0
+PyQt5-sip==12.8.1
 PyQtWebEngine==5.13.2  # rq.filter: < 5.14
diff --git a/misc/requirements/requirements-pyqt-5.14.txt b/misc/requirements/requirements-pyqt-5.14.txt
index c82acedb0..d515e717f 100644
--- a/misc/requirements/requirements-pyqt-5.14.txt
+++ b/misc/requirements/requirements-pyqt-5.14.txt
@@ -1,5 +1,5 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 PyQt5==5.14.2  # rq.filter: < 5.15
-PyQt5-sip==12.8.0
+PyQt5-sip==12.8.1
 PyQtWebEngine==5.14.0  # rq.filter: < 5.15
diff --git a/misc/requirements/requirements-pyqt-5.15.txt b/misc/requirements/requirements-pyqt-5.15.txt
index c21b7b742..21745c814 100644
--- a/misc/requirements/requirements-pyqt-5.15.txt
+++ b/misc/requirements/requirements-pyqt-5.15.txt
@@ -1,5 +1,5 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-PyQt5==5.15.0  # rq.filter: < 6
-PyQt5-sip==12.8.0
-PyQtWebEngine==5.15.0  # rq.filter: < 6
+PyQt5==5.15.1  # rq.filter: < 6
+PyQt5-sip==12.8.1
+PyQtWebEngine==5.15.1  # rq.filter: < 6
diff --git a/misc/requirements/requirements-pyqt.txt b/misc/requirements/requirements-pyqt.txt
index 74d86e8d5..88a04230d 100644
--- a/misc/requirements/requirements-pyqt.txt
+++ b/misc/requirements/requirements-pyqt.txt
@@ -1,5 +1,5 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-PyQt5==5.15.0
-PyQt5-sip==12.8.0
-PyQtWebEngine==5.15.0
+PyQt5==5.15.1
+PyQt5-sip==12.8.1
+PyQtWebEngine==5.15.1
diff --git a/misc/requirements/requirements-pyroma.txt b/misc/requirements/requirements-pyroma.txt
index 6b131e155..c6e751e19 100644
--- a/misc/requirements/requirements-pyroma.txt
+++ b/misc/requirements/requirements-pyroma.txt
@@ -1,5 +1,5 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 docutils==0.16
-Pygments==2.6.1
+Pygments==2.7.1
 pyroma==2.6
diff --git a/misc/requirements/requirements-sphinx.txt b/misc/requirements/requirements-sphinx.txt
index da6447009..7b43a72a1 100644
--- a/misc/requirements/requirements-sphinx.txt
+++ b/misc/requirements/requirements-sphinx.txt
@@ -10,7 +10,7 @@ imagesize==1.2.0
 Jinja2==2.11.2
 MarkupSafe==1.1.1
 packaging==20.4
-Pygments==2.6.1
+Pygments==2.7.1
 pyparsing==2.4.7
 pytz==2020.1
 requests==2.24.0
diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt
index a82ba796c..789c176e6 100644
--- a/misc/requirements/requirements-tests.txt
+++ b/misc/requirements/requirements-tests.txt
@@ -1,18 +1,20 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-attrs==20.1.0
+apipkg==1.5
+attrs==20.2.0
 beautifulsoup4==4.9.1
 certifi==2020.6.20
 chardet==3.0.4
 cheroot==8.4.5
 click==7.1.2
 # colorama==0.4.3
-coverage==5.2.1
+coverage==5.3
 EasyProcess==0.3
+execnet==1.7.1
 Flask==1.1.2
 glob2==0.7
-hunter==3.2.1
-hypothesis==5.29.0
+hunter==3.2.2
+hypothesis==5.35.3 ; python_version>="3.6"
 idna==2.10
 iniconfig==1.0.1
 itsdangerous==1.1.0
@@ -21,24 +23,26 @@ jaraco.functools==3.0.1 ; python_version>="3.6"
 Mako==1.1.3
 manhole==1.6.0
 # MarkupSafe==1.1.1
-more-itertools==8.4.0
+more-itertools==8.5.0
 packaging==20.4
-parse==1.16.0
+parse==1.18.0
 parse-type==0.5.2
 pluggy==0.13.1
 py==1.9.0
 py-cpuinfo==7.0.0
-Pygments==2.6.1
+Pygments==2.7.1
 pyparsing==2.4.7
-pytest==6.0.1
-pytest-bdd==3.4.0
+pytest==6.0.2
+pytest-bdd==4.0.1
 pytest-benchmark==3.2.3
 pytest-cov==2.10.1
+pytest-forked==1.3.0
 pytest-instafail==0.4.2
-pytest-mock==3.3.0
+pytest-mock==3.3.1
 pytest-qt==3.3.0
 pytest-repeat==0.8.0
-pytest-rerunfailures==9.0
+pytest-rerunfailures==9.1
+pytest-xdist==2.1.0
 pytest-xvfb==2.0.0
 PyVirtualDisplay==1.3.2
 requests==2.24.0
@@ -53,3 +57,4 @@ vulture==2.1 ; python_version>="3.6"
 Werkzeug==1.0.1
 jaraco.functools==2.0; python_version<"3.6"
 vulture==1.6; python_version<"3.6"
+hypothesis<5.34.0; python_version<"3.6"
diff --git a/misc/requirements/requirements-tests.txt-raw b/misc/requirements/requirements-tests.txt-raw
index f063a3512..73f58461c 100644
--- a/misc/requirements/requirements-tests.txt-raw
+++ b/misc/requirements/requirements-tests.txt-raw
@@ -25,6 +25,8 @@ pytest-cov
 # To avoid windows from popping up
 pytest-xvfb
 PyVirtualDisplay
+# To run on multiple cores with -n
+pytest-xdist
 
 # Needed to test misc/userscripts/qute-lastpass
 tldextract
@@ -35,4 +37,7 @@ tldextract
 #@ markers: vulture python_version>="3.6"
 #@ add: vulture==1.6; python_version<"3.6"
 
+#@ markers: hypothesis python_version>="3.6"
+#@ add: hypothesis<5.34.0; python_version<"3.6"
+
 #@ ignore: Jinja2, MarkupSafe, colorama
diff --git a/misc/requirements/requirements-tox.txt b/misc/requirements/requirements-tox.txt
index 3fb7595ad..c7f99a2da 100644
--- a/misc/requirements/requirements-tox.txt
+++ b/misc/requirements/requirements-tox.txt
@@ -9,7 +9,7 @@ py==1.9.0
 pyparsing==2.4.7
 six==1.15.0
 toml==0.10.1
-tox==3.19.0
+tox==3.20.0
 tox-pip-version==0.0.7
 tox-venv==0.4.0
 virtualenv==20.0.31
diff --git a/misc/userscripts/README.md b/misc/userscripts/README.md
index 729e63f6e..d2519a672 100644
--- a/misc/userscripts/README.md
+++ b/misc/userscripts/README.md
@@ -40,7 +40,7 @@ The following userscripts are included in the current directory.
 
 The following userscripts can be found on their own repositories.
 
-- [qurlshare](https://github.com/sim590/qurlshare): *secure* sharing of an URL between qutebrowser
+- [qurlshare](https://github.com/sim590/qurlshare): *secure* sharing of a URL between qutebrowser
   instances using a distributed hash table.
 - [qutebrowser-userscripts](https://github.com/cryzed/qutebrowser-userscripts):
   a small pack of userscripts.
diff --git a/misc/userscripts/cast b/misc/userscripts/cast
index f7b64df70..8bbf05a40 100755
--- a/misc/userscripts/cast
+++ b/misc/userscripts/cast
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Behaviour
+# Behavior
 #   Userscript for qutebrowser which casts the url passed in $1 to the default
 #   ChromeCast device in the network using the program `castnow`
 #
diff --git a/misc/userscripts/qute-bitwarden b/misc/userscripts/qute-bitwarden
index d5c4b1e2d..ca9d646e4 100755
--- a/misc/userscripts/qute-bitwarden
+++ b/misc/userscripts/qute-bitwarden
@@ -281,7 +281,7 @@ def main(arguments):
         qute_command('enter-mode insert')
 
     # If it finds a TOTP code, it copies it to the clipboard,
-    # which is the same behaviour as the Firefox add-on.
+    # which is the same behavior as the Firefox add-on.
     if not arguments.totp_only and totp and arguments.totp:
         # The import is done here, to make pyperclip an optional dependency
         import pyperclip
diff --git a/misc/userscripts/qute-pass b/misc/userscripts/qute-pass
index 9d078e94f..b49e87dd8 100755
--- a/misc/userscripts/qute-pass
+++ b/misc/userscripts/qute-pass
@@ -61,11 +61,19 @@ import sys
 
 import tldextract
 
+
+def expanded_path(path):
+    # Expand potential ~ in paths, since this script won't be called from a shell that does it for us
+    expanded = os.path.expanduser(path)
+    # Add trailing slash if not present
+    return os.path.join(expanded, '')
+
+
 argument_parser = argparse.ArgumentParser(description=__doc__, usage=USAGE, epilog=EPILOG)
 argument_parser.add_argument('url', nargs='?', default=os.getenv('QUTE_URL'))
 argument_parser.add_argument('--password-store', '-p',
-                             default=os.getenv('PASSWORD_STORE_DIR', default=os.path.expanduser('~/.password-store')),
-                             help='Path to your pass password-store (only used in pass-mode)')
+                             default=expanded_path(os.getenv('PASSWORD_STORE_DIR', default='~/.password-store')),
+                             help='Path to your pass password-store (only used in pass-mode)', type=expanded_path)
 argument_parser.add_argument('--mode', '-M', choices=['pass', 'gopass'], default="pass",
                              help='Select mode [gopass] to use gopass instead of the standard pass.')
 argument_parser.add_argument('--username-pattern', '-u', default=r'.*/(.+)',
@@ -107,7 +115,7 @@ def qute_command(command):
         fifo.flush()
 
 
-def find_pass_candidates(domain, password_store_path):
+def find_pass_candidates(domain):
     candidates = []
 
     if arguments.mode == "gopass":
@@ -117,13 +125,13 @@ def find_pass_candidates(domain, password_store_path):
             if domain in password:
                 candidates.append(password)
     else:
-        for path, directories, file_names in os.walk(password_store_path, followlinks=True):
+        for path, directories, file_names in os.walk(arguments.password_store, followlinks=True):
             secrets = fnmatch.filter(file_names, '*.gpg')
             if not secrets:
                 continue
 
             # Strip password store path prefix to get the relative pass path
-            pass_path = path[len(password_store_path):]
+            pass_path = path[len(arguments.password_store):]
             split_path = pass_path.split(os.path.sep)
             for secret in secrets:
                 secret_base = os.path.splitext(secret)[0]
@@ -134,25 +142,27 @@ def find_pass_candidates(domain, password_store_path):
     return candidates
 
 
-def _run_pass(pass_arguments, encoding):
+def _run_pass(pass_arguments):
     # The executable is conveniently named after it's mode [pass|gopass].
     pass_command = [arguments.mode]
-    process = subprocess.run(pass_command + pass_arguments, stdout=subprocess.PIPE)
-    return process.stdout.decode(encoding).strip()
+    env = os.environ.copy()
+    env['PASSWORD_STORE_DIR'] = arguments.password_store
+    process = subprocess.run(pass_command + pass_arguments, env=env, stdout=subprocess.PIPE)
+    return process.stdout.decode(arguments.io_encoding).strip()
 
 
-def pass_(path, encoding):
-    return _run_pass([path], encoding)
+def pass_(path):
+    return _run_pass(['show', path])
 
 
-def pass_otp(path, encoding):
-    return _run_pass(['otp', path], encoding)
+def pass_otp(path):
+    return _run_pass(['otp', path])
 
 
-def dmenu(items, invocation, encoding):
+def dmenu(items, invocation):
     command = shlex.split(invocation)
-    process = subprocess.run(command, input='\n'.join(items).encode(encoding), stdout=subprocess.PIPE)
-    return process.stdout.decode(encoding).strip()
+    process = subprocess.run(command, input='\n'.join(items).encode(arguments.io_encoding), stdout=subprocess.PIPE)
+    return process.stdout.decode(arguments.io_encoding).strip()
 
 
 def fake_key_raw(text):
@@ -170,11 +180,6 @@ def main(arguments):
     extractor = tldextract.TLDExtract(extra_suffixes=arguments.extra_url_suffixes.split(','))
     extract_result = extractor(arguments.url)
 
-    # Expand potential ~ in paths, since this script won't be called from a shell that does it for us
-    password_store_path = os.path.expanduser(arguments.password_store)
-    # Add trailing slash if not present
-    password_store_path = os.path.join(password_store_path, '')
-
     # Try to find candidates using targets in the following order: fully-qualified domain name (includes subdomains),
     # the registered domain name, the IPv4 address if that's what the URL represents and finally the private domain
     # (if a non-public suffix was used).
@@ -188,7 +193,7 @@ def main(arguments):
 
     for target in filter(None, [extract_result.fqdn, extract_result.registered_domain, extract_result.ipv4, private_domain]):
         attempted_targets.append(target)
-        target_candidates = find_pass_candidates(target, password_store_path)
+        target_candidates = find_pass_candidates(target)
         if not target_candidates:
             continue
 
@@ -200,8 +205,7 @@ def main(arguments):
             stderr('No pass candidates for URL {!r} found! (I tried {!r})'.format(arguments.url, attempted_targets))
             return ExitCodes.NO_PASS_CANDIDATES
 
-    selection = candidates.pop() if len(candidates) == 1 else dmenu(sorted(candidates), arguments.dmenu_invocation,
-                                                                    arguments.io_encoding)
+    selection = candidates.pop() if len(candidates) == 1 else dmenu(sorted(candidates), arguments.dmenu_invocation)
     # Nothing was selected, simply return
     if not selection:
         return ExitCodes.SUCCESS
@@ -209,7 +213,7 @@ def main(arguments):
     # If username-target is path and user asked for username-only, we don't need to run pass
     secret = None
     if not (arguments.username_target == 'path' and arguments.username_only):
-        secret = pass_(selection, arguments.io_encoding)
+        secret = pass_(selection)
 
         # Match password
         match = re.match(arguments.password_pattern, secret)
@@ -231,7 +235,7 @@ def main(arguments):
     elif arguments.password_only:
         fake_key_raw(password)
     elif arguments.otp_only:
-        otp = pass_otp(selection, arguments.io_encoding)
+        otp = pass_otp(selection)
         fake_key_raw(otp)
     else:
         # Enter username and password using fake-key and <Tab> (which seems to work almost universally), then switch
diff --git a/misc/userscripts/readability-js b/misc/userscripts/readability-js
index 4b9446ed5..207b29b4a 100755
--- a/misc/userscripts/readability-js
+++ b/misc/userscripts/readability-js
@@ -44,6 +44,16 @@ const HEADER = `
         h1, h2, h3 {
             line-height: 1.2;
         }
+        img {
+            max-width:100%;
+            height:auto;
+        }
+        p > img:only-child,
+        p > a:only-child > img:only-child,
+        .wp-caption img,
+        figure img {
+            display: block;
+        }
     </style>
     <!-- This icon is licensed under the Mozilla Public License 2.0 (available at: https://www.mozilla.org/en-US/MPL/2.0/).
     The original icon can be found here: https://dxr.mozilla.org/mozilla-central/source/browser/themes/shared/reader/readerMode.svg -->