Merge remote-tracking branch 'origin/pr/3456'

7 files changed, 277 insertions, 68 deletions

diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py
index 4f390b18b..dd112e00a 100644
--- a/qutebrowser/browser/downloads.py
+++ b/qutebrowser/browser/downloads.py
@@ -238,11 +238,14 @@ class FileDownloadTarget(_DownloadTarget):
 
     Attributes:
         filename: Filename where the download should be saved.
+        force_overwrite: Whether to overwrite the target without
+                         prompting the user.
     """
 
-    def __init__(self, filename):
+    def __init__(self, filename, force_overwrite=False):
         # pylint: disable=super-init-not-called
         self.filename = filename
+        self.force_overwrite = force_overwrite
 
     def suggested_filename(self):
         return os.path.basename(self.filename)
@@ -738,7 +741,8 @@ class AbstractDownloadItem(QObject):
         if isinstance(target, FileObjDownloadTarget):
             self._set_fileobj(target.fileobj, autoclose=False)
         elif isinstance(target, FileDownloadTarget):
-            self._set_filename(target.filename)
+            self._set_filename(
+                target.filename, force_overwrite=target.force_overwrite)
         elif isinstance(target, OpenFileDownloadTarget):
             try:
                 fobj = temp_download_manager.get_tmpfile(self.basename)
diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index fb064f6c1..6879f4cf6 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -23,13 +23,16 @@ import re
 import os
 import json
 import fnmatch
+import functools
 import glob
+import textwrap
 
 import attr
 from PyQt5.QtCore import pyqtSignal, QObject, QUrl
 
-from qutebrowser.utils import log, standarddir, jinja, objreg
+from qutebrowser.utils import log, standarddir, jinja, objreg, utils
 from qutebrowser.commands import cmdutils
+from qutebrowser.browser import downloads
 
 
 def _scripts_dir():
@@ -45,6 +48,7 @@ class GreasemonkeyScript:
         self._code = code
         self.includes = []
         self.excludes = []
+        self.requires = []
         self.description = None
         self.name = None
         self.namespace = None
@@ -66,6 +70,8 @@ class GreasemonkeyScript:
                 self.run_at = value
             elif name == 'noframes':
                 self.runs_on_sub_frames = False
+            elif name == 'require':
+                self.requires.append(value)
 
     HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n'
     PROPS_REGEX = r'// @(?P<prop>[^\s]+)\s*(?P<val>.*)'
@@ -93,7 +99,7 @@ class GreasemonkeyScript:
         """Return the processed JavaScript code of this script.
 
         Adorns the source code with GM_* methods for Greasemonkey
-        compatibility and wraps it in an IFFE to hide it within a
+        compatibility and wraps it in an IIFE to hide it within a
         lexical scope. Note that this means line numbers in your
         browser's debugger/inspector will not match up to the line
         numbers in the source script directly.
@@ -115,6 +121,14 @@ class GreasemonkeyScript:
             'run-at': self.run_at,
         })
 
+    def add_required_script(self, source):
+        """Add the source of a required script to this script."""
+        # The additional source is indented in case it also contains a
+        # metadata block. Because we pass everything at once to
+        # QWebEngineScript and that would parse the first metadata block
+        # found as the valid one.
+        self._code = "\n".join([textwrap.indent(source, "    "), self._code])
+
 
 @attr.s
 class MatchingScripts(object):
@@ -145,15 +159,24 @@ class GreasemonkeyManager(QObject):
 
     def __init__(self, parent=None):
         super().__init__(parent)
+        self._run_start = []
+        self._run_end = []
+        self._run_idle = []
+        self._in_progress_dls = []
+
         self.load_scripts()
 
     @cmdutils.register(name='greasemonkey-reload',
                        instance='greasemonkey')
-    def load_scripts(self):
+    def load_scripts(self, force=False):
         """Re-read Greasemonkey scripts from disk.
 
         The scripts are read from a 'greasemonkey' subdirectory in
         qutebrowser's data directory (see `:version`).
+
+        Args:
+            force: For any scripts that have required dependencies,
+                   re-download them.
         """
         self._run_start = []
         self._run_end = []
@@ -169,24 +192,115 @@ class GreasemonkeyManager(QObject):
                 script = GreasemonkeyScript.parse(script_file.read())
                 if not script.name:
                     script.name = script_filename
-
-                if script.run_at == 'document-start':
-                    self._run_start.append(script)
-                elif script.run_at == 'document-end':
-                    self._run_end.append(script)
-                elif script.run_at == 'document-idle':
-                    self._run_idle.append(script)
-                else:
-                    if script.run_at:
-                        log.greasemonkey.warning(
-                            "Script {} has invalid run-at defined, "
-                            "defaulting to document-end".format(script_path))
-                    # Default as per
-                    # https://wiki.greasespot.net/Metadata_Block#.40run-at
-                    self._run_end.append(script)
-                log.greasemonkey.debug("Loaded script: {}".format(script.name))
+                self.add_script(script, force)
         self.scripts_reloaded.emit()
 
+    def add_script(self, script, force=False):
+        """Add a GreasemonkeyScript to this manager.
+
+        Args:
+            force: Fetch and overwrite any dependancies which are
+                   already locally cached.
+        """
+        if script.requires:
+            log.greasemonkey.debug(
+                "Deferring script until requirements are "
+                "fulfilled: {}".format(script.name))
+            self._get_required_scripts(script, force)
+        else:
+            self._add_script(script)
+
+    def _add_script(self, script):
+        if script.run_at == 'document-start':
+            self._run_start.append(script)
+        elif script.run_at == 'document-end':
+            self._run_end.append(script)
+        elif script.run_at == 'document-idle':
+            self._run_idle.append(script)
+        else:
+            if script.run_at:
+                log.greasemonkey.warning("Script {} has invalid run-at "
+                                         "defined, defaulting to "
+                                         "document-end"
+                                         .format(script.name))
+                # Default as per
+                # https://wiki.greasespot.net/Metadata_Block#.40run-at
+            self._run_end.append(script)
+        log.greasemonkey.debug("Loaded script: {}".format(script.name))
+
+    def _required_url_to_file_path(self, url):
+        requires_dir = os.path.join(_scripts_dir(), 'requires')
+        if not os.path.exists(requires_dir):
+            os.mkdir(requires_dir)
+        return os.path.join(requires_dir, utils.sanitize_filename(url))
+
+    def _on_required_download_finished(self, script, download):
+        self._in_progress_dls.remove(download)
+        if not self._add_script_with_requires(script):
+            log.greasemonkey.debug(
+                "Finished download {} for script {} "
+                "but some requirements are still pending"
+                .format(download.basename, script.name))
+
+    def _add_script_with_requires(self, script, quiet=False):
+        """Add a script with pending downloads to this GreasemonkeyManager.
+
+        Specifically a script that has dependancies specified via an
+        `@require` rule.
+
+        Args:
+            script: The GreasemonkeyScript to add.
+            quiet: True to suppress the scripts_reloaded signal after
+                   adding `script`.
+        Returns: True if the script was added, False if there are still
+                 dependancies being downloaded.
+        """
+        # See if we are still waiting on any required scripts for this one
+        for dl in self._in_progress_dls:
+            if dl.requested_url in script.requires:
+                return False
+
+        # Need to add the required scripts to the IIFE now
+        for url in reversed(script.requires):
+            target_path = self._required_url_to_file_path(url)
+            log.greasemonkey.debug(
+                "Adding required script for {} to IIFE: {}"
+                .format(script.name, url))
+            with open(target_path, encoding='utf8') as f:
+                script.add_required_script(f.read())
+
+        self._add_script(script)
+        if not quiet:
+            self.scripts_reloaded.emit()
+        return True
+
+    def _get_required_scripts(self, script, force=False):
+        required_dls = [(url, self._required_url_to_file_path(url))
+                        for url in script.requires]
+        if not force:
+            required_dls = [(url, path) for (url, path) in required_dls
+                            if not os.path.exists(path)]
+        if not required_dls:
+            # All the required files exist already
+            self._add_script_with_requires(script, quiet=True)
+            return
+
+        download_manager = objreg.get('qtnetwork-download-manager')
+
+        for url, target_path in required_dls:
+            target = downloads.FileDownloadTarget(target_path,
+                                                  force_overwrite=True)
+            download = download_manager.get(QUrl(url), target=target,
+                                            auto_remove=True)
+            download.requested_url = url
+            self._in_progress_dls.append(download)
+            if download.successful:
+                self._on_required_download_finished(script, download)
+            else:
+                download.finished.connect(
+                    functools.partial(self._on_required_download_finished,
+                                      script, download))
+
     def scripts_for(self, url):
         """Fetch scripts that are registered to run for url.
 
diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js
index 2d36220dc..71266755a 100644
--- a/qutebrowser/javascript/greasemonkey_wrapper.js
+++ b/qutebrowser/javascript/greasemonkey_wrapper.js
@@ -110,6 +110,44 @@
         }
     }
 
+    // Stub these two so that the gm4 polyfill script doesn't try to
+    // create broken versions as attributes of window.
+    function GM_getResourceText(caption, commandFunc, accessKey) {
+        console.error(`${GM_info.script.name} called unimplemented GM_getResourceText`);
+    }
+
+    function GM_registerMenuCommand(caption, commandFunc, accessKey) {
+        console.error(`${GM_info.script.name} called unimplemented GM_registerMenuCommand`);
+    }
+
+    // Mock the greasemonkey 4.0 async API.
+    const GM = {};
+    GM.info = GM_info;
+    const entries = {
+        'log': GM_log,
+        'addStyle': GM_addStyle,
+        'deleteValue': GM_deleteValue,
+        'getValue': GM_getValue,
+        'listValues': GM_listValues,
+        'openInTab': GM_openInTab,
+        'setValue': GM_setValue,
+        'xmlHttpRequest': GM_xmlhttpRequest,
+    }
+    for (newKey in entries) {
+        let old = entries[newKey];
+        if (old && (typeof GM[newKey] == 'undefined')) {
+            GM[newKey] = function(...args) {
+                return new Promise((resolve, reject) => {
+                    try {
+                        resolve(old(...args));
+                    } catch (e) {
+                        reject(e);
+                    }
+                });
+            };
+        }
+    };
+
     const unsafeWindow = window;
 
     // ====== The actual user script source ====== //
diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py
index d30514f83..f9f02ba8b 100644
--- a/tests/helpers/fixtures.py
+++ b/tests/helpers/fixtures.py
@@ -507,3 +507,12 @@ class ModelValidator:
 @pytest.fixture
 def model_validator(qtmodeltester):
     return ModelValidator(qtmodeltester)
+
+
+@pytest.fixture
+def download_stub(win_registry, tmpdir, stubs):
+    """Register a FakeDownloadManager."""
+    stub = stubs.FakeDownloadManager(tmpdir)
+    objreg.register('qtnetwork-download-manager', stub)
+    yield stub
+    objreg.delete('qtnetwork-download-manager')
diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py
index 3957a670a..fbe7035e3 100644
--- a/tests/helpers/stubs.py
+++ b/tests/helpers/stubs.py
@@ -22,6 +22,8 @@
 """Fake objects/stubs."""
 
 from unittest import mock
+import contextlib
+import shutil
 
 import attr
 from PyQt5.QtCore import pyqtSignal, QPoint, QProcess, QObject, QUrl
@@ -29,7 +31,7 @@ from PyQt5.QtNetwork import (QNetworkRequest, QAbstractNetworkCache,
                              QNetworkCacheMetaData)
 from PyQt5.QtWidgets import QCommonStyle, QLineEdit, QWidget, QTabBar
 
-from qutebrowser.browser import browsertab
+from qutebrowser.browser import browsertab, downloads
 from qutebrowser.utils import usertypes
 from qutebrowser.mainwindow import mainwindow
 
@@ -558,3 +560,49 @@ class HTTPPostStub(QObject):
     def post(self, url, data=None):
         self.url = url
         self.data = data
+
+
+class FakeDownloadItem(QObject):
+
+    """Mock browser.downloads.DownloadItem."""
+
+    finished = pyqtSignal()
+
+    def __init__(self, fileobj, name, parent=None):
+        super().__init__(parent)
+        self.fileobj = fileobj
+        self.name = name
+        self.successful = False
+
+
+class FakeDownloadManager:
+
+    """Mock browser.downloads.DownloadManager."""
+
+    def __init__(self, tmpdir):
+        self._tmpdir = tmpdir
+        self.downloads = []
+
+    @contextlib.contextmanager
+    def _open_fileobj(self, target):
+        """Ensure a DownloadTarget's fileobj attribute is available."""
+        if isinstance(target, downloads.FileDownloadTarget):
+            target.fileobj = open(target.filename, 'wb')
+            try:
+                yield target.fileobj
+            finally:
+                target.fileobj.close()
+        else:
+            yield target.fileobj
+
+    def get(self, url, target, **kwargs):
+        """Return a FakeDownloadItem instance with a fileobj.
+
+        The content is copied from the file the given url links to.
+        """
+        with self._open_fileobj(target):
+            download_item = FakeDownloadItem(target.fileobj, name=url.path())
+            with (self._tmpdir / url.path()).open('rb') as fake_url_file:
+                shutil.copyfileobj(fake_url_file, download_item.fileobj)
+        self.downloads.append(download_item)
+        return download_item
diff --git a/tests/unit/browser/test_adblock.py b/tests/unit/browser/test_adblock.py
index 09161e806..5b353efb9 100644
--- a/tests/unit/browser/test_adblock.py
+++ b/tests/unit/browser/test_adblock.py
@@ -21,15 +21,13 @@
 import os
 import os.path
 import zipfile
-import shutil
 import logging
 
 import pytest
 
-from PyQt5.QtCore import pyqtSignal, QUrl, QObject
+from PyQt5.QtCore import QUrl
 
 from qutebrowser.browser import adblock
-from qutebrowser.utils import objreg
 
 pytestmark = pytest.mark.usefixtures('qapp', 'config_tmpdir')
 
@@ -69,46 +67,6 @@ def basedir(fake_args):
     fake_args.basedir = None
 
 
-class FakeDownloadItem(QObject):
-
-    """Mock browser.downloads.DownloadItem."""
-
-    finished = pyqtSignal()
-
-    def __init__(self, fileobj, name, parent=None):
-        super().__init__(parent)
-        self.fileobj = fileobj
-        self.name = name
-        self.successful = True
-
-
-class FakeDownloadManager:
-
-    """Mock browser.downloads.DownloadManager."""
-
-    def __init__(self, tmpdir):
-        self._tmpdir = tmpdir
-
-    def get(self, url, target, **kwargs):
-        """Return a FakeDownloadItem instance with a fileobj.
-
-        The content is copied from the file the given url links to.
-        """
-        download_item = FakeDownloadItem(target.fileobj, name=url.path())
-        with (self._tmpdir / url.path()).open('rb') as fake_url_file:
-            shutil.copyfileobj(fake_url_file, download_item.fileobj)
-        return download_item
-
-
-@pytest.fixture
-def download_stub(win_registry, tmpdir):
-    """Register a FakeDownloadManager."""
-    stub = FakeDownloadManager(tmpdir)
-    objreg.register('qtnetwork-download-manager', stub)
-    yield
-    objreg.delete('qtnetwork-download-manager')
-
-
 def create_zipfile(directory, files, zipname='test'):
     """Return a path to a newly created zip file.
 
@@ -248,6 +206,7 @@ def test_disabled_blocking_update(basedir, config_stub, download_stub,
     while host_blocker._in_progress:
         current_download = host_blocker._in_progress[0]
         with caplog.at_level(logging.ERROR):
+            current_download.successful = True
             current_download.finished.emit()
     host_blocker.read_hosts()
     for str_url in URLS_TO_CHECK:
@@ -263,6 +222,8 @@ def test_no_blocklist_update(config_stub, download_stub,
     host_blocker = adblock.HostBlocker()
     host_blocker.adblock_update()
     host_blocker.read_hosts()
+    for dl in download_stub.downloads:
+        dl.successful = True
     for str_url in URLS_TO_CHECK:
         assert not host_blocker.is_blocked(QUrl(str_url))
 
@@ -280,6 +241,7 @@ def test_successful_update(config_stub, basedir, download_stub,
     while host_blocker._in_progress:
         current_download = host_blocker._in_progress[0]
         with caplog.at_level(logging.ERROR):
+            current_download.successful = True
             current_download.finished.emit()
     host_blocker.read_hosts()
     assert_urls(host_blocker, whitelisted=[])
@@ -307,6 +269,8 @@ def test_failed_dl_update(config_stub, basedir, download_stub,
         # if current download is the file we want to fail, make it fail
         if current_download.name == dl_fail_blocklist.path():
             current_download.successful = False
+        else:
+            current_download.successful = True
         with caplog.at_level(logging.ERROR):
             current_download.finished.emit()
     host_blocker.read_hosts()
@@ -336,16 +300,18 @@ def test_invalid_utf8(config_stub, download_stub, tmpdir, data_tmpdir,
 
     host_blocker = adblock.HostBlocker()
     host_blocker.adblock_update()
-    finished_signal = host_blocker._in_progress[0].finished
+    current_download = host_blocker._in_progress[0]
 
     if location == 'content':
         with caplog.at_level(logging.ERROR):
-            finished_signal.emit()
+            current_download.successful = True
+            current_download.finished.emit()
         expected = (r"Failed to decode: "
                     r"b'https://www.example.org/\xa0localhost")
         assert caplog.records[-2].message.startswith(expected)
     else:
-        finished_signal.emit()
+        current_download.successful = True
+        current_download.finished.emit()
 
     host_blocker.read_hosts()
     assert_urls(host_blocker, whitelisted=[])
diff --git a/tests/unit/javascript/test_greasemonkey.py b/tests/unit/javascript/test_greasemonkey.py
index 52af51a4b..7759f5d18 100644
--- a/tests/unit/javascript/test_greasemonkey.py
+++ b/tests/unit/javascript/test_greasemonkey.py
@@ -128,3 +128,33 @@ def test_load_emits_signal(qtbot):
     gm_manager = greasemonkey.GreasemonkeyManager()
     with qtbot.wait_signal(gm_manager.scripts_reloaded):
         gm_manager.load_scripts()
+
+
+def test_required_scripts_are_included(download_stub, tmpdir):
+    test_require_script = textwrap.dedent("""
+        // ==UserScript==
+        // @name qutebrowser test userscript
+        // @namespace invalid.org
+        // @include http://localhost:*/data/title.html
+        // @match http://trolol*
+        // @exclude https://badhost.xxx/*
+        // @run-at document-start
+        // @require http://localhost/test.js
+        // ==/UserScript==
+        console.log("Script is running.");
+    """)
+    _save_script(test_require_script, 'requiring.user.js')
+    with open(str(tmpdir / 'test.js'), 'w', encoding='UTF-8') as f:
+        f.write("REQUIRED SCRIPT")
+
+    gm_manager = greasemonkey.GreasemonkeyManager()
+    assert len(gm_manager._in_progress_dls) == 1
+    for download in gm_manager._in_progress_dls:
+        download.finished.emit()
+
+    scripts = gm_manager.all_scripts()
+    assert len(scripts) == 1
+    assert "REQUIRED SCRIPT" in scripts[0].code()
+    # Additionally check that the base script is still being parsed correctly
+    assert "Script is running." in scripts[0].code()
+    assert scripts[0].excludes