Refactor user agent handling

We now use a format string for the user_agent setting and parse both backend's
default user agents to get the needed information.

Fixes #513

19 files changed, 342 insertions, 85 deletions

diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc
index 8e9386101..3030c7459 100644
--- a/doc/changelog.asciidoc
+++ b/doc/changelog.asciidoc
@@ -32,6 +32,10 @@ Added
 Changed
 ~~~~~~~
 
+- The `content.headers.user_agent` setting now is a format string with the
+  default value resembling the behavior of it being set to null before.
+  This slightly changes the sent user agent for QtWebKit: Instead of mentioning
+  qutebrowser and its version it now mentions the Qt version.
 - The `qute-pass` userscript now has a new `--extra-url-suffixes` (`-s`)
   argument which passes extra URL suffixes to the tldextract library.
 - A stack is now used for `:tab-focus last` rather than just saving one tab.
diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc
index 355c30153..fe34f511d 100644
--- a/doc/help/settings.asciidoc
+++ b/doc/help/settings.asciidoc
@@ -138,7 +138,7 @@
 |<<content.headers.custom,content.headers.custom>>|Custom headers for qutebrowser HTTP requests.
 |<<content.headers.do_not_track,content.headers.do_not_track>>|Value to send in the `DNT` header.
 |<<content.headers.referer,content.headers.referer>>|When to send the Referer header.
-|<<content.headers.user_agent,content.headers.user_agent>>|User agent to send. Unset to send the default.
+|<<content.headers.user_agent,content.headers.user_agent>>|User agent to send.
 |<<content.host_blocking.enabled,content.host_blocking.enabled>>|Enable host blocking.
 |<<content.host_blocking.lists,content.host_blocking.lists>>|List of URLs of lists which contain hosts to block.
 |<<content.host_blocking.whitelist,content.host_blocking.whitelist>>|A list of patterns that should always be loaded, despite being ad-blocked.
@@ -1840,14 +1840,19 @@ Default: +pass:[same-domain]+
 
 [[content.headers.user_agent]]
 === content.headers.user_agent
-User agent to send. Unset to send the default.
+User agent to send.
+The following placeholders are defined:
+* `{os_info}`: Something like "X11; Linux x86_64". * `{webkit_version}`: The underlying WebKit version (set to a fixed value
+  with QtWebEngine).
+* `{qt_key}`: "Qt" for QtWebKit, "QtWebEngine" for QtWebEngine. * `{qt_version}`: The underlying Qt version. * `{upstream_browser_key}`: "Version" for QtWebKit, "Chrome" for QtWebEngine. * `{upstream_browser_version}`: The corresponding Safari/Chrome version. * `{qutebrowser_version}`: The currently running qutebrowser version.
+The default value is equal to the unchanged user agent of QtWebKit/QtWebEngine.
 Note that the value read from JavaScript is always the global value.
 
 This setting supports URL patterns.
 
-Type: <<types,String>>
+Type: <<types,FormatString>>
 
-Default: empty
+Default: +pass:[Mozilla/5.0 ({os_info}) AppleWebKit/{webkit_version} (KHTML, like Gecko) {qt_key}/{qt_version} {upstream_browser_key}/{upstream_browser_version} Safari/{webkit_version}]+
 
 [[content.host_blocking.enabled]]
 === content.host_blocking.enabled
diff --git a/qutebrowser/browser/webengine/interceptor.py b/qutebrowser/browser/webengine/interceptor.py
index 6d15b9c40..02f75fa0c 100644
--- a/qutebrowser/browser/webengine/interceptor.py
+++ b/qutebrowser/browser/webengine/interceptor.py
@@ -25,7 +25,7 @@ from PyQt5.QtCore import QUrl, QByteArray
 from PyQt5.QtWebEngineCore import (QWebEngineUrlRequestInterceptor,
                                    QWebEngineUrlRequestInfo)
 
-from qutebrowser.config import config
+from qutebrowser.config import websettings
 from qutebrowser.browser import shared
 from qutebrowser.utils import utils, log, debug, qtutils
 from qutebrowser.extensions import interceptors
@@ -204,6 +204,5 @@ class RequestInterceptor(QWebEngineUrlRequestInterceptor):
         for header, value in shared.custom_headers(url=url):
             info.setHttpHeader(header, value)
 
-        user_agent = config.instance.get('content.headers.user_agent', url=url)
-        if user_agent is not None:
-            info.setHttpHeader(b'User-Agent', user_agent.encode('ascii'))
+        user_agent = websettings.user_agent(url)
+        info.setHttpHeader(b'User-Agent', user_agent.encode('ascii'))
diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py
index 3cf4a8adc..4a677539f 100644
--- a/qutebrowser/browser/webengine/webenginesettings.py
+++ b/qutebrowser/browser/webengine/webenginesettings.py
@@ -44,7 +44,7 @@ private_profile = None  # type: typing.Optional[QWebEngineProfile]
 # The global WebEngineSettings object
 global_settings = typing.cast('WebEngineSettings', None)
 
-default_user_agent = None
+parsed_user_agent = None
 
 
 class _SettingsWrapper:
@@ -228,7 +228,9 @@ class ProfileSetter:
         per-domain values), but this one still gets used for things like
         window.navigator.userAgent/.languages in JS.
         """
-        self._profile.setHttpUserAgent(config.val.content.headers.user_agent)
+        user_agent = websettings.user_agent()
+        self._profile.setHttpUserAgent(user_agent)
+
         accept_language = config.val.content.headers.accept_language
         if accept_language is not None:
             self._profile.setHttpAcceptLanguage(accept_language)
@@ -296,12 +298,22 @@ def _update_settings(option):
             private_profile.setter.set_dictionary_language(warn=False)
 
 
+def _init_user_agent_str(ua):
+    global parsed_user_agent
+    parsed_user_agent = websettings.UserAgent.parse(ua)
+
+
+def init_user_agent():
+    _init_user_agent_str(QWebEngineProfile.defaultProfile().httpUserAgent())
+
+
 def _init_profiles():
     """Init the two used QWebEngineProfiles."""
-    global default_profile, private_profile, default_user_agent
+    global default_profile, private_profile
 
     default_profile = QWebEngineProfile.defaultProfile()
-    default_user_agent = default_profile.httpUserAgent()
+    init_user_agent()
+
     default_profile.setter = ProfileSetter(default_profile)
     default_profile.setCachePath(
         os.path.join(standarddir.cache(), 'webengine'))
diff --git a/qutebrowser/browser/webkit/webkitsettings.py b/qutebrowser/browser/webkit/webkitsettings.py
index 3c5eeb545..f5e55607d 100644
--- a/qutebrowser/browser/webkit/webkitsettings.py
+++ b/qutebrowser/browser/webkit/webkitsettings.py
@@ -27,8 +27,10 @@ Module attributes:
 import typing
 import os.path
 
+from PyQt5.QtCore import QUrl
 from PyQt5.QtGui import QFont
 from PyQt5.QtWebKit import QWebSettings
+from PyQt5.QtWebKitWidgets import QWebPage
 
 from qutebrowser.config import config, websettings
 from qutebrowser.config.websettings import AttributeInfo as Attr
@@ -39,6 +41,8 @@ from qutebrowser.browser import shared
 # The global WebKitSettings object
 global_settings = typing.cast('WebKitSettings', None)
 
+parsed_user_agent = None
+
 
 class WebKitSettings(websettings.AbstractSettings):
 
@@ -160,6 +164,12 @@ def _update_settings(option):
         _set_cache_maximum_pages(settings)
 
 
+def _init_user_agent():
+    global parsed_user_agent
+    ua = QWebPage().userAgentForUrl(QUrl())
+    parsed_user_agent = websettings.UserAgent.parse(ua)
+
+
 def init(_args):
     """Initialize the global QWebSettings."""
     cache_path = standarddir.cache()
@@ -178,6 +188,8 @@ def init(_args):
     _set_cookie_accept_policy(settings)
     _set_cache_maximum_pages(settings)
 
+    _init_user_agent()
+
     config.instance.changed.connect(_update_settings)
 
     global global_settings
diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py
index 7069c1699..743e550f0 100644
--- a/qutebrowser/browser/webkit/webpage.py
+++ b/qutebrowser/browser/webkit/webpage.py
@@ -29,7 +29,7 @@ from PyQt5.QtWidgets import QFileDialog
 from PyQt5.QtPrintSupport import QPrintDialog
 from PyQt5.QtWebKitWidgets import QWebPage, QWebFrame
 
-from qutebrowser.config import config
+from qutebrowser.config import websettings
 from qutebrowser.browser import pdfjs, shared, downloads, greasemonkey
 from qutebrowser.browser.webkit import http
 from qutebrowser.browser.webkit.network import networkmanager
@@ -411,11 +411,7 @@ class BrowserPage(QWebPage):
 
     def userAgentForUrl(self, url):
         """Override QWebPage::userAgentForUrl to customize the user agent."""
-        ua = config.instance.get('content.headers.user_agent', url=url)
-        if ua is None:
-            return super().userAgentForUrl(url)
-        else:
-            return ua
+        return websettings.user_agent(url)
 
     def supportsExtension(self, ext):
         """Override QWebPage::supportsExtension to provide error pages.
diff --git a/qutebrowser/commands/userscripts.py b/qutebrowser/commands/userscripts.py
index 439af695c..8ff19c7d8 100644
--- a/qutebrowser/commands/userscripts.py
+++ b/qutebrowser/commands/userscripts.py
@@ -28,7 +28,7 @@ from PyQt5.QtCore import pyqtSignal, pyqtSlot, QObject, QSocketNotifier
 
 from qutebrowser.utils import message, log, objreg, standarddir, utils
 from qutebrowser.commands import runners
-from qutebrowser.config import config
+from qutebrowser.config import websettings
 from qutebrowser.misc import guiprocess
 from qutebrowser.browser import downloads
 
@@ -429,10 +429,8 @@ def run_async(tab, cmd, *args, win_id, env, verbose=False):
         lambda cmd:
         log.commands.debug("Got userscript command: {}".format(cmd)))
     runner.got_cmd.connect(commandrunner.run_safely)
-    user_agent = config.val.content.headers.user_agent
-    if user_agent is not None:
-        env['QUTE_USER_AGENT'] = user_agent
 
+    env['QUTE_USER_AGENT'] = websettings.user_agent()
     env['QUTE_CONFIG_DIR'] = standarddir.config()
     env['QUTE_DATA_DIR'] = standarddir.data()
     env['QUTE_DOWNLOAD_DIR'] = downloads.download_dir()
diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml
index 84c157b28..f105f89c8 100644
--- a/qutebrowser/config/configdata.yml
+++ b/qutebrowser/config/configdata.yml
@@ -469,10 +469,20 @@ content.headers.referer:
     No restart is needed with QtWebKit.
 
 content.headers.user_agent:
-  default: null
+  default: 'Mozilla/5.0 ({os_info})
+      AppleWebKit/{webkit_version} (KHTML, like Gecko)
+      {qt_key}/{qt_version} {upstream_browser_key}/{upstream_browser_version}
+      Safari/{webkit_version}'
   type:
-    name: String
-    none_ok: true
+    name: FormatString
+    fields:
+      - os_info
+      - webkit_version
+      - qt_key
+      - qt_version
+      - upstream_browser_key
+      - upstream_browser_version
+      - qutebrowser_version
     completions:
       # To update the following list of user agents, run the script
       # 'ua_fetch.py'
@@ -484,12 +494,23 @@ content.headers.user_agent:
       - - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like
           Gecko) Chrome/76.0.3809.132 Safari/537.36"
         - Chrome 76 Linux
-      - - ""
-        - Use default QtWebKit/QtWebEngine User-Agent
-
   supports_pattern: true
   desc: >-
-    User agent to send. Unset to send the default.
+    User agent to send.
+
+    The following placeholders are defined:
+
+    * `{os_info}`: Something like "X11; Linux x86_64".
+    * `{webkit_version}`: The underlying WebKit version (set to a fixed value
+      with QtWebEngine).
+    * `{qt_key}`: "Qt" for QtWebKit, "QtWebEngine" for QtWebEngine.
+    * `{qt_version}`: The underlying Qt version.
+    * `{upstream_browser_key}`: "Version" for QtWebKit, "Chrome" for QtWebEngine.
+    * `{upstream_browser_version}`: The corresponding Safari/Chrome version.
+    * `{qutebrowser_version}`: The currently running qutebrowser version.
+
+    The default value is equal to the unchanged user agent of
+    QtWebKit/QtWebEngine.
 
     Note that the value read from JavaScript is always the global value.
 
diff --git a/qutebrowser/config/configfiles.py b/qutebrowser/config/configfiles.py
index ebfec1354..8f35ebc7f 100644
--- a/qutebrowser/config/configfiles.py
+++ b/qutebrowser/config/configfiles.py
@@ -271,6 +271,14 @@ class YamlConfig(QObject):
                     settings[name][scope] = true_value if val else false_value
                     self._mark_changed()
 
+    def _migrate_none(self, settings: _SettingsType, name: str,
+                      value: str) -> None:
+        if name in settings:
+            for scope, val in settings[name].items():
+                if val is None:
+                    settings[name][scope] = value
+                    self._mark_changed()
+
     def _migrate_string_value(self, settings: _SettingsType, name: str,
                               source: str, target: str) -> None:
         if name in settings:
@@ -343,6 +351,10 @@ class YamlConfig(QObject):
             self._migrate_string_value(
                 settings, s, r'(?<!{)\{title\}(?!})', r'{current_title}')
 
+        # content.headers.user_agent can't be empty to get the default anymore.
+        s = 'content.headers.user_agent'
+        self._migrate_none(settings, s, configdata.DATA[s].default)
+
         return settings
 
     def _validate(self, settings: _SettingsType) -> None:
diff --git a/qutebrowser/config/websettings.py b/qutebrowser/config/websettings.py
index db8f77387..a4c271e6b 100644
--- a/qutebrowser/config/websettings.py
+++ b/qutebrowser/config/websettings.py
@@ -19,19 +19,65 @@
 
 """Bridge from QWeb(Engine)Settings to our own settings."""
 
+import re
 import typing
 import argparse
+import functools
 
-from PyQt5.QtCore import QUrl, pyqtSlot
+import attr
+from PyQt5.QtCore import QUrl, pyqtSlot, qVersion
 from PyQt5.QtGui import QFont
 
+import qutebrowser
 from qutebrowser.config import config, configutils
 from qutebrowser.utils import log, usertypes, urlmatch, qtutils
-from qutebrowser.misc import objects
+from qutebrowser.misc import objects, debugcachestats
 
 UNSET = object()
 
 
+@attr.s
+class UserAgent:
+
+    """A parsed user agent."""
+
+    os_info = attr.ib()  # type: str
+    webkit_version = attr.ib()  # type: str
+    upstream_browser_key = attr.ib()  # type: str
+    upstream_browser_version = attr.ib()  # type: str
+    qt_key = attr.ib()  # type: str
+
+    @classmethod
+    def parse(cls, ua: str) -> 'UserAgent':
+        """Parse a user agent string into its components."""
+        comment_matches = re.finditer(r'\(([^)]*)\)', ua)
+        os_info = list(comment_matches)[0].group(1)
+
+        version_matches = re.finditer(r'(\S+)/(\S+)', ua)
+        versions = {}
+        for match in version_matches:
+            versions[match.group(1)] = match.group(2)
+
+        webkit_version = versions['AppleWebKit']
+
+        if 'Chrome' in versions:
+            upstream_browser_key = 'Chrome'
+            qt_key = 'QtWebEngine'
+        elif 'Version' in versions:
+            upstream_browser_key = 'Version'
+            qt_key = 'Qt'
+        else:
+            raise ValueError("Invalid upstream browser key: {}".format(ua))
+
+        upstream_browser_version = versions[upstream_browser_key]
+
+        return cls(os_info=os_info,
+                   webkit_version=webkit_version,
+                   upstream_browser_key=upstream_browser_key,
+                   upstream_browser_version=upstream_browser_version,
+                   qt_key=qt_key)
+
+
 class AttributeInfo:
 
     """Info about a settings attribute."""
@@ -183,6 +229,34 @@ class AbstractSettings:
             self.update_setting(setting)
 
 
+@debugcachestats.register(name='user agent cache')
+@functools.lru_cache()
+def _format_user_agent(template: str, backend: usertypes.Backend) -> str:
+    if backend == usertypes.Backend.QtWebEngine:
+        from qutebrowser.browser.webengine import webenginesettings
+        parsed = webenginesettings.parsed_user_agent
+    else:
+        from qutebrowser.browser.webkit import webkitsettings
+        parsed = webkitsettings.parsed_user_agent
+
+    assert parsed is not None
+
+    return template.format(
+        os_info=parsed.os_info,
+        webkit_version=parsed.webkit_version,
+        qt_key=parsed.qt_key,
+        qt_version=qVersion(),
+        upstream_browser_key=parsed.upstream_browser_key,
+        upstream_browser_version=parsed.upstream_browser_version,
+        qutebrowser_version=qutebrowser.__version__,
+    )
+
+
+def user_agent(url: QUrl = None) -> str:
+    template = config.instance.get('content.headers.user_agent', url=url)
+    return _format_user_agent(template=template, backend=objects.backend)
+
+
 def init(args: argparse.Namespace) -> None:
     """Initialize all QWeb(Engine)Settings."""
     if objects.backend == usertypes.Backend.QtWebEngine:
diff --git a/qutebrowser/utils/usertypes.py b/qutebrowser/utils/usertypes.py
index 5214459bb..58fe950b2 100644
--- a/qutebrowser/utils/usertypes.py
+++ b/qutebrowser/utils/usertypes.py
@@ -25,7 +25,7 @@ import typing
 
 import attr
 from PyQt5.QtCore import pyqtSignal, pyqtSlot, QObject, QTimer
-from PyQt5.QtCore import QUrl  # pylint: disable=unused-import
+from PyQt5.QtCore import QUrl
 
 from qutebrowser.utils import log, qtutils, utils
 
diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py
index da17cd6bc..6f8628daf 100644
--- a/qutebrowser/utils/version.py
+++ b/qutebrowser/utils/version.py
@@ -45,11 +45,6 @@ try:
 except ImportError:  # pragma: no cover
     qWebKitVersion = None  # type: ignore  # noqa: N816
 
-try:
-    from PyQt5.QtWebEngineWidgets import QWebEngineProfile
-except ImportError:  # pragma: no cover
-    QWebEngineProfile = None  # type: ignore
-
 import qutebrowser
 from qutebrowser.utils import log, utils, standarddir, usertypes, message
 from qutebrowser.misc import objects, earlyinit, sql, httpclient, pastebin
@@ -369,18 +364,11 @@ def _chromium_version() -> str:
     Also see https://www.chromium.org/developers/calendar
     and https://chromereleases.googleblog.com/
     """
-    if webenginesettings is None or QWebEngineProfile is None:  # type: ignore
-        # This should never happen
-        return 'unavailable'  # type: ignore
-    ua = webenginesettings.default_user_agent
-    if ua is None:
-        profile = QWebEngineProfile.defaultProfile()
-        ua = profile.httpUserAgent()
-    match = re.search(r' Chrome/([^ ]*) ', ua)
-    if not match:
-        log.misc.error("Could not get Chromium version from: {}".format(ua))
-        return 'unknown'
-    return match.group(1)
+    if webenginesettings.parsed_user_agent is None:
+        webenginesettings.init_user_agent()
+        assert webenginesettings.parsed_user_agent is not None
+
+    return webenginesettings.parsed_user_agent.upstream_browser_version
 
 
 def _backend() -> str:
diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature
index e6b7f9fa4..7a26c9dda 100644
--- a/tests/end2end/features/misc.feature
+++ b/tests/end2end/features/misc.feature
@@ -362,13 +362,6 @@ Feature: Various utility commands.
         Then the header User-Agent should be set to toaster
         And the javascript message "toaster" should be logged
 
-    Scenario: Setting the default user-agent header
-        When I set content.headers.user_agent to <empty>
-        And I open headers
-        And I run :jseval console.log(window.navigator.userAgent)
-        Then the header User-Agent should be set to Mozilla/5.0 *
-        And the javascript message "Mozilla/5.0 *" should be logged
-
     ## https://github.com/qutebrowser/qutebrowser/issues/1523
 
     Scenario: Completing a single option argument
diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py
index c76d4f061..880438d93 100644
--- a/tests/helpers/fixtures.py
+++ b/tests/helpers/fixtures.py
@@ -204,12 +204,13 @@ def web_tab_setup(qtbot, tab_registry, session_manager_stub,
 
 @pytest.fixture
 def webkit_tab(web_tab_setup, qtbot, cookiejar_and_cache, mode_manager,
-               widget_container):
+               widget_container, webpage):
     webkittab = pytest.importorskip('qutebrowser.browser.webkit.webkittab')
 
     tab = webkittab.WebKitTab(win_id=0, mode_manager=mode_manager,
                               private=False)
     widget_container.set_widget(tab)
+
     return tab
 
 
@@ -416,6 +417,7 @@ def webengineview(qtbot, monkeypatch, web_tab_setup):
 def webpage(qnam):
     """Get a new QWebPage object."""
     QtWebKitWidgets = pytest.importorskip('PyQt5.QtWebKitWidgets')
+
     class WebPageStub(QtWebKitWidgets.QWebPage):
 
         """QWebPage with default error pages disabled."""
@@ -425,8 +427,13 @@ def webpage(qnam):
             return False
 
     page = WebPageStub()
+
     page.networkAccessManager().deleteLater()
     page.setNetworkAccessManager(qnam)
+
+    from qutebrowser.browser.webkit import webkitsettings
+    webkitsettings._init_user_agent()
+
     return page
 
 
diff --git a/tests/unit/browser/webengine/test_webenginesettings.py b/tests/unit/browser/webengine/test_webenginesettings.py
index 0e369d655..d2a6b96ba 100644
--- a/tests/unit/browser/webengine/test_webenginesettings.py
+++ b/tests/unit/browser/webengine/test_webenginesettings.py
@@ -33,6 +33,7 @@ from qutebrowser.misc import objects
 def init(qapp, config_stub, cache_tmpdir, data_tmpdir, monkeypatch):
     monkeypatch.setattr(webenginesettings.webenginequtescheme, 'init',
                         lambda: None)
+    monkeypatch.setattr(objects, 'backend', usertypes.Backend.QtWebEngine)
     init_args = types.SimpleNamespace(enable_webengine_inspector=False)
     webenginesettings.init(init_args)
     config_stub.changed.disconnect(webenginesettings._update_settings)
@@ -49,7 +50,6 @@ def test_big_cache_size(config_stub):
 @pytest.mark.skipif(
     not qtutils.version_check('5.8'), reason="Needs Qt 5.8 or newer")
 def test_non_existing_dict(config_stub, monkeypatch, message_mock, caplog):
-    monkeypatch.setattr(objects, 'backend', usertypes.Backend.QtWebEngine)
     monkeypatch.setattr(webenginesettings.spell, 'local_filename',
                         lambda _code: None)
     config_stub.val.spellcheck.languages = ['af-ZA']
@@ -66,7 +66,6 @@ def test_non_existing_dict(config_stub, monkeypatch, message_mock, caplog):
 @pytest.mark.skipif(
     not qtutils.version_check('5.8'), reason="Needs Qt 5.8 or newer")
 def test_existing_dict(config_stub, monkeypatch):
-    monkeypatch.setattr(objects, 'backend', usertypes.Backend.QtWebEngine)
     monkeypatch.setattr(webenginesettings.spell, 'local_filename',
                         lambda _code: 'en-US-8-0')
     config_stub.val.spellcheck.languages = ['en-US']
@@ -80,7 +79,6 @@ def test_existing_dict(config_stub, monkeypatch):
 @pytest.mark.skipif(
     not qtutils.version_check('5.8'), reason="Needs Qt 5.8 or newer")
 def test_spell_check_disabled(config_stub, monkeypatch):
-    monkeypatch.setattr(objects, 'backend', usertypes.Backend.QtWebEngine)
     config_stub.val.spellcheck.languages = []
     webenginesettings._update_settings('spellcheck.languages')
     for profile in [webenginesettings.default_profile,
@@ -89,4 +87,11 @@ def test_spell_check_disabled(config_stub, monkeypatch):
 
 
 def test_default_user_agent_saved():
-    assert webenginesettings.default_user_agent is not None
+    assert webenginesettings.parsed_user_agent is not None
+
+
+def test_parsed_user_agent(qapp):
+    webenginesettings.init_user_agent()
+    parsed = webenginesettings.parsed_user_agent
+    assert parsed.upstream_browser_key == 'Chrome'
+    assert parsed.qt_key == 'QtWebEngine'
diff --git a/tests/unit/browser/webkit/test_webkitsettings.py b/tests/unit/browser/webkit/test_webkitsettings.py
new file mode 100644
index 000000000..bb7fbecb8
--- /dev/null
+++ b/tests/unit/browser/webkit/test_webkitsettings.py
@@ -0,0 +1,31 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2019 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+import pytest
+pytest.importorskip('PyQt5.QtWebKitWidgets')
+
+from qutebrowser.browser.webkit import webkitsettings
+
+
+def test_parsed_user_agent(qapp):
+    webkitsettings._init_user_agent()
+
+    parsed = webkitsettings.parsed_user_agent
+    assert parsed.upstream_browser_key == 'Version'
+    assert parsed.qt_key == 'Qt'
diff --git a/tests/unit/config/test_configfiles.py b/tests/unit/config/test_configfiles.py
index 833e1e4fd..36bf88868 100644
--- a/tests/unit/config/test_configfiles.py
+++ b/tests/unit/config/test_configfiles.py
@@ -354,6 +354,17 @@ class TestYaml:
     def test_title_format_migrations(self, migration_test, setting, old, new):
         migration_test(setting, old, new)
 
+    @pytest.mark.parametrize('old, new', [
+        (None, ('Mozilla/5.0 ({os_info}) '
+                'AppleWebKit/{webkit_version} (KHTML, like Gecko) '
+                '{qt_key}/{qt_version} '
+                '{upstream_browser_key}/{upstream_browser_version} '
+                'Safari/{webkit_version}')),
+        ('toaster', 'toaster'),
+    ])
+    def test_user_agent_migration(self, migration_test, old, new):
+        migration_test('content.headers.user_agent', old, new)
+
     def test_renamed_key_unknown_target(self, monkeypatch, yaml,
                                         autoconfig):
         """A key marked as renamed with invalid name should raise an error."""
diff --git a/tests/unit/config/test_websettings.py b/tests/unit/config/test_websettings.py
new file mode 100644
index 000000000..916c8d609
--- /dev/null
+++ b/tests/unit/config/test_websettings.py
@@ -0,0 +1,92 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2019 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+import pytest
+
+from qutebrowser.config import websettings
+from qutebrowser.misc import objects
+from qutebrowser.utils import usertypes
+
+
+@pytest.mark.parametrize([
+    'user_agent', 'os_info', 'webkit_version',
+    'upstream_browser_key', 'upstream_browser_version', 'qt_key'
+], [
+    (
+        # QtWebEngine, Linux
+        # (no differences other than Chrome version with older Qt Versions)
+        ("Mozilla/5.0 (X11; Linux x86_64) "
+         "AppleWebKit/537.36 (KHTML, like Gecko) "
+         "QtWebEngine/5.14.0 Chrome/77.0.3865.98 Safari/537.36"),
+        "X11; Linux x86_64",
+        "537.36",
+        "Chrome", "77.0.3865.98",
+        "QtWebEngine",
+    ), (
+        # QtWebKit, Linux
+        ("Mozilla/5.0 (X11; Linux x86_64) "
+         "AppleWebKit/602.1 (KHTML, like Gecko) "
+         "qutebrowser/1.8.3 "
+         "Version/10.0 Safari/602.1"),
+        "X11; Linux x86_64",
+        "602.1",
+        "Version", "10.0",
+        "Qt",
+    ), (
+        # QtWebEngine, macOS
+        ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) "
+         "AppleWebKit/537.36 (KHTML, like Gecko) "
+         "QtWebEngine/5.13.2 Chrome/73.0.3683.105 Safari/537.36"),
+        "Macintosh; Intel Mac OS X 10_12_6",
+        "537.36",
+        "Chrome", "73.0.3683.105",
+        "QtWebEngine",
+    ), (
+        # QtWebEngine, Windows
+        ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+         "AppleWebKit/537.36 (KHTML, like Gecko) "
+         "QtWebEngine/5.12.5 Chrome/69.0.3497.128 Safari/537.36"),
+        "Windows NT 10.0; Win64; x64",
+        "537.36",
+        "Chrome", "69.0.3497.128",
+        "QtWebEngine",
+    )
+])
+def test_parse_user_agent(user_agent, os_info, webkit_version,
+                          upstream_browser_key, upstream_browser_version,
+                          qt_key):
+    parsed = websettings.UserAgent.parse(user_agent)
+    assert parsed.os_info == os_info
+    assert parsed.webkit_version == webkit_version
+    assert parsed.upstream_browser_key == upstream_browser_key
+    assert parsed.upstream_browser_version == upstream_browser_version
+    assert parsed.qt_key == qt_key
+
+
+def test_user_agent(monkeypatch, config_stub, qapp):
+    webenginesettings = pytest.importorskip(
+        "qutebrowser.browser.webengine.webenginesettings")
+    monkeypatch.setattr(objects, 'backend', usertypes.Backend.QtWebEngine)
+    webenginesettings.init_user_agent()
+
+    config_stub.val.content.headers.user_agent = 'test {qt_key}'
+    assert websettings.user_agent() == 'test QtWebEngine'
+
+    config_stub.val.content.headers.user_agent = 'test2 {qt_key}'
+    assert websettings.user_agent() == 'test2 QtWebEngine'
diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py
index 9ff2b42f9..043621667 100644
--- a/tests/unit/utils/test_version.py
+++ b/tests/unit/utils/test_version.py
@@ -851,36 +851,32 @@ class FakeQSslSocket:
         return self._version
 
 
-@pytest.mark.parametrize('ua, expected', [
-    (None, 'unavailable'),  # No QWebEngineProfile
-    ('Mozilla/5.0', 'unknown'),
-    ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
-     'QtWebEngine/5.8.0 Chrome/53.0.2785.148 Safari/537.36', '53.0.2785.148'),
-])
-def test_chromium_version(monkeypatch, caplog, ua, expected):
+_QTWE_USER_AGENT = ("Mozilla/5.0 (X11; Linux x86_64) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "QtWebEngine/5.14.0 Chrome/{} Safari/537.36")
+
+
+def test_chromium_version(monkeypatch, caplog):
     pytest.importorskip('PyQt5.QtWebEngineWidgets')
-    if ua is None:
-        monkeypatch.setattr(version, 'webenginesettings', None)
-    else:
-        monkeypatch.setattr(version.webenginesettings,
-                            'default_user_agent', ua)
+
+    ver = '77.0.3865.98'
+    version.webenginesettings._init_user_agent_str(
+        _QTWE_USER_AGENT.format(ver))
 
     with caplog.at_level(logging.ERROR):
-        assert version._chromium_version() == expected
+        assert version._chromium_version() == ver
 
 
 def test_chromium_version_prefers_saved_user_agent(monkeypatch):
     pytest.importorskip('PyQt5.QtWebEngineWidgets')
-    monkeypatch.setattr(
-        version.webenginesettings, 'default_user_agent',
-        'QtWebEngine/5.8.0 Chrome/53.0.2785.148 Safari/537.36'
-    )
+    version.webenginesettings._init_user_agent_str(_QTWE_USER_AGENT)
 
     class FakeProfile:
         def defaultProfile(self):
             raise AssertionError("Should not be called")
 
-    monkeypatch.setattr(version, 'QWebEngineProfile', FakeProfile())
+    monkeypatch.setattr(version.webenginesettings, 'QWebEngineProfile',
+                        FakeProfile())
 
     version._chromium_version()
 
@@ -918,12 +914,9 @@ class VersionParams:
 ], ids=lambda param: param.name)
 def test_version_output(params, stubs, monkeypatch, config_stub):
     """Test version.version()."""
-    class FakeWebEngineSettings:
-        default_user_agent = ('Toaster/4.0.4 Chrome/CHROMIUMVERSION '
-                              'Teapot/4.1.8')
-
     config.instance.config_py_loaded = params.config_py_loaded
     import_path = os.path.abspath('/IMPORTPATH')
+
     patches = {
         'qutebrowser.__file__': os.path.join(import_path, '__init__.py'),
         'qutebrowser.__version__': 'VERSION',
@@ -960,6 +953,12 @@ def test_version_output(params, stubs, monkeypatch, config_stub):
         'autoconfig_loaded': "yes" if params.autoconfig_loaded else "no",
     }
 
+    ua = _QTWE_USER_AGENT.format('CHROMIUMVERSION')
+    if version.webenginesettings is None:
+        patches['_chromium_version'] = lambda: 'CHROMIUMVERSION'
+    else:
+        version.webenginesettings._init_user_agent_str(ua)
+
     if params.config_py_loaded:
         substitutions["config_py_loaded"] = "{} has been loaded".format(
             standarddir.config_py())
@@ -975,8 +974,6 @@ def test_version_output(params, stubs, monkeypatch, config_stub):
         monkeypatch.delattr(version, 'qtutils.qWebKitVersion', raising=False)
         patches['objects.backend'] = usertypes.Backend.QtWebEngine
         substitutions['backend'] = 'QtWebEngine (Chromium CHROMIUMVERSION)'
-        patches['webenginesettings'] = FakeWebEngineSettings
-        patches['QWebEngineProfile'] = True
 
     if params.known_distribution:
         patches['distribution'] = lambda: version.DistributionInfo(