diff options
author | toofar <toofar@spalge.com> | 2023-12-02 13:34:55 +1300 |
---|---|---|
committer | toofar <toofar@spalge.com> | 2023-12-02 13:36:14 +1300 |
commit | 23a168926262f2304652baee57b7793c87ffe359 (patch) | |
tree | 01806daeb5c9a4e1b9ee51400d46c4d6866c687f | |
parent | 7d5acb697086c903d485679f3dc754777c8e6ef5 (diff) | |
parent | 75c78cadc4c7391df4654798b4b6de0c96007597 (diff) | |
download | qutebrowser-23a168926262f2304652baee57b7793c87ffe359.tar.gz qutebrowser-23a168926262f2304652baee57b7793c87ffe359.zip |
Merge branch 'main' into pr/7992
This is just to get the CI to re-trigger.
Apparently the checkout action doesn't actually checkout the branch
being proposed for merge but checks out an actual merge commit. So if
you push a PR while main is broken it'll say changes from main are on
your branch. That's a little unexpected.
main is fixed now and I tried re-running the CI jobs from the web UI but
they are still failing with the same errors. Hence this merge of main
just to get a change on the branch. I could have gone and found a typo
to fix instead. I know I've left enough of them around.
ref: https://github.com/actions/checkout/issues/881
41 files changed, 1095 insertions, 222 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccfa69ca3..c2babf437 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,28 +157,28 @@ jobs: - testenv: py310-pyqt65 os: ubuntu-22.04 python: "3.10" - ### PyQt 6.5 (Python 3.11) - - testenv: py311-pyqt65 + ### PyQt 6.6 (Python 3.11) + - testenv: py311-pyqt66 os: ubuntu-22.04 python: "3.11" - ### PyQt 6.5 (Python 3.12) - - testenv: py312-pyqt65 + ### PyQt 6.6 (Python 3.12) + - testenv: py312-pyqt66 os: ubuntu-22.04 - python: "3.12-dev" - ### macOS Big Sur: PyQt 5.15 (Python 3.9 to match PyInstaller env) - - testenv: py39-pyqt515 + python: "3.12" + ### macOS Big Sur + - testenv: py311-pyqt66 os: macos-11 - python: "3.9" + python: "3.11" args: "tests/unit" # Only run unit tests on macOS ### macOS Monterey - - testenv: py39-pyqt515 + - testenv: py311-pyqt66 os: macos-12 - python: "3.9" + python: "3.11" args: "tests/unit" # Only run unit tests on macOS - ### Windows: PyQt 5.15 (Python 3.9 to match PyInstaller env) - - testenv: py39-pyqt515 + ### Windows + - testenv: py311-pyqt66 os: windows-2019 - python: "3.9" + python: "3.11" runs-on: "${{ matrix.os }}" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 76332e8ba..433cd3c0b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -15,24 +15,19 @@ jobs: matrix: include: - os: macos-11 - branch: main toxenv: build-release-qt5 name: qt5-macos - os: windows-2019 - branch: main toxenv: build-release-qt5 name: qt5-windows - os: macos-11 args: --debug - branch: main toxenv: build-release-qt5 name: qt5-macos-debug - os: windows-2019 args: --debug - branch: main toxenv: build-release-qt5 name: qt5-windows-debug - - os: macos-11 toxenv: build-release name: macos @@ -52,7 +47,6 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: "${{ matrix.branch }}" persist-credentials: false - name: Set up Python uses: actions/setup-python@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 786f9742c..fd3bc5cd8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,7 +34,7 @@ jobs: contents: write # To push release commit/tag steps: - name: Find release branch - uses: actions/github-script@v6 + uses: actions/github-script@v7 id: find-branch with: script: | @@ -84,7 +84,7 @@ jobs: id: bump run: "tox -e update-version -- ${{ github.event.inputs.release_type }}" - name: Check milestone - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: script: | const milestones = await github.paginate(github.rest.issues.listMilestones, { @@ -178,7 +178,7 @@ jobs: contents: write # To change release steps: - name: Publish final release - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: script: | await github.rest.repos.updateRelease({ diff --git a/README.asciidoc b/README.asciidoc index 2b6bdfdd6..364f8fa62 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -227,7 +227,7 @@ Active https://tridactyl.xyz/[Tridactyl], https://addons.mozilla.org/en-GB/firefox/addon/vimium-ff/[Vimium-FF] * Addons for Firefox and Chrome: - https://github.com/brookhong/Surfingkeys[Surfingkeys], + https://github.com/brookhong/Surfingkeys[Surfingkeys] (https://github.com/brookhong/Surfingkeys/issues/1796[somewhat sketchy]...), https://lydell.github.io/LinkHints/[Link Hints] (hinting only), https://github.com/ueokande/vimmatic[Vimmatic] diff --git a/doc/backers.asciidoc b/doc/backers.asciidoc index bdabb5f96..81ccc14ab 100644 --- a/doc/backers.asciidoc +++ b/doc/backers.asciidoc @@ -1,6 +1,14 @@ Crowdfunding backers ==================== +2019+ +----- + +Since late 2019, qutebrowser is taking recurring donations via +https://github.com/sponsors/The-Compiler/[GitHub Sponsors] and +https://liberapay.com/The-Compiler/[Liberapay]. You can find Sponsors/Patrons +who opted to be listed as public on the respective pages. **Thank you!** + 2017 ---- diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index a799deaab..43888903e 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -15,6 +15,25 @@ breaking changes (such as renamed commands) can happen in minor releases. // `Fixed` for any bug fixes. // `Security` to invite users to upgrade in case of vulnerabilities. +[[v3.1.0]] +v3.1.0 (unreleased) +------------------- + +Changed +~~~~~~~ + +- (TODO) Upgraded the bundled Qt version to 6.6.1, based on Chromium 112. Note + this is only relevant for the macOS/Windows releases, on Linux those will be + upgraded via your distribution packages. + +Fixed +~~~~~ + +- (TODO) Compatibility with PDF.js v4 +- Added an elaborate workaround for a bug in QtWebEngine 6.6.0 causing crashes + on Google Mail/Meet/Chat. + + [[v3.0.2]] v3.0.2 (2023-10-19) ------------------- diff --git a/misc/requirements/requirements-dev.txt b/misc/requirements/requirements-dev.txt index 3d6a7760a..4d1eb9646 100644 --- a/misc/requirements/requirements-dev.txt +++ b/misc/requirements/requirements-dev.txt @@ -2,19 +2,19 @@ build==1.0.3 bump2version==1.0.1 -certifi==2023.7.22 +certifi==2023.11.17 cffi==1.16.0 -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 cryptography==41.0.5 docutils==0.20.1 github3.py==4.0.1 hunter==3.6.1 -idna==3.4 +idna==3.6 importlib-metadata==6.8.0 -importlib-resources==6.1.0 +importlib-resources==6.1.1 jaraco.classes==3.3.0 jeepney==0.8.0 -keyring==24.2.0 +keyring==24.3.0 manhole==1.8.0 markdown-it-py==3.0.0 mdurl==0.1.2 @@ -24,7 +24,7 @@ packaging==23.2 pkginfo==1.9.6 ply==3.11 pycparser==2.21 -Pygments==2.16.1 +Pygments==2.17.2 PyJWT==2.8.0 Pympler==1.0.1 pyproject_hooks==1.0.0 @@ -34,7 +34,7 @@ readme-renderer==42.0 requests==2.31.0 requests-toolbelt==1.0.0 rfc3986==2.0.0 -rich==13.6.0 +rich==13.7.0 SecretStorage==3.3.3 sip==6.7.12 six==1.16.0 @@ -42,5 +42,5 @@ tomli==2.0.1 twine==4.0.2 typing_extensions==4.8.0 uritemplate==4.1.1 -# urllib3==2.0.7 +# urllib3==2.1.0 zipp==3.17.0 diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt index 95a9cb382..10d0daab5 100644 --- a/misc/requirements/requirements-flake8.txt +++ b/misc/requirements/requirements-flake8.txt @@ -2,11 +2,11 @@ attrs==23.1.0 flake8==6.1.0 -flake8-bugbear==23.9.16 -flake8-builtins==2.1.0 +flake8-bugbear==23.11.26 +flake8-builtins==2.2.0 flake8-comprehensions==3.14.0 flake8-debugger==4.1.2 -flake8-deprecated==2.1.0 +flake8-deprecated==2.2.1 flake8-docstrings==1.7.0 flake8-future-import==0.4.7 flake8-plugin-utils==1.3.3 diff --git a/misc/requirements/requirements-mypy.txt b/misc/requirements/requirements-mypy.txt index 8b5b68b56..c23c115a7 100644 --- a/misc/requirements/requirements-mypy.txt +++ b/misc/requirements/requirements-mypy.txt @@ -1,21 +1,21 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py chardet==5.2.0 -diff_cover==8.0.0 -importlib-resources==6.1.0 +diff_cover==8.0.1 +importlib-resources==6.1.1 Jinja2==3.1.2 lxml==4.9.3 MarkupSafe==2.1.3 -mypy==1.6.1 +mypy==1.7.1 mypy-extensions==1.0.0 pluggy==1.3.0 -Pygments==2.16.1 +Pygments==2.17.2 PyQt5-stubs==5.15.6.0 tomli==2.0.1 types-colorama==0.4.15.12 types-docutils==0.20.0.3 -types-Pygments==2.16.0.0 +types-Pygments==2.17.0.0 types-PyYAML==6.0.12.12 -types-setuptools==68.2.0.0 +types-setuptools==68.2.0.2 typing_extensions==4.8.0 zipp==3.17.0 diff --git a/misc/requirements/requirements-pyinstaller.txt b/misc/requirements/requirements-pyinstaller.txt index d62b99df5..d1a2c18c9 100644 --- a/misc/requirements/requirements-pyinstaller.txt +++ b/misc/requirements/requirements-pyinstaller.txt @@ -3,6 +3,6 @@ altgraph==0.17.4 importlib-metadata==6.8.0 packaging==23.2 -pyinstaller==6.1.0 +pyinstaller==6.2.0 pyinstaller-hooks-contrib==2023.10 zipp==3.17.0 diff --git a/misc/requirements/requirements-pylint.txt b/misc/requirements/requirements-pylint.txt index 6cdd658f8..a782d7182 100644 --- a/misc/requirements/requirements-pylint.txt +++ b/misc/requirements/requirements-pylint.txt @@ -1,17 +1,17 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py astroid==3.0.1 -certifi==2023.7.22 +certifi==2023.11.17 cffi==1.16.0 -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 cryptography==41.0.5 dill==0.3.7 github3.py==4.0.1 -idna==3.4 +idna==3.6 isort==5.12.0 mccabe==0.7.0 pefile==2023.2.7 -platformdirs==3.11.0 +platformdirs==4.0.0 pycparser==2.21 PyJWT==2.8.0 pylint==3.0.2 @@ -20,7 +20,7 @@ python-dateutil==2.8.2 requests==2.31.0 six==1.16.0 tomli==2.0.1 -tomlkit==0.12.1 +tomlkit==0.12.3 typing_extensions==4.8.0 uritemplate==4.1.1 -# urllib3==2.0.7 +# urllib3==2.1.0 diff --git a/misc/requirements/requirements-pyqt-6.6.txt b/misc/requirements/requirements-pyqt-6.6.txt new file mode 100644 index 000000000..0a9c72e25 --- /dev/null +++ b/misc/requirements/requirements-pyqt-6.6.txt @@ -0,0 +1,7 @@ +# This file is automatically generated by scripts/dev/recompile_requirements.py + +PyQt6==6.6.0 +PyQt6-Qt6==6.6.0 +PyQt6-sip==13.6.0 +PyQt6-WebEngine==6.6.0 +PyQt6-WebEngine-Qt6==6.6.0 diff --git a/misc/requirements/requirements-pyqt-6.6.txt-raw b/misc/requirements/requirements-pyqt-6.6.txt-raw new file mode 100644 index 000000000..7cfe6d34c --- /dev/null +++ b/misc/requirements/requirements-pyqt-6.6.txt-raw @@ -0,0 +1,4 @@ +PyQt6 >= 6.6, < 6.7 +PyQt6-Qt6 >= 6.6, < 6.7 +PyQt6-WebEngine >= 6.6, < 6.7 +PyQt6-WebEngine-Qt6 >= 6.6, < 6.7 diff --git a/misc/requirements/requirements-pyqt-6.txt b/misc/requirements/requirements-pyqt-6.txt index 5dca9ab74..0a9c72e25 100644 --- a/misc/requirements/requirements-pyqt-6.txt +++ b/misc/requirements/requirements-pyqt-6.txt @@ -1,7 +1,7 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -PyQt6==6.5.3 -PyQt6-Qt6==6.5.3 +PyQt6==6.6.0 +PyQt6-Qt6==6.6.0 PyQt6-sip==13.6.0 -PyQt6-WebEngine==6.5.0 -PyQt6-WebEngine-Qt6==6.5.3 +PyQt6-WebEngine==6.6.0 +PyQt6-WebEngine-Qt6==6.6.0 diff --git a/misc/requirements/requirements-pyqt.txt b/misc/requirements/requirements-pyqt.txt index 5dca9ab74..0a9c72e25 100644 --- a/misc/requirements/requirements-pyqt.txt +++ b/misc/requirements/requirements-pyqt.txt @@ -1,7 +1,7 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -PyQt6==6.5.3 -PyQt6-Qt6==6.5.3 +PyQt6==6.6.0 +PyQt6-Qt6==6.6.0 PyQt6-sip==13.6.0 -PyQt6-WebEngine==6.5.0 -PyQt6-WebEngine-Qt6==6.5.3 +PyQt6-WebEngine==6.6.0 +PyQt6-WebEngine-Qt6==6.6.0 diff --git a/misc/requirements/requirements-pyroma.txt b/misc/requirements/requirements-pyroma.txt index c171b400e..2ed286912 100644 --- a/misc/requirements/requirements-pyroma.txt +++ b/misc/requirements/requirements-pyroma.txt @@ -1,17 +1,17 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py build==1.0.3 -certifi==2023.7.22 -charset-normalizer==3.3.1 +certifi==2023.11.17 +charset-normalizer==3.3.2 docutils==0.20.1 -idna==3.4 +idna==3.6 importlib-metadata==6.8.0 packaging==23.2 -Pygments==2.16.1 +Pygments==2.17.2 pyproject_hooks==1.0.0 pyroma==4.2 requests==2.31.0 tomli==2.0.1 -trove-classifiers==2023.10.18 -urllib3==2.0.7 +trove-classifiers==2023.11.22 +urllib3==2.1.0 zipp==3.17.0 diff --git a/misc/requirements/requirements-sphinx.txt b/misc/requirements/requirements-sphinx.txt index 77d982519..69856e27c 100644 --- a/misc/requirements/requirements-sphinx.txt +++ b/misc/requirements/requirements-sphinx.txt @@ -2,16 +2,16 @@ alabaster==0.7.13 Babel==2.13.1 -certifi==2023.7.22 -charset-normalizer==3.3.1 +certifi==2023.11.17 +charset-normalizer==3.3.2 docutils==0.20.1 -idna==3.4 +idna==3.6 imagesize==1.4.1 importlib-metadata==6.8.0 Jinja2==3.1.2 MarkupSafe==2.1.3 packaging==23.2 -Pygments==2.16.1 +Pygments==2.17.2 pytz==2023.3.post1 requests==2.31.0 snowballstemmer==2.2.0 @@ -22,5 +22,5 @@ sphinxcontrib-htmlhelp==2.0.1 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 -urllib3==2.0.7 +urllib3==2.1.0 zipp==3.17.0 diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 49028afa4..ec91d0003 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -2,34 +2,34 @@ attrs==23.1.0 beautifulsoup4==4.12.2 -blinker==1.6.3 -certifi==2023.7.22 -charset-normalizer==3.3.1 +blinker==1.7.0 +certifi==2023.11.17 +charset-normalizer==3.3.2 cheroot==10.0.0 click==8.1.7 coverage==7.3.2 -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 execnet==2.0.2 -filelock==3.13.0 +filelock==3.13.1 Flask==3.0.0 hunter==3.6.1 -hypothesis==6.88.1 -idna==3.4 +hypothesis==6.90.0 +idna==3.6 importlib-metadata==6.8.0 iniconfig==2.0.0 itsdangerous==2.1.2 -jaraco.functools==3.9.0 +jaraco.functools==4.0.0 # Jinja2==3.1.2 -Mako==1.2.4 +Mako==1.3.0 manhole==1.8.0 # MarkupSafe==2.1.3 more-itertools==10.1.0 packaging==23.2 -parse==1.19.1 +parse==1.20.0 parse-type==0.6.2 pluggy==1.3.0 py-cpuinfo==9.0.0 -Pygments==2.16.1 +Pygments==2.17.2 pytest==7.4.3 pytest-bdd==7.0.0 pytest-benchmark==4.0.0 @@ -38,8 +38,8 @@ pytest-instafail==0.5.0 pytest-mock==3.12.0 pytest-qt==4.2.0 pytest-repeat==0.9.3 -pytest-rerunfailures==12.0 -pytest-xdist==3.3.1 +pytest-rerunfailures==13.0 +pytest-xdist==3.5.0 pytest-xvfb==3.0.0 PyVirtualDisplay==3.0 requests==2.31.0 @@ -47,11 +47,11 @@ requests-file==1.5.1 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.5 -tldextract==5.0.1 +tldextract==5.1.1 toml==0.10.2 tomli==2.0.1 typing_extensions==4.8.0 -urllib3==2.0.7 +urllib3==2.1.0 vulture==2.10 Werkzeug==3.0.1 zipp==3.17.0 diff --git a/misc/requirements/requirements-tox.txt b/misc/requirements/requirements-tox.txt index b9a2c0508..e72d539ea 100644 --- a/misc/requirements/requirements-tox.txt +++ b/misc/requirements/requirements-tox.txt @@ -4,14 +4,14 @@ cachetools==5.3.2 chardet==5.2.0 colorama==0.4.6 distlib==0.3.7 -filelock==3.13.0 +filelock==3.13.1 packaging==23.2 pip==23.3.1 -platformdirs==3.11.0 +platformdirs==4.0.0 pluggy==1.3.0 pyproject-api==1.6.1 -setuptools==68.2.2 +setuptools==69.0.2 tomli==2.0.1 tox==4.11.3 -virtualenv==20.24.6 -wheel==0.41.2 +virtualenv==20.24.7 +wheel==0.42.0 diff --git a/misc/requirements/requirements-yamllint.txt b/misc/requirements/requirements-yamllint.txt index fd9ea256f..32589d26f 100644 --- a/misc/requirements/requirements-yamllint.txt +++ b/misc/requirements/requirements-yamllint.txt @@ -2,4 +2,4 @@ pathspec==0.11.2 PyYAML==6.0.1 -yamllint==1.32.0 +yamllint==1.33.0 diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py index 1312275dc..4d14c9cd7 100644 --- a/qutebrowser/browser/browsertab.py +++ b/qutebrowser/browser/browsertab.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Base class for a wrapper over QWebView/QWebEngineView.""" +"""Base class for a wrapper over WebView/WebEngineView.""" import enum import pathlib @@ -22,10 +22,9 @@ from qutebrowser.qt.network import QNetworkAccessManager if TYPE_CHECKING: from qutebrowser.qt.webkit import QWebHistory, QWebHistoryItem - from qutebrowser.qt.webkitwidgets import QWebPage, QWebView + from qutebrowser.qt.webkitwidgets import QWebPage from qutebrowser.qt.webenginecore import ( QWebEngineHistory, QWebEngineHistoryItem, QWebEnginePage) - from qutebrowser.qt.webenginewidgets import QWebEngineView from qutebrowser.keyinput import modeman from qutebrowser.config import config, websettings @@ -38,10 +37,12 @@ from qutebrowser.qt import sip if TYPE_CHECKING: from qutebrowser.browser import webelem from qutebrowser.browser.inspector import AbstractWebInspector + from qutebrowser.browser.webengine.webview import WebEngineView + from qutebrowser.browser.webkit.webview import WebView tab_id_gen = itertools.count(0) -_WidgetType = Union["QWebView", "QWebEngineView"] +_WidgetType = Union["WebView", "WebEngineView"] def create(win_id: int, @@ -964,7 +965,7 @@ class AbstractTabPrivate: class AbstractTab(QWidget): - """An adapter for QWebView/QWebEngineView representing a single tab.""" + """An adapter for WebView/WebEngineView representing a single tab.""" #: Signal emitted when a website requests to close this tab. window_close_requested = pyqtSignal() @@ -1058,7 +1059,7 @@ class AbstractTab(QWidget): self.before_load_started.connect(self._on_before_load_started) - def _set_widget(self, widget: Union["QWebView", "QWebEngineView"]) -> None: + def _set_widget(self, widget: _WidgetType) -> None: # pylint: disable=protected-access self._widget = widget # FIXME:v4 ignore needed for QtWebKit diff --git a/qutebrowser/browser/webengine/webengineinspector.py b/qutebrowser/browser/webengine/webengineinspector.py index 64ef24319..d37f41ba5 100644 --- a/qutebrowser/browser/webengine/webengineinspector.py +++ b/qutebrowser/browser/webengine/webengineinspector.py @@ -35,14 +35,19 @@ class WebEngineInspectorView(QWebEngineView): See WebEngineView.createWindow for details. """ - inspected_page = self.page().inspectedPage() + our_page = self.page() + assert our_page is not None + inspected_page = our_page.inspectedPage() + assert inspected_page is not None if machinery.IS_QT5: view = inspected_page.view() assert isinstance(view, QWebEngineView), view return view.createWindow(wintype) else: # Qt 6 newpage = inspected_page.createWindow(wintype) - return webview.WebEngineView.forPage(newpage) + ret = webview.WebEngineView.forPage(newpage) + assert ret is not None + return ret class WebEngineInspector(inspector.AbstractWebInspector): @@ -88,16 +93,17 @@ class WebEngineInspector(inspector.AbstractWebInspector): def inspect(self, page: QWebEnginePage) -> None: if not self._widget: view = WebEngineInspectorView() - inspector_page = QWebEnginePage( + new_page = QWebEnginePage( page.profile(), self ) - inspector_page.windowCloseRequested.connect(self._on_window_close_requested) - view.setPage(inspector_page) + new_page.windowCloseRequested.connect(self._on_window_close_requested) + view.setPage(new_page) self._settings = webenginesettings.WebEngineSettings(view.settings()) self._set_widget(view) inspector_page = self._widget.page() + assert inspector_page is not None assert inspector_page.profile() == page.profile() inspector_page.setInspectedPage(page) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index d0b6b5beb..1275edf0b 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -24,6 +24,7 @@ from qutebrowser.browser.webengine import (spell, webenginequtescheme, cookies, webenginedownloads, notification) from qutebrowser.config import config, websettings from qutebrowser.config.websettings import AttributeInfo as Attr +from qutebrowser.misc import pakjoy from qutebrowser.utils import (standarddir, qtutils, message, log, urlmatch, usertypes, objreg, version) if TYPE_CHECKING: @@ -50,8 +51,12 @@ class _SettingsWrapper: For read operations, the default profile value is always used. """ + def _default_profile_settings(self): + assert default_profile is not None + return default_profile.settings() + def _settings(self): - yield default_profile.settings() + yield self._default_profile_settings() if private_profile: yield private_profile.settings() @@ -76,19 +81,19 @@ class _SettingsWrapper: settings.setUnknownUrlSchemePolicy(policy) def testAttribute(self, attribute): - return default_profile.settings().testAttribute(attribute) + return self._default_profile_settings().testAttribute(attribute) def fontSize(self, fonttype): - return default_profile.settings().fontSize(fonttype) + return self._default_profile_settings().fontSize(fonttype) def fontFamily(self, which): - return default_profile.settings().fontFamily(which) + return self._default_profile_settings().fontFamily(which) def defaultTextEncoding(self): - return default_profile.settings().defaultTextEncoding() + return self._default_profile_settings().defaultTextEncoding() def unknownUrlSchemePolicy(self): - return default_profile.settings().unknownUrlSchemePolicy() + return self._default_profile_settings().unknownUrlSchemePolicy() class WebEngineSettings(websettings.AbstractSettings): @@ -341,7 +346,10 @@ def _init_user_agent_str(ua): def init_user_agent(): - _init_user_agent_str(QWebEngineProfile.defaultProfile().httpUserAgent()) + """Make the default WebEngine user agent available via parsed_user_agent.""" + actual_default_profile = QWebEngineProfile.defaultProfile() + assert actual_default_profile is not None + _init_user_agent_str(actual_default_profile.httpUserAgent()) def _init_profile(profile: QWebEngineProfile) -> None: @@ -546,7 +554,11 @@ def init(): _global_settings = WebEngineSettings(_SettingsWrapper()) log.init.debug("Initializing profiles...") - _init_default_profile() + + # Apply potential resource patches while initializing profiles. + with pakjoy.patch_webengine(): + _init_default_profile() + init_private_profile() config.instance.changed.connect(_update_settings) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 9f1d04b63..1c712db5e 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Wrapper over a QWebEngineView.""" +"""Wrapper over a WebEngineView.""" import math import struct @@ -15,7 +15,6 @@ from typing import cast, Union, Optional from qutebrowser.qt.core import (pyqtSignal, pyqtSlot, Qt, QPoint, QPointF, QTimer, QUrl, QObject, QByteArray) from qutebrowser.qt.network import QAuthenticator -from qutebrowser.qt.webenginewidgets import QWebEngineView from qutebrowser.qt.webenginecore import QWebEnginePage, QWebEngineScript, QWebEngineHistory from qutebrowser.config import config @@ -1267,7 +1266,7 @@ class WebEngineTab(browsertab.AbstractTab): abort_questions = pyqtSignal() - _widget: QWebEngineView + _widget: webview.WebEngineView search: WebEngineSearch audio: WebEngineAudio printing: WebEnginePrinting diff --git a/qutebrowser/browser/webengine/webview.py b/qutebrowser/browser/webengine/webview.py index 3c63c59e4..a6f2ae113 100644 --- a/qutebrowser/browser/webengine/webview.py +++ b/qutebrowser/browser/webengine/webview.py @@ -5,13 +5,16 @@ """The main browser widget for QtWebEngine.""" import mimetypes -from typing import List, Iterable +from typing import List, Iterable, Optional from qutebrowser.qt import machinery from qutebrowser.qt.core import pyqtSignal, pyqtSlot, QUrl from qutebrowser.qt.gui import QPalette from qutebrowser.qt.webenginewidgets import QWebEngineView -from qutebrowser.qt.webenginecore import QWebEnginePage, QWebEngineCertificateError +from qutebrowser.qt.webenginecore import ( + QWebEnginePage, QWebEngineCertificateError, QWebEngineSettings, + QWebEngineHistory, +) from qutebrowser.browser import shared from qutebrowser.browser.webengine import webenginesettings, certificateerror @@ -129,6 +132,25 @@ class WebEngineView(QWebEngineView): return super().contextMenuEvent(ev) + def page(self) -> "WebEnginePage": + """Return the page for this view.""" + maybe_page = super().page() + assert maybe_page is not None + assert isinstance(maybe_page, WebEnginePage) + return maybe_page + + def settings(self) -> "QWebEngineSettings": + """Return the settings for this view.""" + maybe_settings = super().settings() + assert maybe_settings is not None + return maybe_settings + + def history(self) -> "QWebEngineHistory": + """Return the history for this view.""" + maybe_history = super().history() + assert maybe_history is not None + return maybe_history + def extra_suffixes_workaround(upstream_mimetypes): """Return any extra suffixes for mimetypes in upstream_mimetypes. @@ -294,22 +316,28 @@ class WebEnginePage(QWebEnginePage): def chooseFiles( self, mode: QWebEnginePage.FileSelectionMode, - old_files: Iterable[str], - accepted_mimetypes: Iterable[str], + old_files: Iterable[Optional[str]], + accepted_mimetypes: Iterable[Optional[str]], ) -> List[str]: """Override chooseFiles to (optionally) invoke custom file uploader.""" - extra_suffixes = extra_suffixes_workaround(accepted_mimetypes) + accepted_mimetypes_filtered = [m for m in accepted_mimetypes if m is not None] + old_files_filtered = [f for f in old_files if f is not None] + extra_suffixes = extra_suffixes_workaround(accepted_mimetypes_filtered) if extra_suffixes: log.webview.debug( "adding extra suffixes to filepicker: " - f"before={accepted_mimetypes} " + f"before={accepted_mimetypes_filtered} " f"added={extra_suffixes}", ) - accepted_mimetypes = list(accepted_mimetypes) + list(extra_suffixes) + accepted_mimetypes_filtered = list( + accepted_mimetypes_filtered + ) + list(extra_suffixes) handler = config.val.fileselect.handler if handler == "default": - return super().chooseFiles(mode, old_files, accepted_mimetypes) + return super().chooseFiles( + mode, old_files_filtered, accepted_mimetypes_filtered, + ) assert handler == "external", handler try: qb_mode = _QB_FILESELECTION_MODES[mode] @@ -317,6 +345,8 @@ class WebEnginePage(QWebEnginePage): log.webview.warning( f"Got file selection mode {mode}, but we don't support that!" ) - return super().chooseFiles(mode, old_files, accepted_mimetypes) + return super().chooseFiles( + mode, old_files_filtered, accepted_mimetypes_filtered, + ) return shared.choose_file(qb_mode=qb_mode) diff --git a/qutebrowser/config/qtargs.py b/qutebrowser/config/qtargs.py index 934953d0a..4fa6aa43f 100644 --- a/qutebrowser/config/qtargs.py +++ b/qutebrowser/config/qtargs.py @@ -336,10 +336,8 @@ _WEBENGINE_SETTINGS: Dict[str, Dict[Any, Optional[_SettingValueType]]] = { 'qt.workarounds.disable_accelerated_2d_canvas': { 'always': '--disable-accelerated-2d-canvas', 'never': None, - 'auto': lambda versions: 'always' + 'auto': lambda _versions: 'always' if machinery.IS_QT6 - and versions.chromium_major - and versions.chromium_major < 111 else 'never', }, } diff --git a/qutebrowser/misc/binparsing.py b/qutebrowser/misc/binparsing.py new file mode 100644 index 000000000..81e2e6dbb --- /dev/null +++ b/qutebrowser/misc/binparsing.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Florian Bruhin (The-Compiler) <mail@qutebrowser.org> +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Utilities for parsing binary files. + +Used by elf.py as well as pakjoy.py. +""" + +import struct +from typing import Any, IO, Tuple + + +class ParseError(Exception): + + """Raised when the file can't be parsed.""" + + +def unpack(fmt: str, fobj: IO[bytes]) -> Tuple[Any, ...]: + """Unpack the given struct format from the given file.""" + size = struct.calcsize(fmt) + data = safe_read(fobj, size) + + try: + return struct.unpack(fmt, data) + except struct.error as e: + raise ParseError(e) + + +def safe_read(fobj: IO[bytes], size: int) -> bytes: + """Read from a file, handling possible exceptions.""" + try: + return fobj.read(size) + except (OSError, OverflowError) as e: + raise ParseError(e) + + +def safe_seek(fobj: IO[bytes], pos: int) -> None: + """Seek in a file, handling possible exceptions.""" + try: + fobj.seek(pos) + except (OSError, OverflowError) as e: + raise ParseError(e) diff --git a/qutebrowser/misc/elf.py b/qutebrowser/misc/elf.py index aa717e790..35af5af28 100644 --- a/qutebrowser/misc/elf.py +++ b/qutebrowser/misc/elf.py @@ -44,21 +44,16 @@ This is a "best effort" parser. If it errors out, we instead end up relying on t PyQtWebEngine version, which is the next best thing. """ -import struct import enum import re import dataclasses import mmap import pathlib -from typing import Any, IO, ClassVar, Dict, Optional, Tuple, cast +from typing import IO, ClassVar, Dict, Optional, cast from qutebrowser.qt import machinery from qutebrowser.utils import log, version, qtutils - - -class ParseError(Exception): - - """Raised when the ELF file can't be parsed.""" +from qutebrowser.misc import binparsing class Bitness(enum.Enum): @@ -77,33 +72,6 @@ class Endianness(enum.Enum): big = 2 -def _unpack(fmt: str, fobj: IO[bytes]) -> Tuple[Any, ...]: - """Unpack the given struct format from the given file.""" - size = struct.calcsize(fmt) - data = _safe_read(fobj, size) - - try: - return struct.unpack(fmt, data) - except struct.error as e: - raise ParseError(e) - - -def _safe_read(fobj: IO[bytes], size: int) -> bytes: - """Read from a file, handling possible exceptions.""" - try: - return fobj.read(size) - except (OSError, OverflowError) as e: - raise ParseError(e) - - -def _safe_seek(fobj: IO[bytes], pos: int) -> None: - """Seek in a file, handling possible exceptions.""" - try: - fobj.seek(pos) - except (OSError, OverflowError) as e: - raise ParseError(e) - - @dataclasses.dataclass class Ident: @@ -125,17 +93,17 @@ class Ident: @classmethod def parse(cls, fobj: IO[bytes]) -> 'Ident': """Parse an ELF ident header from a file.""" - magic, klass, data, elfversion, osabi, abiversion = _unpack(cls._FORMAT, fobj) + magic, klass, data, elfversion, osabi, abiversion = binparsing.unpack(cls._FORMAT, fobj) try: bitness = Bitness(klass) except ValueError: - raise ParseError(f"Invalid bitness {klass}") + raise binparsing.ParseError(f"Invalid bitness {klass}") try: endianness = Endianness(data) except ValueError: - raise ParseError(f"Invalid endianness {data}") + raise binparsing.ParseError(f"Invalid endianness {data}") return cls(magic, bitness, endianness, elfversion, osabi, abiversion) @@ -172,7 +140,7 @@ class Header: def parse(cls, fobj: IO[bytes], bitness: Bitness) -> 'Header': """Parse an ELF header from a file.""" fmt = cls._FORMATS[bitness] - return cls(*_unpack(fmt, fobj)) + return cls(*binparsing.unpack(fmt, fobj)) @dataclasses.dataclass @@ -203,39 +171,39 @@ class SectionHeader: def parse(cls, fobj: IO[bytes], bitness: Bitness) -> 'SectionHeader': """Parse an ELF section header from a file.""" fmt = cls._FORMATS[bitness] - return cls(*_unpack(fmt, fobj)) + return cls(*binparsing.unpack(fmt, fobj)) def get_rodata_header(f: IO[bytes]) -> SectionHeader: """Parse an ELF file and find the .rodata section header.""" ident = Ident.parse(f) if ident.magic != b'\x7fELF': - raise ParseError(f"Invalid magic {ident.magic!r}") + raise binparsing.ParseError(f"Invalid magic {ident.magic!r}") if ident.data != Endianness.little: - raise ParseError("Big endian is unsupported") + raise binparsing.ParseError("Big endian is unsupported") if ident.version != 1: - raise ParseError(f"Only version 1 is supported, not {ident.version}") + raise binparsing.ParseError(f"Only version 1 is supported, not {ident.version}") header = Header.parse(f, bitness=ident.klass) # Read string table - _safe_seek(f, header.shoff + header.shstrndx * header.shentsize) + binparsing.safe_seek(f, header.shoff + header.shstrndx * header.shentsize) shstr = SectionHeader.parse(f, bitness=ident.klass) - _safe_seek(f, shstr.offset) - string_table = _safe_read(f, shstr.size) + binparsing.safe_seek(f, shstr.offset) + string_table = binparsing.safe_read(f, shstr.size) # Back to all sections for i in range(header.shnum): - _safe_seek(f, header.shoff + i * header.shentsize) + binparsing.safe_seek(f, header.shoff + i * header.shentsize) sh = SectionHeader.parse(f, bitness=ident.klass) name = string_table[sh.name:].split(b'\x00')[0] if name == b'.rodata': return sh - raise ParseError("No .rodata section found") + raise binparsing.ParseError("No .rodata section found") @dataclasses.dataclass @@ -262,7 +230,7 @@ def _find_versions(data: bytes) -> Versions: chromium=match.group(2).decode('ascii'), ) except UnicodeDecodeError as e: - raise ParseError(e) + raise binparsing.ParseError(e) # Here it gets even more crazy: Sometimes, we don't have the full UA in one piece # in the string table somehow (?!). However, Qt 6.2 added a separate @@ -273,20 +241,20 @@ def _find_versions(data: bytes) -> Versions: # We first get the partial Chromium version from the UA: match = re.search(pattern[:-4], data) # without trailing literal \x00 if match is None: - raise ParseError("No match in .rodata") + raise binparsing.ParseError("No match in .rodata") webengine_bytes = match.group(1) partial_chromium_bytes = match.group(2) if b"." not in partial_chromium_bytes or len(partial_chromium_bytes) < 6: # some sanity checking - raise ParseError("Inconclusive partial Chromium bytes") + raise binparsing.ParseError("Inconclusive partial Chromium bytes") # And then try to find the *full* string, stored separately, based on the # partial one we got above. pattern = br"\x00(" + re.escape(partial_chromium_bytes) + br"[0-9.]+)\x00" match = re.search(pattern, data) if match is None: - raise ParseError("No match in .rodata for full version") + raise binparsing.ParseError("No match in .rodata for full version") chromium_bytes = match.group(1) try: @@ -295,7 +263,7 @@ def _find_versions(data: bytes) -> Versions: chromium=chromium_bytes.decode('ascii'), ) except UnicodeDecodeError as e: - raise ParseError(e) + raise binparsing.ParseError(e) def _parse_from_file(f: IO[bytes]) -> Versions: @@ -316,8 +284,8 @@ def _parse_from_file(f: IO[bytes]) -> Versions: return _find_versions(cast(bytes, mmap_data)) except (OSError, OverflowError) as e: log.misc.debug(f"mmap failed ({e}), falling back to reading", exc_info=True) - _safe_seek(f, sh.offset) - data = _safe_read(f, sh.size) + binparsing.safe_seek(f, sh.offset) + data = binparsing.safe_read(f, sh.size) return _find_versions(data) @@ -344,6 +312,6 @@ def parse_webenginecore() -> Optional[Versions]: log.misc.debug(f"Got versions from ELF: {versions}") return versions - except ParseError as e: + except binparsing.ParseError as e: log.misc.debug(f"Failed to parse ELF: {e}", exc_info=True) return None diff --git a/qutebrowser/misc/pakjoy.py b/qutebrowser/misc/pakjoy.py new file mode 100644 index 000000000..a511034a2 --- /dev/null +++ b/qutebrowser/misc/pakjoy.py @@ -0,0 +1,260 @@ +# SPDX-FileCopyrightText: Florian Bruhin (The-Compiler) <mail@qutebrowser.org> +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Chromium .pak repacking. + +This entire file is a great WORKAROUND for https://bugreports.qt.io/browse/QTBUG-118157 +and the fact we can't just simply disable the hangouts extension: +https://bugreports.qt.io/browse/QTBUG-118452 + +It's yet another big hack. If you think this is bad, look at elf.py instead. + +The name of this file might or might not be inspired by a certain vegetable, +as well as the "joy" this bug has caused me. + +Useful references: + +- https://sweetscape.com/010editor/repository/files/PAK.bt (010 editor <3) +- https://textslashplain.com/2022/05/03/chromium-internals-pak-files/ +- https://github.com/myfreeer/chrome-pak-customizer +- https://source.chromium.org/chromium/chromium/src/+/main:tools/grit/pak_util.py +- https://source.chromium.org/chromium/chromium/src/+/main:tools/grit/grit/format/data_pack.py + +This is a "best effort" parser. If it errors out, we don't apply the workaround +instead of crashing. +""" + +import os +import shutil +import pathlib +import dataclasses +import contextlib +from typing import ClassVar, IO, Optional, Dict, Tuple, Iterator + +from qutebrowser.misc import binparsing, objects +from qutebrowser.utils import qtutils, standarddir, version, utils, log + +HANGOUTS_MARKER = b"// Extension ID: nkeimhogjdpnpccoofpliimaahmaaome" +HANGOUTS_ID = 36197 # as found by toofar +PAK_VERSION = 5 +RESOURCES_ENV_VAR = "QTWEBENGINE_RESOURCES_PATH" +DISABLE_ENV_VAR = "QUTE_DISABLE_PAKJOY" +CACHE_DIR_NAME = "webengine_resources_pak_quirk" +PAK_FILENAME = "qtwebengine_resources.pak" + +TARGET_URL = b"https://*.google.com/*" +REPLACEMENT_URL = b"https://qute.invalid/*" +assert len(TARGET_URL) == len(REPLACEMENT_URL) + + +@dataclasses.dataclass +class PakHeader: + + """Chromium .pak header (version 5).""" + + encoding: int # uint32 + resource_count: int # uint16 + _alias_count: int # uint16 + + _FORMAT: ClassVar[str] = "<IHH" + + @classmethod + def parse(cls, fobj: IO[bytes]) -> "PakHeader": + """Parse a PAK version 5 header from a file.""" + return cls(*binparsing.unpack(cls._FORMAT, fobj)) + + +@dataclasses.dataclass +class PakEntry: + + """Entry description in a .pak file.""" + + resource_id: int # uint16 + file_offset: int # uint32 + size: int = 0 # not in file + + _FORMAT: ClassVar[str] = "<HI" + + @classmethod + def parse(cls, fobj: IO[bytes]) -> "PakEntry": + """Parse a PAK entry from a file.""" + return cls(*binparsing.unpack(cls._FORMAT, fobj)) + + +class PakParser: + """Parse webengine pak and find patch location to disable Google Meet extension.""" + + def __init__(self, fobj: IO[bytes]) -> None: + """Parse the .pak file from the given file object.""" + pak_version = binparsing.unpack("<I", fobj)[0] + if pak_version != PAK_VERSION: + raise binparsing.ParseError(f"Unsupported .pak version {pak_version}") + + self.fobj = fobj + entries = self._read_header() + self.manifest_entry, self.manifest = self._find_manifest(entries) + + def find_patch_offset(self) -> int: + """Return byte offset of TARGET_URL into the pak file.""" + try: + return self.manifest_entry.file_offset + self.manifest.index(TARGET_URL) + except ValueError: + raise binparsing.ParseError("Couldn't find URL in manifest") + + def _maybe_get_hangouts_manifest(self, entry: PakEntry) -> Optional[bytes]: + self.fobj.seek(entry.file_offset) + data = self.fobj.read(entry.size) + + if not data.startswith(b"{") or not data.rstrip(b"\n").endswith(b"}"): + # not JSON + return None + + if HANGOUTS_MARKER not in data: + return None + + return data + + def _read_header(self) -> Dict[int, PakEntry]: + """Read the header and entry index from the .pak file.""" + entries = [] + + header = PakHeader.parse(self.fobj) + for _ in range(header.resource_count + 1): # + 1 due to sentinel at end + entries.append(PakEntry.parse(self.fobj)) + + for entry, next_entry in zip(entries, entries[1:]): + if entry.resource_id == 0: + raise binparsing.ParseError("Unexpected sentinel entry") + entry.size = next_entry.file_offset - entry.file_offset + + if entries[-1].resource_id != 0: + raise binparsing.ParseError("Missing sentinel entry") + del entries[-1] + + return {entry.resource_id: entry for entry in entries} + + def _find_manifest(self, entries: Dict[int, PakEntry]) -> Tuple[PakEntry, bytes]: + to_check = list(entries.values()) + if HANGOUTS_ID in entries: + # Most likely candidate, based on previous known ID + to_check.insert(0, entries[HANGOUTS_ID]) + + for entry in to_check: + manifest = self._maybe_get_hangouts_manifest(entry) + if manifest is not None: + return entry, manifest + + raise binparsing.ParseError("Couldn't find hangouts manifest") + + +def _find_webengine_resources() -> pathlib.Path: + """Find the QtWebEngine resources dir. + + Mirrors logic from QtWebEngine: + https://github.com/qt/qtwebengine/blob/v6.6.0/src/core/web_engine_library_info.cpp#L293-L341 + """ + if RESOURCES_ENV_VAR in os.environ: + return pathlib.Path(os.environ[RESOURCES_ENV_VAR]) + + candidates = [] + qt_data_path = qtutils.library_path(qtutils.LibraryPath.data) + if utils.is_mac: # pragma: no cover + # I'm not sure how to arrive at this path without hardcoding it + # ourselves. importlib_resources("PyQt6.Qt6") can serve as a + # replacement for the qtutils bit but it doesn't seem to help find the + # actuall Resources folder. + candidates.append( + qt_data_path / "lib" / "QtWebEngineCore.framework" / "Resources" + ) + + candidates += [ + qt_data_path / "resources", + qt_data_path, + pathlib.Path(objects.qapp.applicationDirPath()), + pathlib.Path.home() / f".{objects.qapp.applicationName()}", + ] + + for candidate in candidates: + if (candidate / PAK_FILENAME).exists(): + return candidate + + raise binparsing.ParseError("Couldn't find webengine resources dir") + + +def copy_webengine_resources() -> Optional[pathlib.Path]: + """Copy qtwebengine resources to local dir for patching.""" + resources_dir = _find_webengine_resources() + work_dir = pathlib.Path(standarddir.cache()) / CACHE_DIR_NAME + + if work_dir.exists(): + log.misc.debug(f"Removing existing {work_dir}") + shutil.rmtree(work_dir) + + versions = version.qtwebengine_versions(avoid_init=True) + if versions.webengine != utils.VersionNumber(6, 6): + # No patching needed + return None + + log.misc.debug( + "Copying webengine resources for quirk patching: " + f"{resources_dir} -> {work_dir}" + ) + + shutil.copytree(resources_dir, work_dir) + return work_dir + + +def _patch(file_to_patch: pathlib.Path) -> None: + """Apply any patches to the given pak file.""" + if not file_to_patch.exists(): + log.misc.error( + "Resource pak doesn't exist at expected location! " + f"Not applying quirks. Expected location: {file_to_patch}" + ) + return + + with open(file_to_patch, "r+b") as f: + try: + parser = PakParser(f) + log.misc.debug(f"Patching pak entry: {parser.manifest_entry}") + offset = parser.find_patch_offset() + binparsing.safe_seek(f, offset) + f.write(REPLACEMENT_URL) + except binparsing.ParseError: + log.misc.exception("Failed to apply quirk to resources pak.") + + +@contextlib.contextmanager +def patch_webengine() -> Iterator[None]: + """Apply any patches to webengine resource pak files.""" + if os.environ.get(DISABLE_ENV_VAR): + log.misc.debug(f"Not applying quirk due to {DISABLE_ENV_VAR}") + yield + return + + try: + # Still calling this on Qt != 6.6 so that the directory is cleaned up + # when not needed anymore. + webengine_resources_path = copy_webengine_resources() + except OSError: + log.misc.exception("Failed to copy webengine resources, not applying quirk") + yield + return + + if webengine_resources_path is None: + yield + return + + _patch(webengine_resources_path / PAK_FILENAME) + + old_value = os.environ.get(RESOURCES_ENV_VAR) + os.environ[RESOURCES_ENV_VAR] = str(webengine_resources_path) + + yield + + # Restore old value for subprocesses or :restart + if old_value is None: + del os.environ[RESOURCES_ENV_VAR] + else: + os.environ[RESOURCES_ENV_VAR] = old_value diff --git a/qutebrowser/utils/qtutils.py b/qutebrowser/utils/qtutils.py index 2c103c6b8..363d5607a 100644 --- a/qutebrowser/utils/qtutils.py +++ b/qutebrowser/utils/qtutils.py @@ -18,8 +18,8 @@ import enum import pathlib import operator import contextlib -from typing import (Any, AnyStr, TYPE_CHECKING, BinaryIO, IO, Iterator, - Optional, Union, Tuple, Protocol, cast, TypeVar) +from typing import (Any, TYPE_CHECKING, BinaryIO, IO, Iterator, Literal, + Optional, Union, Tuple, Protocol, cast, overload, TypeVar) from qutebrowser.qt import machinery, sip from qutebrowser.qt.core import (qVersion, QEventLoop, QDataStream, QByteArray, @@ -236,12 +236,32 @@ def deserialize_stream(stream: QDataStream, obj: _QtSerializableType) -> None: check_qdatastream(stream) +@overload +@contextlib.contextmanager +def savefile_open( + filename: str, + binary: Literal[False] = ..., + encoding: str = 'utf-8' +) -> Iterator[IO[str]]: + ... + + +@overload +@contextlib.contextmanager +def savefile_open( + filename: str, + binary: Literal[True] = ..., + encoding: str = 'utf-8' +) -> Iterator[IO[str]]: + ... + + @contextlib.contextmanager def savefile_open( filename: str, binary: bool = False, encoding: str = 'utf-8' -) -> Iterator[IO[AnyStr]]: +) -> Iterator[Union[IO[str], IO[bytes]]]: """Context manager to easily use a QSaveFile.""" f = QSaveFile(filename) cancelled = False @@ -253,7 +273,7 @@ def savefile_open( dev = cast(BinaryIO, PyQIODevice(f)) if binary: - new_f: IO[Any] = dev # FIXME:mypy Why doesn't AnyStr work? + new_f: Union[IO[str], IO[bytes]] = dev else: new_f = io.TextIOWrapper(dev, encoding=encoding) diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py index a20774617..59da5b5f0 100644 --- a/qutebrowser/utils/version.py +++ b/qutebrowser/utils/version.py @@ -686,7 +686,7 @@ class WebEngineVersions: return cls._CHROMIUM_VERSIONS.get(minor_version) @classmethod - def from_api(cls, qtwe_version: str, chromium_version: str) -> 'WebEngineVersions': + def from_api(cls, qtwe_version: str, chromium_version: Optional[str]) -> 'WebEngineVersions': """Get the versions based on the exact versions. This is called if we have proper APIs to get the versions easily @@ -796,8 +796,10 @@ def qtwebengine_versions(*, avoid_init: bool = False) -> WebEngineVersions: except ImportError: pass # Needs QtWebEngine 6.2+ with PyQtWebEngine 6.3.1+ else: + qtwe_version = qWebEngineVersion() + assert qtwe_version is not None return WebEngineVersions.from_api( - qtwe_version=qWebEngineVersion(), + qtwe_version=qtwe_version, chromium_version=qWebEngineChromiumVersion(), ) diff --git a/requirements.txt b/requirements.txt index dc07bf531..81e0d0606 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,10 +2,10 @@ adblock==0.6.0 colorama==0.4.6 -importlib-resources==6.1.0 ; python_version=="3.8.*" +importlib-resources==6.1.1 ; python_version=="3.8.*" Jinja2==3.1.2 MarkupSafe==2.1.3 -Pygments==2.16.1 +Pygments==2.17.2 PyYAML==6.0.1 zipp==3.17.0 # Unpinned due to recompile_requirements.py limitations diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index a3a9bf644..38a8f6ca1 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -123,6 +123,8 @@ PERFECT_FILES = [ 'qutebrowser/misc/objects.py'), ('tests/unit/misc/test_throttle.py', 'qutebrowser/misc/throttle.py'), + ('tests/unit/misc/test_pakjoy.py', + 'qutebrowser/misc/pakjoy.py'), (None, 'qutebrowser/mainwindow/statusbar/keystring.py'), @@ -328,10 +330,6 @@ def main_check(): print("or check https://codecov.io/github/qutebrowser/qutebrowser") print() - if scriptutils.ON_CI: - print("Keeping coverage.xml on CI.") - else: - os.remove('coverage.xml') return 1 if messages else 0 @@ -352,7 +350,6 @@ def main_check_all(): '--cov-report', 'xml', test_file], check=True) with open('coverage.xml', encoding='utf-8') as f: messages = check(f, [(test_file, src_file)]) - os.remove('coverage.xml') messages = [msg for msg in messages if msg.typ == MsgType.insufficient_coverage] diff --git a/scripts/dev/ci/problemmatchers.py b/scripts/dev/ci/problemmatchers.py index fa623dec7..3316c5597 100644 --- a/scripts/dev/ci/problemmatchers.py +++ b/scripts/dev/ci/problemmatchers.py @@ -160,13 +160,17 @@ MATCHERS = { "tests": [ { # pytest test summary output + # Examples (with ANSI color codes around FAILED|ERROR and the + # function name): + # FAILED tests/end2end/features/test_keyinput_bdd.py::test_fakekey_sending_special_key_to_the_website - end2end.fixtures.testprocess.WaitForTimeout: Timed out after 15000ms waiting for {'category': 'js', 'message': '[*] key press: 27'}. + # ERROR tests/end2end/test_insert_mode.py::test_insert_mode[100-textarea.html-qute-textarea-clipboard-qutebrowser] - Failed: Logged unexpected errors: "severity": "error", "pattern": [ { - "regexp": r'^=+ short test summary info =+$', + "regexp": r'^.*=== short test summary info ===.*$', }, { - "regexp": r"^((ERROR|FAILED) .*)", + "regexp": r"^[^ ]*((ERROR|FAILED)[^ ]* .*)$", "message": 1, "loop": True, } diff --git a/tests/conftest.py b/tests/conftest.py index 2fea48c43..9d7c5c29c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -197,7 +197,12 @@ def qapp_args(): """Make QtWebEngine unit tests run on older Qt versions + newer kernels.""" if testutils.disable_seccomp_bpf_sandbox(): return [sys.argv[0], testutils.DISABLE_SECCOMP_BPF_FLAG] - return [sys.argv[0]] + + # Disabling PaintHoldingCrossOrigin makes tests needing UI interaction with + # QtWebEngine more reliable. + # Only needed with QtWebEngine and Qt 6.5, but Qt just ignores arguments it + # doesn't know about anyways. + return [sys.argv[0], "--webEngineArgs", "--disable-features=PaintHoldingCrossOrigin"] @pytest.fixture(scope='session') diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index a2f870e32..de9b490ca 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -390,7 +390,8 @@ class QuteProc(testprocess.Process): '--json-logging', '--loglevel', 'vdebug', '--backend', backend, '--debug-flag', 'no-sql-history', '--debug-flag', 'werror', '--debug-flag', - 'test-notification-service'] + 'test-notification-service', + '--qt-flag', 'disable-features=PaintHoldingCrossOrigin'] if self.request.config.webengine and testutils.disable_seccomp_bpf_sandbox(): args += testutils.DISABLE_SECCOMP_BPF_ARGS diff --git a/tests/end2end/test_invocations.py b/tests/end2end/test_invocations.py index af81781f6..a55efb129 100644 --- a/tests/end2end/test_invocations.py +++ b/tests/end2end/test_invocations.py @@ -15,6 +15,7 @@ import re import json import platform from contextlib import nullcontext as does_not_raise +from unittest.mock import ANY import pytest from qutebrowser.qt.core import QProcess, QPoint @@ -882,30 +883,86 @@ def test_sandboxing( line.expected = True pytest.skip("chrome://sandbox/ not supported") + if len(text.split("\n")) == 1: + # Try again, maybe the JS hasn't run yet? + text = quteproc_new.get_content() + print(text) + bpf_text = "Seccomp-BPF sandbox" yama_text = "Ptrace Protection with Yama LSM" - header, *lines, empty, result = text.split("\n") - assert not empty - - expected_status = { - "Layer 1 Sandbox": "Namespace" if has_namespaces else "None", + if not utils.is_windows: + header, *lines, empty, result = text.split("\n") + assert not empty - "PID namespaces": "Yes" if has_namespaces else "No", - "Network namespaces": "Yes" if has_namespaces else "No", + expected_status = { + "Layer 1 Sandbox": "Namespace" if has_namespaces else "None", - bpf_text: "Yes" if has_seccomp else "No", - f"{bpf_text} supports TSYNC": "Yes" if has_seccomp else "No", + "PID namespaces": "Yes" if has_namespaces else "No", + "Network namespaces": "Yes" if has_namespaces else "No", - f"{yama_text} (Broker)": "Yes" if has_yama else "No", - f"{yama_text} (Non-broker)": "Yes" if has_yama_non_broker else "No", - } + bpf_text: "Yes" if has_seccomp else "No", + f"{bpf_text} supports TSYNC": "Yes" if has_seccomp else "No", - assert header == "Sandbox Status" - assert result == expected_result + f"{yama_text} (Broker)": "Yes" if has_yama else "No", + f"{yama_text} (Non-broker)": "Yes" if has_yama_non_broker else "No", + } - status = dict(line.split("\t") for line in lines) - assert status == expected_status + assert header == "Sandbox Status" + assert result == expected_result + + status = dict(line.split("\t") for line in lines) + assert status == expected_status + + else: # utils.is_windows + # The sandbox page on Windows if different that Linux and macOS. It's + # a lot more complex. There is a table up top with lots of columns and + # a row per tab and helper process then a json object per row down + # below with even more detail (which we ignore). + # https://www.chromium.org/Home/chromium-security/articles/chrome-sandbox-diagnostics-for-windows/ + + # We're not getting full coverage of the table and there doesn't seem + # to be a simple summary like for linux. The "Sandbox" and "Lockdown" + # column are probably the key ones. + # We are looking at all the rows in the table for the sake of + # completeness, but I expect there will always be just one row with a + # renderer process in it for this test. If other helper processes pop + # up we might want to exclude them. + lines = text.split("\n") + assert lines.pop(0) == "Sandbox Status" + header = lines.pop(0).split("\t") + rows = [] + current_line = lines.pop(0) + while current_line.strip(): + if lines[0].startswith("\t"): + # Continuation line. Not sure how to 100% identify them + # but new rows should start with a process ID. + current_line += lines.pop(0) + continue + + columns = current_line.split("\t") + assert len(header) == len(columns) + rows.append(dict(zip(header, columns))) + current_line = lines.pop(0) + + assert rows + + # I'm using has_namespaces as a proxy for "should be sandboxed" here, + # which is a bit lazy but its either that or match on the text + # "sandboxing" arg. The seccomp-bpf arg does nothing on windows, so + # we only have the off and on states. + for row in rows: + assert row == { + "Process": ANY, + "Type": "Renderer", + "Name": "", + "Sandbox": "Renderer" if has_namespaces else "Not Sandboxed", + "Lockdown": "Lockdown" if has_namespaces else "", + "Integrity": ANY if has_namespaces else "", + "Mitigations": ANY if has_namespaces else "", + "Component Filter": ANY if has_namespaces else "", + "Lowbox/AppContainer": "", + } @pytest.mark.not_frozen diff --git a/tests/unit/config/test_qtargs.py b/tests/unit/config/test_qtargs.py index 419faad12..2414d4ba9 100644 --- a/tests/unit/config/test_qtargs.py +++ b/tests/unit/config/test_qtargs.py @@ -156,14 +156,12 @@ class TestWebEngineArgs: assert '--enable-in-process-stack-traces' not in args @pytest.mark.parametrize( - 'qt_version, qt6, value, has_arg', + 'qt6, value, has_arg', [ - ('5.15.2', False, 'auto', False), - ('6.5.3', True, 'auto', True), - ('6.6.0', True, 'auto', False), - ('6.5.3', True, 'always', True), - ('6.5.3', True, 'never', False), - ('6.6.0', True, 'always', True), + (False, 'auto', False), + (True, 'auto', True), + (True, 'always', True), + (True, 'never', False), ], ) def test_accelerated_2d_canvas( @@ -172,12 +170,10 @@ class TestWebEngineArgs: version_patcher, config_stub, monkeypatch, - qt_version, qt6, value, has_arg, ): - version_patcher(qt_version) config_stub.val.qt.workarounds.disable_accelerated_2d_canvas = value monkeypatch.setattr(machinery, 'IS_QT6', qt6) diff --git a/tests/unit/misc/test_elf.py b/tests/unit/misc/test_elf.py index 88b984dcd..6ae23357c 100644 --- a/tests/unit/misc/test_elf.py +++ b/tests/unit/misc/test_elf.py @@ -9,7 +9,7 @@ import pytest import hypothesis from hypothesis import strategies as hst -from qutebrowser.misc import elf +from qutebrowser.misc import elf, binparsing from qutebrowser.utils import utils @@ -117,7 +117,7 @@ def test_find_versions(data, expected): ), ]) def test_find_versions_invalid(data, message): - with pytest.raises(elf.ParseError) as excinfo: + with pytest.raises(binparsing.ParseError) as excinfo: elf._find_versions(data) assert str(excinfo.value) == message @@ -132,5 +132,5 @@ def test_hypothesis(data): fobj = io.BytesIO(data) try: elf._parse_from_file(fobj) - except elf.ParseError as e: + except binparsing.ParseError as e: print(e) diff --git a/tests/unit/misc/test_pakjoy.py b/tests/unit/misc/test_pakjoy.py new file mode 100644 index 000000000..65d02ec7e --- /dev/null +++ b/tests/unit/misc/test_pakjoy.py @@ -0,0 +1,441 @@ +# SPDX-FileCopyrightText: Florian Bruhin (The-Compiler) <mail@qutebrowser.org> +# +# SPDX-License-Identifier: GPL-3.0-or-later + +import os +import io +import json +import struct +import pathlib +import logging +import shutil + +import pytest + +from qutebrowser.misc import pakjoy, binparsing +from qutebrowser.utils import utils, version, standarddir + + +pytest.importorskip("qutebrowser.qt.webenginecore") + + +pytestmark = pytest.mark.usefixtures("cache_tmpdir") + + +versions = version.qtwebengine_versions(avoid_init=True) + + +# Used to skip happy path tests with the real resources file. +# +# Since we don't know how reliably the Google Meet hangouts extensions is +# reliably in the resource files, and this quirk is only targeting 6.6 +# anyway. +skip_if_unsupported = pytest.mark.skipif( + versions.webengine != utils.VersionNumber(6, 6), + reason="Code under test only runs on 6.6", +) + + +@pytest.fixture(autouse=True) +def prepare_env(qapp, monkeypatch): + monkeypatch.setattr(pakjoy.objects, "qapp", qapp) + monkeypatch.delenv(pakjoy.RESOURCES_ENV_VAR, raising=False) + monkeypatch.delenv(pakjoy.DISABLE_ENV_VAR, raising=False) + + +def patch_version(monkeypatch, *args): + monkeypatch.setattr( + pakjoy.version, + "qtwebengine_versions", + lambda **kwargs: version.WebEngineVersions( + webengine=utils.VersionNumber(*args), + chromium=None, + source="unittest", + ), + ) + + +@pytest.fixture +def unaffected_version(monkeypatch): + patch_version(monkeypatch, 6, 6, 1) + + +@pytest.fixture +def affected_version(monkeypatch): + patch_version(monkeypatch, 6, 6) + + +@pytest.mark.parametrize("workdir_exists", [True, False]) +def test_version_gate(cache_tmpdir, unaffected_version, mocker, workdir_exists): + workdir = cache_tmpdir / pakjoy.CACHE_DIR_NAME + if workdir_exists: + workdir.mkdir() + (workdir / "some_patched_file.pak").ensure() + fake_open = mocker.patch("qutebrowser.misc.pakjoy.open") + + with pakjoy.patch_webengine(): + pass + + assert not fake_open.called + assert not workdir.exists() + + +def test_escape_hatch(affected_version, mocker, monkeypatch): + fake_open = mocker.patch("qutebrowser.misc.pakjoy.open") + monkeypatch.setenv(pakjoy.DISABLE_ENV_VAR, "1") + + with pakjoy.patch_webengine(): + pass + + assert not fake_open.called + + +class TestFindWebengineResources: + @pytest.fixture + def qt_data_path(self, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path): + """Patch qtutils.library_path() to return a temp dir.""" + qt_data_path = tmp_path / "qt_data" + qt_data_path.mkdir() + monkeypatch.setattr(pakjoy.qtutils, "library_path", lambda _which: qt_data_path) + return qt_data_path + + @pytest.fixture + def application_dir_path( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, + qt_data_path: pathlib.Path, # needs patching + ): + """Patch QApplication.applicationDirPath() to return a temp dir.""" + app_dir_path = tmp_path / "app_dir" + app_dir_path.mkdir() + monkeypatch.setattr( + pakjoy.objects.qapp, "applicationDirPath", lambda: app_dir_path + ) + return app_dir_path + + @pytest.fixture + def fallback_path( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, + qt_data_path: pathlib.Path, # needs patching + application_dir_path: pathlib.Path, # needs patching + ): + """Patch the fallback path to return a temp dir.""" + home_path = tmp_path / "home" + monkeypatch.setattr(pakjoy.pathlib.Path, "home", lambda: home_path) + + app_path = home_path / f".{pakjoy.objects.qapp.applicationName()}" + app_path.mkdir(parents=True) + return app_path + + @pytest.mark.parametrize("create_file", [True, False]) + def test_overridden( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path, create_file: bool + ): + """Test the overridden path is used.""" + override_path = tmp_path / "override" + override_path.mkdir() + monkeypatch.setenv(pakjoy.RESOURCES_ENV_VAR, str(override_path)) + if create_file: # should get this no matter if file exists or not + (override_path / pakjoy.PAK_FILENAME).touch() + assert pakjoy._find_webengine_resources() == override_path + + @pytest.mark.parametrize("with_subfolder", [True, False]) + def test_qt_data_path(self, qt_data_path: pathlib.Path, with_subfolder: bool): + """Test qtutils.library_path() is used.""" + resources_path = qt_data_path + if with_subfolder: + resources_path /= "resources" + resources_path.mkdir() + (resources_path / pakjoy.PAK_FILENAME).touch() + assert pakjoy._find_webengine_resources() == resources_path + + def test_application_dir_path(self, application_dir_path: pathlib.Path): + """Test QApplication.applicationDirPath() is used.""" + (application_dir_path / pakjoy.PAK_FILENAME).touch() + assert pakjoy._find_webengine_resources() == application_dir_path + + def test_fallback_path(self, fallback_path: pathlib.Path): + """Test fallback path is used.""" + (fallback_path / pakjoy.PAK_FILENAME).touch() + assert pakjoy._find_webengine_resources() == fallback_path + + def test_nowhere(self, fallback_path: pathlib.Path): + """Test we raise if we can't find the resources.""" + with pytest.raises( + binparsing.ParseError, match="Couldn't find webengine resources dir" + ): + pakjoy._find_webengine_resources() + + +def json_without_comments(bytestring): + str_without_comments = "\n".join( + [ + line + for line in bytestring.decode("utf-8").split("\n") + if not line.strip().startswith("//") + ] + ) + return json.loads(str_without_comments) + + +def read_patched_manifest(): + patched_resources = pathlib.Path(os.environ[pakjoy.RESOURCES_ENV_VAR]) + + with open(patched_resources / pakjoy.PAK_FILENAME, "rb") as fd: + reparsed = pakjoy.PakParser(fd) + + return json_without_comments(reparsed.manifest) + + +@pytest.mark.usefixtures("affected_version") +class TestWithRealResourcesFile: + """Tests that use the real pak file form the Qt installation.""" + + @skip_if_unsupported + def test_happy_path(self): + # Go through the full patching processes with the real resources file from + # the current installation. Make sure our replacement string is in it + # afterwards. + with pakjoy.patch_webengine(): + json_manifest = read_patched_manifest() + + assert ( + pakjoy.REPLACEMENT_URL.decode("utf-8") + in json_manifest["externally_connectable"]["matches"] + ) + + def test_copying_resources(self): + # Test we managed to copy some files over + work_dir = pakjoy.copy_webengine_resources() + + assert work_dir is not None + assert work_dir.exists() + assert work_dir == pathlib.Path(standarddir.cache()) / pakjoy.CACHE_DIR_NAME + assert (work_dir / pakjoy.PAK_FILENAME).exists() + assert len(list(work_dir.glob("*"))) > 1 + + def test_copying_resources_overwrites(self): + work_dir = pakjoy.copy_webengine_resources() + assert work_dir is not None + tmpfile = work_dir / "tmp.txt" + tmpfile.touch() + + pakjoy.copy_webengine_resources() + assert not tmpfile.exists() + + @pytest.mark.parametrize("osfunc", ["copytree", "rmtree"]) + def test_copying_resources_oserror(self, monkeypatch, caplog, osfunc): + # Test errors from the calls to shutil are handled + pakjoy.copy_webengine_resources() # run twice so we hit rmtree too + caplog.clear() + + def raiseme(err): + raise err + + monkeypatch.setattr( + pakjoy.shutil, osfunc, lambda *_args: raiseme(PermissionError(osfunc)) + ) + with caplog.at_level(logging.ERROR, "misc"): + with pakjoy.patch_webengine(): + pass + + assert caplog.messages == [ + "Failed to copy webengine resources, not applying quirk" + ] + + def test_expected_file_not_found(self, cache_tmpdir, monkeypatch, caplog): + with caplog.at_level(logging.ERROR, "misc"): + pakjoy._patch(pathlib.Path(cache_tmpdir) / "doesntexist") + assert caplog.messages[-1].startswith( + "Resource pak doesn't exist at expected location! " + "Not applying quirks. Expected location: " + ) + + +def json_manifest_factory(extension_id=pakjoy.HANGOUTS_MARKER, url=pakjoy.TARGET_URL): + assert isinstance(extension_id, bytes) + assert isinstance(url, bytes) + + return f""" + {{ + {extension_id.decode("utf-8")} + "key": "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDAQt2ZDdPfoSe/JI6ID5bgLHRCnCu9T36aYczmhw/tnv6QZB2I6WnOCMZXJZlRdqWc7w9jo4BWhYS50Vb4weMfh/I0On7VcRwJUgfAxW2cHB+EkmtI1v4v/OU24OqIa1Nmv9uRVeX0GjhQukdLNhAE6ACWooaf5kqKlCeK+1GOkQIDAQAB", + "name": "Google Hangouts", + // Note: Always update the version number when this file is updated. Chrome + // triggers extension preferences update on the version increase. + "version": "1.3.21", + "manifest_version": 2, + "externally_connectable": {{ + "matches": [ + "{url.decode("utf-8")}", + "http://localhost:*/*" + ] + }} + }} + """.strip().encode( + "utf-8" + ) + + +def pak_factory(version=5, entries=None, encoding=1, sentinel_position=-1): + if entries is None: + entries = [json_manifest_factory()] + + buffer = io.BytesIO() + buffer.write(struct.pack("<I", version)) + buffer.write(struct.pack(pakjoy.PakHeader._FORMAT, encoding, len(entries), 0)) + + entry_headers_size = (len(entries) + 1) * 6 + start_of_data = buffer.tell() + entry_headers_size + + # Normally the sentinel sits between the headers and the data. But to get + # full coverage we want to insert it in other positions. + with_indices = list(enumerate(entries, 1)) + if sentinel_position == -1: + with_indices.append((0, b"")) + elif sentinel_position is not None: + with_indices.insert(sentinel_position, (0, b"")) + + accumulated_data_offset = start_of_data + for idx, entry in with_indices: + buffer.write(struct.pack(pakjoy.PakEntry._FORMAT, idx, accumulated_data_offset)) + accumulated_data_offset += len(entry) + + for entry in entries: + assert isinstance(entry, bytes) + buffer.write(entry) + + buffer.seek(0) + return buffer + + +@pytest.mark.usefixtures("affected_version") +class TestWithConstructedResourcesFile: + """Tests that use a constructed pak file to give us more control over it.""" + + @pytest.mark.parametrize( + "offset", + [0, 42, pakjoy.HANGOUTS_ID], # test both slow search and fast path + ) + def test_happy_path(self, offset): + entries = [b""] * offset + [json_manifest_factory()] + assert entries[offset] != b"" + buffer = pak_factory(entries=entries) + + parser = pakjoy.PakParser(buffer) + + json_manifest = json_without_comments(parser.manifest) + + assert ( + pakjoy.TARGET_URL.decode("utf-8") + in json_manifest["externally_connectable"]["matches"] + ) + + def test_bad_version(self): + buffer = pak_factory(version=99) + + with pytest.raises( + binparsing.ParseError, + match="Unsupported .pak version 99", + ): + pakjoy.PakParser(buffer) + + @pytest.mark.parametrize( + "position, error", + [ + (0, "Unexpected sentinel entry"), + (None, "Missing sentinel entry"), + ], + ) + def test_bad_sentinal_position(self, position, error): + buffer = pak_factory(sentinel_position=position) + + with pytest.raises(binparsing.ParseError): + pakjoy.PakParser(buffer) + + @pytest.mark.parametrize( + "entry", + [ + b"{foo}", + b"V2VsbCBoZWxsbyB0aGVyZQo=", + ], + ) + def test_marker_not_found(self, entry): + buffer = pak_factory(entries=[entry]) + + with pytest.raises( + binparsing.ParseError, + match="Couldn't find hangouts manifest", + ): + pakjoy.PakParser(buffer) + + def test_url_not_found(self): + buffer = pak_factory(entries=[json_manifest_factory(url=b"example.com")]) + + parser = pakjoy.PakParser(buffer) + with pytest.raises( + binparsing.ParseError, + match="Couldn't find URL in manifest", + ): + parser.find_patch_offset() + + def test_url_not_found_high_level(self, cache_tmpdir, caplog, affected_version): + buffer = pak_factory(entries=[json_manifest_factory(url=b"example.com")]) + + # Write bytes to file so we can test pakjoy._patch() + tmpfile = pathlib.Path(cache_tmpdir) / "bad.pak" + with open(tmpfile, "wb") as fd: + fd.write(buffer.read()) + + with caplog.at_level(logging.ERROR, "misc"): + pakjoy._patch(tmpfile) + + assert caplog.messages == ["Failed to apply quirk to resources pak."] + + @pytest.fixture + def resources_path( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path + ) -> pathlib.Path: + resources_path = tmp_path / "resources" + resources_path.mkdir() + + buffer = pak_factory() + with open(resources_path / pakjoy.PAK_FILENAME, "wb") as fd: + fd.write(buffer.read()) + + monkeypatch.setattr(pakjoy.qtutils, "library_path", lambda _which: tmp_path) + return resources_path + + @pytest.fixture + def quirk_dir_path(self, tmp_path: pathlib.Path) -> pathlib.Path: + return tmp_path / "cache" / pakjoy.CACHE_DIR_NAME + + def test_patching(self, resources_path: pathlib.Path, quirk_dir_path: pathlib.Path): + """Go through the full patching processes with a fake resources file.""" + with pakjoy.patch_webengine(): + assert os.environ[pakjoy.RESOURCES_ENV_VAR] == str(quirk_dir_path) + json_manifest = read_patched_manifest() + + assert ( + pakjoy.REPLACEMENT_URL.decode("utf-8") + in json_manifest["externally_connectable"]["matches"] + ) + assert pakjoy.RESOURCES_ENV_VAR not in os.environ + + def test_preset_env_var( + self, + resources_path: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, + quirk_dir_path: pathlib.Path, + ): + new_resources_path = resources_path.with_name(resources_path.name + "_moved") + shutil.move(resources_path, new_resources_path) + monkeypatch.setenv(pakjoy.RESOURCES_ENV_VAR, str(new_resources_path)) + + with pakjoy.patch_webengine(): + assert os.environ[pakjoy.RESOURCES_ENV_VAR] == str(quirk_dir_path) + + assert os.environ[pakjoy.RESOURCES_ENV_VAR] == str(new_resources_path) @@ -51,8 +51,9 @@ deps = pyqt63: -r{toxinidir}/misc/requirements/requirements-pyqt-6.3.txt pyqt64: -r{toxinidir}/misc/requirements/requirements-pyqt-6.4.txt pyqt65: -r{toxinidir}/misc/requirements/requirements-pyqt-6.5.txt + pyqt66: -r{toxinidir}/misc/requirements/requirements-pyqt-6.6.txt commands = - !pyqt-!pyqt515-!pyqt5152-!pyqt62-!pyqt63-!pyqt64-!pyqt65: {envpython} scripts/link_pyqt.py --tox {envdir} + !pyqt-!pyqt515-!pyqt5152-!pyqt62-!pyqt63-!pyqt64-!pyqt65-!pyqt66: {envpython} scripts/link_pyqt.py --tox {envdir} {envpython} -bb -m pytest {posargs:tests} cov: {envpython} scripts/dev/check_coverage.py {posargs} |